From 2b05efbbd911421cb7c7290bda6b603f30816699 Mon Sep 17 00:00:00 2001 From: aivsomkar Date: Tue, 15 Sep 2026 21:34:31 +0530 Subject: [PATCH 001/211] fix(composer): give the editor its own line when the composer is narrow The composer is one flex row: attach, the Full access and place chips, the editor, the mic. The editor is the only child that can shrink, so with a bot's settings panel open beside the chat (or a small window) it collapsed to a few pixels; its placeholder then stacked one or two letters per line and the auto-grow made the whole box tall to fit them. Below a 30rem composer width the row wraps and the editor takes a full line of its own above the chips, with the mic/send group pushed to the right of the chip line. Tailwind container query on the composer box, same mechanism the chat header uses to fold its chips. Co-Authored-By: Claude Fable 5.1 --- src/components/ChatView.controls.test.ts | 18 ++++++++++++++++++ src/components/Composer.tsx | 14 +++++++++++--- src/components/MentionTextarea.tsx | 8 ++++++-- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/src/components/ChatView.controls.test.ts b/src/components/ChatView.controls.test.ts index 5d893f2a32..ba5e3fb479 100644 --- a/src/components/ChatView.controls.test.ts +++ b/src/components/ChatView.controls.test.ts @@ -84,6 +84,24 @@ describe("thread control placement", () => { expect(markup).not.toMatch(/class="[^"]*chat-text[^"\n]*bg-bubble-user/); }); + it("moves the editor onto its own line above the chips when the composer is narrow", () => { + // A bot's settings open beside the chat leaves the composer a few hundred + // pixels wide; the editor, the only shrinkable child, used to collapse to + // a sliver and stack its placeholder one letter per line. + const markup = renderToStaticMarkup(createElement(ChatView, { bot: { ...bot, busy: false } })); + const box = markup.indexOf("@container/composer"); + expect(box).toBeGreaterThan(-1); + const row = /data-composer-row="[^"]*" class="([^"]*)"/.exec(markup)!; + expect(row[1].split(" ")).toContain("@max-[30rem]/composer:flex-wrap"); + const editor = /class="mention-editor ([^"]*)"/.exec(markup)!; + expect(editor[1].split(" ")).toEqual(expect.arrayContaining(["min-w-0", "flex-1", "@max-[30rem]/composer:order-first", "@max-[30rem]/composer:basis-full"])); + const actions = /data-composer-actions="[^"]*" class="([^"]*)"/.exec(markup)!; + expect(actions[1].split(" ")).toContain("@max-[30rem]/composer:ml-auto"); + // the chips group still comes before the editor in source order: the + // wrap only reorders visually below the threshold + expect(markup.indexOf("data-test-approval-control")).toBeLessThan(markup.indexOf("mention-editor")); + }); + it("keeps the selected thread's model in the header and permissions inside the composer pill", () => { const markup = renderToStaticMarkup(createElement(ChatView, { bot })); expect(markup.match(/data-test-model-control/g)).toHaveLength(1); diff --git a/src/components/Composer.tsx b/src/components/Composer.tsx index 5533ddfebc..7a1ac2af04 100644 --- a/src/components/Composer.tsx +++ b/src/components/Composer.tsx @@ -803,8 +803,15 @@ export function Composer({ data-composer-backdrop className="pointer-events-none absolute -left-5 -right-5 -bottom-3 top-1/2 bg-app" /> -
-
+ {/* One row while it fits: chips, editor, mic. The editor is the only + child that can shrink, so in a narrow column (a bot's settings open + beside the chat, a small window) it collapsed to a few pixels and + its placeholder stacked one letter per line, while the auto-grow + made the box tall to fit them. Below the container width where the + chips and the placeholder cannot share a line, the editor takes a + full line of its own above the chips instead. */} +
+
)} member.id !== bot?.id)} everyone={Boolean(group && !group.dm)} @@ -979,7 +987,7 @@ export function Composer({ aria-label={t("composer.placeholder.bot", { name: group ? group.name : (bot?.name ?? "") })} className="block max-h-[9rem] min-h-6 w-full resize-none overflow-y-auto bg-transparent px-1 py-1 text-[15px] leading-6 placeholder:text-ink-secondary focus:outline-none" /> -
+
{/* Stop stays a stop. Stop-then-steer is named beside the queued message above, where its effect is visible before activation. */} {busy && !locked && ( diff --git a/src/components/MentionTextarea.tsx b/src/components/MentionTextarea.tsx index b13b98708c..701222c983 100644 --- a/src/components/MentionTextarea.tsx +++ b/src/components/MentionTextarea.tsx @@ -1,10 +1,14 @@ +import { cn } from "@/lib/cn"; import { useCallback, useLayoutEffect, useRef, type RefObject, type TextareaHTMLAttributes } from "react"; import { type MentionPeer } from "@/lib/mentions"; import { MentionText } from "./MentionText"; /** A native textarea retains selection, undo, IME and accessibility. Its * aria-hidden mirror paints mentions without changing wrapping or caret offsets. */ -export function MentionTextarea({ inputRef, peers, everyone = false, ...props }: TextareaHTMLAttributes & { +export function MentionTextarea({ inputRef, peers, everyone = false, wrapperClassName, ...props }: TextareaHTMLAttributes & { + /** Classes for the flex child around the editor: the composer uses it to + * move the editor onto its own line when the row is too narrow. */ + wrapperClassName?: string; inputRef: RefObject; peers: readonly MentionPeer[]; everyone?: boolean; @@ -41,7 +45,7 @@ export function MentionTextarea({ inputRef, peers, everyone = false, ...props }: if (inputRef.current) observer.observe(inputRef.current); return () => observer.disconnect(); }, [inputRef, resize, sync]); - return
+ return
From 2de2351e505a8d17bad21a7ad23b65376189bc2d Mon Sep 17 00:00:00 2001 From: aivsomkar Date: Tue, 15 Sep 2026 21:48:12 +0530 Subject: [PATCH 002/211] fix(layout): keep the chat usable at the default window size with panels open MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on a dev instance at the 1100x780 default and the 840x620 minimum, with the settings panel, the inspector and both open. Three things broke, all from rows or columns that could not shrink: - Chat and group headers: the chip group is shrink-0, so with a panel beside the chat (~330px column) the name truncated to nothing and the rename pencil landed under the find button. The row now wraps below 30rem — name line on top, chips underneath on the right. The query lives on a child row: a container query never matches the container. - Two side panels: bot settings deliberately keep the inspector or computer panel open (their controls open settings), but two static panels plus the sidebar left the default window a sliver of chat with letters stacked vertically. Until the window is 2xl (1536px) wide, settings now floats over the chat as a sheet and the other panel is still there when it closes. New useMediaQuery hook; the global :focus-visible ring is suppressed on the sheet container. - Composer: shipped in the previous commit (editor on its own line). Everything else checked at both sizes held up: sidebar, Tools panel, attention inbox, Settings modal, group setup card. Co-Authored-By: Claude Fable 5.1 --- src/App.tsx | 11 +++++++++-- src/components/BotSettingsDialog.tsx | 19 ++++++++++++++++-- src/components/ChatView.controls.test.ts | 15 ++++++++++++++ src/components/ChatView.tsx | 19 ++++++++++++++---- src/components/GroupView.tsx | 15 ++++++++++---- src/components/RemoteAgentSettingsPanel.tsx | 12 +++++++++-- src/lib/use-media-query.ts | 22 +++++++++++++++++++++ 7 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 src/lib/use-media-query.ts diff --git a/src/App.tsx b/src/App.tsx index 882928e3a0..e42f4fcad3 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -11,6 +11,7 @@ import { Sidebar } from "@/components/Sidebar"; import { ChatView } from "@/components/ChatView"; import { GroupView } from "@/components/GroupView"; import { BotSettingsDialog } from "@/components/BotSettingsDialog"; +import { TWO_SIDE_PANELS_FIT, useMediaQuery } from "@/lib/use-media-query"; import { RemoteAgentSettingsPanel } from "@/components/RemoteAgentSettingsPanel"; import { NewBotDialog } from "@/components/NewBotDialog"; import { PluginsPanel, preloadConnectedApps } from "@/components/PluginsPanel"; @@ -38,6 +39,7 @@ function Shell() { state.bots.filter((bot) => !bot.hidden && bot.unread).length + state.groups.filter((group) => group.unread).length; const remoteClient = window.ogb?.remoteClient?.active === true; + const twoSidePanelsFit = useMediaQuery(TWO_SIDE_PANELS_FIT, true); useEffect(() => { if (!window.ogb?.environments) return; const open = (computerId?: string | null) => { @@ -293,10 +295,15 @@ function Shell() { (Computer panel, then the usage chip): every re-render mounts a fresh settings panel and never removes the previous one, so the panels pile up and Close stops working. */} + {/* Bot settings keep the inspector or computer panel open on purpose + (their own controls open settings). Two static panels beside the + sidebar leave the default 1100px window a sliver of chat, so until + the window is wide enough to seat both, settings floats over the + chat instead and the other panel is still there when it closes. */} {state.settingsOpen && bot && ( remoteClient - ? - : + ? + : )} {state.computerOpen && bot && ( remoteClient ? ( diff --git a/src/components/BotSettingsDialog.tsx b/src/components/BotSettingsDialog.tsx index b61957d0e4..05abafbe57 100644 --- a/src/components/BotSettingsDialog.tsx +++ b/src/components/BotSettingsDialog.tsx @@ -30,7 +30,12 @@ function sectionMatches(entry: (typeof BOT_SECTIONS)[number], query: string): bo return [entry.label, ...entry.keywords].some((part) => part.toLowerCase().includes(query)); } -export function BotSettingsDialog({ bot }: { bot: Bot }) { +export function BotSettingsDialog({ bot, overlay = false }: { + bot: Bot; + /** Float over the chat instead of taking a column: the inspector or + * computer panel is open too and the window cannot seat both (App). */ + overlay?: boolean; +}) { const { state, dispatch, flushBotPatches } = useStore(); const section = state.botSettingsSection; const derived = useBotSettingsDerived(bot); @@ -302,7 +307,17 @@ export function BotSettingsDialog({ bot }: { bot: Bot }) { role="dialog" aria-labelledby="bot-settings-title" tabIndex={-1} - className="animate-panel-in absolute inset-0 z-40 flex h-full min-w-0 flex-col border-l border-hairline/40 bg-panel outline-none lg:static lg:z-auto lg:w-[min(420px,42vw)] lg:shrink-0" + className={cn( + // focus() lands here when the panel opens; the global :focus-visible + // ring would frame the whole sheet, so it is off for the container. + "animate-panel-in absolute inset-0 z-40 flex h-full min-w-0 flex-col border-l border-hairline/40 bg-panel outline-none focus-visible:outline-none", + overlay + // Below lg every panel already covers the window; from lg up + // this one hugs the right edge over the chat, shadowed so it + // reads as a sheet on top of the panel that stays beneath it. + ? "lg:inset-auto lg:right-0 lg:top-0 lg:bottom-0 lg:w-[min(420px,42vw)] lg:shadow-2xl" + : "lg:static lg:z-auto lg:w-[min(420px,42vw)] lg:shrink-0", + )} >
diff --git a/src/components/ChatView.controls.test.ts b/src/components/ChatView.controls.test.ts index ba5e3fb479..9b4a2a4eb1 100644 --- a/src/components/ChatView.controls.test.ts +++ b/src/components/ChatView.controls.test.ts @@ -84,6 +84,21 @@ describe("thread control placement", () => { expect(markup).not.toMatch(/class="[^"]*chat-text[^"\n]*bg-bubble-user/); }); + it("wraps the header into a name line and a chip line when the column is narrow", () => { + // With a settings or inspector panel open beside the chat the header's + // chip group cannot shrink; the wrap keeps the name readable and every + // chip in place. The query lives on the container's child row: a + // container query never matches the container element itself. + const markup = renderToStaticMarkup(createElement(ChatView, { bot: { ...bot, busy: false } })); + expect(markup).toContain("@container/chathead"); + const row = /data-chathead-row="[^"]*" class="([^"]*)"/.exec(markup)!; + expect(row[1].split(" ")).toContain("@max-[30rem]/chathead:flex-wrap"); + const identity = /data-chathead-identity="[^"]*" class="([^"]*)"/.exec(markup)!; + expect(identity[1].split(" ")).toEqual(expect.arrayContaining(["min-w-0", "@max-[30rem]/chathead:basis-full"])); + const controls = /data-chathead-controls="[^"]*" class="([^"]*)"/.exec(markup)!; + expect(controls[1].split(" ")).toContain("@max-[30rem]/chathead:ml-auto"); + }); + it("moves the editor onto its own line above the chips when the composer is narrow", () => { // A bot's settings open beside the chat leaves the composer a few hundred // pixels wide; the editor, the only shrinkable child, used to collapse to diff --git a/src/components/ChatView.tsx b/src/components/ChatView.tsx index 66dc3e1182..a62447f617 100644 --- a/src/components/ChatView.tsx +++ b/src/components/ChatView.tsx @@ -1131,13 +1131,22 @@ export function ChatView({ bot: profile }: { bot: Bot }) { style={headerDragStyle} className={cn( // @container so the chips on the right can fold to icon bubbles - // when the column is narrow (side panel open, small window) - "@container/chathead flex items-center justify-between px-5 py-3", + // when the column is narrow (side panel open, small window). A + // container query never matches the container itself, so the row + // that has to wrap is the child below, not this element. + "@container/chathead px-5 py-3", // Room for the drawer button, which overlays this corner below md. "pl-11 md:pl-5", )} > -
+ {/* Folding the chips to bubbles is not enough once a settings or + inspector panel leaves the chat ~330px wide: the chip group does + not shrink, so the name truncated to nothing and the rename + pencil landed under the find button. Below 30rem the header + wraps — name line on top, chips underneath on the right — so + every control keeps its place and the name stays readable. */} +
+
}
+
diff --git a/src/components/GroupView.tsx b/src/components/GroupView.tsx index a88fcd5107..43fca30137 100644 --- a/src/components/GroupView.tsx +++ b/src/components/GroupView.tsx @@ -1105,21 +1105,27 @@ export function GroupView({ group }: { group: Group }) { {membersOpen && !remoteClient && !group.dm && ( )} - {/* Header: static member avatars; a ring + dot marks the working bot. */} + {/* Header: static member avatars; a ring + dot marks the working bot. + Same shape as ChatView's header: the container is the outer box and + the row inside it wraps below 30rem — name line on top, controls + underneath on the right — since a container query never matches + the container itself and the control group cannot shrink. */}
-
+
+
{group.name} {!setupPending && !group.dm && }
)}
+
{findOpen && setFindOpen(false)} />} diff --git a/src/components/RemoteAgentSettingsPanel.tsx b/src/components/RemoteAgentSettingsPanel.tsx index c23b201674..299aa802e7 100644 --- a/src/components/RemoteAgentSettingsPanel.tsx +++ b/src/components/RemoteAgentSettingsPanel.tsx @@ -15,7 +15,12 @@ type RemoteProfilePatch = Partial< Pick >; -export function RemoteAgentSettingsPanel({ bot }: { bot: Bot }) { +export function RemoteAgentSettingsPanel({ bot, overlay = false }: { + bot: Bot; + /** Float over the chat: the remote computer panel is open too and the + * window cannot seat both columns (see App). */ + overlay?: boolean; +}) { const { dispatch } = useStore(); // Docked flush under the Windows caption corner: drop the header 16px. const { padClass } = useCaptionChrome(); @@ -73,7 +78,10 @@ export function RemoteAgentSettingsPanel({ bot }: { bot: Bot }) { }; return ( -
@@ -1459,12 +1443,13 @@ export function ChatView({ bot: profile }: { bot: Bot }) { /** What the open task has spent — quiet until the first turn settles. * Click opens the bot's settings, where the Usage card has the breakdown. */ -function UsageChip({ bot }: { bot: Bot }) { - const { state, dispatch } = useStore(); +/** The thread's usage, folded to one figure for the header menu — cost when + * the engine reports one, else new tokens — with the full breakdown as the + * tooltip. Null while the thread has no usage yet. */ +function usageSummary(bot: Bot, instances: AppState["instances"]): { short: string; detail: string; tone?: "danger" | "warning" } | null { const usage = bot.tasks?.find((t) => t.threadId === bot.threadId)?.usage; - const text = usage ? usageChip(usage) : ""; - if (!usage || !text) return null; - const billing = state.instances.find((i) => i.instanceId === bot.modelSelection.instanceId)?.snapshot.billing; + if (!usage || !usageChip(usage)) return null; + const billing = instances.find((i) => i.instanceId === bot.modelSelection.instanceId)?.snapshot.billing; const share = contextShare(usage); const detail = [ usage.turns === 1 ? t("chat.usage.turnsOne") : t("chat.usage.turnsMany", { count: usage.turns }), @@ -1480,19 +1465,83 @@ function UsageChip({ bot }: { bot: Bot }) { ] .filter(Boolean) .join("\n"); - // folded: one figure — cost when the engine reports one, else new tokens const short = hasFiniteCost(usage.costUsd) ? formatUsd(usage.costUsd) : formatTokens(cachedKnown(usage) ? freshTokens(usage) : usage.input + usage.output); const ctx = contextChip(usage); + return { short: ctx ? `${short} · ${ctx}` : short, detail, tone: share?.tone === "danger" ? "danger" : share?.tone === "warning" ? "warning" : undefined }; +} + +/** The header's "more" menu: find, export, usage and the inspector, behind + * one button that opens on hover. Keeps the header to four controls in a + * narrow column instead of eight chips that ran under the side panels. */ +function ChatHeaderMenu({ bot, messages, findOpen, onFind }: { + bot: Bot; + messages: readonly Message[]; + findOpen: boolean; + onFind: () => void; +}) { + const { state, dispatch } = useStore(); + const remoteClient = window.ogb?.remoteClient?.active === true; + const usage = usageSummary(bot, state.instances); + const hasMessages = messages.length > 0; + const transcript = () => formatTranscriptMarkdown({ title: bot.name, messages, botName: bot.name, isGroup: false }); + const items: SidebarMenuItem[] = [ + { + key: "find", + label: t("chat.find"), + icon: , + active: findOpen, + trailing: ⌘F, + onSelect: onFind, + }, + { + key: "copy", + label: t("chat.export.copy"), + icon: , + disabled: !hasMessages, + onSelect: () => { void copyTranscriptToClipboard(transcript()); }, + }, + { + key: "download", + label: t("chat.export.download"), + icon: , + disabled: !hasMessages, + onSelect: () => downloadMarkdownTranscript(slugifyTranscriptFilename(bot.name), transcript()), + }, + ...(usage ? [{ + key: "usage", + label: t("chat.usage.menu"), + icon: , + separatorBefore: true, + trailing: {usage.short}, + onSelect: () => dispatch({ type: "toggleSettings", open: true, section: "usage" }), + } satisfies SidebarMenuItem] : []), + ...(remoteClient ? [] : [{ + key: "inspector", + label: t("chat.inspector"), + icon: , + active: state.inspectorOpen, + separatorBefore: !usage, + onSelect: () => dispatch({ type: "toggleInspector" }), + } satisfies SidebarMenuItem]), + ]; return ( - + ( + + + + )} + /> ); } diff --git a/src/components/Composer.tsx b/src/components/Composer.tsx index 7a1ac2af04..a272bc16ae 100644 --- a/src/components/Composer.tsx +++ b/src/components/Composer.tsx @@ -824,7 +824,7 @@ export function Composer({ }} /> {!locked && ( -
+
; + })} + ; +} + /** The escape hatch for other ongoing conversations when their tree is hidden. * These are selection-only buttons: no create, rename, move, or delete menu. */ export function SidebarBotActivity({ bot, density }: { bot: Bot; density: SidebarDensity }) { diff --git a/src/components/SidebarPinnedThreadsPanel.test.ts b/src/components/SidebarPinnedThreadsPanel.test.ts new file mode 100644 index 0000000000..b05ca297cd --- /dev/null +++ b/src/components/SidebarPinnedThreadsPanel.test.ts @@ -0,0 +1,77 @@ +import { Children, createElement, isValidElement, type MouseEvent, type ReactElement, type ReactNode } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { describe, expect, it, vi } from "vitest"; + +import { t } from "@/lib/i18n"; +import type { AttentionThread } from "./SidebarBotActivity"; +import { SidebarPinnedThreadsPanel } from "./SidebarPinnedThreadsPanel"; + +const entry: AttentionThread = { + kind: "bot", + botId: "atlas", + botName: "Atlas", + task: { threadId: "pinned-1", title: "Quarterly plan", createdAt: 1, queued: false }, +}; + +type ElementProps = { children?: ReactNode; onClick?: (event: MouseEvent) => void; [key: string]: unknown }; +function findElement(tree: ReactNode, attribute: string, value: string): ReactElement | undefined { + for (const child of Children.toArray(tree)) { + if (!isValidElement(child)) continue; + if (child.props[attribute] === value) return child; + const found = findElement(child.props.children, attribute, value); + if (found) return found; + } +} + +function renderPanel( + entries: AttentionThread[] = [entry], + options: { collapsed?: boolean; onToggle?: () => void } = {}, +) { + let tree: ReactNode; + const onJump = vi.fn(); + const onToggle = options.onToggle ?? vi.fn(); + function Capture() { + tree = SidebarPinnedThreadsPanel({ + entries, + density: "comfortable", + now: 1, + onJump, + collapsed: options.collapsed ?? false, + onToggle, + }); + return tree; + } + const markup = renderToStaticMarkup(createElement(Capture)); + return { markup, tree: () => tree as ReactNode, onJump, onToggle }; +} + +describe("pinned threads panel", () => { + it("renders nothing when there is no pinned thread", () => { + const { markup } = renderPanel([]); + expect(markup).toBe(""); + }); + + it("renders the pinned rows under the Pinned threads name", () => { + const { markup } = renderPanel(); + expect(markup).toContain(t("sidebar.pinnedThreads.title")); + expect(markup).toContain("Quarterly plan"); + expect(markup).toContain("Atlas"); + expect(markup).toContain('data-testid="sidebar-pinned-threads-panel"'); + }); + + it("hides the rows but keeps the header when collapsed", () => { + const { markup } = renderPanel([entry], { collapsed: true }); + expect(markup).toContain(t("sidebar.pinnedThreads.title")); + expect(markup).not.toContain("Quarterly plan"); + const label = t("sidebar.section.expand", { name: t("sidebar.pinnedThreads.title") }); + expect(markup).toContain('aria-label="' + label + '"'); + expect(markup).toContain('aria-expanded="false"'); + }); + + it("toggles from the collapse/expand control", () => { + const { tree, onToggle } = renderPanel([entry], { collapsed: false }); + const label = t("sidebar.section.collapse", { name: t("sidebar.pinnedThreads.title") }); + findElement(tree(), "aria-label", label)!.props.onClick!({} as MouseEvent); + expect(onToggle).toHaveBeenCalledOnce(); + }); +}); diff --git a/src/components/SidebarPinnedThreadsPanel.tsx b/src/components/SidebarPinnedThreadsPanel.tsx new file mode 100644 index 0000000000..04ee4e11ad --- /dev/null +++ b/src/components/SidebarPinnedThreadsPanel.tsx @@ -0,0 +1,49 @@ +import { ChevronDown, ChevronRight, Pin } from "lucide-react"; +import { cn } from "@/lib/cn"; +import { t } from "@/lib/i18n"; +import type { SidebarDensity } from "@/lib/sidebar-preferences"; +import { PinnedThreadRows, type AttentionThread } from "./SidebarBotActivity"; + +/** A cross-bot, cross-room list of every pinned thread, living between + * search and the bots list — pinned bots already get this top-level view + * (the built-in Pinned section); pinned threads did not. Renders nothing + * when there is no pin, so it never costs space it isn't using. */ +export function SidebarPinnedThreadsPanel({ entries, density, now, onJump, collapsed, onToggle }: { + entries: AttentionThread[]; + density: SidebarDensity; + now: number; + onJump: (entry: AttentionThread) => void; + collapsed: boolean; + onToggle: () => void; +}) { + if (entries.length === 0) return null; + const compact = density === "compact"; + const Chevron = collapsed ? ChevronRight : ChevronDown; + return ( +
+
+ +
+ {!collapsed && ( +
+ +
+ )} +
+ ); +} diff --git a/src/lib/sidebar-layout.ts b/src/lib/sidebar-layout.ts index 4de8dcfc6c..35e96d4b70 100644 --- a/src/lib/sidebar-layout.ts +++ b/src/lib/sidebar-layout.ts @@ -4,6 +4,8 @@ export const PINNED_SECTION_ID = "builtin:pinned"; export const CHANNELS_SECTION_ID = "builtin:channels"; export const BOT_CHATS_SECTION_ID = "builtin:bot-chats"; export const BOTS_SECTION_ID = "builtin:bots"; +export const ATTENTION_SECTION_ID = "builtin:attention"; +export const PINNED_THREADS_SECTION_ID = "builtin:pinned-threads"; const USER_SECTION_PREFIX = "section:"; diff --git a/src/locales/en.json b/src/locales/en.json index 70565ce823..6f21861224 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1142,6 +1142,8 @@ "attention.item": "{title} · {name} · {status}", "attention.pin": "Pin", "attention.unpin": "Unpin", + "sidebar.pinnedThreads.title": "Pinned threads", + "sidebar.pinnedThreads.item": "{title} · {name}", "task.search": "Search threads", "task.renameAria": "Rename thread", "task.renameNamed": "Rename {title}", From 151f1eed54c6368892961790d1c8ed22e45b37a6 Mon Sep 17 00:00:00 2001 From: buttonsjasper360-lang Date: Fri, 2 Oct 2026 08:12:51 +0100 Subject: [PATCH 021/211] fix(android): copy and select a failed routine run's error (#2110) Copying a failed or missed routine.run card gave the headline plus the (often stale) summary, never the error the card actually shows, and the error text sat outside SelectionContainer so it could not be selected. Include the error in MessageActions.copyableText and wrap it in SelectionContainer. Follow-up to the cross-bot/expansion items from #1903, which main already covers via RoutineRunCardView (8caa6bff). Co-authored-by: Claude Sonnet 5.5 --- .../kotlin/com/openmausbot/companion/ui/ChatPolicy.kt | 10 ++++++++-- .../com/openmausbot/companion/ui/RoutineRunCardView.kt | 4 +++- .../com/openmausbot/companion/ui/ChatPolicyTest.kt | 8 ++++++++ 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt index 3110d0ecd2..b7fc34a025 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/ChatPolicy.kt @@ -478,9 +478,15 @@ object MessageActions { // A tool chip is context, a screenshot is pixels, a digest is a log line. Message.Kind.ACTIVITY, Message.Kind.SCREEN, Message.Kind.DIGEST -> null Message.Kind.COMPACTION -> message.compaction?.summary ?: message.text?.takeIf { it.isNotBlank() } - // The run's report is the part worth keeping; the headline without one. + // The run's report and error are the parts worth keeping; the headline without either. Message.Kind.ROUTINE_RUN -> message.routineRun - ?.let { run -> listOfNotNull(run.headline, run.summary?.takeIf { it.isNotBlank() }).joinToString("\n\n") } + ?.let { run -> + listOfNotNull( + run.headline, + run.summary?.takeIf { it.isNotBlank() }, + run.error?.takeIf { it.isNotBlank() }, + ).joinToString("\n\n") + } ?: message.text?.takeIf { it.isNotBlank() } } diff --git a/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt b/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt index 966bf621a5..f6e72c918e 100644 --- a/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt +++ b/android/app/src/main/kotlin/com/openmausbot/companion/ui/RoutineRunCardView.kt @@ -161,7 +161,9 @@ internal fun RoutineRunCardView(message: Message, openRun: (() -> Unit)?) { } run.error?.takeIf { it.isNotBlank() }?.let { error -> - Text(error.trim(), fontSize = 13.sp, color = MaterialTheme.colorScheme.error) + SelectionContainer { + Text(error.trim(), fontSize = 13.sp, color = MaterialTheme.colorScheme.error) + } } } } diff --git a/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt b/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt index f7131144ab..709b8dcc6c 100644 --- a/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt +++ b/android/app/src/test/kotlin/com/openmausbot/companion/ui/ChatPolicyTest.kt @@ -8,6 +8,7 @@ import com.openmausbot.companion.core.ChatTarget import com.openmausbot.companion.core.CompanionState import com.openmausbot.companion.core.GroupResponder import com.openmausbot.companion.core.Message +import com.openmausbot.companion.core.RoutineRunCard import com.openmausbot.companion.core.ModelSelection import com.openmausbot.companion.core.OptionCard import com.openmausbot.companion.core.Reaction @@ -606,6 +607,13 @@ class MessageActionsTest { assertEquals("hello", MessageActions.copyableText(message(Message.Kind.UNKNOWN, "hello"))) } + @Test + fun `a failed routine run copies the error the card shows`() { + val run = RoutineRunCard(routineName = "Brief", status = "failed", summary = "old report", error = "boom") + val copied = MessageActions.copyableText(message(Message.Kind.ROUTINE_RUN, null).copy(routineRun = run)) + assertEquals(true, copied?.contains("boom")) + } + @Test fun `empty or absent text offers nothing`() { assertNull(MessageActions.copyableText(message(Message.Kind.TEXT, null))) From 9e3ec305c2d34a478cf8861d54882a0209881674 Mon Sep 17 00:00:00 2001 From: Aditya Umale Date: Fri, 2 Oct 2026 12:44:32 +0530 Subject: [PATCH 022/211] fix(browser): let the Browser panel's full screen button lead back out (#2125) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Browser panel's expand button only ever called requestFullscreen(). On macOS a full-screen element takes the whole window into native full screen, where the yellow minimize button and Cmd-M are disabled, and clicking the same button again did nothing — its label and icon never changed. The only exits were Esc, the hidden green button or View > Toggle Full Screen, none of which the app pointed to. The button is now a toggle, like the desktop viewer's: a second click calls exitFullscreen(), and a fullscreenchange listener keeps its label ("Full screen" / "Exit full screen") and icon in step with the document, so Esc and the window's own controls leave it right too. A failed exit says to press Esc. Fixes MOCA-266 Co-authored-by: Claude Opus 5.5 --- src/components/BrowserPanel.test.ts | 64 +++++++++++++++++++++++++++++ src/components/BrowserPanel.tsx | 20 ++++++++- 2 files changed, 82 insertions(+), 2 deletions(-) diff --git a/src/components/BrowserPanel.test.ts b/src/components/BrowserPanel.test.ts index 47f08e8ddb..dcc85118a6 100644 --- a/src/components/BrowserPanel.test.ts +++ b/src/components/BrowserPanel.test.ts @@ -381,3 +381,67 @@ describe("live browser control affordance", () => { expect(html).not.toContain('Take control'); }); }); + +// MOCA-266: the expand button only ever entered full screen. On macOS that +// takes the whole window into native full screen, where minimize is disabled, +// and clicking the button again did nothing. +describe("browser full screen", () => { + const fullscreenDocument = () => { + const listeners = new Map void>(); + const doc = { + fullscreenElement: null as unknown, + exitFullscreen: vi.fn().mockResolvedValue(undefined), + addEventListener: vi.fn((name: string, listener: () => void) => { listeners.set(name, listener); }), + removeEventListener: vi.fn((name: string) => { listeners.delete(name); }), + }; + vi.stubGlobal("document", doc); + return { doc, listeners }; + }; + const panelElement = () => { + const element = { requestFullscreen: vi.fn().mockResolvedValue(undefined) }; + (fixture.refs[3] as RefObject).current = element; + return element; + }; + + it("leaves full screen on the second click instead of asking for it again", async () => { + const { doc } = fullscreenDocument(); + const nodes = renderElements(); + const panel = panelElement(); + + click(nodes, "Full screen"); + expect(panel.requestFullscreen).toHaveBeenCalledOnce(); + doc.fullscreenElement = panel; + click(nodes, "Full screen"); + await settle(); + expect(doc.exitFullscreen).toHaveBeenCalledOnce(); + expect(panel.requestFullscreen).toHaveBeenCalledOnce(); + }); + + it("follows the document, so Esc and the window's own controls keep the button right", () => { + const { doc, listeners } = fullscreenDocument(); + renderElements(); + const panel = panelElement(); + const fullscreenSetter = fixture.setters[11]!; + const cleanup = fixture.effects[3]!(); + + doc.fullscreenElement = panel; + listeners.get("fullscreenchange")!(); + expect(fullscreenSetter).toHaveBeenLastCalledWith(true); + doc.fullscreenElement = null; + listeners.get("fullscreenchange")!(); + expect(fullscreenSetter).toHaveBeenLastCalledWith(false); + cleanup?.(); + expect(listeners.has("fullscreenchange")).toBe(false); + }); + + it("says when leaving full screen fails", async () => { + const { doc } = fullscreenDocument(); + doc.exitFullscreen.mockRejectedValue(new Error("denied")); + const nodes = renderElements(); + const panel = panelElement(); + doc.fullscreenElement = panel; + click(nodes, "Full screen"); + await settle(); + expect(fixture.setters[7]).toHaveBeenLastCalledWith("Could not leave full screen. Press Esc instead."); + }); +}); diff --git a/src/components/BrowserPanel.tsx b/src/components/BrowserPanel.tsx index c51811814f..9e3fc1227d 100644 --- a/src/components/BrowserPanel.tsx +++ b/src/components/BrowserPanel.tsx @@ -1,5 +1,5 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { ArrowLeft, ArrowRight, EllipsisVertical, Globe, Hand, Loader2, Maximize2, Plus, RotateCw, UserRound, X } from "lucide-react"; +import { ArrowLeft, ArrowRight, EllipsisVertical, Globe, Hand, Loader2, Maximize2, Minimize2, Plus, RotateCw, UserRound, X } from "lucide-react"; import { browserUnavailableReason } from "@/lib/feature-flags"; import { api, useStore, type Bot } from "@/state/store"; import { BrowserProfilesManager } from "./BrowserProfilesManager"; @@ -27,6 +27,7 @@ export function LiveBrowser({ bot }: { bot: Bot }) { const [showProfiles, setShowProfiles] = useState(false); const [showTyping, setShowTyping] = useState(false); const [viewport, setViewport] = useState({ width: 1280, height: 720 }); + const [fullscreen, setFullscreen] = useState(false); const viewer = useRef(""); const generation = useRef(0); const pendingOperation = useRef(null); @@ -157,6 +158,21 @@ export function LiveBrowser({ bot }: { bot: Bot }) { }; }, [bot.id, bot.browserProfile, attempt, action]); + // On macOS a full-screen element takes the whole window into native full + // screen, where minimize is disabled. The button has to lead back out, and + // it follows the document so Esc, the green button and View > Toggle Full + // Screen keep it honest too (MOCA-266). + useEffect(() => { + const update = () => setFullscreen(Boolean(panel.current) && document.fullscreenElement === panel.current); + document.addEventListener("fullscreenchange", update); + return () => document.removeEventListener("fullscreenchange", update); + }, []); + const toggleFullscreen = () => { + const leaving = Boolean(panel.current) && document.fullscreenElement === panel.current; + const request = leaving ? document.exitFullscreen() : panel.current?.requestFullscreen(); + void request?.catch(() => setError(leaving ? "Could not leave full screen. Press Esc instead." : "Full screen is unavailable in this browser.")); + }; + const execute = async (body: Record) => { if (pendingOperation.current !== null) return; const expected = viewer.current; @@ -192,7 +208,7 @@ export function LiveBrowser({ bot }: { bot: Bot }) {
) :
New tab
}
- +
{ e.preventDefault(); if (driving && address.trim()) void execute({ type: "navigate", url: /^https?:\/\//i.test(address.trim()) ? address.trim() : `https://${address.trim()}` }); }}> From 0c8d4d5f0579bb6c8ea8483de0c7519579d4589d Mon Sep 17 00:00:00 2001 From: jakequade Date: Fri, 2 Oct 2026 17:14:37 +1000 Subject: [PATCH 023/211] Show date and time in routine run thread titles (#2112) * feat(routines): timestamp run thread titles * test(routines): cover timestamped room task titles --- server/group-goal-run.e2e.test.ts | 2 +- server/routines.test.ts | 29 ++++++++++++++++++++++++++++- server/routines.ts | 9 ++++++--- 3 files changed, 35 insertions(+), 5 deletions(-) diff --git a/server/group-goal-run.e2e.test.ts b/server/group-goal-run.e2e.test.ts index 6b744c6dcb..567bc90211 100644 --- a/server/group-goal-run.e2e.test.ts +++ b/server/group-goal-run.e2e.test.ts @@ -979,7 +979,7 @@ describe("goal-driven channel runs", () => { expect(backgroundRoom.threadId).toBe(completedRun.threadId); expect(backgroundRoom.tasks).toContainEqual(expect.objectContaining({ threadId: completedRun.threadId, - title: "Daily team review", + title: expect.stringMatching(/^Daily team review · [A-Z][a-z]{2} \d{1,2}, \d{1,2}:\d{2} [AP]M$/), })); const switched = await api("POST", `/api/groups/${room.id}/tasks/${completedRun.threadId}`); diff --git a/server/routines.test.ts b/server/routines.test.ts index 787c251ec3..bbcbc26292 100644 --- a/server/routines.test.ts +++ b/server/routines.test.ts @@ -41,6 +41,7 @@ function harness(start = new Date(2026, 7, 17, 8, 0, 0).getTime()) { const runOns: string[] = []; const triggerSources: string[] = []; const taskActivations: boolean[] = []; + const taskTitles: string[] = []; const goalTasks: Array<{ groupId: string; title: string }> = []; const interruptedTurns: Array<{ botId: string; threadId: string; runOn: string }> = []; const interruptedGoals: Array<{ @@ -60,6 +61,7 @@ function harness(start = new Date(2026, 7, 17, 8, 0, 0).getTime()) { goalState: () => goal, createTask: (_botId, _title, activate = false) => { taskActivations.push(activate); + taskTitles.push(_title); return { threadId: `thread-${++task}` }; }, createGoalTask: (groupId, title) => { @@ -94,6 +96,7 @@ function harness(start = new Date(2026, 7, 17, 8, 0, 0).getTime()) { runOns, triggerSources, taskActivations, + taskTitles, goalTasks, interruptedTurns, interruptedGoals, @@ -108,6 +111,7 @@ function harness(start = new Date(2026, 7, 17, 8, 0, 0).getTime()) { afterEach(() => { vi.restoreAllMocks(); + vi.unstubAllEnvs(); for (const dir of dirs.splice(0)) rmSync(dir, { recursive: true, force: true }); }); @@ -249,6 +253,28 @@ describe("bounded scheduled overlap and run health", () => { }); }); +it("names a new routine thread with the local dispatch date and time", async () => { + vi.stubEnv("TZ", "UTC"); + const at = Date.parse("2026-10-01T21:26:00Z"); + const h = harness(at); + const routine = h.manager.create({ name: "Morning brief", prompt: "Summarize", botId: "maus-1", + schedule: { type: "once", at } }); + h.manager.runNow(routine.id); + const createTask = h.options.createTask; + h.options.createTask = (...args) => { h.setNow(at + 60_000); return createTask(...args); }; + await h.manager.tick(); + expect(h.taskTitles).toEqual(["Morning brief · Oct 1, 9:26 PM"]); + expect(h.manager.listRuns()[0]).toMatchObject({ routineName: "Morning brief", startedAt: at }); + + const long = harness(at); + const named = long.manager.create({ name: "x".repeat(100), prompt: "Summarize", botId: "maus-1", + schedule: { type: "once", at } }); + long.manager.runNow(named.id); + await long.manager.tick(); + expect(long.taskTitles[0]).toHaveLength(80); + expect(long.taskTitles[0]).toMatch(/ · Oct 1, 9:26 PM$/); +}); + describe("cron routines use the existing persistent scheduler", () => { const start = Date.parse("2026-09-13T08:00:00Z"); const monthly = { type: "cron" as const, expression: "0 9 1 * *", timeZone: "UTC" }; @@ -1827,6 +1853,7 @@ describe("RoutineManager", () => { }); it("queues behind a busy room goal, then dispatches it into a detached room task", async () => { + vi.stubEnv("TZ", "UTC"); const h = harness(); h.setGoal("busy"); const routine = h.manager.create({ @@ -1852,7 +1879,7 @@ describe("RoutineManager", () => { h.setGoal("ready"); await h.manager.tick(); const run = h.manager.listRuns()[0]!; - expect(h.goalTasks).toEqual([{ groupId: "room-1", title: "Team launch" }]); + expect(h.goalTasks).toEqual([{ groupId: "room-1", title: "Team launch · Aug 17, 8:01 AM" }]); expect(h.startedGoals[0]).toMatchObject({ groupId: "room-1", threadId: "goal-thread-1", diff --git a/server/routines.ts b/server/routines.ts index bacced94e3..786b0735a0 100644 --- a/server/routines.ts +++ b/server/routines.ts @@ -1581,14 +1581,17 @@ export class RoutineManager { // A webhook is an incoming message, so make its task the bot's live // chat immediately. Scheduled work stays in its own task unless the // workspace has asked for runs to join the conversation they report to. + const startedAt = this.now(); + const suffix = ` · ${new Date(startedAt).toLocaleString("en-US", { month: "short", day: "numeric", hour: "numeric", minute: "2-digit" })}`; + const title = `${run.routineName.slice(0, 80 - suffix.length).trimEnd()}${suffix}`; const joined = run.target === "room-goal" ? null : this.options.joinConversation?.(run) || null; const task = joined ? { threadId: joined } : run.target === "room-goal" ? run.groupId - ? this.options.createGoalTask?.(run.groupId, run.routineName) ?? null + ? this.options.createGoalTask?.(run.groupId, title) ?? null : null - : this.options.createTask(run.botId, run.routineName, run.triggerSource === "webhook", run.routineId); + : this.options.createTask(run.botId, title, run.triggerSource === "webhook", run.routineId); if (!task) { this.failRun(run, run.target === "room-goal" ? "Could not create a room task for this goal" @@ -1596,7 +1599,7 @@ export class RoutineManager { continue; } run.threadId = task.threadId; - run.startedAt = this.now(); + run.startedAt = startedAt; run.status = "running"; this.save(); this.emitRun(run); From 5ddc6fda810b3f57b0eca5dfbaed0e2883b26ef3 Mon Sep 17 00:00:00 2001 From: Sathiyan Kutty Date: Fri, 2 Oct 2026 00:14:44 -0700 Subject: [PATCH 024/211] Show live computer/browser session indicator in rooms (#2119) * feat(rooms): show live computer/browser session indicator - GroupView.tsx: render "screen" messages (live screenshot frames) inline, same as ChatView already does in 1:1 chats. - GroupView.tsx: give a busy member's avatar badge the place icon (PlaceIcon) instead of a generic dot when that bot is actively in a concrete computer/browser place, mirroring the 1:1 composer's PlaceChip live indicator. Fixes: a bot running a background browser/computer session inside a room gave no visual indication at all that it was happening, even though the server already posted the screen messages to the room's own thread. Co-Authored-By: Claude Sonnet 5 * fix(rooms): give the busy-member place badge an accessible label CodeRabbit flagged on PR #2119: the place icon on a busy avatar badge conveyed its meaning visually (which computer/browser place a bot is in) but had no screen-reader-accessible text. Reuse the same place.chipAria label PlaceChip already uses for the 1:1 composer. Co-Authored-By: Claude Sonnet 5 --------- Co-authored-by: Claude Sonnet 5 --- .../GroupView.screen-and-place.test.ts | 85 +++++++++++++++++++ src/components/GroupView.tsx | 56 ++++++++---- 2 files changed, 125 insertions(+), 16 deletions(-) create mode 100644 src/components/GroupView.screen-and-place.test.ts diff --git a/src/components/GroupView.screen-and-place.test.ts b/src/components/GroupView.screen-and-place.test.ts new file mode 100644 index 0000000000..8bf97c89b7 --- /dev/null +++ b/src/components/GroupView.screen-and-place.test.ts @@ -0,0 +1,85 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import type { AppState, Bot, Group, Message } from "@/state/store"; + +const fixture = vi.hoisted(() => { + vi.stubGlobal("window", {}); + vi.stubGlobal("localStorage", { getItem: () => null, setItem: () => {} }); + return { state: null as Partial | null, dispatch: vi.fn() }; +}); +vi.mock("@/state/store", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + useStore: () => ({ state: { ...original.initialState, ...fixture.state }, dispatch: fixture.dispatch }), + useStreaming: () => ({ streaming: {} }), + }; +}); +vi.mock("./DesktopCapabilities", async (importOriginal) => ({ + ...await importOriginal(), + useDesktopCapabilities: () => ({ capabilities: { dictation: { available: false } }, ready: true }), +})); +vi.mock("@/lib/analytics", () => ({ track: vi.fn() })); + +const { GroupView } = await import("./GroupView"); + +afterEach(() => { + fixture.state = null; + vi.clearAllMocks(); +}); + +const bot = (patch: Partial = {}): Bot => ({ + id: "aleta", threadId: "t-aleta", name: "Aleta", title: "", description: "", color: "green", + notifications: true, unread: false, busy: false, messages: [], + modelSelection: { instanceId: "test", model: "profile-default" }, + ...patch, +}); + +const group = (patch: Partial = {}): Group => ({ + id: "room", threadId: "room-thread", name: "Job search", memberIds: ["aleta"], + defaultResponder: { kind: "everyone" }, bulletin: "", unread: false, createdAt: 1, + messages: [], + ...patch, +}); + +const render = (members: Bot[], g: Group) => { + fixture.state = { bots: members }; + return renderToStaticMarkup(createElement(GroupView, { group: g })); +}; + +describe("GroupView: computer/browser session visibility", () => { + it("renders a live screenshot message inline, matching the 1:1 ChatView", () => { + const markup = render([bot()], group({ + messages: [{ id: "shot", role: "bot", kind: "screen", png: "PRIVATE_BASE64_PIXELS", mime: "image/png", at: 1, from: { botId: "aleta", name: "Aleta", color: "green" } }] as Message[], + })); + expect(markup).toContain("data:image/png;base64,PRIVATE_BASE64_PIXELS"); + }); + + it("drops a screen message with no image, same as ChatView", () => { + const markup = render([bot()], group({ + messages: [{ id: "shot", role: "bot", kind: "screen", at: 1, from: { botId: "aleta", name: "Aleta", color: "green" } }] as Message[], + })); + expect(markup).not.toContain("data:image"); + }); + + it("marks a busy member working a concrete place (e.g. browser) with that place's icon", () => { + const markup = render([bot({ busy: true, computer: "browser" })], group({ busyBotId: "aleta" })); + expect(markup).toMatch(/width="9"[^>]*height="9"/); + }); + + it("gives the place badge an accessible label naming the place", () => { + const markup = render([bot({ busy: true, computer: "browser" })], group({ busyBotId: "aleta" })); + expect(markup).toContain('aria-label="Where this conversation works: Browser"'); + }); + + it("falls back to the plain working dot when the busy member's place is Auto", () => { + const markup = render([bot({ busy: true })], group({ busyBotId: "aleta" })); + expect(markup).not.toMatch(/width="9"[^>]*height="9"/); + }); + + it("shows no place badge at all for an idle member", () => { + const markup = render([bot({ computer: "browser" })], group({ busyBotId: null })); + expect(markup).not.toMatch(/width="9"[^>]*height="9"/); + }); +}); diff --git a/src/components/GroupView.tsx b/src/components/GroupView.tsx index 8b2fa48edc..105727c737 100644 --- a/src/components/GroupView.tsx +++ b/src/components/GroupView.tsx @@ -17,6 +17,9 @@ import { type Message, } from "@/state/store"; import { BotAvatar } from "./Avatar"; +import { PlaceIcon } from "./PlaceIcon"; +import { ScreenFrame } from "./ScreenFrame"; +import { effectivePlace, placeLabelKey } from "@/lib/place"; import { ThreadChip } from "./ThreadChip"; import { ToolActivity } from "./ToolActivity"; import { ThreadRefText } from "./ThreadRefs"; @@ -288,6 +291,8 @@ export const Transcript = memo(function Transcript({ roomActivityVisible(m, showToolCalls) ? ( isStatusActivity(m) ? : ) : null + ) : m.kind === "screen" ? ( + m.png ? : null ) : m.kind === "compaction" ? ( ) : m.kind === "digest" ? ( @@ -1219,22 +1224,41 @@ export function GroupView({ group }: { group: Group }) { } }; - // Static profile avatars: one per member, a ring + dot on whoever is working. - const memberMauses = members.map((b) => ( - - - {group.busyBotId === b.id && ( - - )} - - )); + // Static profile avatars: one per member, a ring + dot on whoever is + // working. A member actively driving a computer/browser session for this + // room gets the place icon instead of the plain dot, matching the 1:1 + // composer's PlaceChip live indicator. + const memberMauses = members.map((b) => { + const busy = group.busyBotId === b.id; + const task = b.tasks?.find((candidate) => candidate.threadId === group.threadId); + const effective = busy ? effectivePlace(b, task) : "off"; + const showPlace = busy && effective !== "off" && effective !== "auto"; + return ( + + + {busy && ( + showPlace ? ( + + + ) : ( + + ) + )} + + ); + }); return (
From cc6122c4600de73654f6f3a754dac0dc2b8b6987 Mon Sep 17 00:00:00 2001 From: jakequade Date: Fri, 2 Oct 2026 17:15:17 +1000 Subject: [PATCH 025/211] fix(server): release stalled direct Local VM claims safely (#2138) * fix(server): release stalled direct Local VM claims safely * fix(server): keep prompt stall completions on fast path --- server/group-local-vm.e2e.test.ts | 62 +++++++++++++++++++++++++ server/index.ts | 20 ++++---- server/testing/group-local-vm-hooks.mjs | 5 ++ 3 files changed, 79 insertions(+), 8 deletions(-) diff --git a/server/group-local-vm.e2e.test.ts b/server/group-local-vm.e2e.test.ts index e1d42a8e1e..4fb8d283d1 100644 --- a/server/group-local-vm.e2e.test.ts +++ b/server/group-local-vm.e2e.test.ts @@ -618,6 +618,68 @@ describe("Group Local VM ownership on the real isolated server", () => { } }); + it.each(["shared", "per-bot"])("recovers a direct %s VM after a missing completion", async (mode) => { + vmState(); + await api("PATCH", "/api/config", { localVm: { mode, maxInstances: 2 } }); + const { bot } = await api("POST", "/api/bots", { name: `Stalled ${mode} VM` }); + try { + await api("PATCH", `/api/bots/${bot.id}`, { computer: "vm", browser: false }); + rmSync(dumpFile, { force: true }); rmSync(finishFile, { force: true }); + await api("POST", `/api/bots/${bot.id}/messages`, { text: "Hold the VM" }); + const first = computer(await dump()); + expect((await gate(first)).status).toBe(200); + vmState({ dropCompletion: true, stall: true }); + await until(() => api("GET", "/api/bots?messages=30"), state => JSON.stringify(state).includes("the turn was stopped")); + await idle(bot.id); + expect((await gate(first)).status).toBe(401); + vmState(); rmSync(dumpFile, { force: true }); + const { task } = await api("POST", `/api/bots/${bot.id}/tasks`, {}); + await api("POST", `/api/bots/${bot.id}/messages`, { text: "Use the VM again", threadId: task.threadId }); + const next = computer(await dump()); + expect((await gate(next)).status).toBe(200); + } finally { + vmState({ containers: [] }); writeFileSync(finishFile, "finish"); + await api("POST", `/api/bots/${bot.id}/interrupt`, {}); + await idle(bot.id); + await api("DELETE", `/api/bots/${bot.id}`); + if (mode === "per-bot") await api("PATCH", "/api/config", { localVm: { mode: "shared", maxInstances: 2 } }); + vmState(); + } + }, 40_000); + + it.each([["shared", false], ["shared", true], ["per-bot", false], ["per-bot", true]] as const)("keeps a replacement %s VM turn after a late completion (new task: %s)", async (mode, newTask) => { + vmState(); + await api("PATCH", "/api/config", { localVm: { mode, maxInstances: 2 } }); + const { bot } = await api("POST", "/api/bots", { name: "Late VM completion" }); + try { + await api("PATCH", `/api/bots/${bot.id}`, { computer: "vm", browser: false }); + rmSync(dumpFile, { force: true }); rmSync(finishFile, { force: true }); + await api("POST", `/api/bots/${bot.id}/messages`, { text: "Hold the VM" }); + const first = computer(await dump()); + rmSync(stateFile + ".latecompleted", { force: true }); + vmState({ delayCompletion: 8_000, stall: true }); + await until(() => api("GET", "/api/bots?messages=30"), state => JSON.stringify(state).includes("the turn was stopped")); + await idle(bot.id); + expect((await gate(first)).status).toBe(401); + vmState(); rmSync(dumpFile, { force: true }); + const threadId = newTask ? (await api("POST", `/api/bots/${bot.id}/tasks`, {})).task.threadId : bot.threadId; + await api("POST", `/api/bots/${bot.id}/messages`, { text: "Keep using the VM", threadId }); + const next = computer(await dump()); + expect(existsSync(stateFile + ".latecompleted")).toBe(false); + await until(() => existsSync(stateFile + ".latecompleted"), Boolean); + await new Promise(resolve => setTimeout(resolve, 100)); + expect((await gate(next)).status).toBe(200); + expect((await api("GET", "/api/bots?messages=0")).bots.find((entry: any) => entry.id === bot.id).busy).toBe(true); + } finally { + vmState({ containers: [] }); writeFileSync(finishFile, "finish"); + await api("POST", `/api/bots/${bot.id}/interrupt`, {}); + await idle(bot.id); + await api("DELETE", `/api/bots/${bot.id}`); + if (mode === "per-bot") await api("PATCH", "/api/config", { localVm: { mode: "shared", maxInstances: 2 } }); + vmState(); + } + }, 30_000); + // Linux accepts only its own validated runtime descriptor, which a fixture // cannot forge; the macOS and Windows descriptor is a plain file. it.skipIf(process.platform === "linux")("mounts a channel speaker's own This computer destination behind the control gate", async () => { diff --git a/server/index.ts b/server/index.ts index 4baa75c514..540f2915da 100644 --- a/server/index.ts +++ b/server/index.ts @@ -5696,10 +5696,9 @@ const watchdog = new TurnWatchdog({ onStall: (turn) => { const stalledResourceOwner = turnResourceOwners.get(turn.threadId); const stalledGeneration = directTurnGenerationByThread.get(turn.threadId); + const stalledProviderTurn = directRequestOwners.get(turn.threadId); cancelDirectTurnDispatch(turn.botId, turn.threadId); - // Room targets carry an invocation identity; only those claims belong - // to the room grace cleanup added here. - const stalledVmTarget = groupSpeakers.has(turn.threadId) ? localVmThreadTargets.get(turn.threadId) : undefined; + const stalledVmTarget = localVmThreadTargets.get(turn.threadId); revokeInternalCapabilitiesForThread(turn.threadId); repeats.settle(turn.threadId); const bot = botForThread(turn.botId, turn.threadId); @@ -5735,6 +5734,9 @@ const watchdog = new TurnWatchdog({ const releaseOwnership = () => { if (stalledGeneration && directTurnGenerationByThread.get(turn.threadId) !== stalledGeneration) return; if (stalledResourceOwner && turnResourceOwners.get(turn.threadId)?.generation !== stalledResourceOwner.generation) return; + if (stalledGeneration && stalledProviderTurn?.generation === stalledGeneration && stalledProviderTurn.turnId) { + retireProviderTurn(stalledProviderTurn.turnId); + } // A goal coordinator can stall before sendTurn reveals its provider // turn id. Reusing the room during that ambiguous pre-id window would // make old and replacement events indistinguishable. Keep ownership @@ -5749,7 +5751,8 @@ const watchdog = new TurnWatchdog({ } const group = store.groupByThread(turn.threadId); const speaker = groupSpeakers.get(turn.threadId); - if (group && group.busyBotId === turn.botId && speaker?.botId === turn.botId) { + const stalledRoomSpeaker = group && group.busyBotId === turn.botId && speaker?.botId === turn.botId; + if (stalledRoomSpeaker) { groupSpeakers.delete(turn.threadId); store.patchGroup(group.id, { busyBotId: null, unread: true }); } @@ -5757,7 +5760,7 @@ const watchdog = new TurnWatchdog({ // computer claim. The generation checks above protect replacements. releaseTurnResources(stalledResourceOwner); const currentBot = store.bot(turn.botId); - if (currentBot?.busy) { + if (currentBot?.busy && (threadBusy(currentBot.id, turn.threadId) || stalledRoomSpeaker)) { stopScreenPoller(currentBot.id, turn.threadId); vpsThreadEnded(currentBot.id, turn.threadId); if (store.taskByThread(currentBot.id, turn.threadId)) store.setTaskActivity(currentBot.id, turn.threadId, "idle"); @@ -9295,7 +9298,7 @@ async function startTurn( * desktop, not whatever localVmTargetForBot resolves to by the time * the first screen call arrives. */ const claimAutoLocalVm = async (claimThreadId: string, pinnedTarget?: LocalVmTarget): Promise<{ target: LocalVmTarget; runtime: Runtime }> => { - const localVmTarget = pinnedTarget ?? localVmTargetForThread(bot.id, claimThreadId); + const localVmTarget = pinnedTarget ?? localVmThreadTargets.get(claimThreadId) ?? { ...localVmTargetForThread(bot.id, claimThreadId) }; await bindTurnComputer(resourceOwner, `computer:vm:${localVmTarget.key}`, true); if (localVmImageBusy || localVmModeChangeBusy || localVmLifecycleBusy.has(localVmTarget.key)) { throw new Error("this Local VM is being started, stopped, or replaced — wait for setup to finish"); @@ -9318,6 +9321,7 @@ async function startTurn( // without this the exclusive lease would sit held for the rest of a // turn that never got the VM — the very serialisation #1361 removes. const dropLease = () => { + if (localVmThreadTargets.get(claimThreadId) !== localVmTarget) return; localVmLeaseFor(localVmTarget).release(claimThreadId); if (localVmActiveThreads.get(localVmTarget.key) === claimThreadId) localVmActiveThreads.delete(localVmTarget.key); localVmThreadTargets.delete(claimThreadId); @@ -9407,7 +9411,7 @@ async function startTurn( const poolCandidate = poolLocalVmTarget(localVmSeatPool.candidate(threadId, localVmPoolSeatHolder)); if (!localVmSeen.has(poolCandidate.key) && !opts?.automationSource) return false; } - const localVmTarget = localVmTargetForThread(bot.id, threadId); + const localVmTarget = { ...localVmTargetForThread(bot.id, threadId) }; let lazyReadyVm: { runtime: Runtime } | null = null; if (!strict) { // Nothing this process has ever seen for this target, and nobody is @@ -9479,7 +9483,7 @@ async function startTurn( return true; } catch (error) { if (strict) throw error; - releaseLocalVmThread(threadId); + if (directTurnGenerationByThread.get(threadId) === dispatchClaimId) releaseLocalVmThread(threadId); return false; } }; diff --git a/server/testing/group-local-vm-hooks.mjs b/server/testing/group-local-vm-hooks.mjs index e098be3249..610e71d1f7 100644 --- a/server/testing/group-local-vm-hooks.mjs +++ b/server/testing/group-local-vm-hooks.mjs @@ -52,6 +52,11 @@ registerHooks({ return { ...result, source: `import { readFileSync as readVmClock } from 'node:fs';\n` + String(result.source).replaceAll('Date.now()', `(Date.now() + (JSON.parse(readVmClock(${JSON.stringify(state)}, 'utf8')).clockOffset || 0))`) }; } + if (url.endsWith('/drivers/claude.ts')) { + return { ...result, source: `import { readFileSync as readVmEvents, writeFileSync as writeVmEvent } from 'node:fs';\n` + + String(result.source).replace('for (const l of Array.from(listeners)) l(event);', + `if (event.type === 'turn.completed') {\n const fixture = JSON.parse(readVmEvents(${JSON.stringify(state)}, 'utf8'));\n if (fixture.dropCompletion) return;\n if (fixture.delayCompletion) { setTimeout(() => { writeVmEvent(${JSON.stringify(state)} + '.latecompleted', event.turnId ?? ''); for (const l of Array.from(listeners)) l(event); }, fixture.delayCompletion); return; }\n }\n for (const l of Array.from(listeners)) l(event);`) }; + } if (url.endsWith('/turn-watchdog.ts')) { return { ...result, source: `import { readFileSync as readVmWatch } from 'node:fs';\n` + String(result.source).replace('this.opts = opts;', 'this.opts = { ...opts, checkMs: 30 };') From 208bccc8c99e907d2709a3289ca4c17020cdaffb Mon Sep 17 00:00:00 2001 From: guylfe <127800290+guylfe@users.noreply.github.com> Date: Fri, 2 Oct 2026 10:18:10 +0300 Subject: [PATCH 026/211] Add an option to refresh a thread's permissions (#2132) Settings changes apply to new threads. Refresh permissions on a thread copies the bot's current approval level and saved approvals onto that conversation. --- electron/approval-trusted-mode.cjs | 14 +++- electron/approval-trusted-mode.node-test.mjs | 11 +++ server/independent-task-store.test.ts | 38 +++++++++ server/index.ts | 77 ++++++++++++++++++- server/store.ts | 19 +++++ src/components/Sidebar.tsx | 38 ++++++++- src/components/SidebarThreadRow.test.ts | 28 +++++++ src/components/SidebarThreadRow.tsx | 5 +- .../bot-settings/PermissionsSection.tsx | 2 +- src/locales/en.json | 2 + src/state/store.tsx | 32 +++++++- src/types/ogb.d.ts | 1 + 12 files changed, 257 insertions(+), 10 deletions(-) diff --git a/electron/approval-trusted-mode.cjs b/electron/approval-trusted-mode.cjs index 9e000d6f15..b010e0fe61 100644 --- a/electron/approval-trusted-mode.cjs +++ b/electron/approval-trusted-mode.cjs @@ -8,7 +8,7 @@ function plainObject(value) { return value && typeof value === "object" && !Array.isArray(value); } -function trustedApprovalModeRequest(requestId, botId, mode, acknowledgeLocalAuto = false, threadId, modelSelection, updateBotDefault, threadOnly = false, allThreads = false) { +function trustedApprovalModeRequest(requestId, botId, mode, acknowledgeLocalAuto = false, threadId, modelSelection, updateBotDefault, threadOnly = false, allThreads = false, refreshPermissions = false) { if (typeof requestId !== "string" || !REQUEST_ID.test(requestId)) { throw new Error("invalid trusted approval-mode request id"); } @@ -35,6 +35,11 @@ function trustedApprovalModeRequest(requestId, botId, mode, acknowledgeLocalAuto (!threadOnly && mode !== "full" && mode !== "custom" && modelSelection === undefined))) { throw new Error("invalid thread for trusted approval mode"); } + // A refresh copies the bot's saved level onto one thread. It stays + // thread-scoped so a lost reply cannot downgrade the bot default. + if (typeof refreshPermissions !== "boolean" || (refreshPermissions && (!threadOnly || threadId === undefined || allThreads || modelSelection !== undefined))) { + throw new Error("invalid permission refresh"); + } return { type: "approval-trusted-mode-set", requestId, @@ -44,6 +49,7 @@ function trustedApprovalModeRequest(requestId, botId, mode, acknowledgeLocalAuto ...(threadId !== undefined ? { threadId } : {}), ...(threadOnly ? { threadOnly: true } : {}), ...(allThreads ? { allThreads: true } : {}), + ...(refreshPermissions ? { refreshPermissions: true } : {}), ...(modelSelection !== undefined ? { modelSelection, updateBotDefault } : {}), }; } @@ -124,8 +130,8 @@ function createTrustedApprovalModeCoordinator({ randomId, timeoutMs = 10_000 } = const usedRequestIds = new Set(); const latestRequestByBot = new Map(); - function nextMessage(botId, mode, acknowledgeLocalAuto = false, threadId, modelSelection, updateBotDefault, threadOnly = false, allThreads = false) { - const message = trustedApprovalModeRequest(randomId(), botId, mode, acknowledgeLocalAuto, threadId, modelSelection, updateBotDefault, threadOnly, allThreads); + function nextMessage(botId, mode, acknowledgeLocalAuto = false, threadId, modelSelection, updateBotDefault, threadOnly = false, allThreads = false, refreshPermissions = false) { + const message = trustedApprovalModeRequest(randomId(), botId, mode, acknowledgeLocalAuto, threadId, modelSelection, updateBotDefault, threadOnly, allThreads, refreshPermissions); if (usedRequestIds.has(message.requestId)) { throw new Error("Trusted approval-mode request id was reused"); } @@ -148,7 +154,7 @@ function createTrustedApprovalModeCoordinator({ randomId, timeoutMs = 10_000 } = } let message; try { - message = nextMessage(botId, mode, options.acknowledgeLocalAuto ?? false, options.threadId, options.modelSelection, options.updateBotDefault, options.threadOnly ?? false, options.allThreads ?? false); + message = nextMessage(botId, mode, options.acknowledgeLocalAuto ?? false, options.threadId, options.modelSelection, options.updateBotDefault, options.threadOnly ?? false, options.allThreads ?? false, options.refreshPermissions ?? false); } catch (error) { return Promise.reject(error); } diff --git a/electron/approval-trusted-mode.node-test.mjs b/electron/approval-trusted-mode.node-test.mjs index 0a1fbedd2e..3a4d17bda0 100644 --- a/electron/approval-trusted-mode.node-test.mjs +++ b/electron/approval-trusted-mode.node-test.mjs @@ -77,6 +77,17 @@ test("thread-only options reject missing or mixed scopes", () => { assert.throws(() => trustedApprovalModeRequest(REQUEST_ID, "bot-1", "full", false, "thread-1", undefined, true, true), /thread-only/); }); +test("permission refresh stays on one thread", () => { + const message = trustedApprovalModeRequest(REQUEST_ID, "bot-1", "auto", false, "thread-1", undefined, undefined, true, false, true); + assert.equal(message.refreshPermissions, true); + assert.equal(message.threadOnly, true); + assert.equal(message.threadId, "thread-1"); + // A refresh without thread scope would let a lost reply downgrade the bot. + assert.throws(() => trustedApprovalModeRequest(REQUEST_ID, "bot-1", "full", false, "thread-1", undefined, undefined, false, false, true), /permission refresh/); + assert.throws(() => trustedApprovalModeRequest(REQUEST_ID, "bot-1", "full", false, undefined, undefined, undefined, false, true, true), /permission refresh/); + assert.throws(() => trustedApprovalModeRequest(REQUEST_ID, "bot-1", "ask", false, undefined, undefined, undefined, false, false, "yes"), /permission refresh/); +}); + function idSequence(...ids) { let index = 0; return () => ids[index++] ?? (() => { throw new Error("test request id sequence exhausted"); })(); diff --git a/server/independent-task-store.test.ts b/server/independent-task-store.test.ts index 0062cb0352..8020a4eba5 100644 --- a/server/independent-task-store.test.ts +++ b/server/independent-task-store.test.ts @@ -41,6 +41,44 @@ describe("independent bot task state", () => { expect(new Store(selection).tasks(bot.id).every(task => task.approvalMode === "ask")).toBe(true); }); + it("refreshes one thread's permissions from the bot default and leaves the rest", () => { + const store = new Store(selection); + const bot = store.createBot({}, { seedMessages: false }); + const first = bot.threadId; + // The opening thread inherits until it has its own copy. A later thread, + // and any thread whose level was chosen in the composer, keeps that copy. + store.patchTask(bot.id, first, { approvalMode: "ask", autoApprove: false, alwaysAllow: ["Read"] }); + const model = store.taskByThread(bot.id, first)?.modelSelection; + store.appendMessage(first, { role: "user", kind: "text", text: "Keep this conversation" }); + const history = store.messagesFor(first); + const sibling = store.createTask(bot.id, "Sibling", false)!; + store.patchBot(bot.id, { approvalMode: "auto", autoApprove: true, alwaysAllow: ["Read", "Bash"] }); + expect(store.taskByThread(bot.id, first)).toMatchObject({ approvalMode: "ask", autoApprove: false, alwaysAllow: ["Read"], modelSelection: model }); + expect(store.taskByThread(bot.id, sibling.threadId)).toMatchObject({ approvalMode: "ask", alwaysAllow: [] }); + + const refreshed = store.refreshTaskPermissions(bot.id, first); + expect(refreshed).toMatchObject({ approvalMode: "auto", autoApprove: true, alwaysAllow: ["Read", "Bash"], modelSelection: model }); + expect(store.taskByThread(bot.id, sibling.threadId)).toMatchObject({ approvalMode: "ask", autoApprove: false, alwaysAllow: [] }); + expect(store.bot(bot.id)).toMatchObject({ approvalMode: "auto", autoApprove: true, alwaysAllow: ["Read", "Bash"] }); + expect(store.messagesFor(first)).toEqual(history); + expect(store.refreshTaskPermissions(bot.id, "missing")).toBeNull(); + + const reloaded = new Store(selection); + expect(reloaded.taskByThread(bot.id, first)).toMatchObject({ approvalMode: "auto", autoApprove: true, alwaysAllow: ["Read", "Bash"] }); + expect(reloaded.taskByThread(bot.id, sibling.threadId)).toMatchObject({ approvalMode: "ask", alwaysAllow: [] }); + expect(reloaded.messagesFor(first)).toEqual(history); + + reloaded.patchBot(bot.id, { + approvalMode: "full", + approvalGrant: { requestId: "grant-1", mode: "full", phase: "prepared", threadId: first, threadOnly: true, refreshPermissions: true }, + }); + expect(approvalModeFor(reloaded.bot(bot.id)!)).toBe("ask"); + const caughtUp = reloaded.refreshTaskPermissions(bot.id, sibling.threadId); + expect(caughtUp).toMatchObject({ approvalMode: "full", autoApprove: false, alwaysAllow: ["Read", "Bash"] }); + expect(reloaded.taskByThread(bot.id, first)?.approvalMode).toBe("auto"); + expect(reloaded.bot(bot.id)?.approvalMode).toBe("full"); + }); + it("does not partially elevate any thread when saving the all-threads change fails", () => { const store = new Store(selection); const bot = store.createBot({}, { seedMessages: false }); diff --git a/server/index.ts b/server/index.ts index 540f2915da..c9c53ad9ea 100644 --- a/server/index.ts +++ b/server/index.ts @@ -3896,6 +3896,8 @@ function handleDesktopTrustedApprovalMessage(raw: unknown): boolean { ) { if (bot.approvalGrant.allThreads && mode === "full") { store.setAllThreadApprovalMode(botId, mode); + } else if (bot.approvalGrant.refreshPermissions && bot.approvalGrant.threadId) { + store.refreshTaskPermissions(botId, bot.approvalGrant.threadId); } else if (bot.approvalGrant.threadId) { store.patchTask(botId, bot.approvalGrant.threadId, { approvalMode: mode, autoApprove: false }); } @@ -4104,6 +4106,38 @@ function handleDesktopTrustedApprovalMessage(raw: unknown): boolean { respond({ ok: false, error: "Invalid thread approval scope" }); return true; } + if (message.refreshPermissions !== undefined && message.refreshPermissions !== true) { + respond({ ok: false, error: "Invalid permission refresh" }); + return true; + } + if (message.refreshPermissions === true) { + const target = typeof threadId === "string" ? store.projectBotForTask(botId, threadId) : null; + const savedMode = approvalModeFor({ ...existing, approvalGrant: undefined }); + if (message.threadOnly !== true || !target || message.allThreads === true || message.modelSelection !== undefined || message.updateBotDefault !== undefined || savedMode !== mode) { + respond({ ok: false, error: "Choose this bot's approval level in bot settings before refreshing a thread" }); + return true; + } + if (existing.approvalGrant || threadBusy(botId, target.threadId)) { + respond({ ok: false, error: "Stop this thread and finish its pending approval change first" }); + return true; + } + if (!supportsApprovalMode(target.modelSelection, mode)) { + respond({ ok: false, error: "This thread's provider does not support that approval level" }); + return true; + } + if (mode === "auto" && target.computer === "local" && approvalModeFor(target) !== "auto" && message.acknowledgeLocalAuto !== true) { + respond({ ok: false, error: "Auto mode on this computer requires confirming the warning" }); + return true; + } + if (mode === "full" || mode === "custom") { + store.patchBot(botId, { approvalGrant: { requestId, mode, phase: "prepared", threadId: target.threadId, threadOnly: true, refreshPermissions: true } }); + } else if (!store.refreshTaskPermissions(botId, target.threadId)) { + respond({ ok: false, error: "That thread is no longer available" }); + return true; + } + respond({ ok: true, bot: wireTrustedApprovalBot(store.bot(botId)!) }); + return true; + } if (message.threadOnly === true) { const target = typeof threadId === "string" ? store.projectBotForTask(botId, threadId) : null; if (!target || message.modelSelection !== undefined || message.updateBotDefault !== undefined) { @@ -21319,7 +21353,7 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (!body || typeof body !== "object" || Array.isArray(body)) return json(res, 400, { error: "body must be a JSON object" }); const current = store.projectBotForTask(m[1], m[2]); if (!current) return json(res, 404, { error: "no such task" }); - const allowed = new Set(["title", "projectId", "modelSelection", "updateBotDefault", "resetApprovalToAsk", "approvalMode", "autoApprove", "requireAvailableModel", "pinnedMessageId", "acknowledgeLocalAuto", "archivedAt", "pinned", "snoozedUntil", "surface"]); + const allowed = new Set(["title", "projectId", "modelSelection", "updateBotDefault", "resetApprovalToAsk", "approvalMode", "autoApprove", "requireAvailableModel", "pinnedMessageId", "acknowledgeLocalAuto", "archivedAt", "pinned", "snoozedUntil", "surface", "refreshPermissions"]); if (Object.keys(body).some((key) => !allowed.has(key))) return json(res, 400, { error: "unsupported thread setting" }); const notYours = cloudThreadRefusal(auth, m[2]); if (notYours) return json(res, 403, { error: notYours }); @@ -21327,7 +21361,7 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { // asking them, even in a conversation someone else started, and what // the bot itself defaults to. if (CLOUD_HOME && !cloudOwnerSession(auth) && - (body.approvalMode !== undefined || body.autoApprove !== undefined || body.acknowledgeLocalAuto !== undefined || body.updateBotDefault === true)) { + (body.approvalMode !== undefined || body.autoApprove !== undefined || body.acknowledgeLocalAuto !== undefined || body.updateBotDefault === true || body.refreshPermissions === true)) { return json(res, 403, { error: "On this Cloud only its owner can change how a bot asks for approval, or its default model." }); } for (const key of ["requireAvailableModel", "acknowledgeLocalAuto", "updateBotDefault", "resetApprovalToAsk"] as const) { @@ -21339,6 +21373,45 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { (body.approvalMode !== undefined && body.approvalMode !== "ask") || body.autoApprove === true)) { return json(res, 400, { error: "resetApprovalToAsk requires a model selection and cannot be combined with another approval mode" }); } + if (body.refreshPermissions !== undefined && body.refreshPermissions !== true) { + return json(res, 400, { error: "refreshPermissions must be true" }); + } + if (body.refreshPermissions === true) { + const extra = Object.keys(body).filter((key) => key !== "refreshPermissions" && key !== "acknowledgeLocalAuto"); + if (extra.length) return json(res, 400, { error: "refreshPermissions cannot be combined with other thread settings" }); + const profile = store.bot(m[1]); + if (!profile) return json(res, 404, { error: "no such bot" }); + if (profile.approvalGrant) return json(res, 409, { error: "the bot's approval mode is still being confirmed" }); + if (threadBusy(profile.id, m[2])) return json(res, 409, { error: "stop this thread before changing its approval mode" }); + const nextMode = approvalModeFor({ ...profile, approvalGrant: undefined }); + // Full and Custom never travel over the bot-reachable HTTP surface. + // The desktop's private channel copies those levels, and it is also + // the only way to leave Custom. + if (nextMode === "full" || nextMode === "custom") { + return json(res, 403, { error: "Refresh Full or Custom access from the packaged desktop app" }); + } + if (approvalModeFor(current) === "custom") { + return json(res, 403, { error: "Leaving Custom approval requires the packaged desktop app" }); + } + if (!supportsApprovalMode(current.modelSelection, nextMode)) { + return json(res, 400, { error: "This provider does not support the selected approval level" }); + } + if (nextMode === "auto" && current.computer === "local" && approvalModeFor(current) !== "auto" && body.acknowledgeLocalAuto !== true) { + return json(res, 400, { error: "Auto mode on this computer requires confirming the warning first (acknowledgeLocalAuto)" }); + } + const approvalRank = { ask: 0, edits: 1, auto: 2, full: 3, custom: 4 } as const; + const widensMode = approvalRank[nextMode] > approvalRank[approvalModeFor(current)]; + const nextAllow = profile.alwaysAllow ?? []; + const widensAllow = nextAllow.some((key) => !(current.alwaysAllow ?? []).includes(key)); + if ((widensMode || widensAllow) && auth.kind === "loopback" && !DESKTOP_MANAGED && !req.headers.origin && store.bots.some((bot) => bot.busy)) { + return json(res, 409, { error: "Change approval mode from the app or a paired device while bots are working." }); + } + const task = store.refreshTaskPermissions(profile.id, m[2]); + if (!task) return json(res, 404, { error: "no such task" }); + const fresh = botWithThread(store.bot(profile.id)!); + broadcast({ kind: "bot", bot: fresh }); + return json(res, 200, { task: wireTask(task), bot: fresh }); + } const patch: Parameters[2] = {}; if (body.projectId !== undefined) { if (body.projectId === null) patch.projectId = undefined; diff --git a/server/store.ts b/server/store.ts index 626ae7d69c..3c922f23c9 100644 --- a/server/store.ts +++ b/server/store.ts @@ -379,6 +379,8 @@ export interface BotRecord extends Omit { threadId?: string; /** Composer grant: leave the bot default and other threads unchanged. */ threadOnly?: true; + /** Copy the saved bot default onto threadId, including standing approvals. */ + refreshPermissions?: true; /** Explicit bot-wide grant, including existing threads. */ allThreads?: true; }; @@ -2516,6 +2518,23 @@ export class Store { return bot; } + /** Copy this bot's saved approval level and standing approvals onto one + * thread. A new thread already gets them; this is how an existing + * conversation catches up. Other threads, the transcript, and the bot + * default stay put. A grant that has not committed yet is ignored, so a + * refresh cannot copy the temporary Ask mask. */ + refreshTaskPermissions(botId: string, threadId: string): TaskRecord | null { + const bot = this.bot(botId); + const task = this.taskByThread(botId, threadId); + if (!bot || !task) return null; + const mode = approvalModeFor({ ...bot, approvalGrant: undefined }); + const alwaysAllow = structuredClone(bot.alwaysAllow ?? []); + const autoApprove = mode === "auto"; + const sameAllow = JSON.stringify(task.alwaysAllow ?? []) === JSON.stringify(alwaysAllow); + if (task.approvalMode === mode && task.autoApprove === autoApprove && sameAllow) return task; + return this.patchTask(botId, threadId, { approvalMode: mode, autoApprove, alwaysAllow }); + } + private mirrorActiveTask(bot: BotRecord, task: TaskRecord) { bot.threadId = task.threadId; bot.resumeCursors = structuredClone(task.resumeCursors); diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index acf611e022..a49d4aa2b4 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -34,6 +34,7 @@ import { X, } from "lucide-react"; import { api, useStore, formatTime, visibleMessages, currentTaskBot, type AppState, type Bot, type Group } from "@/state/store"; +import { approvalModeFor } from "../../shared/approval-mode"; import { peerLine } from "@/lib/peer-message"; import { liveActivityLabel } from "@/lib/live-activity"; import { llmThreadTitlesEnabled } from "@/lib/feature-flags"; @@ -47,6 +48,8 @@ import { t } from "@/lib/i18n"; import { isRoutineProblemRun } from "@/lib/routines"; import type { LocaleKey } from "@/locales"; import { ConfirmDialog } from "./ConfirmDialog"; +import { FullAccessWarning } from "./FullAccessWarning"; +import { LocalComputerAutoWarning } from "./LocalComputerAutoWarning"; import { WorkingDots } from "./WorkingIndicator"; import { nextRename } from "@/lib/rename"; import { useDesktopCapabilities } from "./DesktopCapabilities"; @@ -959,6 +962,7 @@ export function BotThreadList({ bot, selected, density = "comfortable", query = const [folderDrop, setFolderDrop] = useState<{ id: string; place: "before" | "after" } | null>(null); const draggingFolder = useRef(null); const [showAll, setShowAll] = useState(false); + const [permissionRefresh, setPermissionRefresh] = useState<{ threadId: string; kind: "full" | "local-auto" } | null>(null); const currentProjectId = tasks.find((task) => task.threadId === bot.threadId)?.projectId; useEffect(() => { if (selected && currentProjectId) setCollapsed((previous) => { @@ -996,7 +1000,20 @@ export function BotThreadList({ bot, selected, density = "comfortable", query = onMove={(projectId) => dispatch({ type: "updateTask", botId: bot.id, threadId: task.threadId, patch: { projectId } })} onArchive={(archivedAt) => dispatch({ type: "updateTask", botId: bot.id, threadId: task.threadId, patch: { archivedAt } })} onPin={(pinned) => dispatch({ type: "updateTask", botId: bot.id, threadId: task.threadId, patch: { pinned } })} - onSnooze={(snoozedUntil) => dispatch({ type: "updateTask", botId: bot.id, threadId: task.threadId, patch: { snoozedUntil } })} />; + onSnooze={(snoozedUntil) => dispatch({ type: "updateTask", botId: bot.id, threadId: task.threadId, patch: { snoozedUntil } })} + onRefreshPermissions={() => { + const mode = approvalModeFor(bot); + const threadMode = approvalModeFor(thread); + if (mode === "full" && threadMode !== "full") { + setPermissionRefresh({ threadId: task.threadId, kind: "full" }); + return; + } + if (mode === "auto" && bot.computer === "local" && threadMode !== "auto") { + setPermissionRefresh({ threadId: task.threadId, kind: "local-auto" }); + return; + } + dispatch({ type: "refreshTaskPermissions", botId: bot.id, threadId: task.threadId }); + }} />; }; const ungrouped = visibleTasks.filter((task) => !projects.some((project) => project.id === task.projectId)); const projectToEdit = projects.find((project) => project.id === editingProject); @@ -1100,6 +1117,25 @@ export function BotThreadList({ bot, selected, density = "comfortable", query = {projects.length > 0 && ungrouped.length > 0 &&
{t("task.list")}
} {ungrouped.map(renderThread)} {!query && !showAll && tasks.length > visibleTasks.length && } + setPermissionRefresh(null)} + onConfirm={() => { + const threadId = permissionRefresh?.threadId; + setPermissionRefresh(null); + if (threadId) dispatch({ type: "refreshTaskPermissions", botId: bot.id, threadId }); + }} + /> + setPermissionRefresh(null)} + onConfirm={() => { + const threadId = permissionRefresh?.threadId; + setPermissionRefresh(null); + if (threadId) dispatch({ type: "refreshTaskPermissions", botId: bot.id, threadId, acknowledgeLocalAuto: true }); + }} + /> {projectToEdit && setEditingProject(null)} />} }
diff --git a/src/components/SidebarThreadRow.test.ts b/src/components/SidebarThreadRow.test.ts index 7e2883dbed..d9a87a8e24 100644 --- a/src/components/SidebarThreadRow.test.ts +++ b/src/components/SidebarThreadRow.test.ts @@ -528,6 +528,34 @@ describe("Copy link", () => { }); }); +describe("Refresh permissions", () => { + const openMenu = (tree: RowNode) => { + (moreMenuButton(tree)!.props!.onClick as (event: unknown) => void)({ currentTarget: { getBoundingClientRect: () => ({ left: 100, bottom: 200 }) } }); + }; + + it("is offered only when the row can refresh, and stays disabled while the thread is working", () => { + vi.stubGlobal("window", { innerWidth: 1024, innerHeight: 768 }); + vi.stubGlobal("document", { body: { nodeType: 1 } }); + const task = { threadId: "t1", title: "Fix the login" }; + openMenu(renderRow(task, "scout")); + expect(buttonWithLabel(renderRow(task, "scout", false), "Refresh permissions")).toBeUndefined(); + + const onRefreshPermissions = vi.fn(); + openMenu(renderRow(task, "scout", true, { onRefreshPermissions })); + const offered = buttonWithLabel(renderRow(task, "scout", false, { onRefreshPermissions }), "Refresh permissions"); + expect(offered?.props?.disabled).toBe(false); + expect(offered?.props?.title).toBe("Apply this bot's current approval level and saved approvals to this thread. Other threads stay as they are."); + (offered!.props!.onClick as () => void)(); + expect(onRefreshPermissions).toHaveBeenCalledTimes(1); + + const working = { ...task, activity: "working" as const }; + openMenu(renderRow(working, "scout", true, { onRefreshPermissions })); + const busy = buttonWithLabel(renderRow(working, "scout", false, { onRefreshPermissions }), "Refresh permissions"); + expect(busy?.props?.disabled).toBe(true); + vi.unstubAllGlobals(); + }); +}); + describe("Regenerate title", () => { const openMenu = (tree: RowNode) => { (moreMenuButton(tree)!.props!.onClick as (event: unknown) => void)({ currentTarget: { getBoundingClientRect: () => ({ left: 100, bottom: 200 }) } }); diff --git a/src/components/SidebarThreadRow.tsx b/src/components/SidebarThreadRow.tsx index 958b44f5b0..f660fafc6b 100644 --- a/src/components/SidebarThreadRow.tsx +++ b/src/components/SidebarThreadRow.tsx @@ -226,7 +226,7 @@ export function orderedSidebarThreads(tasks: T[], activ /** One quiet row for bot and group histories. Surface denotes selection; * working/waiting/unread remain independent signals, never different cards. */ -export function SidebarThreadRow({ task, ownerId, current, compact, folders, onSelect, onRename, onRegenerateTitle, onDelete, onMove, onArchive, onPin, onSnooze, activityLabel, now }: { +export function SidebarThreadRow({ task, ownerId, current, compact, folders, onSelect, onRename, onRegenerateTitle, onDelete, onMove, onArchive, onPin, onSnooze, onRefreshPermissions, activityLabel, now }: { task: ThreadRowTask; /** the bot or room that owns the thread: the link's ?bot= */ ownerId: string; @@ -248,6 +248,8 @@ export function SidebarThreadRow({ task, ownerId, current, compact, folders, onS onArchive?: (archivedAt: number | null) => void; onPin?: (pinned: boolean) => void; onSnooze?: (snoozedUntil: number | null) => void; + /** Copy this bot's current approval level and saved approvals onto this thread. */ + onRefreshPermissions?: () => void; }) { const [menu, setMenu] = useState<{ left: number; top: number } | null>(null); const menuMotion = useHeldMenuMotion(menu); @@ -368,6 +370,7 @@ export function SidebarThreadRow({ task, ownerId, current, compact, folders, onS
} {onSnooze && snoozed && } + {onRefreshPermissions && }
, document.body)}
Approval level
- {draft ? "Default for the new bot's threads, routines and delegated work." : "Default for new threads, routines and delegated work. When enabling Full access, you can also apply it to every existing thread."} + {draft ? "Default for the new bot's threads, routines and delegated work." : "Default for new threads, routines and delegated work. When enabling Full access, you can also apply it to every existing thread. Use Refresh permissions on a thread to apply the current level to that conversation."}
diff --git a/src/locales/en.json b/src/locales/en.json index 6f21861224..bc0e26ac66 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -1207,6 +1207,8 @@ "task.closed": "Closed", "task.archive": "Archive", "task.copyLink": "Copy link", + "task.refreshPermissions": "Refresh permissions", + "task.refreshPermissionsHint": "Apply this bot's current approval level and saved approvals to this thread. Other threads stay as they are.", "task.unarchive": "Unarchive", "task.archived": "Archived", "task.snooze": "Snooze", diff --git a/src/state/store.tsx b/src/state/store.tsx index cd0be64eb9..0fc6c6efa4 100644 --- a/src/state/store.tsx +++ b/src/state/store.tsx @@ -1167,6 +1167,7 @@ export type Action = | { type: "newBot"; role?: BotRole; visibility?: BotVisibility; section?: string; preserveSelection?: boolean; onCreated?: (bot: Bot) => void; onError?: (message: string) => void } | { type: "botCreationPending"; on: boolean } | { type: "updateTask"; botId: string; threadId: string; patch: TaskUpdatePatch } + | { type: "refreshTaskPermissions"; botId: string; threadId: string; acknowledgeLocalAuto?: boolean } | { type: "createProject"; botId: string; name: string; emoji?: string | null; onCreated?: (project: BotProject) => void; onError?: (message: string) => void } | { type: "updateProject"; botId: string; projectId: string; patch: ProjectUpdatePatch; onSaved?: () => void; onError?: (message: string) => void } | { type: "deleteProject"; botId: string; projectId: string; onDeleted?: () => void; onError?: (message: string) => void } @@ -2310,6 +2311,7 @@ export function reducer(state: AppState, action: Action): AppState { case "cancelRoutineRun": case "markRoutineRunSeen": case "markAllRoutineRunsSeen": + case "refreshTaskPermissions": return state; case "sendGroup": { if (!action.sendId) return state; @@ -2453,7 +2455,7 @@ type TrustedApprovalBridge = { setMode( botId: string, mode: ApprovalMode, - options?: { acknowledgeLocalAuto?: boolean; threadId?: string; threadOnly?: boolean; allThreads?: boolean }, + options?: { acknowledgeLocalAuto?: boolean; threadId?: string; threadOnly?: boolean; allThreads?: boolean; refreshPermissions?: boolean }, ): Promise; }; @@ -3444,6 +3446,34 @@ export function StoreProvider({ children }: { children: ReactNode }) { case "updateTask": persistTaskPatch(action.botId, action.threadId, action.patch); break; + case "refreshTaskPermissions": { + const bot = stateRef.current.bots.find((candidate) => candidate.id === action.botId); + const task = bot?.tasks?.find((candidate) => candidate.threadId === action.threadId); + if (!bot || !task) { + showError(new Error("That thread is no longer available")); + break; + } + const mode = approvalModeFor(bot); + const current = approvalModeFor(currentTaskBot(bot, action.threadId)); + const acknowledgeLocalAuto = action.acknowledgeLocalAuto === true; + // Full, Custom, and leaving Custom stay on the private desktop + // channel. Ask, Edits, and Auto can use the thread settings route. + const needsDesktop = mode === "full" || mode === "custom" || current === "custom"; + const refreshed = needsDesktop + ? window.ogb?.approvals + ? window.ogb.approvals.setMode(action.botId, mode, { + threadId: action.threadId, threadOnly: true, refreshPermissions: true, acknowledgeLocalAuto, + }) + : Promise.reject(new Error("This approval change requires the packaged desktop app")) + : api<{ bot: BotAnnouncement }>(`/api/bots/${action.botId}/tasks/${action.threadId}`, { + method: "PATCH", + body: JSON.stringify({ refreshPermissions: true, ...(acknowledgeLocalAuto ? { acknowledgeLocalAuto: true } : {}) }), + }).then((result) => result.bot); + void refreshed.then((updated) => { + if (updated) rawDispatch({ type: "botPatched", bot: withTaskWrites(updated) }); + }).catch(showError); + break; + } case "createProject": api(`/api/bots/${action.botId}/projects`, { method: "POST", body: JSON.stringify({ name: action.name, emoji: action.emoji }) }) .then(({ bot, project }) => { diff --git a/src/types/ogb.d.ts b/src/types/ogb.d.ts index 9718e4e46d..7a7d5020c9 100644 --- a/src/types/ogb.d.ts +++ b/src/types/ogb.d.ts @@ -188,6 +188,7 @@ const __APP_VERSION__: string; botId: string, mode: import("../../shared/approval-mode").ApprovalMode, options?: { acknowledgeLocalAuto?: boolean; threadId?: string; threadOnly?: boolean; allThreads?: boolean; + refreshPermissions?: boolean; modelSelection?: import("../state/store").ModelSelection; updateBotDefault?: boolean }, ): Promise; }; From 858c1a2ae7dd18071031971694e09c895adbfb65 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 12:49:09 +0530 Subject: [PATCH 027/211] test(acp): keep quiet-agent checks deterministic on every platform --- server/drivers/acp/acp.test.ts | 5 +++++ server/drivers/acp/quiet-status.test.ts | 4 +++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/server/drivers/acp/acp.test.ts b/server/drivers/acp/acp.test.ts index 64386778f8..75aa93a70a 100644 --- a/server/drivers/acp/acp.test.ts +++ b/server/drivers/acp/acp.test.ts @@ -26,6 +26,7 @@ import { CursorAgentDriver } from "./cursor.ts"; import { QwenAgentDriver } from "./qwen.ts"; import { removeTempDir } from "../../testing/cleanup.ts"; import * as procs from "../../procs.ts"; +import * as quietStatus from "./quiet-status.ts"; const FAKE_CLI = join(dirname(fileURLToPath(import.meta.url)), "..", "..", "testing", "fake-acp-cli.ts"); @@ -1291,6 +1292,9 @@ describe("ACP turns (fake CLI)", () => { // A quiet agent is not a black box: the person is told what it is doing. it("tells the person what a quiet agent is doing, then finishes the turn", async () => { + // Test the driver's notices, not how fast this OS starts ps/PowerShell. + // The process probe has its own checks in quiet-status.test.ts. + vi.spyOn(quietStatus, "sampleProcessTree").mockResolvedValue({ cpuMs: 0, connections: 1 }); process.env.OMB_ACP_QUIET_NOTICE_MS = "150"; process.env.OMB_ACP_QUIET_TICK_MS = "50"; process.env.FAKE_ACP_QUIET_MS = "700"; @@ -1304,6 +1308,7 @@ describe("ACP turns (fake CLI)", () => { }); it("reads Qwen's debug log: a logged rate-limit retry is reported and keeps the turn alive", async () => { + vi.spyOn(quietStatus, "sampleProcessTree").mockResolvedValue({ cpuMs: 0, connections: 1 }); process.env.QWEN_HOME = scratch; process.env.OPENMAUS_ACP_PROMPT_IDLE_TIMEOUT_MS = "400"; process.env.OMB_ACP_QUIET_NOTICE_MS = "150"; diff --git a/server/drivers/acp/quiet-status.test.ts b/server/drivers/acp/quiet-status.test.ts index 0f520ca638..91d26de8ed 100644 --- a/server/drivers/acp/quiet-status.test.ts +++ b/server/drivers/acp/quiet-status.test.ts @@ -126,7 +126,9 @@ describe("process probe", () => { await new Promise((resolve) => socket.once("connect", () => resolve())); try { const sample = await sampleProcessTree(process.pid); - expect(sample.cpuMs).toBeGreaterThan(0); + // Linux ps reports whole CPU seconds: a healthy young worker can read 0. + expect(sample.cpuMs).not.toBeNull(); + expect(sample.cpuMs!).toBeGreaterThanOrEqual(0); // lsof may be missing on a minimal Linux box: unknown, never zero if (sample.connections !== null) expect(sample.connections).toBeGreaterThanOrEqual(1); } finally { From 5550aa1529011ec8a5d115eacbf86c2c66e65cf3 Mon Sep 17 00:00:00 2001 From: Aditya Umale Date: Thu, 1 Oct 2026 21:34:51 +0530 Subject: [PATCH 028/211] fix(engines): find Cursor installed on Windows without a restart OpenMausBot installs Cursor from Settings with cursor.com's Windows script, which puts cursor-agent.* (and `agent` copies) in %LOCALAPPDATA%\cursor-agent and adds that folder to the user PATH. Windows never pushes PATH changes into a running process, so the engine stayed "`cursor-agent` CLI not found" until the app restarted, while `agent` worked in a terminal. Every other engine we ship an install command for already had its install folder scanned. Scan %LOCALAPPDATA%\cursor-agent with the other Windows install locations, and say in the Cursor docs that the installer's `cursor-agent` sits beside the `agent` command Cursor's docs use. Fixes MOCA-272 Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 87fd06a4227c8133bd31a235081be1e1eff221da) --- docs/cursor.md | 12 ++++++++---- server/env-path.test.ts | 30 ++++++++++++++++++++++++++++++ server/env-path.ts | 7 +++++-- 3 files changed, 43 insertions(+), 6 deletions(-) diff --git a/docs/cursor.md b/docs/cursor.md index be15149e6d..e14ca46487 100644 --- a/docs/cursor.md +++ b/docs/cursor.md @@ -22,10 +22,14 @@ key. 2. Sign in with `cursor-agent login`, or set `CURSOR_API_KEY` / `CURSOR_AUTH_TOKEN` in the environment of the Cursor instance. -3. Confirm `cursor-agent --version` works. The binary installs to `~/.local/bin` by - default; OpenMausBot already looks there when launched from a GUI. - -The engine stays unavailable until the `cursor-agent` executable is on PATH. A +3. Confirm `cursor-agent --version` works. Cursor's docs call the command `agent`; + the installer adds `cursor-agent` beside it, and OpenMausBot runs that name, + because other tools also install an `agent`. The binary installs to + `~/.local/bin` by default (`%LOCALAPPDATA%\cursor-agent` on Windows); + OpenMausBot already looks in both, so a CLI installed while the app is open + is found without restarting. + +The engine stays unavailable until the `cursor-agent` executable is found. A missing login shows as unauthenticated rather than crashing the fleet. ## Models diff --git a/server/env-path.test.ts b/server/env-path.test.ts index e868a9e084..f5b1359f20 100644 --- a/server/env-path.test.ts +++ b/server/env-path.test.ts @@ -9,6 +9,7 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { augmentedPath, + findCliCandidates, harnessHome, registerPathDir, resetPathCache, @@ -201,6 +202,35 @@ describe("augmentedPath", () => { rmSync(programFiles, { recursive: true, force: true }); } }); + + // MOCA-272: OMB installs Cursor from Settings with cursor.com's Windows + // script, which puts cursor-agent.* (and `agent` copies) in + // %LOCALAPPDATA%\cursor-agent and adds that to the user PATH — which a + // running app never sees. Simulated so it runs on every platform. + it("finds Cursor installed after launch on Windows", () => { + const realPlatform = process.platform; + const previous = { LOCALAPPDATA: process.env.LOCALAPPDATA, PATHEXT: process.env.PATHEXT }; + const localAppData = mkdtempSync(join(tmpdir(), "omb-localappdata-")); + try { + Object.defineProperty(process, "platform", { value: "win32" }); + process.env.LOCALAPPDATA = localAppData; + process.env.PATHEXT = ".COM;.EXE;.BAT;.CMD"; + const cursorDir = join(localAppData, "cursor-agent"); + mkdirSync(join(cursorDir, "versions", "2026.09.28-64d2043"), { recursive: true }); + for (const name of ["cursor-agent.cmd", "cursor-agent.ps1", "agent.cmd", "agent.ps1"]) writeFileSync(join(cursorDir, name), "@echo off\n"); + resetPathCacheForTests(); + expect(augmentedPath().split(delimiter)).toContain(cursorDir); + expect(findCliCandidates("cursor-agent").map((path) => path.toLowerCase())).toContain(join(cursorDir, "cursor-agent.cmd").toLowerCase()); + } finally { + Object.defineProperty(process, "platform", { value: realPlatform }); + for (const [name, value] of Object.entries(previous)) { + if (value === undefined) delete process.env[name]; + else process.env[name] = value; + } + resetPathCacheForTests(); + rmSync(localAppData, { recursive: true, force: true }); + } + }); }); describe("userHome / harnessHome", () => { diff --git a/server/env-path.ts b/server/env-path.ts index 662ce926cd..718b27c658 100644 --- a/server/env-path.ts +++ b/server/env-path.ts @@ -67,8 +67,9 @@ function knownDirs(): string[] { * invisible until it restarts, because Windows never pushes PATH changes * into a live process. Scanning the standard install locations recovers * those without a restart — `~/.grok/bin` (the x.ai installer) and - * `%APPDATA%\npm` (global npm shims), plus `%LOCALAPPDATA%\agy\bin`, cover - * every engine we ship an install command for. */ + * `%APPDATA%\npm` (global npm shims), plus `%LOCALAPPDATA%\agy\bin` and + * `%LOCALAPPDATA%\cursor-agent`, cover every engine we ship an install + * command for. */ function windowsKnownDirs(): string[] { const home = homedir(); const appData = process.env.APPDATA ?? join(home, "AppData", "Roaming"); @@ -77,6 +78,8 @@ function windowsKnownDirs(): string[] { join(appData, "npm"), // npm -g shims: claude, codex join(home, ".grok", "bin"), // x.ai installer join(localAppData, "agy", "bin"), // Antigravity installer + // Cursor installer: cursor-agent.* and its `agent` copies (MOCA-272) + join(localAppData, "cursor-agent"), join(home, ".local", "bin"), // claude native installer join(home, ".claude", "local"), join(home, "bin"), // Factory droid installer (%USERPROFILE%\bin) From 970fae42fd9c037e22dddde82c3bb39c137d21a4 Mon Sep 17 00:00:00 2001 From: Aditya Umale Date: Thu, 1 Oct 2026 21:48:50 +0530 Subject: [PATCH 029/211] fix(rooms): take a deleted bot out of the rooms it was in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deleting a bot removed its record, transcripts and workspace but left its id in every room's memberIds. The room then counted it on Manage members' "Save · N bots" (5 for 4 visible bots), the roster check refused every save from that panel ("unknown room member"), and a room the bot led kept pointing its default responder at a bot that no longer existed. deleteBot now removes the bot from each room it was in, lets the room's lead fall back to the next member, and saves and announces the rooms. Startup repairs rooms that already list a deleted bot, but only when the bot list was actually read, so an unreadable bots.json can never empty rooms. Bot-to-bot channels keep their pair. Fixes MOCA-264 Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 6e88f73674606f0f61e7e7ac4292d322669b8207) --- server/store.test.ts | 51 ++++++++++++++++++++++++++++++++++++++++++++ server/store.ts | 23 ++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/server/store.test.ts b/server/store.test.ts index 9cc153d8bf..f45977c0c2 100644 --- a/server/store.test.ts +++ b/server/store.test.ts @@ -27,6 +27,57 @@ describe("Store", () => { rmSync(DATA_DIR, { recursive: true, force: true }); }); + // MOCA-264: a deleted bot stayed in its rooms for good — counted on the + // Save button, refused by the roster check on every save, and still the + // room's lead. + it("takes a deleted bot out of every room it was in and passes its lead role on", () => { + const store = new Store(selection); + const [ada, ben, cleo] = [store.createBot({}), store.createBot({}), store.createBot({})]; + const room = store.createGroup("Launch team", [ada.id, ben.id, cleo.id], false); + store.patchGroup(room.id, { defaultResponder: { kind: "member", botId: ben.id } }); + const pair = store.createGroup("Ada & Ben", [ada.id, ben.id], true); + const changes: unknown[] = []; + store.onChange((change) => changes.push(change)); + + expect(store.deleteBot(ben.id)).toBe(true); + expect(store.group(room.id)).toMatchObject({ memberIds: [ada.id, cleo.id], defaultResponder: { kind: "member", botId: ada.id } }); + expect(store.group(pair.id)?.memberIds).toEqual([ada.id, ben.id]); + expect(changes).toContainEqual({ type: "group", groupId: room.id }); + expect(new Store(selection).group(room.id)?.memberIds).toEqual([ada.id, cleo.id]); + }); + + it("repairs rooms that still list a deleted bot when it starts", () => { + const store = new Store(selection); + const [ada, cleo] = [store.createBot({}), store.createBot({})]; + const room = store.createGroup("Launch team", [ada.id, cleo.id], false); + const groupsFile = join(DATA_DIR, "groups.json"); + const saved = JSON.parse(readFileSync(groupsFile, "utf8")); + const ghost = "8a2acb50-6276-4ce2-926a-9e112b848acc"; + Object.assign(saved.find((g: { id: string }) => g.id === room.id), { + memberIds: [ghost, ada.id, cleo.id], + defaultResponder: { kind: "member", botId: ghost }, + }); + writeFileSync(groupsFile, JSON.stringify(saved)); + + const restarted = new Store(selection); + expect(restarted.group(room.id)).toMatchObject({ memberIds: [ada.id, cleo.id], defaultResponder: { kind: "member", botId: ada.id } }); + const persisted = JSON.parse(readFileSync(groupsFile, "utf8")).find((g: { id: string }) => g.id === room.id); + expect(persisted.memberIds).toEqual([ada.id, cleo.id]); + }); + + it("never empties rooms when the bot list cannot be read", () => { + const store = new Store(selection); + const [ada, cleo] = [store.createBot({}), store.createBot({})]; + const room = store.createGroup("Launch team", [ada.id, cleo.id], false); + const groupsFile = join(DATA_DIR, "groups.json"); + const before = readFileSync(groupsFile, "utf8"); + writeFileSync(join(DATA_DIR, "bots.json"), "{ not json"); + + expect(new Store(selection).group(room.id)?.memberIds).toEqual([ada.id, cleo.id]); + expect(JSON.parse(readFileSync(groupsFile, "utf8")).find((g: { id: string }) => g.id === room.id).memberIds) + .toEqual(JSON.parse(before).find((g: { id: string }) => g.id === room.id).memberIds); + }); + it("renames populated teams without changing members, conversations, grants or computer identity", () => { const store = new Store(selection); const chief = store.createBot({ section: "Delivery" }); diff --git a/server/store.ts b/server/store.ts index 3c922f23c9..1d863bc20c 100644 --- a/server/store.ts +++ b/server/store.ts @@ -622,8 +622,13 @@ export class Store { this.completeNewBotSelection = completeNewBotSelection; mkdirSync(DATA_DIR, { recursive: true }); for (const file of [BOTS_FILE, GROUPS_FILE]) tightenRegistryFile(file); + // Whether the bot list is the real one. Room repair below trusts it to + // say which members no longer exist; an unreadable file must never read + // as "every member was deleted". + let botsLoaded = false; try { this.bots = JSON.parse(readFileSync(BOTS_FILE, "utf8")); + botsLoaded = Array.isArray(this.bots); } catch { this.bots = []; } @@ -776,9 +781,17 @@ export class Store { botsMigrated = true; } } + const botIds = new Set(this.bots.map((b) => b.id)); for (const g of this.groups) { g.busyBotId = null; delete g.turnStartedAt; + // A deleted bot used to stay a member for good: counted on the room's + // Save button and refused by the roster check on every save (MOCA-264). + // Bot-to-bot channels keep their pair; they are not edited as rooms. + if (botsLoaded && !g.dm && g.memberIds.some((id) => !botIds.has(id))) { + g.memberIds = g.memberIds.filter((id) => botIds.has(id)); + groupsMigrated = true; + } const normalized = normalizeGroupDefaultResponder(g.defaultResponder, g.memberIds, Boolean(g.dm)); if (JSON.stringify(normalized) !== JSON.stringify(g.defaultResponder)) groupsMigrated = true; g.defaultResponder = normalized; @@ -1892,6 +1905,15 @@ export class Store { this.saveBots(nextBots); this.bots = nextBots; this.legacyActivities.delete(id); + // A deleted bot leaves every room it was in, and a room it led falls back + // to its next member. Bot-to-bot channels keep their pair. + const rooms = this.groups.filter((g) => !g.dm && g.memberIds.includes(id)); + for (const g of rooms) { + g.memberIds = g.memberIds.filter((member) => member !== id); + if (g.busyBotId === id) g.busyBotId = null; + g.defaultResponder = normalizeGroupDefaultResponder(g.defaultResponder, g.memberIds, false); + } + if (rooms.length) this.saveGroups(); // every task's transcript goes with the bot, not just the open one for (const threadId of new Set([bot.threadId, ...(bot.tasks ?? []).map((t) => t.threadId)])) { this.deleteThreadRecord(threadId); @@ -1912,6 +1934,7 @@ export class Store { // The bot folder (SOUL.md mirror) is the bot's too. removeBotFolder(id); this.emit({ type: "bot.deleted", botId: id }); + for (const g of rooms) this.emit({ type: "group", groupId: g.id }); return true; } From a207c3461d0ad932985d1855bc4ab29704886598 Mon Sep 17 00:00:00 2001 From: asemabdallah Date: Thu, 1 Oct 2026 15:28:48 +0300 Subject: [PATCH 030/211] fix(claude): detect --autocompact from claude --help instead of trusting the version floor CLI 2.1.129 is above the 2.1.122 floor but rejects --autocompact as an unknown option, which fails every Claude turn. snapshot() now reads `claude --help` once per CLI version and records whether the flag is listed; turns use that result and fall back to the version floor only when the CLI has not been probed or the probe failed. The earlier detection from #1213 was lost in a later merge, so this adds it back in a smaller form along with a regression test. The fake CLI now answers --help. Fixes #1187 (cherry picked from commit 47be3c3ee3da18e5a400d7f9ffbecb33cefb6997) --- server/drivers/claude.test.ts | 13 +++++++++++++ server/drivers/claude.ts | 15 ++++++++++++++- server/testing/fake-claude-cli.ts | 12 ++++++++++++ 3 files changed, 39 insertions(+), 1 deletion(-) diff --git a/server/drivers/claude.test.ts b/server/drivers/claude.test.ts index 6b4a6f7bc2..ba9b10dc9e 100644 --- a/server/drivers/claude.test.ts +++ b/server/drivers/claude.test.ts @@ -1434,6 +1434,19 @@ describe("ClaudeDriver turns (fake CLI)", () => { }); }); + it("withholds --autocompact from a CLI above the floor whose --help does not list it", async () => { + // 2.1.129 clears the 2.1.122 floor yet rejects the flag ("unknown option") + const dump = join(scratch, "no-autocompact-cli.json"); + await create(undefined, { FAKE_CLAUDE_DUMP: dump, FAKE_CLAUDE_VERSION: "2.1.129", FAKE_CLAUDE_AUTOCOMPACT: "0" }); + await instance.snapshot(); + await instance.adapter.sendTurn({ threadId: "t-no-autocompact", text: "hi" }); + await recorder.until((e) => e.type === "turn.completed"); + + const seen = JSON.parse(readFileSync(dump, "utf8")); + expect(seen.argv).not.toContain("--autocompact"); + expect(seen.argv).toContain("--strict-mcp-config"); + }); + it("keeps only the isolation flag a very old CLI accepts", async () => { // 1.0.100: --strict-mcp-config exists (1.0.60), --setting-sources does // not yet (1.0.122) diff --git a/server/drivers/claude.ts b/server/drivers/claude.ts index ec9feeceb8..b2cb3311ef 100644 --- a/server/drivers/claude.ts +++ b/server/drivers/claude.ts @@ -1182,6 +1182,12 @@ export const ClaudeDriver: ProviderDriver = { // other context controls and an unknown flag would reject that request. let cliVersion: ClaudeCliVersion | null = null; let cliVersionChecked = false; + // Whether `claude --help` lists --autocompact, read once per CLI version + // by snapshot(). The flag is not in every build above its version floor + // (2.1.129 rejects it), so the listing wins over the floor; null until + // probed, or when the probe fails. + let cliHasAutocompact: boolean | null = null; + let cliHelpVersion: string | null = null; const readCliVersion = (env: NodeJS.ProcessEnv): Promise => new Promise((resolve) => { execCli(config.cli, ["--version"], { timeout: 8000, env }, (err, stdout) => @@ -1439,7 +1445,7 @@ export const ClaudeDriver: ProviderDriver = { if (claudeCliSupports(cliVersion, "--setting-sources")) args.push("--setting-sources", "project"); } const compactWindow = autoCompactWindow(turnEnvironment); - if (compactWindow && claudeCliSupports(cliVersion, "--autocompact")) { + if (compactWindow && (cliHasAutocompact ?? claudeCliSupports(cliVersion, "--autocompact"))) { args.push("--autocompact", compactWindow); } // An old pair conversation can still carry its first assignment in @@ -2431,6 +2437,13 @@ export const ClaudeDriver: ProviderDriver = { if (!version) return { state: "unavailable", reason: `\`${config.cli}\` CLI not found` }; cliVersion = parseClaudeCliVersion(version); cliVersionChecked = true; + if (version !== cliHelpVersion) { + const help = await new Promise((resolve) => { + execCli(config.cli, ["--help"], { timeout: 8000, env }, (err, stdout) => resolve(err ? null : stdout)); + }); + cliHasAutocompact = help === null ? null : /^\s*--autocompact\b/m.test(help); + cliHelpVersion = version; + } const update = claudeCliUpdate(version, config.cli); const warning = claudeInheritWarning(env); if (config.requireApiKey) { diff --git a/server/testing/fake-claude-cli.ts b/server/testing/fake-claude-cli.ts index 0008bee95a..6864125bf3 100755 --- a/server/testing/fake-claude-cli.ts +++ b/server/testing/fake-claude-cli.ts @@ -237,6 +237,18 @@ if (argv[0] === "--version") { process.exit(0); } +if (argv[0] === "--help") { + // Lists --autocompact in the option column like the real CLI, unless the + // fake stands in for a build without it: FAKE_CLAUDE_AUTOCOMPACT=0, or a + // version below the 2.1.122 floor. + const [maj = 0, min = 0, pat = 0] = (process.env.FAKE_CLAUDE_VERSION ?? "2.1.232").split(".").map(Number); + const has = process.env.FAKE_CLAUDE_AUTOCOMPACT !== "0" && (maj > 2 || (maj === 2 && (min > 1 || (min === 1 && pat >= 122)))); + process.stdout.write( + `Usage: claude [options]\n\nOptions:\n --model Model\n${has ? " --autocompact Compaction window\n" : ""} -h, --help Display help\n`, + ); + process.exit(0); +} + if (argv[0] === "update") { if (process.env.FAKE_CLAUDE_UPDATE === "fail") { process.stderr.write("fake-claude: simulated update failure\n"); From 38ae67e952100560aa4c7040fb10215b397f0f16 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 12:47:24 +0530 Subject: [PATCH 031/211] fix(runtime): preserve room erasure and align provider fixtures (cherry picked from commit e181dd21d9cd07942b7fb23e4789e47193067424) --- server/drivers/claude-accounts.test.ts | 4 +++- server/env-path.test.ts | 3 ++- server/store.test.ts | 18 +++++++++++++++++- server/store.ts | 8 ++++++-- server/team-backup.test.ts | 2 ++ 5 files changed, 30 insertions(+), 5 deletions(-) diff --git a/server/drivers/claude-accounts.test.ts b/server/drivers/claude-accounts.test.ts index 405164c92c..536e56f53b 100644 --- a/server/drivers/claude-accounts.test.ts +++ b/server/drivers/claude-accounts.test.ts @@ -24,6 +24,7 @@ appendFileSync(join(configDir, "calls.ndjson"), JSON.stringify({ }) + "\\n"); const out = value => process.stdout.write(JSON.stringify(value) + "\\n"); if (args[0] === "--version") { console.log("fixture-claude"); process.exit(0); } +if (args[0] === "--help") { console.log("Usage: claude [options]\\n --model Model"); process.exit(0); } if (args[0] === "auth") { out(auth); process.exit(auth.loggedIn ? 0 : 1); } if (args.includes("text")) { process.stdin.resume(); @@ -113,7 +114,8 @@ describe("Claude account configuration", () => { recorder.stop(); expect(await provider.reviewPermission?.("review fixture only")).toBe(`${name}@example.test`); const recorded = calls(dirs[index]!); - expect(recorded).toHaveLength(4); + expect(recorded).toHaveLength(5); + expect(recorded.filter(call => call.args[0] === "--help")).toHaveLength(1); for (const call of recorded) { expect(call.configDir).toBe(dirs[index]); expect(call.home).toBe(process.env.HOME); diff --git a/server/env-path.test.ts b/server/env-path.test.ts index f5b1359f20..04187192ec 100644 --- a/server/env-path.test.ts +++ b/server/env-path.test.ts @@ -214,7 +214,8 @@ describe("augmentedPath", () => { try { Object.defineProperty(process, "platform", { value: "win32" }); process.env.LOCALAPPDATA = localAppData; - process.env.PATHEXT = ".COM;.EXE;.BAT;.CMD"; + // The simulated Windows platform still has the host's case-sensitive filesystem. + process.env.PATHEXT = ".com;.exe;.bat;.cmd"; const cursorDir = join(localAppData, "cursor-agent"); mkdirSync(join(cursorDir, "versions", "2026.09.28-64d2043"), { recursive: true }); for (const name of ["cursor-agent.cmd", "cursor-agent.ps1", "agent.cmd", "agent.ps1"]) writeFileSync(join(cursorDir, name), "@echo off\n"); diff --git a/server/store.test.ts b/server/store.test.ts index f45977c0c2..378696b992 100644 --- a/server/store.test.ts +++ b/server/store.test.ts @@ -34,18 +34,34 @@ describe("Store", () => { const store = new Store(selection); const [ada, ben, cleo] = [store.createBot({}), store.createBot({}), store.createBot({})]; const room = store.createGroup("Launch team", [ada.id, ben.id, cleo.id], false); - store.patchGroup(room.id, { defaultResponder: { kind: "member", botId: ben.id } }); + store.patchGroup(room.id, { defaultResponder: { kind: "member", botId: ben.id }, busyBotId: ben.id }); const pair = store.createGroup("Ada & Ben", [ada.id, ben.id], true); const changes: unknown[] = []; store.onChange((change) => changes.push(change)); expect(store.deleteBot(ben.id)).toBe(true); expect(store.group(room.id)).toMatchObject({ memberIds: [ada.id, cleo.id], defaultResponder: { kind: "member", botId: ada.id } }); + expect(store.group(room.id)?.turnStartedAt).toBeUndefined(); expect(store.group(pair.id)?.memberIds).toEqual([ada.id, ben.id]); expect(changes).toContainEqual({ type: "group", groupId: room.id }); expect(new Store(selection).group(room.id)?.memberIds).toEqual([ada.id, cleo.id]); }); + it("finishes bot erasure if the room registry cannot persist, then repairs it on restart", () => { + const store = new Store(selection); + const [ada, ben] = [store.createBot({}), store.createBot({})]; + const room = store.createGroup("Launch team", [ada.id, ben.id], false); + const internals = store as unknown as { saveGroups: () => void }; + vi.spyOn(internals, "saveGroups").mockImplementationOnce(() => { throw new Error("fixture write failure"); }); + + expect(store.deleteBot(ben.id)).toBe(true); + expect(store.bot(ben.id)).toBeNull(); + expect(store.messagesFor(ben.threadId)).toEqual([]); + expect(existsSync(soulFile(ben.id))).toBe(false); + expect(JSON.parse(readFileSync(join(DATA_DIR, "groups.json"), "utf8"))[0].memberIds).toContain(ben.id); + expect(new Store(selection).group(room.id)?.memberIds).toEqual([ada.id]); + }); + it("repairs rooms that still list a deleted bot when it starts", () => { const store = new Store(selection); const [ada, cleo] = [store.createBot({}), store.createBot({})]; diff --git a/server/store.ts b/server/store.ts index 1d863bc20c..8417297c5c 100644 --- a/server/store.ts +++ b/server/store.ts @@ -1910,10 +1910,14 @@ export class Store { const rooms = this.groups.filter((g) => !g.dm && g.memberIds.includes(id)); for (const g of rooms) { g.memberIds = g.memberIds.filter((member) => member !== id); - if (g.busyBotId === id) g.busyBotId = null; + if (g.busyBotId === id) { g.busyBotId = null; delete g.turnStartedAt; } g.defaultResponder = normalizeGroupDefaultResponder(g.defaultResponder, g.memberIds, false); } - if (rooms.length) this.saveGroups(); + if (rooms.length) { + // Bot removal is already durable. Finish erasing its data even if this + // write fails; startup repair removes these stale memberships later. + try { this.saveGroups(); } catch (error) { console.warn("store: room cleanup will retry on restart", error); } + } // every task's transcript goes with the bot, not just the open one for (const threadId of new Set([bot.threadId, ...(bot.tasks ?? []).map((t) => t.threadId)])) { this.deleteThreadRecord(threadId); diff --git a/server/team-backup.test.ts b/server/team-backup.test.ts index 4ba031673e..070785c263 100644 --- a/server/team-backup.test.ts +++ b/server/team-backup.test.ts @@ -309,6 +309,8 @@ describe("additive portable team backups", () => { const dm = store.createGroup("Old direct message", [chief.id, scout.id], true); store.appendMessage(dm.threadId, { role: "bot", kind: "text", text: "Keep this old reply", from: { botId: chief.id, name: chief.name, color: chief.color } }); store.deleteBot(chief.id); + // Recreate the pre-repair records this legacy-export regression covers. + Object.assign(group, { memberIds: [chief.id, scout.id], defaultResponder: { kind: "member", botId: chief.id } }); const backup = createTeamBackup(store, routines.listRoutines(), "My team"); expect(backup.warnings).toHaveLength(4); expect(backup.routines).toEqual([]); From 21ad4d9abfb2eb0b9b1adf3c1fa2a32e9197cdf0 Mon Sep 17 00:00:00 2001 From: Rui Gomes <5658301+ruigomeseu@users.noreply.github.com> Date: Thu, 1 Oct 2026 21:57:10 +0000 Subject: [PATCH 032/211] Preserve idle Local VMs and offer guarded resume with startup progress (cherry picked from commit 11e83b70318daf047211d60e459c50162a96bbec) --- docs/verification/README.md | 3 ++ docs/verification/local-vm-resume.md | 37 ++++++++++++++ scripts/testing/cloud-preview.tsx | 37 +++++++++++--- scripts/verify-local-vm-resume.ts | 63 ++++++++++++++++++++++++ server/container-computer.test.ts | 41 ++++++++++++---- server/container-computer.ts | 48 +++++++++---------- server/group-local-vm.e2e.test.ts | 47 +++++++++++++++++- server/index.ts | 64 +++++++++++-------------- server/local-vm-stop-reason.test.ts | 18 +++++++ server/local-vm-stop-reason.ts | 27 +++++++++++ server/testing/group-local-vm-hooks.mjs | 18 +++++-- server/vps-computer.ts | 2 +- shared/local-vm-lifecycle.ts | 15 ++++++ src/components/ComputerPanel.tsx | 59 ++++++++++++++++++----- src/components/LocalComputerSection.tsx | 45 ++++++++++++----- src/lib/local-vm-readiness.ts | 23 +++++++++ src/locales/en.json | 10 ++-- 17 files changed, 450 insertions(+), 107 deletions(-) create mode 100644 docs/verification/local-vm-resume.md create mode 100644 scripts/verify-local-vm-resume.ts create mode 100644 server/local-vm-stop-reason.test.ts create mode 100644 server/local-vm-stop-reason.ts create mode 100644 shared/local-vm-lifecycle.ts create mode 100644 src/lib/local-vm-readiness.ts diff --git a/docs/verification/README.md b/docs/verification/README.md index c0ec1a73ed..e11682e677 100644 --- a/docs/verification/README.md +++ b/docs/verification/README.md @@ -125,6 +125,9 @@ The [live browser fixture](browser-live.md) mounts the real Browser panel with an explicitly selected native engine and Chrome in a disposable home, covering watching, takeover, input, and profile switching. +The [Local VM resume fixture](local-vm-resume.md) checks idle stop, restart +recovery, guarded resume and the stopped-to-ready Computer panel flow. + The [local computer launch regression](local-computer-launch.md) starts the host CUA gate through real Electron in a disposable home, without opening the desktop app or controlling the user's computer. diff --git a/docs/verification/local-vm-resume.md b/docs/verification/local-vm-resume.md new file mode 100644 index 0000000000..ec04a70b8f --- /dev/null +++ b/docs/verification/local-vm-resume.md @@ -0,0 +1,37 @@ +# Local VM stop and resume + +Run the focused lifecycle checks in disposable homes: + +```sh +pnpm exec vitest run server/container-computer.test.ts server/local-vm-stop-reason.test.ts server/local-vm-idle.test.ts server/group-local-vm.e2e.test.ts server/routes/desktop-viewer.test.ts src/components/LocalComputerSection.test.ts src/components/ComputerPanel.test.ts src/components/ComputerPanel.i18n.test.ts +node --experimental-strip-types scripts/verify-local-vm-resume.ts +``` + +The browser recipe uses `launchVerificationServer`, the fake engine and a +private browser profile. It mounts the real ComputerPanel with synthetic +desktop transport; it never contacts the host Docker daemon. Set +`OMB_AGENT_BROWSER_PATH` and `AGENT_BROWSER_EXECUTABLE_PATH` to reuse installed +tools. Its receipt prints the isolated server URL and persistent log path; +stopped and starting screenshots remain beside that log. + +The browser check proves an existing shared VM shows “stopped” and an idle +explanation, Start issues exactly one request, the button stays disabled and +busy after that request returns while Cua is still warming up, and the ready +desktop replaces the empty state without a remove or recreate request. It also +checks that Settings offers Start for a stopped VM and waits for readiness. + +The server fixture runs the actual HTTP routes and idle timer, replacing only +the container boundary and shortening the idle window through a test-only +loader. It verifies idle shutdown stops without deleting, the stop reason +survives a server restart, shared and per-bot starts work, and an Auto turn +resumes an existing desktop even when the per-bot capacity limit is reached. +The reason record matches the runtime's finish timestamp; a later external +stop or an unknown timestamp gets the neutral stopped explanation. + +2026-10-01, Docker on Linux: a separately created, disposable container using +the pinned driver-0.20.0-v5 image passed two stop/start cycles. Each cycle +passed the production readiness probe (driver version, health report, and +complete screenshot) and preserved marker files under `/home/cua` and `/opt`. +The fixture used an empty temporary workspace and was removed afterward. +No image rebuild was required. That live acceptance does not cover Podman, +Apple container, native iOS, or persistence across image replacement. diff --git a/scripts/testing/cloud-preview.tsx b/scripts/testing/cloud-preview.tsx index 0d5a15e219..8d25d19e3f 100644 --- a/scripts/testing/cloud-preview.tsx +++ b/scripts/testing/cloud-preview.tsx @@ -1,5 +1,6 @@ import { useEffect, useState } from "react"; import { createRoot } from "react-dom/client"; +import { LocalComputerSection } from "../../src/components/LocalComputerSection"; import { ComputerPanel } from "../../src/components/ComputerPanel"; import { BotSettingsDialog } from "../../src/components/BotSettingsDialog"; import { RemoteDesktopPanel } from "../../src/components/remote-desktop-panel"; @@ -27,10 +28,21 @@ const screenshot = frame("Cloud screen connected", "#134e4a"); const vmScreenshot = `data:image/png;base64,${frame("Local VM connected", "#1e3a8a")}`; let mode = "connected"; let surfaceScenario = "default"; +let vmRunning = false; +let vmDesktopReady = false; +const vmResumeStatus = () => ({ + platform: "linux", runtime: "docker", available: ["docker"], mode: "shared", daemonUp: true, + image: true, create_supported: true, container: vmRunning ? "running" : "stopped", + imageMatches: true, managed: true, network: "loopback", security: "hardened", persistence: "durable", + desktopReady: vmDesktopReady, ready: vmDesktopReady, stop_reason: vmRunning ? null : "idle", + problem: vmDesktopReady ? null : vmRunning ? "Desktop is starting" : "The Local VM is stopped", + viewer_url: "http://127.0.0.1/fixture-viewer", idle_timeout_ms: 480 * 60_000, max_instances: 2, + commands: {}, workspace_path: "/fixture/workspace", workspace_guest_path: "/home/cua/workspace", +}); // A host capture outlives an aborted renderer fetch. Keep this work pending // until explicitly released, so reconnects exercise real lifecycle contention. const transport = { - requests: 0, aborted: 0, conflicts: 0, capturing: false, + resetVm: () => { vmRunning = false; vmDesktopReady = false; }, vmStarts: 0, requests: 0, aborted: 0, conflicts: 0, capturing: false, joining: false, duringJoin: 0, controlCalls: 0, opened: 0, abortedJoins: 0, releaseCapture: () => {}, releaseJoin: () => {}, screenshot: `data:image/png;base64,${screenshot}`, @@ -60,7 +72,13 @@ window.fetch = async (input, init) => { status, headers: { "content-type": "application/json" }, }); if (/^\/api\/bots\/[\w-]+\/computer$/.test(path)) return json({ surface: surfaceScenario === "auto-vm" ? "vm" : "cloud", configured: true, box: { state: "idle" } }); - if (path.endsWith("/local-computer")) return json({ mode: "per-bot", max_instances: 2, image: true, create_supported: true, + if (path.endsWith("/local-computer/start")) { + transport.vmStarts++; + vmRunning = true; + return json(vmResumeStatus()); + } + if (path.endsWith("/local-computer") && surfaceScenario === "vm-stopped") return json(vmResumeStatus()); + if (path.endsWith("/local-computer")) return json({ daemonUp: true, mode: "per-bot", max_instances: 2, image: true, create_supported: true, container: "running", imageMatches: true, managed: true, network: "loopback", security: "hardened", persistence: "durable", desktopReady: true, ready: true, problem: null, viewer_url: "http://127.0.0.1/fixture-viewer" }); if (path.endsWith("/local-computer/screenshot")) { @@ -168,7 +186,7 @@ function Fixture() { const fixtureBot: Bot | undefined = bot && (scenario === "default" ? { ...bot, busy, tasks: bot.tasks?.map((task) => ({ ...task, busy })) } : { ...bot, busy: false, browser: true, - computer: scenario === "auto-vm" ? undefined : scenario === "cloud-pin" ? "local" : scenario === "off" ? "off" : "cloud", + computer: scenario === "vm-stopped" ? "vm" : scenario === "auto-vm" ? undefined : scenario === "cloud-pin" ? "local" : scenario === "off" ? "off" : "cloud", modelSelection: scenario === "vm-pin" ? { ...bot.modelSelection, instanceId: "unavailable-profile-engine" } : bot.modelSelection, threadId: `fixture-${scenario}`, tasks: [{ threadId: `fixture-${scenario}`, title: scenario, createdAt: 1, busy: false, modelSelection: bot.modelSelection, @@ -180,23 +198,30 @@ function Fixture() { {["connected", "slow", "held", "contended", "failed", "unconfigured", "corrupt", "timeout"].map((value) => )} +
{state.settingsOpen && bot && } {state.computerOpen && fixtureBot ? panel === "computer" ? + : panel === "settings" ?
: : !state.settingsOpen && }
; diff --git a/scripts/verify-local-vm-resume.ts b/scripts/verify-local-vm-resume.ts new file mode 100644 index 0000000000..cd62f5a534 --- /dev/null +++ b/scripts/verify-local-vm-resume.ts @@ -0,0 +1,63 @@ +// Real ComputerPanel in an isolated fake-engine workspace; only the desktop +// transport is simulated. No host Docker or user profile is accessed. +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, dirname } from "node:path"; +import { launchVerificationServer, runControlOmb } from "./control-omb.ts"; +import { mountPreview } from "./testing/preview-fixture.ts"; +import { agentBrowser, ensureUiBrowser, sessionEnv } from "./testing/control-omb-ui.ts"; + +const fixture = await launchVerificationServer(); +const home = mkdtempSync(join(tmpdir(), "omb-resume-browser-")); +let preview: Awaited> | undefined; +let browser: { binary: string; env: NodeJS.ProcessEnv } | undefined; +try { + console.log(JSON.stringify(fixture.info)); + await runControlOmb(["new-bot", "--name", "Resume test", "--url", fixture.info.url]); + preview = await mountPreview(fixture, { + entry: "/scripts/testing/cloud-preview.tsx", route: "/__vm-resume.html", title: "Local VM resume verification", logLevel: "silent", + }); + const { binary, chrome } = await ensureUiBrowser(process.env); + const env = sessionEnv({ home, session: `vm-resume-${process.pid}`, chrome }); + browser = { binary, env }; + const command = (...args: string[]) => agentBrowser(binary, env, args); + const evaluate = async (js: string) => (await command("eval", js)).result as T; + const wait = (condition: string) => { console.log("Waiting:", condition); return command("wait", "--fn", condition); }; + await command("open", preview.previewUrl); + await wait(`document.querySelector('select[aria-label="Conversation surface"]') !== null`); + await wait(`Array.from(document.querySelectorAll('img')).some(img => img.src === window.cloudPreviewFixture.screenshot && img.complete)`); + await evaluate(`(() => { const s = document.querySelector('select[aria-label="Conversation surface"]'); s.value='vm-stopped'; s.dispatchEvent(new Event('change', {bubbles:true})); })()`); + await wait(`document.body.textContent.includes('The Local VM is stopped')`); + assert.equal(await evaluate(`document.body.textContent.includes('Stopped after inactivity')`), true); + assert.equal(await evaluate(`document.body.textContent.includes("isn't available for this bot")`), false); + await command("screenshot", join(dirname(fixture.info.logPath), `vm-stopped-${process.pid}.png`)); + await command("find", "role", "button", "click", "--name", "Start Local VM", "--exact"); + await wait(`window.cloudPreviewFixture.vmStarts === 1`); + await wait(`Array.from(document.querySelectorAll('button')).some(b => b.textContent.includes('Starting Local VM') && b.disabled && b.getAttribute('aria-busy') === 'true')`); + // The POST has returned but Cua is still warming up: the button must keep + // spinning rather than become clickable or claim the desktop is ready. + await evaluate(`new Promise(resolve => setTimeout(resolve, 1500))`); + assert.equal(await evaluate(`Array.from(document.querySelectorAll('button')).some(b => b.textContent.includes('Starting Local VM') && b.disabled)`), true); + await command("screenshot", join(dirname(fixture.info.logPath), `vm-starting-${process.pid}.png`)); + await command("find", "role", "button", "click", "--name", "Finish VM startup", "--exact"); + await wait(`Array.from(document.querySelectorAll('img')).some(img => img.src === window.cloudPreviewFixture.vmScreenshot && img.complete && img.naturalWidth > 1)`); + assert.equal(await evaluate(`window.cloudPreviewFixture.vmStarts`), 1); + assert.equal(await evaluate(`window.cloudPreviewFixture.paths.some(p => p.endsWith('/local-computer/remove') || p.endsWith('/local-computer/run'))`), false); + await evaluate(`window.cloudPreviewFixture.resetVm(); const s = document.querySelector('select[aria-label="Panel"]'); s.value='settings'; s.dispatchEvent(new Event('change', {bubbles:true}));`); + await wait(`document.body.textContent.includes('Stopped after inactivity')`); + assert.equal(await evaluate(`document.body.textContent.includes('Delete and recreate')`), false); + await command("find", "role", "button", "click", "--name", "Start Local VM", "--exact"); + await wait(`document.body.textContent.includes('Waiting for the desktop')`); + await command("find", "role", "button", "click", "--name", "Finish VM startup", "--exact"); + await wait(`Array.from(document.querySelectorAll('[aria-live]')).some(el => el.textContent.trim() === 'Ready')`); + console.log("PASS: stopped explanation → one start → pending through warmup → live preview; Settings also resumes without replacement"); +} catch (error) { + if (browser) console.error(JSON.stringify(await agentBrowser(browser.binary, browser.env, ["snapshot"]).catch(() => ({})))); + throw error; +} finally { + if (browser) await agentBrowser(browser.binary, browser.env, ["close"]).catch(() => {}); + await preview?.close(); + await fixture.close(); + rmSync(home, { recursive: true, force: true }); +} diff --git a/server/container-computer.test.ts b/server/container-computer.test.ts index b15f61d172..0bd3ffc316 100644 --- a/server/container-computer.test.ts +++ b/server/container-computer.test.ts @@ -29,6 +29,7 @@ import { containerRunArgs, dockerSecurityIsHardened, localVmRecreatableOnDemand, + localVmResumable, localVmWorkspaceExists, managedImageDockerfile, perBotLocalVmTarget, @@ -829,17 +830,22 @@ describe("containerComputerAction", () => { expect(fake.calls.some((call) => call.startsWith("docker run "))).toBe(false); }); - it("never starts a stopped desktop because its stale X lock makes resume unsafe", async () => { + it("resumes a compatible stopped desktop without removing it", async () => { const fake = runner({ "/usr/bin/which docker": "docker\n", "/usr/bin/which podman": new Error("missing"), "docker info --format {{.ServerVersion}}": "29\n", [`docker image inspect ${IMAGE}`]: preparedImageInspect(), - [`docker inspect ${CONTAINER}`]: readyInspect({ State: { Running: false } }), + [`docker inspect ${CONTAINER}`]: readyInspect({ State: { Running: false, FinishedAt: "2026-10-01T12:00:00Z" } }), + [`docker start ${CONTAINER}`]: CONTAINER, }); - await expect(containerComputerAction("start", fake.run, "linux")).rejects.toThrow("cannot safely resume"); - expect(fake.calls).not.toContain(`docker start ${CONTAINER}`); + const stopped = await containerComputerStatus(fake.run, "linux"); + expect(localVmResumable(stopped)).toBe(true); + expect(stopped.stopped_at).toBe("2026-10-01T12:00:00Z"); + await containerComputerAction("start", fake.run, "linux"); + expect(fake.calls).toContain(`docker start ${CONTAINER}`); + expect(fake.calls.some(call => call.includes(" rm ") || call.includes(" run "))).toBe(false); }); }); @@ -888,8 +894,8 @@ describe("setupCommands", () => { expect(command).toContain("VNC_PW=CHANGE_ME"); }); - it("does not suggest docker start for an image that must be recreated", () => { - expect(setupCommands("docker", "linux").start).toBeNull(); + it("offers a start command for a stopped compatible desktop", () => { + expect(setupCommands("docker", "linux").start).toBe(`docker start ${CONTAINER}`); }); it("limits resources and retains only the sandbox supervisor's identity-switch caps", () => { @@ -979,7 +985,7 @@ describe("localVmRecreatableOnDemand", () => { expect(localVmRecreatableOnDemand(status)).toBe(true); }); - it("leaves a stopped container alone, because it is asked to be recreated not started", async () => { + it("does not recreate an existing stopped container", async () => { const target = SHARED_LOCAL_VM_TARGET; const detail = JSON.parse(readyInspect())[0]; detail.State = { Running: false, Status: "exited" }; @@ -1027,7 +1033,7 @@ describe("Auto's Local VM eligibility", () => { it("attaches a ready desktop or one whose prepared image can be recreated, and nothing else", () => { expect(autoLocalVmAttachable({ ...base, ready: true, container: "running" })).toBe(true); expect(autoLocalVmAttachable(base)).toBe(true); - // never a first-time setup, a stopped image that cannot resume, or a dead daemon + // Never a first-time setup, an unverified stopped image, or a dead daemon expect(autoLocalVmAttachable({ ...base, image: false })).toBe(false); expect(autoLocalVmAttachable({ ...base, daemonUp: false })).toBe(false); expect(autoLocalVmAttachable({ ...base, container: "stopped" })).toBe(false); @@ -1035,3 +1041,22 @@ describe("Auto's Local VM eligibility", () => { expect(autoLocalVmAttachable({ ...base, create_supported: false })).toBe(false); }); }); + + +describe("Local VM resume safety", () => { + it.each([ + { Config: { Image: "foreign" } }, + { HostConfig: { Privileged: true } }, + { Mounts: [] }, + { State: { Running: true } }, + ])("refuses an incompatible, unsafe, or running desktop: %j", async patch => { + const fake = runner({ + "/usr/bin/which docker": "docker\n", + "docker info --format {{.ServerVersion}}": "29\n", + [`docker image inspect ${IMAGE}`]: preparedImageInspect(), + [`docker inspect ${CONTAINER}`]: readyInspect({ State: { Running: false }, ...patch }), + }); + await expect(containerComputerAction("start", fake.run, "linux")).rejects.toThrow(); + expect(fake.calls).not.toContain(`docker start ${CONTAINER}`); + }); +}); diff --git a/server/container-computer.ts b/server/container-computer.ts index 38a61c7b3a..07a90c712e 100644 --- a/server/container-computer.ts +++ b/server/container-computer.ts @@ -15,6 +15,7 @@ import { promisify } from "node:util"; import { augmentedPath, resolveCliSpawn } from "./env-path.ts"; import { DATA_DIR } from "./config.ts"; +import { canResumeLocalVm } from "../shared/local-vm-lifecycle.ts"; import { SPAWNED_PROXIES } from "./proxy-paths.ts"; const run = promisify(execFile); @@ -110,7 +111,7 @@ export function poolLocalVmTarget(seat: number): LocalVmTarget { }; } -/** Only provisioning creates this durable directory; idle removal keeps it. */ +/** Only provisioning creates this durable directory; idle shutdown keeps it. */ export function localVmWorkspaceExists(target: LocalVmTarget): boolean { try { return lstatSync(target.workspaceDir, { throwIfNoEntry: false })?.isDirectory() === true; @@ -317,6 +318,8 @@ export interface ContainerComputerStatus { persistence: "durable" | "unsafe" | "unknown"; desktopReady: boolean; desktop_error: string | null; + /** Runtime timestamp used to match an idle-stop record, never an inferred cause. */ + stopped_at?: string | null; create_supported: boolean; ready: boolean; problem: string | null; @@ -363,21 +366,8 @@ function emptyStatus(platform: NodeJS.Platform, target: LocalVmTarget): Containe }; } -/** Whether a turn may recreate this Local VM itself instead of failing. - * - * True for exactly one state: the container is gone, and a plain `run` is all - * that is needed to bring it back. That is what `LocalVmIdleTimer` leaves - * behind — it removes an unused Local VM rather than pausing it — so a turn - * arriving after an idle period should not have to send the person to App - * Settings for a container the app itself deleted. - * - * Every other problem in `statusProblem` stays the person's call and returns - * false here: no runtime, daemon down, image never prepared, `create_supported` - * false, and any existing container — stale image, unmanaged, unsafe network, - * security or persistence. A stopped container is excluded deliberately, since - * `statusProblem` says this desktop image cannot safely resume and asks for a - * recreate rather than a start. - */ +/** Recreate a missing desktop when its image and runtime are already prepared. + * This also recovers desktops deleted by older versions' idle cleanup. */ export function localVmRecreatableOnDemand( status: ContainerComputerStatus, ): status is ContainerComputerStatus & { runtime: Runtime } { @@ -388,12 +378,19 @@ export function localVmRecreatableOnDemand( && status.create_supported; } +/** Start only an existing, compatible desktop with the managed safety boundary. */ +export function localVmResumable( + status: ContainerComputerStatus, +): status is ContainerComputerStatus & { runtime: Runtime } { + return Boolean(status.runtime) && canResumeLocalVm(status); +} + /** Whether Auto may attach this Local VM without a person choosing it: the * desktop is ready, or its image is prepared and the container can simply be * recreated after idling away. Anything else — no runtime, daemon down, image * never prepared, an unmanaged or unsafe container — stays the person's call. */ export function autoLocalVmAttachable(status: ContainerComputerStatus): boolean { - return status.ready === true || localVmRecreatableOnDemand(status); + return status.ready === true || localVmRecreatableOnDemand(status) || localVmResumable(status); } function statusProblem(status: ContainerComputerStatus): string | null { @@ -409,7 +406,7 @@ function statusProblem(status: ContainerComputerStatus): string | null { if (status.network === "unsafe") return "The existing Local VM exposes its viewer publicly; recreate it"; if (status.security === "unsafe") return "The existing Local VM is missing safety limits; recreate it"; if (status.persistence === "unsafe") return "The existing Local VM is missing its durable folder; recreate it"; - if (status.container === "stopped") return "This desktop image cannot safely resume; recreate the Local VM"; + if (status.container === "stopped") return "The Local VM is stopped; start it to continue"; if (status.desktop_error) return `The Local VM desktop failed to start: ${status.desktop_error}`; if (!status.desktopReady) return "The Local VM started, but Cua Driver is not ready yet"; return null; @@ -586,11 +583,12 @@ export async function containerComputerStatus( }>; EffectiveCaps?: string[]; BoundingCaps?: string[]; - State?: { Running?: boolean }; + State?: { Running?: boolean; FinishedAt?: string }; Image?: string; }>; const detail = inspected[0]; status.container = detail?.State?.Running ? "running" : "stopped"; + status.stopped_at = status.container === "stopped" ? detail?.State?.FinishedAt ?? null : null; status.network = dockerPortsAreLocal(detail?.HostConfig?.PortBindings) ? "loopback" : "unsafe"; status.viewer_port = dockerViewerPort(detail?.NetworkSettings?.Ports, target.viewerPort); status.imageMatches = @@ -817,8 +815,8 @@ export interface DockerHardeningConfig { * runtime-specific capability exception is Podman's Firefox sandbox chroot. * Callers also differ on restart policy — the VPS * container must survive a reboot nobody is watching ("unless-stopped"), - * while the Local VM must NOT auto-resume: its desktop leaves a stale X lock - * on stop, so a restarted container is a broken one. */ + * while Local VM starts remain controlled by OMB's idle policy and turn + * lifecycle rather than a daemon restart policy. */ export function dockerSecurityIsHardened( config: DockerHardeningConfig | undefined, options: { restartPolicy?: "no" | "unless-stopped"; podmanBrowserSandbox?: boolean } = {}, @@ -1017,10 +1015,8 @@ export async function containerComputerAction( if (action === "run" && !before.create_supported) { throw Object.assign(new Error(before.problem ?? "This runtime cannot create a per-bot Local VM"), { status: 409 }); } - if (action === "start") { - throw Object.assign(new Error("This desktop image cannot safely resume; remove and recreate the Local VM"), { - status: 409, - }); + if (action === "start" && !localVmResumable(before)) { + throw Object.assign(new Error(before.problem ?? "The Local VM is not stopped"), { status: 409 }); } if (action === "stop" && before.container !== "running") { throw Object.assign(new Error("The Local VM is not running"), { status: 409 }); @@ -1306,7 +1302,7 @@ export function setupCommands( runtime === "container" && target.key !== SHARED_LOCAL_VM_TARGET.key ? null : command(containerRunArgs(runtime, "CHANGE_ME", target)), - start: null, + start: command(["start", target.containerName]), stop: command(["stop", target.containerName]), remove: command(["rm", runtime === "container" ? "--force" : "-f", target.containerName]), view: target.viewerPort ? `http://127.0.0.1:${target.viewerPort}/vnc.html` : "", diff --git a/server/group-local-vm.e2e.test.ts b/server/group-local-vm.e2e.test.ts index b4488e8aba..4cd2c950a2 100644 --- a/server/group-local-vm.e2e.test.ts +++ b/server/group-local-vm.e2e.test.ts @@ -170,6 +170,51 @@ async function room() { const send = (id: string) => api("POST", `/api/groups/${id}/messages`, { text: "Reply once." }); const stop = (id: string) => api("POST", `/api/groups/${id}/interrupt`, {}); +describe("Local VM stop and resume", () => { + it("stops an idle shared VM without deleting it, remembers why across restart, and starts it from the bot panel", async () => { + vmState({ containers: ["shared"], idleMs: 300 }); + await api("PATCH", "/api/config", { localVm: { mode: "shared", idleTimeoutMinutes: 5 } }); + const { bot } = await api("POST", "/api/bots", { name: "Resume fixture", computer: "vm" }); + await until(() => api("GET", "/api/local-computer"), s => s.container === "stopped"); + expect((await api("GET", "/api/local-computer")).stop_reason).toBe("idle"); + const stopped = JSON.parse(readFileSync(stateFile, "utf8")); + expect(stopped.containers).toEqual(["shared"]); + expect(stopped.actions).toEqual([{ action: "stop", target: "shared" }]); + vmState({ ...stopped, idleMs: 60_000 }); + await waitForExit(child, { signal: "SIGTERM" }); + await startServer(); + expect((await api("GET", `/api/bots/${bot.id}/local-computer`)).stop_reason).toBe("idle"); + const started = await api("POST", `/api/bots/${bot.id}/local-computer/start`, {}); + expect(started.ready).toBe(true); + expect((await api("GET", "/api/local-computer")).stop_reason).toBeNull(); + expect(JSON.parse(readFileSync(stateFile, "utf8")).actions).toEqual([ + { action: "stop", target: "shared" }, { action: "start", target: "shared" }, + ]); + await api("DELETE", `/api/bots/${bot.id}`); + }); + + it("resumes an existing per-bot VM even at the instance cap", async () => { + vmState({ containers: [] }); + await api("PATCH", "/api/config", { localVm: { mode: "per-bot", maxInstances: 1 } }); + const { bot } = await api("POST", "/api/bots", { name: "Per-bot resume", computer: "vm" }); + await api("POST", `/api/bots/${bot.id}/local-computer/run`, {}); + await api("POST", `/api/bots/${bot.id}/local-computer/stop`, {}); + expect((await api("POST", `/api/bots/${bot.id}/local-computer/start`, {})).ready).toBe(true); + await api("POST", `/api/bots/${bot.id}/local-computer/stop`, {}); + await api("PATCH", `/api/bots/${bot.id}`, { computer: null, browser: false }); + rmSync(dumpFile, { force: true }); rmSync(finishFile, { force: true }); + await api("POST", `/api/bots/${bot.id}/messages`, { text: "Use the existing computer." }); + expect(computer(await dump())).toBeTruthy(); + writeFileSync(finishFile, "finish"); + await idle(bot.id); + expect(JSON.parse(readFileSync(stateFile, "utf8")).actions.map((entry: any) => entry.action)).toEqual([ + "run", "stop", "start", "stop", "start", + ]); + await api("DELETE", `/api/bots/${bot.id}`); + await api("PATCH", "/api/config", { localVm: { mode: "shared", maxInstances: 2 } }); + }); +}); + describe("Group Local VM ownership on the real isolated server", () => { it("cleans the remaining rooms after one cleanup operation fails", async () => { rooms.set("missing-fixture-room", []); @@ -293,7 +338,7 @@ describe("Group Local VM ownership on the real isolated server", () => { expect(created.workspace_path.startsWith(fixtureHome)).toBe(true); const saved = join(created.workspace_path, "saved.txt"); writeFileSync(saved, "survives idle removal"); - // Idle cleanup removes only this container; its workspace survives. + // Older versions removed idle containers; keep that recovery path working. await api("POST", `/api/bots/${returning.id}/local-computer/remove`, {}); await api("POST", `/api/bots/${holder.id}/local-computer/run`, {}); await waitForExit(child, { signal: "SIGTERM" }); diff --git a/server/index.ts b/server/index.ts index c9c53ad9ea..3ed51adbd3 100644 --- a/server/index.ts +++ b/server/index.ts @@ -130,6 +130,7 @@ import { containerExec, containerRuntimeStatus, localVmRecreatableOnDemand, + localVmResumable, localVmWorkspaceExists, perBotLocalVmTarget, poolLocalVmTarget, @@ -418,6 +419,7 @@ import { localVmInventoryEntry, shouldArmLocalVmIdle, } from "./local-vm-inventory.ts"; +import { localVmStopReason, recordLocalVmIdleStop } from "./local-vm-stop-reason.ts"; import { LocalVmIdleTimer } from "./local-vm-idle.ts"; import { LocalVmLease, LocalVmLeasePool } from "./local-vm-lease.ts"; import { LocalVmSeatPool, type LocalVmSeatHolder } from "./local-vm-seat-pool.ts"; @@ -6890,11 +6892,11 @@ function localVmIdleFor(target: LocalVmTarget): LocalVmIdleTimer { localVmLifecycleBusy.add(target.key); try { const status = await containerComputerStatus(undefined, undefined, target); - // The desktop leaves a stale X lock after stop, so idle cleanup - // removes only the disposable container. Its target-specific durable - // workspace and the shared prepared image remain. - if (status.container === "running") { - await containerComputerAction("remove", undefined, undefined, target); + // The pinned VNC startup clears stale X locks. Keep the container's + // home and installed software so inactivity is a reversible stop. + if (status.container === "running" && status.managed) { + const stopped = await containerComputerAction("stop", undefined, undefined, target); + recordLocalVmIdleStop(target.key, stopped.stopped_at); } } finally { localVmLifecycleBusy.delete(target.key); @@ -6934,7 +6936,7 @@ function releaseLocalVmThread(threadId: string): void { // Browser, This computer, Auto, or Off does not delete its old Local VM. void (async () => { if (localVmMode(cfg) === "pool") { - // Same restore rule as per-bot: idle cleanup removes the container, not + // Same restore rule as per-bot: idle shutdown preserves the container and // its provisioned workspace, so every surviving seat stays Auto-eligible. const seats = localVmMaxInstances(cfg); for (let seat = 0; seat < seats; seat += 1) { @@ -14044,6 +14046,7 @@ async function localVmPayload(target: LocalVmTarget, auth: RequestAuth) { const status = await containerComputerStatus(undefined, undefined, target); return { ...localVmViewerStatus(status, auth), + stop_reason: localVmStopReason(target.key, status), commands: setupCommands(status.runtime, process.platform, target), idle_timeout_ms: localVmIdleMs(), mode: localVmMode(cfg), @@ -14051,28 +14054,8 @@ async function localVmPayload(target: LocalVmTarget, auth: RequestAuth) { }; } -/** The Local VM a turn is about to use, recreated if the idle timer took it. - * - * `LocalVmIdleTimer` REMOVES an unused Local VM rather than pausing it. The - * turn then failed with "Create the Local VM (App Settings → Local VM)" — - * which reads like a fault the person must repair by hand, for a container the - * app itself deleted eight hours earlier. Someone who steps away overnight - * comes back to an error on their first message. - * - * The cloud branch below already does the opposite: an absent boat is - * provisioned on first use behind a `provisioning` broadcast. This gives the - * Local VM the same lifecycle for the same reason. - * - * Only `missing` is recovered, and only when a fresh `run` is all it takes. - * Every other problem still surfaces: no runtime installed, no image pulled, - * `create_supported` false, or an existing container that is stale, unmanaged - * or unsafe. Those need a decision — install podman, download 1.4 GB, replace - * a container someone else made — and a stopped container is deliberately not - * resumed here, because `localVmProblem` says this desktop image cannot safely - * resume and asks for a recreate rather than a start. Per-bot mode keeps its - * instance cap; creating past it would quietly do what the lifecycle route - * refuses. - */ +/** Wake an idle desktop, or recreate a missing one from an already prepared + * image. Incompatible or unsafe containers still require an explicit decision. */ async function readyLocalVmForTurn(botId: string, target: LocalVmTarget, isCurrent = () => true) { localVmLifecycleBusy.add(target.key); // Fence this target, and the cross-target capacity decision for creates, @@ -14090,20 +14073,20 @@ async function readyLocalVmForTurn(botId: string, target: LocalVmTarget, isCurre status = await containerComputerStatus(undefined, undefined, target); noteLocalVmSeen(target, status); if (!isCurrent()) return status; - if (status.ready || !localVmRecreatableOnDemand(status)) return status; + if (status.ready || (!localVmRecreatableOnDemand(status) && !localVmResumable(status))) return status; // Another creation is already mid-flight and its container is not yet // visible to a count, so the safe answer is the inspected status — // exactly what the over-cap path below returns. - if (!pooled && !ownsProvision) return status; + if (status.container === "missing" && !pooled && !ownsProvision) return status; - if (target.key.startsWith("bot:")) { + if (status.container === "missing" && target.key.startsWith("bot:")) { const count = await existingPerBotLocalVmCount(status.runtime); if (!isCurrent() || count >= localVmMaxInstances(cfg)) return status; } broadcast({ kind: "computer", botId, state: "provisioning" }); try { - status = await containerComputerAction("run", undefined, undefined, target); + status = await containerComputerAction(status.container === "stopped" ? "start" : "run", undefined, undefined, target); } catch { // Keep the inspected status: its `problem` names the real obstacle, // which is more use to the person than "podman run exited non-zero". @@ -21772,7 +21755,10 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { else localVmLifecycleBusy.add(SHARED_LOCAL_VM_TARGET.key); try { const status = await containerComputerAction(action, undefined, undefined, SHARED_LOCAL_VM_TARGET); - if (action === "run" || action === "start") localVmIdleFor(SHARED_LOCAL_VM_TARGET).touch(); + if (action === "run" || action === "start") { + localVmSeen.add(SHARED_LOCAL_VM_TARGET.key); + localVmIdleFor(SHARED_LOCAL_VM_TARGET).touch(); + } if (action === "stop" || action === "remove") localVmIdleFor(SHARED_LOCAL_VM_TARGET).cancel(); return json(res, 200, { ...localVmViewerStatus(status, auth), @@ -21800,7 +21786,7 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (!bot) return json(res, 404, { error: "no such bot" }); return json(res, 200, await localVmPayload(localVmTargetForStatus(bot.id, bot.threadId), auth)); } - m = path.match(/^\/api\/bots\/([\w-]+)\/local-computer\/(run|stop|remove)$/); + m = path.match(/^\/api\/bots\/([\w-]+)\/local-computer\/(run|start|stop|remove)$/); if (m && method === "POST") { if (!String(req.headers["content-type"] ?? "").toLowerCase().startsWith("application/json")) { return json(res, 415, { error: "content-type must be application/json" }); @@ -21810,9 +21796,10 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (boatLifecycleBusyBots.has(bot.id)) { return json(res, 409, { error: "this bot's computer is being changed or deleted — wait for it to finish" }); } - const action = z.enum(["run", "stop", "remove"]).parse(m[2]); + const action = z.enum(["run", "start", "stop", "remove"]).parse(m[2]); const target = localVmTargetForBot(bot.id); - if (target.key === SHARED_LOCAL_VM_TARGET.key) { + if (localVmMode(cfg) === "pool") return json(res, 409, { error: "Pool desktops start automatically when a conversation needs one" }); + if (localVmMode(cfg) !== "per-bot" && action !== "start") { return json(res, 409, { error: "Shared mode manages this desktop in App Settings → Computers" }); } if (localVmImageBusy || localVmModeChangeBusy || localVmLifecycleBusy.has(target.key)) { @@ -21846,7 +21833,10 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { } } const status = await containerComputerAction(action, undefined, undefined, target); - if (action === "run") localVmIdleFor(target).touch(); + if (action === "run" || action === "start") { + localVmSeen.add(target.key); + localVmIdleFor(target).touch(); + } if (action === "stop" || action === "remove") localVmIdleFor(target).cancel(); return json(res, 200, { ...localVmViewerStatus(status, auth), diff --git a/server/local-vm-stop-reason.test.ts b/server/local-vm-stop-reason.test.ts new file mode 100644 index 0000000000..7d9fbe99ce --- /dev/null +++ b/server/local-vm-stop-reason.test.ts @@ -0,0 +1,18 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { expect, it } from "vitest"; +import { localVmStopReason, recordLocalVmIdleStop } from "./local-vm-stop-reason.ts"; + +it("remembers an idle stop across reads without mislabelling another target or a later external stop", () => { + const dir = mkdtempSync(join(tmpdir(), "omb-stop-reason-")); + try { + const status = { container: "stopped", stopped_at: "2026-10-01T12:00:00Z" }; + recordLocalVmIdleStop("shared", status.stopped_at, dir); + expect(localVmStopReason("shared", status, dir)).toBe("idle"); + expect(localVmStopReason("bot:other", status, dir)).toBeNull(); + expect(localVmStopReason("shared", { ...status, stopped_at: "2026-10-01T13:00:00Z" }, dir)).toBeNull(); + expect(localVmStopReason("shared", { ...status, container: "running" }, dir)).toBeNull(); + expect(localVmStopReason("shared", { container: "stopped" }, dir)).toBeNull(); + } finally { rmSync(dir, { recursive: true, force: true }); } +}); diff --git a/server/local-vm-stop-reason.ts b/server/local-vm-stop-reason.ts new file mode 100644 index 0000000000..08b55650a7 --- /dev/null +++ b/server/local-vm-stop-reason.ts @@ -0,0 +1,27 @@ +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync, renameSync } from "node:fs"; +import { join } from "node:path"; +import { DATA_DIR } from "./config.ts"; + +function recordPath(key: string, dataDir: string): string { + return join(dataDir, "local-vm-stops", `${createHash("sha256").update(key).digest("hex")}.json`); +} + +/** Record the runtime's actual finish timestamp. An external stop later gets + * a different timestamp and must not be misreported as OMB's idle shutdown. */ +export function recordLocalVmIdleStop(key: string, stoppedAt: string | null | undefined, dataDir = DATA_DIR): void { + if (!stoppedAt) return; + const file = recordPath(key, dataDir); + mkdirSync(join(dataDir, "local-vm-stops"), { recursive: true, mode: 0o700 }); + writeFileSync(`${file}.tmp`, JSON.stringify({ stoppedAt }), { mode: 0o600 }); + renameSync(`${file}.tmp`, file); +} + +export function localVmStopReason(key: string, status: { container: string; stopped_at?: string | null }, dataDir = DATA_DIR): "idle" | null { + if (status.container !== "stopped" || !status.stopped_at) return null; + try { + return JSON.parse(readFileSync(recordPath(key, dataDir), "utf8")).stoppedAt === status.stopped_at ? "idle" : null; + } catch { + return null; + } +} diff --git a/server/testing/group-local-vm-hooks.mjs b/server/testing/group-local-vm-hooks.mjs index 610e71d1f7..9c554b0f67 100644 --- a/server/testing/group-local-vm-hooks.mjs +++ b/server/testing/group-local-vm-hooks.mjs @@ -30,17 +30,25 @@ registerHooks({ writeFileSync(file + '.entered', target.key); while (read().blocked || read().blockedTarget === target.key) await new Promise(r => setTimeout(r, 30)); const missing = !(await containerComputerExists('podman', target)); - const ready = !missing && !read().failed; + const stopped = read().stopped?.includes(target.key); + const ready = !missing && !stopped && !read().failed; return { runtime: 'podman', daemonUp: true, image: true, create_supported: true, managed: !missing, - container: missing ? 'missing' : 'running', ready, problem: ready ? null : 'fixture desktop unavailable', + container: missing ? 'missing' : stopped ? 'stopped' : 'running', ready, + imageMatches: true, network: 'loopback', security: 'hardened', persistence: 'durable', + stopped_at: stopped ? read().stoppedAt : null, problem: ready ? null : 'fixture desktop unavailable', container_name: target.containerName, target_key: target.key, workspace_path: target.workspaceDir }; } export async function containerComputerAction(action, _run, _platform, target = SHARED_LOCAL_VM_TARGET) { const state = read(); - if (!state.containers || !['run', 'remove'].includes(action)) throw new Error('Unexpected container mutation in VM routing test'); + if (!state.containers || !['run', 'start', 'stop', 'remove'].includes(action)) throw new Error('Unexpected container mutation in VM routing test'); if (action === 'run') { mkdirSync(target.workspaceDir, { recursive: true }); state.containers.push(target.key); + } else if (action === 'stop') { + state.stopped = [...(state.stopped ?? []), target.key]; + state.stoppedAt = new Date().toISOString(); + } else if (action === 'start') { + state.stopped = (state.stopped ?? []).filter(key => key !== target.key); } else state.containers = state.containers.filter(key => key !== target.key); state.actions = [...(state.actions ?? []), { action, target: target.key }]; writeFileSync(file, JSON.stringify(state)); @@ -48,6 +56,10 @@ registerHooks({ } ` }; const result = nextLoad(url, context); + if (url.endsWith('/local-vm-idle.ts')) { + return { ...result, source: `import { readFileSync as readVmIdle } from 'node:fs';\n` + + String(result.source).replaceAll('checkedIdleMs(idleMs)', `(JSON.parse(readVmIdle(${JSON.stringify(state)}, 'utf8')).idleMs ?? checkedIdleMs(idleMs))`) }; + } if (url.endsWith('/local-vm-lease.ts')) { return { ...result, source: `import { readFileSync as readVmClock } from 'node:fs';\n` + String(result.source).replaceAll('Date.now()', `(Date.now() + (JSON.parse(readVmClock(${JSON.stringify(state)}, 'utf8')).clockOffset || 0))`) }; diff --git a/server/vps-computer.ts b/server/vps-computer.ts index f79a175bd8..4c5d0acc36 100644 --- a/server/vps-computer.ts +++ b/server/vps-computer.ts @@ -962,7 +962,7 @@ export function vpsContainerRunArgs( // someone opens the panel. unless-stopped survives reboots while still // honoring an explicit Stop. The shared hardening check accepts exactly // this policy for the VPS caller (and only "no"/unset for the Local VM, - // whose desktop cannot safely resume). + // whose starts are controlled by OMB's turn lifecycle). "--restart", "unless-stopped", "-e", diff --git a/shared/local-vm-lifecycle.ts b/shared/local-vm-lifecycle.ts new file mode 100644 index 0000000000..02e67fe2a0 --- /dev/null +++ b/shared/local-vm-lifecycle.ts @@ -0,0 +1,15 @@ +/** The same resume boundary is used by the server and the renderer. */ +export function canResumeLocalVm(status: { + daemonUp: boolean; + image: boolean; + container: string; + imageMatches: boolean; + managed: boolean; + network: string; + security: string; + persistence: string; +}): boolean { + return status.daemonUp === true && status.image === true && status.container === "stopped" + && status.imageMatches === true && status.managed === true && status.network === "loopback" + && status.security === "hardened" && status.persistence === "durable"; +} diff --git a/src/components/ComputerPanel.tsx b/src/components/ComputerPanel.tsx index e4e556aee9..177aa3960f 100644 --- a/src/components/ComputerPanel.tsx +++ b/src/components/ComputerPanel.tsx @@ -1,3 +1,5 @@ +import { canResumeLocalVm } from "../../shared/local-vm-lifecycle"; +import { waitForLocalVmReady } from "@/lib/local-vm-readiness"; import { cloudRunner } from "@/lib/remote-desktop"; // The bot's computer, in the right-side slot. Where it runs decides the // whole flow: explicit cloud → provision the boat on open (idempotent) and preview @@ -118,6 +120,8 @@ type Phase = | "error"; interface LocalVmStatus { + daemonUp: boolean; + stop_reason?: "idle" | null; mode: "shared" | "per-bot" | "pool"; max_instances: number; image: boolean; @@ -344,7 +348,7 @@ export function ComputerPanel({ const [vpsStatus, setVpsStatus] = useState(null); const [localFrame, setLocalFrame] = useState(null); const [pending, setPending] = useState< - "join" | "sleep" | "provision" | "vps-replace" | "vm-create" | "vm-recreate" | "vm-delete" | null + "join" | "sleep" | "provision" | "vps-replace" | "vm-start" | "vm-create" | "vm-recreate" | "vm-delete" | null >(null); const [controlPending, setControlPending] = useState(false); const [viewerOpen, setViewerOpen] = useState(false); @@ -380,6 +384,14 @@ export function ComputerPanel({ return () => controller.abort(); }, [connectionKey, livePlace, computerSelectionPersisted, threadPath, retry]); const vmReadinessAttempts = useRef(0); + const vmActionController = useRef(null); + useEffect(() => { + if (vmActionController.current?.signal.aborted) { + vmActionController.current = null; + setPending(null); + } + return () => { vmActionController.current?.abort(); }; + }, [bot.id, bot.threadId, bot.computer, panelView]); const selectedInstance = state.instances.find( (instance) => instance.instanceId === bot.modelSelection.instanceId, ); @@ -551,7 +563,7 @@ export function ComputerPanel({ status.container === "missing" && status.image && status.create_supported; - setError(canCreateHere ? null : new LocalizedPanelError( + setError(canCreateHere || canResumeLocalVm(status) ? null : new LocalizedPanelError( "computer.err.vmOpenSettings", status.problem, "computer.err.vmNotReady", )); setPhase("vm-unavailable"); @@ -1116,7 +1128,7 @@ export function ComputerPanel({ .finally(() => setPending(null)); }; - const runVmAction = async (action: "vm-create" | "vm-recreate" | "vm-delete") => { + const runVmAction = async (action: "vm-start" | "vm-create" | "vm-recreate" | "vm-delete") => { if ( (action === "vm-recreate" || action === "vm-delete") && !window.confirm( @@ -1125,34 +1137,44 @@ export function ComputerPanel({ : t("computer.confirm.replaceVm", { name: bot.name }), ) ) return; + if (vmActionController.current) return; + const controller = new AbortController(); + vmActionController.current = controller; setPending(action); setError(null); - setVmStatus(null); vmReadinessAttempts.current = 0; try { - if (action !== "vm-create") { + if (action === "vm-recreate" || action === "vm-delete") { await api(`/api/bots/${bot.id}/local-computer/remove`, { method: "POST", body: "{}", + signal: controller.signal, }); } if (action !== "vm-delete") { - const status: LocalVmStatus = await api(`/api/bots/${bot.id}/local-computer/run`, { + const started: LocalVmStatus = await api(`/api/bots/${bot.id}/local-computer/${action === "vm-start" ? "start" : "run"}`, { method: "POST", body: "{}", + signal: controller.signal, }); + const status = await waitForLocalVmReady(started, () => api(threadPath("local-computer"), { signal: controller.signal }), controller.signal); + if (!status.ready) throw new Error(status.problem ?? t("computer.err.vmNotReady")); setVmStatus(status); - setPhase(status.ready ? "vm" : "checking"); + setPhase("vm"); } else { setVmStatus((current) => current ? { ...current, container: "missing", ready: false } : current); setPhase("vm-unavailable"); } } catch (e) { + if (controller.signal.aborted) return; setError(e instanceof Error ? e.message : String(e)); setPhase("vm-unavailable"); } finally { - setPending(null); - setRetry((n) => n + 1); + if (vmActionController.current === controller && !controller.signal.aborted) { + vmActionController.current = null; + setPending(null); + setRetry((n) => n + 1); + } } }; @@ -1414,7 +1436,9 @@ export function ComputerPanel({ : localMisses >= 3 ? t("computer.needsScreenPerm") : t("computer.capturingLocal") - : emptyState[phase]} + : phase === "vm-unavailable" && vmStatus && canResumeLocalVm(vmStatus) + ? t(pending === "vm-start" ? "vm.setup.starting" : "computer.phase.vmStopped") + : emptyState[phase]} {currentTeamComputer && <>

Shared files and signed-in accounts. Auto uses this Boat, not a private computer.

@@ -1452,8 +1476,21 @@ export function ComputerPanel({ : t("computer.chooseCloudManage")} )} + {phase === "vm-unavailable" && pending !== "vm-start" && vmStatus && canResumeLocalVm(vmStatus) && ( +

{t(vmStatus.stop_reason === "idle" ? "vm.stopped.idle" : "vm.stopped.detail")}

+ )} {phase === "vm-unavailable" && ( - canManageVm && vmStatus?.mode === "per-bot" && vmStatus.image && vmStatus.create_supported ? ( + canManageVm && vmStatus && vmStatus.mode !== "pool" && canResumeLocalVm(vmStatus) ? ( + + ) : canManageVm && vmStatus?.mode === "per-bot" && vmStatus.image && vmStatus.create_supported ? ( )} - {phase === "vm-unavailable" && pending !== "vm-start" && vmStatus && canResumeLocalVm(vmStatus) && ( -

{t(vmStatus.stop_reason === "idle" ? "vm.stopped.idle" : "vm.stopped.detail")}

+ {vmResumable && pending !== "vm-start" && ( +

{t(vmStatus?.stop_reason === "idle" ? "vm.stopped.idle" : "vm.stopped.detail")}

)} {phase === "vm-unavailable" && ( - canManageVm && vmStatus && vmStatus.mode !== "pool" && canResumeLocalVm(vmStatus) ? ( + canManageVm && vmResumable && vmStatus.mode !== "pool" ? ( ; + const unpinLabel = t("sidebar.bot.unpin"); + const key = `${entry.kind}-${entry.kind === "bot" ? entry.botId : entry.groupId}-${entry.task.threadId}`; + const Icon = status.active ? status.Icon : Pin; + return
+ + +
; })} ; } diff --git a/src/components/SidebarPinnedThreadsPanel.test.ts b/src/components/SidebarPinnedThreadsPanel.test.ts index b05ca297cd..20c012b208 100644 --- a/src/components/SidebarPinnedThreadsPanel.test.ts +++ b/src/components/SidebarPinnedThreadsPanel.test.ts @@ -3,7 +3,7 @@ import { renderToStaticMarkup } from "react-dom/server"; import { describe, expect, it, vi } from "vitest"; import { t } from "@/lib/i18n"; -import type { AttentionThread } from "./SidebarBotActivity"; +import { PinnedThreadRows, type AttentionThread } from "./SidebarBotActivity"; import { SidebarPinnedThreadsPanel } from "./SidebarPinnedThreadsPanel"; const entry: AttentionThread = { @@ -25,11 +25,12 @@ function findElement(tree: ReactNode, attribute: string, value: string): ReactEl function renderPanel( entries: AttentionThread[] = [entry], - options: { collapsed?: boolean; onToggle?: () => void } = {}, + options: { collapsed?: boolean; onToggle?: () => void; onUnpin?: (entry: AttentionThread) => void } = {}, ) { let tree: ReactNode; const onJump = vi.fn(); const onToggle = options.onToggle ?? vi.fn(); + const onUnpin = options.onUnpin ?? vi.fn(); function Capture() { tree = SidebarPinnedThreadsPanel({ entries, @@ -38,11 +39,24 @@ function renderPanel( onJump, collapsed: options.collapsed ?? false, onToggle, + onUnpin, }); return tree; } const markup = renderToStaticMarkup(createElement(Capture)); - return { markup, tree: () => tree as ReactNode, onJump, onToggle }; + return { markup, tree: () => tree as ReactNode, onJump, onToggle, onUnpin }; +} + +function renderRows(entries: AttentionThread[]) { + let tree: ReactNode; + const onJump = vi.fn(); + const onUnpin = vi.fn(); + function Capture() { + tree = PinnedThreadRows({ entries, now: 1, onJump, onUnpin }); + return tree; + } + const markup = renderToStaticMarkup(createElement(Capture)); + return { markup, tree: () => tree as ReactNode, onJump, onUnpin }; } describe("pinned threads panel", () => { @@ -74,4 +88,44 @@ describe("pinned threads panel", () => { findElement(tree(), "aria-label", label)!.props.onClick!({} as MouseEvent); expect(onToggle).toHaveBeenCalledOnce(); }); + +}); + +describe("pinned thread rows", () => { + it("shows the plain Pin icon and a time byline for an idle pinned thread", () => { + const { markup } = renderRows([entry]); + expect(markup).toContain("lucide-pin "); + expect(markup).not.toContain("lucide-circle-alert"); + expect(markup).not.toContain("lucide-loader"); + expect(markup).toContain("Atlas · just now"); + }); + + it("shows the live status icon and word for an actively working pinned thread", () => { + const working: AttentionThread = { + kind: "bot", botId: "atlas", botName: "Atlas", + task: { threadId: "pinned-2", title: "Build report", createdAt: 1, queued: false, busy: true, activity: "working" }, + }; + const { markup } = renderRows([working]); + expect(markup).toContain("lucide-loader"); + expect(markup).not.toContain("lucide-pin "); + expect(markup).toContain("Atlas · " + t("chat.activity.working")); + }); + + it("shows the waiting icon and word for a pinned thread waiting on the person", () => { + const waiting: AttentionThread = { + kind: "bot", botId: "atlas", botName: "Atlas", + task: { threadId: "pinned-3", title: "Needs approval", createdAt: 1, queued: false, activity: "waiting-on-you" }, + }; + const { markup } = renderRows([waiting]); + expect(markup).toContain("lucide-circle-alert"); + expect(markup).toContain("Atlas · " + t("task.waiting")); + }); + + it("calls onUnpin for the row's own entry, not onJump, from the per-row unpin button", () => { + const { tree, onUnpin, onJump } = renderRows([entry]); + const label = t("sidebar.bot.unpin"); + findElement(tree(), "aria-label", label)!.props.onClick!({} as MouseEvent); + expect(onUnpin).toHaveBeenCalledWith(entry); + expect(onJump).not.toHaveBeenCalled(); + }); }); diff --git a/src/components/SidebarPinnedThreadsPanel.tsx b/src/components/SidebarPinnedThreadsPanel.tsx index 04ee4e11ad..822dfda9ae 100644 --- a/src/components/SidebarPinnedThreadsPanel.tsx +++ b/src/components/SidebarPinnedThreadsPanel.tsx @@ -8,11 +8,12 @@ import { PinnedThreadRows, type AttentionThread } from "./SidebarBotActivity"; * search and the bots list — pinned bots already get this top-level view * (the built-in Pinned section); pinned threads did not. Renders nothing * when there is no pin, so it never costs space it isn't using. */ -export function SidebarPinnedThreadsPanel({ entries, density, now, onJump, collapsed, onToggle }: { +export function SidebarPinnedThreadsPanel({ entries, density, now, onJump, onUnpin, collapsed, onToggle }: { entries: AttentionThread[]; density: SidebarDensity; now: number; onJump: (entry: AttentionThread) => void; + onUnpin: (entry: AttentionThread) => void; collapsed: boolean; onToggle: () => void; }) { @@ -41,7 +42,7 @@ export function SidebarPinnedThreadsPanel({ entries, density, now, onJump, colla
{!collapsed && (
- +
)} From dcb0148d41ae19cab001381bb72da6a877d9414f Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 12:53:55 +0530 Subject: [PATCH 055/211] chore(chat): document intentional normalization sentinels (cherry picked from commit dec8093177e51816ecd5b5d8422a18fdb27c1332) --- src/components/ChatMarkdown.tsx | 1 + 1 file changed, 1 insertion(+) diff --git a/src/components/ChatMarkdown.tsx b/src/components/ChatMarkdown.tsx index 41c59a4a8e..4550b8dce0 100644 --- a/src/components/ChatMarkdown.tsx +++ b/src/components/ChatMarkdown.tsx @@ -808,6 +808,7 @@ export function normalizeMathDelimiters(text: string, imageOffsets?: Map `$$\n${math}\n$$`); normalized = escapeLiteralDollars(normalized); let shift = 0; + // oxlint-disable-next-line no-control-regex -- restore opaque code and image sentinels normalized = normalized.replace(/\u0000OMB_CODE_(\d+)\u0000/g, (token, index: string, at: number) => { const part = protectedCode[Number(index)]; if (!part) return token; From efc91dc6cbb9f3e49e8d4ff3dda01756e893f044 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 12:53:56 +0530 Subject: [PATCH 056/211] fix(sidebar): derive pinned room status from room activity (cherry picked from commit 7fd82d5cfbd288017916cabc51ca1115b43b7197) --- src/components/SidebarBotActivity.test.ts | 14 ++++++++++++++ src/components/SidebarBotActivity.tsx | 9 ++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/components/SidebarBotActivity.test.ts b/src/components/SidebarBotActivity.test.ts index 85fbeec041..17bf53e7a9 100644 --- a/src/components/SidebarBotActivity.test.ts +++ b/src/components/SidebarBotActivity.test.ts @@ -167,6 +167,20 @@ describe("cross-bot pinned threads", () => { const [entry] = crossBotPinnedThreads([alpha], [], { a0: [{}] }); expect(entry.task.queued).toBe(true); }); + + it("derives a pinned room's live status without attributing it to an idle sibling", () => { + const alpha = bot("a", "Alpha", "a0", [], { activity: "waiting-on-you" }); + const room = group("g", "Crew", "g0", { busyBotId: "a", unread: true, tasks: [ + { threadId: "g0", title: "Crew", createdAt: 0, pinned: true }, + { threadId: "g1", title: "Idle sibling", createdAt: 0, pinned: true }, + ] }); + const status = (patch: Partial) => crossBotPinnedThreads([alpha], [{ ...room, ...patch }], {}) + .map((entry) => attentionRowStatus(entry.task)); + expect(status({})[0].label).toBe(t("task.waiting")); + expect(status({ busyBotId: null, working: true })[0].label).toBe(t("chat.activity.working")); + expect(status({ busyBotId: null })[0].label).toBe(t("task.unread")); + expect(status({})[1].active).toBe(false); + }); }); describe("attentionRowStatus", () => { diff --git a/src/components/SidebarBotActivity.tsx b/src/components/SidebarBotActivity.tsx index 84036abb07..6fd654c3cc 100644 --- a/src/components/SidebarBotActivity.tsx +++ b/src/components/SidebarBotActivity.tsx @@ -140,9 +140,12 @@ export function crossBotPinnedThreads(bots: Bot[], groups: Group[], queued: Reco .filter((task) => task.pinned === true && !task.routineRunId) .map((task) => ({ ...task, queued: Boolean(queued[task.threadId]?.length), botId: bot.id, botName: bot.name }))), ...groups - .flatMap((group): FlatAttentionEntry[] => (group.tasks ?? []) - .filter((task) => task.pinned === true) - .map((task) => ({ ...task, queued: Boolean(queued[task.threadId]?.length), groupId: group.id, groupName: group.name, groupThreadId: group.threadId }))), + .flatMap((group): FlatAttentionEntry[] => { + const activity = new Map(sidebarGroupActivityTasks(group, bots, queued).map((task) => [task.threadId, task])); + return (group.tasks ?? []) + .filter((task) => task.pinned === true) + .map((task) => ({ ...task, ...activity.get(task.threadId), queued: Boolean(queued[task.threadId]?.length), groupId: group.id, groupName: group.name, groupThreadId: group.threadId })); + }), ]; return orderedThreadList(flat).map(({ botId, botName, groupId, groupName, groupThreadId, ...task }): AttentionThread => groupId !== undefined From f1faa9fc307f32ba9cfc32cef861d2923fb86c04 Mon Sep 17 00:00:00 2001 From: virtuousityai Date: Fri, 2 Oct 2026 01:43:12 -0400 Subject: [PATCH 057/211] feat(sidebar): make the Active Threads panel collapsible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Active Threads panel previously had no way to fold its body — unlike every bot/group section, which already supports this via the shared collapsedSections mechanism (sidebar-preferences.ts). Wires the panel into the same mechanism with a new fixed id (ATTENTION_SECTION_ID) and a chevron toggle in its header, matching the pattern SidebarSectionHeader already uses. Reuses the existing generic sidebar.section.expand/collapse i18n strings, so no new translations are needed. Collapsed state persists the same way other sections' does. Co-Authored-By: Claude Sonnet 5 (cherry picked from commit df9bdb51d9fc32d091435e882de77c62cdca93a0) --- src/components/Sidebar.tsx | 3 ++ src/components/SidebarAttentionPanel.test.ts | 38 ++++++++++++++++++-- src/components/SidebarAttentionPanel.tsx | 20 ++++++++--- 3 files changed, 54 insertions(+), 7 deletions(-) diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index eba05a36d0..bbeb5cb11a 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -79,6 +79,7 @@ import { type SidebarDensity, } from "@/lib/sidebar-preferences"; import { + ATTENTION_SECTION_ID, BOT_CHATS_SECTION_ID, BOTS_SECTION_ID, CHANNELS_SECTION_ID, @@ -2207,6 +2208,8 @@ export function Sidebar({ open, onClose, collapseToIcons = false }: { density={density} onUnpin={() => setAttentionPinned(false)} onJump={(entry) => dispatch(attentionJumpAction(entry))} + collapsed={sectionCollapsed(ATTENTION_SECTION_ID)} + onToggle={() => toggleSection(ATTENTION_SECTION_ID)} /> )} diff --git a/src/components/SidebarAttentionPanel.test.ts b/src/components/SidebarAttentionPanel.test.ts index bb9a3689bf..5f7a341ea4 100644 --- a/src/components/SidebarAttentionPanel.test.ts +++ b/src/components/SidebarAttentionPanel.test.ts @@ -23,16 +23,27 @@ function findElement(tree: ReactNode, attribute: string, value: string): ReactEl } } -function renderPanel(entries: AttentionThread[] = [entry]) { +function renderPanel( + entries: AttentionThread[] = [entry], + options: { collapsed?: boolean; onToggle?: () => void } = {}, +) { let tree: ReactNode; const onUnpin = vi.fn(); const onJump = vi.fn(); + const onToggle = options.onToggle ?? vi.fn(); function Capture() { - tree = SidebarAttentionPanel({ entries, density: "comfortable", onUnpin, onJump }); + tree = SidebarAttentionPanel({ + entries, + density: "comfortable", + onUnpin, + onJump, + collapsed: options.collapsed ?? false, + onToggle, + }); return tree; } const markup = renderToStaticMarkup(createElement(Capture)); - return { markup, tree: () => tree as ReactNode, onUnpin, onJump }; + return { markup, tree: () => tree as ReactNode, onUnpin, onJump, onToggle }; } describe("pinned attention panel", () => { @@ -72,4 +83,25 @@ describe("pinned attention panel", () => { findElement(tree(), "aria-label", "Unpin")!.props.onClick!({} as MouseEvent); expect(onUnpin).toHaveBeenCalledOnce(); }); + + it("hides the rows but keeps the header when collapsed", () => { + const { markup } = renderPanel([entry], { collapsed: true }); + expect(markup).toContain("Active Threads"); + expect(markup).not.toContain("Review permission"); + const label = t("sidebar.section.expand", { name: t("attention.title") }); + expect(markup).toContain('aria-label="' + label + '"'); + expect(markup).toContain('aria-expanded="false"'); + }); + + it("shows the empty label only when expanded", () => { + const { markup } = renderPanel([], { collapsed: true }); + expect(markup).not.toContain("No active threads"); + }); + + it("toggles from the collapse/expand control", () => { + const { tree, onToggle } = renderPanel([entry], { collapsed: false }); + const label = t("sidebar.section.collapse", { name: t("attention.title") }); + findElement(tree(), "aria-label", label)!.props.onClick!({} as MouseEvent); + expect(onToggle).toHaveBeenCalledOnce(); + }); }); diff --git a/src/components/SidebarAttentionPanel.tsx b/src/components/SidebarAttentionPanel.tsx index 41a172d597..9f16ff42ca 100644 --- a/src/components/SidebarAttentionPanel.tsx +++ b/src/components/SidebarAttentionPanel.tsx @@ -1,4 +1,4 @@ -import { Activity, PinOff } from "lucide-react"; +import { Activity, ChevronDown, ChevronRight, PinOff } from "lucide-react"; import { cn } from "@/lib/cn"; import { t } from "@/lib/i18n"; import type { SidebarDensity } from "@/lib/sidebar-preferences"; @@ -6,13 +6,16 @@ import { AttentionThreadRows, type AttentionThread } from "./SidebarBotActivity" /** The pinned form of the attention panel: the same rows the popover lists, * living between search and the bots list so active work stays in view. */ -export function SidebarAttentionPanel({ entries, density, onUnpin, onJump }: { +export function SidebarAttentionPanel({ entries, density, onUnpin, onJump, collapsed, onToggle }: { entries: AttentionThread[]; density: SidebarDensity; onUnpin: () => void; onJump: (entry: AttentionThread) => void; + collapsed: boolean; + onToggle: () => void; }) { const compact = density === "compact"; + const Chevron = collapsed ? ChevronRight : ChevronDown; return (
+
- {entries.length === 0 ? ( + {!collapsed && (entries.length === 0 ? (
{t("attention.empty")}
) : (
- )} + ))}
); } From d985dac107514133fb86414a2a809069396446e3 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 12:58:17 +0530 Subject: [PATCH 058/211] fix(ios): bound untrusted VNC updates (cherry picked from commit ca64159357a38338f639910992f2510f1df21531) --- ios/Sources/CompanionCore/RFB.swift | 16 ++++++++++-- ios/Tests/CompanionCoreTests/RFBTests.swift | 27 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/ios/Sources/CompanionCore/RFB.swift b/ios/Sources/CompanionCore/RFB.swift index ca12eba2ca..b5bb73d67b 100644 --- a/ios/Sources/CompanionCore/RFB.swift +++ b/ios/Sources/CompanionCore/RFB.swift @@ -105,6 +105,8 @@ public final class RFBClient { /// Larger than any desktop a Local VM runs, and small enough that a /// misbehaving server cannot make the phone allocate gigabytes. static let maxDimension = 8192 + /// One largest supported framebuffer plus protocol headers/clipboard. + static let maxPendingBytes = maxDimension * maxDimension * 4 + (1 << 20) private let password: String? private var phase = Phase.version @@ -134,6 +136,9 @@ public final class RFBClient { /// when the session cannot continue. @discardableResult public func receive(_ data: Data) throws -> [RFBEvent] { + guard data.count <= Self.maxPendingBytes - pending.count else { + throw RFBError.malformed("incoming buffer") + } pending.append(data) var events: [RFBEvent] = [] while let event = try step() { @@ -331,6 +336,7 @@ public final class RFBClient { guard pending.count >= 4 else { return nil } let count = Int(pending.readUInt16(at: 2)) var offset = 4 + var pictureWidth = width, pictureHeight = height var rects: [(x: Int, y: Int, w: Int, h: Int, encoding: Int32, data: Int)] = [] for _ in 0 ..< count { guard pending.count >= offset + 12 else { return nil } @@ -339,8 +345,14 @@ public final class RFBClient { let w = Int(pending.readUInt16(at: offset + 4)) let h = Int(pending.readUInt16(at: offset + 6)) let encoding = Int32(bitPattern: pending.readUInt32(at: offset + 8)) - if encoding == Self.encodingDesktopSize, w > Self.maxDimension || h > Self.maxDimension { - throw RFBError.malformed("desktop size") + if encoding == Self.encodingDesktopSize { + guard w > 0, h > 0, w <= Self.maxDimension, h <= Self.maxDimension else { + throw RFBError.malformed("desktop size") + } + pictureWidth = w + pictureHeight = h + } else if encoding == Self.encodingRaw, x + w > pictureWidth || y + h > pictureHeight { + throw RFBError.malformed("rectangle bounds") } let body = try rectangleLength(width: w, height: h, encoding: encoding) guard pending.count >= offset + 12 + body else { return nil } diff --git a/ios/Tests/CompanionCoreTests/RFBTests.swift b/ios/Tests/CompanionCoreTests/RFBTests.swift index 80b9f06f4a..e6dc571d9b 100644 --- a/ios/Tests/CompanionCoreTests/RFBTests.swift +++ b/ios/Tests/CompanionCoreTests/RFBTests.swift @@ -149,6 +149,33 @@ final class RFBTests: XCTestCase { XCTAssertEqual(client.framebuffer.enumerated().filter { $0.offset % 4 == 0 }.map(\.element), [1, 1, 2]) } + func testRefusesRawRectanglesOutsideTheDesktopBeforeWaitingForPixels() throws { + for (x, y, w, h) in [(0, 0, 65_535, 65_535), (3, 0, 2, 1), (0, 2, 1, 2)] { + let client = try connected(width: 4, height: 3) + let header = Data([0, 0] + u16(1) + u16(x) + u16(y) + u16(w) + u16(h) + s32(0)) + XCTAssertThrowsError(try client.receive(header)) { error in + XCTAssertEqual(error as? RFBError, .malformed("rectangle bounds")) + } + } + } + + func testAcceptsRawPixelsAfterAResizeInTheSameUpdate() throws { + let client = try connected(width: 2, height: 1) + let resize = u16(0) + u16(0) + u16(3) + u16(2) + s32(-223) + let pixels = [UInt8](repeating: 7, count: 3 * 2 * 4) + let raw = u16(0) + u16(0) + u16(3) + u16(2) + s32(0) + pixels + XCTAssertEqual(try client.receive(Data([0, 0] + u16(2) + resize + raw)), [.updated(resized: true)]) + XCTAssertEqual(client.framebuffer, pixels) + } + + func testBoundsPendingBytesAcrossMessagesBeforeAppending() throws { + let client = RFBClient(password: nil) + try client.receive(Data("R".utf8)) + XCTAssertThrowsError(try client.receive(Data(repeating: 0, count: RFBClient.maxPendingBytes))) { error in + XCTAssertEqual(error as? RFBError, .malformed("incoming buffer")) + } + } + func testRefusesAnEncodingItDidNotAskFor() throws { let client = try connected() XCTAssertThrowsError(try client.receive(Data([0, 0] + u16(1) + u16(0) + u16(0) + u16(1) + u16(1) + s32(7)))) { error in From 752275381ba2343b91b19815f17e1af1831044b5 Mon Sep 17 00:00:00 2001 From: Milind Soni <46266943+milind-soni@users.noreply.github.com> Date: Fri, 2 Oct 2026 13:03:06 +0530 Subject: [PATCH 059/211] test(acp): make the quiet-agent tests pass on Linux and Windows (#2154) #2145 merged with its Ubuntu and Windows shard 4 red, and main has failed the same three tests on every run since: - quiet-status.test: Linux procps prints `ps` TIME in whole seconds, so a test worker with under a second of CPU reads 0. Still require a found (non-null) sample everywhere; require non-zero CPU only on macOS. - acp.test (2 tests): on Windows each probe is a PowerShell process that takes 1-3 s to start, longer than the tests' sub-second quiet windows. Run them where the probe is fast; the real guard ticks every 15 s. Co-authored-by: Claude Opus 5.5 --- server/drivers/acp/acp.test.ts | 9 +++++++-- server/drivers/acp/quiet-status.test.ts | 7 ++++++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/server/drivers/acp/acp.test.ts b/server/drivers/acp/acp.test.ts index 64386778f8..9f4f8d77e6 100644 --- a/server/drivers/acp/acp.test.ts +++ b/server/drivers/acp/acp.test.ts @@ -1290,7 +1290,12 @@ describe("ACP turns (fake CLI)", () => { }); // A quiet agent is not a black box: the person is told what it is doing. - it("tells the person what a quiet agent is doing, then finishes the turn", async () => { + // Not on Windows: there each probe is a PowerShell process that takes one + // to three seconds to start, longer than these tests' sub-second quiet + // windows (the real guard ticks every 15 s and waits minutes, so the + // probe's speed does not matter there). + const quietProbeIsFast = process.platform !== "win32"; + it.runIf(quietProbeIsFast)("tells the person what a quiet agent is doing, then finishes the turn", async () => { process.env.OMB_ACP_QUIET_NOTICE_MS = "150"; process.env.OMB_ACP_QUIET_TICK_MS = "50"; process.env.FAKE_ACP_QUIET_MS = "700"; @@ -1303,7 +1308,7 @@ describe("ACP turns (fake CLI)", () => { expect(recorder.events.some(e => e.type === "runtime.error")).toBe(false); }); - it("reads Qwen's debug log: a logged rate-limit retry is reported and keeps the turn alive", async () => { + it.runIf(quietProbeIsFast)("reads Qwen's debug log: a logged rate-limit retry is reported and keeps the turn alive", async () => { process.env.QWEN_HOME = scratch; process.env.OPENMAUS_ACP_PROMPT_IDLE_TIMEOUT_MS = "400"; process.env.OMB_ACP_QUIET_NOTICE_MS = "150"; diff --git a/server/drivers/acp/quiet-status.test.ts b/server/drivers/acp/quiet-status.test.ts index 0f520ca638..a3422bd7c7 100644 --- a/server/drivers/acp/quiet-status.test.ts +++ b/server/drivers/acp/quiet-status.test.ts @@ -126,7 +126,12 @@ describe("process probe", () => { await new Promise((resolve) => socket.once("connect", () => resolve())); try { const sample = await sampleProcessTree(process.pid); - expect(sample.cpuMs).toBeGreaterThan(0); + // Found, not "unknown". macOS `ps` prints TIME to the hundredth of a + // second; Linux procps prints whole seconds, so a test worker that has + // used under a second of CPU honestly reads 0 there. + expect(sample.cpuMs).not.toBeNull(); + expect(sample.cpuMs!).toBeGreaterThanOrEqual(0); + if (process.platform === "darwin") expect(sample.cpuMs!).toBeGreaterThan(0); // lsof may be missing on a minimal Linux box: unknown, never zero if (sample.connections !== null) expect(sample.connections).toBeGreaterThanOrEqual(1); } finally { From 92e1d2e1c0417f6d626cf2feb27cbcd3fc31e62d Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:15:59 +0530 Subject: [PATCH 060/211] fix(desktop-viewer): don't re-close or re-count a viewer that is already closed closeForOwner walked every viewer still listed, and a viewer leaves the list only on its socket's close event. On Windows that event arrives after the client has already seen the close, so a hand-back followed by a sign-out counted the handed-back viewer twice (3 instead of 2), failing desktop-viewer.test.ts on Windows since #2135 and blocking 0.1.93. Skip viewers whose socket is already destroyed. The new assertion (asking twice before the close event) reproduces it on every platform. Co-Authored-By: Claude Opus 5.5 --- server/routes/desktop-viewer.test.ts | 2 ++ server/routes/desktop-viewer.ts | 3 +++ 2 files changed, 5 insertions(+) diff --git a/server/routes/desktop-viewer.test.ts b/server/routes/desktop-viewer.test.ts index 9fc8cc6580..77375d7168 100644 --- a/server/routes/desktop-viewer.test.ts +++ b/server/routes/desktop-viewer.test.ts @@ -407,6 +407,8 @@ it("keeps the session and scope checks ahead of the lease, and closes a session' const closedMine = once(mine, "close"); expect(viewer.closeForOwner("nobody", "test-bot")).toBe(0); expect(viewer.closeForOwner(admin.session.id, "test-bot")).toBe(1); + // Asked again before the socket's close event: nothing left to close. + expect(viewer.closeForOwner(admin.session.id, "test-bot")).toBe(0); await closedMine; expect(other.destroyed).toBe(false); expect(browser.destroyed).toBe(false); diff --git a/server/routes/desktop-viewer.ts b/server/routes/desktop-viewer.ts index 1f4cac14f3..f531f086a3 100644 --- a/server/routes/desktop-viewer.ts +++ b/server/routes/desktop-viewer.ts @@ -210,6 +210,9 @@ export function createDesktopViewer(deps: { let closed = 0; for (const upgrade of upgrades.values()) { if (upgrade.owner !== owner || (botId !== undefined && upgrade.botId !== botId)) continue; + // A viewer already closed stays listed until its socket's close + // event, which arrives later on Windows: not closed or counted again. + if (upgrade.socket.destroyed) continue; upgrade.close(); closed++; } From d98d25667149b5fbe9bfa495a673ea3609ff5fb3 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:26:25 +0530 Subject: [PATCH 061/211] test(api): assert persisted Local VM idle timeout --- server/index.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/index.test.ts b/server/index.test.ts index a980233da0..040c492de8 100644 --- a/server/index.test.ts +++ b/server/index.test.ts @@ -8637,7 +8637,7 @@ describe("harness HTTP API", () => { expect(invalid.body.error).toContain("localVm.maxInstances"); const disk = JSON.parse(readFileSync(join(home, ".openmausbot", "config.json"), "utf8")); - expect(disk.localVm).toEqual({ mode: "per-bot", maxInstances: 5 }); + expect(disk.localVm).toEqual({ mode: "per-bot", maxInstances: 5, idleTimeoutMinutes: 480 }); await api("PATCH", "/api/config", { localVm: { mode: "shared", maxInstances: 2 } }); }); From f08c9e6481ac1e5ee993e673e5b09b23c0252254 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:31:10 +0530 Subject: [PATCH 062/211] fix(chat): preserve CommonMark syntax while normalizing currency (cherry picked from commit a3e537c264368fabb07e3c88926d3a629326e6fa) --- src/components/ChatMarkdown.test.ts | 27 +++++++--- src/components/ChatMarkdown.tsx | 83 +++++++++++------------------ 2 files changed, 52 insertions(+), 58 deletions(-) diff --git a/src/components/ChatMarkdown.test.ts b/src/components/ChatMarkdown.test.ts index 8cc322304f..53d649554d 100644 --- a/src/components/ChatMarkdown.test.ts +++ b/src/components/ChatMarkdown.test.ts @@ -115,14 +115,14 @@ describe("math rendering", () => { }); it("keeps prices literal instead of rendering the text between them as math", () => { - for (const text of [ - "**1. R$ 120:** o plano custa R$ 120 por mês.", - "**2. Os R$1.500,00: à vista ou parcelado?** O total fica em R$ 1.500,00.", - "It costs $5 and the upgrade costs $10.", - "Plans: US$5, $20 per month, or $x$ per seat.", - ]) { + for (const [text, expected] of [ + ["**1. R$ 120:** o plano custa R$ 120 por mês.", 0], + ["**2. Os R$1.500,00: à vista ou parcelado?** O total fica em R$ 1.500,00.", 0], + ["It costs $5 and the upgrade costs $10.", 0], + ["Plans: US$5, $20 per month, or $x$ per seat.", 1], + ] as const) { const html = renderToStaticMarkup(createElement(ChatMarkdown, { text })); - expect(html.match(/class="katex"/g)?.length ?? 0).toBeLessThanOrEqual(1); + expect(html.match(/class="katex"/g)?.length ?? 0).toBe(expected); expect(html).toContain("$"); } const prose = renderToStaticMarkup(createElement(ChatMarkdown, { @@ -141,6 +141,12 @@ describe("math rendering", () => { expect(html).toContain("Pay $5 now"); }); + it("does not treat a math closer as a currency sign", () => { + const html = renderToStaticMarkup(createElement(ChatMarkdown, { text: "$R$ 120 and US$5." })); + expect(html.match(/class="katex"/g)).toHaveLength(1); + expect(html).toContain("120 and US$5."); + }); + it("does not pair dollars across paragraphs", () => { const html = renderToStaticMarkup(createElement(ChatMarkdown, { text: "Costs $5.\n\nThen pay later$" })); expect(html).not.toContain('class="katex"'); @@ -149,13 +155,20 @@ describe("math rendering", () => { it.each([ "R$ 120.\n\n![receipt](/workspace/receipt.png)", "Pay $5.\n\n![receipt][image]\n\n[image]: /workspace/receipt.png", + "` lone ![receipt](/workspace/receipt.png) ``code``", + "![receipt][R$5]\n\n[R$5]: /workspace/receipt.png", + "![R$5]\n\n[R$5]: /workspace/receipt.png", + "[R$5]: /workspace/receipt.png\n\nPay $10.\n\n![receipt][R$5]", + "Price R$ 120.\n\n[![receipt](/workspace/receipt.png)][R$5]\n\n[R$5]: https://example.test", ])("keeps local image authorization offsets after prices: %s", (text) => { const preview = vi.spyOn(AttachmentPreview, "MarkdownImagePreview"); try { renderToStaticMarkup(createElement(ChatMarkdown, { text, message: { threadId: "thread-1", messageId: "message-1" }, })); + expect(preview).toHaveBeenCalledOnce(); expect(preview.mock.calls[0][0].sourceOffset).toBe(text.indexOf("![")); + expect(preview.mock.calls[0][0].filePath).toBe("/workspace/receipt.png"); } finally { preview.mockRestore(); } diff --git a/src/components/ChatMarkdown.tsx b/src/components/ChatMarkdown.tsx index 4550b8dce0..191a24db11 100644 --- a/src/components/ChatMarkdown.tsx +++ b/src/components/ChatMarkdown.tsx @@ -700,42 +700,12 @@ function Spoiler({ children }: { children?: ReactNode }) { const NO_MENTION_PEERS: readonly MentionPeer[] = []; -// A markdown image resolves its attachment by source offset, so a message -// holding one must reach the parser byte-for-byte as written. +// A markdown image resolves its attachment by its original source offset. const MARKDOWN_IMAGE = "!["; -/** Replace CommonMark fenced code blocks with opaque tokens while text is normalized. */ -function protectFencedCode(text: string, protect: (value: string) => string): string { - const opener = - /(^|\r?\n)((?: {0,3}>[ \t]?)* {0,3})(?:(`{3,})([^`\r\n]*)|(~{3,})([^\r\n]*))(?:\r?\n|$)/g; - let cursor = 0; - let tokenized = ""; - let match: RegExpExecArray | null; - - while ((match = opener.exec(text)) !== null) { - const fence = match[3] ?? match[5]; - const fenceCharacter = fence[0]; - const closer = new RegExp( - `(^|\\r?\\n)(?: {0,3}>[ \\t]?)* {0,3}${fenceCharacter}{${fence.length},}[ \\t]*(?=\\r?\\n|$)`, - "g", - ); - closer.lastIndex = opener.lastIndex; - const closingMatch = closer.exec(text); - const end = closingMatch === null - ? text.length - : closingMatch.index + closingMatch[0].length; - tokenized += text.slice(cursor, match.index); - tokenized += protect(text.slice(match.index, end)); - cursor = end; - opener.lastIndex = end; - } - - return tokenized + text.slice(cursor); -} - // A currency sign glued to its code and followed by an amount ("R$ 120", // "US$5") is money, never a math delimiter. -const CURRENCY_DOLLAR = /(?): string { - const protectedCode: Array<{ value: string; imageOffset?: number }> = []; - const protect = (value: string, imageOffset?: number): string => { + const protectedCode: Array<{ value: string; sourceOffset: number }> = []; + const protect = (value: string, sourceOffset: number): string => { const token = `\u0000OMB_CODE_${protectedCode.length}\u0000`; - protectedCode.push({ value, imageOffset }); + protectedCode.push({ value, sourceOffset }); return token; }; - if (imageOffsets) { - const images: Array<{ start: number; end: number }> = []; - const visit = (node: { type: string; children?: any[]; position?: { start: { offset?: number }; end: { offset?: number } } }) => { - const start = node.position?.start.offset; - const end = node.position?.end.offset; - if ((node.type === "image" || node.type === "imageReference") && start !== undefined && end !== undefined) images.push({ start, end }); - node.children?.forEach(visit); - }; - visit(fromMarkdown(text, { mdastExtensions: [windowsPathDestinations] })); - for (const { start, end } of images.reverse()) { - text = text.slice(0, start) + protect(text.slice(start, end), start) + text.slice(end); + const spans: Array<{ start: number; end: number }> = []; + const imageStarts: number[] = []; + const visit = (node: { type: string; children?: any[]; position?: { start: { offset?: number }; end: { offset?: number } } }, protectedParent = false) => { + const start = node.position?.start.offset; + const end = node.position?.end.offset; + const image = node.type === "image" || node.type === "imageReference"; + if (image && start !== undefined) imageStarts.push(start); + // Use the same CommonMark parser as attachment authorization. Protect + // whole syntax nodes once: code cannot swallow an image sentinel, and + // escaping a reference label cannot break its matching definition. + const protectedNode = node.type === "code" || node.type === "inlineCode" || node.type === "definition" || node.type === "linkReference" || (imageOffsets !== undefined && image); + if (!protectedParent && protectedNode && start !== undefined && end !== undefined) { + spans.push({ start, end }); } + node.children?.forEach((child) => visit(child, protectedParent || protectedNode)); + }; + visit(fromMarkdown(text, { mdastExtensions: [windowsPathDestinations] })); + for (const { start, end } of spans.reverse()) { + text = text.slice(0, start) + protect(text.slice(start, end), start) + text.slice(end); } - const tokenized = protectFencedCode(text, protect) - .replace(/(`+)[\s\S]*?\1/g, (value) => protect(value)); - let normalized = tokenized + let normalized = text .replace(/\\\[([\s\S]*?)\\\]/g, (_match, math: string) => `$$\n${math}\n$$`) .replace(/\\\(([\s\S]*?)\\\)/g, (_match, math: string) => `$${math.trim()}$`) // remark-math treats flow math as a block only when the fences occupy @@ -812,8 +787,14 @@ export function normalizeMathDelimiters(text: string, imageOffsets?: Map { const part = protectedCode[Number(index)]; if (!part) return token; - const { value, imageOffset } = part; - if (imageOffset !== undefined) imageOffsets?.set(at + shift, imageOffset); + const { value, sourceOffset } = part; + // A protected reference link can contain images of its own. Their raw + // positions stay relative to that unchanged span when it is restored. + for (const imageOffset of imageStarts) { + if (imageOffset >= sourceOffset && imageOffset < sourceOffset + value.length) { + imageOffsets?.set(at + shift + imageOffset - sourceOffset, imageOffset); + } + } shift += value.length - token.length; return value; }); From 0799f3075bdae8b25a5653fcbd69dfcfe73ccad5 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:34:18 +0530 Subject: [PATCH 063/211] test(ios): clarify fixed VNC receive budget (cherry picked from commit f76179f38702f95e56a3242beaee5c056ebd4eaa) --- ios/Sources/CompanionCore/RFB.swift | 6 +++++- ios/Tests/CompanionCoreTests/RFBTests.swift | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/ios/Sources/CompanionCore/RFB.swift b/ios/Sources/CompanionCore/RFB.swift index b5bb73d67b..173268fae4 100644 --- a/ios/Sources/CompanionCore/RFB.swift +++ b/ios/Sources/CompanionCore/RFB.swift @@ -105,7 +105,11 @@ public final class RFBClient { /// Larger than any desktop a Local VM runs, and small enough that a /// misbehaving server cannot make the phone allocate gigabytes. static let maxDimension = 8192 - /// One largest supported framebuffer plus protocol headers/clipboard. + /// Fixed budget: one maximum supported framebuffer plus 1 MiB overhead, + /// not the current frame size. Ordinary multi-rectangle updates fit. + /// ponytail: whole updates are buffered; even valid extreme overlapping + /// updates above this budget are refused. Consume rectangles incrementally + /// if those updates need support. static let maxPendingBytes = maxDimension * maxDimension * 4 + (1 << 20) private let password: String? diff --git a/ios/Tests/CompanionCoreTests/RFBTests.swift b/ios/Tests/CompanionCoreTests/RFBTests.swift index e6dc571d9b..7bbb39c14f 100644 --- a/ios/Tests/CompanionCoreTests/RFBTests.swift +++ b/ios/Tests/CompanionCoreTests/RFBTests.swift @@ -168,6 +168,24 @@ final class RFBTests: XCTestCase { XCTAssertEqual(client.framebuffer, pixels) } + func testAcceptsMultipleRawRectanglesBeyondTheCurrentFramebufferBudget() throws { + let width = 1280, height = 800 + let client = try connected(width: width, height: height) + let frameBytes = width * height * 4 + var update = Data([0, 0] + u16(2)) + for value: UInt8 in [7, 9] { + update.append(contentsOf: u16(0) + u16(0) + u16(width) + u16(height) + s32(0)) + update.append(Data(repeating: value, count: frameBytes)) + } + XCTAssertGreaterThan(update.count, frameBytes + (1 << 20)) + XCTAssertLessThan(update.count, RFBClient.maxPendingBytes) + let split = 4 + 12 + frameBytes + XCTAssertEqual(try client.receive(Data(update.prefix(split))), []) + XCTAssertEqual(client.framebuffer.first, 0, "the first rectangle waits for the complete update") + XCTAssertEqual(try client.receive(Data(update.dropFirst(split))), [.updated(resized: false)]) + XCTAssertEqual(client.framebuffer, [UInt8](repeating: 9, count: frameBytes)) + } + func testBoundsPendingBytesAcrossMessagesBeforeAppending() throws { let client = RFBClient(password: nil) try client.receive(Data("R".utf8)) From 875dd249c77db82ee28cc59cc54b22b5e80ac25b Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:37:26 +0530 Subject: [PATCH 064/211] fix(acp): retain usage peaks and invalidate compacted failed prompts (cherry picked from commit c6ed45336d398eedf3e5996ab131fbf18ee6b3f8) --- server/drivers/acp/acp.test.ts | 33 +++++++++++++++++++++++++++-- server/drivers/acp/core.ts | 21 ++++++++++++------ server/drivers/prompt-split.test.ts | 13 ++++++++++++ server/drivers/prompt-split.ts | 5 ++++- server/testing/fake-acp-cli.ts | 25 +++++++++++++--------- 5 files changed, 77 insertions(+), 20 deletions(-) diff --git a/server/drivers/acp/acp.test.ts b/server/drivers/acp/acp.test.ts index 82be81ed4a..4b6a7ffe58 100644 --- a/server/drivers/acp/acp.test.ts +++ b/server/drivers/acp/acp.test.ts @@ -235,6 +235,7 @@ describe("ACP turns (fake CLI)", () => { delete process.env.FAKE_ACP_RPC_FAILURE_FILE; delete process.env.FAKE_ACP_RPC_FAILURE_METHOD; delete process.env.FAKE_ACP_RPC_FAILURE_AFTER_OUTPUT; + delete process.env.FAKE_ACP_RPC_FAILURE_AFTER_USAGE; delete process.env.FAKE_ACP_LOAD_ERROR; delete process.env.FAKE_ACP_ALLOW_ALWAYS; delete process.env.FAKE_ACP_PERMISSION_ANSWER; @@ -432,7 +433,7 @@ describe("ACP turns (fake CLI)", () => { await create(); const threadId = `t-acp-${name}-` + randomUUID(); const usage = (...used: number[]) => writeFileSync(usageFile, JSON.stringify(used)); - const send = async (text: string) => { + const send = async (text: string, ok = true) => { const { turnId } = await instance.adapter.sendTurn({ threadId, text, @@ -440,7 +441,7 @@ describe("ACP turns (fake CLI)", () => { systemStable: "Standing rules.", systemVolatile: "Memory: likes quiet hours.", }); - await recorder.until((event) => event.type === "turn.completed" && event.turnId === turnId); + expect(await recorder.until((event) => event.type === "turn.completed" && event.turnId === turnId)).toMatchObject({ ok }); return (JSON.parse(readFileSync(dump + ".prompt.json", "utf8")) as Array<{ type: string; text: string }>)[0]?.text; }; return { threadId, usage, send }; @@ -481,6 +482,34 @@ describe("ACP turns (fake CLI)", () => { expect(await send("turn 4")).toBe(FULL + "\n\nturn 4"); }); + it("detects a cumulative collapse against an earlier turn's high-water mark", async () => { + const { usage, send } = await usageThread("cumulative-collapse"); + usage(100000, 75000); + expect(await send("turn 1")).toBe(FULL + "\n\nturn 1"); + // Neither adjacent dip exceeds 40%, but the total fall from the peak does. + usage(50000); + expect(await send("turn 2")).toBe("turn 2"); + usage(); + expect(await send("turn 3")).toBe(FULL + "\n\nturn 3"); + expect(await send("turn 4")).toBe("turn 4"); + }); + + it.each([40000, 75000])("invalidates a rejected prompt's receipt only after compaction (usage: %s)", async (used) => { + const failureFile = join(scratch, "compaction-failure.json"); + process.env.FAKE_ACP_RPC_FAILURE_FILE = failureFile; + process.env.FAKE_ACP_RPC_FAILURE_AFTER_USAGE = "1"; + const { usage, send } = await usageThread("rejected-compaction"); + usage(100000); + expect(await send("turn 1")).toBe(FULL + "\n\nturn 1"); + usage(used); + writeFileSync(failureFile, JSON.stringify({ code: -32603, message: "Internal error after compaction" })); + expect(await send("turn 2", false)).toBe("turn 2"); + unlinkSync(failureFile); + usage(); + expect(await send("turn 3")).toBe(used < 60000 ? FULL + "\n\nturn 3" : "turn 3"); + expect(await send("turn 4")).toBe("turn 4"); + }); + it("ignores a zero usage report", async () => { const { usage, send } = await usageThread("zero-usage"); usage(150000); diff --git a/server/drivers/acp/core.ts b/server/drivers/acp/core.ts index c55f4a0b41..e4fdf0401f 100644 --- a/server/drivers/acp/core.ts +++ b/server/drivers/acp/core.ts @@ -1707,6 +1707,7 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver session.current = current; (async () => { + let pendingSplitReceipt: { key: string; receipt: PromptSplitReceipt; previous: PromptSplitReceipt | null } | null = null; try { // The handshake is paid once per process, not once per turn. It // is a function so the establishment retry below can pay it @@ -1982,13 +1983,12 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver // adapter call) keeps the legacy full-prompt shape. const halves = promptHalves(turn); let promptInput = promptTurn; - let pendingSplitReceipt: { key: string; receipt: PromptSplitReceipt; previous: PromptSplitReceipt | null } | null = null; if (halves.stable !== null) { const receiptKey = JSON.stringify([threadId, sessionId]); const previousReceipt = readPromptSplitReceipt(DRIVER_KIND, receiptKey); - // seed the peak from the last turn, so a compaction before this - // turn's first report still shows as a collapse - state.usagePeak = typeof previousReceipt?.lastUsed === "number" ? previousReceipt.lastUsed : null; + // Carry the high-water mark, not just the final report: several + // ordinary dips across turns can add up to a compaction. + state.usagePeak = previousReceipt?.peakUsed ?? previousReceipt?.lastUsed ?? null; const composed = splitSessionPrompt( halves.stable, halves.volatile, @@ -2048,9 +2048,11 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver writePromptSplitReceipt( DRIVER_KIND, pendingSplitReceipt.key, - lastUsed === undefined - ? pendingSplitReceipt.receipt - : { ...pendingSplitReceipt.receipt, lastUsed }, + { + ...pendingSplitReceipt.receipt, + ...(lastUsed === undefined ? {} : { lastUsed }), + ...(state.usagePeak === null ? {} : { peakUsed: state.usagePeak }), + }, ); } } @@ -2102,6 +2104,11 @@ export function createAcpDriver(support: AcpSupport): ProviderDriver settle(threadId, session, false, reason ?? "failed"); } } catch (e) { + // A rejected prompt can still have compacted native history. Its + // old receipt must not suppress the next turn's standing rules. + if (pendingSplitReceipt && state.promptSent && state.usageCompacted) { + deletePromptSplitReceipt(DRIVER_KIND, pendingSplitReceipt.key); + } if (!state.settled) { const message = e instanceof Error ? e.message : String(e); const code = support.classifyError?.(e); diff --git a/server/drivers/prompt-split.test.ts b/server/drivers/prompt-split.test.ts index 62c228cd56..e788c69b35 100644 --- a/server/drivers/prompt-split.test.ts +++ b/server/drivers/prompt-split.test.ts @@ -78,6 +78,19 @@ describe("prompt-split receipts", () => { } deletePromptSplitReceipt(scope, key); }); + + it("retains the context high-water mark separately from the final report", () => { + const scope = "test-driver"; + const key = randomUUID(); + const receipt = { ...promptSplitFingerprints("stable rules", "memory"), lastUsed: 75000, peakUsed: 100000 }; + writePromptSplitReceipt(scope, key, receipt); + expect(readPromptSplitReceipt(scope, key)).toEqual(receipt); + for (const invalid of [0, -4096, "100000", null, undefined]) { + writePromptSplitReceipt(scope, key, { ...receipt, peakUsed: invalid } as unknown as PromptSplitReceipt); + expect(readPromptSplitReceipt(scope, key)?.peakUsed).toBeUndefined(); + } + deletePromptSplitReceipt(scope, key); + }); }); describe("splitSessionPrompt", () => { diff --git a/server/drivers/prompt-split.ts b/server/drivers/prompt-split.ts index 4e6e8c5b5a..a9bcbe2e89 100644 --- a/server/drivers/prompt-split.ts +++ b/server/drivers/prompt-split.ts @@ -63,6 +63,8 @@ export interface PromptSplitReceipt { /** Last reported context size, for compaction detection; absent on * receipts written before it existed. */ lastUsed?: number; + /** Highest reported context size since the last detected compaction. */ + peakUsed?: number; } const digest = (value: string) => createHash("sha256").update(value).digest("hex"); @@ -84,13 +86,14 @@ export function readPromptSplitReceipt(scope: string, key: string): PromptSplitR try { const raw = JSON.parse(readFileSync(receiptPath(scope, key), "utf8")) as unknown; if (raw && typeof raw === "object" && !Array.isArray(raw)) { - const record = raw as { stable?: unknown; volatile?: unknown; turnsSinceFull?: unknown; lastUsed?: unknown }; + const record = raw as { stable?: unknown; volatile?: unknown; turnsSinceFull?: unknown; lastUsed?: unknown; peakUsed?: unknown }; if (typeof record.stable === "string" && typeof record.volatile === "string") { return { stable: record.stable, volatile: record.volatile, ...(typeof record.turnsSinceFull === "number" ? { turnsSinceFull: record.turnsSinceFull } : {}), ...(typeof record.lastUsed === "number" && record.lastUsed > 0 ? { lastUsed: record.lastUsed } : {}), + ...(typeof record.peakUsed === "number" && record.peakUsed > 0 ? { peakUsed: record.peakUsed } : {}), }; } } diff --git a/server/testing/fake-acp-cli.ts b/server/testing/fake-acp-cli.ts index 9225d78c9a..d45c1bb8c2 100755 --- a/server/testing/fake-acp-cli.ts +++ b/server/testing/fake-acp-cli.ts @@ -79,6 +79,7 @@ // FAKE_ACP_RPC_FAILURE_METHOD initialize, session/new or session/prompt (default). // FAKE_ACP_RPC_FAILURE_GATE hold the error until this file exists. // FAKE_ACP_RPC_FAILURE_AFTER_OUTPUT emit text + a tool result before failing. +// FAKE_ACP_RPC_FAILURE_AFTER_USAGE emit scripted usage updates before failing. // FAKE_ACP_LOAD_ERROR JSON-RPC error object returned by session/load. // FAKE_ACP_MODELS comma-separated model ids. Enables the opencode-shaped // surface: session/new and session/load return @@ -420,11 +421,24 @@ let agentsMcp: McpEntry | null = null; // the session this process established, for FAKE_ACP_REJECT_LIVE_LOAD_FILE let liveSession: string | null = null; let rpcFailure: unknown = null; +function emitUsageUpdates(): void { + const usageFile = process.env.FAKE_ACP_USAGE_UPDATES_FILE; + if (usageFile && existsSync(usageFile)) { + try { + for (const used of JSON.parse(readFileSync(usageFile, "utf8")) as unknown[]) { + if (typeof used === "number") { + out({ jsonrpc: "2.0", method: "session/update", params: { update: { sessionUpdate: "usage_update", used, size: 200000 } } }); + } + } + } catch {} + } +} function failRpc(msg: { method: string; id: unknown }): boolean { if (msg.method !== (process.env.FAKE_ACP_RPC_FAILURE_METHOD ?? "session/prompt")) return false; const failureFile = process.env.FAKE_ACP_RPC_FAILURE_FILE; if (failureFile && existsSync(failureFile)) rpcFailure = JSON.parse(readFileSync(failureFile, "utf8")); if (!rpcFailure) return false; + if (msg.method === "session/prompt" && process.env.FAKE_ACP_RPC_FAILURE_AFTER_USAGE === "1") emitUsageUpdates(); if (msg.method === "session/prompt" && process.env.FAKE_ACP_RPC_FAILURE_AFTER_OUTPUT === "1") playTurn(); const fail = () => { recordMethod(`${msg.method}.error`); @@ -805,16 +819,7 @@ function handle(msg: any) { return; } const complete = () => { - const usageFile = process.env.FAKE_ACP_USAGE_UPDATES_FILE; - if (usageFile && existsSync(usageFile)) { - try { - for (const used of JSON.parse(readFileSync(usageFile, "utf8")) as unknown[]) { - if (typeof used === "number") { - out({ jsonrpc: "2.0", method: "session/update", params: { update: { sessionUpdate: "usage_update", used, size: 200000 } } }); - } - } - } catch {} - } + emitUsageUpdates(); recordMethod("session/prompt.result"); result( msg.id, From 55c7912899c6162b733361a854af87f3b229a343 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:37:26 +0530 Subject: [PATCH 065/211] fix(claude): omit autocompact until CLI support is confirmed (cherry picked from commit 949059e3c633c112881d3b2e66f2df2916f2fce4) --- server/drivers/claude.test.ts | 10 +++++----- server/drivers/claude.ts | 7 ++++--- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/server/drivers/claude.test.ts b/server/drivers/claude.test.ts index ba9b10dc9e..38f1b55954 100644 --- a/server/drivers/claude.test.ts +++ b/server/drivers/claude.test.ts @@ -1262,6 +1262,7 @@ describe("ClaudeDriver turns (fake CLI)", () => { it("compacts the CLI session at a window the harness picks", async () => { await create(); + await instance.snapshot(); const dump = join(scratch, "compact.json"); process.env.FAKE_CLAUDE_DUMP = dump; @@ -1488,14 +1489,13 @@ describe("ClaudeDriver turns (fake CLI)", () => { expect((await instance.snapshot()).warning).toBeUndefined(); }); - it("assumes a current CLI on a turn that runs before any snapshot", async () => { - // no CLI start-up of its own: the flags are the default, and the next - // snapshot corrects an older install + it.each(["2.1.100", "2.1.129", "2.1.267"])("omits unconfirmed --autocompact before a snapshot on Claude %s", async (version) => { + // Version alone is insufficient: even some newer builds reject the flag. const dump = join(scratch, "unsnapshotted.json"); - await create(undefined, { FAKE_CLAUDE_DUMP: dump, FAKE_CLAUDE_VERSION: "2.1.100" }); + await create(undefined, { FAKE_CLAUDE_DUMP: dump, FAKE_CLAUDE_VERSION: version }); await instance.adapter.sendTurn({ threadId: "t-unsnapshotted", text: "hi" }); await recorder.until((e) => e.type === "turn.completed"); - expect(JSON.parse(readFileSync(dump, "utf8")).argv).toContain("--autocompact"); + expect(JSON.parse(readFileSync(dump, "utf8")).argv).not.toContain("--autocompact"); }); it("maps a CLI version onto the flags it accepts", () => { diff --git a/server/drivers/claude.ts b/server/drivers/claude.ts index b2cb3311ef..d46e88c23e 100644 --- a/server/drivers/claude.ts +++ b/server/drivers/claude.ts @@ -1177,8 +1177,9 @@ export const ClaudeDriver: ProviderDriver = { // harness snapshots every instance whenever it describes them — app // load, the Engines page, and right after `claude update`, which is // exactly when the answer changes — so a turn normally finds it filled. - // Most turns before any snapshot assume a current CLI. A coordinated - // turn checks first because the snapshot-refresh flag is newer than the + // Most flags before any snapshot assume a current CLI; autocompact + // requires confirmed help support. A coordinated turn checks first + // because the snapshot-refresh flag is newer than the // other context controls and an unknown flag would reject that request. let cliVersion: ClaudeCliVersion | null = null; let cliVersionChecked = false; @@ -1445,7 +1446,7 @@ export const ClaudeDriver: ProviderDriver = { if (claudeCliSupports(cliVersion, "--setting-sources")) args.push("--setting-sources", "project"); } const compactWindow = autoCompactWindow(turnEnvironment); - if (compactWindow && (cliHasAutocompact ?? claudeCliSupports(cliVersion, "--autocompact"))) { + if (compactWindow && cliHasAutocompact === true) { args.push("--autocompact", compactWindow); } // An old pair conversation can still carry its first assignment in From 948730192bbfe46d892faa7db5dbdee15aabe8f9 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:37:26 +0530 Subject: [PATCH 066/211] fix(cerebras): compact against the free-tier context window (cherry picked from commit cec19d2930561d1630344fd800548b3017a87027) --- server/drivers/cerebras.test.ts | 10 +++++++++- server/drivers/cerebras.ts | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/server/drivers/cerebras.test.ts b/server/drivers/cerebras.test.ts index 167d85eeda..f5a248467b 100644 --- a/server/drivers/cerebras.test.ts +++ b/server/drivers/cerebras.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { ASK_USER_TOOL_DEFINITION } from "../../shared/ask-question.ts"; import { recordEvents } from "../testing/events.ts"; +import { compactBudget, contextWindowFor, shouldCompact } from "../context-budget.ts"; import { CerebrasDriver } from "./cerebras.ts"; afterEach(() => { vi.unstubAllGlobals(); vi.unstubAllEnvs(); }); @@ -11,6 +12,13 @@ const create = (config = {}) => CerebrasDriver.create({ }); describe("Cerebras provider", () => { + it("compacts known models before the free-tier context limit", () => { + for (const { id } of CerebrasDriver.models.options) { + const window = contextWindowFor(id, CerebrasDriver.models); + expect(window).toBeLessThanOrEqual(65536); + expect(shouldCompact({ contextTokens: 60000, estimatedBytes: 0, budget: compactBudget(window), window })).toBe(true); + } + }); it("rejects invalid tools flags and non-TLS remote endpoints", () => { expect(() => CerebrasDriver.decodeConfig({ tools: "false" })).toThrow(); expect(() => CerebrasDriver.decodeConfig({ url: "http://example.com/v1" })).toThrow("HTTPS"); @@ -38,7 +46,7 @@ describe("Cerebras provider", () => { expect(instance.models.default).toBe("private-model"); expect(instance.models.options).toEqual([ { id: "private-model", label: "private-model" }, - { id: "gpt-oss-120b", label: "GPT OSS 120B", contextWindow: 131072 }, + { id: "gpt-oss-120b", label: "GPT OSS 120B", contextWindow: 65536 }, { id: "brand-new-model", label: "brand-new-model" }, ]); expect(fetcher.mock.calls[0]).toMatchObject(["https://api.cerebras.ai/v1/models", { diff --git a/server/drivers/cerebras.ts b/server/drivers/cerebras.ts index bd1ace7dc9..f209f79ee4 100644 --- a/server/drivers/cerebras.ts +++ b/server/drivers/cerebras.ts @@ -5,11 +5,13 @@ import { createOpenAIChatRuntime } from "./openai-chat.ts"; // Cerebras serves open models on wafer-scale chips: same OpenAI chat // contract, several times the tokens per second of a GPU cloud. const DEFAULT_URL = "https://api.cerebras.ai/v1"; +// The account tier is unknown: use the shared free-tier limit, not paid 128K. +// https://inference-docs.cerebras.ai/models/overview const DEFAULT_MODELS: ModelCatalog = { default: "gpt-oss-120b", options: [ - { id: "gpt-oss-120b", label: "GPT OSS 120B", contextWindow: 131072 }, - { id: "qwen-3.8-27b", label: "Qwen3.8 27B", contextWindow: 131072 }, + { id: "gpt-oss-120b", label: "GPT OSS 120B", contextWindow: 65536 }, + { id: "qwen-3.8-27b", label: "Qwen3.8 27B", contextWindow: 65536 }, ], }; const configSchema = z.object({ From 4438f8f206a82d47d18b8c858006a38a3ca6aef7 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 13:39:39 +0530 Subject: [PATCH 067/211] fix(ui): guard search toggles and contain long formulas (cherry picked from commit f297685b88b675eb078ef29e72ba3e0709b91a59) --- src/components/ChatMarkdown.test.ts | 12 ++++++++++++ src/components/Sidebar.tsx | 4 ++-- src/components/SidebarAttentionPanel.test.ts | 8 ++++++++ src/components/SidebarAttentionPanel.tsx | 3 ++- src/components/SidebarBotActivity.test.ts | 10 ++++++++++ src/components/SidebarPinnedThreadsPanel.test.ts | 8 ++++++++ src/components/SidebarPinnedThreadsPanel.tsx | 3 ++- src/styles.css | 12 +++++++----- 8 files changed, 51 insertions(+), 9 deletions(-) diff --git a/src/components/ChatMarkdown.test.ts b/src/components/ChatMarkdown.test.ts index 53d649554d..1e1a3133bd 100644 --- a/src/components/ChatMarkdown.test.ts +++ b/src/components/ChatMarkdown.test.ts @@ -1,3 +1,4 @@ +import { readFileSync } from "node:fs"; import { createElement } from "react"; import * as React from "react"; import { renderToStaticMarkup } from "react-dom/server"; @@ -147,6 +148,17 @@ describe("math rendering", () => { expect(html).toContain("120 and US$5."); }); + it("contains long inline formulas in a horizontal scroll container", () => { + const html = renderToStaticMarkup(createElement(ChatMarkdown, { + text: `Inline $${"abcdefghijklmnopqrstuvwxyz".repeat(3)}$.`, + })); + expect(html.match(/class="katex"/g)).toHaveLength(1); + const css = readFileSync(new URL("../styles.css", import.meta.url), "utf8"); + const rule = css.match(/\.chat-md :not\(\.katex-display\) > \.katex \{([^}]*)\}/)?.[1]; + expect(rule).toContain("max-width: 100%"); + expect(rule).toContain("overflow-x: auto"); + }); + it("does not pair dollars across paragraphs", () => { const html = renderToStaticMarkup(createElement(ChatMarkdown, { text: "Costs $5.\n\nThen pay later$" })); expect(html).not.toContain('class="katex"'); diff --git a/src/components/Sidebar.tsx b/src/components/Sidebar.tsx index bbeb5cb11a..5c8e7be9e2 100644 --- a/src/components/Sidebar.tsx +++ b/src/components/Sidebar.tsx @@ -2209,7 +2209,7 @@ export function Sidebar({ open, onClose, collapseToIcons = false }: { onUnpin={() => setAttentionPinned(false)} onJump={(entry) => dispatch(attentionJumpAction(entry))} collapsed={sectionCollapsed(ATTENTION_SECTION_ID)} - onToggle={() => toggleSection(ATTENTION_SECTION_ID)} + onToggle={layoutInteractive ? () => toggleSection(ATTENTION_SECTION_ID) : undefined} /> )} @@ -2221,7 +2221,7 @@ export function Sidebar({ open, onClose, collapseToIcons = false }: { onJump={(entry) => dispatch(attentionJumpAction(entry))} onUnpin={(entry) => dispatch(attentionUnpinAction(entry))} collapsed={sectionCollapsed(PINNED_THREADS_SECTION_ID)} - onToggle={() => toggleSection(PINNED_THREADS_SECTION_ID)} + onToggle={layoutInteractive ? () => toggleSection(PINNED_THREADS_SECTION_ID) : undefined} /> )} diff --git a/src/components/SidebarAttentionPanel.test.ts b/src/components/SidebarAttentionPanel.test.ts index 5f7a341ea4..2960ca2a05 100644 --- a/src/components/SidebarAttentionPanel.test.ts +++ b/src/components/SidebarAttentionPanel.test.ts @@ -104,4 +104,12 @@ describe("pinned attention panel", () => { findElement(tree(), "aria-label", label)!.props.onClick!({} as MouseEvent); expect(onToggle).toHaveBeenCalledOnce(); }); + + it("disables collapse when search prevents layout changes", () => { + const markup = renderToStaticMarkup(createElement(SidebarAttentionPanel, { + entries: [entry], density: "comfortable", onUnpin: vi.fn(), onJump: vi.fn(), collapsed: false, + })); + expect(markup).toMatch(/]+disabled=""[^>]+aria-expanded="true"/); + expect(markup).toContain("Review permission"); + }); }); diff --git a/src/components/SidebarAttentionPanel.tsx b/src/components/SidebarAttentionPanel.tsx index 9f16ff42ca..1f2facfa0c 100644 --- a/src/components/SidebarAttentionPanel.tsx +++ b/src/components/SidebarAttentionPanel.tsx @@ -12,7 +12,7 @@ export function SidebarAttentionPanel({ entries, density, onUnpin, onJump, colla onUnpin: () => void; onJump: (entry: AttentionThread) => void; collapsed: boolean; - onToggle: () => void; + onToggle?: () => void; }) { const compact = density === "compact"; const Chevron = collapsed ? ChevronRight : ChevronDown; @@ -26,6 +26,7 @@ export function SidebarAttentionPanel({ entries, density, onUnpin, onJump, colla
} {signed &&
{account.account &&

{account.account.email}

} - {account.status === "connected" &&

{activePro ? t("cloudAccount.pro") : t("cloudAccount.free")}

} + {account.status === "connected" &&

{activePlan ? t("cloudAccount.pro", { plan: cloudPlanLabel(account.entitlement?.tier) }) : t("cloudAccount.free")}

}

{t("cloudAccount.purchaseHelp")}

- + {!confirm && }
diff --git a/src/components/DecisionModelSettings.test.ts b/src/components/DecisionModelSettings.test.ts index 506d08b563..8f9292a28c 100644 --- a/src/components/DecisionModelSettings.test.ts +++ b/src/components/DecisionModelSettings.test.ts @@ -133,7 +133,7 @@ describe("DecisionModelSettings", () => { it("shows Cloud Pro's included decisions as included, on, with nothing to clear", () => { fixture.config = status({ provider: "jev", configured: true, included: true, enabled: true, jobs: { roomRouting: true } }); const view = render(); - expect(view.html).toContain("Included with Cloud Pro"); + expect(view.html).toContain("Included with your Cloud plan"); expect(view.html).not.toContain("Connected"); expect(view.html).not.toContain("Not connected"); expect(view.html).not.toContain("Save a key below to turn this on."); @@ -152,7 +152,7 @@ describe("DecisionModelSettings", () => { fixture.config = status({ provider: "jev", configured: true, enabled: true, jobs: { roomRouting: true } }); const view = render(); expect(view.html).toContain("Connected"); - expect(view.html).not.toContain("Included with Cloud Pro"); + expect(view.html).not.toContain("Included with your Cloud plan"); expect(view.html).toContain("Clear"); expect(view.save.props.disabled).toBe(false); }); @@ -184,9 +184,9 @@ describe("DecisionModelSettings", () => { expect(fixture.api).toHaveBeenLastCalledWith("/api/decider/test", { method: "POST", body: "{}" }); expect(render().html).toContain("Jev answered in 210 ms."); for (const [result, text] of [ - [{ ok: false, reason: "http_error", status: 402 }, "Decisions are included with an active Cloud Pro subscription."], - [{ ok: false, reason: "rate_limited", status: 429 }, "Cloud Pro decisions are busy or used up for this month. Try again later."], - [{ ok: false, reason: "rejected", status: 401 }, "Cloud Pro did not accept this machine's decisions. Try again later."], + [{ ok: false, reason: "http_error", status: 402 }, "Decisions are included with an active Cloud subscription."], + [{ ok: false, reason: "rate_limited", status: 429 }, "Your Cloud plan's decisions are busy or used up for this month. Try again later."], + [{ ok: false, reason: "rejected", status: 401 }, "OMB Cloud did not accept this machine's decisions. Try again later."], ] as const) { fixture.api.mockResolvedValueOnce(result); click(render().test); diff --git a/src/components/ServerPairingCard.test.ts b/src/components/ServerPairingCard.test.ts index 3e21237eb3..e59c93c248 100644 --- a/src/components/ServerPairingCard.test.ts +++ b/src/components/ServerPairingCard.test.ts @@ -62,7 +62,7 @@ describe("pairing devices from a hosted server's settings", () => { expect(cloud).toContain("Create pairing code"); expect(cloud).not.toContain("Chat and approvals only"); expect(cloud).toContain("data-server-pairing-personal"); - expect(cloud).toContain("Cloud Pro is personal: only your own devices can connect"); + expect(cloud).toContain("OMB Cloud is personal: only your own devices can connect"); const elsewhere = renderToStaticMarkup(createElement(ServerPairingCard, { initialSession: admin })); expect(elsewhere).toContain("Chat and approvals only"); expect(elsewhere).not.toContain("data-server-pairing-personal"); diff --git a/src/locales/en.json b/src/locales/en.json index bc0e26ac66..6af2f251bc 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -216,17 +216,17 @@ "cloudAccount.loading": "Loading Cloud account…", "cloudAccount.signIn": "Sign in to OMB Cloud", "cloudAccount.browser": "Finish email-code sign-in and approve this computer in your browser. The app will connect automatically.", - "cloudAccount.pro": "Pro active · verified by OMB Cloud", + "cloudAccount.pro": "{plan} active · verified by OMB Cloud", "cloudAccount.free": "Free account", - "cloudAccount.upgrade": "Get Pro in your browser", + "cloudAccount.upgrade": "Choose a Cloud plan in your browser", "cloudAccount.manage": "Manage Cloud subscription", - "cloudAccount.purchaseHelp": "Complete billing in your browser using this account. After purchase, Refresh checks activation with OMB Cloud; a checkout return alone never activates Pro.", + "cloudAccount.purchaseHelp": "Complete billing in your browser using this account. After purchase, Refresh checks activation with OMB Cloud; a checkout return alone never activates a plan.", "cloudAccount.signOut": "Sign out of OMB Cloud", "cloudAccount.signoutTitle": "Sign out on this computer?", "cloudAccount.signoutHelp": "This removes personal Cloud access from this computer. It does not cancel your subscription, disconnect your organization, or change local chats and model accounts.", "cloudAccount.keep": "Keep signed in", "cloudAccount.reauth": "Cloud access expired or was revoked. Sign out below, then sign in again. Free local use is unchanged.", - "cloudAccount.unavailable": "Cloud status cannot currently be verified. Pro is unavailable until verification succeeds; free local use is unchanged.", + "cloudAccount.unavailable": "Cloud status cannot currently be verified. Your Cloud plan is unavailable until verification succeeds; free local use is unchanged.", "cloudAccount.signoutLocalOnly": "Signed out on this computer. Cloud could not confirm remote revocation; revoke this computer in your Cloud account too.", "cloudAccount.storageFailed": "The saved Cloud sign-in could not be read or cleared. Unlock your system keychain, then sign out again before reconnecting.", "cloudAccount.signinFailed": "Cloud sign-in ended or could not be completed. Start again on this computer.", @@ -1509,7 +1509,7 @@ "keys.helpAria": "{label} help", "keys.optional": "Optional", "keys.configured": "Configured", - "keys.includedWithCloudPro": "Included with Cloud Pro", + "keys.includedWithCloudPro": "Included with your Cloud plan", "keys.testAuthenticated": "API key authenticated. Chat not tested.", "keys.testCatalog": "Model catalog reachable: {models}. Authentication and chat not verified.", "keys.testCatalogNoModels": "Model catalog reachable. Authentication and chat not verified.", @@ -1566,9 +1566,9 @@ "decider.key.getKey": "Get a key at typesafe.ai", "decider.status.connected": "Connected", "decider.status.notConnected": "Not connected", - "decider.included.error.rejected": "Cloud Pro did not accept this machine's decisions. Try again later.", - "decider.included.error.subscription": "Decisions are included with an active Cloud Pro subscription.", - "decider.included.error.limited": "Cloud Pro decisions are busy or used up for this month. Try again later.", + "decider.included.error.rejected": "OMB Cloud did not accept this machine's decisions. Try again later.", + "decider.included.error.subscription": "Decisions are included with an active Cloud subscription.", + "decider.included.error.limited": "Your Cloud plan's decisions are busy or used up for this month. Try again later.", "decider.test.ok": "Jev answered in {ms} ms.", "decider.error.rejected": "Jev rejected this key. Check it at typesafe.ai.", "decider.error.unreachable": "Could not reach Jev. Check your connection.", @@ -2762,7 +2762,7 @@ "remote.serverPairing.copied": "Copied", "remote.serverPairing.noLink": "This server has no public address for a link yet: open /pair on the address you use and type the code.", "remote.serverPairing.chatOnly": "This device is connected with chat and approvals only, so it cannot pair others. On the computer running the server, run `openmausbot pair` (or open Settings there) to make a full-access code, and connect this device again with it.", - "remote.serverPairing.cloudPersonal": "Cloud Pro is personal: only your own devices can connect, each with full access.", + "remote.serverPairing.cloudPersonal": "OMB Cloud is personal: only your own devices can connect, each with full access.", "remote.serverPairing.devices": "Paired devices", "remote.serverPairing.noDevices": "No devices paired yet.", "remote.serverPairing.thisBrowser": "this browser", From 5a74d9237b0dc5f423a8982af0082ef463214303 Mon Sep 17 00:00:00 2001 From: milind-soni Date: Fri, 2 Oct 2026 14:48:24 +0530 Subject: [PATCH 070/211] Pro card: launch price without a struck-through former price MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Pro introduction card showed "$89 $49/month" with an aria-label reading "Was $89, now $49 a month". $89 was only live for about a day, so presenting it as a former price is a fictitious reference price under FTC 16 CFR 233.1 and the EU 30-day prior-price rule. The card now says, as plain text: "$49/month — launch price for the first 100 users, then $89/month". No , no line-through, no "was" label, and the role="img" wrapper that only carried that label is gone. - PRO_REGULAR_PRICE is renamed PRO_LATER_PRICE; prices are unchanged. - pro.launchPrice gains a {laterPrice} placeholder; pro.launchPriceLabel is removed (it was the only "was" string). - PRO_DISMISSED is unchanged, so nobody who dismissed the card sees it again; the test now pins it. - Every locale gets the Pro card strings translated (none had any pro.* keys before, so translating only the price line would have left one translated sentence inside an English card). source-hashes.json updated with `generate-locale.mjs --accept`. No other struck-through or "was" price exists in src/ or electron/. Co-Authored-By: Claude Opus 5.5 --- src/components/ProIntroduction.test.ts | 9 ++- src/components/ProIntroduction.tsx | 12 ++- src/locales/de.json | 12 +++ src/locales/en.json | 3 +- src/locales/es.json | 12 +++ src/locales/fr.json | 12 +++ src/locales/hi.json | 12 +++ src/locales/ja.json | 12 +++ src/locales/pt-br.json | 12 +++ src/locales/source-hashes.json | 108 +++++++++++++++++++++++++ src/locales/uk.json | 12 +++ src/locales/zh-tw.json | 12 +++ src/locales/zh.json | 12 +++ 13 files changed, 229 insertions(+), 11 deletions(-) diff --git a/src/components/ProIntroduction.test.ts b/src/components/ProIntroduction.test.ts index b7418e4e03..c3aea45a97 100644 --- a/src/components/ProIntroduction.test.ts +++ b/src/components/ProIntroduction.test.ts @@ -42,12 +42,17 @@ it("shows the live benefits without enrolling, charging or refreshing an account expect(html).not.toContain("Coming soon"); expect(api).not.toHaveBeenCalled(); expect(html).not.toContain('aria-modal="true"'); }); -it("shows the $49 launch price beside the struck-through $89", () => { +it("states the $49 launch price and the later $89 as plain text, never as a former price", () => { const html = render(); - expect(html).toContain('$89 $49/month: launch price for the first 100 users'); + expect(html).toContain('

$49/month — launch price for the first 100 users, then $89/month

'); + // No struck-through or "was" reference price (FTC 16 CFR 233.1, EU 30-day prior-price rule). + for (const markup of ["", " { storage.set(OLD_DISMISSED, "1"); diff --git a/src/components/ProIntroduction.tsx b/src/components/ProIntroduction.tsx index f289ed701a..d53f5263d4 100644 --- a/src/components/ProIntroduction.tsx +++ b/src/components/ProIntroduction.tsx @@ -14,9 +14,11 @@ import { t } from "@/lib/i18n"; // more. After that, updates and replaying the welcome tour never reset it. export const PRO_DISMISSED = "pro-introduction-dismissed-v2"; -/** The launch price, and the regular price shown struck through beside it. */ +/** The launch price, and the price after the launch offer. Both are plain + * text: never a struck-through or "was" former price (FTC 16 CFR 233.1 and + * the EU 30-day prior-price rule). */ export const PRO_LAUNCH_PRICE = "$49"; -export const PRO_REGULAR_PRICE = "$89"; +export const PRO_LATER_PRICE = "$89"; export function proOfferAvailable(account: CloudAccountState | null): boolean { return account?.status === "signed-out" || (account?.status === "connected" && account.entitlement?.plan === "free"); @@ -81,11 +83,7 @@ export function ProIntroductionCard({ onDismiss }: { onDismiss: () => void }) {
    • -

    - - {PRO_REGULAR_PRICE} {t("pro.launchPrice", { price: PRO_LAUNCH_PRICE })} - -

    +

    {t("pro.launchPrice", { price: PRO_LAUNCH_PRICE, laterPrice: PRO_LATER_PRICE })}

    diff --git a/src/locales/de.json b/src/locales/de.json index 16e147179a..8ab4ef5f6f 100644 --- a/src/locales/de.json +++ b/src/locales/de.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Pro holen", + "pro.headline": "Lass deine Bots in der Cloud weiterlaufen.", + "pro.priority": "Neue Funktionen zuerst und bevorzugter Support", + "pro.alwaysOn": "Deine Bots laufen rund um die Uhr in der Cloud", + "pro.computers": "Cloud-Computer und Sprachfunktionen inklusive", + "pro.schedule": "Geplante Aufgaben in der Cloud", + "pro.launchPrice": "{price}/Monat — Einführungspreis für die ersten 100 Nutzer, danach {laterPrice}/Monat", + "pro.settingsSummary": "Neue Funktionen zuerst, bevorzugter Support und Cloud-Bots rund um die Uhr.", + "pro.dismiss": "Pro-Vorstellung dauerhaft ausblenden", + "pro.noThanks": "Nicht mehr anzeigen", + "pro.disclaimer": "Lokal bleibt kostenlos. Modellnutzung ist nicht inbegriffen.", + "pro.openFailed": "Dein Browser konnte nicht geöffnet werden. Bitte versuche es erneut.", "noEngines.title": "Installiere einen Modellanbieter, um loszulegen", "noEngines.intro": "{app} bringt kein eigenes Modell mit — deine Bots laufen über eine auf diesem Rechner installierte KI-CLI mit deinem bestehenden Login. Richte eine davon ein und deine Bots erwachen zum Leben.", "engines.cloud": "Cloud", diff --git a/src/locales/en.json b/src/locales/en.json index 6af2f251bc..4ecd80fb3c 100644 --- a/src/locales/en.json +++ b/src/locales/en.json @@ -6,8 +6,7 @@ "pro.alwaysOn": "Your bots stay always-on in the cloud", "pro.computers": "Cloud computers and voice included", "pro.schedule": "Cloud scheduled tasks", - "pro.launchPrice": "{price}/month: launch price for the first 100 users", - "pro.launchPriceLabel": "Was {was}, now {price} a month: launch price for the first 100 users", + "pro.launchPrice": "{price}/month — launch price for the first 100 users, then {laterPrice}/month", "pro.settingsSummary": "New features first, priority support, and always-on cloud bots.", "pro.dismiss": "Dismiss Pro introduction forever", "pro.noThanks": "Don’t show again", diff --git a/src/locales/es.json b/src/locales/es.json index 5da1112ef9..b91d255b78 100644 --- a/src/locales/es.json +++ b/src/locales/es.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Consigue Pro", + "pro.headline": "Mantén tus bots funcionando en la nube.", + "pro.priority": "Novedades antes que nadie y soporte prioritario", + "pro.alwaysOn": "Tus bots siempre activos en la nube", + "pro.computers": "Equipos en la nube y voz incluidos", + "pro.schedule": "Tareas programadas en la nube", + "pro.launchPrice": "{price}/mes — precio de lanzamiento para los primeros 100 usuarios; después, {laterPrice}/mes", + "pro.settingsSummary": "Novedades antes que nadie, soporte prioritario y bots siempre activos en la nube.", + "pro.dismiss": "Descartar para siempre la presentación de Pro", + "pro.noThanks": "No volver a mostrar", + "pro.disclaimer": "Lo local sigue siendo gratis. El uso de modelos no está incluido.", + "pro.openFailed": "No se pudo abrir tu navegador. Inténtalo de nuevo.", "noEngines.title": "Instala un proveedor de modelos para empezar", "noEngines.intro": "{app} no incluye un modelo propio: tus bots funcionan con una CLI de IA instalada en este equipo, usando tu sesión existente. Configura cualquiera de ellas y tus bots cobrarán vida.", "engines.cloud": "Nube", diff --git a/src/locales/fr.json b/src/locales/fr.json index 524631158d..f2b4efd57e 100644 --- a/src/locales/fr.json +++ b/src/locales/fr.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Passer à Pro", + "pro.headline": "Gardez vos bots actifs dans le cloud.", + "pro.priority": "Nouveautés en avant-première et support prioritaire", + "pro.alwaysOn": "Vos bots restent actifs en permanence dans le cloud", + "pro.computers": "Ordinateurs cloud et voix inclus", + "pro.schedule": "Tâches planifiées dans le cloud", + "pro.launchPrice": "{price}/mois — prix de lancement pour les 100 premiers utilisateurs, puis {laterPrice}/mois", + "pro.settingsSummary": "Nouveautés en avant-première, support prioritaire et bots actifs en permanence dans le cloud.", + "pro.dismiss": "Masquer définitivement la présentation de Pro", + "pro.noThanks": "Ne plus afficher", + "pro.disclaimer": "Le local reste gratuit. L'utilisation des modèles n'est pas incluse.", + "pro.openFailed": "Impossible d'ouvrir votre navigateur. Veuillez réessayer.", "noEngines.title": "Installez un fournisseur de modèles pour commencer", "noEngines.intro": "{app} n'embarque pas de modèle : vos bots s'appuient sur une CLI d'IA installée sur cet ordinateur, avec votre session existante. Configurez l'une d'elles et vos bots prennent vie.", "engines.cloud": "Cloud", diff --git a/src/locales/hi.json b/src/locales/hi.json index 9c79253484..0652d68171 100644 --- a/src/locales/hi.json +++ b/src/locales/hi.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Pro पाएँ", + "pro.headline": "अपने बॉट क्लाउड में लगातार चलते रखें।", + "pro.priority": "नई सुविधाएँ सबसे पहले और प्राथमिकता सहायता", + "pro.alwaysOn": "आपके बॉट क्लाउड में हमेशा चालू रहते हैं", + "pro.computers": "क्लाउड कंप्यूटर और आवाज़ सुविधा शामिल", + "pro.schedule": "क्लाउड में शेड्यूल किए गए काम", + "pro.launchPrice": "{price}/माह — पहले 100 उपयोगकर्ताओं के लिए लॉन्च कीमत, उसके बाद {laterPrice}/माह", + "pro.settingsSummary": "नई सुविधाएँ सबसे पहले, प्राथमिकता सहायता, और क्लाउड में हमेशा चालू बॉट।", + "pro.dismiss": "Pro परिचय हमेशा के लिए हटाएँ", + "pro.noThanks": "फिर से न दिखाएँ", + "pro.disclaimer": "लोकल मुफ़्त रहेगा। मॉडल का उपयोग इसमें शामिल नहीं है।", + "pro.openFailed": "आपका ब्राउज़र नहीं खुल सका। कृपया फिर से कोशिश करें।", "noEngines.title": "शुरू करने के लिए एक मॉडल प्रदाता इंस्टॉल करें", "noEngines.intro": "{app} अपना कोई मॉडल नहीं देता — आपके बॉट इस कंप्यूटर पर इंस्टॉल की गई AI CLI पर, आपके मौजूदा लॉगिन से चलते हैं। इनमें से कोई भी सेट कर लें, और आपके बॉट चल पड़ेंगे।", "engines.cloud": "क्लाउड", diff --git a/src/locales/ja.json b/src/locales/ja.json index f86e246080..ca1f170e29 100644 --- a/src/locales/ja.json +++ b/src/locales/ja.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Pro を入手", + "pro.headline": "ボットをクラウドで動かし続けましょう。", + "pro.priority": "新機能をいち早く、優先サポート付き", + "pro.alwaysOn": "ボットがクラウドで常時稼働", + "pro.computers": "クラウドコンピューターと音声機能を含む", + "pro.schedule": "クラウドでの予約タスク", + "pro.launchPrice": "月額 {price} — 先着 100 ユーザー向けのローンチ価格、以降は月額 {laterPrice}", + "pro.settingsSummary": "新機能をいち早く、優先サポート、クラウドで常時稼働するボット。", + "pro.dismiss": "Pro の紹介を閉じて今後表示しない", + "pro.noThanks": "今後表示しない", + "pro.disclaimer": "ローカルは引き続き無料です。モデルの利用料は含まれません。", + "pro.openFailed": "ブラウザーを開けませんでした。もう一度お試しください。", "noEngines.title": "モデルプロバイダーをインストールして始めましょう", "noEngines.intro": "{app} 自体はモデルを持ちません。ボットはこのコンピューターにインストール済みの AI CLI 上で、既存のログインを使って動きます。どれか一つをセットアップすれば、ボットが動き出します。", "engines.cloud": "クラウド", diff --git a/src/locales/pt-br.json b/src/locales/pt-br.json index 535f4aceeb..21a84830da 100644 --- a/src/locales/pt-br.json +++ b/src/locales/pt-br.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Obter o Pro", + "pro.headline": "Mantenha seus bots rodando na nuvem.", + "pro.priority": "Novos recursos primeiro e suporte prioritário", + "pro.alwaysOn": "Seus bots ficam sempre ativos na nuvem", + "pro.computers": "Computadores na nuvem e voz incluídos", + "pro.schedule": "Tarefas agendadas na nuvem", + "pro.launchPrice": "{price}/mês — preço de lançamento para os primeiros 100 usuários, depois {laterPrice}/mês", + "pro.settingsSummary": "Novos recursos primeiro, suporte prioritário e bots sempre ativos na nuvem.", + "pro.dismiss": "Dispensar a apresentação do Pro para sempre", + "pro.noThanks": "Não mostrar novamente", + "pro.disclaimer": "O modo local continua gratuito. O uso de modelos não está incluído.", + "pro.openFailed": "Não foi possível abrir seu navegador. Tente novamente.", "noEngines.title": "Instale um provedor de modelos para começar", "noEngines.intro": "O {app} não traz um modelo próprio — seus bots rodam em uma CLI de IA instalada neste computador, usando seu login existente. Configure qualquer uma delas e seus bots ganham vida.", "engines.cloud": "Nuvem", diff --git a/src/locales/source-hashes.json b/src/locales/source-hashes.json index c3fe883500..97feb90cbe 100644 --- a/src/locales/source-hashes.json +++ b/src/locales/source-hashes.json @@ -2,6 +2,18 @@ "version": 1, "locales": { "de": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -1422,6 +1434,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "es": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -2824,6 +2848,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "fr": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -4226,6 +4262,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "hi": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -5628,6 +5676,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "ja": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -7032,6 +7092,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "pt-br": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -8448,6 +8520,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "zh": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -9850,6 +9934,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "zh-tw": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", @@ -11768,6 +11864,18 @@ "computer.addBoatKey": "fcf349af3c05d6750fdc7ebdaac778bfcf27f8514faf5084c3882479c084cc36" }, "uk": { + "pro.getPro": "12657108fe5682e937f179e5f394191b99fad614dbb3bf8759376d5da35d4992", + "pro.headline": "f62e3e2a9a16c7090dec92c08e7f043d33aa9b5d6edd6895d70d34309a4cb99d", + "pro.priority": "6f4f2f892e0dc4dbc52356c0347527679758e98fba95e99b6d767ab5a0753cea", + "pro.alwaysOn": "36f4ee1946d7663331ac5818cf5fe119fec42960069ed6915fbc5c18d0e3d18d", + "pro.computers": "3212d5d19a34109981874a7f10f5643736011acfab0d0066702df6c6306e4eea", + "pro.schedule": "5ef1b4cacf7a607879d8d5ec91631ea7ecefec9163ddd18e65e863a8fbf33608", + "pro.launchPrice": "35223738d1452896d0d672413f4fb42522525eeac29c6512f8dbd1655456961e", + "pro.settingsSummary": "90e4d72b5a95773bd90a419e833c1a7eb08b656b3ca61fc679a8c9de37acae8c", + "pro.dismiss": "f6b7875491d933fec536ea9e5f0717e85e884f66e540b5d089237e4610e3b215", + "pro.noThanks": "f6f62e01a13bcae93b71dafdb9eba6310bfec86a9f907f6ea71d1a0fdea9f47c", + "pro.disclaimer": "a36901f7339a4e086aee9dcdc3e6b02d8eb1acaee5161054419841a742f1dce3", + "pro.openFailed": "6051a5f580f72d5d1483489665cab2cfa54334223b3df6ce8f5896c6f232b608", "noEngines.title": "77a6e0966b7323ceae6973abb581b31d52263bb0ec8249e2b1c1b7ae1e586f04", "noEngines.intro": "771dc420a107df9c9df61b2ee02d2bd30860060d482e9b7918e67082a28744fe", "engines.cloud": "b977b950c1ae31e5aeb9ef778cc20a66fc034eb81e738e0206104b677962c465", diff --git a/src/locales/uk.json b/src/locales/uk.json index eb7f112f8e..801e4f3ce4 100644 --- a/src/locales/uk.json +++ b/src/locales/uk.json @@ -1,4 +1,16 @@ { + "pro.getPro": "Отримати Pro", + "pro.headline": "Нехай ваші боти працюють у хмарі без перерви.", + "pro.priority": "Нові функції першими та пріоритетна підтримка", + "pro.alwaysOn": "Ваші боти завжди увімкнені в хмарі", + "pro.computers": "Хмарні комп'ютери та голос у комплекті", + "pro.schedule": "Заплановані завдання в хмарі", + "pro.launchPrice": "{price}/міс. — стартова ціна для перших 100 користувачів, далі {laterPrice}/міс.", + "pro.settingsSummary": "Нові функції першими, пріоритетна підтримка та боти, що завжди працюють у хмарі.", + "pro.dismiss": "Назавжди приховати знайомство з Pro", + "pro.noThanks": "Більше не показувати", + "pro.disclaimer": "Локальна робота залишається безкоштовною. Використання моделей не входить у вартість.", + "pro.openFailed": "Не вдалося відкрити браузер. Спробуйте ще раз.", "noEngines.title": "Встановіть провайдера моделей, щоб почати", "noEngines.intro": "{app} не має власної моделі — ваші боти працюють на AI-CLI, встановленому на цьому комп'ютері, з вашим наявним логіном. Налаштуйте будь-який із перелічених — і боти оживуть.", "engines.cloud": "Хмара", diff --git a/src/locales/zh-tw.json b/src/locales/zh-tw.json index cdf5cf1073..470a2c0c74 100644 --- a/src/locales/zh-tw.json +++ b/src/locales/zh-tw.json @@ -1,4 +1,16 @@ { + "pro.getPro": "取得 Pro", + "pro.headline": "讓你的機器人在雲端持續運作。", + "pro.priority": "搶先使用新功能，並享有優先支援", + "pro.alwaysOn": "你的機器人在雲端隨時在線", + "pro.computers": "包含雲端電腦與語音", + "pro.schedule": "雲端排程工作", + "pro.launchPrice": "{price}/月——前 100 位使用者的首發價，之後為 {laterPrice}/月", + "pro.settingsSummary": "搶先使用新功能、優先支援，以及在雲端隨時在線的機器人。", + "pro.dismiss": "永久關閉 Pro 介紹", + "pro.noThanks": "不再顯示", + "pro.disclaimer": "本機使用仍然免費。模型用量不包含在內。", + "pro.openFailed": "無法開啟瀏覽器，請再試一次。", "noEngines.title": "安裝模型供應商以開始使用", "noEngines.intro": "{app} 本身不隨附任何模型——你的機器人是透過安裝在這台電腦上的 AI CLI 執行，並使用你既有的登入資訊。只要設定好其中任何一項，你的機器人就能開始運作。", "engines.cloud": "雲端", diff --git a/src/locales/zh.json b/src/locales/zh.json index d8cd622f36..4f4c225e8e 100644 --- a/src/locales/zh.json +++ b/src/locales/zh.json @@ -1,4 +1,16 @@ { + "pro.getPro": "获取 Pro", + "pro.headline": "让你的机器人在云端持续运行。", + "pro.priority": "抢先体验新功能，享受优先支持", + "pro.alwaysOn": "你的机器人在云端始终在线", + "pro.computers": "包含云端电脑和语音", + "pro.schedule": "云端定时任务", + "pro.launchPrice": "{price}/月——前 100 位用户的首发价，之后为 {laterPrice}/月", + "pro.settingsSummary": "抢先体验新功能、优先支持，以及在云端始终在线的机器人。", + "pro.dismiss": "永久关闭 Pro 介绍", + "pro.noThanks": "不再显示", + "pro.disclaimer": "本地使用依然免费。模型用量不包含在内。", + "pro.openFailed": "无法打开浏览器，请重试。", "noEngines.title": "安装一个模型提供方，开始使用", "noEngines.intro": "{app} 本身不自带模型——你的机器人依托这台电脑上已安装的 AI 命令行工具运行，使用你现有的登录。任选其一完成设置，机器人即可开始工作。", "engines.cloud": "云端", From 667579e05bd3c5e986ef7908777d2ea486bb133f Mon Sep 17 00:00:00 2001 From: Brad Hallett <53977268+bradhallett@users.noreply.github.com> Date: Mon, 28 Sep 2026 13:57:12 -0400 Subject: [PATCH 071/211] fix(ios): resolve chat entities without trapping on duplicate rows The snapshot does not promise one row per thread, and Dictionary(uniqueKeysWithValues:) would crash the extension during entity resolution. Both the row dictionary and the persisted identity store now use a merging initializer that keeps the snapshot's first-ranked row. Signed-off-by: Brad Hallett <53977268+bradhallett@users.noreply.github.com> (cherry picked from commit 326ad4de5f757d3718db6907df3b6dd1916c7f5f) --- ios/Widgets/BotWidget.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ios/Widgets/BotWidget.swift b/ios/Widgets/BotWidget.swift index 7514f65103..37bc25eeac 100644 --- a/ios/Widgets/BotWidget.swift +++ b/ios/Widgets/BotWidget.swift @@ -156,7 +156,7 @@ enum ChatIdentityStore { forSecurityApplicationGroupIdentifier: OpenMausSharedConfiguration.appGroupIdentifier ), let data = try? Data(contentsOf: directory.appendingPathComponent(fileName)) else { return [:] } let identities = (try? JSONDecoder().decode([ChatEntity].self, from: data)) ?? [] - return Dictionary(uniqueKeysWithValues: identities.map { ($0.id, $0) }) + return Dictionary(identities.map { ($0.id, $0) }, uniquingKeysWith: { first, _ in first }) } } From be0fab36f00a32ed8ceafab4ae3a05a764bb48ed Mon Sep 17 00:00:00 2001 From: Nevil Date: Sun, 27 Sep 2026 23:01:20 +0300 Subject: [PATCH 072/211] refactor(server): one direct-send path for a person's message POST /api/bots/:id/messages (and its guarded variant) kept the whole send inline: spend cap, sendId idempotency, steer into a running turn, queue, or start. Move that into acceptDirectSend, with the spend-cap and task checks in directSendRefusal, so another caller can send a person's words the same way. No behaviour change: the route answers the same refusals in the same order (nothing awaits between its early check and the shared one), the guarded start runs at the same point inside the send sequencer, and a steered message still clears the unattended mark only for a proven person (a paired session, or the desktop's owner capability). (cherry picked from commit e3118d2ca3a277b64db0ed72752a90a0ef0456ba) --- server/index.ts | 338 +++++++++++++++++++++++++++++------------------- 1 file changed, 203 insertions(+), 135 deletions(-) diff --git a/server/index.ts b/server/index.ts index c9c53ad9ea..79b472e637 100644 --- a/server/index.ts +++ b/server/index.ts @@ -103,7 +103,7 @@ import { groupTurnCwd } from "./room-cwd.ts"; import { RoomTurnDeadline, RoomTurnStallRegistry, roomTurnTimeoutMessage } from "./room-turn-timeout.ts"; import * as boat from "./boat.ts"; import { TeamComputers, teamComputerAssignment, teamComputerCreate, teamComputerOwner, type TeamComputerRecord } from "./team-computers.ts"; -import { isEffortLevel, type BotVisibility, type CardAnswerer, type ResolvedSender, type WireBot, type WireGroup, type WireTask } from "../shared/wire.ts"; +import { isEffortLevel, type BotVisibility, type CardAnswerer, type ResolvedSender, type SteerQueueReason, type WireBot, type WireGroup, type WireTask } from "../shared/wire.ts"; import type { TeamComputersPayload } from "../shared/team-computer.ts"; import { boatCreateRecoverySnapshot, retireDeletedBoatCreate } from "./boat-create-idempotency.ts"; import { boatDeletionSnapshot } from "./boat-delete-journal.ts"; @@ -8494,6 +8494,180 @@ async function startOrQueueDirectMessage(botId: string, threadId: string, text: return { ok: true as const, threadId, message }; } +/** What a person's direct message became: a line in the transcript (a new + * turn, or words steered into the running one), or a place in the queue. */ +type DirectSendReceipt = + | { ok: true; threadId: string; message: Message; steered?: true } + | { ok: true; queued: true; queueId: string; threadId: string; reason?: SteerQueueReason }; + +/** Refusal from acceptDirectSend: the route turns it into its HTTP answer. */ +class DirectSendRefused extends Error { + readonly status: number; + readonly body: Record; + constructor(status: number, body: Record) { + super(typeof body.error === "string" ? body.error : "send refused"); + this.status = status; + this.body = body; + } +} + +/** The refusals a direct send meets before it is sequenced, or null. */ +function directSendRefusal(botId: string, threadId: string): DirectSendRefused | null { + // The send is acknowledged before the turn starts, so a workspace at its + // spend limit is refused here, where the person can see it. + try { + assertWithinBudget(cfg, DATA_DIR); + } catch (error) { + return new DirectSendRefused(409, { error: error instanceof Error ? error.message : String(error), code: "spend_cap" }); + } + if (!store.taskByThread(botId, threadId)) { + return new DirectSendRefused(409, { error: "the bot switched tasks before it could receive the message" }); + } + return null; +} + +/** The one path a person's direct message takes into a bot turn: spend cap, + * idempotency, steer into a running turn, queue, or start. Used by + * POST /api/bots/:id/messages and its guarded variant. + * `guardedStart` is POST /api/bots/:id/messages/guarded's own start, in + * place of steer, queue or start: it checks its preconditions against the + * task as it stands, then starts a turn or refuses. */ +async function acceptDirectSend( + input: { + botId: string; + threadId: string; + text: string; + sendId?: string; + replyTo?: Message; + sender?: ResolvedSender; + trigger: UsageTrigger; + /** A person is proven present (a paired session, or the desktop's owner + * capability): steering their words in clears the unattended mark. */ + personPresent: boolean; + }, + guardedStart?: (currentAtStart: BotRecord) => Promise, +): Promise { + const { botId, threadId, text, sendId, replyTo, sender, trigger, personPresent } = input; + const refused = directSendRefusal(botId, threadId); + if (refused) throw refused; + return sendSequencer.run( + sendId ? `bot:${botId}:${threadId}:${sendId}` : undefined, + sendFingerprint(text, replyTo?.id), + async (): Promise => { + if (sendId) { + if (cancelledChatFollowup("bot", botId, threadId, sendId)) { + throw Object.assign(new Error("this queued sendId was cancelled; send a new message to try again"), { status: 409 }); + } + const accepted = acceptedSendMatch(store.messagesFor(threadId), sendId, text, replyTo?.id); + if (accepted.kind === "conflict") { + throw Object.assign(new Error("sendId already belongs to another message"), { status: 409 }); + } + if (accepted.kind === "match") { + const canonical = { + ok: true as const, + threadId, + message: accepted.message, + }; + return accepted.message.steered + ? { ...canonical, steered: true as const } + : canonical; + } + const queued = queuedSteeredMessage(botId, threadId, sendId); + if (queued) { + if (queued.text !== text || queued.replyToId !== replyTo?.id) { + throw Object.assign(new Error("sendId already belongs to another message"), { status: 409 }); + } + return { ok: true as const, queued: true as const, queueId: queued.id, threadId, reason: queued.reason }; + } + } + + const currentAtStart = store.projectBotForTask(botId, threadId); + if (!currentAtStart) throw Object.assign(new Error("no such bot"), { status: 404 }); + if (!store.taskByThread(currentAtStart.id, threadId)) { + throw Object.assign(new Error("the target task no longer exists"), { status: 409 }); + } + + if (guardedStart) return guardedStart(currentAtStart); + + // Claude can accept the message inside its live turn. If the write + // loses a race with turn settlement, or the engine cannot steer, the + // existing server-side queue records it atomically for the next turn. + if (currentAtStart.busy) { + const instance = runningTurnInstance(currentAtStart, threadId); + let steered: SteerOutcome = "refused"; + // A live text steer has no image side channel. Keep an attachment + // message intact for the next ordinary turn, where central image + // admission can hand it to the provider natively. + const carriesImages = extractTurnImages(text).images.length > 0; + const steerTarget = handoffs.current(threadId); + const busyAdmission = admit("direct-busy", { + carriesImages, + pendingComputerSelection: Boolean(computerSelectionTurns.get(threadId)?.selected), + engineCanSteer: Boolean(instance?.adapter.capabilities.queueing && instance.adapter.steer), + }); + // steer was offered only when a live instance could take it; + // the second check carries that fact to the type system. + if (busyAdmission.action === "steer" && instance?.adapter.steer) { + steered = await instance.adapter + .steer(threadId, promptWithReply(text, replyTo, cfg.profile?.name?.trim() || "User")) + .catch((): SteerOutcome => "indeterminate"); + } + // steer() is awaited adapter work. The turn can settle, the task can + // switch, or the whole bot can be deleted before its acknowledgement + // arrives. Re-read every ownership invariant before appending even a + // successful steer; otherwise that late acknowledgement writes a user + // message into a task the bot no longer owns. A conflict leaves the + // text in the client's composer/outbox to resend deliberately. + const current = store.projectBotForTask(botId, threadId); + if (!current) throw Object.assign(new Error("no such bot"), { status: 404 }); + if (!store.taskByThread(botId, threadId)) { + throw Object.assign(new Error("the target task no longer exists"), { status: 409 }); + } + const delivered = steered !== "refused"; + if (delivered) { + if (steered === "steered" && !current.busy) { + throw Object.assign( + new Error("the running turn ended before the steered message could be recorded"), + { status: 409 }, + ); + } + // "indeterminate" falls through to the same record: the words + // may already be folded into a turn whose acknowledgement was + // lost, and handing them back for a resend could run them + // twice. Recording them once is the honest outcome. + // A person steering a webhook turn is present, and auto mode may + // follow them again; words that do not prove a person never lift it. + if (personPresent) clearUnattended(threadId); + const message = store.appendMessage(threadId, { + role: "user", + kind: "text", + text, + replyToId: replyTo?.id, + sendId, + steered: true, + sender, + }); + // Offered to the next turn again unless the person stops this one. + handoffs.steered(threadId, steerTarget, instance?.instanceId, message.id); + return { ok: true as const, steered: true as const, threadId, message }; + } + if (!current.busy) { + return startOrQueueDirectMessage(botId, threadId, text, replyTo, sendId, sender, trigger); + } + const queued = queueSteeredMessage(current.id, threadId, text, { + replyToId: replyTo?.id, + sendId, + prompt: promptWithReply(text, replyTo, cfg.profile?.name?.trim() || "User"), + sender, + trigger, + }); + return { ok: true as const, queued: true as const, queueId: queued.id, threadId }; + } + return startOrQueueDirectMessage(botId, threadId, text, replyTo, sendId, sender, trigger); + }, + ); +} + /** How many start_thread calls one turn may make. Same spirit as the * create-bot ceiling above: a handful is a plan, more is a fan-out. */ const MAX_THREADS_OPENED_PER_TURN = 5; @@ -20635,56 +20809,16 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { const trigger: UsageTrigger = onBehalfOf ? { kind: "user", ...(onBehalfOf.email ? { email: onBehalfOf.email } : {}), ...(onBehalfOf.name ? { label: onBehalfOf.name } : {}) } : usageTriggerFor(auth); - // The send is acknowledged before the turn starts, so a workspace at its - // spend limit is refused here, where the person can see it. - try { - assertWithinBudget(cfg, DATA_DIR); - } catch (error) { - return json(res, 409, { error: error instanceof Error ? error.message : String(error), code: "spend_cap" }); - } - if (!store.taskByThread(bot.id, threadId)) { - return json(res, 409, { error: "the bot switched tasks before it could receive the message" }); - } + // The spend cap and a task that is gone answer before a malformed + // sendId or reply target does. acceptDirectSend checks both again for + // callers that do not come through this route; nothing awaits between. + const refused = directSendRefusal(bot.id, threadId); + if (refused) return json(res, refused.status, refused.body); const sendId = parseSendId(body.sendId); const replyTo = resolveReplyTarget(threadId, body.replyToId); - const receipt = await sendSequencer.run( - sendId ? `bot:${bot.id}:${threadId}:${sendId}` : undefined, - sendFingerprint(text, replyTo?.id), - async () => { - if (sendId) { - if (cancelledChatFollowup("bot", bot.id, threadId, sendId)) { - throw Object.assign(new Error("this queued sendId was cancelled; send a new message to try again"), { status: 409 }); - } - const accepted = acceptedSendMatch(store.messagesFor(threadId), sendId, text, replyTo?.id); - if (accepted.kind === "conflict") { - throw Object.assign(new Error("sendId already belongs to another message"), { status: 409 }); - } - if (accepted.kind === "match") { - const canonical = { - ok: true as const, - threadId, - message: accepted.message, - }; - return accepted.message.steered - ? { ...canonical, steered: true as const } - : canonical; - } - const queued = queuedSteeredMessage(bot.id, threadId, sendId); - if (queued) { - if (queued.text !== text || queued.replyToId !== replyTo?.id) { - throw Object.assign(new Error("sendId already belongs to another message"), { status: 409 }); - } - return { ok: true as const, queued: true as const, queueId: queued.id, threadId, reason: queued.reason }; - } - } - - const currentAtStart = store.projectBotForTask(bot.id, threadId); - if (!currentAtStart) throw Object.assign(new Error("no such bot"), { status: 404 }); - if (!store.taskByThread(currentAtStart.id, threadId)) { - throw Object.assign(new Error("the target task no longer exists"), { status: 409 }); - } - - if (guarded) { + const sender = messageSender(auth); + const guardedStart = guarded + ? async (currentAtStart: BotRecord): Promise => { // There is no await between these checks and startTurn's // synchronous transcript append / runtime reservation. In // particular, never steer or enqueue under stale permissions. @@ -20706,94 +20840,28 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (guardedAdmission.action === "refuse") { throw Object.assign(new Error("wait for a free thread slot before retrying this message"), { status: 409, code: "guarded_busy" }); } - const message = await startTurn(bot.id, text, { threadId, replyTo, sendId, sender: messageSender(auth), trigger }); + const message = await startTurn(bot.id, text, { threadId, replyTo, sendId, sender, trigger }); return { ok: true as const, threadId, message }; } - - // Claude can accept the message inside its live turn. If the write - // loses a race with turn settlement, or the engine cannot steer, the - // existing server-side queue records it atomically for the next turn. - if (currentAtStart.busy) { - const instance = runningTurnInstance(currentAtStart, threadId); - let steered: SteerOutcome = "refused"; - // A live text steer has no image side channel. Keep an attachment - // message intact for the next ordinary turn, where central image - // admission can hand it to the provider natively. - const carriesImages = extractTurnImages(text).images.length > 0; - const steerTarget = handoffs.current(threadId); - const busyAdmission = admit("direct-busy", { - carriesImages, - pendingComputerSelection: Boolean(computerSelectionTurns.get(threadId)?.selected), - engineCanSteer: Boolean(instance?.adapter.capabilities.queueing && instance.adapter.steer), - }); - // steer was offered only when a live instance could take it; - // the second check carries that fact to the type system. - if (busyAdmission.action === "steer" && instance?.adapter.steer) { - steered = await instance.adapter - .steer(threadId, promptWithReply(text, replyTo, cfg.profile?.name?.trim() || "User")) - .catch((): SteerOutcome => "indeterminate"); - } - // steer() is awaited adapter work. The turn can settle, the task can - // switch, or the whole bot can be deleted before its acknowledgement - // arrives. Re-read every ownership invariant before appending even a - // successful steer; otherwise that late acknowledgement writes a user - // message into a task the bot no longer owns. A conflict leaves the - // text in the client's composer/outbox to resend deliberately. - const current = store.projectBotForTask(bot.id, threadId); - if (!current) throw Object.assign(new Error("no such bot"), { status: 404 }); - if (!store.taskByThread(bot.id, threadId)) { - throw Object.assign(new Error("the target task no longer exists"), { status: 409 }); - } - const delivered = steered !== "refused"; - if (delivered) { - if (steered === "steered" && !current.busy) { - throw Object.assign( - new Error("the running turn ended before the steered message could be recorded"), - { status: 409 }, - ); - } - // "indeterminate" falls through to the same record: the words - // may already be folded into a turn whose acknowledgement was - // lost, and handing them back for a resend could run them - // twice. Recording them once is the honest outcome. - // A person steering a webhook turn is present, and auto mode may - // follow them again. But this route is also reachable from the - // bot's own shell on a headless server (loopback is the owner - // there), and "continue" typed by the turn itself must not be - // the thing that lifts the block written against it — so only - // a request that proves a person (a paired session, or the - // desktop's owner capability, which every mutation there has - // already shown) clears the mark. - if (auth.kind === "session" || DESKTOP_MANAGED) clearUnattended(threadId); - const message = store.appendMessage(threadId, { - role: "user", - kind: "text", - text, - replyToId: replyTo?.id, - sendId, - steered: true, - sender: messageSender(auth), - }); - // Offered to the next turn again unless the person stops this one. - handoffs.steered(threadId, steerTarget, instance?.instanceId, message.id); - return { ok: true as const, steered: true as const, threadId, message }; - } - if (!current.busy) { - return startOrQueueDirectMessage(bot.id, threadId, text, replyTo, sendId, messageSender(auth), trigger); - } - const queued = queueSteeredMessage(current.id, threadId, text, { - replyToId: replyTo?.id, - sendId, - prompt: promptWithReply(text, replyTo, cfg.profile?.name?.trim() || "User"), - sender: messageSender(auth), - trigger, - }); - return { ok: true as const, queued: true as const, queueId: queued.id, threadId }; - } - return startOrQueueDirectMessage(bot.id, threadId, text, replyTo, sendId, messageSender(auth), trigger); - }, - ); - return json(res, 202, receipt); + : undefined; + try { + const receipt = await acceptDirectSend({ + botId: bot.id, threadId, text, sendId, replyTo, sender, trigger, + // A person steering a webhook turn is present, and auto mode may + // follow them again. But this route is also reachable from the + // bot's own shell on a headless server (loopback is the owner + // there), and "continue" typed by the turn itself must not be the + // thing that lifts the block written against it — so only a + // request that proves a person (a paired session, or the desktop's + // owner capability, which every mutation there has already shown) + // clears the mark. + personPresent: auth.kind === "session" || DESKTOP_MANAGED, + }, guardedStart); + return json(res, 202, receipt); + } catch (error) { + if (error instanceof DirectSendRefused) return json(res, error.status, error.body); + throw error; + } } m = path.match(/^\/api\/bots\/([\w-]+)\/queue\/([\w-]+)$/); From f3886bb7bc4df8a83de53111e2ee001edf2e6dc2 Mon Sep 17 00:00:00 2001 From: Nevil Date: Sun, 27 Sep 2026 23:05:01 +0300 Subject: [PATCH 073/211] feat(calls): Live calls on the harness, with GPT-Live as the voice and the bot as the brain A Live call runs the conversation through OpenAI GPT-Live. Its client delegation hands every real request back to the application, so the harness sends it to the bot as an ordinary message marked `via: "call"` (through acceptDirectSend), and the bot keeps its own engine, tools, memory and approvals. The voice only talks, listens and takes interruptions. LiveCallController holds the sideband WebSocket to the call's session and runs every rule that joins the voice to the bot, once, for every client: a delegation becomes a message on the bot's thread; the bot's progress becomes quiet context and a status note every 30 s while it works; its answer is read back; a permission card is decided by a strict spoken yes or no (shared/call-consent.ts), never by the voice model, and a card decided that way is marked "by voice" on the card and in the decision log; proposals that change the bot or the workspace stay on screen. A typed message during a call is answered by the bot and read once, or, with "read replies to typed messages" off, nothing about it reaches OpenAI. A call hangs up after 5 idle minutes (configurable), and ends at once when the sign-in that started it ends or its phone is unpaired. Routes (server/routes/live.ts, admin scope): POST /api/live/session exchanges the client's WebRTC offer for OpenAI's answer, so the key never leaves the harness; POST /api/live/call/end, GET /api/live/call and PATCH /api/live/settings (voice, typed replies, idle minutes; a key in the body is refused). POST /api/live/device-revoked is the companion's own notice that it unpaired a phone; the relay token opens it, a phone never can (companion/src/routes.ts isCompanionNotice). State travels in admin-only `live.call` frames. The key is write-only, like every other credential: `live.key`, the OMB_OPENAI_LIVE_KEY environment variable, or the desktop credential `openaiLiveKey`, kept apart from every other OpenAI key; the API reports only `configured`. Words spoken on a call stay out of server.log, whose call lines carry counters only. Tests run against server/testing/fake-openai-live.ts, a loopback fake GPT-Live with a sideband WebSocket; the fixture launcher lets a loopback OMB_OPENAI_LIVE_URL (and only then the key) through to the child. (cherry picked from commit 32b01a7266c98c848ca59aecf37697f81c5a3903) --- companion/src/routes.ts | 17 + companion/test/routes.test.ts | 14 +- docs/verification/README.md | 7 +- docs/verification/guarded-messages.md | 6 + electron/diagnostics.mjs | 1 + electron/main.mjs | 1 + electron/workspace-credentials.mjs | 1 + scripts/control-omb.test.ts | 29 + scripts/control-omb.ts | 7 + server/channel-queue.ts | 5 +- server/chat-followups-restart.test.ts | 11 +- server/config.test.ts | 54 ++ server/config.ts | 32 +- server/decision-log.test.ts | 20 + server/decision-log.ts | 19 +- server/index.ts | 190 +++- server/live-call-controller.test.ts | 1058 +++++++++++++++++++++++ server/live-call-controller.ts | 915 ++++++++++++++++++++ server/live-call.e2e.test.ts | 377 ++++++++ server/live-call.test.ts | 197 +++++ server/live-call.ts | 252 ++++++ server/message-db.ts | 4 +- server/request-auth.test.ts | 8 + server/request-auth.ts | 5 +- server/routes/live.test.ts | 195 +++++ server/routes/live.ts | 114 +++ server/steer-queue.test.ts | 39 + server/steer-queue.ts | 20 +- server/testing/fake-openai-live.test.ts | 177 ++++ server/testing/fake-openai-live.ts | 251 ++++++ server/turn-log.test.ts | 19 + server/turn-log.ts | 22 + shared/call-consent.test.ts | 52 ++ shared/call-consent.ts | 25 + shared/live-approval.test.ts | 95 ++ shared/live-approval.ts | 142 +++ shared/live-call.test.ts | 99 +++ shared/live-call.ts | 185 ++++ shared/wire.ts | 52 +- src/types/ogb.d.ts | 2 +- 40 files changed, 4680 insertions(+), 39 deletions(-) create mode 100644 scripts/control-omb.test.ts create mode 100644 server/live-call-controller.test.ts create mode 100644 server/live-call-controller.ts create mode 100644 server/live-call.e2e.test.ts create mode 100644 server/live-call.test.ts create mode 100644 server/live-call.ts create mode 100644 server/routes/live.test.ts create mode 100644 server/routes/live.ts create mode 100644 server/testing/fake-openai-live.test.ts create mode 100644 server/testing/fake-openai-live.ts create mode 100644 server/turn-log.test.ts create mode 100644 server/turn-log.ts create mode 100644 shared/call-consent.test.ts create mode 100644 shared/call-consent.ts create mode 100644 shared/live-approval.test.ts create mode 100644 shared/live-approval.ts create mode 100644 shared/live-call.test.ts create mode 100644 shared/live-call.ts diff --git a/companion/src/routes.ts b/companion/src/routes.ts index 0cc153ec7f..5dcff74615 100644 --- a/companion/src/routes.ts +++ b/companion/src/routes.ts @@ -222,6 +222,23 @@ const ALLOWED: ReadonlyArray<{ method: string; path: RegExp }> = [ { method: "POST", path: /^\/api\/bots\/[\w-]+\/secret-cards\/[\w-]+\/(?:resume|dismiss)$/ }, ]; +/** Notices the companion itself sends the harness, never a device: they are + * not in ALLOWED, so the proxy refuses them from a phone, and the harness + * accepts them only with the companion's private relay token + * (server/request-auth.ts) or, for a standalone harness, from loopback. + * + * `POST /api/live/device-revoked`: a phone was just unpaired (the device id + * rides in `x-openmausbot-companion-device`), so the harness ends the Live + * call that phone holds. A phone's requests reach the harness as this + * computer's own, so nothing else would tell it the phone lost its access. */ +const COMPANION_NOTICES: ReadonlyArray<{ method: string; path: RegExp }> = [ + { method: "POST", path: /^\/api\/live\/device-revoked$/ }, +]; + +export function isCompanionNotice(method: string, path: string): boolean { + return COMPANION_NOTICES.some((route) => route.method === method && route.path.test(path)); +} + /** Route families worth naming in the refusal. * * Everything not allowed is denied either way; this only decides whether the diff --git a/companion/test/routes.test.ts b/companion/test/routes.test.ts index ea7567e5bd..14d6940e95 100644 --- a/companion/test/routes.test.ts +++ b/companion/test/routes.test.ts @@ -7,7 +7,7 @@ // and the one that quietly stopped being true once before. import { describe, expect, it } from "vitest"; -import { denyReason, isCloudDesktopAccess } from "../src/routes.ts"; +import { denyReason, isCloudDesktopAccess, isCompanionNotice } from "../src/routes.ts"; const ask = (method: string, path: string, authenticated = true) => denyReason({ method, path, authenticated }); @@ -169,6 +169,18 @@ describe("what it may not", () => { expect(ask("POST", "/api/routines/routine_1/run")).toBeNull(); }); + // The companion tells the harness itself when it unpaired a phone, so the + // call that phone holds ends. That notice is the companion's, never a + // phone's: no paired device may send it, even about itself. + it("keeps the unpaired-phone notice for the companion alone", () => { + expect(ask("POST", "/api/live/device-revoked")?.status).toBe(404); + expect(ask("POST", "/api/live/device-revoked", false)?.status).toBe(401); + expect(isCompanionNotice("POST", "/api/live/device-revoked")).toBe(true); + expect(isCompanionNotice("GET", "/api/live/device-revoked")).toBe(false); + expect(isCompanionNotice("POST", "/api/live/device-revoked/x")).toBe(false); + expect(isCompanionNotice("POST", "/api/live/call/end")).toBe(false); + }); + it("denies the peer-agent endpoints exist at all", () => { expect(ask("GET", "/api/internal/peers")?.status).toBe(404); expect(ask("POST", "/api/internal/ask-bot")?.status).toBe(404); diff --git a/docs/verification/README.md b/docs/verification/README.md index c0ec1a73ed..482e9d906d 100644 --- a/docs/verification/README.md +++ b/docs/verification/README.md @@ -22,7 +22,12 @@ the URL, PID, data directory, and persistent log path, then stays attached to that exact child. The parent shell and the user's OpenMausBot data are untouched. Only `FAKE_CLAUDE_*` variables cross from the launcher's environment into that child, so a recipe can script the fake engine's mode, -replies and tool calls without writing a wrapper CLI. +replies and tool calls without writing a wrapper CLI. Live calls add one +exception: `OMB_OPENAI_LIVE_URL` crosses when it is a loopback +`http://127.0.0.1:PORT` (the fake GPT-Live that +`node --experimental-strip-types server/testing/fake-openai-live.ts` prints), +and `OMB_OPENAI_LIVE_KEY` crosses only with it, so that key can only ever +reach the fake. Pass the printed URL explicitly from a second terminal: diff --git a/docs/verification/guarded-messages.md b/docs/verification/guarded-messages.md index cf1a8309e9..166f00a00b 100644 --- a/docs/verification/guarded-messages.md +++ b/docs/verification/guarded-messages.md @@ -20,6 +20,12 @@ of to this machine. It changes nothing else: not the transcript's sender, not permissions. The route stays admin-only, so a member's session cannot name someone else. +Every guarded send's user line is stored with `relayed: true`: an external +interface relayed it, and nobody typed it in one of this workspace's clients. +A Live call running on that thread never reads such a line, or the bot's +answer to it, back to the caller as something they typed +(`server/live-call.e2e.test.ts` sends one during a call). + `capabilities.guardedFullAccess: 1` additionally accepts `expectedApprovalMode: "full"` for a task that **already has** Full access. It does not grant or change permissions. The exact task, not its bot default, diff --git a/electron/diagnostics.mjs b/electron/diagnostics.mjs index 455449b393..6a70edff3e 100644 --- a/electron/diagnostics.mjs +++ b/electron/diagnostics.mjs @@ -29,6 +29,7 @@ export const CREDENTIAL_ENV_NAMES = [ "OMB_JEV_API_KEY", "OMB_OPENAI_IMAGE_KEY", "OMB_CUSTOM_IMAGE_KEY", + "OMB_OPENAI_LIVE_KEY", "COMPOSIO_API_KEY", "OMB_COMPOSIO_BROKER_TOKEN", "OMB_CLOUD_BOAT_TOKEN", diff --git a/electron/main.mjs b/electron/main.mjs index 408381985a..28a1e5e171 100644 --- a/electron/main.mjs +++ b/electron/main.mjs @@ -3039,6 +3039,7 @@ const CREDENTIAL_PATCH = { jevApiKey: (value) => ({ decider: { key: value } }), openaiImageApiKey: (value) => ({ imageGen: { key: value } }), customImageApiKey: (value) => ({ imageGen: { customApiKey: value } }), + openaiLiveKey: (value) => ({ live: { key: value } }), }; async function saveWorkspaceCredential(name, value) { diff --git a/electron/workspace-credentials.mjs b/electron/workspace-credentials.mjs index 5b50100082..16cefb0462 100644 --- a/electron/workspace-credentials.mjs +++ b/electron/workspace-credentials.mjs @@ -15,6 +15,7 @@ export const WORKSPACE_CREDENTIALS = [ { section: "decider", field: "key", name: "jevApiKey", env: "OMB_JEV_API_KEY" }, { section: "imageGen", field: "key", name: "openaiImageApiKey", env: "OMB_OPENAI_IMAGE_KEY" }, { section: "imageGen", field: "customApiKey", name: "customImageApiKey", env: "OMB_CUSTOM_IMAGE_KEY" }, + { section: "live", field: "key", name: "openaiLiveKey", env: "OMB_OPENAI_LIVE_KEY" }, { section: "opencodeGo", field: "apiKey", name: "opencodeGoApiKey", env: "OPENCODE_API_KEY" }, ]; diff --git a/scripts/control-omb.test.ts b/scripts/control-omb.test.ts new file mode 100644 index 0000000000..c640faf5ca --- /dev/null +++ b/scripts/control-omb.test.ts @@ -0,0 +1,29 @@ +// What crosses from the launching shell into a fixture server: only what a +// test scripted on purpose. A real OpenAI key must never reach a fixture that +// could send it to OpenAI. +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { describe, expect, it } from "vitest"; + +import { verificationServerEnvironment } from "./control-omb.ts"; + +const childEnv = (parentEnv: NodeJS.ProcessEnv) => verificationServerEnvironment(parentEnv, join(tmpdir(), "omb-fixture-env"), 9100); + +describe("the fixture's Live call environment", () => { + it("passes a loopback fake GPT-Live and its key through", () => { + expect(childEnv({ OMB_OPENAI_LIVE_URL: " http://127.0.0.1:4100 ", OMB_OPENAI_LIVE_KEY: "sk-fake" })) + .toMatchObject({ OMB_OPENAI_LIVE_URL: "http://127.0.0.1:4100", OMB_OPENAI_LIVE_KEY: "sk-fake" }); + const noKey = childEnv({ OMB_OPENAI_LIVE_URL: "http://127.0.0.1:4100" }); + expect(noKey.OMB_OPENAI_LIVE_URL).toBe("http://127.0.0.1:4100"); + expect(noKey).not.toHaveProperty("OMB_OPENAI_LIVE_KEY"); + }); + + it("drops any other Live URL, and never lets the key cross without the fake", () => { + for (const url of ["https://api.openai.com", "http://localhost:4100", "http://127.0.0.1:4100/v1", "http://192.0.2.1:4100", "http://127.0.0.1", ""]) { + const env = childEnv({ OMB_OPENAI_LIVE_URL: url, OMB_OPENAI_LIVE_KEY: "sk-real" }); + expect(env, url).not.toHaveProperty("OMB_OPENAI_LIVE_URL"); + expect(env, url).not.toHaveProperty("OMB_OPENAI_LIVE_KEY"); + } + expect(childEnv({ OMB_OPENAI_LIVE_KEY: "sk-real" })).not.toHaveProperty("OMB_OPENAI_LIVE_KEY"); + }); +}); diff --git a/scripts/control-omb.ts b/scripts/control-omb.ts index 81192ec1c0..a36152fb97 100644 --- a/scripts/control-omb.ts +++ b/scripts/control-omb.ts @@ -381,6 +381,13 @@ export function verificationServerEnvironment(parentEnv: NodeJS.ProcessEnv, data // A test's key for relaying an organization library into the fixture // (POST /api/testing/org-library); the route does not exist without it. if (parentEnv.OMB_TEST_ORG_LIBRARY_KEY) childEnv.OMB_TEST_ORG_LIBRARY_KEY = parentEnv.OMB_TEST_ORG_LIBRARY_KEY; + // Live calls against server/testing/fake-openai-live.ts only: a loopback + // URL, and a key that only ever reaches that fake. + const liveUrl = parentEnv.OMB_OPENAI_LIVE_URL?.trim() ?? ""; + if (/^http:\/\/127\.0\.0\.1:\d{1,5}$/.test(liveUrl)) { + childEnv.OMB_OPENAI_LIVE_URL = liveUrl; + if (parentEnv.OMB_OPENAI_LIVE_KEY) childEnv.OMB_OPENAI_LIVE_KEY = parentEnv.OMB_OPENAI_LIVE_KEY; + } // Voice-note e2e fault injection: arms the one-shot audio-append failure // prelude inside the fixture server (see fail-audio-append-once.mjs). if (parentEnv.OMB_TEST_FAIL_AUDIO_APPEND_ONCE) { diff --git a/server/channel-queue.ts b/server/channel-queue.ts index 4d7f5e5454..de3ae808f2 100644 --- a/server/channel-queue.ts +++ b/server/channel-queue.ts @@ -42,8 +42,11 @@ export function restoreChannelMessages(): void { if (row.status !== "pending") continue; const entry = queues.get(row.threadId) ?? { groupId: row.ownerId, items: [] }; if (entry.groupId !== row.ownerId) throw new Error("queued task belongs to another channel"); + // a channel line is only ever stamped "api"; "call" belongs to 1:1 rows + const { via, ...payload } = row.payload; entry.items.push({ - ...row.payload, + ...payload, + ...(via === "api" ? { via } : {}), id: row.id, mode: row.payload.mode ?? "chat", // rows queued before timestamps were kept read as queued at restore diff --git a/server/chat-followups-restart.test.ts b/server/chat-followups-restart.test.ts index 221e036947..3162b41506 100644 --- a/server/chat-followups-restart.test.ts +++ b/server/chat-followups-restart.test.ts @@ -182,12 +182,14 @@ it("survives a real server crash: queued sends keep receipts, cancellation and u try { // The narrowest crash window: the dispatch claim reached disk, the // transcript line did not. One row names its sender; one was written by - // a build that did not keep one and must still recover. + // a build that did not keep one and must still recover; one was spoken + // in a Live call and must still say so. const claim = crashed.prepare( "INSERT INTO chat_followups(id, kind, owner_id, thread_id, send_id, status, payload) VALUES (?, 'bot', ?, ?, NULL, 'dispatching', ?)", ); claim.run("claimed_before_append_named", unappended.id, unappended.threadId, JSON.stringify({ text: "Claimed, never appended", sender: PAIRED_ROW })); claim.run("claimed_before_append_legacy", unappended.id, unappended.threadId, JSON.stringify({ text: "Claimed by an older build" })); + claim.run("claimed_before_append_call", unappended.id, unappended.threadId, JSON.stringify({ text: "Claimed from a call", via: "call" })); crashed.prepare("UPDATE messages SET json = json_set(json, '$.text', '') WHERE thread_id = ? AND id = ?") .run(lost.threadId, lostTarget.message.id); } finally { crashed.close(); } @@ -200,9 +202,10 @@ it("survives a real server crash: queued sends keep receipts, cancellation and u expect((await api("POST", `/api/bots/${bot.id}/messages`, body, 202)).message).toMatchObject({ sendId: body.sendId, queueId: queued.queueId, replyToId, text, sender: PAIRED, }); - expect((await messages(unappended.threadId)).filter((message) => message.role === "user").map((message) => [message.queueId, message.text, message.sender])).toEqual([ - ["claimed_before_append_named", "Claimed, never appended", PAIRED_ROW], - ["claimed_before_append_legacy", "Claimed by an older build", undefined], + expect((await messages(unappended.threadId)).filter((message) => message.role === "user").map((message) => [message.queueId, message.text, message.sender, message.via])).toEqual([ + ["claimed_before_append_named", "Claimed, never appended", PAIRED_ROW, undefined], + ["claimed_before_append_legacy", "Claimed by an older build", undefined, undefined], + ["claimed_before_append_call", "Claimed from a call", undefined, "call"], ]); await expect.poll(async () => (await messages(lost.threadId)).some((message) => message.tool?.name?.includes("queued channel message could not start")), { timeout: 15_000 }).toBe(true); expect((await messages(lost.threadId)).filter((message) => message.queueId === lostQueued.queueId)).toEqual([ diff --git a/server/config.test.ts b/server/config.test.ts index ecb776e4ed..7ac6c7bbcf 100644 --- a/server/config.test.ts +++ b/server/config.test.ts @@ -40,6 +40,8 @@ import { customMcpServers, browserEngineAttachCdpUrl, withInstanceCli, WORKSPACE_CREDENTIAL_ENV, + liveSettingsFor, + LIVE_IDLE_MINUTES_DEFAULT, type AppConfig, } from "./config.ts"; @@ -1712,6 +1714,58 @@ describe("customMcpServers with url entries", () => { }); }); +describe("live settings", () => { + it("defaults to a 5 minute idle hang-up and reading typed replies", () => { + expect(LIVE_IDLE_MINUTES_DEFAULT).toBe(5); + expect(liveSettingsFor({} as AppConfig)).toEqual({ configured: false, voice: "", readTypedReplies: true, idleMinutes: 5 }); + }); + it("reports saved values and never the key", () => { + const settings = liveSettingsFor({ live: { key: "sk-test", voice: "sol", readTypedReplies: false, idleMinutes: 12 } } as AppConfig); + expect(settings).toEqual({ configured: true, voice: "sol", readTypedReplies: false, idleMinutes: 12 }); + expect(JSON.stringify(settings)).not.toContain("sk-test"); + }); + it("accepts idle minutes from 1 to 60 only", () => { + expect(() => parseConfigPatch({ live: { idleMinutes: 0 } })).toThrow(); + expect(() => parseConfigPatch({ live: { idleMinutes: 61 } })).toThrow(); + expect(() => parseConfigPatch({ live: { idleMinutes: 2.5 } })).toThrow(); + expect(parseConfigPatch({ live: { idleMinutes: 60, readTypedReplies: false } })).toMatchObject({ live: { idleMinutes: 60, readTypedReplies: false } }); + }); + it("does not reload providers for live changes", () => { + expect(providerReloadKeys({ live: { idleMinutes: 3 } } as never)).toEqual([]); + }); + + describe("saving settings from PATCH /api/live/settings", () => { + const path = join(DATA_DIR, "config.json"); + let envKey: string | undefined; + beforeEach(() => { + envKey = process.env.OMB_OPENAI_LIVE_KEY; + delete process.env.OMB_OPENAI_LIVE_KEY; + mkdirSync(DATA_DIR, { recursive: true }); + rmSync(path, { force: true }); + }); + afterEach(() => { + if (envKey === undefined) delete process.env.OMB_OPENAI_LIVE_KEY; + else process.env.OMB_OPENAI_LIVE_KEY = envKey; + rmSync(path, { force: true }); + }); + + it("keeps the Live key when only settings change", () => { + saveConfig({ live: { key: "sk-keep" } }); + saveConfig({ live: { idleMinutes: 9 } }); + expect(loadConfig().live).toMatchObject({ key: "sk-keep", idleMinutes: 9 }); + }); + + it("keeps a key from the desktop credential store, which reaches the harness as env", () => { + // the desktop leaves an empty tombstone in the file and hands the key over as env + saveConfig({ live: { key: "" } }); + process.env.OMB_OPENAI_LIVE_KEY = "sk-from-keychain"; + saveConfig({ live: { readTypedReplies: false, voice: "cedar" } }); + expect(loadConfig().live).toEqual({ key: "sk-from-keychain", readTypedReplies: false, voice: "cedar" }); + expect(JSON.parse(readFileSync(path, "utf8")).live).toEqual({ key: "", readTypedReplies: false, voice: "cedar" }); + }); + }); +}); + describe("loadConfig with an unusable config.json", () => { const path = join(DATA_DIR, "config.json"); let warn: ReturnType; diff --git a/server/config.ts b/server/config.ts index f5ef36ca40..1284715866 100644 --- a/server/config.ts +++ b/server/config.ts @@ -9,7 +9,7 @@ import { normalizeImageGenerationUrl, type ImageGenerationConfig } from "../shar import { writeFileAtomic } from "./atomic.ts"; import { newBotDefaultsSchema, type NewBotDefaults } from "./new-bot-defaults.ts"; -import { EFFORT_LEVELS, type EffortLevel } from "../shared/wire.ts"; +import { EFFORT_LEVELS, type EffortLevel, type LiveSettings } from "../shared/wire.ts"; import { isModelVariant, type InstanceConfigMap, type ModelSelection } from "./contracts.ts"; import { PROVIDER_ICON_PRESETS, providerIconError } from "../shared/provider-icon.ts"; import type { McpServerSpec } from "./contracts.ts"; @@ -506,6 +506,15 @@ const appConfigSchema = z.object({ .optional(), jobs: z.object({ roomRouting: z.boolean().optional() }).optional(), }).optional(), + /** Live calls: an OpenAI project key for GPT-Live, kept apart from every + * other OpenAI credential so a Live call never bills an image or engine key + * the user did not hand to it. `voice` is a GPT-Live built-in voice name. */ + live: z.object({ + key: optionalText, + voice: z.string().trim().max(40).regex(/^[a-z]*$/, "a Live voice is a lowercase built-in voice name").optional(), + readTypedReplies: z.boolean().optional(), + idleMinutes: z.number().int().min(1).max(60).optional(), + }).optional(), /** Avatar provider credentials stay separate; choosing a router never reuses a cloud key. */ imageGen: z.object({ provider: z.enum(["openai", "xai", "custom"]).optional(), @@ -614,6 +623,7 @@ export interface AppConfig { /** The decision model; see the schema above and server/decider. */ decider?: { enabled?: boolean; provider?: "jev" | "off"; key?: string; baseUrl?: string; jobs?: { roomRouting?: boolean } }; imageGen?: ImageGenerationConfig; + live?: { key?: string; voice?: string; readTypedReplies?: boolean; idleMinutes?: number }; profile?: { name?: string; email?: string; aboutMe?: string }; rooms?: { turnTimeoutMinutes: number; handoffLifetimeMinutes?: number; handoffMinRunwayMinutes?: number; handoffHardCapMinutes?: number }; threads?: { maxConcurrentPerBot: number; eventLogMaxBytes?: number; eventLogRetentionDays?: number }; @@ -759,6 +769,19 @@ export function roomTurnTimeoutMinutes(cfg: AppConfig): number { return cfg.rooms?.turnTimeoutMinutes ?? DEFAULT_ROOM_TURN_TIMEOUT_MINUTES; } +export const LIVE_IDLE_MINUTES_DEFAULT = 5; + +/** Non-secret Live settings. The key only shows up as `configured`. */ +export function liveSettingsFor(cfg: AppConfig): LiveSettings { + const minutes = cfg.live?.idleMinutes; + return { + configured: Boolean(cfg.live?.key?.trim()), + voice: cfg.live?.voice ?? "", + readTypedReplies: cfg.live?.readTypedReplies ?? true, + idleMinutes: Number.isInteger(minutes) && minutes! >= 1 && minutes! <= 60 ? minutes! : LIVE_IDLE_MINUTES_DEFAULT, + }; +} + export interface RoomHandoffLimitsMs { lifetimeMs: number; minRunwayMs: number; @@ -936,6 +959,7 @@ export const FLEET_NEUTRAL_KEYS: ReadonlySet = new Set([ // no engine reads it: the harness asks it before a turn starts "decider", "imageGen", + "live", "vps", "rooms", "threads", @@ -1071,6 +1095,8 @@ export function loadConfig(): AppConfig { if (process.env.OMB_FISH_AUDIO_API_KEY !== undefined) cfg.tts.fishKey = process.env.OMB_FISH_AUDIO_API_KEY; cfg.decider = { ...cfg.decider }; if (process.env.OMB_JEV_API_KEY !== undefined) cfg.decider.key = process.env.OMB_JEV_API_KEY; + cfg.live = { ...cfg.live }; + if (process.env.OMB_OPENAI_LIVE_KEY !== undefined) cfg.live.key = process.env.OMB_OPENAI_LIVE_KEY; cfg.imageGen = { ...cfg.imageGen }; if (process.env.OMB_OPENAI_IMAGE_KEY !== undefined) cfg.imageGen.key = process.env.OMB_OPENAI_IMAGE_KEY; if (process.env.OMB_CUSTOM_IMAGE_KEY !== undefined) cfg.imageGen.customApiKey = process.env.OMB_CUSTOM_IMAGE_KEY; @@ -1108,6 +1134,7 @@ export function syncCredentialEnv(patch: Partial { expect(rows.find((r) => r.requestId === "outside")?.actor).toBeUndefined(); }); + // A card decided by voice on a Live call says so, on the row and in the + // export: "yes" heard on a call is a weaker signal than a tap. + it("marks a person's decision made by voice on a Live call", async () => { + await withDecisionActor({ kind: "loopback" }, async () => { + appendDecision(dir, row({ requestId: "spoken", decision: "user-approved", source: "user" })); + appendDecision(dir, row({ requestId: "rule", decision: "auto-approved", source: "full-access" })); + }, "call"); + await withDecisionActor({ kind: "loopback" }, async () => { + appendDecision(dir, row({ requestId: "tapped", decision: "user-approved", source: "user" })); + }); + await flushDecisionLog(dir); + const rows = readDecisions(dir, 10); + expect(rows.find((r) => r.requestId === "spoken")).toMatchObject({ actor: { kind: "loopback" }, via: "call" }); + expect(rows.find((r) => r.requestId === "rule")?.via).toBeUndefined(); + expect(rows.find((r) => r.requestId === "tapped")?.via).toBeUndefined(); + const csv = decisionsCsv(rows); + expect(csv).toContain("This computer (by voice)"); + expect(csv.split("\n").filter((line) => line.includes("(by voice)"))).toHaveLength(1); + }); + it("exports a date range as CSV with formula cells neutralised and secrets still redacted", async () => { const legacy = { at: "2026-02-10T10:00:00.000Z", threadId: "t1", requestId: "r1", botName: "=HYPERLINK(\"https://evil\")", tool: "Bash", summary: "export STRIPE_API_KEY=sk-live-abcdefghijklmnop1234", decision: "user-approved", source: "user", diff --git a/server/decision-log.ts b/server/decision-log.ts index 0cee4fbb74..f82a13bb01 100644 --- a/server/decision-log.ts +++ b/server/decision-log.ts @@ -97,6 +97,8 @@ export interface DecisionRow { unattended?: boolean; /** who answered, on rows a person's answer produced */ actor?: DecisionActor; + /** "call": that person answered by voice on a Live call, not with a tap */ + via?: "call"; /** how the ask reached the fold: a tool call (absent) or a block parsed * out of model-authored output ("output", the BoatAgent transport). * Question cards only. */ @@ -135,12 +137,13 @@ export function decisionRetentionDays(configured: number | undefined, env: NodeJ return DEFAULT_DECISION_RETENTION_DAYS; } -const actorScope = new AsyncLocalStorage(); +const actorScope = new AsyncLocalStorage<{ actor: DecisionActor; via?: "call" }>(); /** Run `work` as a person's card answer: every `source: "user"` row it - * writes names `actor`, without each resolver having to thread it through. */ -export function withDecisionActor(actor: DecisionActor, work: () => T): T { - return actorScope.run(actor, work); + * writes names `actor` (and `via`, for an answer spoken on a Live call), + * without each resolver having to thread it through. */ +export function withDecisionActor(actor: DecisionActor, work: () => T, via?: "call"): T { + return actorScope.run(via ? { actor, via } : { actor }, work); } function monthKey(at: Date): string { @@ -165,10 +168,12 @@ async function writeDecision(dataDir: string, record: DecisionRow): Promise): void { - const actor = row.actor ?? (row.source === "user" ? actorScope.getStore() : undefined); + const scope = row.source === "user" ? actorScope.getStore() : undefined; + const actor = row.actor ?? scope?.actor; + const via = row.via ?? scope?.via; // Redact now, not when the queue drains: the row is what was true at the // moment of the decision. - const record = redactSecrets({ at: new Date().toISOString(), ...row, ...(actor ? { actor } : {}) }) as DecisionRow; + const record = redactSecrets({ at: new Date().toISOString(), ...row, ...(actor ? { actor } : {}), ...(via ? { via } : {}) }) as DecisionRow; const previous = writeQueues.get(dataDir) ?? Promise.resolve(); // Serialize appends (and the occasional prune) per directory, so two // simultaneous approvals keep their decision order. @@ -330,7 +335,7 @@ export function decisionsCsv(rows: DecisionRow[]): string { row.summary ?? "", row.rule ?? "", row.unattended ? "yes" : "", - actorLabel(row.actor), + row.via === "call" ? `${actorLabel(row.actor)} (by voice)` : actorLabel(row.actor), row.threadId, row.requestId ?? "", ].map(csvCell).join(",")); diff --git a/server/index.ts b/server/index.ts index 79b472e637..7bd1878b5c 100644 --- a/server/index.ts +++ b/server/index.ts @@ -107,6 +107,7 @@ import { isEffortLevel, type BotVisibility, type CardAnswerer, type ResolvedSend import type { TeamComputersPayload } from "../shared/team-computer.ts"; import { boatCreateRecoverySnapshot, retireDeletedBoatCreate } from "./boat-create-idempotency.ts"; import { boatDeletionSnapshot } from "./boat-delete-journal.ts"; +import { liveDecisionRefusal } from "../shared/live-approval.ts"; import { boatAccountResourceChangeError, cloudBackendChangeError, @@ -186,6 +187,7 @@ import { roomHandoffLimits, onConfigSaved, CLAUDE_API_INSTANCE, + liveSettingsFor, } from "./config.ts"; import { sweepThreadEventLogs, type ThreadLogRetentionCandidate } from "./thread-retention.ts"; import { ComputerControl } from "./computer-control.ts"; @@ -264,6 +266,7 @@ import { drainSteeredMessages, hasQueuedSteeredMessages, holdSteeredQueue, + isSteeredMessageQueued, onSteeredQueueChange, queuedSteerSnapshot, queuedSteeredMessage, @@ -328,7 +331,10 @@ import { import * as tts from "./tts/index.ts"; import { createDecider, deciderIncludedHere, deciderReady, deciderSavePatch, describeDecider } from "./decider/index.ts"; import { decideRoomResponder, type RoomRoutingInput } from "./decider/room-routing.ts"; +import { createLiveSession, liveAttachUrl, LiveSessionError, type LiveBot, type LiveHistoryMessage } from "./live-call.ts"; +import { LiveCallController, LiveCallSignedOutError, type LiveSocket } from "./live-call-controller.ts"; import { narrateTool, toUtterances } from "./tts/speech-text.ts"; +import { turnStartLogLine } from "./turn-log.ts"; import { buildRecoveryText, buildTurnContext, engineIsFresh, NATIVELY_REPLAYING_DRIVER_KINDS, peerMessageText } from "./turn-context.ts"; import { Handoffs, handedStateUsable, recordHanded, renderUnseen, sessionStart, unseenMessages, withUnseenMessages, type ContextMessage } from "./delta-context.ts"; import { extractTurnImages } from "./turn-images.ts"; @@ -583,6 +589,7 @@ import { createDesktopViewer, desktopViewerUrl } from "./routes/desktop-viewer.t import { localDesktopTarget, localVmViewerStatus, viewerTargetId } from "./desktop-viewer-targets.ts"; import { createAntigravityLeftoverRoutes } from "./routes/antigravity-leftovers.ts"; import { findAntigravityLeftovers, removeAntigravityLeftovers } from "./drivers/antigravity-temp.ts"; +import { createLiveRoutes } from "./routes/live.ts"; const PORT = Number(process.env.OMB_PORT || process.env.OGB_PORT || 8799); const WEBHOOK_PORT = Number(process.env.OMB_WEBHOOK_PORT || PORT + 1); @@ -1335,21 +1342,22 @@ function cloudCardAnswerer(card: { requestId?: string; answeredBy?: { kind: stri } /** Answer a card as `auth`: the decision rows written meanwhile name the - * answerer, and a card this answer settled records who settled it. A card - * that was already settled keeps whatever it said. */ -async function answeringCardAs(auth: RequestAuth, threadId: string, requestId: string, work: () => Promise): Promise { + * answerer, and a card this answer settled records who settled it (and + * `via: "call"` when it was decided by voice on a Live call). A card that + * was already settled keeps whatever it said. */ +async function answeringCardAs(auth: RequestAuth, threadId: string, requestId: string, work: () => Promise, via?: "call"): Promise { const open = (() => { const card = store.messagesFor(threadId).find((message) => message.card?.requestId === requestId)?.card; return Boolean(card && !card.answered && !card.dismissed && !card.expired); })(); if (CLOUD_HOME && open) cloudCardAnswersInFlight.set(requestId, auth.kind === "session" ? actorKey(auth) : undefined); try { - await withDecisionActor(decisionActorFor(auth), work); + await withDecisionActor(decisionActorFor(auth), work, via); } finally { const message = open ? store.messagesFor(threadId).find((candidate) => candidate.card?.requestId === requestId) : undefined; const card = message?.card; if (message && card && !card.answeredBy && card.answered !== "unavailable" && (card.answered || card.dismissed)) { - store.patchMessage(threadId, message.id, { card: { ...card, answeredBy: cardAnswererFor(auth) } }); + store.patchMessage(threadId, message.id, { card: { ...card, answeredBy: { ...cardAnswererFor(auth), ...(via ? { via } : {}) } } }); } if (CLOUD_HOME && open) cloudCardAnswersInFlight.delete(requestId); } @@ -5639,6 +5647,48 @@ async function answerRequest( return outcome; } +type CardRespondResult = { ok: true } | { ok: false; status: number; error: string }; + +/** A decision or answer on a provider/peer card, made on behalf of `auth` + * (a Live call answers as the person who started the call). Harness-native + * proposals (skill, routine, profile, default model, tightening, team setup) + * are not handled here; they are reviewed on screen (liveDecisionRefusal). + * Mirrors POST /api/threads/:id/respond. */ +async function respondToCard(input: { + auth: RequestAuth; + threadId: string; + requestId: string; + behavior: "allow" | "deny" | "answer"; + message?: string; +}): Promise { + const { auth, threadId, requestId, behavior, message } = input; + // A call outlives the request that started it: the session must still be valid. + if (auth.kind === "session" && !sessions.isLive(auth.session.id)) { + return { ok: false, status: 401, error: "The session that started this call has ended." }; + } + const refusal = cardAnswerRefusal(auth, threadId, requestId, behavior); + if (refusal) return { ok: false, status: 403, error: refusal }; + const bot = store.botByThread(threadId); + if (!bot) return { ok: false, status: 404, error: "The chat is gone." }; + const card = store.messagesFor(threadId).find((candidate) => candidate.card?.requestId === requestId)?.card; + // Refused before anything runs: answerRequest would close a proposal it + // cannot deliver as "unavailable", and would add a false "not run" line to + // a card someone settled on screen a moment ago. + const refused = liveDecisionRefusal(card); + if (refused) return { ok: false, status: 409, error: refused }; + let result: CardRespondResult = { ok: true }; + await answeringCardAs(auth, threadId, requestId, async () => { + // peer-approval intercept, as the route: only a card on this thread + if (card && resolvePeerComms(approvalBus, requestId, behavior)) return; + const outcome = await answerRequest( + threadId, botForThread(bot.id, threadId)?.modelSelection.instanceId ?? "", requestId, behavior, message, + { id: bot.id, name: bot.name }, + ); + if (outcome === "unavailable") result = { ok: false, status: 409, error: "The request is no longer open." }; + }, "call"); + return result; +} + /** Close every provider-owned approval still open on a thread. Interrupting a * turn kills the process that raised its questions, so those cards can never * be answered. Routine proposals are harness-owned and durable, so they stay @@ -8469,7 +8519,7 @@ function drainAsideLane() { /** Keep a person's words off the transcript until a direct-thread slot is * available. Reuse the existing cancellable, idempotent composer queue. */ -async function startOrQueueDirectMessage(botId: string, threadId: string, text: string, replyTo?: Message, sendId?: string, sender?: ResolvedSender, trigger?: UsageTrigger) { +async function startOrQueueDirectMessage(botId: string, threadId: string, text: string, replyTo?: Message, sendId?: string, sender?: ResolvedSender, trigger?: UsageTrigger, via?: "call") { const decision = admit("direct", {}, { // A room turn holds the bot exactly like the sibling opened-thread queue // below: the drain's own block check waits it out, so the words queue @@ -8487,10 +8537,11 @@ async function startOrQueueDirectMessage(botId: string, threadId: string, text: prompt: promptWithReply(text, replyTo, cfg.profile?.name?.trim() || "User"), sender, trigger, + via, }); return { ok: true as const, queued: true as const, queueId: queued.id, threadId, reason: decision.reason }; } - const message = await startTurn(botId, text, { threadId, replyTo, sendId, sender, trigger }); + const message = await startTurn(botId, text, { threadId, replyTo, sendId, sender, trigger, via }); return { ok: true as const, threadId, message }; } @@ -8528,7 +8579,7 @@ function directSendRefusal(botId: string, threadId: string): DirectSendRefused | /** The one path a person's direct message takes into a bot turn: spend cap, * idempotency, steer into a running turn, queue, or start. Used by - * POST /api/bots/:id/messages and its guarded variant. + * POST /api/bots/:id/messages and by Live calls (via "call"). * `guardedStart` is POST /api/bots/:id/messages/guarded's own start, in * place of steer, queue or start: it checks its preconditions against the * task as it stands, then starts a turn or refuses. */ @@ -8541,13 +8592,14 @@ async function acceptDirectSend( replyTo?: Message; sender?: ResolvedSender; trigger: UsageTrigger; + via?: "call"; /** A person is proven present (a paired session, or the desktop's owner * capability): steering their words in clears the unattended mark. */ personPresent: boolean; }, guardedStart?: (currentAtStart: BotRecord) => Promise, ): Promise { - const { botId, threadId, text, sendId, replyTo, sender, trigger, personPresent } = input; + const { botId, threadId, text, sendId, replyTo, sender, trigger, via, personPresent } = input; const refused = directSendRefusal(botId, threadId); if (refused) throw refused; return sendSequencer.run( @@ -8646,13 +8698,14 @@ async function acceptDirectSend( sendId, steered: true, sender, + ...(via ? { via } : {}), }); // Offered to the next turn again unless the person stops this one. handoffs.steered(threadId, steerTarget, instance?.instanceId, message.id); return { ok: true as const, steered: true as const, threadId, message }; } if (!current.busy) { - return startOrQueueDirectMessage(botId, threadId, text, replyTo, sendId, sender, trigger); + return startOrQueueDirectMessage(botId, threadId, text, replyTo, sendId, sender, trigger, via); } const queued = queueSteeredMessage(current.id, threadId, text, { replyToId: replyTo?.id, @@ -8660,10 +8713,11 @@ async function acceptDirectSend( prompt: promptWithReply(text, replyTo, cfg.profile?.name?.trim() || "User"), sender, trigger, + via, }); return { ok: true as const, queued: true as const, queueId: queued.id, threadId }; } - return startOrQueueDirectMessage(botId, threadId, text, replyTo, sendId, sender, trigger); + return startOrQueueDirectMessage(botId, threadId, text, replyTo, sendId, sender, trigger, via); }, ); } @@ -8941,6 +8995,11 @@ async function startTurn( /** Stable identity supplied by the composer so a network retry cannot * dispatch the same user action twice. */ sendId?: string; + /** The words were spoken in a Live call, not typed. */ + via?: "call"; + /** An external interface relayed the words through the guarded send + * route (Message.relayed): nobody typed them in a client here. */ + relayed?: boolean; onDispatchError?: (message: string) => void; /** Summarize this conversation without asking the agent to do more work. */ compactOnly?: boolean; @@ -9071,7 +9130,14 @@ async function startTurn( } } - console.error(`[omb-turn] bot=${botId} text=${JSON.stringify(resolvedImages.text.slice(0, 70))} images=${turnImages.length} depth=${commsDepth} card=${Boolean(opts?.cardContinuation)}`); + // Spoken words never reach the log. A direct send says so itself; a + // drained queue (or a continuation) carries it on the user lines it runs. + const turnLineIds = new Set(opts?.excludeMessageIds ?? []); + const spoken = opts?.via === "call" || opts?.userMessage?.via === "call" || + (turnLineIds.size > 0 && store.messagesFor(threadId).some((message) => turnLineIds.has(message.id) && message.via === "call")); + console.error(turnStartLogLine({ + botId, text: resolvedImages.text, images: turnImages.length, depth: commsDepth, card: Boolean(opts?.cardContinuation), spoken, + })); const instanceId = instance.instanceId; if (providerInstancesChanging.has(instanceId)) { throw Object.assign(new Error("this provider account is being updated — try again shortly"), { status: 409 }); @@ -9112,6 +9178,8 @@ async function startTurn( sendId: opts?.sendId, peerAsk: opts?.peerAsk, sender: opts?.sender, + ...(opts?.via ? { via: opts.via } : {}), + ...(opts?.relayed ? { relayed: true } : {}), }); } const recoveryUserMessageId = opts?.coordination @@ -14377,6 +14445,8 @@ function configStatus() { // the decision model: switches and configured-or-not, never the key decider: describeDecider(cfg), imageGen: avatarImageStatus(cfg), + // Live calls: configured-or-not only; the voice name is a setting + live: liveSettingsFor(cfg), // not a secret — the sidebar shows it profile: { name: cfg.profile?.name ?? "", email: cfg.profile?.email ?? "", aboutMe: cfg.profile?.aboutMe ?? "" }, // the enrolled organisation's read-only desktop policy; null when not enrolled @@ -14840,6 +14910,94 @@ ROUTES.push(createAntigravityLeftoverRoutes({ ROUTES.push(desktopViewer.route); +// Live calls (GPT-Live as the voice, the bot as the brain). A client holds +// the WebRTC audio; the harness creates the session with the key (which +// never leaves it) and runs the call in LiveCallController. +/** Who the voice speaks for, for its instructions. */ +function liveBotFor(botId: string): LiveBot { + const bot = store.bot(botId); + if (!bot) throw new LiveSessionError("That bot no longer exists.", 404); + return { name: bot.name, title: bot.title, description: bot.description }; +} +/** The chat's text so far, so the voice can follow "and the other one?". */ +function liveHistoryFor(threadId: string): LiveHistoryMessage[] { + return store.activePath(threadId) + .filter((message) => message.kind === "text" && typeof message.text === "string" && (message.role === "user" || message.role === "bot")) + .map((message) => ({ role: message.role === "user" ? "user" as const : "assistant" as const, text: message.text ?? "" })); +} +/** A call outlives the request that started it: a signed-out or removed + * person's call must not keep reaching the bot (or spending the key). */ +function liveSignedIn(auth: RequestAuth): boolean { + return auth.kind !== "session" || sessions.isLive(auth.session.id); +} +const liveCalls = new LiveCallController({ + store, + send: async ({ auth, botId, threadId, text }) => { + // the controller ends the call on this error + if (!liveSignedIn(auth)) throw new LiveCallSignedOutError(); + const receipt = await acceptDirectSend({ + botId, threadId, text, + sendId: randomUUID().replaceAll("-", ""), + sender: messageSender(auth), + trigger: usageTriggerFor(auth), + via: "call", + // a person is on the call: these are their words + personPresent: true, + }); + if ("queued" in receipt) return { kind: "queued", queueId: receipt.queueId }; + return { kind: receipt.steered ? "steered" : "started", messageId: receipt.message.id }; + }, + respond: async (input) => { + const result = await respondToCard(input); + if (result.ok) return { ok: true }; + // respondToCard's 401: the session that started the call has ended + if (result.status === 401) throw new LiveCallSignedOutError(); + return { ok: false, error: result.error }; + }, + // send() queues through the steer queue; an edit or cancel there removes a request undelivered + queued: isSteeredMessageQueued, + signedIn: liveSignedIn, + // the caller's own typed lines carry this sender (none for the owner) + personKey: (auth) => messageSender(auth)?.id, + activity: (botId, threadId) => { + const task = store.taskByThread(botId, threadId); + if (!task?.busy) return "idle"; + return task.activity === "waiting-on-you" ? "waiting" : "working"; + }, + broadcast: (frame) => broadcast(frame, { adminOnly: true }), + settings: () => ({ key: cfg.live?.key ?? "", ...liveSettingsFor(cfg) }), + createSession: ({ key, sdp, botId, threadId, voice }) => createLiveSession({ key, sdp, voice, bot: liveBotFor(botId), history: liveHistoryFor(threadId) }), + // Node's WebSocket (undici) accepts headers in its second argument. + openSocket: (url, key) => new WebSocket(url, { headers: { authorization: `Bearer ${key}` } } as unknown as string[]) as unknown as LiveSocket, + attachUrl: (sessionId) => liveAttachUrl(sessionId), + speakable: (text) => toUtterances(text), + log: (line) => console.log(line), +}); +// A signed-out or revoked sign-in ends the call it started at once (the idle +// check would only notice within 15 s). A paired phone's unpairing arrives +// from the companion instead (POST /api/live/device-revoked). +sessions.onSessionRevoked((sessionId) => liveCalls.sessionRevoked(sessionId)); +ROUTES.push(createLiveRoutes({ + calls: liveCalls, + resolveTarget: (botId, threadId) => { + const bot = store.bot(botId); + if (!bot) return null; + const target = threadId ?? bot.threadId; + if (store.botByThread(target)?.id !== bot.id) return null; + return { botId: bot.id, botName: bot.name, threadId: target }; + }, + settings: () => liveSettingsFor(cfg), + // Non-secret settings only, written the way PUT /api/config writes a + // section: saveConfig merges into `live`, so the key stays where it is. + saveSettings: async (patch) => { + if (providerConfigBusy) throw Object.assign(new Error("Settings are already being updated. Try again in a moment."), { status: 409 }); + saveConfig({ live: patch }); + Object.assign(cfg, loadConfig()); + broadcast({ kind: "config", ...configStatus() }); + return liveSettingsFor(cfg); + }, +})); + const toolResults = new ToolResults(); const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { let url: URL; @@ -20840,7 +20998,9 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (guardedAdmission.action === "refuse") { throw Object.assign(new Error("wait for a free thread slot before retrying this message"), { status: 409, code: "guarded_busy" }); } - const message = await startTurn(bot.id, text, { threadId, replyTo, sendId, sender, trigger }); + // Stored as relayed: a worker's line for someone else, which a + // Live call on this thread must not read back as typed there. + const message = await startTurn(bot.id, text, { threadId, replyTo, sendId, sender, trigger, relayed: true }); return { ok: true as const, threadId, message }; } : undefined; @@ -20930,6 +21090,7 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { peerAsk: item.peerAsk, steered: true, sender: item.sender, + ...(item.via ? { via: item.via } : {}), })); // Offered to the next turn again unless the person stops this one. for (const message of messages) handoffs.steered(bot.threadId, steerTarget, instance?.instanceId, message.id); @@ -23100,6 +23261,7 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (persisted.decider?.key !== undefined) persisted.decider.key = ""; if (persisted.imageGen?.key !== undefined) persisted.imageGen.key = ""; if (persisted.imageGen?.customApiKey !== undefined) persisted.imageGen.customApiKey = ""; + if (persisted.live?.key !== undefined) persisted.live.key = ""; saveConfig(persisted); configWriteCommitted = true; syncCredentialEnv(patch); @@ -23804,6 +23966,7 @@ for (const row of chatFollowups()) { role: "user", kind: "text", text: row.kind === "aside" ? row.payload.prompt ?? row.payload.text : row.payload.text, replyToId: row.payload.replyToId, sendId: row.payload.sendId, queueId: row.id, sender: row.payload.sender, ...(row.kind === "channel" ? { channelMode: row.payload.mode, via: row.payload.via } : {}), + ...(row.kind === "bot" && row.payload.via === "call" ? { via: "call" as const } : {}), ...(row.kind === "aside" ? { aside: true, peerAsk: row.payload.aside @@ -23914,6 +24077,7 @@ const gracefulShutdown = createGracefulShutdown({ await Promise.all([...temporaryBrowserSessions.keys()].map((botId) => forgetTemporaryBrowser(botId))); await browserRuntime.closeAll(); }, + () => liveCalls.shutdown(), () => flushAllProfileHistory(), () => flushAllMemoryJournals(), () => flushUsageLedger(DATA_DIR), diff --git a/server/live-call-controller.test.ts b/server/live-call-controller.test.ts new file mode 100644 index 0000000000..5899de8438 --- /dev/null +++ b/server/live-call-controller.test.ts @@ -0,0 +1,1058 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { LiveCallState } from "../shared/wire.ts"; +import { + ATTACH_TIMEOUT_MS, CLOSE_TIMEOUT_MS, CONSENT_SETTLE_MS, DELEGATION_SETTLE_MS, IDLE_CHECK_MS, PROGRESS_INTERVAL_MS, + LiveCallBusyError, LiveCallController, LiveCallSignedOutError, type LiveActivity, type LiveCallDeps, type LiveSocket, +} from "./live-call-controller.ts"; +import { LIVE_COPY } from "../shared/live-approval.ts"; +import { LiveSessionError } from "./live-call.ts"; +import type { RequestAuth } from "./request-auth.ts"; +import type { Message, StoreChange } from "./store.ts"; + +class FakeSocket implements LiveSocket { + readyState = 0; + url: string; + key: string; + sent: Array> = []; + onopen: ((event: unknown) => void) | null = null; + onmessage: ((event: { data: unknown }) => void) | null = null; + onclose: ((event: unknown) => void) | null = null; + onerror: ((event: unknown) => void) | null = null; + constructor(url: string, key: string) { this.url = url; this.key = key; } + send(data: string) { this.sent.push(JSON.parse(data) as Record); } + close() { if (this.readyState === 3) return; this.readyState = 3; this.onclose?.({}); } + open() { this.readyState = 1; this.onopen?.({}); } + receive(event: Record) { this.onmessage?.({ data: JSON.stringify(event) }); } + refuse() { this.onerror?.({}); this.readyState = 3; } + drop() { this.readyState = 3; this.onclose?.({}); } + appends(kind: string) { return this.sent.filter((e) => e.type === `session.${kind}.append`); } +} + +const owner: RequestAuth = { kind: "loopback", scopes: ["admin", "client"] }; +const BOT = { botId: "bot1", botName: "Ada", threadId: "t1" }; + +function setup(overrides: Partial = {}) { + const listeners = new Set<(change: StoreChange) => void>(); + const sockets: FakeSocket[] = []; + const frames: Array = []; + const logs: string[] = []; + let activity: LiveActivity = "idle"; + const settings = { key: "sk-test", voice: "sol", readTypedReplies: true, idleMinutes: 5 }; + /** the thread's queue: ids of sends still waiting there */ + const queue = new Set(); + let messageCounter = 0; + const deps: LiveCallDeps = { + store: { onChange: (listener) => { listeners.add(listener); return () => listeners.delete(listener); } }, + send: vi.fn(async () => ({ kind: "started" as const, messageId: `m${++messageCounter}` })), + respond: vi.fn(async () => ({ ok: true as const })), + queued: vi.fn((_botId: string, _threadId: string, queueId: string) => queue.has(queueId)), + activity: () => activity, + broadcast: (frame) => frames.push(frame.call), + settings: () => settings, + createSession: vi.fn(async () => ({ sessionId: "sess_1", sdp: "answer-sdp" })), + openSocket: (url, key) => { const socket = new FakeSocket(url, key); sockets.push(socket); return socket; }, + attachUrl: (id) => `ws://fake/${id}/attach`, + speakable: (text) => text.split(/(?<=[.!?])\s+/).filter(Boolean), + log: (line) => logs.push(line), + now: () => Date.now(), + ...overrides, + }; + const controller = new LiveCallController(deps); + const emit = (change: StoreChange) => { for (const listener of Array.from(listeners)) listener(change); }; + const message = (m: Partial & { id: string }) => emit({ type: "message", threadId: "t1", message: { role: "bot", kind: "text", ...m } as Message }); + const patch = (m: Partial & { id: string }) => emit({ type: "message.patch", threadId: "t1", message: { role: "bot", kind: "text", ...m } as Message }); + const setActivity = (next: LiveActivity) => { activity = next; emit({ type: "bot", botId: "bot1" }); }; + const start = async () => { + const result = await controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "offer-sdp" }); + sockets.at(-1)!.open(); + return result; + }; + return { controller, deps, sockets, frames, logs, settings, queue, listeners, emit, message, patch, setActivity, start, socket: () => sockets.at(-1)! }; +} + +beforeEach(() => vi.useFakeTimers()); +afterEach(() => vi.useRealTimers()); + +describe("LiveCallController lifecycle", () => { + it("creates the session with the key and voice, attaches the sideband and goes live", async () => { + const t = setup(); + const result = await t.controller.start({ auth: owner, ...BOT, client: "ios", sdp: "offer-sdp" }); + expect(t.deps.createSession).toHaveBeenCalledWith({ key: "sk-test", sdp: "offer-sdp", botId: "bot1", threadId: "t1", voice: "sol" }); + expect(result.sdp).toBe("answer-sdp"); + expect(result.call).toMatchObject({ botId: "bot1", threadId: "t1", client: "ios", voice: "sol", status: "connecting" }); + expect(t.socket().url).toBe("ws://fake/sess_1/attach"); + expect(t.socket().key).toBe("sk-test"); + t.socket().open(); + expect(t.controller.current()?.status).toBe("live"); + expect(t.frames.map((f) => f?.status)).toEqual(["connecting", "live"]); + }); + + it("refuses a second call while one is active, before creating a session", async () => { + const t = setup(); + const first = t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" }); + await expect(t.controller.start({ auth: owner, ...BOT, client: "ios", sdp: "b" })).rejects.toBeInstanceOf(LiveCallBusyError); + await first; + expect(t.deps.createSession).toHaveBeenCalledTimes(1); + }); + + it("asks for a key when none is set", async () => { + const t = setup(); + t.settings.key = " "; + await expect(t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" })).rejects.toMatchObject({ status: 409 }); + expect(t.deps.createSession).not.toHaveBeenCalled(); + }); + + it("frees the slot when OpenAI refuses the session", async () => { + const t = setup({ createSession: vi.fn().mockRejectedValueOnce(new LiveSessionError("nope", 502)).mockResolvedValue({ sessionId: "sess_2", sdp: "x" }) }); + await expect(t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" })).rejects.toBeInstanceOf(LiveSessionError); + expect(t.controller.current()).toBeNull(); + await expect(t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" })).resolves.toBeTruthy(); + }); + + it("ends with sideband-lost when attach is refused", async () => { + const t = setup(); + await t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" }); + t.socket().refuse(); + expect(t.controller.current()).toBeNull(); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "sideband-lost" }); + expect(t.logs.some((line) => line.startsWith("[live] call ended") && line.includes("end=sideband-lost"))).toBe(true); + }); + + it("ends with sideband-lost and leaves no timer when the sideband cannot be opened", async () => { + const t = setup({ openSocket: () => { throw new Error("bad url"); } }); + const result = await t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" }); + expect(result.call).toMatchObject({ status: "ended", endReason: "sideband-lost" }); + expect(t.controller.current()).toBeNull(); + expect(t.listeners.size).toBe(0); + expect(vi.getTimerCount()).toBe(0); + }); + + it("sends one session.close when asked to hang up twice", async () => { + const t = setup(); + const { call } = await t.start(); + const first = t.controller.end(call.callId); + const second = t.controller.end(call.callId); + expect(t.socket().sent.filter((e) => e.type === "session.close")).toHaveLength(1); + t.socket().receive({ type: "session.closed", reason: "close_requested" }); + await expect(first).resolves.toMatchObject({ status: "ended", endReason: "hung-up" }); + await expect(second).resolves.toMatchObject({ status: "ended", endReason: "hung-up" }); + }); + + it("ends with sideband-lost when attach never opens", async () => { + const t = setup(); + await t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" }); + await vi.advanceTimersByTimeAsync(ATTACH_TIMEOUT_MS + 1); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "sideband-lost" }); + }); + + it("ends with sideband-lost when the sideband drops mid-call", async () => { + const t = setup(); + await t.start(); + t.socket().drop(); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "sideband-lost" }); + }); + + it("hangs up gracefully: session.close, then session.closed", async () => { + const t = setup(); + const { call } = await t.start(); + const ending = t.controller.end(call.callId); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + expect(t.frames.at(-1)?.status).toBe("ending"); + t.socket().receive({ type: "session.closed", reason: "close_requested", usage: { seconds: 61 } }); + await expect(ending).resolves.toMatchObject({ status: "ended", endReason: "hung-up" }); + expect(t.logs.at(-1)).toContain("seconds=61"); + }); + + it("finishes a hang-up after 5 s without session.closed", async () => { + const t = setup(); + const { call } = await t.start(); + const ending = t.controller.end(call.callId); + await vi.advanceTimersByTimeAsync(CLOSE_TIMEOUT_MS + 1); + await expect(ending).resolves.toMatchObject({ status: "ended", endReason: "hung-up" }); + }); + + it("returns null when asked to end an unknown call", async () => { + const t = setup(); + await t.start(); + await expect(t.controller.end("other")).resolves.toBeNull(); + }); + + it("maps OpenAI's close reasons", async () => { + const t = setup(); + await t.start(); + t.socket().receive({ type: "session.closed", reason: "expired", usage: { seconds: 3600 } }); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "expired" }); + }); + + it("hangs up after the idle minutes without speech or work", async () => { + const t = setup(); + await t.start(); + await vi.advanceTimersByTimeAsync(4 * 60_000); + t.socket().receive({ type: "session.input_transcript.delta", delta: "hi", start_ms: 1, end_ms: 2 }); + await vi.advanceTimersByTimeAsync(4 * 60_000); + expect(t.socket().sent.some((e) => e.type === "session.close")).toBe(false); + await vi.advanceTimersByTimeAsync(60_000 + IDLE_CHECK_MS); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + t.socket().receive({ type: "session.closed", reason: "close_requested" }); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "idle" }); + }); + + it("does not hang up while the bot works, but does while an approval waits", async () => { + const t = setup(); + await t.start(); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(10 * 60_000); + expect(t.socket().sent.some((e) => e.type === "session.close")).toBe(false); + t.setActivity("waiting"); + await vi.advanceTimersByTimeAsync(5 * 60_000 + IDLE_CHECK_MS); + expect(t.socket().sent.some((e) => e.type === "session.close")).toBe(true); + }); + + it("ends the call when the thread is deleted", async () => { + const t = setup(); + await t.start(); + t.emit({ type: "thread.deleted", threadId: "t1" }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + await vi.advanceTimersByTimeAsync(CLOSE_TIMEOUT_MS + 1); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "deleted" }); + expect(t.listeners.size).toBe(0); + }); + + it("shutdown closes the session and clears every timer", async () => { + const t = setup(); + await t.start(); + await t.controller.shutdown(); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "shutdown" }); + expect(vi.getTimerCount()).toBe(0); + }); + + // Hung up (or the harness shut down) while OpenAI was creating the + // session: nobody will ever attach to that session, so the harness closes + // it instead of leaving it open until OpenAI gives up on it. + describe("a start cancelled while OpenAI creates the session", () => { + function pendingSession() { + let resolve!: (value: { sessionId: string; sdp: string }) => void; + const createSession = vi.fn(() => new Promise<{ sessionId: string; sdp: string }>((done) => { resolve = done; })); + return { createSession, answer: () => resolve({ sessionId: "sess_9", sdp: "answer-sdp" }) }; + } + + it("closes the session it created through the sideband", async () => { + const pending = pendingSession(); + const t = setup({ createSession: pending.createSession }); + const starting = t.controller.start({ auth: owner, ...BOT, client: "ios", sdp: "offer-sdp" }); + const callId = t.controller.current()!.callId; + await t.controller.end(callId); + pending.answer(); + await expect(starting).rejects.toMatchObject({ status: 503 }); + const orphan = t.socket(); + expect(orphan.url).toBe("ws://fake/sess_9/attach"); + expect(orphan.key).toBe("sk-test"); + orphan.open(); + expect(orphan.sent).toEqual([expect.objectContaining({ type: "session.close" })]); + expect(orphan.readyState).toBe(3); + expect(vi.getTimerCount()).toBe(0); + }); + + it("gives up on a sideband that never opens, leaving no timer behind", async () => { + const pending = pendingSession(); + const t = setup({ createSession: pending.createSession }); + const starting = t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "offer-sdp" }); + await t.controller.shutdown(); + pending.answer(); + await expect(starting).rejects.toMatchObject({ status: 503 }); + await vi.advanceTimersByTimeAsync(ATTACH_TIMEOUT_MS + 1); + expect(t.socket().readyState).toBe(3); + expect(t.socket().sent).toEqual([]); + expect(vi.getTimerCount()).toBe(0); + }); + }); + + it("frees the slot when setting up the call fails after the session exists", async () => { + let fail = true; + const t = setup({ activity: () => { if (fail) throw new Error("no such bot"); return "idle"; } }); + await expect(t.controller.start({ auth: owner, ...BOT, client: "desktop", sdp: "a" })).rejects.toThrow("no such bot"); + expect(t.controller.current()).toBeNull(); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "error" }); + expect(vi.getTimerCount()).toBe(0); + fail = false; + await expect(t.start()).resolves.toBeTruthy(); + }); + + it("survives a failing relay step or broadcast and counts it, without crashing the harness", async () => { + let failing = false; + const boom = (what: string) => { if (failing) throw new Error(what); }; + const t = setup({ + speakable: () => { throw new Error("tts prep broke"); }, + activity: () => { boom("no such bot"); return "idle"; }, + broadcast: () => boom("sse down"), + }); + await t.start(); + failing = true; + t.message({ id: "u1", role: "user", kind: "text", text: "hi", sendId: "s1" }); + t.patch({ id: "b1", text: "Hello.", requestMessageId: "u1", turnTerminal: true }); + t.emit({ type: "bot", botId: "bot1" }); + await vi.advanceTimersByTimeAsync(IDLE_CHECK_MS + 1); + expect(t.controller.current()?.status).toBe("live"); + t.socket().receive({ type: "session.closed", reason: "close_requested" }); + expect(t.controller.current()).toBeNull(); + expect(t.logs.at(-1)).toMatch(/errors=internal,internal,internal,broadcast$/); + }); + + // A phone's requests reach the harness as the computer's own (loopback), so + // the sign-in check never fires for them. The call is bound to the paired + // phone the companion vouches for instead, and ends when that phone is + // unpaired. + describe("a call bound to the phone or sign-in that started it", () => { + const fromPhone = (t: ReturnType, device = "phone-1") => + t.controller.start({ auth: owner, device, ...BOT, client: "ios", sdp: "offer-sdp" }); + + it("ends at once, and says why, when that phone is unpaired", async () => { + const t = setup(); + await fromPhone(t); + t.socket().open(); + const ending = t.controller.deviceRevoked("phone-1"); + expect(ending).toMatchObject({ status: "ending" }); + expect(t.socket().appends("commentary")).toEqual([expect.objectContaining({ content: LIVE_COPY.unpaired })]); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + t.socket().receive({ type: "session.closed", reason: "close_requested" }); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "signed-out", error: LIVE_COPY.unpaired }); + expect(t.logs.at(-1)).toContain("end=signed-out"); + }); + + it("ends a call that is still connecting to OpenAI", async () => { + const t = setup(); + await fromPhone(t); + t.controller.deviceRevoked("phone-1"); + expect(t.controller.current()).toBeNull(); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "signed-out" }); + }); + + it("leaves a call from another phone, or from the computer, alone", async () => { + const t = setup(); + await fromPhone(t, "phone-2"); + t.socket().open(); + expect(t.controller.deviceRevoked("phone-1")).toBeNull(); + expect(t.controller.current()?.status).toBe("live"); + await t.controller.shutdown(); + + const desktop = setup(); + await desktop.start(); + expect(desktop.controller.deviceRevoked("phone-1")).toBeNull(); + expect(desktop.controller.current()?.status).toBe("live"); + }); + + it("refuses a call from a phone that was already unpaired, before creating a session", async () => { + const t = setup(); + t.controller.deviceRevoked("phone-1"); + await expect(fromPhone(t)).rejects.toBeInstanceOf(LiveCallSignedOutError); + expect(t.deps.createSession).not.toHaveBeenCalled(); + await expect(fromPhone(t, "phone-2")).resolves.toBeTruthy(); + }); + + it("ends at once when the sign-in that started it is revoked", async () => { + const t = setup(); + const session = { id: "s1", tokenHash: "x".repeat(64), label: "Safari on Mac", scopes: ["admin" as const], createdAt: 0, lastSeenAt: 0, expiresAt: 0 }; + await t.controller.start({ auth: { kind: "session", session, via: "cookie", scopes: ["admin"] }, ...BOT, client: "desktop", sdp: "offer-sdp" }); + t.socket().open(); + expect(t.controller.sessionRevoked("s2")).toBeNull(); + expect(t.controller.sessionRevoked("s1")).toMatchObject({ status: "ending" }); + expect(t.socket().appends("commentary")).toEqual([expect.objectContaining({ content: LIVE_COPY.signedOut })]); + t.socket().receive({ type: "session.closed", reason: "close_requested" }); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "signed-out", error: LIVE_COPY.signedOut }); + }); + }); + + it("skips mirrored audio without parsing it and never logs speech", async () => { + const t = setup(); + await t.start(); + t.socket().onmessage?.({ data: `{"type":"session.input_audio.append","audio":"${"A".repeat(50_000)}"}` }); + t.socket().receive({ type: "session.input_transcript.delta", delta: "my secret plan", start_ms: 1, end_ms: 2 }); + await t.controller.shutdown(); + expect(t.logs.join("\n")).not.toContain("secret"); + }); +}); + +describe("LiveCallController relay", () => { + async function live(overrides: Partial = {}) { + const t = setup(overrides); + await t.start(); + return t; + } + const hear = (t: ReturnType, text: string, at: number, until = at + 100) => + t.socket().receive({ type: "session.input_transcript.delta", delta: text, start_ms: at, end_ms: until }); + const delegate = async (t: ReturnType, id: string, offset: number) => { + t.socket().receive({ type: "session.delegation.created", offset_ms: offset, delegation: { id, target: "client", type: "delegation" } }); + await vi.advanceTimersByTimeAsync(DELEGATION_SETTLE_MS + 1); + }; + + describe("when the sign-in that started the call ends", () => { + const signedOut = (t: ReturnType) => t.socket().appends("commentary").filter((e) => e.content === LIVE_COPY.signedOut); + + it("says so once and hangs up instead of retrying a request", async () => { + const t = await live({ send: vi.fn(async () => { throw new LiveCallSignedOutError(); }) }); + hear(t, "what is on my calendar", 100); + await delegate(t, "del_1", 400); + expect(signedOut(t)).toEqual([expect.objectContaining({ delegation_id: "del_1" })]); + expect(t.socket().appends("commentary").some((e) => String(e.content).includes("could not be sent"))).toBe(false); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + // a second request while the call ends is refused the same way, and not answered again + hear(t, "and tomorrow", 2_000); + await delegate(t, "del_2", 2_300); + expect(signedOut(t)).toHaveLength(1); + expect(t.socket().appends("commentary").some((e) => String(e.content).includes("could not be sent"))).toBe(false); + expect(t.socket().sent.filter((e) => e.type === "session.close")).toHaveLength(1); + t.socket().receive({ type: "session.closed", reason: "close_requested" }); + expect(t.frames.at(-1)).toMatchObject({ status: "ended", endReason: "signed-out", error: LIVE_COPY.signedOut }); + expect(t.logs.at(-1)).toContain("end=signed-out"); + expect(t.logs.at(-1)).toContain("errors=signed-out"); + }); + + it("hangs up when a spoken decision is refused for the same reason", async () => { + const t = await live({ respond: vi.fn(async () => { throw new LiveCallSignedOutError(); }) }); + t.message({ id: "c1", kind: "options", card: { title: "Approval needed", subtitle: "ls", options: ["Allow", "Deny"], requestId: "r1", tool: "Bash" } }); + await vi.advanceTimersByTimeAsync(0); + hear(t, "yes", 5_000); + await delegate(t, "del_2", 5_100); + expect(signedOut(t)).toEqual([expect.objectContaining({ delegation_id: "del_2" })]); + expect(t.socket().appends("commentary").some((e) => String(e.content).includes("could not be saved"))).toBe(false); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + }); + + it("hangs up when a spoken answer to a question is refused for the same reason", async () => { + const t = await live({ respond: vi.fn(async () => { throw new LiveCallSignedOutError(); }) }); + t.message({ id: "q1", kind: "options", card: { title: "A question", subtitle: "Which account?", options: [], requestId: "r7" } }); + await vi.advanceTimersByTimeAsync(0); + hear(t, "savings", 5_000); + await delegate(t, "del_2", 5_100); + expect(signedOut(t)).toHaveLength(1); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + }); + + it("hangs up at the next idle check even while the person keeps talking", async () => { + let signedIn = true; + const t = await live({ signedIn: () => signedIn }); + hear(t, "hello", 100); + await vi.advanceTimersByTimeAsync(IDLE_CHECK_MS); + expect(t.socket().sent.some((e) => e.type === "session.close")).toBe(false); + signedIn = false; + hear(t, "still here", 20_000); + await vi.advanceTimersByTimeAsync(IDLE_CHECK_MS); + expect(signedOut(t)).toEqual([expect.objectContaining({ delegation_id: null })]); + expect(t.socket().sent.at(-1)).toMatchObject({ type: "session.close" }); + }); + }); + + it("sends the words since the last request to the bot, as the call's starter", async () => { + const t = await live(); + hear(t, "what is on ", 100); + hear(t, "my calendar", 200); + await delegate(t, "del_1", 400); + expect(t.deps.send).toHaveBeenCalledWith({ auth: owner, botId: "bot1", threadId: "t1", text: "what is on my calendar" }); + expect(t.socket().appends("thinking").at(-1)).toMatchObject({ delegation_id: "del_1", content: expect.stringContaining("You are working on the request") }); + }); + + it("asks to repeat when nothing was heard", async () => { + const t = await live(); + await delegate(t, "del_1", 400); + expect(t.deps.send).not.toHaveBeenCalled(); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ delegation_id: "del_1", content: expect.stringContaining("not heard clearly") }); + }); + + it("ignores delegations aimed at OpenAI's own tools", async () => { + const t = await live(); + hear(t, "hello", 100); + t.socket().receive({ type: "session.delegation.created", offset_ms: 300, delegation: { id: "d", target: "responses" } }); + await vi.advanceTimersByTimeAsync(DELEGATION_SETTLE_MS + 1); + expect(t.deps.send).not.toHaveBeenCalled(); + }); + + it("speaks only the turn's final answer, in the delegation it answers", async () => { + const t = await live(); + hear(t, "check my mail", 100); + await delegate(t, "del_1", 300); + t.message({ id: "b1", text: "Let me look.", requestMessageId: "m1" }); + t.message({ id: "a1", kind: "activity", tool: { name: "gmail", spoken: "Reading your inbox" } }); + t.message({ id: "b2", text: "You have two new emails. One is from Sam.", requestMessageId: "m1" }); + t.patch({ id: "b2", text: "You have two new emails. One is from Sam.", requestMessageId: "m1", turnTerminal: true }); + t.patch({ id: "b2", text: "You have two new emails. One is from Sam.", requestMessageId: "m1", turnTerminal: true, turnSucceeded: true }); + await vi.advanceTimersByTimeAsync(0); + const commentary = t.socket().appends("commentary"); + expect(commentary).toHaveLength(1); + expect(commentary[0]).toMatchObject({ delegation_id: "del_1", content: "You have two new emails. One is from Sam." }); + expect(t.socket().appends("thinking").some((e) => String(e.content).includes("Reading your inbox"))).toBe(true); + expect(JSON.stringify(t.socket().sent)).not.toContain("Let me look."); + }); + + it("rate-limits progress to one every 4 s", async () => { + const t = await live(); + t.message({ id: "a1", kind: "activity", tool: { name: "x", spoken: "Step one" } }); + t.message({ id: "a2", kind: "activity", tool: { name: "x", spoken: "Step two" } }); + await vi.advanceTimersByTimeAsync(PROGRESS_INTERVAL_MS + 1); + t.message({ id: "a3", kind: "activity", tool: { name: "x", spoken: "Step three" } }); + await vi.advanceTimersByTimeAsync(0); + const progress = t.socket().appends("thinking").map((e) => String(e.content)); + expect(progress.filter((c) => c.startsWith("Progress:"))).toEqual(["Progress: Step one", "Progress: Step three"]); + }); + + it("says the result is in the chat when the turn ends without text", async () => { + const t = await live(); + hear(t, "do it", 100); + await delegate(t, "del_1", 300); + // store listeners run in a microtask: flush between the two changes + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("commentary").at(-1)).toMatchObject({ content: "I'm done. The result, or what went wrong, is in the chat." }); + }); + + // A request spoken while the bot works can wait in the thread's queue. It + // holds back "the result is in the chat" until it is delivered. Edited or + // cancelled in a client, it leaves the queue without ever arriving: from + // then on it must hold nothing back. + describe("a spoken request that waits in the queue", () => { + const noAnswer = (t: ReturnType) => t.socket().appends("commentary").filter((e) => e.content === LIVE_COPY.noAnswer); + /** The next request is queued, the way the harness queues a send. */ + const queueNext = (t: ReturnType, queueId: string) => + vi.mocked(t.deps.send).mockImplementationOnce(async () => { + t.queue.add(queueId); + return { kind: "queued", queueId }; + }); + + it("is answered once it is delivered, with no missing-answer line", async () => { + const t = await live(); + t.setActivity("working"); + queueNext(t, "q1"); + hear(t, "and then the weather", 100); + await delegate(t, "del_1", 300); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("commentary")).toHaveLength(0); + // the drain takes it out of the queue and onto the thread, and its turn answers + t.queue.delete("q1"); + t.message({ id: "m9", role: "user", kind: "text", text: "and then the weather", via: "call", queueId: "q1" }); + t.setActivity("working"); + t.patch({ id: "b9", text: "Sunny.", requestMessageId: "m9", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("commentary")).toEqual([expect.objectContaining({ delegation_id: "del_1", content: "Sunny." })]); + }); + + it("still holds the missing-answer line back while it waits", async () => { + const t = await live(); + hear(t, "do it", 100); + await delegate(t, "del_1", 300); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + queueNext(t, "q1"); + hear(t, "and then the weather", 2_000); + await delegate(t, "del_2", 2_200); + // the first request's turn ends without an answer; the second still waits + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(noAnswer(t)).toHaveLength(0); + expect(t.deps.queued).toHaveBeenCalledWith("bot1", "t1", "q1"); + }); + + // A drain delivers every waiting line at once, as one turn whose request + // is the last line. The spoken lines before it are answered by it too. + it("answers a drained batch of spoken lines once, without saying the result is in the chat", async () => { + const t = await live(); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + queueNext(t, "q1"); + hear(t, "what is the weather", 100); + await delegate(t, "del_1", 300); + queueNext(t, "q2"); + hear(t, "and tomorrow", 2_000); + await delegate(t, "del_2", 2_200); + t.queue.clear(); + t.message({ id: "m1", role: "user", kind: "text", text: "what is the weather", via: "call", queueId: "q1" }); + t.message({ id: "m2", role: "user", kind: "text", text: "and tomorrow", via: "call", queueId: "q2" }); + t.patch({ id: "b1", text: "Sunny today and tomorrow.", requestMessageId: "m2", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("commentary")).toEqual([expect.objectContaining({ delegation_id: "del_2", content: "Sunny today and tomorrow." })]); + }); + + it("answers aloud a drained batch that holds a spoken line, even when a typed line came last", async () => { + const t = await live(); + t.settings.readTypedReplies = false; + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + queueNext(t, "q1"); + hear(t, "what is the weather", 100); + await delegate(t, "del_1", 300); + t.queue.clear(); + t.message({ id: "m1", role: "user", kind: "text", text: "what is the weather", via: "call", queueId: "q1" }); + t.message({ id: "m2", role: "user", kind: "text", text: "in Utrecht", sendId: "s2", queueId: "q2" }); + t.patch({ id: "b1", text: "Sunny in Utrecht.", requestMessageId: "m2", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("commentary")).toEqual([expect.objectContaining({ delegation_id: "del_1", content: "Sunny in Utrecht." })]); + }); + + it("holds nothing back once it left the queue without being delivered", async () => { + const t = await live(); + t.setActivity("working"); + queueNext(t, "q1"); + hear(t, "and then the weather", 100); + await delegate(t, "del_1", 300); + // the person edits (or cancels) the waiting line: it leaves the queue and never arrives + t.queue.delete("q1"); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + // a later spoken request's turn ends without an answer + hear(t, "do it", 2_000); + await delegate(t, "del_2", 2_200); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(noAnswer(t)).toEqual([expect.objectContaining({ delegation_id: "del_2" })]); + }); + }); + + // Without facts about a long turn the voice guessed ("I see it is stuck"). + // While the bot works it now gets a quiet status note every 30 s. + describe("status notes while the bot works", () => { + const notes = (t: ReturnType) => + t.socket().appends("thinking").filter((e) => String(e.content).startsWith("Status note")); + + it("tells the voice every 30 s how long the bot has worked and what its last step was", async () => { + const t = await live(); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + t.message({ id: "a1", kind: "activity", tool: { name: "mcp__team-notes__describe_database", summary: "Admin/Instruction" } }); + t.message({ id: "a2", kind: "activity", tool: { name: "mcp__team-notes__query_database", summary: "SELECT secrets FROM /Users/someone" } }); + await vi.advanceTimersByTimeAsync(29_000); + expect(notes(t)).toHaveLength(0); + await vi.advanceTimersByTimeAsync(2_000); + expect(notes(t)).toHaveLength(1); + const note = String(notes(t)[0].content); + expect(note).toContain("you are still working on it"); + expect(note).toContain("2 steps"); + expect(note).toContain("team notes: query database"); + expect(note).not.toMatch(/secrets|\/Users/); + expect(notes(t)[0].delegation_id).toBeNull(); + await vi.advanceTimersByTimeAsync(15_000); + expect(notes(t)).toHaveLength(1); + await vi.advanceTimersByTimeAsync(15_000); + expect(notes(t)).toHaveLength(2); + expect(String(notes(t)[1].content)).toContain("1 minute"); + }); + + it("stops when the bot is done, and sends none while it waits for an answer", async () => { + const t = await live(); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(31_000); + expect(notes(t)).toHaveLength(1); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(90_000); + expect(notes(t)).toHaveLength(1); + t.setActivity("waiting"); + await vi.advanceTimersByTimeAsync(0); + await vi.advanceTimersByTimeAsync(90_000); + expect(notes(t)).toHaveLength(1); + }); + + it("counts a new piece of work from zero", async () => { + const t = await live(); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + t.message({ id: "a1", kind: "activity", tool: { name: "Bash" } }); + await vi.advanceTimersByTimeAsync(31_000); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + t.setActivity("working"); + // notes ride on the 15 s idle tick: the next one lands 30–45 s after the work began + await vi.advanceTimersByTimeAsync(46_000); + const last = String(notes(t).at(-1)?.content); + expect(last).toContain("0 steps"); + expect(last).not.toContain("run a command"); + }); + }); + + // A typed message goes to the bot, not to the voice. Mirroring it into the + // voice when it is typed made GPT-Live answer it at once, and then again + // when the bot's answer was relayed: the person heard two answers. + it("says nothing when a message is typed, then reads the bot's answer once, with what was typed", async () => { + const t = await live(); + const before = t.socket().sent.length; + t.message({ id: "u1", role: "user", kind: "text", text: "kun je mij verstaan", sendId: "s1" }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.length).toBe(before); + t.patch({ id: "b1", text: "Ja, ik kan je verstaan.", requestMessageId: "u1", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + const commentary = t.socket().appends("commentary"); + expect(commentary).toHaveLength(1); + expect(commentary[0]).toMatchObject({ delegation_id: null, content: expect.stringContaining("kun je mij verstaan") }); + expect(commentary[0].content).toContain("Ja, ik kan je verstaan."); + expect(t.socket().appends("thinking")).toHaveLength(0); + }); + + // "Read replies to typed messages" off means nothing about a typed + // exchange reaches OpenAI: not what was typed, not the answer, not the + // steps the bot took for it. + it("sends OpenAI nothing about a typed exchange when reading typed replies is off", async () => { + const t = await live(); + t.settings.readTypedReplies = false; + const before = t.socket().sent.length; + t.message({ id: "u1", role: "user", kind: "text", text: "also book a table", sendId: "s1" }); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + t.message({ id: "a1", kind: "activity", tool: { name: "mcp__resy__book", spoken: "Booking at Luigi's" } }); + await vi.advanceTimersByTimeAsync(31_000 + IDLE_CHECK_MS); + t.patch({ id: "b1", text: "Booked for eight.", requestMessageId: "u1", turnTerminal: true }); + t.setActivity("idle"); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.slice(before)).toEqual([]); + }); + + it("keeps telling the voice about work on a spoken request when typed replies are off", async () => { + const t = await live(); + t.settings.readTypedReplies = false; + hear(t, "check my mail", 100); + await delegate(t, "del_1", 300); + t.setActivity("working"); + await vi.advanceTimersByTimeAsync(0); + t.message({ id: "a1", kind: "activity", tool: { name: "gmail", spoken: "Reading your inbox" } }); + await vi.advanceTimersByTimeAsync(31_000 + IDLE_CHECK_MS); + const thinking = t.socket().appends("thinking").map((e) => String(e.content)); + expect(thinking).toContain("Progress: Reading your inbox"); + expect(thinking.some((c) => c.startsWith("Status note"))).toBe(true); + }); + + // A peer bot's words land as user-role lines too: an ask_bot request, or + // context from the aside lane folded into the running turn. The person did + // not type them, so neither they nor the bot's answer to them is relayed. + it("never relays a peer bot's line, or the answer to it, as something the person typed", async () => { + const t = await live(); + const before = t.socket().sent.length; + t.message({ id: "p1", role: "user", kind: "text", text: "[aside from Bo] the invoice is paid", aside: true, peerAsk: { botId: "bo", name: "Bo" } }); + t.message({ id: "p2", role: "user", kind: "text", text: "Bo asks: is the report ready?", peerAsk: { botId: "bo", name: "Bo" } }); + await vi.advanceTimersByTimeAsync(0); + t.patch({ id: "b1", text: "Noted, thanks.", requestMessageId: "p1", turnTerminal: true }); + t.patch({ id: "b2", text: "Yes, it is in the shared folder.", requestMessageId: "p2", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.length).toBe(before); + t.settings.readTypedReplies = false; + t.message({ id: "p3", role: "user", kind: "text", text: "Bo asks: and the invoice?", peerAsk: { botId: "bo", name: "Bo" } }); + await vi.advanceTimersByTimeAsync(0); + t.patch({ id: "b3", text: "Paid yesterday.", requestMessageId: "p3", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.length).toBe(before); + }); + + // Only what the caller typed in a client is "what you typed": not a + // member's line in a shared workspace, not a routine's or a webhook's + // line, not a line sent through the local API. + it("relays only a line the caller typed in a client", async () => { + const t = await live(); + const before = t.socket().sent.length; + const lines: Array> = [ + { sendId: "s1", sender: { name: "sam@example.test", id: "p_sam" } }, + {}, + { sendId: "s3", via: "api" }, + ]; + lines.forEach((line, index) => t.message({ id: `u${index}`, role: "user", kind: "text", text: `line ${index}`, ...line })); + await vi.advanceTimersByTimeAsync(0); + lines.forEach((_, index) => t.patch({ id: `b${index}`, text: `Answer ${index}.`, requestMessageId: `u${index}`, turnTerminal: true })); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.slice(before)).toEqual([]); + + t.message({ id: "u9", role: "user", kind: "text", text: "and mine", sendId: "s9" }); + await vi.advanceTimersByTimeAsync(0); + t.patch({ id: "b9", text: "Done.", requestMessageId: "u9", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("commentary")).toEqual([expect.objectContaining({ content: expect.stringContaining("and mine") })]); + }); + + // A worker (the Slack relay, or any send through the guarded route) posts + // for someone else as this computer: no sender, a sendId of its own, so it + // looked exactly like the owner's typed line. Its line is marked relayed, + // and neither it nor its answer is read back to the caller. + it("never reads a line a worker relayed into the call's chat back as the caller's own", async () => { + const t = await live(); + const before = t.socket().sent.length; + t.message({ id: "r1", role: "user", kind: "text", text: "Ada asks: is the report ready?", sendId: "slackjob_report_1", relayed: true }); + await vi.advanceTimersByTimeAsync(0); + t.patch({ id: "b1", text: "Yes, it is in the shared folder.", requestMessageId: "r1", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().sent.slice(before)).toEqual([]); + }); + + it("knows a signed-in caller's own lines by who sent them", async () => { + const t = setup({ personKey: (auth) => (auth.kind === "session" ? "p_me" : undefined) }); + const session = { id: "s1", tokenHash: "x".repeat(64), label: "Safari", scopes: ["admin" as const], createdAt: 0, lastSeenAt: 0, expiresAt: 0 }; + await t.controller.start({ auth: { kind: "session", session, via: "cookie", scopes: ["admin"] }, ...BOT, client: "desktop", sdp: "offer-sdp" }); + t.socket().open(); + t.message({ id: "u1", role: "user", kind: "text", text: "from the owner's desk", sendId: "s1" }); + t.message({ id: "u2", role: "user", kind: "text", text: "from me", sendId: "s2", sender: { name: "me@example.test", id: "p_me" } }); + await vi.advanceTimersByTimeAsync(0); + t.patch({ id: "b1", text: "One.", requestMessageId: "u1", turnTerminal: true }); + t.patch({ id: "b2", text: "Two.", requestMessageId: "u2", turnTerminal: true }); + await vi.advanceTimersByTimeAsync(0); + const said = t.socket().appends("commentary").map((e) => String(e.content)); + expect(said).toHaveLength(1); + expect(said[0]).toContain("from me"); + }); + + it("keeps a typed message's idle clock running like speech", async () => { + const t = await live(); + await vi.advanceTimersByTimeAsync(4 * 60_000); + t.message({ id: "u1", role: "user", kind: "text", text: "still there?" }); + await vi.advanceTimersByTimeAsync(4 * 60_000); + expect(t.socket().sent.some((e) => e.type === "session.close")).toBe(false); + }); + + describe("approvals", () => { + const approval = (extra: Partial = {}) => ({ + id: "c1", kind: "options" as const, + card: { title: "Approval needed", subtitle: "rm -rf build", options: ["Allow", "Deny"], requestId: "r1", tool: "Bash", ...extra }, + }); + + it("reads the request and decides on a clear spoken yes", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ content: expect.stringContaining("I want to run a command. rm -rf build. May I?") }); + hear(t, "yes go ahead", 5_000); + await delegate(t, "del_2", 5_200); + expect(t.deps.respond).toHaveBeenCalledWith({ auth: owner, threadId: "t1", requestId: "r1", behavior: "allow", message: undefined }); + expect(t.deps.send).not.toHaveBeenCalled(); + expect(t.socket().appends("commentary").at(-1)).toMatchObject({ content: "Thanks, I'll go ahead." }); + }); + + it("denies with the call's reason", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "no", 5_000); + await delegate(t, "del_2", 5_100); + expect(t.deps.respond).toHaveBeenCalledWith(expect.objectContaining({ behavior: "deny", message: "Denied by the user, on a live call." })); + }); + + it("asks again when the answer is not a clear yes or no", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "hmm what does that do", 5_000); + await delegate(t, "del_2", 5_200); + expect(t.deps.respond).not.toHaveBeenCalled(); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ content: expect.stringContaining("not a clear yes or no") }); + }); + + it("accepts a yes heard in a quiet moment, without a delegation", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "yes", 5_000); + await vi.advanceTimersByTimeAsync(CONSENT_SETTLE_MS + 1); + expect(t.deps.respond).toHaveBeenCalledWith(expect.objectContaining({ behavior: "allow" })); + }); + + // The voice's own opener ("Okay, I need your permission…") comes back + // through a phone's speaker as input, and "okay…" is a hedge, not a yes. + describe("on the open microphone of a Live call", () => { + const speak = (t: ReturnType, text: string, at: number, until: number) => + t.socket().receive({ type: "session.output_transcript.delta", delta: text, start_ms: at, end_ms: until }); + + it("never takes a hedging okay heard in a quiet moment as a yes", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "okay", 5_000); + await vi.advanceTimersByTimeAsync(CONSENT_SETTLE_MS + 1); + hear(t, " wait, what does it delete", 7_000); + await vi.advanceTimersByTimeAsync(CONSENT_SETTLE_MS + 1); + expect(t.deps.respond).not.toHaveBeenCalled(); + }); + + it("asks again when a delegated answer is only a hedge", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "okay", 5_000); + await delegate(t, "del_2", 5_100); + expect(t.deps.respond).not.toHaveBeenCalled(); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ delegation_id: "del_2", content: LIVE_COPY.notClear }); + }); + + it("does not take the voice's own words, heard back, for the person's answer", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + speak(t, "Okay, I need your permission to run a command. May I?", 4_000, 6_000); + hear(t, "Okay I need your permission to run a command", 4_100, 5_900); + await vi.advanceTimersByTimeAsync(CONSENT_SETTLE_MS + 1); + expect(t.deps.respond).not.toHaveBeenCalled(); + hear(t, "no", 7_000); + await delegate(t, "del_2", 7_200); + expect(t.deps.respond).toHaveBeenCalledWith(expect.objectContaining({ behavior: "deny" })); + }); + + it("asks for a clear answer when all it heard was the voice itself", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + speak(t, "May I run it?", 4_000, 5_000); + hear(t, "may I run it", 4_100, 4_900); + await delegate(t, "del_2", 5_000); + expect(t.deps.respond).not.toHaveBeenCalled(); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ delegation_id: "del_2", content: LIVE_COPY.notClear }); + }); + + it("still takes a clear yes after a hedge", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "okay, yes", 5_000); + await vi.advanceTimersByTimeAsync(CONSENT_SETTLE_MS + 1); + expect(t.deps.respond).toHaveBeenCalledWith(expect.objectContaining({ behavior: "allow" })); + }); + }); + + it("does not ask to repeat a yes the quiet window already decided when its delegation arrives late", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "yes", 5_000); + await vi.advanceTimersByTimeAsync(CONSENT_SETTLE_MS + 1); + expect(t.deps.respond).toHaveBeenCalledTimes(1); + t.patch({ ...approval({ answered: "allow" }) }); + await delegate(t, "del_2", 5_300); + expect(t.socket().appends("instructions").some((e) => String(e.content).includes("not heard clearly"))).toBe(false); + expect(t.socket().appends("thinking").at(-1)).toMatchObject({ delegation_id: "del_2", content: "Thanks, I'll go ahead." }); + expect(t.deps.send).not.toHaveBeenCalled(); + // an empty delegation well after the decision is a new, unheard request + await delegate(t, "del_3", 20_000); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ delegation_id: "del_3", content: expect.stringContaining("not heard clearly") }); + }); + + it("lets the user try again when saving the decision fails", async () => { + const t = await live({ respond: vi.fn().mockResolvedValueOnce({ ok: false, error: "The bot is not running." }).mockResolvedValue({ ok: true }) }); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "yes", 5_000); + await delegate(t, "del_2", 5_100); + expect(t.socket().appends("commentary").at(-1)).toMatchObject({ content: expect.stringContaining("could not be saved") }); + hear(t, "yes", 9_000); + await delegate(t, "del_3", 9_100); + expect(t.deps.respond).toHaveBeenCalledTimes(2); + }); + + it("lets the user try again when saving the decision throws", async () => { + const t = await live({ respond: vi.fn().mockRejectedValueOnce(new Error("store offline")).mockResolvedValue({ ok: true }) }); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "yes", 5_000); + await delegate(t, "del_2", 5_100); + expect(t.socket().appends("commentary").at(-1)).toMatchObject({ content: "The decision could not be saved. Ask the user to try again." }); + hear(t, "yes", 9_000); + await delegate(t, "del_3", 9_100); + expect(t.deps.respond).toHaveBeenCalledTimes(2); + expect(t.controller.current()?.status).toBe("live"); + }); + + it("stays quiet when a tap on screen settled the card just before the spoken decision", async () => { + let t!: ReturnType; + const respond = vi.fn(async () => { + // the tap landed first: the card is settled, and the harness refuses without side effects + t.patch({ ...approval({ answered: "allow" }) }); + return { ok: false as const, error: "The request is no longer open." }; + }); + t = await live({ respond }); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + hear(t, "yes", 5_000); + await delegate(t, "del_2", 5_100); + expect(respond).toHaveBeenCalledTimes(1); + const said = t.socket().appends("commentary").map((e) => String(e.content)); + expect(said).toEqual(["Thanks, I'll go ahead."]); + expect(t.controller.current()?.status).toBe("live"); + }); + + it("forgets an approval answered in the chat", async () => { + const t = await live(); + t.message(approval()); + await vi.advanceTimersByTimeAsync(0); + t.patch({ ...approval({ answered: "allow" }) }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("thinking").at(-1)).toMatchObject({ content: expect.stringContaining("answered that request in the chat") }); + hear(t, "yes and also check the logs", 8_000); + await delegate(t, "del_2", 8_100); + expect(t.deps.respond).not.toHaveBeenCalled(); + expect(t.deps.send).toHaveBeenCalledWith(expect.objectContaining({ text: "yes and also check the logs" })); + }); + + it("sends harness reviews to the screen and never decides them by voice", async () => { + const t = await live(); + t.message(approval({ tool: "stage_skill", title: "Enable the invoice skill", skillRequest: { action: "create" } as never })); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ content: expect.stringContaining("you need their decision in the chat") }); + hear(t, "yes", 5_000); + await delegate(t, "del_2", 5_100); + expect(t.deps.respond).not.toHaveBeenCalled(); + }); + + // A connect-an-app or credential card waits on the person in the chat; + // without a word the call went silent until the idle hang-up. + it("points the person at the chat when the bot needs an app connected", async () => { + const t = await live(); + const connector = { slug: "gmail", label: "Gmail", description: "Read mail", status: "required" as const, resumeKey: "k1" }; + t.message({ id: "k1", kind: "connector", connector }); + t.patch({ id: "k1", kind: "connector", connector: { ...connector, status: "authorizing" } }); + await vi.advanceTimersByTimeAsync(0); + const said = t.socket().appends("instructions").map((e) => String(e.content)); + expect(said).toHaveLength(1); + expect(said[0]).toContain("Gmail"); + expect(said[0]).toContain("in the chat"); + t.message({ id: "k2", kind: "connector", connector: { ...connector, label: "Slack", status: "connected" } }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("instructions")).toHaveLength(1); + }); + + it("points at the chat for a credential, and never asks for it aloud", async () => { + const t = await live(); + const secret = { target: "openaiImageApiKey", label: "OpenAI image key", description: "", placeholder: "sk-", helpUrl: "", requestKey: "r1" } as never; + t.message({ id: "k1", kind: "secret", secret }); + t.message({ id: "k2", kind: "secret", secret: { ...(secret as object), requestKey: "r2", superseded: true } as never }); + await vi.advanceTimersByTimeAsync(0); + const said = t.socket().appends("instructions").map((e) => String(e.content)); + expect(said).toHaveLength(1); + expect(said[0]).toContain("OpenAI image key"); + expect(said[0]).toMatch(/never ask .*aloud/i); + }); + + it("answers a bot question with the next request", async () => { + const t = await live(); + t.message({ id: "q1", kind: "options", card: { title: "Your bot has a question", subtitle: "Which account?", options: ["Main", "Savings"], requestId: "r7" } }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ content: expect.stringContaining("Which account?") }); + hear(t, "savings", 5_000); + await delegate(t, "del_2", 5_100); + expect(t.deps.respond).toHaveBeenCalledWith({ auth: owner, threadId: "t1", requestId: "r7", behavior: "answer", message: "savings" }); + expect(t.deps.send).not.toHaveBeenCalled(); + }); + + it("announces a second approval after the first one settles", async () => { + const t = await live(); + t.message(approval()); + t.message({ ...approval({ requestId: "r2", subtitle: "ls" }), id: "c2" }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("instructions")).toHaveLength(1); + hear(t, "yes", 5_000); + await delegate(t, "del_2", 5_100); + t.patch({ ...approval({ answered: "allow" }) }); + await vi.advanceTimersByTimeAsync(0); + expect(t.socket().appends("instructions").at(-1)).toMatchObject({ content: expect.stringContaining("ls") }); + }); + }); +}); diff --git a/server/live-call-controller.ts b/server/live-call-controller.ts new file mode 100644 index 0000000000..ac6af6ce47 --- /dev/null +++ b/server/live-call-controller.ts @@ -0,0 +1,915 @@ +// The harness side of a Live call (spec: docs/superpowers/specs/2026-09-25-live-call-bar-design.md). +// +// A client (desktop renderer, iPhone or Android app) holds the microphone +// and speaker over WebRTC, straight to OpenAI. This controller holds the +// sideband WebSocket to the same GPT-Live session and runs every rule that +// connects the voice to the bot, so all three clients behave the same: +// delegation → a user message "via call" on the bot's thread; +// bot progress → quiet thinking; the turn's final text → spoken commentary; +// an approval card → a strict spoken yes/no; a question card → the next request; +// a typed message → nothing until the bot answers it; then the answer is read +// with what was typed (readTypedReplies on), or nothing about it at all +// reaches OpenAI (off); +// no speech and no work for idleMinutes → hang up. +// Nothing said on the call is logged; the summary line has counters only. +import { randomUUID } from "node:crypto"; +import { spokenConsent } from "../shared/call-consent.ts"; +import { + LIVE_COPY, liveCardKind, liveStepLabel, spokenApprovalPrompt, spokenConnectorPrompt, spokenQuestionPrompt, spokenReviewPrompt, spokenSecretPrompt, +} from "../shared/live-approval.ts"; +import { clampAppend, commentaryChunks, LiveTranscript } from "../shared/live-call.ts"; +import type { LiveCallState, LiveClient, LiveEndReason } from "../shared/wire.ts"; +import { liveCallSummaryLine, LiveSessionError, liveVoice } from "./live-call.ts"; +import type { RequestAuth } from "./request-auth.ts"; +import type { Message, StoreChange } from "./store.ts"; + +export const DELEGATION_SETTLE_MS = 700; +export const CONSENT_SETTLE_MS = 1_200; +export const PROGRESS_INTERVAL_MS = 4_000; +export const ATTACH_TIMEOUT_MS = 10_000; +export const CLOSE_TIMEOUT_MS = 5_000; +export const IDLE_CHECK_MS = 15_000; +/** How often the voice gets a quiet status note while the bot works. */ +export const STATUS_INTERVAL_MS = 30_000; +const SOCKET_OPEN = 1; +const MAX_ERRORS = 5; +/** Unpaired phones remembered, so a start already in flight when its phone + * was unpaired is refused too. Ids are never reused: a new pairing gets a new one. */ +const MAX_REVOKED_DEVICES = 200; +/** How far past a spoken decision GPT-Live's own delegation of the same + * yes/no may land on the session timeline (takeRequest allows 1.5 s of + * trailing transcript; the delegation point trails the words a little). */ +const DECISION_ECHO_MS = 3_000; + +export interface LiveSocket { + readonly readyState: number; + onopen: ((event: unknown) => void) | null; + onmessage: ((event: { data: unknown }) => void) | null; + onclose: ((event: unknown) => void) | null; + onerror: ((event: unknown) => void) | null; + send(data: string): void; + close(code?: number, reason?: string): void; +} + +export type LiveActivity = "idle" | "working" | "waiting"; +export type LiveSendResult = { kind: "started" | "steered"; messageId: string } | { kind: "queued"; queueId: string }; +export type LiveRespondResult = { ok: true } | { ok: false; error: string }; + +export interface LiveCallDeps { + store: { onChange(listener: (change: StoreChange) => void): () => void }; + /** Throws LiveCallSignedOutError once the sign-in that started the call has ended. */ + send(input: { auth: RequestAuth; botId: string; threadId: string; text: string }): Promise; + /** Throws LiveCallSignedOutError once the sign-in that started the call has ended. */ + respond(input: { auth: RequestAuth; threadId: string; requestId: string; behavior: "allow" | "deny" | "answer"; message?: string }): Promise; + /** Whether a call request that send() queued still waits in the thread's + * queue. Editing or cancelling it in a client removes it undelivered. */ + queued(botId: string, threadId: string, queueId: string): boolean; + /** Whether the sign-in that started a call is still valid (checked with the idle timer). Omitted: always. */ + signedIn?(auth: RequestAuth): boolean; + /** Who a message from `auth` names as its sender (`message.sender.id`): + * undefined for the owner on this computer, whose lines carry no sender. */ + personKey?(auth: RequestAuth): string | undefined; + activity(botId: string, threadId: string): LiveActivity; + broadcast(frame: { kind: "live.call"; botId: string; threadId: string; call: LiveCallState | null }): void; + settings(): { key: string; voice: string; readTypedReplies: boolean; idleMinutes: number }; + createSession(input: { key: string; sdp: string; botId: string; threadId: string; voice: string }): Promise<{ sessionId: string; sdp: string }>; + openSocket(url: string, key: string): LiveSocket; + attachUrl(sessionId: string): string; + speakable(text: string): string[]; + log(line: string): void; + now?(): number; +} + +export class LiveCallBusyError extends Error { + readonly call: LiveCallState; + constructor(call: LiveCallState) { + super("A Live call is already running."); + this.call = call; + } +} + +/** The person who started the call signed out, or their session was + * revoked: the call must not keep reaching the bot on their behalf. */ +export class LiveCallSignedOutError extends LiveSessionError { + constructor() { + super("The sign-in that started this call has ended.", 401); + } +} + +type AppendKind = "instructions" | "thinking" | "commentary"; +type Timer = ReturnType; + +interface OpenCard { + requestId: string; + messageId: string; + submitted: boolean; +} + +interface Call { + state: LiveCallState; + auth: RequestAuth; + /** The paired phone that started the call, as the companion vouched for it + * (null for the computer's own window or a signed-in browser). A phone's + * requests arrive as loopback, so this is what its unpairing ends. */ + device: string | null; + /** the sender id the caller's own typed lines carry (undefined: the owner) */ + personKey: string | undefined; + botName: string; + key: string; + sessionId: string; + socket: LiveSocket | null; + attached: boolean; + transcript: LiveTranscript; + /** end of the latest input transcript fragment, on the session timeline */ + heardThroughMs: number; + /** the last spoken approval decision: where its words ended, what was said back */ + lastDecision: { throughMs: number; copy: string } | null; + eventCounter: number; + activeDelegation: string | null; + lastProgressAt: number; + lastActivityAt: number; + lastActivity: LiveActivity; + /** the current piece of bot work, for the voice's status notes (null while idle) */ + workStartedAt: number | null; + workSteps: number; + lastStep: string | null; + lastStepAt: number; + lastStatusAt: number; + approval: OpenCard | null; + question: OpenCard | null; + /** cards that opened while another one was open, oldest first */ + waitingCards: Message[]; + announced: Set; + /** call requests (user message ids) still waiting for an answer */ + pendingCall: Set; + /** a call request was steered into a running turn: its next answer is ours */ + claimNextTerminal: boolean; + /** call requests waiting in the thread's queue (queue ids), until delivered or removed */ + queuedIds: Set; + /** typed user messages on the thread (id → text), until their answer arrives */ + typedTexts: Map; + /** Lines a drain delivered together, as one turn answering the last of + * them: each drained line → its batch. `drainBatch` is the batch still + * being appended (drained lines arrive back to back). */ + batches: Map; + drainBatch: string[] | null; + spokenAnswers: Set; + pendingEnd: LiveEndReason | null; + timers: Set; + consentTimer: Timer | null; + idleTimer: ReturnType | null; + unsubscribe: (() => void) | null; + closeWaiters: Array<() => void>; + stats: { delegations: number; sentToBot: number; answers: number; approvals: number; notHeard: number; replies: number; seconds: number | null; errors: string[] }; +} + +const CLOSE_REASONS: Record = { + expired: "expired", + content: "content", + remote_hangup: "remote-hangup", + connection_lost: "connection-lost", +}; + +export class LiveCallController { + private readonly deps: LiveCallDeps; + private call: Call | null = null; + private readonly revokedDevices = new Set(); + + constructor(deps: LiveCallDeps) { + this.deps = deps; + } + + current(): LiveCallState | null { + return this.call && this.call.state.status !== "ended" ? { ...this.call.state } : null; + } + + /** `device`: the paired phone the companion vouched for, when the request came through it. */ + async start(input: { auth: RequestAuth; device?: string; botId: string; botName: string; threadId: string; client: LiveClient; sdp: string }): Promise<{ call: LiveCallState; sdp: string }> { + if (this.call && this.call.state.status !== "ended") throw new LiveCallBusyError({ ...this.call.state }); + if (input.device && this.revokedDevices.has(input.device)) throw new LiveCallSignedOutError(); + const settings = this.deps.settings(); + const key = settings.key.trim(); + if (!key) throw new LiveSessionError("Add an OpenAI API key to use Live calls.", 409); + const voice = liveVoice(settings.voice); + const call = this.newCall(input, key, voice); + // Taken before the first await: a second start in the same tick is refused. + this.call = call; + let session: { sessionId: string; sdp: string }; + try { + session = await this.deps.createSession({ key, sdp: input.sdp, botId: input.botId, threadId: input.threadId, voice }); + } catch (error) { + if (this.call === call) this.call = null; + this.deps.log(`[live] call failed bot=${input.botId} client=${input.client} status=${error instanceof LiveSessionError ? error.status : "error"}`); + throw error; + } + if (this.call !== call) { + this.closeOrphan(session.sessionId, key); + throw new LiveSessionError("The call was cancelled.", 503); + } + try { + call.sessionId = session.sessionId; + call.state.startedAt = this.now(); + call.lastActivityAt = this.now(); + call.lastActivity = this.deps.activity(input.botId, input.threadId); + if (call.lastActivity === "working") this.beginWork(call); + call.unsubscribe = this.deps.store.onChange((change) => this.onStoreChange(call, change)); + this.deps.log(`[live] call started bot=${input.botId} voice=${voice} client=${input.client}`); + this.emit(call); + // Before attach: a sideband that fails at once finishes the call, and + // finish must find the interval to clear it. + call.idleTimer = setInterval(() => this.guarded(call, () => this.checkIdle(call)), IDLE_CHECK_MS); + call.idleTimer.unref?.(); + this.attach(call); + } catch (error) { + // Never leave the one call slot taken by a call that did not start. + this.finish(call, "error", "The call could not start."); + throw error; + } + return { call: { ...call.state }, sdp: session.sdp }; + } + + async end(callId: string): Promise { + const call = this.call; + if (!call || call.state.callId !== callId || call.state.status === "ended") return null; + await this.hangUp(call, "hung-up"); + return { ...call.state }; + } + + async shutdown(): Promise { + const call = this.call; + if (!call || call.state.status === "ended") return; + this.command(call, { type: "session.close" }); + this.finish(call, "shutdown"); + } + + /** A paired phone was unpaired (the companion says so). Its call, if it + * holds the line, ends at once; a start of its still in flight is refused. + * Returns the call it ended, or null. */ + deviceRevoked(deviceId: string): LiveCallState | null { + this.revokedDevices.add(deviceId); + if (this.revokedDevices.size > MAX_REVOKED_DEVICES) this.revokedDevices.delete(this.revokedDevices.values().next().value!); + const call = this.call; + if (!call || call.device !== deviceId) return null; + return this.endSignedOut(call, null, LIVE_COPY.unpaired) ? { ...call.state } : null; + } + + /** A sign-in was revoked or signed out. The idle check would notice within + * IDLE_CHECK_MS; this ends the call it started at once. */ + sessionRevoked(sessionId: string): LiveCallState | null { + const call = this.call; + if (!call || call.auth.kind !== "session" || call.auth.session.id !== sessionId) return null; + return this.endSignedOut(call, null) ? { ...call.state } : null; + } + + // ── lifecycle ──────────────────────────────────────────────────────── + + private newCall(input: { auth: RequestAuth; device?: string; botId: string; botName: string; threadId: string; client: LiveClient }, key: string, voice: string): Call { + return { + state: { callId: randomUUID(), botId: input.botId, threadId: input.threadId, client: input.client, voice, startedAt: this.now(), status: "connecting" }, + auth: input.auth, + device: input.device ?? null, + personKey: this.deps.personKey?.(input.auth), + botName: input.botName, + key, + sessionId: "", + socket: null, + attached: false, + transcript: new LiveTranscript(), + heardThroughMs: 0, + lastDecision: null, + eventCounter: 0, + activeDelegation: null, + lastProgressAt: 0, + lastActivityAt: this.now(), + lastActivity: "idle", + workStartedAt: null, + workSteps: 0, + lastStep: null, + lastStepAt: 0, + lastStatusAt: 0, + approval: null, + question: null, + waitingCards: [], + announced: new Set(), + pendingCall: new Set(), + claimNextTerminal: false, + queuedIds: new Set(), + typedTexts: new Map(), + batches: new Map(), + drainBatch: null, + spokenAnswers: new Set(), + pendingEnd: null, + timers: new Set(), + consentTimer: null, + idleTimer: null, + unsubscribe: null, + closeWaiters: [], + stats: { delegations: 0, sentToBot: 0, answers: 0, approvals: 0, notHeard: 0, replies: 0, seconds: null, errors: [] }, + }; + } + + private attach(call: Call): void { + let socket: LiveSocket; + try { + socket = this.deps.openSocket(this.deps.attachUrl(call.sessionId), call.key); + } catch { + this.finish(call, "sideband-lost", "The call could not connect to OpenAI."); + return; + } + call.socket = socket; + const attachTimer = this.later(call, () => { + if (!call.attached) this.finish(call, "sideband-lost", "The call could not connect to OpenAI."); + }, ATTACH_TIMEOUT_MS); + socket.onopen = () => { + call.attached = true; + this.clear(call, attachTimer); + if (call.state.status === "connecting") { + call.state.status = "live"; + this.emit(call); + } + }; + socket.onmessage = (event) => this.guarded(call, () => this.onSideband(call, event.data)); + // Node's WebSocket reports a refused handshake as an error without a close. + socket.onerror = () => { + if (!call.attached) this.finish(call, "sideband-lost", "The call could not connect to OpenAI."); + }; + socket.onclose = () => { + if (call.state.status === "ended") return; + if (call.pendingEnd) this.finish(call, call.pendingEnd); + else this.finish(call, "sideband-lost", "The call connection to OpenAI dropped."); + }; + } + + /** A start cancelled while OpenAI created the session (a hang-up, an + * unpairing, a shutdown): no client will attach to it and no call owns it, + * so close it through its sideband instead of leaving it open until OpenAI + * gives up on it. Best effort, bounded by ATTACH_TIMEOUT_MS. */ + private closeOrphan(sessionId: string, key: string): void { + let socket: LiveSocket; + try { + socket = this.deps.openSocket(this.deps.attachUrl(sessionId), key); + } catch { + return; + } + const done = () => { + clearTimeout(timer); + socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null; + try { socket.close(1000, "call cancelled"); } catch { /* already closed */ } + }; + const timer = setTimeout(done, ATTACH_TIMEOUT_MS); + timer.unref?.(); + socket.onopen = () => { + try { socket.send(JSON.stringify({ type: "session.close", event_id: "omb_cancelled" })); } catch { /* closing anyway */ } + done(); + }; + socket.onerror = done; + socket.onclose = () => clearTimeout(timer); + } + + private hangUp(call: Call, reason: LiveEndReason): Promise { + if (call.state.status === "ended") return Promise.resolve(); + call.pendingEnd ??= reason; + const done = new Promise((resolve) => call.closeWaiters.push(resolve)); + // Already ending: session.close was sent and the close timer runs. + if (call.state.status === "ending") return done; + call.state.status = "ending"; + this.emit(call); + if (!this.command(call, { type: "session.close" })) this.finish(call, call.pendingEnd); + else this.later(call, () => this.finish(call, call.pendingEnd ?? reason), CLOSE_TIMEOUT_MS); + return done; + } + + private finish(call: Call, reason: LiveEndReason, error?: string): void { + if (call.state.status === "ended") return; + call.state.status = "ended"; + call.state.endReason = reason; + if (error) call.state.error = error; + for (const timer of call.timers) clearTimeout(timer); + call.timers.clear(); + if (call.idleTimer) clearInterval(call.idleTimer); + call.idleTimer = null; + call.unsubscribe?.(); + call.unsubscribe = null; + const socket = call.socket; + call.socket = null; + if (socket) { + socket.onopen = socket.onmessage = socket.onerror = socket.onclose = null; + try { socket.close(1000, "call ended"); } catch { /* already closed */ } + } + this.emit(call); + this.deps.log(liveCallSummaryLine({ + botId: call.state.botId, + voice: call.state.voice, + client: call.state.client, + seconds: call.stats.seconds ?? (this.now() - call.state.startedAt) / 1000, + delegations: call.stats.delegations, + sentToBot: call.stats.sentToBot, + answers: call.stats.answers, + approvals: call.stats.approvals, + notHeard: call.stats.notHeard, + replies: call.stats.replies, + end: reason, + errors: call.stats.errors, + })); + if (this.call === call) this.call = null; + for (const resolve of call.closeWaiters.splice(0)) resolve(); + } + + private checkIdle(call: Call): void { + if (call.state.status !== "live" && call.state.status !== "connecting") return; + // Talking keeps a call from idling out; it must not keep a signed-out person's call up. + if (this.deps.signedIn && !this.deps.signedIn(call.auth)) { + this.endSignedOut(call, null); + return; + } + if (this.deps.activity(call.state.botId, call.state.threadId) === "working") { + this.touch(call); + this.maybeStatus(call); + return; + } + const idleMs = this.deps.settings().idleMinutes * 60_000; + if (this.now() - call.lastActivityAt >= idleMs) void this.hangUp(call, "idle"); + } + + // ── sideband events ────────────────────────────────────────────────── + + private onSideband(call: Call, raw: unknown): void { + const text = typeof raw === "string" ? raw : Buffer.isBuffer(raw) ? raw.toString("utf8") : raw instanceof ArrayBuffer ? Buffer.from(raw).toString("utf8") : ""; + // The sideband mirrors the audio; skip it without parsing megabytes of base64. + if (!text || text.includes('"session.input_audio.append"') || text.includes('"session.output_audio.delta"')) return; + let event: Record; + try { + event = JSON.parse(text) as Record; + } catch { + return; + } + if (call.state.status === "ended") return; + switch (event.type) { + case "session.started": + if (call.state.status === "connecting") { + call.state.status = "live"; + this.emit(call); + } + return; + case "session.input_transcript.delta": + call.transcript.addInput(String(event.delta ?? ""), Number(event.start_ms), Number(event.end_ms)); + call.heardThroughMs = Math.max(call.heardThroughMs, Number(event.end_ms) || 0); + this.touch(call); + if (call.approval && !call.approval.submitted) this.scheduleConsentCheck(call); + return; + case "session.output_transcript.delta": + // its timing, when given, marks where the voice itself was talking + call.transcript.addOutput(Number(event.start_ms), Number(event.end_ms)); + this.touch(call); + return; + case "session.delegation.created": { + const delegation = event.delegation as { id?: unknown; target?: unknown } | undefined; + if (delegation?.target !== "client" || typeof delegation.id !== "string") return; + const id = delegation.id; + const offset = Number(event.offset_ms) || 0; + call.stats.delegations += 1; + this.touch(call); + this.later(call, () => this.delegate(call, id, offset), DELEGATION_SETTLE_MS); + return; + } + case "session.usage.updated": + case "session.closed": { + const seconds = Number((event.usage as { seconds?: unknown } | undefined)?.seconds); + if (Number.isFinite(seconds)) call.stats.seconds = seconds; + if (event.type === "session.closed") { + const reason = CLOSE_REASONS[String(event.reason)] ?? call.pendingEnd ?? "hung-up"; + this.finish(call, reason); + } + return; + } + case "error": { + this.recordError(call, String((event.error as { code?: unknown } | undefined)?.code ?? "error")); + return; + } + default: + return; + } + } + + // ── voice → bot ────────────────────────────────────────────────────── + + private async delegate(call: Call, id: string, offsetMs: number): Promise { + if (call.state.status === "ended") return; + const heard = call.transcript.takeRequestParts(offsetMs); + const said = heard.text; + if (call.approval && !call.approval.submitted) { + // the voice's own words, heard back, are never the person's answer + await this.decide(call, heard.withoutEcho, id, said); + return; + } + const question = call.question; + if (question && !question.submitted && said) { + question.submitted = true; + call.activeDelegation = id; + call.stats.answers += 1; + const result = await this.respond(call, { auth: call.auth, threadId: call.state.threadId, requestId: question.requestId, behavior: "answer", message: said }, id); + if (!result) return; + if (result.ok) { + this.append(call, "thinking", LIVE_COPY.answerPassed, id); + } else if (call.question === question) { + question.submitted = false; + this.append(call, "commentary", LIVE_COPY.saveFailed(result.error.trim().slice(0, 200)), id); + } + return; + } + const decision = call.lastDecision; + if (!said && decision && offsetMs <= decision.throughMs + DECISION_ECHO_MS) { + // The quiet-window rule already decided on these words; this is + // GPT-Live delegating the same yes/no. Do not ask the user to repeat it. + this.append(call, "thinking", decision.copy, id); + return; + } + if (!said) { + call.stats.notHeard += 1; + this.append(call, "instructions", LIVE_COPY.notHeard, id); + return; + } + call.activeDelegation = id; + call.stats.sentToBot += 1; + this.append(call, "thinking", LIVE_COPY.working, id); + try { + const result = await this.deps.send({ auth: call.auth, botId: call.state.botId, threadId: call.state.threadId, text: said }); + if (result.kind === "queued") call.queuedIds.add(result.queueId); + else if (result.kind === "steered") { + call.pendingCall.add(result.messageId); + call.claimNextTerminal = true; + } else call.pendingCall.add(result.messageId); + } catch (error) { + if (error instanceof LiveCallSignedOutError) { + this.endSignedOut(call, id); + return; + } + const detail = error instanceof Error ? error.message.trim().slice(0, 200) : ""; + this.recordError(call, "send-failed"); + this.append(call, "commentary", `The request could not be sent to ${call.botName}${detail ? `: ${detail}` : "."}`, id); + } + } + + /** `said`: the person's words; `heard`: everything heard, echo included + * (asking again only makes sense when something was heard at all). */ + private async decide(call: Call, said: string, delegationId: string | null, heard: string = said): Promise { + const open = call.approval; + if (!open || open.submitted) return; + // Live's rule: hedges ("okay…", "sure") never decide on an open microphone + const decision = spokenConsent(said, "live"); + if (!decision) { + if (heard.trim()) this.append(call, "instructions", LIVE_COPY.notClear, delegationId); + return; + } + call.transcript.consumeAll(); + open.submitted = true; + call.stats.approvals += 1; + const copy = decision === "allow" ? LIVE_COPY.granted : LIVE_COPY.denied; + call.lastDecision = { throughMs: call.heardThroughMs, copy }; + this.append(call, "commentary", copy, delegationId); + const result = await this.respond(call, { + auth: call.auth, + threadId: call.state.threadId, + requestId: open.requestId, + behavior: decision, + message: decision === "deny" ? LIVE_COPY.deniedMessage : undefined, + }, delegationId); + if (!result) return; + if (!result.ok && call.approval === open) { + open.submitted = false; + this.append(call, "commentary", LIVE_COPY.saveFailed(result.error.trim().slice(0, 200)), delegationId); + } + } + + private scheduleConsentCheck(call: Call): void { + if (call.consentTimer) this.clear(call, call.consentTimer); + call.consentTimer = this.later(call, () => { + call.consentTimer = null; + if (!call.approval || call.approval.submitted) return; + const pending = call.transcript.pending({ skipEcho: true }); + return spokenConsent(pending, "live") ? this.decide(call, pending, call.activeDelegation) : undefined; + }, CONSENT_SETTLE_MS); + } + + // ── bot → voice ────────────────────────────────────────────────────── + + private onStoreChange(call: Call, change: StoreChange): void { + if (call.state.status === "ended") return; + // Listeners run inside the store's write; do the work after it. + if (change.type === "thread.deleted" && change.threadId === call.state.threadId) this.soon(call, () => this.hangUp(call, "deleted")); + else if (change.type === "bot.deleted" && change.botId === call.state.botId) this.soon(call, () => this.hangUp(call, "deleted")); + else if ((change.type === "message" || change.type === "message.patch") && change.threadId === call.state.threadId) { + const { type, message } = change; + this.soon(call, () => this.onMessage(call, type, message)); + } else if (change.type === "bot" && change.botId === call.state.botId) this.soon(call, () => this.onBotChange(call)); + } + + private onBotChange(call: Call): void { + if (call.state.status === "ended") return; + const activity = this.deps.activity(call.state.botId, call.state.threadId); + const was = call.lastActivity; + call.lastActivity = activity; + if (activity === "working") this.touch(call); + if (activity === "working" && (was === "idle" || call.workStartedAt === null)) this.beginWork(call); + if (activity === "idle") call.workStartedAt = null; + if (was === "idle" || activity !== "idle") return; + // A turn settled. If it carried a call request and said nothing, say so. + this.forgetUnqueued(call); + if (call.approval || call.question || call.queuedIds.size) return; + if (!call.pendingCall.size && !call.claimNextTerminal) return; + call.pendingCall.clear(); + call.claimNextTerminal = false; + this.append(call, "commentary", LIVE_COPY.noAnswer); + } + + /** A queued call request leaves queuedIds when it is delivered (onMessage). + * Edited or cancelled in a client, it leaves the queue without arriving; + * waiting for it would keep "the result is in the chat" unsaid for good. */ + private forgetUnqueued(call: Call): void { + for (const queueId of call.queuedIds) { + if (!this.deps.queued(call.state.botId, call.state.threadId, queueId)) call.queuedIds.delete(queueId); + } + } + + private onMessage(call: Call, type: "message" | "message.patch", message: Message): void { + if (call.state.status === "ended") return; + this.touch(call); + if (type === "message") this.trackDrain(call, message); + if (message.role === "user") { + if (type !== "message" || message.kind !== "text") return; + if (message.via === "call") { + if (message.queueId) call.queuedIds.delete(message.queueId); + call.pendingCall.add(message.id); + } else if (this.typedByCaller(call, message)) { + // Only the bot answers a typed message. Telling the voice now made it + // answer too, and then again when the bot's answer was relayed. + call.typedTexts.set(message.id, message.text ?? ""); + } + return; + } + if (message.kind === "options") { + this.onCard(call, message); + return; + } + if (message.kind === "connector" || message.kind === "secret") { + this.onSetupCard(call, message); + return; + } + if (message.kind === "activity") { + if (type !== "message" || !message.tool) return; + if (call.workStartedAt !== null) { + call.workSteps += 1; + call.lastStep = liveStepLabel(message.tool); + call.lastStepAt = this.now(); + } + if (!this.mayNarrate(call)) return; + if (message.tool.spoken && this.now() - call.lastProgressAt > PROGRESS_INTERVAL_MS) { + call.lastProgressAt = this.now(); + this.append(call, "thinking", LIVE_COPY.progress(message.tool.spoken)); + } + return; + } + if (message.kind === "text" && message.turnTerminal && !call.spokenAnswers.has(message.id)) this.onAnswer(call, message); + } + + /** A line the caller typed in a client (every client sends a sendId with + * what is typed), as the same person. Not a peer bot's line (ask_bot, + * start_thread, an aside: the answer goes back to that bot), not a line a + * routine, a webhook or the local API added (no sendId, or via "api"), not + * a line a worker relayed for someone else (the Slack relay, any guarded + * send: relayed), and not another member's line in a shared workspace. */ + private typedByCaller(call: Call, message: Message): boolean { + if (message.peerAsk || message.aside || message.relayed || message.via === "api" || !message.sendId) return false; + return message.sender?.id === call.personKey; + } + + /** Keep the lines one drain delivered together: they arrive back to back, + * and any other message closes the batch. */ + private trackDrain(call: Call, message: Message): void { + if (message.role !== "user" || message.kind !== "text" || !message.queueId) { + call.drainBatch = null; + return; + } + call.drainBatch ??= []; + call.drainBatch.push(message.id); + call.batches.set(message.id, call.drainBatch); + if (call.batches.size > 200) call.batches.delete(call.batches.keys().next().value!); + } + + private onAnswer(call: Call, message: Message): void { + const request = message.requestMessageId; + // A drained turn answers every line of its batch, not only the last one. + const batch = request === undefined ? [] : call.batches.get(request) ?? [request]; + // A batch that holds a spoken line is the call's: answered aloud, even + // when a typed line came last. + const forCall = batch.some((id) => call.pendingCall.has(id)) || call.claimNextTerminal || (request === undefined && call.pendingCall.size > 0); + const typed = !forCall && request !== undefined ? call.typedTexts.get(request) : undefined; + const settle = () => { + for (const id of batch) { + call.pendingCall.delete(id); + call.typedTexts.delete(id); + call.batches.delete(id); + } + }; + if (!forCall && typed === undefined) return; + if (!forCall && !this.deps.settings().readTypedReplies) { + // Off means nothing about a typed exchange reaches OpenAI, not even as context. + settle(); + call.spokenAnswers.add(message.id); + return; + } + const lead = !forCall && typed !== undefined ? [LIVE_COPY.typedAnswerLead(typed)] : []; + const utterances = this.deps.speakable(message.text ?? ""); + if (!utterances.length) return; + const chunks = commentaryChunks([...lead, ...utterances]); + if (!chunks.length) return; + call.spokenAnswers.add(message.id); + settle(); + if (forCall) { + if (request === undefined || call.claimNextTerminal) call.pendingCall.clear(); + call.claimNextTerminal = false; + } + call.stats.replies += 1; + const delegationId = forCall ? call.activeDelegation : null; + for (const chunk of chunks) this.append(call, "commentary", chunk, delegationId); + } + + private onCard(call: Call, message: Message): void { + const card = message.card; + if (!card?.requestId) return; + const kind = liveCardKind(card); + if (!kind) { + this.onCardSettled(call, card.requestId); + return; + } + if (call.announced.has(card.requestId)) return; + if ((kind === "approval" || kind === "question") && (call.approval || call.question)) { + if (!call.waitingCards.some((waiting) => waiting.card?.requestId === card.requestId)) call.waitingCards.push(message); + return; + } + call.announced.add(card.requestId); + if (kind === "review") { + this.append(call, "instructions", spokenReviewPrompt(card)); + } else if (kind === "approval") { + call.approval = { requestId: card.requestId, messageId: message.id, submitted: false }; + call.transcript.consumeAll(); + this.append(call, "instructions", LIVE_COPY.permissionRequest(spokenApprovalPrompt(card))); + } else { + call.question = { requestId: card.requestId, messageId: message.id, submitted: false }; + call.transcript.consumeAll(); + this.append(call, "instructions", spokenQuestionPrompt(card)); + } + } + + /** A connect-an-app or credential card: the bot waits on the person in + * the chat. The voice says so once; neither can be done by voice, and a + * credential must never be spoken into the call. */ + private onSetupCard(call: Call, message: Message): void { + const { connector, secret } = message; + const waiting = message.kind === "connector" + ? Boolean(connector && connector.status !== "connected" && !connector.dismissed && !connector.resumed) + : Boolean(secret && !secret.provided && !secret.dismissed && !secret.superseded && !secret.resumed); + const key = `${message.kind}:${message.id}`; + if (!waiting || call.announced.has(key)) return; + call.announced.add(key); + this.append(call, "instructions", connector ? spokenConnectorPrompt(connector.label) : spokenSecretPrompt(secret?.label ?? "")); + } + + private onCardSettled(call: Call, requestId: string): void { + call.waitingCards = call.waitingCards.filter((waiting) => waiting.card?.requestId !== requestId); + const open = call.approval?.requestId === requestId ? call.approval : call.question?.requestId === requestId ? call.question : null; + if (!open) return; + if (call.approval === open) call.approval = null; + else call.question = null; + if (!open.submitted) this.append(call, "thinking", LIVE_COPY.answeredInChat, null); + const next = call.waitingCards.shift(); + if (next) this.onCard(call, next); + } + + // ── status notes ───────────────────────────────────────────────────── + + private beginWork(call: Call): void { + const now = this.now(); + call.workStartedAt = now; + call.workSteps = 0; + call.lastStep = null; + call.lastStepAt = now; + call.lastStatusAt = now; + } + + /** A quiet fact for the voice while the bot works, so "is it stuck?" gets an + * answer from the real state instead of a guess (or a question steered into + * the running turn). Names steps, never their arguments. */ + private maybeStatus(call: Call): void { + if (call.workStartedAt === null) { + this.beginWork(call); + return; + } + const now = this.now(); + if (now - call.lastStatusAt < STATUS_INTERVAL_MS) return; + call.lastStatusAt = now; + if (!this.mayNarrate(call)) return; + this.append(call, "thinking", LIVE_COPY.status(now - call.workStartedAt, call.workSteps, call.lastStep, now - call.lastStepAt), null); + } + + /** Whether the voice may hear about the bot's current work (progress and + * status notes). With typed replies off, only work on a spoken request: + * the steps the bot takes for a typed message are about that exchange. */ + private mayNarrate(call: Call): boolean { + return this.deps.settings().readTypedReplies || call.pendingCall.size > 0 || call.claimNextTerminal; + } + + // ── plumbing ───────────────────────────────────────────────────────── + + private append(call: Call, kind: AppendKind, content: string, delegationId: string | null = call.activeDelegation): boolean { + return this.command(call, { type: `session.${kind}.append`, delegation_id: delegationId, content: clampAppend(content) }); + } + + private command(call: Call, event: Record): boolean { + const socket = call.socket; + if (!socket || socket.readyState !== SOCKET_OPEN || call.state.status === "ended") return false; + try { + socket.send(JSON.stringify({ ...event, event_id: `omb_${++call.eventCounter}` })); + return true; + } catch { + return false; + } + } + + private emit(call: Call): void { + try { + this.deps.broadcast({ kind: "live.call", botId: call.state.botId, threadId: call.state.threadId, call: { ...call.state } }); + } catch { + this.recordError(call, "broadcast"); + } + } + + private touch(call: Call): void { + call.lastActivityAt = this.now(); + } + + private later(call: Call, fn: () => unknown, ms: number): Timer { + const timer = setTimeout(() => { + call.timers.delete(timer); + this.guarded(call, fn); + }, ms); + timer.unref?.(); + call.timers.add(timer); + return timer; + } + + /** Run deferred work in the next microtask, after the store's write. */ + private soon(call: Call, fn: () => unknown): void { + queueMicrotask(() => this.guarded(call, fn)); + } + + /** Timers, microtasks and socket events run outside any caller's + * try/catch, and the server has no unhandled-rejection handler: a bug in + * the voice relay must cost a counter, never the harness process. */ + private guarded(call: Call, fn: () => unknown): void { + try { + const result = fn(); + if (result instanceof Promise) result.catch(() => this.recordError(call, "internal")); + } catch { + this.recordError(call, "internal"); + } + } + + /** Null when the call is ending because its sign-in ended: say nothing more. */ + private async respond(call: Call, input: Parameters[0], delegationId: string | null): Promise { + try { + return await this.deps.respond(input); + } catch (error) { + if (error instanceof LiveCallSignedOutError) { + this.endSignedOut(call, delegationId); + return null; + } + this.recordError(call, "respond-failed"); + return { ok: false, error: "" }; + } + } + + /** The sign-in (or paired phone) that started the call ended: say so once, + * then hang up. False when the call was already ending. */ + private endSignedOut(call: Call, delegationId: string | null, why: string = LIVE_COPY.signedOut): boolean { + if (call.state.status === "ending" || call.state.status === "ended") return false; + this.recordError(call, "signed-out"); + this.append(call, "commentary", why, delegationId); + call.state.error = why; + void this.hangUp(call, "signed-out"); + return true; + } + + private recordError(call: Call, code: string): void { + if (call.stats.errors.length < MAX_ERRORS) call.stats.errors.push(code); + } + + private clear(call: Call, timer: Timer): void { + clearTimeout(timer); + call.timers.delete(timer); + } + + private now(): number { + return this.deps.now?.() ?? Date.now(); + } +} diff --git a/server/live-call.e2e.test.ts b/server/live-call.e2e.test.ts new file mode 100644 index 0000000000..c818f8bf33 --- /dev/null +++ b/server/live-call.e2e.test.ts @@ -0,0 +1,377 @@ +// A Live call, end to end: boots the real harness against the fake GPT-Live +// (server/testing/fake-openai-live.ts) and the fake claude CLI, starts a call +// the way a client does, and plays the voice's side of it over the sideband. +// What is pinned here is the wiring the unit tests cannot see: the route +// creates the session with the restricted data channel, the controller +// attaches, a delegation becomes a user message "via call" on the bot's +// thread, the bot's answer comes back as spoken commentary on that +// delegation, and hanging up (or losing the sideband) ends the call for +// every client — with one summary line in the log and no words in it. +// +// POSIX-gated like the other CLI e2es (the fakes are shebang scripts). +import { spawn, type ChildProcess } from "node:child_process"; +import { chmodSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { afterAll, beforeAll, describe, expect, it } from "vitest"; +import { LIVE_COPY } from "../shared/live-approval.ts"; +import type { LiveCallState } from "../shared/wire.ts"; +import { removeTempDir, waitForExit } from "./testing/cleanup.ts"; +import { startFakeOpenAiLive, type FakeOpenAiLive } from "./testing/fake-openai-live.ts"; +import { freePortBlock } from "./testing/ports.ts"; +import { openSse } from "./testing/sse.ts"; + +const SERVER_DIR = dirname(fileURLToPath(import.meta.url)); +const FAKE_CLAUDE = join(SERVER_DIR, "testing", "fake-claude-cli.ts"); +const FAKE_ACP = join(SERVER_DIR, "testing", "fake-acp-cli.ts"); +const LIVE_KEY = "sk-fake-e2e"; +const SDP = "v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\n"; +const posixOnly = describe.skipIf(process.platform === "win32"); + +posixOnly("Live call e2e", () => { + let child: ChildProcess; + let home = ""; + let base = ""; + let output = ""; + let live: FakeOpenAiLive; + + /** Everything the harness printed: stdout is where server.log comes from. */ + const serverOutput = () => output; + const post = async (path: string, body: unknown, headers: Record = {}): Promise<{ status: number; body: unknown }> => { + const res = await fetch(`${base}${path}`, { + method: "POST", + headers: { "content-type": "application/json", ...headers }, + body: JSON.stringify(body), + }); + return { status: res.status, body: await res.json() }; + }; + const createBot = async (instanceId = "claude", model = "claude-fake"): Promise<{ id: string; threadId: string }> => { + const { bot } = (await post("/api/bots", {})).body as { bot: { id: string; threadId: string } }; + const res = await fetch(`${base}/api/bots/${bot.id}`, { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ modelSelection: { instanceId, model } }), + }); + expect(res.status).toBe(200); + return bot; + }; + const isBusy = async (botId: string): Promise => { + const { bots } = (await (await fetch(`${base}/api/bots`)).json()) as { bots: Array<{ id: string; busy?: boolean }> }; + return bots.find((bot) => bot.id === botId)?.busy === true; + }; + const spokenTurnsLogged = () => serverOutput().split("text=(spoken)").length - 1; + /** Starts a call and returns it with the fake's record of its session. */ + const startCall = async (botId: string, client: LiveCallState["client"] = "desktop") => { + const before = live.sessions.length; + const started = await post("/api/live/session", { botId, sdp: SDP, client }); + expect(started.status).toBe(201); + const { call } = started.body as { call: LiveCallState }; + const session = live.sessions[before]; + expect(session).toBeDefined(); + return { started, call, session }; + }; + + beforeAll(async () => { + chmodSync(FAKE_CLAUDE, 0o755); + chmodSync(FAKE_ACP, 0o755); + live = await startFakeOpenAiLive(); + home = mkdtempSync(join(tmpdir(), "omb-live-call-")); + mkdirSync(join(home, ".openmausbot"), { recursive: true }); + writeFileSync( + join(home, ".openmausbot", "config.json"), + JSON.stringify({ + instances: { + claude: { + driver: "claudeAgent", + environment: { FAKE_CLAUDE_REPLIES: JSON.stringify(["Six times seven is 42."]) }, + config: { cli: FAKE_CLAUDE, permissionMode: "bypassPermissions" }, + }, + // cannot steer and never finishes: a request made while it works waits in the queue + acp: { driver: "grokAgent", environment: { FAKE_ACP_MODE: "hang" }, config: { cli: FAKE_ACP, fullAuto: true } }, + // every turn asks permission to run a command + asks: { driver: "grokAgent", environment: { FAKE_ACP_MODE: "permission" }, config: { cli: FAKE_ACP, fullAuto: false } }, + }, + }), + ); + const port = await freePortBlock([0, 1]); + base = `http://127.0.0.1:${port}`; + child = spawn(process.execPath, [join(SERVER_DIR, "index.ts")], { + cwd: join(SERVER_DIR, ".."), + env: { + ...(process.env.PATH ? { PATH: process.env.PATH } : {}), + HOME: home, + USERPROFILE: home, + OMB_PORT: String(port), + OMB_WEBHOOK_PORT: String(port + 1), + OMB_OPENAI_LIVE_URL: live.url, + OMB_OPENAI_LIVE_KEY: LIVE_KEY, + }, + stdio: ["ignore", "pipe", "pipe"], + }); + child.stdout!.on("data", (chunk) => (output += chunk)); + child.stderr!.on("data", (chunk) => (output += chunk)); + // Generous: a loaded machine can take most of a minute to boot the harness. + const deadline = Date.now() + 90_000; + for (;;) { + try { + if ((await fetch(`${base}/api/health`)).ok) break; + } catch { + /* not up yet */ + } + if (Date.now() > deadline) throw new Error(`server never came up. output:\n${output}`); + if (child.exitCode !== null) throw new Error(`server exited ${child.exitCode}. output:\n${output}`); + await new Promise((r) => setTimeout(r, 150)); + } + }, 120_000); + + afterAll(async () => { + await waitForExit(child, { signal: "SIGTERM" }); + await live?.stop(); + if (home) await removeTempDir(home); + }); + + it("runs a spoken request through the bot and speaks the answer", async () => { + const bot = await createBot(); + const sse = await openSse(`${base}/api/events`); + try { + const { started, call, session } = await startCall(bot.id); + // the key goes to OpenAI and nowhere else + expect(JSON.stringify(started.body)).not.toContain(LIVE_KEY); + expect(call).toMatchObject({ botId: bot.id, threadId: bot.threadId, client: "desktop" }); + // the client's data channel may only hang up + expect(session.body).toMatchObject({ session: { client: { data_channel: { allowed_client_events: ["session.close"] } } } }); + await live.waitForAttach(session.id); + await sse.until((frame) => frame.kind === "live.call" && frame.call?.callId === call.callId && frame.call?.status === "live"); + + live.emit(session.id, { type: "session.input_transcript.delta", delta: "what is six times seven", start_ms: 100, end_ms: 900 }); + live.emit(session.id, { type: "session.delegation.created", offset_ms: 950, delegation: { id: "del_1", target: "client", type: "delegation" } }); + + const asked = await sse.until( + (frame) => frame.kind === "message" && frame.threadId === bot.threadId && frame.message?.role === "user" && frame.message?.via === "call", + ); + expect(asked.message.text).toBe("what is six times seven"); + const spoken = await live.waitForCommand(session.id, (c) => c.type === "session.commentary.append" && String(c.content).includes("42"), 20_000); + expect(spoken.delegation_id).toBe("del_1"); + + const ended = await post("/api/live/call/end", { callId: call.callId }); + expect(ended).toMatchObject({ status: 200, body: { call: { status: "ended", endReason: "hung-up" } } }); + expect(session.commands.some((c) => c.type === "session.close")).toBe(true); + await sse.until((frame) => frame.kind === "live.call" && frame.call?.callId === call.callId && frame.call?.status === "ended"); + // The summary line is printed as the call finishes; the pipe can + // deliver it a moment after the HTTP answer. + await expect.poll(serverOutput, { timeout: 5_000 }).toMatch(/\[live\] call ended bot=\S+ .*client=desktop .*end=hung-up/); + // the turn ran and was logged, without the words that started it + expect(serverOutput()).toContain("text=(spoken)"); + expect(serverOutput()).not.toMatch(/six times seven/i); + expect(serverOutput()).not.toContain(LIVE_KEY); + expect(JSON.stringify(sse.frames)).not.toContain(LIVE_KEY); + } finally { + sse.close(); + } + }, 40_000); + + it("answers 409 with the running call to a second client", async () => { + const bot = await createBot(); + const { call, session } = await startCall(bot.id); + await live.waitForAttach(session.id); + + const second = await post("/api/live/session", { botId: bot.id, sdp: SDP, client: "ios" }); + expect(second.status).toBe(409); + expect((second.body as { activeCall: LiveCallState }).activeCall).toMatchObject({ callId: call.callId, client: "desktop" }); + // the refused start did not reach OpenAI + expect(live.sessions.at(-1)).toBe(session); + + const ended = await post("/api/live/call/end", { callId: call.callId }); + expect(ended).toMatchObject({ status: 200, body: { call: { callId: call.callId, status: "ended" } } }); + }, 40_000); + + it("queues a spoken request while the bot is busy, and the drained turn still keeps the words out of the log", async () => { + const bot = await createBot("acp", "fake-model"); + const sse = await openSse(`${base}/api/events`); + try { + expect((await post(`/api/bots/${bot.id}/messages`, { text: "first" })).status).toBe(202); + await expect.poll(() => isBusy(bot.id), { timeout: 20_000 }).toBe(true); + const { call, session } = await startCall(bot.id); + await live.waitForAttach(session.id); + + live.emit(session.id, { type: "session.input_transcript.delta", delta: "spell the word banana", start_ms: 100, end_ms: 900 }); + live.emit(session.id, { type: "session.delegation.created", offset_ms: 950, delegation: { id: "del_q", target: "client", type: "delegation" } }); + await sse.until( + (frame) => frame.kind === "bot.queued" && (frame.queues?.[bot.threadId] ?? []).some((item: { text?: string }) => item.text === "spell the word banana"), + 20_000, + ); + + // Stop frees the thread: the spoken line leaves the queue as its own turn + const logged = spokenTurnsLogged(); + await post(`/api/bots/${bot.id}/interrupt`, {}); + const drained = await sse.until( + (frame) => frame.kind === "message" && frame.threadId === bot.threadId && frame.message?.text === "spell the word banana", + 20_000, + ); + expect(drained.message).toMatchObject({ role: "user", via: "call" }); + await expect.poll(spokenTurnsLogged, { timeout: 10_000 }).toBeGreaterThan(logged); + expect(serverOutput()).not.toMatch(/banana/i); + + await post(`/api/bots/${bot.id}/interrupt`, {}); + await expect.poll(() => isBusy(bot.id), { timeout: 20_000 }).toBe(false); + expect(await post("/api/live/call/end", { callId: call.callId })).toMatchObject({ status: 200 }); + } finally { + sse.close(); + } + }, 60_000); + + it("stops waiting for a queued spoken request once it is cancelled in the chat", async () => { + const bot = await createBot("acp", "fake-model"); + const sse = await openSse(`${base}/api/events`); + try { + const { call, session } = await startCall(bot.id); + await live.waitForAttach(session.id); + // a spoken request starts a turn that does not finish on its own + live.emit(session.id, { type: "session.input_transcript.delta", delta: "check the build", start_ms: 100, end_ms: 900 }); + live.emit(session.id, { type: "session.delegation.created", offset_ms: 950, delegation: { id: "del_a", target: "client", type: "delegation" } }); + await expect.poll(() => isBusy(bot.id), { timeout: 20_000 }).toBe(true); + // the next one waits in the queue + live.emit(session.id, { type: "session.input_transcript.delta", delta: "and the tests", start_ms: 2_000, end_ms: 2_800 }); + live.emit(session.id, { type: "session.delegation.created", offset_ms: 2_850, delegation: { id: "del_b", target: "client", type: "delegation" } }); + const waiting = await sse.until( + (frame) => frame.kind === "bot.queued" && (frame.queues?.[bot.threadId] ?? []).some((item: { text?: string }) => item.text === "and the tests"), + 20_000, + ); + const { queueId } = (waiting.queues[bot.threadId] as Array<{ queueId: string; text: string }>).find((item) => item.text === "and the tests")!; + + // cancelled in the chat (editing a queued line cancels it too), it never arrives + const cancelled = await fetch(`${base}/api/bots/${bot.id}/queue/${queueId}`, { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ threadId: bot.threadId }), + }); + expect(cancelled.status).toBe(200); + // Stop ends the first request's turn without an answer: with nothing + // left waiting, the voice says where the result is + await post(`/api/bots/${bot.id}/interrupt`, {}); + await live.waitForCommand(session.id, (c) => c.type === "session.commentary.append" && c.content === LIVE_COPY.noAnswer, 20_000); + expect(await post("/api/live/call/end", { callId: call.callId })).toMatchObject({ status: 200 }); + } finally { + sse.close(); + } + }, 60_000); + + // A phone reaches the harness through the companion, as this computer's + // own requests: only the companion can say the phone was unpaired. + it("ends a phone's call, and refuses it another, once the companion says it was unpaired", async () => { + const bot = await createBot(); + const phone = { "x-openmausbot-companion": "1", "x-openmausbot-companion-device": "phone-e2e" }; + const sse = await openSse(`${base}/api/events`); + try { + const before = live.sessions.length; + const started = await post("/api/live/session", { botId: bot.id, sdp: SDP, client: "ios" }, phone); + expect(started.status).toBe(201); + const { call } = started.body as { call: LiveCallState }; + const session = live.sessions[before]; + await live.waitForAttach(session.id); + await sse.until((frame) => frame.kind === "live.call" && frame.call?.callId === call.callId && frame.call?.status === "live"); + + // another phone's unpairing is not this call's business + expect(await post("/api/live/device-revoked", {}, { ...phone, "x-openmausbot-companion-device": "phone-other" })) + .toEqual({ status: 200, body: { call: null } }); + expect(await post("/api/live/device-revoked", {}, phone)).toMatchObject({ status: 200, body: { call: { callId: call.callId } } }); + await live.waitForCommand(session.id, (c) => c.type === "session.close", 5_000); + const ended = await sse.until( + (frame) => frame.kind === "live.call" && frame.call?.callId === call.callId && frame.call?.status === "ended", + 10_000, + ); + expect(ended.call).toMatchObject({ endReason: "signed-out", error: LIVE_COPY.unpaired }); + const again = await post("/api/live/session", { botId: bot.id, sdp: SDP, client: "ios" }, phone); + expect(again.status).toBe(401); + await expect.poll(serverOutput, { timeout: 5_000 }).toMatch(/\[live\] call ended .*client=ios .*end=signed-out/); + } finally { + sse.close(); + } + }, 40_000); + + // A worker such as the Slack relay sends through the guarded route, for + // someone else and as this computer. The caller did not type that line, so + // the voice never reads it (or its answer) back as "what you typed". + it("never reads back a line a worker relayed into the call's chat as the caller's own", async () => { + const bot = await createBot(); + const { call, session } = await startCall(bot.id); + type Line = { id: string; role: string; text?: string; relayed?: boolean; requestMessageId?: string; turnTerminal?: boolean }; + const lines = async () => ((await (await fetch(`${base}/api/threads/${bot.threadId}/messages`)).json()) as { messages: Line[] }).messages; + try { + await live.waitForAttach(session.id); + const page = (await (await fetch(`${base}/api/threads/${bot.threadId}/messages?limit=0`)).json()) as { activeLeafId?: string | null }; + const relayed = await post(`/api/bots/${bot.id}/messages/guarded`, { + threadId: bot.threadId, sendId: "slackjob_live_call_relay_1", text: "relayed from Slack", + expectedActiveLeafId: page.activeLeafId ?? null, onBehalfOf: { email: "ada@example.test", name: "Ada" }, + }); + expect(relayed.status, JSON.stringify(relayed.body)).toBe(202); + const relayedId = (relayed.body as { message: { id: string } }).message.id; + // the bot answers it in the chat, and the thread is free again + await expect.poll(async () => (await lines()).some((m) => m.requestMessageId === relayedId && m.turnTerminal), { timeout: 20_000 }).toBe(true); + await expect.poll(() => isBusy(bot.id), { timeout: 20_000 }).toBe(false); + + // A line the caller typed is read back; the relayed one before it was not. + expect((await post(`/api/bots/${bot.id}/messages`, { threadId: bot.threadId, text: "typed on the desktop", sendId: "typed-live-call-e2e-0001" })).status).toBe(202); + await live.waitForCommand(session.id, (c) => c.type === "session.commentary.append" && String(c.content).includes("typed on the desktop"), 20_000); + const said = session.commands.filter((c) => c.type === "session.commentary.append").map((c) => String(c.content)); + expect(said).toHaveLength(1); + expect(said[0]).not.toContain("relayed from Slack"); + + const stored = await lines(); + expect(stored.find((m) => m.id === relayedId)).toMatchObject({ role: "user", relayed: true }); + expect(stored.find((m) => m.text === "typed on the desktop")?.relayed).toBeUndefined(); + } finally { + await post("/api/live/call/end", { callId: call.callId }); + } + }, 60_000); + + // A card approved by voice says so, on the card and in the decision log. + it("approves a card by voice and marks it as decided on the call", async () => { + const bot = await createBot("asks", "fake-model"); + const { call, session } = await startCall(bot.id); + try { + await live.waitForAttach(session.id); + live.emit(session.id, { type: "session.input_transcript.delta", delta: "run the check", start_ms: 100, end_ms: 900 }); + live.emit(session.id, { type: "session.delegation.created", offset_ms: 950, delegation: { id: "del_p", target: "client", type: "delegation" } }); + await live.waitForCommand(session.id, (c) => c.type === "session.instructions.append" && String(c.content).includes("May I?"), 20_000); + live.emit(session.id, { type: "session.input_transcript.delta", delta: "yes", start_ms: 5_000, end_ms: 5_300 }); + live.emit(session.id, { type: "session.delegation.created", offset_ms: 5_400, delegation: { id: "del_y", target: "client", type: "delegation" } }); + type Card = { requestId?: string; answered?: string; answeredBy?: unknown }; + const approved = async () => { + const { messages } = (await (await fetch(`${base}/api/threads/${bot.threadId}/messages`)).json()) as { messages: Array<{ card?: Card }> }; + return messages.find((m) => m.card?.requestId && m.card.answered === "allow")?.card; + }; + await expect.poll(approved, { timeout: 20_000 }).toMatchObject({ answeredBy: { kind: "loopback", via: "call" } }); + const requestId = (await approved())!.requestId; + const decision = async () => ((await (await fetch(`${base}/api/decisions`)).json()) as { decisions: Array<{ requestId?: string; decision: string; via?: string }> }) + .decisions.find((row) => row.requestId === requestId && row.decision === "user-approved"); + await expect.poll(decision, { timeout: 10_000 }).toMatchObject({ via: "call" }); + } finally { + await post("/api/live/call/end", { callId: call.callId }); + } + }, 60_000); + + it("ends the call and tells clients when the sideband drops", async () => { + const bot = await createBot(); + const sse = await openSse(`${base}/api/events`); + try { + const { call, session } = await startCall(bot.id); + await live.waitForAttach(session.id); + await sse.until((frame) => frame.kind === "live.call" && frame.call?.callId === call.callId && frame.call?.status === "live"); + + live.dropSideband(session.id); + + const ended = await sse.until( + (frame) => frame.kind === "live.call" && frame.call?.callId === call.callId && frame.call?.status === "ended", + ); + expect(ended).toMatchObject({ botId: bot.id, threadId: bot.threadId, call: { endReason: "sideband-lost" } }); + // a client that connects now sees no call, and the slot is free again + const current = await (await fetch(`${base}/api/live/call`)).json(); + expect(current).toEqual({ call: null }); + await expect.poll(serverOutput, { timeout: 5_000 }).toContain("end=sideband-lost"); + } finally { + sse.close(); + } + }, 40_000); +}); diff --git a/server/live-call.test.ts b/server/live-call.test.ts new file mode 100644 index 0000000000..2a53138a87 --- /dev/null +++ b/server/live-call.test.ts @@ -0,0 +1,197 @@ +import { describe, expect, it, vi } from "vitest"; + +import { + createLiveSession, + DEFAULT_LIVE_VOICE, + LIVE_MODEL, + liveAttachUrl, + liveBaseUrl, + liveErrorMessage, + LiveSessionError, + liveInitialInput, + liveInstructions, + liveSessionsUrl, + MAX_SDP_BYTES, +} from "./live-call.ts"; + +const OFFER = "v=0\r\no=- 1 2 IN IP4 127.0.0.1\r\n"; +const BOT = { name: "Ada", title: "Tech Lead", description: "Leads the engineering team." }; + +function okFetch(answer = { session: { id: "live_123" }, transport: { type: "webrtc", sdp: "v=0 answer" } }) { + return vi.fn(async (_url: string | URL | Request, _init?: RequestInit) => new Response(JSON.stringify(answer), { status: 201 })); +} + +describe("createLiveSession", () => { + it("creates a client-delegation session and returns only the answer", async () => { + const fetchImpl = okFetch(); + const result = await createLiveSession({ + key: " sk-live-secret ", + sdp: OFFER, + bot: BOT, + history: [{ role: "user", text: "Check the release notes" }, { role: "assistant", text: "Done: two fixes." }], + voice: "cedar", + fetchImpl, + }); + expect(result).toEqual({ sessionId: "live_123", sdp: "v=0 answer" }); + + const [url, init] = fetchImpl.mock.calls[0]; + expect(url).toBe("https://api.openai.com/v1/live/sessions"); + expect(init).toBeDefined(); + expect((init!.headers as Record).authorization).toBe("Bearer sk-live-secret"); + const body = JSON.parse(String(init?.body)); + expect(body.transport).toEqual({ type: "webrtc", sdp: OFFER }); + expect(body.session.model).toBe(LIVE_MODEL); + expect(body.session.delegation).toEqual({ type: "client" }); + expect(body.session.audio).toEqual({ output: { voice: "cedar" } }); + expect(body.session.instructions).toContain("You are Ada, Tech Lead"); + expect(body.session.instructions).toContain("Delegation policy:"); + expect(body.session.input).toEqual([ + { type: "message", role: "user", content: [{ type: "input_text", text: "Check the release notes" }] }, + { type: "message", role: "assistant", content: [{ type: "output_text", text: "Done: two fixes." }] }, + ]); + // the key is a header, never part of the session body + expect(String(init?.body)).not.toContain("sk-live-secret"); + }); + + it("falls back to the default voice and omits empty history", async () => { + const fetchImpl = okFetch(); + await createLiveSession({ key: "k", sdp: OFFER, bot: BOT, history: [], voice: "not a voice!", fetchImpl }); + const body = JSON.parse(String(fetchImpl.mock.calls[0][1]?.body)); + expect(body.session.audio.output.voice).toBe(DEFAULT_LIVE_VOICE); + expect(body.session).not.toHaveProperty("input"); + }); + + it("passes an unlisted but well-formed voice name through for OpenAI to judge", async () => { + const fetchImpl = okFetch(); + await createLiveSession({ key: "k", sdp: OFFER, bot: BOT, history: [], voice: " Sol ", fetchImpl }); + expect(JSON.parse(String(fetchImpl.mock.calls[0][1]?.body)).session.audio.output.voice).toBe("sol"); + }); + + it("refuses without a key or with a bad offer before calling OpenAI", async () => { + const fetchImpl = okFetch(); + await expect(createLiveSession({ key: " ", sdp: OFFER, bot: BOT, history: [], fetchImpl })).rejects.toMatchObject({ status: 409 }); + await expect(createLiveSession({ key: "k", sdp: " ", bot: BOT, history: [], fetchImpl })).rejects.toMatchObject({ status: 400 }); + await expect(createLiveSession({ key: "k", sdp: "x".repeat(MAX_SDP_BYTES + 1), bot: BOT, history: [], fetchImpl })).rejects.toMatchObject({ status: 400 }); + expect(fetchImpl).not.toHaveBeenCalled(); + }); + + it("turns OpenAI refusals into plain messages without echoing the body", async () => { + const refuse = (status: number) => vi.fn(async () => new Response(JSON.stringify({ error: { message: "sk-live-secret is invalid" } }), { status })); + for (const [status, text, returned] of [[400, "rejected the call settings", 502], [401, "rejected the API key", 502], [403, "no access to GPT-Live", 502], [429, "limiting Live sessions", 429], [500, "had a problem", 502]] as const) { + const error = await createLiveSession({ key: "sk-live-secret", sdp: OFFER, bot: BOT, history: [], fetchImpl: refuse(status) }).catch((e) => e); + expect(error).toBeInstanceOf(LiveSessionError); + expect(error.status).toBe(returned); + expect(error.message).toContain(text); + expect(error.message).not.toContain("sk-live-secret"); + } + }); + + it("reports network failures and malformed answers", async () => { + const offline = vi.fn(async () => { throw new TypeError("fetch failed"); }); + await expect(createLiveSession({ key: "k", sdp: OFFER, bot: BOT, history: [], fetchImpl: offline })).rejects.toMatchObject({ status: 502, message: expect.stringContaining("Could not reach OpenAI") }); + const odd = vi.fn(async () => new Response(JSON.stringify({ session: {} }), { status: 201 })); + await expect(createLiveSession({ key: "k", sdp: OFFER, bot: BOT, history: [], fetchImpl: odd })).rejects.toMatchObject({ status: 502, message: expect.stringContaining("unexpected answer") }); + }); +}); + +describe("live startup context", () => { + it("keeps the newest messages inside the history budget", () => { + const history = Array.from({ length: 30 }, (_, index) => ({ role: index % 2 ? "assistant" as const : "user" as const, text: `message ${index} ${"x".repeat(900)}` })); + const input = liveInitialInput(history); + expect(input.length).toBeLessThanOrEqual(8); + expect(input.at(-1)?.content[0].text.startsWith("message 29")).toBe(true); + const total = input.reduce((sum, item) => sum + item.content[0].text.length, 0); + expect(total).toBeLessThanOrEqual(6_000); + for (const item of input) expect(item.content[0].text.length).toBeLessThanOrEqual(600); + }); + + it("writes one-line instructions from the bot's own profile", () => { + const text = liveInstructions({ name: " Rigel\n", title: "QA", description: "Line one\nline two" }); + expect(text.split("\n")[0]).toBe("You are Rigel, QA, an AI agent that runs in OpenMausBot on the user's own computer. Line one line two"); + expect(text).toContain("Never answer it yourself."); + }); + + it("tells the voice to answer 'is it still working?' from the status notes, not by delegating", () => { + const text = liveInstructions({ name: "Rigel" }); + expect(text).toContain("- The user only asks whether you are still working or stuck."); + expect(text).toContain("While you work you get quiet status notes"); + }); + + // The voice called itself "the voice layer" and offered to "ask the backend" + // when asked which AI model it is. It is the bot, and it checks without asking. + it("makes the voice the bot itself: first person, no backend talk, no asking to check", () => { + const text = liveInstructions({ name: "CFO", title: "Chief Financial Officer" }); + expect(text.split("\n")[0]).toBe("You are CFO, Chief Financial Officer, an AI agent that runs in OpenMausBot on the user's own computer."); + expect(text).not.toContain("only the voice"); + expect(text).not.toContain("voice of"); + expect(text).toContain("Never mention a backend, delegation, a voice layer, or another system or model doing the work"); + expect(text).toContain("Never ask the user whether you may look something up or check something"); + expect(text).toContain("- The user asks something about you that this conversation does not already answer, for example which AI model you run on."); + // OpenAI's trained delegation labels stay as they are + for (const label of ["Backchannel policy:", "Interruption policy:", "Delegation policy:", "Backend tools:", "Delegate to the backend when:", "Do not delegate to the backend when:"]) { + expect(text).toContain(label); + } + }); +}); + +describe("live call summary line", () => { + it("keeps counters and short codes only", async () => { + const { liveCallSummaryLine } = await import("./live-call.ts"); + const line = liveCallSummaryLine({ + botId: "bot-1", voice: "sol", client: "ios", seconds: 83.6, delegations: 3, sentToBot: 2, answers: 0, approvals: 1, + notHeard: 1, replies: 2, end: "close_requested", errors: ["rate_limit", "bad code; rm -rf /", 7], + said: "this must never be logged", + }); + expect(line).toBe("[live] call ended bot=bot-1 voice=sol client=ios seconds=84 delegations=3 sentToBot=2 answers=0 approvals=1 notHeard=1 replies=2 end=close_requested errors=rate_limit,badcoderm-rf"); + expect(line).not.toContain("never be logged"); + expect(liveCallSummaryLine({ seconds: -5, delegations: "x" })).toContain("seconds=0 delegations=0"); + expect(liveCallSummaryLine({ client: "desk top!" })).toContain(" client=desktop "); + expect(liveCallSummaryLine({})).toContain(" client=? "); + }); +}); + +describe("session config for the untrusted client", () => { + it("lets the client data channel send only session.close and receive captions", async () => { + const fetchImpl = okFetch(); + await createLiveSession({ key: "sk-test", sdp: "v=0\r\n", bot: { name: "Ada" } as never, history: [], fetchImpl }); + const body = JSON.parse(String(fetchImpl.mock.calls[0][1]?.body)); + expect(body.session.client).toEqual({ + data_channel: { + allowed_client_events: ["session.close"], + allowed_server_events: [ + { type: "session.started" }, + { type: "session.input_transcript.delta" }, + { type: "session.output_transcript.delta" }, + { type: "session.closed" }, + { type: "error" }, + { type: "info" }, + ], + }, + }); + expect(body.transport).toEqual({ type: "webrtc", sdp: "v=0\r\n" }); + }); +}); + +describe("live URLs", () => { + it("uses OpenAI unless a loopback test server is set", () => { + expect(liveSessionsUrl({})).toBe("https://api.openai.com/v1/live/sessions"); + expect(liveAttachUrl("sess_1", {})).toBe("wss://api.openai.com/v1/live/sessions/sess_1/attach"); + const env = { OMB_OPENAI_LIVE_URL: "http://127.0.0.1:4555" }; + expect(liveSessionsUrl(env)).toBe("http://127.0.0.1:4555/v1/live/sessions"); + expect(liveAttachUrl("sess_1", env)).toBe("ws://127.0.0.1:4555/v1/live/sessions/sess_1/attach"); + }); + it("ignores an override that is not loopback", () => { + expect(liveBaseUrl({ OMB_OPENAI_LIVE_URL: "http://evil.example:80" })).toBe("https://api.openai.com"); + }); + it("escapes the session id", () => { + expect(liveAttachUrl("a/b", {})).toBe("wss://api.openai.com/v1/live/sessions/a%2Fb/attach"); + }); +}); + +describe("liveErrorMessage", () => { + it("blames the voice only when OpenAI names the voice", () => { + expect(liveErrorMessage(400, "session.audio.output.voice")).toMatch(/voice/); + expect(liveErrorMessage(400, "session.client")).not.toMatch(/voice/); + expect(liveErrorMessage(400)).toMatch(/call settings/); + }); +}); diff --git a/server/live-call.ts b/server/live-call.ts new file mode 100644 index 0000000000..95215b5504 --- /dev/null +++ b/server/live-call.ts @@ -0,0 +1,252 @@ +// Live calls: OpenAI GPT-Live as the voice, the bot as the brain. +// +// The voice design doc (docs/voice-mode.md) rejected speech-to-speech models +// because "they replace the brain". GPT-Live's *client delegation* does not: +// the voice model only runs the conversation — listening while it speaks, +// taking interruptions, filling the wait — and hands every real request back +// to the application. Here that application is the harness, and the request +// becomes an ordinary turn on the bot the person called, with its own engine, +// tools, memory and approvals. Nothing about the bot changes. +// +// This file owns the one server-side step: exchanging the renderer's WebRTC +// offer for an answer. The OpenAI key never leaves the harness; the renderer +// only ever holds the SDP answer and the media connection it describes. + +export const LIVE_MODEL = "gpt-live-1"; +const OPENAI_BASE = "https://api.openai.com"; +export const LIVE_SESSIONS_URL = `${OPENAI_BASE}/v1/live/sessions`; +export const DEFAULT_LIVE_VOICE = "marin"; +/** GPT-Live built-in voices (openai-node 7.23 `BuiltInVoice`). */ +export const LIVE_VOICES = [ + "alloy", "ash", "ballad", "beacon", "bossa", "cedar", "cinder", "coral", "delta", "echo", "gleam", + "marin", "meridian", "quartz", "ripple", "sage", "shimmer", "stone", "tempo", "verse", "vesper", "willow", +] as const; +/** Largest SDP offer accepted from the renderer — a real offer is a few KB. */ +export const MAX_SDP_BYTES = 64 * 1024; + +/** The client's data channel is untrusted (it runs on a phone or in a + * renderer). It may only hang up; the harness sends every append over the + * sideband. Server events are limited to what captions and state need. */ +export const LIVE_DATA_CHANNEL = { + allowed_client_events: ["session.close"], + allowed_server_events: [ + { type: "session.started" }, + { type: "session.input_transcript.delta" }, + { type: "session.output_transcript.delta" }, + { type: "session.closed" }, + { type: "error" }, + { type: "info" }, + ], +}; + +/** Tests and fixtures point this at server/testing/fake-openai-live.ts. + * Only a loopback http URL is accepted, read at call time. */ +export function liveBaseUrl(env: NodeJS.ProcessEnv = process.env): string { + const override = env.OMB_OPENAI_LIVE_URL?.trim() ?? ""; + return /^http:\/\/127\.0\.0\.1:\d{1,5}$/.test(override) ? override : OPENAI_BASE; +} + +export function liveSessionsUrl(env: NodeJS.ProcessEnv = process.env): string { + return `${liveBaseUrl(env)}/v1/live/sessions`; +} + +export function liveAttachUrl(sessionId: string, env: NodeJS.ProcessEnv = process.env): string { + return `${liveBaseUrl(env).replace(/^http/, "ws")}/v1/live/sessions/${encodeURIComponent(sessionId)}/attach`; +} + +// Startup history is capped by the API at 128 messages / 8,192 tokens; stay +// well inside it. It is only there so "and the other one?" makes sense to +// the voice — the bot's own thread remains the source of truth. +const HISTORY_MESSAGES = 8; +const HISTORY_MESSAGE_CHARS = 600; +const HISTORY_TOTAL_CHARS = 6_000; +const CREATE_TIMEOUT_MS = 20_000; + +export interface LiveBot { + name: string; + title?: string; + description?: string; +} + +export interface LiveHistoryMessage { + role: "user" | "assistant"; + text: string; +} + +export class LiveSessionError extends Error { + readonly status: number; + constructor(message: string, status: number) { + super(message); + this.status = status; + } +} + +/** Frontend instructions for the voice. Business rules stay with the bot; + * this only says who is speaking and when to hand work over. Structure + * follows the GPT-Live prompting guide's template. */ +export function liveInstructions(bot: LiveBot): string { + const name = oneLine(bot.name) || "the agent"; + const title = oneLine(bot.title ?? ""); + const description = oneLine(bot.description ?? "").slice(0, 400); + return [ + `You are ${name}${title ? `, ${title}` : ""}, an AI agent that runs in OpenMausBot on the user's own computer.${description ? ` ${description}` : ""}`, + `To the user you are one assistant, ${name}, and you speak in the first person. Your work — looking things up, using your tools, files and memory, researching, deciding, and answering anything that needs facts this conversation does not hold — happens when you delegate. Delegating is how you think and act; it is not someone else.`, + "Never mention a backend, delegation, a voice layer, or another system or model doing the work, and never say you are only a voice. Say \"I\" about the work.", + "Never ask the user whether you may look something up or check something. When it needs checking, delegate at once and say briefly that you are checking.", + "Speak warmly and naturally, at an unhurried pace. Keep replies short: one or two sentences unless the user asks for more. Speak the language the user speaks.", + "", + "Backchannel policy: Use light backchannels. Acknowledge naturally without competing with the user.", + "Interruption policy: Stop speaking when the user interrupts. Listen to what they say.", + "Delegation policy:", + "Backend tools:", + `- ${name}: your own files, tools, memory, settings and this conversation's history. It researches, writes, changes things on the computer, and answers questions, including questions about yourself such as which AI model you run on, your settings, your tools and your memory.`, + "Delegate to the backend when:", + "- The user asks a question, asks for work, or gives an instruction.", + "- The user asks something about you that this conversation does not already answer, for example which AI model you run on.", + "- The user answers a question you asked, including yes or no to a permission request.", + "- A correction or a new detail changes work already requested.", + "Do not delegate to the backend when:", + "- You can answer from this conversation or from a result that is still current.", + "- The user only greets you, thanks you, or asks you to repeat something you already said.", + "- The user only asks whether you are still working or stuck.", + "- You cannot tell what they are asking for without a brief clarification.", + "Delegate before giving an answer that depends on backend work.", + "Do not guess the result while waiting. You may say briefly that you are on it, then wait for the update.", + "While you work you get quiet status notes: how long you have worked, how many steps you took, and your last step. Answer questions about progress from the latest note in one short sentence. Only say you are stuck when the note shows no new step for several minutes.", + "Never say that an action happened until the result reports it.", + "When a permission question comes up, ask it clearly and wait for a clear yes or no. Never answer it yourself.", + ].join("\n"); +} + +/** Recent thread text as startup history, newest last, inside the budget. */ +export function liveInitialInput(history: LiveHistoryMessage[]) { + const picked: LiveHistoryMessage[] = []; + let total = 0; + for (const message of [...history].reverse()) { + if (picked.length >= HISTORY_MESSAGES) break; + const text = message.text.replace(/\s+/g, " ").trim().slice(0, HISTORY_MESSAGE_CHARS); + if (!text) continue; + if (total + text.length > HISTORY_TOTAL_CHARS) break; + total += text.length; + picked.unshift({ role: message.role, text }); + } + return picked.map((message) => message.role === "user" + ? { type: "message" as const, role: "user" as const, content: [{ type: "input_text" as const, text: message.text }] } + : { type: "message" as const, role: "assistant" as const, content: [{ type: "output_text" as const, text: message.text }] }); +} + +/** The configured voice name, or the default. Names outside LIVE_VOICES are + * passed through: OpenAI adds voices faster than this list, and it rejects + * an unknown one with a clear 400 (see liveErrorMessage). */ +export function liveVoice(voice: string | undefined): string { + const name = voice?.trim().toLowerCase() ?? ""; + return /^[a-z]{2,40}$/.test(name) ? name : DEFAULT_LIVE_VOICE; +} + +export interface CreateLiveSessionInput { + key: string; + sdp: string; + bot: LiveBot; + history: LiveHistoryMessage[]; + voice?: string; + fetchImpl?: typeof fetch; + url?: string; + timeoutMs?: number; +} + +/** POST /v1/live/sessions with client delegation. Returns the session id + * and the SDP answer. Errors carry a status and a message fit for the user — + * never the key, never OpenAI's raw body. */ +export async function createLiveSession(input: CreateLiveSessionInput): Promise<{ sessionId: string; sdp: string }> { + const key = input.key.trim(); + if (!key) throw new LiveSessionError("Add an OpenAI API key to use Live calls.", 409); + if (!input.sdp.trim() || Buffer.byteLength(input.sdp) > MAX_SDP_BYTES) { + throw new LiveSessionError("The call could not start: the connection offer was invalid.", 400); + } + const initialInput = liveInitialInput(input.history); + const body = { + session: { + model: LIVE_MODEL, + instructions: liveInstructions(input.bot), + // client delegation: every request comes back to the harness, which + // runs it as a normal turn on the bot + delegation: { type: "client" }, + audio: { output: { voice: liveVoice(input.voice) } }, + client: { data_channel: LIVE_DATA_CHANNEL }, + ...(initialInput.length ? { input: initialInput } : {}), + }, + transport: { type: "webrtc", sdp: input.sdp }, + }; + let response: Response; + try { + response = await (input.fetchImpl ?? fetch)(input.url ?? liveSessionsUrl(), { + method: "POST", + headers: { authorization: `Bearer ${key}`, "content-type": "application/json" }, + body: JSON.stringify(body), + signal: AbortSignal.timeout(input.timeoutMs ?? CREATE_TIMEOUT_MS), + }); + } catch (error) { + const timedOut = error instanceof Error && (error.name === "TimeoutError" || error.name === "AbortError"); + throw new LiveSessionError( + timedOut ? "OpenAI did not answer in time. Try the call again." : "Could not reach OpenAI. Check the internet connection.", + 502, + ); + } + if (!response.ok) { + const failure = await response.json().catch(() => null) as { error?: { param?: unknown } } | null; + const param = typeof failure?.error?.param === "string" ? failure.error.param : undefined; + throw new LiveSessionError(liveErrorMessage(response.status, param), response.status === 429 ? 429 : 502); + } + const payload = await response.json().catch(() => null) as { session?: { id?: unknown }; transport?: { sdp?: unknown } } | null; + const sessionId = payload?.session?.id; + const sdp = payload?.transport?.sdp; + if (typeof sessionId !== "string" || typeof sdp !== "string" || !sdp.trim()) { + throw new LiveSessionError("OpenAI returned an unexpected answer. Try the call again.", 502); + } + return { sessionId, sdp }; +} + +export function liveErrorMessage(status: number, param?: string): string { + if (status === 400) { + return param && /voice/i.test(param) + ? "OpenAI rejected the voice for this call. Choose another voice." + : `OpenAI rejected the call settings (HTTP 400${param ? `, ${param.replace(/[^\w.]/g, "").slice(0, 60)}` : ""}).`; + } + if (status === 401) return "OpenAI rejected the API key. Check the key for Live calls."; + if (status === 403) return "This OpenAI project has no access to GPT-Live. Check the project's model access and billing."; + if (status === 404) return "GPT-Live is not available for this OpenAI project."; + if (status === 429) return "OpenAI is limiting Live sessions right now (rate limit or quota). Try again in a moment."; + if (status >= 500) return "OpenAI had a problem starting the call. Try again in a moment."; + return `OpenAI could not start the call (HTTP ${status}).`; +} + +function oneLine(value: string): string { + return value.replace(/\s+/g, " ").trim(); +} + +/** One server.log line per finished Live call, from the call's counters + * (LiveCallController). Numbers and short codes only — never what anyone + * said. */ +export function liveCallSummaryLine(body: Record): string { + const count = (value: unknown) => { + const n = Number(value); + return Number.isFinite(n) && n >= 0 ? Math.min(Math.round(n), 1_000_000) : 0; + }; + const code = (value: unknown) => typeof value === "string" ? value.replace(/[^\w.-]/g, "").slice(0, 40) : ""; + const errors = Array.isArray(body.errors) ? body.errors.slice(0, 5).map(code).filter(Boolean) : []; + return [ + "[live] call ended", + `bot=${code(body.botId) || "?"}`, + `voice=${code(body.voice) || DEFAULT_LIVE_VOICE}`, + `client=${code(body.client) || "?"}`, + `seconds=${count(body.seconds)}`, + `delegations=${count(body.delegations)}`, + `sentToBot=${count(body.sentToBot)}`, + `answers=${count(body.answers)}`, + `approvals=${count(body.approvals)}`, + `notHeard=${count(body.notHeard)}`, + `replies=${count(body.replies)}`, + `end=${code(body.end) || "unknown"}`, + `errors=${errors.length ? errors.join(",") : "none"}`, + ].join(" "); +} diff --git a/server/message-db.ts b/server/message-db.ts index ae1724e3f9..adf4f25715 100644 --- a/server/message-db.ts +++ b/server/message-db.ts @@ -228,7 +228,9 @@ export interface FollowupPayload { unattended?: boolean; peerAsk?: Message["peerAsk"]; mode?: "chat" | "goal"; - via?: "api"; + /** "api": a room line sent through the local API with no session behind + * it. "call": a person's words relayed from a Live call. */ + via?: "api" | "call"; /** Who queued these words. Absent on the owner's own sends and on every * row written before this existed; both read as the profile name. */ sender?: ResolvedSender; diff --git a/server/request-auth.test.ts b/server/request-auth.test.ts index 84b2ec7b22..70c7b57f87 100644 --- a/server/request-auth.test.ts +++ b/server/request-auth.test.ts @@ -118,6 +118,9 @@ describe("scopes", () => { ["POST", "/api/bots/x/checkpoints/restore"], ["GET", "/api/mcp/servers"], ["POST", "/api/mcp/servers"], ["POST", "/api/connectors/slack/authorize"], ["POST", "/api/bots/x/slack-management"], ["GET", "/api/bots/x/slack-management/extra"], ["PUT", "/api/config"], ["POST", "/api/auth/pairing"], ["GET", "/api/auth/sessions"], ["DELETE", "/api/auth/sessions/abc"], + // Live calls spend the owner's OpenAI key and reach any bot: admins only + ["POST", "/api/live/session"], ["POST", "/api/live/call/end"], ["GET", "/api/live/call"], ["PATCH", "/api/live/settings"], + ["POST", "/api/live/device-revoked"], ["POST", "/api/auth/pair"], // handled before the gate; the gate itself never grants it ["GET", "/api/something-new"], // anything unlisted is admin until listed ] as const) expect(requiredScope(method, path), `${method} ${path}`).toBe("admin"); @@ -182,6 +185,11 @@ describe("resolveRequestAuth", () => { ["POST", "/api/internal/anything"], ["GET", "/api/auth/sessions"], ["POST", "/api/not-yet-supported"], ]) expect(check(method, path).auth, path).toBeNull(); + // The companion's own notice that it unpaired a phone: not on the phone + // allowlist, but the relay's private token opens it; a forged one does not. + expect(check("POST", "/api/live/device-revoked").auth?.kind).toBe("loopback"); + for (const overrides of forged) expect(check("POST", "/api/live/device-revoked", overrides).auth).toBeNull(); + expect(check("GET", "/api/live/device-revoked").auth).toBeNull(); }); function pairedToken(scopes: Array<"admin" | "client"> = ["admin", "client"]): string { diff --git a/server/request-auth.ts b/server/request-auth.ts index df23a8ff98..5d6dc629dc 100644 --- a/server/request-auth.ts +++ b/server/request-auth.ts @@ -13,7 +13,7 @@ import { timingSafeEqual } from "node:crypto"; import { isIP } from "node:net"; import type { Scope, SessionRecord, SessionRegistry } from "./sessions.ts"; -import { denyReason as companionDenial } from "../companion/src/routes.ts"; +import { denyReason as companionDenial, isCompanionNotice } from "../companion/src/routes.ts"; /** How much a loopback request without a session is trusted. * @@ -516,7 +516,8 @@ export function resolveRequestAuth(req: IncomingMessage, options: ResolveOptions !secureTokenMatch(companionToken, options.companionMutationToken ?? "") || req.headers["x-openmausbot-companion"] !== "1" || !/^[\w-]{1,128}$/.test(headerValue(req.headers["x-openmausbot-companion-device"]) ?? "") || - companionDenial({ path, method, authenticated: true }) + // a phone's allowlisted route, or the companion's own notice (an unpaired phone) + (companionDenial({ path, method, authenticated: true }) && !isCompanionNotice(method, path)) ) return deny(403, "forbidden: invalid companion request"); return { auth: { kind: "loopback", scopes: LOOPBACK_SCOPES }, status: 401, error: "" }; } diff --git a/server/routes/live.test.ts b/server/routes/live.test.ts new file mode 100644 index 0000000000..4194c76753 --- /dev/null +++ b/server/routes/live.test.ts @@ -0,0 +1,195 @@ +// The Live call routes, the way a request reaches them: through the table, +// on a real HTTP server, with a stand-in for index.ts's inline routes behind +// it. The controller and the config are stand-ins; their own tests cover them. +import { createServer, type Server } from "node:http"; +import type { AddressInfo } from "node:net"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import type { LiveCallState, LiveSettings } from "../../shared/wire.ts"; +import { json, readBody } from "../harness/http.ts"; +import { LiveCallBusyError, LiveCallSignedOutError } from "../live-call-controller.ts"; +import { LiveSessionError } from "../live-call.ts"; +import { requiredScope } from "../request-auth.ts"; +import { createLiveRoutes, type LiveRouteDeps } from "./live.ts"; +import { dispatchRoutes } from "./table.ts"; + +type SettingsPatch = Parameters[0]; + +const servers: Server[] = []; +afterEach(async () => { + await Promise.all(servers.splice(0).map((server) => new Promise((done) => server.close(done)))); +}); + +async function serve(deps: LiveRouteDeps): Promise { + const routes = [createLiveRoutes(deps)]; + const server = createServer(async (req, res) => { + const url = new URL(req.url ?? "/", "http://localhost"); + const handled = await dispatchRoutes(routes, { + req, res, url, path: url.pathname, method: req.method ?? "GET", + auth: { kind: "loopback", scopes: ["admin", "client"] }, json, readBody, + }); + if (!handled) json(res, 404, { from: "inline routes" }); + }); + servers.push(server); + await new Promise((ready) => server.listen(0, "127.0.0.1", ready)); + return `http://127.0.0.1:${(server.address() as AddressInfo).port}`; +} + +async function request(deps: LiveRouteDeps, method: string, path: string, body?: unknown, headers: Record = {}): Promise<{ status: number; body: unknown }> { + const base = await serve(deps); + const response = await fetch(`${base}${path}`, { + method, + headers: { ...(body === undefined ? {} : { "content-type": "application/json" }), ...headers }, + ...(body === undefined ? {} : { body: typeof body === "string" ? body : JSON.stringify(body) }), + }); + return { status: response.status, body: await response.json() }; +} + +/** What the companion adds to a paired phone's request (companion/src/proxy.ts). */ +const fromPhone = (device = "phone-1") => ({ "x-openmausbot-companion": "1", "x-openmausbot-companion-device": device }); + +function deps(overrides: Partial = {}): LiveRouteDeps & { saved: SettingsPatch[] } { + const saved: SettingsPatch[] = []; + const call: LiveCallState = { callId: "c1", botId: "bot1", threadId: "t1", client: "desktop", voice: "marin", startedAt: 1, status: "connecting" }; + return { + saved, + calls: { + start: vi.fn(async () => ({ call, sdp: "answer" })), + end: vi.fn(async (id: string) => (id === "c1" ? { ...call, status: "ended" as const, endReason: "hung-up" as const } : null)), + current: () => call, + deviceRevoked: vi.fn((device: string) => (device === "phone-1" ? { ...call, client: "ios" as const, status: "ending" as const } : null)), + }, + resolveTarget: (botId, threadId) => (botId === "bot1" && (threadId ?? "t1") === "t1" ? { botId, botName: "Ada", threadId: threadId ?? "t1" } : null), + settings: () => ({ configured: true, voice: "marin", readTypedReplies: true, idleMinutes: 5 }), + saveSettings: vi.fn(async (patch: SettingsPatch): Promise => { + saved.push(patch); + return { configured: true, voice: "marin", readTypedReplies: true, idleMinutes: 5, ...patch }; + }), + ...overrides, + }; +} + +describe("live routes", () => { + it("starts a call and returns the answer", async () => { + const d = deps(); + const res = await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0\r\n", client: "ios" }); + expect(res.status).toBe(201); + expect(res.body).toEqual({ call: expect.objectContaining({ callId: "c1" }), transport: { type: "webrtc", sdp: "answer" } }); + expect(d.calls.start).toHaveBeenCalledWith(expect.objectContaining({ botId: "bot1", botName: "Ada", threadId: "t1", client: "ios", sdp: "v=0\r\n" })); + }); + it("starts the call as the person who asked", async () => { + const d = deps(); + await request(d, "POST", "/api/live/session", { botId: "bot1", threadId: "t1", sdp: "v=0\r\n", client: "android" }); + expect(d.calls.start).toHaveBeenCalledWith(expect.objectContaining({ auth: { kind: "loopback", scopes: ["admin", "client"] }, threadId: "t1", client: "android" })); + }); + it("keeps the SDP byte for byte", async () => { + const d = deps(); + await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0\r\na=x \r\n", client: "desktop" }); + expect(d.calls.start).toHaveBeenCalledWith(expect.objectContaining({ sdp: "v=0\r\na=x \r\n" })); + }); + it("rejects bad bodies and unknown targets", async () => { + expect((await request(deps(), "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "fridge" })).status).toBe(400); + expect((await request(deps(), "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios", extra: 1 })).status).toBe(400); + expect((await request(deps(), "POST", "/api/live/session", { botId: "bot1", sdp: "", client: "ios" })).status).toBe(400); + expect((await request(deps(), "POST", "/api/live/session", { botId: "bot1", sdp: "x".repeat(64 * 1024 + 1), client: "ios" })).status).toBe(400); + expect((await request(deps(), "POST", "/api/live/session", { botId: "../bot1", sdp: "v=0", client: "ios" })).status).toBe(400); + expect((await request(deps(), "POST", "/api/live/session", "{not json")).status).toBe(400); + expect((await request(deps(), "POST", "/api/live/session", { botId: "nope", sdp: "v=0", client: "ios" })).status).toBe(404); + expect((await request(deps(), "POST", "/api/live/session", { botId: "bot1", threadId: "other", sdp: "v=0", client: "ios" })).status).toBe(404); + }); + it("does not start a call for a bad body or an unknown target", async () => { + const d = deps(); + await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "fridge" }); + await request(d, "POST", "/api/live/session", { botId: "nope", sdp: "v=0", client: "ios" }); + expect(d.calls.start).not.toHaveBeenCalled(); + }); + it("says when a key is needed and when a call is already running", async () => { + const noKey = deps({ calls: { ...deps().calls, start: vi.fn(async () => { throw new LiveSessionError("Add an OpenAI API key to use Live calls.", 409); }) } }); + expect(await request(noKey, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios" })).toMatchObject({ status: 409, body: { needsKey: true } }); + const active = { callId: "c0", botId: "bot2", threadId: "t2", client: "android", voice: "marin", startedAt: 1, status: "live" } as const; + const busy = deps({ calls: { ...deps().calls, start: vi.fn(async () => { throw new LiveCallBusyError(active); }) } }); + const refused = await request(busy, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios" }); + expect(refused).toMatchObject({ status: 409, body: { activeCall: active } }); + expect(refused.body).not.toHaveProperty("needsKey"); + }); + it("passes OpenAI's refusals through with their status and message", async () => { + const refusal = deps({ calls: { ...deps().calls, start: vi.fn(async () => { throw new LiveSessionError("OpenAI rejected the API key. Check the key for Live calls.", 502); }) } }); + expect(await request(refusal, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "desktop" })) + .toEqual({ status: 502, body: { error: "OpenAI rejected the API key. Check the key for Live calls." } }); + }); + it("ends a call by id and 404s for another id", async () => { + expect(await request(deps(), "POST", "/api/live/call/end", { callId: "c1" })).toMatchObject({ status: 200, body: { call: { status: "ended" } } }); + expect((await request(deps(), "POST", "/api/live/call/end", { callId: "zz" })).status).toBe(404); + expect((await request(deps(), "POST", "/api/live/call/end", {})).status).toBe(400); + }); + it("reports the current call, or none", async () => { + expect(await request(deps(), "GET", "/api/live/call")).toMatchObject({ status: 200, body: { call: { callId: "c1" } } }); + expect(await request(deps({ calls: { ...deps().calls, current: () => null } }), "GET", "/api/live/call")).toEqual({ status: 200, body: { call: null } }); + }); + it("saves non-secret settings and refuses the key", async () => { + const d = deps(); + expect(await request(d, "PATCH", "/api/live/settings", { idleMinutes: 10, readTypedReplies: false })).toMatchObject({ status: 200, body: { live: { idleMinutes: 10, readTypedReplies: false } } }); + expect((await request(d, "PATCH", "/api/live/settings", { key: "sk-x" })).status).toBe(400); + expect((await request(d, "PATCH", "/api/live/settings", { idleMinutes: 0 })).status).toBe(400); + expect((await request(d, "PATCH", "/api/live/settings", { idleMinutes: 61 })).status).toBe(400); + expect((await request(d, "PATCH", "/api/live/settings", { voice: "Marin!" })).status).toBe(400); + expect(d.saved).toEqual([{ idleMinutes: 10, readTypedReplies: false }]); + }); + it("saves nothing for an empty patch", async () => { + const d = deps(); + expect(await request(d, "PATCH", "/api/live/settings", {})).toEqual({ status: 400, body: { error: "nothing to save" } }); + expect(d.saveSettings).not.toHaveBeenCalled(); + }); + it("never answers with the key", async () => { + const res = await request(deps(), "PATCH", "/api/live/settings", { voice: "cedar" }); + expect(res).toEqual({ status: 200, body: { live: { configured: true, voice: "cedar", readTypedReplies: true, idleMinutes: 5 } } }); + }); + it("passes other paths and methods", async () => { + expect(await request(deps(), "GET", "/api/live/summary")).toEqual({ status: 404, body: { from: "inline routes" } }); + expect(await request(deps(), "POST", "/api/live/summary", {})).toEqual({ status: 404, body: { from: "inline routes" } }); + expect(await request(deps(), "GET", "/api/live/session")).toEqual({ status: 404, body: { from: "inline routes" } }); + expect(await request(deps(), "PUT", "/api/live/settings", {})).toEqual({ status: 404, body: { from: "inline routes" } }); + expect(await request(deps(), "GET", "/api/bots")).toEqual({ status: 404, body: { from: "inline routes" } }); + }); + it("is for admins only", () => { + expect(requiredScope("POST", "/api/live/session")).toBe("admin"); + expect(requiredScope("POST", "/api/live/call/end")).toBe("admin"); + expect(requiredScope("GET", "/api/live/call")).toBe("admin"); + expect(requiredScope("PATCH", "/api/live/settings")).toBe("admin"); + expect(requiredScope("POST", "/api/live/device-revoked")).toBe("admin"); + }); + + describe("a call from a paired phone", () => { + it("is bound to the phone the companion vouched for", async () => { + const d = deps(); + await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios" }, fromPhone()); + expect(d.calls.start).toHaveBeenCalledWith(expect.objectContaining({ device: "phone-1", client: "ios" })); + }); + it("is bound to no phone without the companion's word, or with a malformed id", async () => { + const d = deps(); + await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "desktop" }); + await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios" }, { "x-openmausbot-companion-device": "phone-1" }); + await request(d, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios" }, fromPhone("../phone")); + for (const [input] of vi.mocked(d.calls.start).mock.calls) expect(input.device).toBeUndefined(); + }); + it("is refused once its phone was unpaired", async () => { + const refused = deps({ calls: { ...deps().calls, start: vi.fn(async () => { throw new LiveCallSignedOutError(); }) } }); + expect(await request(refused, "POST", "/api/live/session", { botId: "bot1", sdp: "v=0", client: "ios" }, fromPhone())) + .toEqual({ status: 401, body: { error: "The sign-in that started this call has ended." } }); + }); + it("ends when the companion says its phone was unpaired", async () => { + const d = deps(); + expect(await request(d, "POST", "/api/live/device-revoked", undefined, fromPhone())) + .toMatchObject({ status: 200, body: { call: { status: "ending" } } }); + expect(d.calls.deviceRevoked).toHaveBeenCalledWith("phone-1"); + expect(await request(d, "POST", "/api/live/device-revoked", undefined, fromPhone("phone-2"))).toEqual({ status: 200, body: { call: null } }); + }); + it("takes the unpairing only from the companion, for a well-formed phone", async () => { + const d = deps(); + expect((await request(d, "POST", "/api/live/device-revoked", undefined, { "x-openmausbot-companion-device": "phone-1" })).status).toBe(403); + expect((await request(d, "POST", "/api/live/device-revoked", undefined, { "x-openmausbot-companion": "1" })).status).toBe(400); + expect((await request(d, "POST", "/api/live/device-revoked", undefined, fromPhone("a/b"))).status).toBe(400); + expect(d.calls.deviceRevoked).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/server/routes/live.ts b/server/routes/live.ts new file mode 100644 index 0000000000..96a8454ac7 --- /dev/null +++ b/server/routes/live.ts @@ -0,0 +1,114 @@ +// Live calls (GPT-Live as the voice, the bot as the brain). The call itself +// runs in LiveCallController; these routes start it, end it, report it and +// change its non-secret settings. The OpenAI key is never read or written +// here. All are admin-scoped by default in server/request-auth.ts; the +// companion allows the first four for the phone apps (companion/src/routes.ts). +// The fifth, device-revoked, is the companion's own notice that it unpaired a +// phone; a phone can never send it (companion/src/routes.ts COMPANION_NOTICES). +import type { IncomingMessage } from "node:http"; +import { z } from "zod"; +import type { LiveSettings } from "../../shared/wire.ts"; +import { LiveSessionError, MAX_SDP_BYTES } from "../live-call.ts"; +import { LiveCallBusyError, type LiveCallController } from "../live-call-controller.ts"; +import { PASS, type RouteContext, type RouteHandler } from "./table.ts"; + +export interface LiveRouteDeps { + calls: Pick; + /** The bot and chat a call goes to; no threadId means the bot's current chat. Null when either is unknown. */ + resolveTarget(botId: string, threadId: string | undefined): { botId: string; botName: string; threadId: string } | null; + settings(): LiveSettings; + saveSettings(patch: { voice?: string; readTypedReplies?: boolean; idleMinutes?: number }): Promise; +} + +const id = z.string().regex(/^[\w-]{1,120}$/); +const sessionBody = z.object({ + botId: id, + threadId: id.optional(), + // Not trimmed: the SDP offer goes to OpenAI byte for byte. + sdp: z.string().min(1).refine((sdp) => Buffer.byteLength(sdp) <= MAX_SDP_BYTES), + client: z.enum(["desktop", "ios", "android"]), +}).strict(); +const endBody = z.object({ callId: z.string().min(1).max(120) }).strict(); +// Strict, so `key` (or anything else) is refused rather than ignored: the key +// is only ever set on the computer that runs the harness. +const settingsBody = z.object({ + voice: z.string().trim().max(40).regex(/^[a-z]*$/).optional(), + readTypedReplies: z.boolean().optional(), + idleMinutes: z.number().int().min(1).max(60).optional(), +}).strict(); + +const DEVICE_ID = /^[\w-]{1,128}$/; + +/** The paired phone the companion vouched for, or undefined. The companion + * sends its own id for the phone that authenticated (never one the phone + * chose); in the desktop app the harness has already checked the companion's + * private token before this runs (server/request-auth.ts). */ +function companionDevice(req: IncomingMessage): string | undefined { + if (req.headers["x-openmausbot-companion"] !== "1") return undefined; + const device = req.headers["x-openmausbot-companion-device"]; + return typeof device === "string" && DEVICE_ID.test(device) ? device : undefined; +} + +/** The parsed JSON body, or undefined when it is not JSON (the schema then refuses it). */ +async function bodyOf(req: IncomingMessage, readBody: RouteContext["readBody"]): Promise { + try { + return await readBody(req); + } catch { + return undefined; + } +} + +export function createLiveRoutes(deps: LiveRouteDeps): RouteHandler { + return async ({ req, res, path, method, auth, json, readBody }) => { + if (!path.startsWith("/api/live/")) return PASS; + + if (method === "POST" && path === "/api/live/session") { + const parsed = sessionBody.safeParse(await bodyOf(req, readBody)); + if (!parsed.success) return json(res, 400, { error: "The call request was not valid." }); + const target = deps.resolveTarget(parsed.data.botId, parsed.data.threadId); + if (!target) return json(res, 404, { error: "That bot or chat does not exist." }); + try { + // A phone's call is bound to the phone, so unpairing it ends the call. + const device = companionDevice(req); + const { call, sdp } = await deps.calls.start({ auth, ...(device ? { device } : {}), ...target, client: parsed.data.client, sdp: parsed.data.sdp }); + return json(res, 201, { call, transport: { type: "webrtc", sdp } }); + } catch (error) { + if (error instanceof LiveCallBusyError) return json(res, 409, { error: error.message, activeCall: error.call }); + if (error instanceof LiveSessionError) { + // The one 409 a session refuses with is a missing key. + return json(res, error.status, error.status === 409 ? { error: error.message, needsKey: true } : { error: error.message }); + } + throw error; + } + } + + if (method === "POST" && path === "/api/live/call/end") { + const parsed = endBody.safeParse(await bodyOf(req, readBody)); + if (!parsed.success) return json(res, 400, { error: "The request was not valid." }); + const call = await deps.calls.end(parsed.data.callId); + return call ? json(res, 200, { call }) : json(res, 404, { error: "That call is not running." }); + } + + if (method === "GET" && path === "/api/live/call") return json(res, 200, { call: deps.calls.current() }); + + // The companion unpaired a phone: end the call that phone holds, if any. + if (method === "POST" && path === "/api/live/device-revoked") { + if (req.headers["x-openmausbot-companion"] !== "1") return json(res, 403, { error: "Only the phone companion can report an unpaired phone." }); + const device = companionDevice(req); + if (!device) return json(res, 400, { error: "The unpaired phone was not named." }); + return json(res, 200, { call: deps.calls.deviceRevoked(device) }); + } + + if (method === "PATCH" && path === "/api/live/settings") { + const parsed = settingsBody.safeParse(await bodyOf(req, readBody)); + if (!parsed.success) { + return json(res, 400, { error: "Those Live settings are not valid. The OpenAI key can only be changed on the computer that runs OpenMausBot." }); + } + // as PUT /api/config: an empty patch is not saved (or broadcast) + if (!Object.keys(parsed.data).length) return json(res, 400, { error: "nothing to save" }); + return json(res, 200, { live: await deps.saveSettings(parsed.data) }); + } + + return PASS; + }; +} diff --git a/server/steer-queue.test.ts b/server/steer-queue.test.ts index 7e0c9a9507..6d0514d3f0 100644 --- a/server/steer-queue.test.ts +++ b/server/steer-queue.test.ts @@ -23,6 +23,7 @@ import { drainSteeredMessages, hasQueuedSteeredMessages, holdSteeredQueue, + isSteeredMessageQueued, onSteeredQueueChange, queuedThreadPosition, queuedSteerSnapshot, @@ -171,6 +172,23 @@ describe("steer-queue module", () => { expect(run.mock.calls[0][3].sender).toBeUndefined(); }); + it("keeps a call's via on the message it drains", () => { + const bot = fakeBot("bot-via-drain", "thread-via-drain", true); + const store = fakeStore([bot]); + const run = vi.fn(); + queueSteeredMessage(bot.id, bot.threadId, "typed while the bot worked"); + queueSteeredMessage(bot.id, bot.threadId, "what is on my calendar", { via: "call" }); + restoreSteeredMessages(); // a restart reads via back from the durable row + bot.busy = false; + drainSteeredMessages(store, run); + expect(store.messages.map((message) => [message.text, message.via])).toEqual([ + ["typed while the bot worked", undefined], + ["what is on my calendar", "call"], + ]); + expect("via" in store.messages[0]).toBe(false); + expect(run.mock.calls[0][3].via).toBe("call"); + }); + it("still loads and drains a durable row written before senders were kept", () => { const bot = fakeBot("bot-sender-legacy", "thread-sender-legacy", false); const store = fakeStore([bot]); @@ -421,6 +439,27 @@ describe("steer-queue module", () => { expect(run).toHaveBeenCalledTimes(1); }); + // A Live call waits for a spoken request it saw queued; it asks here + // whether that one send still waits, since an edit or cancel removes it + // without ever delivering it. + it("tells whether one send still waits in its thread's queue", () => { + const bot = fakeBot("bot-waiting", "thread-waiting", true); + const store = fakeStore([bot]); + const kept = queueSteeredMessage(bot.id, bot.threadId, "keep waiting"); + const edited = queueSteeredMessage(bot.id, bot.threadId, "edited away"); + expect(isSteeredMessageQueued(bot.id, bot.threadId, kept.id)).toBe(true); + expect(isSteeredMessageQueued("other-bot", bot.threadId, kept.id)).toBe(false); + expect(isSteeredMessageQueued(bot.id, "other-thread", kept.id)).toBe(false); + // editing a queued line cancels it first + expect(cancelSteeredMessage(bot.id, edited.id, bot.threadId)).toBe(true); + expect(isSteeredMessageQueued(bot.id, bot.threadId, edited.id)).toBe(false); + expect(isSteeredMessageQueued(bot.id, bot.threadId, kept.id)).toBe(true); + bot.busy = false; + drainSteeredMessages(store, vi.fn()); + expect(store.messages.map((m) => m.queueId)).toEqual([kept.id]); + expect(isSteeredMessageQueued(bot.id, bot.threadId, kept.id)).toBe(false); + }); + it("drops a cancelled message so drain does not send it", () => { const bot = fakeBot("bot-cancel", "thread-cancel", true); const store = fakeStore([bot]); diff --git a/server/steer-queue.ts b/server/steer-queue.ts index bc839cc5ff..b1ad16f872 100644 --- a/server/steer-queue.ts +++ b/server/steer-queue.ts @@ -56,6 +56,9 @@ interface QueueEntry { /** When the words were queued (epoch ms): drain-time coalescing splits * one sender's items when the gap between them outgrows the window. */ queuedAt: number; + /** The words were spoken in a Live call, not typed: the drained line + * says so, like an immediate send would. */ + via?: "call"; }>; } @@ -77,8 +80,11 @@ export function restoreSteeredMessages(): void { if (row.status !== "pending") continue; const entry = queues.get(row.threadId) ?? { botId: row.ownerId, items: [] }; if (entry.botId !== row.ownerId) throw new Error("queued task belongs to another bot"); + // a 1:1 line is only ever stamped "call"; "api" belongs to channel rows + const { via, ...payload } = row.payload; entry.items.push({ - ...row.payload, + ...payload, + ...(via === "call" ? { via } : {}), messageId: row.id, prompt: row.payload.prompt ?? row.payload.text, // rows queued before timestamps were kept read as queued at restore @@ -121,7 +127,7 @@ export function queueSteeredMessage( botId: string, threadId: string, text: string, - options: { prompt?: string; replyToId?: string; sendId?: string; reason?: SteerQueueReason; unattended?: boolean; peerAsk?: Message["peerAsk"]; sender?: ResolvedSender; trigger?: UsageTrigger } = {}, + options: { prompt?: string; replyToId?: string; sendId?: string; reason?: SteerQueueReason; unattended?: boolean; peerAsk?: Message["peerAsk"]; sender?: ResolvedSender; trigger?: UsageTrigger; via?: "call" } = {}, ): QueuedSteer { const id = newId(); const entry = queues.get(threadId) ?? { botId, items: [] }; @@ -140,6 +146,7 @@ export function queueSteeredMessage( sender: options.sender, trigger: options.trigger, queuedAt: Date.now(), + via: options.via, }; saveChatFollowup({ id, kind: "bot", ownerId: botId, threadId, payload: item }); entry.items.push(item); @@ -170,6 +177,14 @@ export function hasQueuedSteeredMessages(botId: string, threadId: string): boole return entry?.botId === botId && entry.items.length > 0; } +/** Whether one send still waits in its thread's queue. False while it is + * out of the queue: drained onto the thread, held for a steer into the + * running turn, or cancelled (editing a queued line cancels it). */ +export function isSteeredMessageQueued(botId: string, threadId: string, queueId: string): boolean { + const entry = queues.get(threadId); + return entry?.botId === botId && entry.items.some((item) => item.messageId === queueId); +} + /** Drain every queue whose task is idle: append the held lines (leaf is now * the finished turn's last item), then one run per thread whose prompt is * the drained group's texts separated by a blank line. `userMessage` is the last appended line @@ -229,6 +244,7 @@ export function drainSteeredMessages( queueId: item.messageId, peerAsk: item.peerAsk, sender: item.sender, + ...(item.via ? { via: item.via } : {}), }), ); } diff --git a/server/testing/fake-openai-live.test.ts b/server/testing/fake-openai-live.test.ts new file mode 100644 index 0000000000..4f6358e0b0 --- /dev/null +++ b/server/testing/fake-openai-live.test.ts @@ -0,0 +1,177 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { fakeAnswerSdp, startFakeOpenAiLive, type FakeOpenAiLive } from "./fake-openai-live.ts"; + +let fake: FakeOpenAiLive | undefined; +afterEach(async () => { await fake?.stop(); fake = undefined; }); + +async function create(url: string, key = "sk-fake") { + return fetch(`${url}/v1/live/sessions`, { + method: "POST", + headers: { authorization: `Bearer ${key}`, "content-type": "application/json" }, + body: JSON.stringify({ session: { model: "gpt-live-1" }, transport: { type: "webrtc", sdp: "v=0\r\n" } }), + }); +} + +function attach(url: string, id: string, key = "sk-fake") { + const ws = new WebSocket(`${url.replace(/^http/, "ws")}/v1/live/sessions/${id}/attach`, { headers: { authorization: `Bearer ${key}` } } as never); + const received: Array> = []; + ws.addEventListener("message", (event) => received.push(JSON.parse(String((event as MessageEvent).data)))); + return { ws, received }; +} + +const until = async (check: () => boolean, ms = 3_000) => { + const end = Date.now() + ms; + while (!check()) { if (Date.now() > end) throw new Error("timeout"); await new Promise((r) => setTimeout(r, 10)); } +}; + +describe("fake GPT-Live", () => { + it("creates a session and records the body", async () => { + fake = await startFakeOpenAiLive(); + const response = await create(fake.url); + expect(response.status).toBe(201); + const payload = await response.json() as { session: { id: string }; transport: { sdp: string } }; + expect(payload.session.id).toMatch(/^sess_fake_/); + expect(payload.transport.sdp).toContain("v=0"); + expect(fake.sessions[0].body).toMatchObject({ session: { model: "gpt-live-1" } }); + }); + + it("refuses a create without a key and can fail on demand", async () => { + fake = await startFakeOpenAiLive(); + expect((await create(fake.url, "")).status).toBe(401); + fake.failNextCreate(400, { error: { param: "session.audio.output.voice" } }); + expect((await create(fake.url)).status).toBe(400); + }); + + it("attaches a sideband, plays events and records commands", async () => { + fake = await startFakeOpenAiLive(); + const { session } = await (await create(fake.url)).json() as { session: { id: string } }; + const { ws, received } = attach(fake.url, session.id); + await fake.waitForAttach(session.id); + await until(() => received.some((e) => e.type === "session.started")); + fake.emit(session.id, { type: "session.input_transcript.delta", delta: "hello", start_ms: 10, end_ms: 20 }); + await until(() => received.some((e) => e.type === "session.input_transcript.delta")); + ws.send(JSON.stringify({ type: "session.thinking.append", delegation_id: null, content: "x".repeat(70_000) })); + const command = await fake.waitForCommand(session.id, (c) => c.type === "session.thinking.append"); + expect(String(command.content)).toHaveLength(70_000); + ws.send(JSON.stringify({ type: "session.close" })); + await until(() => received.some((e) => e.type === "session.closed")); + await until(() => ws.readyState === WebSocket.CLOSED); + }); + + it("refuses an attach with an HTTP status and can drop the socket", async () => { + fake = await startFakeOpenAiLive(); + const { session } = await (await create(fake.url)).json() as { session: { id: string } }; + fake.refuseNextAttach(404); + const refused = attach(fake.url, session.id); + const errored = new Promise((resolve) => refused.ws.addEventListener("error", () => resolve())); + await errored; + const { ws } = attach(fake.url, session.id); + await fake.waitForAttach(session.id); + const closed = new Promise((resolve) => ws.addEventListener("close", () => resolve())); + fake.dropSideband(session.id); + await closed; + }); +}); + +// A browser's offer: audio (Opus first) and the events data channel, bundled. +const OFFER = [ + "v=0", + "o=- 4611731400430051336 2 IN IP4 127.0.0.1", + "s=-", + "t=0 0", + "a=group:BUNDLE 0 1", + "a=extmap-allow-mixed", + "a=msid-semantic: WMS stream", + "m=audio 9 UDP/TLS/RTP/SAVPF 111 63 9 0 8 13 110 126", + "c=IN IP4 0.0.0.0", + "a=rtcp:9 IN IP4 0.0.0.0", + "a=ice-ufrag:abcd", + "a=ice-pwd:0123456789abcdefghijklmn", + "a=ice-options:trickle", + "a=fingerprint:sha-256 AA:BB", + "a=setup:actpass", + "a=mid:0", + "a=sendrecv", + "a=rtcp-mux", + "a=rtpmap:111 opus/48000/2", + "a=rtcp-fb:111 transport-cc", + "a=fmtp:111 minptime=10;useinbandfec=1", + "a=rtpmap:63 red/48000/2", + "a=fmtp:63 111/111", + "a=rtpmap:9 G722/8000", + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel", + "c=IN IP4 0.0.0.0", + "a=ice-ufrag:abcd", + "a=ice-pwd:0123456789abcdefghijklmn", + "a=fingerprint:sha-256 AA:BB", + "a=setup:actpass", + "a=mid:1", + "a=sctp-port:5000", + "a=max-message-size:262144", + "", +].join("\r\n"); + +describe("fake GPT-Live answer", () => { + const lines = (sdp: string) => sdp.split("\r\n").filter(Boolean); + + it("answers every offered section in order, with the same mids, bundled", () => { + const answer = fakeAnswerSdp(OFFER); + expect(answer.endsWith("\r\n")).toBe(true); + // CRLF only: no bare LF anywhere + expect(answer.replaceAll("\r\n", "")).not.toContain("\n"); + const all = lines(answer); + expect(all.slice(0, 4)).toEqual(["v=0", expect.stringMatching(/^o=- \d+ 2 IN IP4 127\.0\.0\.1$/), "s=-", "t=0 0"]); + expect(all).toContain("a=group:BUNDLE 0 1"); + expect(all.filter((line) => line.startsWith("m="))).toEqual([ + "m=audio 9 UDP/TLS/RTP/SAVPF 111", + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel", + ]); + expect(all.filter((line) => line.startsWith("a=mid:"))).toEqual(["a=mid:0", "a=mid:1"]); + }); + + it("takes the offer's first audio codec and the data channel's settings, and plays the active DTLS side", () => { + const answer = fakeAnswerSdp(OFFER); + const [audio, application] = answer.split("\r\nm=").slice(1).map((section) => lines(`m=${section}`)); + expect(audio).toEqual(expect.arrayContaining(["a=rtpmap:111 opus/48000/2", "a=fmtp:111 minptime=10;useinbandfec=1", "a=rtcp-mux", "a=sendrecv", "a=setup:active"])); + expect(audio.some((line) => line.startsWith("a=rtpmap:63") || line.startsWith("a=rtpmap:9 "))).toBe(false); + expect(application).toEqual(expect.arrayContaining(["a=sctp-port:5000", "a=max-message-size:262144", "a=setup:active"])); + for (const section of [audio, application]) { + expect(section).toContain("a=candidate:1 1 udp 2122260223 127.0.0.1 9 typ host"); + expect(section.at(-1)).toBe("a=end-of-candidates"); + expect(section.find((line) => line.startsWith("a=ice-ufrag:"))).toMatch(/^a=ice-ufrag:[0-9a-f]{8}$/); + expect(section.find((line) => line.startsWith("a=ice-pwd:"))).toMatch(/^a=ice-pwd:[0-9a-f]{32}$/); + expect(section.find((line) => line.startsWith("a=fingerprint:"))).toMatch(/^a=fingerprint:sha-256 [0-9A-F]{2}(:[0-9A-F]{2}){31}$/); + } + // its own credentials, not the offer's + expect(answer).not.toContain("a=ice-ufrag:abcd"); + expect(fakeAnswerSdp(OFFER)).not.toBe(answer); + }); + + it("rejects media it does not speak and mirrors a one-way direction", () => { + const offer = OFFER.replace("m=application", "m=video 9 UDP/TLS/RTP/SAVPF 96\r\na=mid:2\r\nm=application").replace("a=sendrecv", "a=sendonly"); + const all = lines(fakeAnswerSdp(offer)); + expect(all.filter((line) => line.startsWith("m="))).toEqual([ + "m=audio 9 UDP/TLS/RTP/SAVPF 111", + "m=video 0 UDP/TLS/RTP/SAVPF 96", + "m=application 9 UDP/DTLS/SCTP webrtc-datachannel", + ]); + expect(all.filter((line) => line.startsWith("a=mid:"))).toEqual(["a=mid:0", "a=mid:2", "a=mid:1"]); + expect(all).toContain("a=group:BUNDLE 0 1"); + expect(all).toContain("a=recvonly"); + }); + + it("answers an offer without media with session lines only", () => { + expect(lines(fakeAnswerSdp("v=0\r\no=- 1 1 IN IP4 127.0.0.1\r\n"))).toHaveLength(4); + }); + + it("answers a create with the answer to its own offer", async () => { + fake = await startFakeOpenAiLive(); + const response = await fetch(`${fake.url}/v1/live/sessions`, { + method: "POST", + headers: { authorization: "Bearer sk-fake", "content-type": "application/json" }, + body: JSON.stringify({ session: { model: "gpt-live-1" }, transport: { type: "webrtc", sdp: OFFER } }), + }); + const { transport } = await response.json() as { transport: { sdp: string } }; + expect(lines(transport.sdp).filter((line) => line.startsWith("a=mid:"))).toEqual(["a=mid:0", "a=mid:1"]); + }); +}); diff --git a/server/testing/fake-openai-live.ts b/server/testing/fake-openai-live.ts new file mode 100644 index 0000000000..81812a2d05 --- /dev/null +++ b/server/testing/fake-openai-live.ts @@ -0,0 +1,251 @@ +#!/usr/bin/env node +// A stand-in for OpenAI's GPT-Live API, for tests and the isolated fixture. +// It speaks just enough HTTP and RFC 6455 WebSocket for the harness: +// POST /v1/live/sessions and GET /v1/live/sessions/:id/attach. Nothing here +// touches audio. Self-contained on purpose (node built-ins only). +import { createHash, randomBytes, randomInt } from "node:crypto"; +import { createServer, type IncomingMessage } from "node:http"; +import type { Socket } from "node:net"; +import { pathToFileURL } from "node:url"; + +export interface FakeLiveSession { + id: string; + body: Record; + commands: Array>; + attached: boolean; + closed: boolean; +} + +export interface FakeOpenAiLive { + url: string; + sessions: FakeLiveSession[]; + failNextCreate(status: number, body?: unknown): void; + refuseNextAttach(status: number): void; + emit(sessionId: string, event: Record): void; + waitForAttach(sessionId: string, timeoutMs?: number): Promise; + waitForCommand(sessionId: string, match: (command: Record) => boolean, timeoutMs?: number): Promise>; + dropSideband(sessionId: string): void; + stop(): Promise; +} + +const GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; + +/** An SDP answer to `offer` that a real WebRTC client (browser, simulator, + * emulator) accepts in setRemoteDescription, so it reaches its "live" state + * against this fake. Media never connects: nothing listens on the one host + * candidate. Audio answers with the offer's first codec, the data channel + * with the offer's SCTP settings, anything else is rejected (port 0). */ +export function fakeAnswerSdp(offer: string): string { + const lines = offer.split(/\r?\n/).filter(Boolean); + const firstMedia = lines.findIndex((line) => line.startsWith("m=")); + const session = firstMedia === -1 ? lines : lines.slice(0, firstMedia); + const sections: string[][] = []; + for (const line of firstMedia === -1 ? [] : lines.slice(firstMedia)) { + if (line.startsWith("m=")) sections.push([line]); + else sections.at(-1)!.push(line); + } + const ufrag = randomBytes(4).toString("hex"); + const pwd = randomBytes(16).toString("hex"); + const fingerprint = Array.from(randomBytes(32), (byte) => byte.toString(16).padStart(2, "0").toUpperCase()).join(":"); + const bundled = session.some((line) => line.startsWith("a=group:BUNDLE")); + const accepted: string[] = []; + const media = sections.map((section) => { + const [kind = "", port = "0", proto = "", ...formats] = section[0].slice(2).split(" "); + const attr = (name: string) => section.find((line) => line.startsWith(`a=${name}`)); + const mid = attr("mid:")?.slice("a=mid:".length) ?? ""; + const midLine = mid ? [`a=mid:${mid}`] : []; + const supported = (kind === "audio" && formats.length > 0) || (kind === "application" && proto.includes("SCTP")); + if (!supported || port === "0") return [`m=${kind} 0 ${proto} ${formats[0] ?? "0"}`, ...midLine]; + if (mid) accepted.push(mid); + const transport = [ + "c=IN IP4 127.0.0.1", + ...midLine, + `a=ice-ufrag:${ufrag}`, + `a=ice-pwd:${pwd}`, + `a=fingerprint:sha-256 ${fingerprint}`, + // the offer is actpass (or passive); an active offer gets a passive answer + `a=setup:${attr("setup:") === "a=setup:active" ? "passive" : "active"}`, + ]; + const candidate = ["a=candidate:1 1 udp 2122260223 127.0.0.1 9 typ host", "a=end-of-candidates"]; + if (kind === "application") { + const sctp = [attr("sctp-port:"), attr("max-message-size:")].filter((line): line is string => Boolean(line)); + return [`m=application 9 ${proto} ${formats.join(" ")}`, ...transport, ...sctp, ...candidate]; + } + const codec = formats[0]; + const direction = attr("sendonly") ? "recvonly" : attr("recvonly") ? "sendonly" : attr("inactive") ? "inactive" : "sendrecv"; + return [ + `m=audio 9 ${proto} ${codec}`, + ...transport, + `a=${direction}`, + "a=rtcp-mux", + ...[attr(`rtpmap:${codec} `), attr(`fmtp:${codec} `)].filter((line): line is string => Boolean(line)), + ...candidate, + ]; + }); + const head = [ + "v=0", + `o=- ${randomInt(1, 2 ** 47)} 2 IN IP4 127.0.0.1`, + "s=-", + "t=0 0", + ...(bundled && accepted.length ? [`a=group:BUNDLE ${accepted.join(" ")}`] : []), + ]; + return `${[...head, ...media.flat()].join("\r\n")}\r\n`; +} + +function bearer(req: IncomingMessage): string { + const match = /^Bearer\s+(\S+)$/i.exec(String(req.headers.authorization ?? "")); + return match?.[1] ?? ""; +} + +function frame(opcode: number, payload: Buffer): Buffer { + const length = payload.length; + const head = length < 126 ? Buffer.from([0x80 | opcode, length]) + : length < 65_536 ? Buffer.from([0x80 | opcode, 126, length >> 8, length & 0xff]) + : (() => { const b = Buffer.alloc(10); b[0] = 0x80 | opcode; b[1] = 127; b.writeBigUInt64BE(BigInt(length), 2); return b; })(); + return Buffer.concat([head, payload]); +} + +/** Pulls whole client frames (always masked) out of `buffer`. */ +function readFrames(buffer: Buffer): { frames: Array<{ opcode: number; payload: Buffer }>; rest: Buffer } { + const frames: Array<{ opcode: number; payload: Buffer }> = []; + let offset = 0; + while (buffer.length - offset >= 2) { + const opcode = buffer[offset] & 0x0f; + const masked = (buffer[offset + 1] & 0x80) !== 0; + let length = buffer[offset + 1] & 0x7f; + let cursor = offset + 2; + if (length === 126) { if (buffer.length - cursor < 2) break; length = buffer.readUInt16BE(cursor); cursor += 2; } + else if (length === 127) { if (buffer.length - cursor < 8) break; length = Number(buffer.readBigUInt64BE(cursor)); cursor += 8; } + const maskLength = masked ? 4 : 0; + if (buffer.length - cursor < maskLength + length) break; + const mask = masked ? buffer.subarray(cursor, cursor + 4) : null; + cursor += maskLength; + const payload = Buffer.from(buffer.subarray(cursor, cursor + length)); + if (mask) for (let i = 0; i < payload.length; i++) payload[i] ^= mask[i & 3]; + frames.push({ opcode, payload }); + offset = cursor + length; + } + return { frames, rest: buffer.subarray(offset) }; +} + +export async function startFakeOpenAiLive(options: { port?: number; closeOnRequest?: boolean } = {}): Promise { + const sessions: FakeLiveSession[] = []; + const sockets = new Map(); + const waiters = new Set<() => void>(); + const wake = () => { for (const fn of Array.from(waiters)) fn(); }; + let nextCreateFailure: { status: number; body: unknown } | null = null; + let nextAttachRefusal: number | null = null; + let counter = 0; + let eventCounter = 0; + + const send = (socket: Socket, event: Record) => { + if (!socket.destroyed) socket.write(frame(0x1, Buffer.from(JSON.stringify({ event_id: `evt_${++eventCounter}`, ...event })))); + }; + + const server = createServer((req, res) => { + if (req.method !== "POST" || req.url !== "/v1/live/sessions") { res.writeHead(404).end(); return; } + const chunks: Buffer[] = []; + req.on("data", (chunk: Buffer) => chunks.push(chunk)); + req.on("end", () => { + if (!bearer(req)) { res.writeHead(401, { "content-type": "application/json" }).end(JSON.stringify({ error: { message: "missing key" } })); return; } + if (nextCreateFailure) { + const failure = nextCreateFailure; + nextCreateFailure = null; + res.writeHead(failure.status, { "content-type": "application/json" }).end(JSON.stringify(failure.body ?? { error: { message: "fake failure" } })); + return; + } + let body: Record = {}; + try { body = JSON.parse(Buffer.concat(chunks).toString("utf8")) as Record; } catch { /* recorded as {} */ } + const session: FakeLiveSession = { id: `sess_fake_${++counter}`, body, commands: [], attached: false, closed: false }; + sessions.push(session); + wake(); + res.writeHead(201, { "content-type": "application/json" }).end(JSON.stringify({ + session: { id: session.id, object: "live.session" }, + transport: { type: "webrtc", sdp: fakeAnswerSdp(String((body.transport as { sdp?: unknown } | undefined)?.sdp ?? "")) }, + })); + }); + }); + + server.on("upgrade", (req: IncomingMessage, socket: Socket) => { + const match = /^\/v1\/live\/sessions\/([^/]+)\/attach$/.exec(req.url ?? ""); + const session = match ? sessions.find((s) => s.id === decodeURIComponent(match[1])) : undefined; + const refuse = (status: number) => { socket.end(`HTTP/1.1 ${status} Refused\r\nConnection: close\r\nContent-Length: 0\r\n\r\n`); }; + if (nextAttachRefusal !== null) { const status = nextAttachRefusal; nextAttachRefusal = null; refuse(status); return; } + if (!session || !bearer(req) || session.closed) { refuse(session ? 401 : 404); return; } + const accept = createHash("sha1").update(`${String(req.headers["sec-websocket-key"])}${GUID}`).digest("base64"); + socket.write(`HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\nSec-WebSocket-Accept: ${accept}\r\n\r\n`); + session.attached = true; + sockets.set(session.id, socket); + wake(); + send(socket, { type: "session.started", session: { id: session.id, status: "active", expires_at: Math.floor(Date.now() / 1000) + 3600 } }); + let pending: Buffer = Buffer.alloc(0); + socket.on("data", (chunk: Buffer) => { + const { frames, rest } = readFrames(Buffer.concat([pending, chunk])); + pending = rest; + for (const { opcode, payload } of frames) { + if (opcode === 0x8) { socket.end(frame(0x8, Buffer.alloc(0))); continue; } + if (opcode === 0x9) { socket.write(frame(0xa, payload)); continue; } + if (opcode !== 0x1) continue; + let command: Record; + try { command = JSON.parse(payload.toString("utf8")) as Record; } catch { continue; } + session.commands.push(command); + wake(); + if (command.type === "session.close" && options.closeOnRequest !== false) { + session.closed = true; + send(socket, { type: "session.closed", reason: "close_requested", usage: { seconds: 42 }, session: { id: session.id } }); + socket.end(frame(0x8, Buffer.from([0x03, 0xe8]))); + } + } + }); + socket.on("error", () => {}); + socket.on("close", () => { if (sockets.get(session.id) === socket) sockets.delete(session.id); }); + }); + + await new Promise((resolve) => server.listen(options.port ?? 0, "127.0.0.1", resolve)); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + + const waitFor = (check: () => T | undefined, timeoutMs: number, what: string) => + new Promise((resolve, reject) => { + const tryNow = () => { + const value = check(); + if (value === undefined) return false; + waiters.delete(tryNow); + clearTimeout(timer); + resolve(value); + return true; + }; + const timer = setTimeout(() => { waiters.delete(tryNow); reject(new Error(`fake GPT-Live: timed out waiting for ${what}`)); }, timeoutMs); + if (!tryNow()) waiters.add(tryNow); + }); + + return { + url: `http://127.0.0.1:${port}`, + sessions, + failNextCreate(status, body) { nextCreateFailure = { status, body }; }, + refuseNextAttach(status) { nextAttachRefusal = status; }, + emit(sessionId, event) { + const socket = sockets.get(sessionId); + if (!socket) throw new Error(`fake GPT-Live: ${sessionId} has no sideband`); + send(socket, event); + }, + waitForAttach(sessionId, timeoutMs = 5_000) { + return waitFor(() => (sessions.find((s) => s.id === sessionId)?.attached ? true : undefined), timeoutMs, `attach of ${sessionId}`).then(() => undefined); + }, + waitForCommand(sessionId, match, timeoutMs = 5_000) { + return waitFor(() => sessions.find((s) => s.id === sessionId)?.commands.find(match), timeoutMs, `a command on ${sessionId}`); + }, + dropSideband(sessionId) { sockets.get(sessionId)?.destroy(); }, + async stop() { + for (const socket of sockets.values()) socket.destroy(); + await new Promise((resolve) => server.close(() => resolve())); + }, + }; +} + +// `node --experimental-strip-types server/testing/fake-openai-live.ts [port]` +// prints the base URL for OMB_OPENAI_LIVE_URL and runs until killed. +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + const fake = await startFakeOpenAiLive({ port: Number(process.argv[2]) || 0 }); + process.stdout.write(`${fake.url}\n`); +} diff --git a/server/turn-log.test.ts b/server/turn-log.test.ts new file mode 100644 index 0000000000..50da5c954c --- /dev/null +++ b/server/turn-log.test.ts @@ -0,0 +1,19 @@ +import { describe, expect, it } from "vitest"; +import { turnStartLogLine } from "./turn-log.ts"; + +describe("turnStartLogLine", () => { + const turn = { botId: "bot-1", text: "what is six times seven", images: 0, depth: 0, card: false }; + + it("shows the start of a typed prompt, so a stuck turn can be traced", () => { + expect(turnStartLogLine({ ...turn, spoken: false })).toBe( + '[omb-turn] bot=bot-1 text="what is six times seven" images=0 depth=0 card=false', + ); + expect(turnStartLogLine({ ...turn, text: "x".repeat(200), spoken: false })).toContain(`text="${"x".repeat(70)}" `); + }); + + it("never logs words spoken on a Live call", () => { + const line = turnStartLogLine({ ...turn, images: 1, depth: 0, card: true, spoken: true }); + expect(line).not.toContain("six times seven"); + expect(line).toBe("[omb-turn] bot=bot-1 text=(spoken) images=1 depth=0 card=true"); + }); +}); diff --git a/server/turn-log.ts b/server/turn-log.ts new file mode 100644 index 0000000000..039ad03202 --- /dev/null +++ b/server/turn-log.ts @@ -0,0 +1,22 @@ +// The line the harness prints when a turn starts (stderr, which the desktop +// app keeps in server.log). It shows the start of the prompt, so a stuck turn +// can be traced to what was asked. Words spoken on a Live call are never +// logged — the call's summary line (liveCallSummaryLine) has counters only — +// so a turn carrying any of them shows `text=(spoken)` instead. + +export interface TurnStartLog { + botId: string; + text: string; + images: number; + depth: number; + card: boolean; + /** some of the turn's words were spoken on a Live call */ + spoken: boolean; +} + +const LOGGED_CHARS = 70; + +export function turnStartLogLine(turn: TurnStartLog): string { + const text = turn.spoken ? "(spoken)" : JSON.stringify(turn.text.slice(0, LOGGED_CHARS)); + return `[omb-turn] bot=${turn.botId} text=${text} images=${turn.images} depth=${turn.depth} card=${turn.card}`; +} diff --git a/shared/call-consent.test.ts b/shared/call-consent.test.ts new file mode 100644 index 0000000000..e4a23ea4c8 --- /dev/null +++ b/shared/call-consent.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from "vitest"; + +import { spokenConsent } from "./call-consent.ts"; + +describe("spokenConsent", () => { + it("reads a clear yes or no at the start of the answer", () => { + expect(spokenConsent("Yes, go ahead")).toBe("allow"); + expect(spokenConsent("okay")).toBe("allow"); + expect(spokenConsent("No, don't do that")).toBe("deny"); + expect(spokenConsent("cancel it")).toBe("deny"); + }); + + it("drops leading filler words", () => { + expect(spokenConsent("uh, yes")).toBe("allow"); + expect(spokenConsent("Hmm... no")).toBe("deny"); + }); + + it("never reads consent from a sentence that only contains the word", () => { + expect(spokenConsent("I'm not sure")).toBeNull(); + expect(spokenConsent("what does it want to do? sure, maybe")).toBeNull(); + expect(spokenConsent("")).toBeNull(); + expect(spokenConsent("yesterday was fine")).toBeNull(); + }); + + // On a Live call the microphone stays open while the voice speaks, and + // nothing waits for the person to finish a thought: "okay…" then a pause + // then "wait, what does it delete?" granted the card. Take turns closes the + // microphone while the bot talks and keeps its rule. + describe("on a Live call", () => { + it("does not take a hedging okay, sure or fine as a yes", () => { + for (const said of ["okay", "OK.", "ok", "sure", "fine", "Okay... wait, what does it delete?", "okay I need your permission to run a command"]) { + expect(spokenConsent(said, "live"), said).toBeNull(); + } + }); + + it("still reads a clear yes or no, after a hedge or filler", () => { + expect(spokenConsent("yes", "live")).toBe("allow"); + expect(spokenConsent("Okay, yes", "live")).toBe("allow"); + expect(spokenConsent("sure, go ahead", "live")).toBe("allow"); + expect(spokenConsent("uh, do it", "live")).toBe("allow"); + expect(spokenConsent("okay no", "live")).toBe("deny"); + expect(spokenConsent("no", "live")).toBe("deny"); + expect(spokenConsent("fine, don't", "live")).toBe("deny"); + }); + + it("leaves take turns as it was", () => { + expect(spokenConsent("okay")).toBe("allow"); + expect(spokenConsent("sure", "turns")).toBe("allow"); + expect(spokenConsent("fine")).toBe("allow"); + }); + }); +}); diff --git a/shared/call-consent.ts b/shared/call-consent.ts new file mode 100644 index 0000000000..f2773cdcb7 --- /dev/null +++ b/shared/call-consent.ts @@ -0,0 +1,25 @@ +// Spoken answers to a permission card, shared by both call modes. Anything +// else is read as a reply to the bot, not as consent — an approval must +// never be granted by a sentence that merely contained the word "sure". +export const YES = /^(yes|yeah|yep|yup|sure|ok|okay|go ahead|do it|allow|approve|approved|fine|please do)\b/i; +export const NO = /^(no|nope|don'?t|do not|stop|deny|denied|cancel|never|skip it)\b/i; + +const FILLER = /^(?:(?:um+|uh+|er+|so|well|hmm+)[\s,.!?-]+)+/i; +/** On a Live call these only hedge: "okay…" then a pause then "wait, what + * does it delete?", a bystander's "okay", or the voice's own "Okay, I need + * your permission" heard back through a phone's speaker. They are filler + * there, never the answer; a clear yes or no after them still counts. */ +const LIVE_HEDGES = /^(?:(?:um+|uh+|er+|so|well|hmm+|ok|okay|sure|fine)(?:[\s,.!?-]+|$))+/i; + +/** "allow", "deny", or null when the words are not a clear decision. The + * answer must open the utterance; filler before it ("uh, yes") is dropped. + * `live`: a Live call, whose microphone stays open while the voice speaks + * and which decides after a short quiet window — hedges never decide there. + * `turns` (take turns) keeps its own rule: its microphone is closed while + * the bot talks. */ +export function spokenConsent(text: string, mode: "turns" | "live" = "turns"): "allow" | "deny" | null { + const said = text.trim().replace(mode === "live" ? LIVE_HEDGES : FILLER, ""); + if (YES.test(said)) return "allow"; + if (NO.test(said)) return "deny"; + return null; +} diff --git a/shared/live-approval.test.ts b/shared/live-approval.test.ts new file mode 100644 index 0000000000..69be4617d0 --- /dev/null +++ b/shared/live-approval.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "vitest"; + +import { LIVE_COPY, liveCardKind, liveStepLabel, liveDecisionRefusal, spokenApprovalPrompt, spokenQuestionPrompt, spokenReviewPrompt, toolPhrase } from "./live-approval.ts"; +import type { OptionCardData } from "./wire.ts"; + +const card = (extra: Partial): OptionCardData => ({ title: "Approval needed", subtitle: "rm -rf build", options: ["Allow", "Deny"], ...extra }); + +describe("liveCardKind", () => { + it("classifies open provider approvals, harness reviews and questions", () => { + expect(liveCardKind(card({ requestId: "r1", tool: "Bash" }))).toBe("approval"); + expect(liveCardKind(card({ requestId: "r1", tool: "stage_skill", skillRequest: { action: "create" } as OptionCardData["skillRequest"] }))).toBe("review"); + expect(liveCardKind(card({ requestId: "r1", tool: "schedule_routine", routineRequest: {} as OptionCardData["routineRequest"] }))).toBe("review"); + expect(liveCardKind(card({ requestId: "r1" }))).toBe("question"); + }); + it("sends a default-model or tightening proposal to the screen, though it names a tool", () => { + // propose_model and propose_tightening cards carry update_model and + // tighten_permissions; a spoken yes must never reach them as an approval + expect(liveCardKind(card({ requestId: "r1", tool: "update_model", modelRequest: {} as OptionCardData["modelRequest"] }))).toBe("review"); + expect(liveCardKind(card({ requestId: "r1", tool: "tighten_permissions", tighteningRequest: {} as OptionCardData["tighteningRequest"] }))).toBe("review"); + expect(liveDecisionRefusal(card({ requestId: "r1", tool: "update_model", modelRequest: {} as OptionCardData["modelRequest"] }))) + .toBe("This request is reviewed on screen."); + }); + it("ignores settled cards and cards without a request", () => { + expect(liveCardKind(card({ requestId: "r1", tool: "Bash", answered: "allow" }))).toBeNull(); + expect(liveCardKind(card({ requestId: "r1", tool: "Bash", dismissed: true }))).toBeNull(); + expect(liveCardKind(card({ requestId: "r1", tool: "update_model", modelRequest: {} as OptionCardData["modelRequest"], expired: true }))).toBeNull(); + expect(liveCardKind(card({}))).toBeNull(); + expect(liveCardKind(undefined)).toBeNull(); + }); +}); + +describe("liveDecisionRefusal", () => { + it("delivers a decision to an open approval or question", () => { + expect(liveDecisionRefusal(card({ requestId: "r1", tool: "Bash" }))).toBeNull(); + expect(liveDecisionRefusal(card({ requestId: "r1" }))).toBeNull(); + }); + it("refuses a card settled on screen a moment earlier, and a harness review", () => { + expect(liveDecisionRefusal(card({ requestId: "r1", tool: "Bash", answered: "allow" }))).toBe("The request is no longer open."); + expect(liveDecisionRefusal(card({ requestId: "r1", dismissed: true }))).toBe("The request is no longer open."); + expect(liveDecisionRefusal(card({ requestId: "r1", tool: "Bash", expired: true }))).toBe("The request is no longer open."); + expect(liveDecisionRefusal(card({ requestId: "r1", tool: "stage_skill", skillRequest: { action: "create" } as OptionCardData["skillRequest"] }))) + .toBe("This request is reviewed on screen."); + }); + it("leaves a card that is not on the thread to the normal answer path", () => { + expect(liveDecisionRefusal(undefined)).toBeNull(); + }); +}); + +describe("spoken prompts", () => { + it("names the tool as a verb phrase, never the raw tool id", () => { + expect(toolPhrase("Bash")).toBe("run a command"); + expect(toolPhrase("mcp__ogb__computer_batch")).toBe("computer batch"); + expect(toolPhrase(undefined)).toBe("take an action"); + expect(spokenApprovalPrompt(card({ requestId: "r1", tool: "Bash" }))).toBe("I want to run a command. rm -rf build. May I?"); + }); + it("keeps a long detail short enough to read aloud", () => { + const prompt = spokenApprovalPrompt(card({ requestId: "r1", tool: "Bash", subtitle: "x ".repeat(800) }), "Ada"); + expect(prompt.length).toBeLessThan(520); + }); + it("sends reviews to the chat and reads questions with their options", () => { + expect(spokenReviewPrompt(card({ requestId: "r1", title: "Enable the invoice skill" }))).toBe( + "Tell the user, in your own words, that you need their decision in the chat: Enable the invoice skill. They review it on screen and choose there; it cannot be decided by voice.", + ); + expect(spokenQuestionPrompt(card({ requestId: "r1", subtitle: "Which account?", options: ["Main", "Savings"] }))).toBe( + "Ask the user, in your own words: Which account? The options are Main, Savings. Then delegate their answer.", + ); + }); + it("never doubles the question mark of a question that ends in one", () => { + expect(spokenQuestionPrompt(card({ requestId: "r1", subtitle: "Which one??", options: [] }))).toBe( + "Ask the user, in your own words: Which one? Then delegate their answer.", + ); + }); + it("has fixed copy for the call", () => { + expect(LIVE_COPY.granted).toBe("Thanks, I'll go ahead."); + expect(LIVE_COPY.denied).toBe("Okay, I won't do it."); + expect(LIVE_COPY.noAnswer).toBe("I'm done. The result, or what went wrong, is in the chat."); + expect(LIVE_COPY.deniedMessage).toBe("Denied by the user, on a live call."); + }); +}); + +describe("liveStepLabel", () => { + it("prefers the tool's spoken label, then its name, and never its arguments", () => { + expect(liveStepLabel({ name: "mcp__agents__list_bots", spoken: "Checking the team" })).toBe("Checking the team"); + expect(liveStepLabel({ name: "mcp__team-notes__query_database" })).toBe("team notes: query database"); + expect(liveStepLabel({ name: "mcp__composio__COMPOSIO_MULTI_EXECUTE_TOOL" })).toBe("composio: COMPOSIO MULTI EXECUTE TOOL"); + expect(liveStepLabel({ name: "Bash" })).toBe("run a command"); + expect(liveStepLabel({ name: `mcp__x__${"y".repeat(200)}` }).length).toBeLessThanOrEqual(81); + }); + it("words a status note with the time, the steps and the last step", () => { + expect(LIVE_COPY.status(250_000, 12, "team notes: query database", 40_000)).toBe( + "Status note, do not announce it: you are still working on it (4 minutes so far, 12 steps; last step: team notes: query database, 40 seconds ago).", + ); + expect(LIVE_COPY.status(1_000, 1, null, 0)).toBe("Status note, do not announce it: you are still working on it (1 second so far, 1 step)."); + }); +}); diff --git a/shared/live-approval.ts b/shared/live-approval.ts new file mode 100644 index 0000000000..5d50a5da27 --- /dev/null +++ b/shared/live-approval.ts @@ -0,0 +1,142 @@ +// What the harness tells GPT-Live about the bot's cards during a Live call. +// English only: the voice speaks the user's language on its own, and the +// server has no i18n. The strings mirror src/locales/en.json approval.voice.* +// and approval.tool.* so the call and the card say the same thing. +import type { OptionCardData } from "./wire.ts"; + +export type LiveCardKind = "approval" | "review" | "question"; + +/** "approval": a provider or peer ask the voice may decide with a strict yes/no. + * "review": a harness-native proposal (skill, routine, profile, default + * model, tightening, team setup) that must be reviewed on screen. + * "question": a provider question. An expired proposal is settled: nothing + * can answer it any more. */ +export function liveCardKind(card: OptionCardData | undefined): LiveCardKind | null { + if (!card?.requestId || card.answered || card.dismissed || card.expired) return null; + if (card.skillRequest || card.routineRequest || card.profileRequest || card.modelRequest || card.tighteningRequest || card.teamSetupRequest) { + return "review"; + } + return card.tool ? "approval" : "question"; +} + +/** Why a decision made on a Live call must not reach this card, or null to + * deliver it. A card settled a moment earlier (answered or dismissed on + * screen) is refused before anything runs: delivering to it would add a + * false "the action was not run" line after the tap did run it. A card that + * is not on the thread is left to the normal answer path. */ +export function liveDecisionRefusal(card: OptionCardData | undefined): string | null { + if (!card) return null; + const kind = liveCardKind(card); + if (kind === "review") return "This request is reviewed on screen."; + return kind ? null : "The request is no longer open."; +} + +const TOOL_PHRASES: Record = { + Bash: "run a command", + Read: "read a file", + Write: "write a file", + Edit: "edit a file", + WebFetch: "fetch a web page", + WebSearch: "search the web", + schedule_routine: "schedule a routine", + manage_routine: "change a routine", + stage_skill: "enable a learned skill", + update_skill: "update a learned skill", + update_profile: "update its profile", + shell: "run a command", + edit: "edit a file", + read: "read a file", + fetch: "fetch a web page", + delete: "delete a file", + think: "think", + other: "take an action", + tool: "use a tool", +}; + +/** Same rule as toolLabel in src/components/ApprovalCard.tsx, in English. */ +export function toolPhrase(tool?: string): string { + if (!tool) return "take an action"; + return TOOL_PHRASES[tool] ?? tool.replace(/^mcp__[^_]+__/, "").replace(/_/g, " "); +} + +const DETAIL_LIMIT = 400; + +function duration(ms: number): string { + const seconds = Math.max(0, Math.round(ms / 1_000)); + if (seconds < 60) return `${seconds} second${seconds === 1 ? "" : "s"}`; + const minutes = Math.round(seconds / 60); + return `${minutes} minute${minutes === 1 ? "" : "s"}`; +} + +/** A bot's step as the voice may name it: the tool's own spoken label, or + * the tool's name ("team notes: query database") — never its arguments, + * which can hold paths, queries or private text. */ +export function liveStepLabel(tool: { name: string; spoken?: string }): string { + if (tool.spoken?.trim()) return brief(tool.spoken, 80); + const parts = tool.name.split("__"); + if (parts[0] === "mcp" && parts.length >= 3) { + const server = parts[1].replace(/[-_]+/g, " ").trim(); + const action = parts.slice(2).join(" ").replace(/_+/g, " ").trim(); + return brief(`${server}: ${action}`, 80); + } + return toolPhrase(tool.name); +} + +function brief(text: string, limit = DETAIL_LIMIT): string { + const flat = text.replace(/\s+/g, " ").trim().replace(/[.!?]+$/, ""); + return flat.length <= limit ? flat : `${flat.slice(0, limit).replace(/\s+\S*$/, "")}…`; +} + +// The voice IS the bot to the person on the call, so everything below is +// either said as the bot ("I …") or addressed to the voice as "you". Never +// name the bot in the third person: the voice then talks about itself as a +// separate "backend" (seen in the first test calls). + +export function spokenApprovalPrompt(card: OptionCardData): string { + const detail = brief(card.subtitle); + return `I want to ${toolPhrase(card.tool)}.${detail ? ` ${detail}.` : ""} May I?`; +} + +export function spokenReviewPrompt(card: OptionCardData): string { + const title = brief(card.title || "a proposal", 200); + return `Tell the user, in your own words, that you need their decision in the chat: ${title}. They review it on screen and choose there; it cannot be decided by voice.`; +} + +/** The bot waits for an app to be connected in the chat (a connector card). */ +export function spokenConnectorPrompt(label: string): string { + return `Tell the user, in your own words, that you need them to connect ${brief(label || "an app", 80)} in the chat before you can go on. It cannot be done by voice.`; +} + +/** The bot waits for a credential entered on screen (a secret card). It must + * never travel through the call, so the voice never asks for it aloud. */ +export function spokenSecretPrompt(label: string): string { + return `Tell the user, in your own words, that you need ${brief(label || "a credential", 80)} entered in the chat on screen before you can go on. Never ask them to say it aloud.`; +} + +export function spokenQuestionPrompt(card: OptionCardData): string { + const detail = brief(card.subtitle || card.title); + const choices = card.options.length ? ` The options are ${card.options.join(", ")}.` : ""; + // brief() drops the detail's own closing mark, so the "?" is never doubled + return `Ask the user, in your own words: ${detail}?${choices} Then delegate their answer.`; +} + +export const LIVE_COPY = { + notHeard: "The request was not heard clearly. Ask the user to say it again.", + notClear: "That was not a clear yes or no. Ask the user again for a clear yes or no.", + deniedMessage: "Denied by the user, on a live call.", + answeredInChat: "The user answered that request in the chat. Do not ask about it again.", + working: "You are working on the request now. Nothing is done until the result comes back; do not guess it.", + answerPassed: "You are using the user's answer now. Wait for the result.", + granted: "Thanks, I'll go ahead.", + denied: "Okay, I won't do it.", + saveFailed: (detail: string) => `The decision could not be saved${detail ? `: ${detail}` : "."} Ask the user to try again.`, + noAnswer: "I'm done. The result, or what went wrong, is in the chat.", + typedAnswerLead: (typed: string) => `About what you typed in the chat ("${brief(typed, 200)}"):`, + progress: (spoken: string) => `Progress: ${brief(spoken, 200)}`, + status: (workedMs: number, steps: number, lastStep: string | null, lastStepAgoMs: number) => + `Status note, do not announce it: you are still working on it (${duration(workedMs)} so far, ${steps} step${steps === 1 ? "" : "s"}${ + lastStep ? `; last step: ${lastStep}, ${duration(lastStepAgoMs)} ago` : ""}).`, + signedOut: "The call has ended because the sign-in that started it has ended. Sign in again to start a new call.", + unpaired: "The call has ended because the phone that started it was unpaired from this computer.", + permissionRequest: (prompt: string) => `You need the user's permission. Say it in your own words: ${prompt} Ask for a clear yes or no, then delegate their answer.`, +}; diff --git a/shared/live-call.test.ts b/shared/live-call.test.ts new file mode 100644 index 0000000000..a9882d764f --- /dev/null +++ b/shared/live-call.test.ts @@ -0,0 +1,99 @@ +import { describe, expect, it } from "vitest"; + +import { APPEND_CHAR_LIMIT, clampAppend, commentaryChunks, FULL_ANSWER_IN_CHAT, joinFragments, LiveTranscript } from "./live-call.ts"; + +describe("LiveTranscript", () => { + it("rebuilds each delegated request from the words spoken since the previous one", () => { + const transcript = new LiveTranscript(); + transcript.addInput("Can you check", 100, 600); + transcript.addInput(" the open pull", 600, 1_100); + transcript.addInput(" requests?", 1_100, 1_500); + expect(transcript.takeRequest(1_500)).toBe("Can you check the open pull requests?"); + + transcript.addInput("And merge the green", 9_000, 9_600); + transcript.addInput(" ones.", 9_600, 9_900); + expect(transcript.takeRequest(9_700)).toBe("And merge the green ones."); + expect(transcript.takeRequest(12_000)).toBe(""); + }); + + it("includes the tail of a sentence that finished just after the delegation", () => { + const transcript = new LiveTranscript(); + transcript.addInput("Rename the file", 0, 800); + transcript.addInput(" to notes.md", 900, 1_600); + transcript.addInput(" later remark", 5_000, 5_500); + expect(transcript.takeRequest(800)).toBe("Rename the file to notes.md"); + expect(transcript.pending()).toBe("later remark"); + }); + + it("can mark everything heard as handled without building a request", () => { + const transcript = new LiveTranscript(); + transcript.addInput("yes", 0, 300); + expect(transcript.pending()).toBe("yes"); + transcript.consumeAll(); + expect(transcript.pending()).toBe(""); + expect(transcript.takeRequest(10_000)).toBe(""); + }); + + // The voice's own words, picked up again by the microphone (a phone on + // speaker), arrive as input. For a spoken yes or no they must not count. + it("can leave out what was heard while the voice itself was speaking", () => { + const transcript = new LiveTranscript(); + transcript.addOutput(900, 1_400); + transcript.addOutput(1_400, 2_000); + transcript.addInput("Okay, I need your permission", 1_000, 1_900); + transcript.addInput(" no", 3_000, 3_200); + expect(transcript.pending({ skipEcho: true })).toBe("no"); + expect(transcript.pending()).toBe("Okay, I need your permission no"); + expect(transcript.takeRequestParts(3_200)).toEqual({ text: "Okay, I need your permission no", withoutEcho: "no" }); + // both are consumed either way + expect(transcript.pending()).toBe(""); + }); + + it("keeps input that only touches the voice's speech, and ignores spans without timing", () => { + const transcript = new LiveTranscript(); + transcript.addOutput(Number.NaN, Number.NaN); + transcript.addOutput(500, 1_000); + transcript.addInput("yes", 1_000, 1_300); + expect(transcript.pending({ skipEcho: true })).toBe("yes"); + }); + + it("glues fragments as delivered and normalizes whitespace", () => { + expect(joinFragments(["hel", "lo ", " world\n"])).toBe("hello world"); + }); +}); + +describe("commentaryChunks", () => { + it("keeps a short answer as one append", () => { + expect(commentaryChunks(["The tests pass.", "Two files changed."])).toEqual(["The tests pass. Two files changed."]); + }); + + it("stays within the per-append limit and points to the chat when it cuts", () => { + const sentence = "This sentence is about seventy characters long, give or take a word. "; + const utterances = Array.from({ length: 200 }, () => sentence.trim()); + const chunks = commentaryChunks(utterances); + expect(chunks.length).toBe(3); + for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(APPEND_CHAR_LIMIT); + expect(chunks.at(-1)?.endsWith(FULL_ANSWER_IN_CHAT)).toBe(true); + expect(chunks.slice(0, -1).some((chunk) => chunk.includes(FULL_ANSWER_IN_CHAT))).toBe(false); + }); + + it("splits a single overlong utterance instead of dropping it", () => { + const long = "word ".repeat(600).trim(); + const chunks = commentaryChunks([long], 500, 10); + expect(chunks.length).toBeGreaterThan(1); + for (const chunk of chunks) expect(chunk.length).toBeLessThanOrEqual(500); + }); + + it("returns nothing for an empty answer", () => { + expect(commentaryChunks(["", " "])).toEqual([]); + }); +}); + +describe("clampAppend", () => { + it("leaves short content alone and trims long content at a word", () => { + expect(clampAppend(" hello there ")).toBe("hello there"); + const clamped = clampAppend("alpha ".repeat(400)); + expect(clamped.length).toBeLessThanOrEqual(APPEND_CHAR_LIMIT); + expect(clamped.endsWith("…")).toBe(true); + }); +}); diff --git a/shared/live-call.ts b/shared/live-call.ts new file mode 100644 index 0000000000..b38dd0559e --- /dev/null +++ b/shared/live-call.ts @@ -0,0 +1,185 @@ +// Pure pieces of a Live (GPT-Live) call, kept out of the component so they +// can be tested without WebRTC. +// +// With client delegation GPT-Live says *that* it wants help, never *what*: +// `session.delegation.created` carries an id and a timeline offset, and the +// request has to be rebuilt from the input transcript. That reconstruction, +// and the shaping of the bot's answer into the 500-token appends GPT-Live +// accepts, live here. + +export interface TranscriptSegment { + text: string; + startMs: number; + endMs: number; +} + +/** Collects `session.input_transcript.delta` fragments on the session + * timeline. Fragments are appended in delivery order, as the API asks. */ +export class LiveTranscript { + private input: TranscriptSegment[] = []; + /** when the voice itself was speaking, on the same timeline */ + private output: Array<{ startMs: number; endMs: number }> = []; + private cutoffMs = -1; + + addInput(text: string, startMs: number, endMs: number): void { + if (!text) return; + this.input.push({ text, startMs: Number.isFinite(startMs) ? startMs : 0, endMs: Number.isFinite(endMs) ? endMs : 0 }); + // keep memory bounded on long calls: only the unconsumed tail matters + if (this.input.length > 2_000) this.input = this.input.filter((segment) => segment.startMs > this.cutoffMs); + } + + /** A stretch of the voice's own speech (a `session.output_transcript.delta` + * with its timing). Input heard during it may be the voice itself, picked + * up again by the microphone of a phone on speaker. Deltas without timing + * are ignored. */ + addOutput(startMs: number, endMs: number): void { + if (!Number.isFinite(startMs) || !Number.isFinite(endMs) || endMs < startMs) return; + const last = this.output.at(-1); + if (last && startMs <= last.endMs) last.endMs = Math.max(last.endMs, endMs); + else this.output.push({ startMs, endMs }); + if (this.output.length > 500) this.output.splice(0, this.output.length - 500); + } + + /** The words spoken since the previous request, up to a little past the + * delegation point (transcription can trail the delegation event, and the + * person may finish the sentence that triggered it). Consumes them, so the + * next request starts after this one. */ + takeRequest(offsetMs: number, graceMs = 1_500): string { + return this.takeRequestParts(offsetMs, graceMs).text; + } + + /** takeRequest, plus the same words without those heard while the voice + * spoke (`withoutEcho`), for reading a spoken yes/no. Consumes all of them. */ + takeRequestParts(offsetMs: number, graceMs = 1_500): { text: string; withoutEcho: string } { + const until = offsetMs + graceMs; + const taken = this.input.filter((segment) => segment.startMs > this.cutoffMs && segment.startMs <= until); + if (!taken.length) return { text: "", withoutEcho: "" }; + this.cutoffMs = Math.max(this.cutoffMs, ...taken.map((segment) => segment.startMs)); + return { + text: joinFragments(taken.map((segment) => segment.text)), + withoutEcho: joinFragments(taken.filter((segment) => !this.echoed(segment)).map((segment) => segment.text)), + }; + } + + /** Everything heard since the last taken request, without consuming it — + * used to read a spoken yes/no while a permission question is open. + * `skipEcho` leaves out what was heard while the voice itself spoke. */ + pending(options: { skipEcho?: boolean } = {}): string { + return joinFragments(this.input + .filter((segment) => segment.startMs > this.cutoffMs && !(options.skipEcho && this.echoed(segment))) + .map((segment) => segment.text)); + } + + private echoed(segment: TranscriptSegment): boolean { + return this.output.some((span) => segment.startMs < span.endMs && segment.endMs > span.startMs); + } + + /** Mark everything heard so far as handled (e.g. after a spoken decision). */ + consumeAll(): void { + for (const segment of this.input) this.cutoffMs = Math.max(this.cutoffMs, segment.startMs); + } +} + +/** Transcript deltas are fragments of words, not words: glue them as sent. */ +export function joinFragments(fragments: string[]): string { + return fragments.join("").replace(/\s+/g, " ").trim(); +} + +/** GPT-Live caps each append at 500 tokens. Characters are a safe stand-in: + * ~1,400 characters stays well under 500 tokens for ordinary prose. */ +export const APPEND_CHAR_LIMIT = 1_400; +const MAX_SPOKEN_CHUNKS = 3; +export const FULL_ANSWER_IN_CHAT = "The full answer is in the chat."; + +/** Group speakable utterances (from /api/tts/prepare) into appends. Long + * answers are cut at a sentence boundary with a pointer to the chat, because + * a voice reading four minutes of a report is not a conversation. */ +export function commentaryChunks( + utterances: string[], + maxChars = APPEND_CHAR_LIMIT, + maxChunks = MAX_SPOKEN_CHUNKS, +): string[] { + const chunks: string[] = []; + let current = ""; + let truncated = false; + for (const raw of utterances) { + const utterance = raw.replace(/\s+/g, " ").trim(); + if (!utterance) continue; + const pieces = utterance.length > maxChars ? splitLong(utterance, maxChars) : [utterance]; + for (const piece of pieces) { + if (current && current.length + 1 + piece.length > maxChars) { + chunks.push(current); + current = ""; + } + if (chunks.length >= maxChunks) { + truncated = true; + break; + } + current = current ? `${current} ${piece}` : piece; + } + if (truncated) break; + } + if (current && chunks.length < maxChunks) chunks.push(current); + else if (current) truncated = true; + if (truncated && chunks.length) { + const last = chunks[chunks.length - 1]; + chunks[chunks.length - 1] = last.length + 1 + FULL_ANSWER_IN_CHAT.length <= maxChars + ? `${last} ${FULL_ANSWER_IN_CHAT}` + : `${last.slice(0, maxChars - FULL_ANSWER_IN_CHAT.length - 1).replace(/\s+\S*$/, "")} ${FULL_ANSWER_IN_CHAT}`; + } + return chunks; +} + +function splitLong(text: string, maxChars: number): string[] { + const out: string[] = []; + let rest = text; + while (rest.length > maxChars) { + const window = rest.slice(0, maxChars); + const sentence = Math.max(window.lastIndexOf(". "), window.lastIndexOf("! "), window.lastIndexOf("? ")); + const space = window.lastIndexOf(" "); + const cut = sentence > maxChars / 2 ? sentence + 1 : space > 0 ? space : maxChars; + out.push(rest.slice(0, cut).trim()); + rest = rest.slice(cut).trim(); + } + if (rest) out.push(rest); + return out; +} + +/** Client events this call sends on the `oai-events` data channel. */ +export type LiveClientEvent = + | { type: "session.close"; event_id?: string } + | { type: "session.instructions.append" | "session.thinking.append" | "session.commentary.append"; event_id?: string; delegation_id: string | null; content: string }; + +/** Keep appended content inside the per-append limit. */ +export function clampAppend(content: string, maxChars = APPEND_CHAR_LIMIT): string { + const text = content.replace(/\s+/g, " ").trim(); + return text.length <= maxChars ? text : `${text.slice(0, maxChars - 1).replace(/\s+\S*$/, "")}…`; +} + +/** GPT-Live voices from OpenAI's documentation (Managing GPT-Live sessions, + * "Voice options", and the SDK's BuiltInVoice list), with the regional + * notes the docs give for the additional ones. */ +export const LIVE_VOICE_OPTIONS: ReadonlyArray<{ id: string; label: string }> = [ + { id: "marin", label: "Marin (default)" }, + { id: "cedar", label: "Cedar" }, + { id: "alloy", label: "Alloy" }, + { id: "ash", label: "Ash" }, + { id: "ballad", label: "Ballad" }, + { id: "coral", label: "Coral" }, + { id: "echo", label: "Echo" }, + { id: "sage", label: "Sage" }, + { id: "shimmer", label: "Shimmer" }, + { id: "verse", label: "Verse" }, + { id: "gleam", label: "Gleam — North American, feminine" }, + { id: "meridian", label: "Meridian — North American, masculine" }, + { id: "quartz", label: "Quartz — Australian, feminine" }, + { id: "ripple", label: "Ripple — Australian, masculine" }, + { id: "vesper", label: "Vesper — British, masculine" }, + { id: "willow", label: "Willow — Irish, feminine" }, + { id: "stone", label: "Stone — Irish, masculine" }, + { id: "delta", label: "Delta — Southern U.S., feminine" }, + { id: "cinder", label: "Cinder — Southern U.S., masculine" }, + { id: "beacon", label: "Beacon — Filipino, masculine" }, + { id: "bossa", label: "Bossa — Brazilian Portuguese, feminine" }, + { id: "tempo", label: "Tempo — Brazilian Portuguese, masculine" }, +]; diff --git a/shared/wire.ts b/shared/wire.ts index 3aefc72f1c..a2e65fdee9 100644 --- a/shared/wire.ts +++ b/shared/wire.ts @@ -354,13 +354,17 @@ export interface ResolvedSender { /** Who answered a card: a signed-in person (named as their messages are), the * owner on this machine, or a session-less local caller on a shared server * (`worker`: the Slack worker, or any other process on that machine). */ -export type CardAnswerer = +export type CardAnswerer = ( /** `person`: the answering session's opaque person key, recorded on an OMB * Cloud home only, where it decides whether an answer came from the owner * (server/cloud-lending.ts). */ | { kind: "session"; name: string; person?: string } | { kind: "loopback" } - | { kind: "worker" }; + | { kind: "worker" } +) & { + /** "call": decided by voice on a Live call, not tapped. */ + via?: "call"; +}; /** One transcript line. Serialized as stored — the durable delivery * identity (roomRequest) rides the wire unchanged. */ @@ -413,8 +417,14 @@ export interface WireMessage { * a new request. The text is stored enveloped exactly as injected, so any * later reader sees the sender and the not-steering framing. */ aside?: boolean; - /** A user-role message that arrived through the server's HTTP API. */ - via?: "api"; + /** A user-role message that arrived through the server's HTTP API + * ("api"), or a request a person spoke on a Live call ("call"). */ + via?: "api" | "call"; + /** A user line an external interface relayed through the guarded send + * route (the Slack worker, for someone else, as this computer): nobody + * typed it in one of this workspace's clients. A Live call never reads it + * back as what the caller typed. */ + relayed?: boolean; /** Which person sent this user message, when the workspace has more than * one. The server authenticates per person but used to attribute every * user turn to the single profile name, so on a shared or paired instance @@ -518,6 +528,39 @@ export interface OptionCardData { questionRequest?: QuestionRequestCardData; } +/** Which app holds the microphone of a Live call. Self-declared; for display and logs only. */ +export type LiveClient = "desktop" | "ios" | "android"; +export type LiveCallStatus = "connecting" | "live" | "ending" | "ended"; +/** Why a call ended. "signed-out": the sign-in or paired phone that started + * it was signed out, revoked or unpaired. A client that does not know a + * reason shows the call's `error` text, or plain "Call ended.". */ +export type LiveEndReason = + | "hung-up" | "idle" | "expired" | "content" | "remote-hangup" | "connection-lost" + | "sideband-lost" | "deleted" | "shutdown" | "signed-out" | "error"; + +/** The one Live call a harness runs. Never carries the key or any speech. */ +export interface LiveCallState { + callId: string; + botId: string; + threadId: string; + client: LiveClient; + voice: string; + /** epoch ms when the session was created */ + startedAt: number; + status: LiveCallStatus; + endReason?: LiveEndReason; + /** short, user-facing; present when the call ended on a problem */ + error?: string; +} + +/** Non-secret Live settings, as GET /api/config and PATCH /api/live/settings report them. */ +export interface LiveSettings { + configured: boolean; + voice: string; + readTypedReplies: boolean; + idleMinutes: number; +} + export interface ConnectorCardData { /** Composio toolkit slug. It is validated server-side before every action. */ slug: string; @@ -657,6 +700,7 @@ export type ServerFrame = | { kind: "computer"; botId: string; state: "provisioning" | "waking" } | { kind: "computer-control"; botId: string; held: boolean; helpReason: string | null } | { kind: "bot.deleted"; botId: string } + | { kind: "live.call"; botId: string; threadId: string; call: LiveCallState | null } /** The config status object spread flat into the frame; its full typing * is the deferred client-model extraction (see j1-phase-bc-progress). */ | ({ kind: "config" } & Record); diff --git a/src/types/ogb.d.ts b/src/types/ogb.d.ts index 7a7d5020c9..0b24c0542b 100644 --- a/src/types/ogb.d.ts +++ b/src/types/ogb.d.ts @@ -292,7 +292,7 @@ const __APP_VERSION__: string; saveFile?(filePath: string): Promise; /** Save a provider credential through Electron's OS-backed store. */ setCredential?( - name: "composioApiKey" | "xaiApiKey" | "boxToken" | "opencodeGoApiKey" | "ttsKey" | "fishAudioKey" | "jevApiKey" | "openaiImageApiKey" | "customImageApiKey", + name: "composioApiKey" | "xaiApiKey" | "boxToken" | "opencodeGoApiKey" | "ttsKey" | "fishAudioKey" | "jevApiKey" | "openaiImageApiKey" | "customImageApiKey" | "openaiLiveKey", value: string, ): Promise; /** In-app auto-update (packaged app only; dormant in dev). onState From 04b812efcaec07abda5e2f28bef0a9f4d0387233 Mon Sep 17 00:00:00 2001 From: Nevil Date: Sun, 27 Sep 2026 23:05:32 +0300 Subject: [PATCH 074/211] feat(calls): the desktop Live call bar, with the chat open The call button now starts the kind of call picked last, and the arrow next to it picks Take turns (the existing call) or Live. A Live call keeps the chat on screen: a bar above the composer shows the call time, a caption of what the voice says and what it heard, a settings gear, Mute and Hang up (Cmd/Ctrl+Shift+M and Cmd/Ctrl+Shift+H). The call's media lives at app level (src/lib/live-call-media.ts), so switching chats keeps it, and a pill shows the call from any other chat. The renderer holds only the microphone, the speaker and a data channel that may send session.close; everything the voice is told runs on the harness. The first Live call asks for an OpenAI key (saved through the desktop credential store, or PUT /api/config in a browser), and the gear changes the voice, typed-reply reading, the idle minutes and the key. Both say what a Live call sends to OpenAI. A spoken request shows "via call" in the chat, a card decided by voice shows "by voice", voice notes do not play during a call, and the call button is hidden while a phone holds the line (the bar then says which device, with a hang-up). A call whose audio does not connect within 20 s drops and offers Try again; Try again appears only where a retry can help. (cherry picked from commit d4693db0f8ab1faa488463708d65f44bf09be8f4) --- src/App.tsx | 2 + src/components/ApprovalCard.test.ts | 27 + src/components/ApprovalCard.tsx | 3 + src/components/CallView.live-mode.test.ts | 140 ++++ src/components/CallView.mode-menu.test.ts | 186 +++++ src/components/CallView.tsx | 279 ++++++- src/components/ChatView.tsx | 16 +- src/components/ChatView.via-call.test.ts | 57 ++ src/components/LiveCallBar.render.test.ts | 144 ++++ src/components/LiveCallBar.test.ts | 99 +++ src/components/LiveCallBar.tsx | 254 +++++++ src/components/LiveCallHost.tsx | 22 + src/components/LiveCallPill.render.test.ts | 106 +++ src/components/LiveCallPill.test.ts | 18 + src/components/LiveCallPill.tsx | 121 +++ src/components/LiveCallSettings.tsx | 126 ++++ src/components/LiveKeySetup.tsx | 67 ++ src/components/Sidebar.tsx | 32 +- .../SidebarBotListItem.expansion.test.ts | 4 + src/components/SidebarBotListItem.test.ts | 37 +- src/components/VoiceNoteBubble.test.ts | 15 + src/components/VoiceNoteBubble.tsx | 14 +- src/lib/call-mode.test.ts | 41 ++ src/lib/call-mode.ts | 66 ++ src/lib/keyboard-shortcuts.test.ts | 6 + src/lib/keyboard-shortcuts.ts | 17 + src/lib/live-call-media.test.ts | 689 ++++++++++++++++++ src/lib/live-call-media.ts | 521 +++++++++++++ src/locales/en.json | 60 +- src/state/store.test.ts | 42 ++ src/state/store.tsx | 70 +- 31 files changed, 3235 insertions(+), 46 deletions(-) create mode 100644 src/components/CallView.live-mode.test.ts create mode 100644 src/components/CallView.mode-menu.test.ts create mode 100644 src/components/ChatView.via-call.test.ts create mode 100644 src/components/LiveCallBar.render.test.ts create mode 100644 src/components/LiveCallBar.test.ts create mode 100644 src/components/LiveCallBar.tsx create mode 100644 src/components/LiveCallHost.tsx create mode 100644 src/components/LiveCallPill.render.test.ts create mode 100644 src/components/LiveCallPill.test.ts create mode 100644 src/components/LiveCallPill.tsx create mode 100644 src/components/LiveCallSettings.tsx create mode 100644 src/components/LiveKeySetup.tsx create mode 100644 src/lib/call-mode.test.ts create mode 100644 src/lib/call-mode.ts create mode 100644 src/lib/live-call-media.test.ts create mode 100644 src/lib/live-call-media.ts diff --git a/src/App.tsx b/src/App.tsx index e9cbc0dfd0..45a93ff548 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -5,6 +5,7 @@ import { useWelcomeViewer, WelcomeGate } from "@/components/onboarding/WelcomeGa import { cloudSignInDue, spotlightsQuiet, type WelcomeViewer } from "@/lib/onboarding"; import { FirstConversationTour } from "@/components/onboarding/FirstConversationTour"; import { GuidedTour } from "@/components/onboarding/GuidedTour"; +import { LiveCallHost } from "@/components/LiveCallHost"; import { ThreadRefsProvider } from "@/components/ThreadRefs"; import { initAnalytics } from "@/lib/analytics"; import { Sidebar } from "@/components/Sidebar"; @@ -391,6 +392,7 @@ function Application() { + diff --git a/src/components/ApprovalCard.test.ts b/src/components/ApprovalCard.test.ts index aeb9762d43..2ef9e43425 100644 --- a/src/components/ApprovalCard.test.ts +++ b/src/components/ApprovalCard.test.ts @@ -26,6 +26,33 @@ const createRoutineOperation = { }, }; +describe("ApprovalCard decided by voice", () => { + const bash = (answered: string, via?: "call"): Message => ({ + id: "bash-card", + role: "bot", + kind: "options", + at: 1, + card: { + title: "Approval needed", + subtitle: "rm -rf build", + options: ["Allow", "Deny"], + requestId: "r1", + tool: "Bash", + answered, + answeredBy: { kind: "loopback", ...(via ? { via } : {}) }, + }, + }); + + it("says a card was decided by voice on a Live call", () => { + expect(renderToStaticMarkup(createElement(ApprovalCard, { message: bash("allow", "call") }))).toMatch(/Allowed.*by voice/); + expect(renderToStaticMarkup(createElement(ApprovalCard, { message: bash("deny", "call") }))).toMatch(/Denied.*by voice/); + }); + + it("says nothing extra for a tap", () => { + expect(renderToStaticMarkup(createElement(ApprovalCard, { message: bash("allow") }))).not.toContain("by voice"); + }); +}); + describe("ApprovalCard routine proposals", () => { it("describes a chat-created routine as scheduling rather than a raw tool call", () => { const message: Message = { diff --git a/src/components/ApprovalCard.tsx b/src/components/ApprovalCard.tsx index 19c406bee2..f2f2f347eb 100644 --- a/src/components/ApprovalCard.tsx +++ b/src/components/ApprovalCard.tsx @@ -97,6 +97,8 @@ export function ApprovalCard({ if (!card) return null; const settled = card.answered; const expired = card.expired === true; + // decided by voice on a Live call rather than tapped + const byVoice = card.answeredBy?.via === "call" ? · {t("approval.status.byVoice")} : null; const isRoutineRequest = Boolean(card.routineRequest); const isSkillRequest = Boolean(card.skillRequest); const isProfileRequest = Boolean(card.profileRequest); @@ -177,6 +179,7 @@ export function ApprovalCard({ {outcome ? ( <> {settled === "allow" && !expired ? : } {outcome} + {byVoice} ) : ( <> diff --git a/src/components/CallView.live-mode.test.ts b/src/components/CallView.live-mode.test.ts new file mode 100644 index 0000000000..05fee66d80 --- /dev/null +++ b/src/components/CallView.live-mode.test.ts @@ -0,0 +1,140 @@ +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { ApiError, StoreProvider, type Bot } from "@/state/store"; + +vi.mock("./DesktopCapabilities", () => ({ + // capabilities still loading: a take-turns call cannot start yet + useDesktopCapabilities: () => ({ capabilities: null, ready: false }), +})); + +import { CallButton, CallOverlay, CallTargetButton } from "./CallView"; +import { setCallMode } from "@/lib/call-mode"; +import { endCall, startCall } from "@/lib/call"; +import { configureLiveMedia, dismissKeyPrompt, resetLiveMedia, startLiveCall } from "@/lib/live-call-media"; + +const bot: Bot = { + id: "atlas", + threadId: "thread-atlas", + name: "Atlas", + title: "", + description: "", + notifications: true, + color: "green", + unread: false, + modelSelection: { instanceId: "claude", model: "test" }, + messages: [], +}; + +const render = (element: ReturnType) => + renderToStaticMarkup(createElement(StoreProvider, null, element)); + +afterEach(() => { + setCallMode("turns"); + resetLiveMedia(); + endCall(); + vi.unstubAllGlobals(); +}); + +describe("call modes", () => { + it("offers a Live call to one bot without on-device dictation or a configured voice", () => { + setCallMode("turns"); + expect(render(createElement(CallButton, { bot }))).toContain('aria-label="Checking call availability"'); + + setCallMode("live"); + const live = render(createElement(CallButton, { bot })); + expect(live).toContain('aria-label="Live call with Atlas"'); + expect(live).not.toContain("bg-warning"); + }); + + it("keeps group calls on the take-turns path", () => { + setCallMode("live"); + const group = render(createElement(CallTargetButton, { + targetId: "room", targetName: "Room", voices: [undefined], requireExplicitVoices: true, onStart: vi.fn(), + })); + expect(group).toContain('aria-label="Checking call availability"'); + }); + + it("puts a call mode chevron next to a one-to-one call button, not a room's", () => { + const one = render(createElement(CallButton, { bot })); + expect(one).toContain('aria-label="Call mode"'); + expect(one).toContain('aria-haspopup="menu"'); + const group = render(createElement(CallTargetButton, { + targetId: "room", targetName: "Room", voices: [undefined], requireExplicitVoices: true, onStart: vi.fn(), + })); + expect(group).not.toContain('aria-label="Call mode"'); + }); + + it("hangs up from the call button while this window is on a Live call", () => { + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + setCallMode("live"); + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + const markup = render(createElement(CallButton, { bot })); + expect(markup).toContain('aria-label="Hang up on Atlas"'); + // a mode is picked before a call, not during one + expect(markup).toMatch(/]*disabled=""[^>]*aria-label="Call mode"/); + }); + + it("blocks other chats' call buttons while this window is on a Live call", () => { + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + for (const mode of ["live", "turns"] as const) { + setCallMode(mode); + const other = render(createElement(CallButton, { bot: { ...bot, id: "juniper", name: "Juniper" } })); + expect(other).toMatch(/]*disabled=""[^>]*aria-label="On a Live call"/); + } + const room = render(createElement(CallTargetButton, { + targetId: "room", targetName: "Room", voices: [undefined], requireExplicitVoices: true, onStart: vi.fn(), + })); + expect(room).toMatch(/]*disabled=""[^>]*aria-label="On a Live call"/); + }); + + it("asks for the OpenAI key under the call button when the harness has none", async () => { + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + const track = { enabled: true, stop: vi.fn() }; + configureLiveMedia({ + getUserMedia: async () => ({ getTracks: () => [track], getAudioTracks: () => [track] }) as unknown as MediaStream, + createPeer: () => ({ + iceGatheringState: "complete", + localDescription: { sdp: "v=0" }, + addTrack() {}, + createDataChannel: () => ({ close() {} }), + createOffer: async () => ({ type: "offer", sdp: "v=0" }), + setLocalDescription: async () => {}, + close() {}, + }) as unknown as RTCPeerConnection, + request: async () => { + throw new ApiError("Add an OpenAI API key to use Live calls.", 409, { needsKey: true }); + }, + stopRemote: () => {}, + }); + setCallMode("live"); + await startLiveCall({ botId: bot.id, threadId: bot.threadId }); + expect(render(createElement(CallButton, { bot }))).toContain('aria-label="OpenAI API key for Live calls"'); + expect(render(createElement(CallButton, { bot: { ...bot, id: "juniper" } }))).not.toContain("OpenAI API key"); + // Leaving the chat (or closing the prompt) drops it: it does not open + // again by itself on a later visit. Another chat leaving keeps it. + dismissKeyPrompt("juniper"); + expect(render(createElement(CallButton, { bot }))).toContain("OpenAI API key"); + dismissKeyPrompt(bot.id); + expect(render(createElement(CallButton, { bot }))).not.toContain("OpenAI API key"); + }); + + it("leaves a Live call to the call bar and covers the chat only for Take turns", async () => { + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + // the microphone prompt never answers: the call stays "starting" + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + expect(render(createElement(CallOverlay, { bot }))).toBe(""); + + // a failed Live call's notice stays in the bar; a take-turns call still opens + resetLiveMedia(); + configureLiveMedia({ getUserMedia: () => Promise.reject(new Error("no microphone")) }); + await startLiveCall({ botId: bot.id, threadId: bot.threadId }); + startCall(bot.id); + expect(render(createElement(CallOverlay, { bot }))).toContain("backdrop-blur-sm"); + }); +}); diff --git a/src/components/CallView.mode-menu.test.ts b/src/components/CallView.mode-menu.test.ts new file mode 100644 index 0000000000..c91da561d5 --- /dev/null +++ b/src/components/CallView.mode-menu.test.ts @@ -0,0 +1,186 @@ +// What the call button and its mode menu start. Pressing a button here reads +// the rendered element tree and calls its handler; the calls themselves are +// observed through the call and Live media modules. +import { Children, createElement, isValidElement, type ReactElement, type ReactNode } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import type { AppState, Bot } from "@/state/store"; + +const fixture = vi.hoisted(() => ({ liveConfigured: true, liveCall: null as AppState["liveCall"] })); +vi.mock("@/state/store", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + useStore: () => ({ + state: { + ...original.initialState, + liveCall: fixture.liveCall, + config: { + tts: { configured: true, ready: true, voice: "v" }, + live: { configured: fixture.liveConfigured, voice: "marin", readTypedReplies: true, idleMinutes: 5 }, + } as AppState["config"], + }, + dispatch: vi.fn(), + }), + }; +}); +vi.mock("./DesktopCapabilities", () => ({ + // a Mac that can take turns + useDesktopCapabilities: () => ({ capabilities: { dictation: { available: true, engine: "apple-speech", onDevice: true } }, ready: true }), +})); + +import { CallModeMenu, CallTargetButton } from "./CallView"; +import { setCallMode, type CallMode } from "@/lib/call-mode"; +import { currentCall, endCall, startCall } from "@/lib/call"; +import { configureLiveMedia, liveMedia, resetLiveMedia, startLiveCall } from "@/lib/live-call-media"; + +type ElementProps = { children?: ReactNode; onClick?: () => void; [key: string]: unknown }; +function findElement(tree: ReactNode, match: (props: ElementProps) => boolean): ReactElement | undefined { + for (const child of Children.toArray(tree)) { + if (!isValidElement(child)) continue; + if (match(child.props)) return child; + const found = findElement(child.props.children, match); + if (found) return found; + } +} + +const bot: Pick = { id: "atlas", threadId: "thread-atlas", name: "Atlas" }; +/** A phone's Live call with another bot: it holds the one Live line. */ +const phoneCall: NonNullable = { + callId: "c9", botId: "juniper", threadId: "t9", client: "ios", voice: "marin", startedAt: 0, status: "live", +}; + +/** The one-to-one call button's markup. */ +function renderButton(): string { + return renderToStaticMarkup(createElement(CallTargetButton, { + targetId: bot.id, targetName: bot.name, threadId: bot.threadId, voices: ["v"], + requireExplicitVoices: false, liveCapable: true, onStart: vi.fn(), + })); +} + +/** Render the one-to-one call button and press the phone. */ +function pressPhone(onStart: (mode: CallMode) => void) { + let tree: ReactNode = null; + function Capture() { + tree = CallTargetButton({ + targetId: bot.id, targetName: bot.name, threadId: bot.threadId, voices: ["v"], + requireExplicitVoices: false, liveCapable: true, onStart, + }); + return tree; + } + renderToStaticMarkup(createElement(Capture)); + const phone = findElement(tree, (props) => typeof props["aria-label"] === "string" && /Atlas/.test(props["aria-label"] as string)); + if (!phone) throw new Error("no phone button"); + phone.props.onClick?.(); +} + +beforeEach(() => { + fixture.liveConfigured = true; + fixture.liveCall = null; + vi.stubGlobal("window", { ogb: { speechStart: vi.fn(), speechStop: vi.fn(async () => {}) } }); + // the microphone prompt never answers: a Live call stays "starting" + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); +}); +afterEach(() => { + setCallMode("turns"); + resetLiveMedia(); + endCall(); + vi.unstubAllGlobals(); +}); + +describe("the call button", () => { + it("starts a Live call through the Live media, not the take-turns overlay", () => { + setCallMode("live"); + const onStart = vi.fn(); + pressPhone(onStart); + expect(liveMedia()).toMatchObject({ phase: "starting", botId: "atlas", threadId: "thread-atlas" }); + expect(onStart).toHaveBeenCalledWith("live"); + }); + + it("asks for a key instead of calling when Live has none", () => { + fixture.liveConfigured = false; + setCallMode("live"); + const onStart = vi.fn(); + pressPhone(onStart); + expect(liveMedia().phase).toBe("idle"); + expect(currentCall()).toBeNull(); + expect(onStart).not.toHaveBeenCalled(); + }); + + it("hangs up this window's Live call", () => { + setCallMode("live"); + void startLiveCall({ botId: "atlas", threadId: "thread-atlas" }); + pressPhone(vi.fn()); + // hung up before the harness knew the call: nothing to wait for + expect(liveMedia().phase).toBe("idle"); + expect(currentCall()).toBeNull(); + }); + + // Another device holds the one Live line: no Live call button at all (the + // iPhone's rule), instead of one that can only be refused as busy. + it("is not there in Live mode while another device holds the Live line, on any bot", () => { + setCallMode("live"); + expect(renderButton()).toContain('aria-label="Live call with Atlas"'); + fixture.liveCall = phoneCall; + expect(renderButton()).toBe(""); + fixture.liveCall = { ...phoneCall, status: "ended", endReason: "hung-up" }; + expect(renderButton()).toContain('aria-label="Live call with Atlas"'); + }); + + // Take turns never uses the Live line, so a phone's Live call leaves it be. + it("keeps Take turns while another device holds the Live line", () => { + setCallMode("turns"); + fixture.liveCall = phoneCall; + expect(renderButton()).toContain('aria-label="Call Atlas"'); + const onStart = vi.fn(); + pressPhone(onStart); + expect(currentCall()).toBe("atlas"); + expect(liveMedia().phase).toBe("idle"); + expect(onStart).toHaveBeenCalledWith("turns"); + expect(renderButton()).toContain('aria-label="Hang up on Atlas"'); + }); + + it("keeps a running Take-turns call's Hang up when a phone takes the Live line, in either mode", () => { + startCall("atlas"); + fixture.liveCall = phoneCall; + for (const mode of ["turns", "live"] as const) { + setCallMode(mode); + expect(renderButton()).toContain('aria-label="Hang up on Atlas"'); + } + }); + + it("keeps Take turns as it was: the overlay's call, no Live media", () => { + setCallMode("turns"); + const onStart = vi.fn(); + pressPhone(onStart); + expect(currentCall()).toBe("atlas"); + expect(liveMedia().phase).toBe("idle"); + expect(onStart).toHaveBeenCalledWith("turns"); + }); +}); + +describe("the call mode menu", () => { + it("offers both modes as radio items, the current one checked", () => { + const markup = renderToStaticMarkup(createElement(CallModeMenu, { id: "m", mode: "live", onChoose: vi.fn(), onClose: vi.fn() })); + expect(markup).toContain('role="menu"'); + expect(markup).toContain('aria-label="Call mode"'); + const items = markup.split('role="menuitemradio"').slice(1); + expect(items).toHaveLength(2); + expect(items[0]).toContain(">Take turns<"); + expect(items[0]).toMatch(/^ aria-checked="false"/); + expect(items[1]).toContain(">Live<"); + expect(items[1]).toMatch(/^ aria-checked="true"/); + }); + + it("reports the chosen mode", () => { + const onChoose = vi.fn(); + let tree: ReactNode = null; + function Capture() { + tree = CallModeMenu({ id: "m", mode: "live", onChoose, onClose: vi.fn() }); + return tree; + } + renderToStaticMarkup(createElement(Capture)); + findElement(tree, (props) => props.role === "menuitemradio" && props["aria-checked"] === false)?.props.onClick?.(); + expect(onChoose).toHaveBeenCalledWith("turns"); + }); +}); diff --git a/src/components/CallView.tsx b/src/components/CallView.tsx index fea26354ca..0176dd38d6 100644 --- a/src/components/CallView.tsx +++ b/src/components/CallView.tsx @@ -18,28 +18,29 @@ // it happens, which is why waiting feels like listening to someone work // rather than listening to nothing. import { useCallback, useEffect, useId, useRef, useState } from "react"; -import { Loader2, Phone, PhoneOff, X } from "lucide-react"; +import { Check, ChevronDown, Loader2, Phone, PhoneOff, X } from "lucide-react"; import { useStore, visibleMessages, type Bot } from "@/state/store"; import { cn } from "@/lib/cn"; import { useMenuMotion } from "./MenuMotion"; import { currentCall, deferCallCleanup, endCall, startCall, useOnCall } from "@/lib/call"; +import { CALL_MODES, callModeHint, setCallMode, useCallMode, type CallMode } from "@/lib/call-mode"; +import { NO, YES } from "../../shared/call-consent"; +import { dismissKeyPrompt, hangUpLiveCall, isLiveCallRunning, startLiveCall, useLiveMedia } from "@/lib/live-call-media"; +import { t } from "@/lib/i18n"; import { speaker } from "@/lib/tts"; import { localSystemVoiceActive } from "@/lib/local-voice"; import { useSpeech } from "@/lib/tts/useSpeech"; import { usePushToTalk } from "@/lib/push-to-talk"; import { BotAvatar } from "./Avatar"; +import { navigateThreadMenu } from "./BotProjects"; +import { LiveKeySetup } from "./LiveKeySetup"; +import { liveLineHeldElsewhere } from "./LiveCallBar"; import { isRoutineApproval, isSkillApproval, pendingApprovals, spokenApprovalPrompt } from "./PendingApproval"; import { track } from "@/lib/analytics"; import { useDesktopCapabilities } from "./DesktopCapabilities"; import { callCapabilityHelp } from "@/lib/call-capability"; -/** Spoken answers to a permission card. Anything else is read as a reply - * to the bot, not as consent — an approval must never be granted by a - * sentence that merely contained the word "sure". */ -const YES = /^(yes|yeah|yep|yup|sure|ok|okay|go ahead|do it|allow|approve|approved|fine|please do)\b/i; -const NO = /^(no|nope|don'?t|do not|stop|deny|denied|cancel|never|skip it)\b/i; - type Phase = "listening" | "sending" | "working" | "speaking"; const CALL_ENDPOINT_MS = 850; @@ -48,10 +49,12 @@ export function CallButton({ bot }: { bot: Bot }) { track("call_started", { driver: bot.modelSelection?.instanceId })} + liveCapable + onStart={(mode) => track("call_started", { driver: bot.modelSelection?.instanceId, mode })} /> ); } @@ -59,23 +62,43 @@ export function CallButton({ bot }: { bot: Bot }) { export function CallTargetButton({ targetId, targetName, + threadId, voices, setupBotId, requireExplicitVoices, + liveCapable = false, onStart, }: { targetId: string; targetName: string; + /** The thread a Live call joins (the chat on screen). Live needs it. */ + threadId?: string; voices: Array; /** Agent profile to open when voice setup is missing (rooms choose a member). */ setupBotId?: string; /** Rooms cannot rely on one workspace fallback for multiple speakers. */ requireExplicitVoices: boolean; - onStart: () => void; + /** One-to-one calls can also run as a Live (GPT-Live) call, which needs + * neither on-device dictation nor a configured voice. */ + liveCapable?: boolean; + /** A call started, in this mode (analytics). The call itself is started + * here: Take turns opens the overlay, Live goes to the call bar. */ + onStart: (mode: CallMode) => void; }) { const { state, dispatch } = useStore(); const { capabilities, ready: capabilitiesReady } = useDesktopCapabilities(); - const active = useOnCall() === targetId; + const media = useLiveMedia(); + const liveThreadId = liveCapable ? threadId : undefined; + const canLive = liveThreadId !== undefined; + // This window's Live call with this target. The media module also marks + // the target as on a call (startCall), so check Live first. + const liveRunning = isLiveCallRunning(media.phase); + const onLiveCall = canLive && liveRunning && media.botId === targetId; + // One call at a time: a Live call with someone else blocks this button + // (a second Live call cannot start, and Take turns would talk over it). + const liveElsewhere = liveRunning && media.botId !== targetId; + const onCall = useOnCall() === targetId; + const active = onCall || onLiveCall; const capabilityHelp = capabilitiesReady ? callCapabilityHelp(capabilities, Boolean(window.ogb?.speechStart)) : null; @@ -86,16 +109,34 @@ export function CallTargetButton({ const voiceReady = localVoice || (configured && (requireExplicitVoices ? everyTargetHasVoice : Boolean(state.config?.tts?.ready || everyTargetHasVoice))); - const unavailable = !active && (!capabilitiesReady || !supported || !voiceReady); + const mode = useCallMode(); + const liveMode = canLive && mode === "live"; + const turnsReady = capabilitiesReady && supported && voiceReady; + const unavailable = !active && !liveElsewhere && !liveMode && !turnsReady; const voiceSetupRequired = capabilitiesReady && supported && !voiceReady; + const liveConfigured = Boolean(state.config?.live?.configured); const [helpOpen, setHelpOpen] = useState(false); const helpMotion = useMenuMotion(Boolean(unavailable && helpOpen)); + const [menuOpen, setMenuOpen] = useState(false); + const [keyOpen, setKeyOpen] = useState(false); + // the harness answered "no key" to this window's call attempt (the key was + // removed, or this window's config was stale): ask for it here too + const keyPopover = canLive && !active && (keyOpen || (media.needsKey && media.botId === targetId)); const rootRef = useRef(null); const buttonRef = useRef(null); + const chevronRef = useRef(null); + const keyRef = useRef(null); const helpId = useId(); + const menuId = useId(); + const keyId = useId(); + const elsewhereName = liveElsewhere ? state.bots.find((candidate) => candidate.id === media.botId)?.name : undefined; const label = active - ? `Hang up on ${targetName}` - : !capabilitiesReady + ? t("call.hangUpOn", { name: targetName }) + : liveElsewhere + ? elsewhereName ? t("call.live.pill", { name: elsewhereName }) : t("call.live.badge") + : liveMode + ? t("call.live.callWith", { name: targetName }) + : !capabilitiesReady ? "Checking call availability" : !supported ? capabilityHelp?.label ?? "Call unavailable" @@ -117,15 +158,50 @@ export function CallTargetButton({ : "Choose a voice before starting a call." : ""; + const closePopovers = useCallback(() => { + setHelpOpen(false); + setMenuOpen(false); + setKeyOpen(false); + dismissKeyPrompt(targetId); + }, [targetId]); + + // The "no key" answer belongs to this chat's call attempt: leaving the + // chat drops it, so it never reopens (and takes focus) on a later visit. + useEffect(() => () => dismissKeyPrompt(targetId), [targetId]); + + /** Start a call in this mode. Live never opens the overlay: the media + * module marks the call and the call bar shows it. */ + const start = (next: CallMode) => { + setHelpOpen(false); + setMenuOpen(false); + if (next === "live" && liveThreadId !== undefined) { + if (!liveConfigured) { + setKeyOpen(true); + return; + } + setKeyOpen(false); + onStart("live"); + void startLiveCall({ botId: targetId, threadId: liveThreadId }); + return; + } + if (!turnsReady) { + setHelpOpen(true); + return; + } + onStart("turns"); + startCall(targetId); + }; + + const popoverOpen = helpOpen || menuOpen || keyPopover; useEffect(() => { - if (!helpOpen) return; + if (!popoverOpen) return; const closeOnOutsideClick = (event: PointerEvent) => { - if (event.target instanceof Node && !rootRef.current?.contains(event.target)) setHelpOpen(false); + if (event.target instanceof Node && !rootRef.current?.contains(event.target)) closePopovers(); }; const closeOnEscape = (event: KeyboardEvent) => { if (event.key !== "Escape") return; - setHelpOpen(false); - buttonRef.current?.focus(); + closePopovers(); + (menuOpen ? chevronRef : buttonRef).current?.focus(); }; document.addEventListener("pointerdown", closeOnOutsideClick); document.addEventListener("keydown", closeOnEscape); @@ -133,27 +209,52 @@ export function CallTargetButton({ document.removeEventListener("pointerdown", closeOnOutsideClick); document.removeEventListener("keydown", closeOnEscape); }; - }, [helpOpen]); + }, [popoverOpen, menuOpen, closePopovers]); + // Keyboard users land in the key field when the prompt opens, not when a + // prompt that was already open is shown again. + const keyShown = useRef(keyPopover); + useEffect(() => { + const opened = keyPopover && !keyShown.current; + keyShown.current = keyPopover; + if (opened) keyRef.current?.querySelector("input")?.focus(); + }, [keyPopover]); + + const opensKey = liveMode && !active && (!liveConfigured || keyPopover); + // Another device (a phone, another window) holds the one Live line: no + // button that would start a Live call here, as on the iPhone. The remote + // bar in that call's chat says who is on the line and can hang up. Take + // turns never uses the Live line, so its call, and the Hang up of one + // already running, stay. + if (liveMode && !onCall && liveLineHeldElsewhere(media, state.liveCall)) return null; return ( -
    +
    + {canLive && ( + + )} + + {menuOpen && ( + { + // picking a mode remembers it and starts a call in it + setCallMode(next); + start(next); + }} + /> + )} + + {keyPopover && liveThreadId !== undefined && ( +
    + { + setKeyOpen(false); + onStart("live"); + void startLiveCall({ botId: targetId, threadId: liveThreadId }); + }} + /> +
    + )} {helpMotion.shown && (
    Call unavailable
    {reason}
    @@ -188,6 +340,19 @@ export function CallTargetButton({ Choose This computer )} + {canLive && ( + + )} {voiceSetupRequired && ( + ))} +
    + ); +} + export function CallOverlay({ bot }: { bot: Bot }) { const active = useOnCall() === bot.id; - if (!active) return null; - return ; + const media = useLiveMedia(); + // A Live call lives in the call bar; only Take turns uses the overlay. + const onLiveCall = media.botId === bot.id && isLiveCallRunning(media.phase); + if (!active || onLiveCall) return null; + return ( +
    + +
    + ); } function Call({ bot }: { bot: Bot }) { @@ -554,7 +773,7 @@ function Call({ bot }: { bot: Bot }) { : "Working"; return ( -
    + <>
    + ); } diff --git a/src/components/ChatView.tsx b/src/components/ChatView.tsx index eb53fdab8f..22b58d1865 100644 --- a/src/components/ChatView.tsx +++ b/src/components/ChatView.tsx @@ -35,6 +35,7 @@ import { formatTime, messageVersions, openNotificationTarget, + openThread, visibleMessages, type Bot, type InstanceInfo, @@ -91,6 +92,8 @@ import { CitationSelectionToolbar, SentCitations } from "./CitationUI"; import { SpeakButton } from "./SpeakButton"; import { CallButton, CallOverlay } from "./CallView"; +import { LiveCallBar } from "./LiveCallBar"; +import { LiveCallChip } from "./LiveCallPill"; import { effectivePlace, toolPlace, type EffectivePlace } from "@/lib/place"; import { cn } from "@/lib/cn"; import { activeLocale, t } from "@/lib/i18n"; @@ -507,6 +510,11 @@ function Bubble({ {t("chat.sentMidTurn")}
    )} + {message.via === "call" && ( + + {t("chat.viaCall")} + + )} {collapsible && (
    )} + {/* A Live call on this chat: its controls and captions sit above the + composer so the transcript stays in view. In the dock, so the + transcript pad grows with it. */} + {canWrite === false ? ( dispatch({ type: "newTask", botId: bot.id })} /> ) : ( diff --git a/src/components/ChatView.via-call.test.ts b/src/components/ChatView.via-call.test.ts new file mode 100644 index 0000000000..19687b9f68 --- /dev/null +++ b/src/components/ChatView.via-call.test.ts @@ -0,0 +1,57 @@ +// A request spoken on a Live call is an ordinary user message with a small +// "via call" line under it; typed messages have none. +import { createElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterAll, describe, expect, it, vi } from "vitest"; +import type { Bot, InstanceInfo, Message } from "@/state/store"; + +vi.hoisted(() => { + vi.stubGlobal("window", {}); + vi.stubGlobal("localStorage", { getItem: () => null, setItem: () => {} }); +}); +vi.mock("@/state/store", async (importOriginal) => { + const original = await importOriginal(); + return { ...original, useStore: () => ({ + state: { ...original.initialState, instances: [{ instanceId: "test", driverKind: "codex", displayName: "Test" } as InstanceInfo] }, + dispatch: vi.fn(), + }) }; +}); +vi.mock("./DesktopCapabilities", async (importOriginal) => ({ + ...await importOriginal(), + useDesktopCapabilities: () => ({ capabilities: { dictation: { available: false }, host: { packaged: true, platform: "other" }, localComputer: { available: false, reasonCode: "cua-driver-unavailable", message: "" } }, ready: true }), +})); +vi.mock("@/lib/analytics", () => ({ track: vi.fn() })); +// the thread controls are not what this test is about +vi.mock("./ModelPicker", () => ({ ModelPicker: () => createElement("span") })); +vi.mock("./ApprovalModeSelector", () => ({ ApprovalModeSelector: () => createElement("span") })); + +const { ChatView } = await import("./ChatView"); +afterAll(() => vi.unstubAllGlobals()); + +const message = (id: string, text: string, extra: Partial = {}): Message => + ({ id, role: "user", kind: "text", text, at: 1_700_000_000_000, ...extra }) as Message; + +const bot = (messages: Message[]): Bot => ({ + id: "bot", threadId: "t1", name: "Pepper", title: "", description: "", color: "green", + notifications: true, unread: false, busy: false, messages, + modelSelection: { instanceId: "test", model: "m" }, +}); + +describe("via call label", () => { + it("marks a request spoken on a Live call, and only that one", () => { + const markup = renderToStaticMarkup(createElement(ChatView, { + bot: bot([message("m1", "typed words"), message("m2", "spoken words", { via: "call" })]), + })); + expect(markup).toContain("typed words"); + expect(markup).toContain("spoken words"); + expect(markup.match(/>via callvia call<")).toBeGreaterThan(markup.indexOf("spoken words")); + }); + + it("shows no label without a call", () => { + const markup = renderToStaticMarkup(createElement(ChatView, { bot: bot([message("m1", "typed words")]) })); + expect(markup).toContain("typed words"); + expect(markup).not.toContain("via call"); + }); +}); diff --git a/src/components/LiveCallBar.render.test.ts b/src/components/LiveCallBar.render.test.ts new file mode 100644 index 0000000000..abfcee2395 --- /dev/null +++ b/src/components/LiveCallBar.render.test.ts @@ -0,0 +1,144 @@ +import { createElement, type ReactElement } from "react"; +import { renderToStaticMarkup } from "react-dom/server"; +import { afterEach, describe, expect, it, vi } from "vitest"; + +import { StoreProvider, type Bot } from "@/state/store"; +import { endCall } from "@/lib/call"; +import { configureLiveMedia, resetLiveMedia, startLiveCall } from "@/lib/live-call-media"; +import { LiveCallBar } from "./LiveCallBar"; +import { LiveCallSettings } from "./LiveCallSettings"; +import { LiveKeySetup } from "./LiveKeySetup"; + +const bot: Bot = { + id: "atlas", + threadId: "thread-atlas", + name: "Atlas", + title: "", + description: "", + notifications: true, + color: "green", + unread: false, + modelSelection: { instanceId: "claude", model: "test" }, + messages: [], +}; + +const render = (element: ReturnType) => + renderToStaticMarkup(createElement(StoreProvider, null, element)); + +afterEach(() => { + resetLiveMedia(); + endCall(); + vi.unstubAllGlobals(); +}); + +describe("LiveCallBar", () => { + it("renders nothing without a call", () => { + expect(render(createElement(LiveCallBar, { bot }))).toBe(""); + }); + + it("shows the hint when the window blocked the call's audio", async () => { + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + const track = { enabled: true, stop: () => {} }; + const peer = { + ontrack: null as ((event: { track: unknown }) => void) | null, + localDescription: { sdp: "v=0\r\n" }, + iceGatheringState: "complete", + addTrack: () => {}, + createDataChannel: () => ({ readyState: "connecting", close: () => {} }), + createOffer: async () => ({ type: "offer", sdp: "v=0\r\n" }), + setLocalDescription: async () => {}, + close: () => {}, + }; + configureLiveMedia({ + getUserMedia: async () => ({ getTracks: () => [track], getAudioTracks: () => [track] }) as unknown as MediaStream, + createPeer: () => peer as unknown as RTCPeerConnection, + // the session answer never comes: the call stays connecting + request: () => new Promise(() => {}), + playRemote: async () => { throw new Error("autoplay blocked"); }, + }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + await vi.waitFor(() => expect(peer.ontrack).not.toBeNull()); + peer.ontrack!({ track: {} }); + await vi.waitFor(() => expect(render(createElement(LiveCallBar, { bot }))).toContain("Click anywhere in the window to hear the call.")); + }); + + it("shows the call's controls while this window connects", () => { + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + const markup = render(createElement(LiveCallBar, { bot })); + expect(markup).toContain("Live with Atlas · Connecting…"); + expect(markup).toContain('aria-label="Call settings"'); + expect(markup).toContain('aria-label="Mute"'); + expect(markup).toContain('aria-label="Hang up"'); + }); + + it("takes clicks itself, inside the composer dock that lets them through", () => { + // ChatView's composer dock is pointer-events-none so a blank band beside + // its cards reaches the transcript; without its own auto the bar's + // buttons would never receive a click. + vi.stubGlobal("window", { ogb: { speechStop: vi.fn(async () => {}) } }); + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + const markup = render(createElement(LiveCallBar, { bot })); + expect(markup).toMatch(/^
    ]* class="pointer-events-auto /); + }); + + it("names the keyboard chords on the mute and hang-up buttons", () => { + vi.stubGlobal("window", { ogb: { platform: "darwin", speechStop: vi.fn(async () => {}) } }); + configureLiveMedia({ getUserMedia: () => new Promise(() => {}) }); + void startLiveCall({ botId: bot.id, threadId: bot.threadId }); + const markup = render(createElement(LiveCallBar, { bot })); + expect(markup).toContain('title="Mute (⌘⇧M)"'); + expect(markup).toContain('aria-keyshortcuts="Meta+Shift+M"'); + expect(markup).toContain('title="Hang up (⌘⇧H)"'); + expect(markup).toContain('aria-keyshortcuts="Meta+Shift+H"'); + }); +}); + +describe("LiveCallSettings", () => { + it("offers voice, typed replies, idle minutes and the key, defaulting to 5 minutes", () => { + const markup = render(createElement(LiveCallSettings, { onClose: vi.fn() })); + expect(markup).toContain('aria-label="Call settings"'); + expect(markup).toContain("Marin (default)"); + expect(markup).toContain("Read replies to typed messages"); + expect(markup).toContain(`When this is off, messages you type during a call and the bot's answers to them are not sent to OpenAI.`); + expect(markup).toContain(`A Live call sends your voice to OpenAI, along with the chat's recent messages, the bot's answers and the details of any approval it asks for. The OpenAI key stays on your computer.`); + expect(markup).toMatch(/
    + + + +
    { + e.preventDefault(); + void importSkill(); + }} + > +