diff --git a/companion/README.md b/companion/README.md index 9ecc56cee2..bfcdef1129 100644 --- a/companion/README.md +++ b/companion/README.md @@ -30,7 +30,7 @@ upstream hardened its loopback gate. | | | |---|---| | **Pairing** | A high-entropy QR credential plus a six-digit manual fallback, valid two minutes and single-use. Redeeming either returns a device token stored only as a SHA-256 digest. | -| **Authorisation** | Every request needs that token. Full cloud-desktop access is a separate per-device capability, off by default. A rebinding page cannot obtain either. | +| **Authorisation** | Every request needs that token. Computer access — the cloud desktop, and seeing or taking control of the Local VM — is a separate per-device capability, off by default. A rebinding page cannot obtain either. | | **The allowlist** | Default deny, per method and path (`src/routes.ts`) — the list is every request the app makes, and nothing else. General bot/room PATCH routes stay closed; read state and approval grants use narrow verbs. A route that appears in the harness later is closed to devices until someone adds it here on purpose. | | **Scrubbing** | `resumeCursors` — the harness's own provider session ids — never reach a device, whether or not the harness still sends them. | | **Discovery** | Bonjour, so a phone finds the computer by name instead of by typed address. | @@ -73,11 +73,12 @@ trusted-network-only rather than described as something it is not. request that carries one is a browser that has found this port. Refused before the token is even looked at — stricter than the harness's own rule, which allows loopback origins. -- **Hold credentials, settings, or Local VM control.** Credential plaintext +- **Hold credentials, settings, or the Local VM's lifecycle.** Credential plaintext remains transient on the phone and desktop only: the sidecar can carry one QR-keyed HPKE envelope for an exact pending card, but cannot open or retain - it. General credential/configuration routes, settings, and Local VM control - remain unavailable. See `src/routes.ts` for the exact boundary. + it. General credential/configuration routes, settings, and the Local VM's + lifecycle remain unavailable; driving the Local VM goes through the same + per-device viewer relay as a VPS desktop. See `src/routes.ts` for the exact boundary. ## Running it diff --git a/companion/src/proxy.ts b/companion/src/proxy.ts index 088a33f8fa..41e6b8f59c 100644 --- a/companion/src/proxy.ts +++ b/companion/src/proxy.ts @@ -262,8 +262,54 @@ const forwardHeaders = (req: IncomingMessage, authenticatedDeviceId?: string, mu /** The device-facing handler: refuse a browser, check the allowlist, check * the token, then replay the request to the harness over loopback and scrub * what comes back. Pairing is the one route that stops here. */ +/** Ask the harness, as the paired device, whether a control lease still + * holds a bot's computer. Read-only (`action: "check"`); every failure — + * no token yet, a refusal, a timeout, an unreadable answer — is "no". */ +function harnessControlCheck(options: ProxyOptions) { + return (deviceId: string, botId: string, controlLeaseId: string): Promise => new Promise((resolve) => { + const mutationToken = options.mutationToken?.(); + if (options.mutationToken && !mutationToken) return resolve(false); + const body = JSON.stringify({ action: "check", controlLeaseId }); + const headers: Record = { + accept: "application/json", + "content-type": "application/json", + "content-length": String(Buffer.byteLength(body)), + "x-openmausbot-companion": "1", + "x-openmausbot-companion-device": deviceId, + }; + if (mutationToken) headers["x-openmausbot-companion-auth"] = mutationToken; + const request = httpRequest({ + hostname: "127.0.0.1", + port: options.harnessPort, + path: `/api/bots/${encodeURIComponent(botId)}/computer/control`, + method: "POST", + headers, + }, (response) => { + const chunks: Buffer[] = []; + let size = 0; + response.on("data", (chunk: Buffer) => { + size += chunk.length; + if (size > 16_384) return request.destroy(); + chunks.push(chunk); + }); + response.once("end", () => { + try { + const parsed = JSON.parse(Buffer.concat(chunks).toString("utf8")) as { owned?: unknown }; + resolve(response.statusCode === 200 && parsed.owned === true); + } catch { + resolve(false); + } + }); + response.once("error", () => resolve(false)); + }); + request.setTimeout(5_000, () => request.destroy()); + request.once("error", () => resolve(false)); + request.end(body); + }); +} + export function createProxyHandler(options: ProxyOptions) { - const viewers = new CompanionViewerRelay(); + const viewers = new CompanionViewerRelay({ checkControl: harnessControlCheck(options) }); const handle = function handle(req: IncomingMessage, res: ServerResponse): void { const path = (req.url ?? "/").split("?")[0]; const method = req.method ?? "GET"; @@ -293,11 +339,11 @@ export function createProxyHandler(options: ProxyOptions) { if (denial) return sendJson(res, denial.status, { error: denial.error }); // Pairing a phone grants the ordinary companion surface, not a browser - // session with every credential that may exist inside the cloud desktop. + // session with every credential that may exist inside a bot's computer. // The computer owner enables this capability per device, off by default. if (isCloudDesktopAccess(method, path) && !device?.cloudDesktopAccess) { return sendJson(res, 403, { - error: "cloud desktop access is off for this device — enable it in OpenMausBot → Settings → Remote access", + error: "computer access is off for this device — enable it in OpenMausBot → Settings → Remote access", }); } @@ -570,7 +616,12 @@ export function createProxyHandler(options: ProxyOptions) { // JSON.parse handles it fine. let text: string; try { - parsed = viewers.rewriteJoinResponse(path, parsed, device?.id); + parsed = viewers.rewriteJoinResponse( + path, + parsed, + device?.id, + new URL(req.url ?? "/", "http://companion.invalid").searchParams.get("controlLeaseId"), + ); text = JSON.stringify(scrub(parsed)); } catch { sendJson(res, 502, { error: "the response could not be prepared for this device" }); diff --git a/companion/src/routes.ts b/companion/src/routes.ts index 236a861ded..0cc153ec7f 100644 --- a/companion/src/routes.ts +++ b/companion/src/routes.ts @@ -53,6 +53,21 @@ export const CLOUD_DESKTOP_CONTROL_ROUTE = { path: /^\/api\/bots\/[\w-]+\/computer\/(?:control|screenshot|viewer-close)$/, } as const; +/** The Local VM's live desktop, relayed by the sidecar like a VPS viewer. + * The harness grants it only while a person holds that bot's computer. */ +export const LOCAL_VM_JOIN_ROUTE = { + method: "POST", + path: /^\/api\/bots\/[\w-]+\/local-computer\/join$/, +} as const; + +/** A still of a bot's Local VM, on demand. The VM's lifecycle stays on the + * host; this only reads a picture of it, behind the same per-device + * computer-access capability as the cloud desktop. */ +export const LOCAL_VM_SCREENSHOT_ROUTE = { + method: "POST", + path: /^\/api\/bots\/[\w-]+\/local-computer\/screenshot$/, +} as const; + export function isCloudDesktopJoin(method: string, path: string): boolean { return method === CLOUD_DESKTOP_JOIN_ROUTE.method && CLOUD_DESKTOP_JOIN_ROUTE.path.test(path); } @@ -61,9 +76,13 @@ export function isMessageFileDownload(method: string, path: string): boolean { return method === MESSAGE_FILE_ROUTE.method && MESSAGE_FILE_ROUTE.path.test(path); } +/** Every route that shows or drives a bot's computer — cloud or Local VM — + * and so needs the device's computer-access capability, not just a token. */ export function isCloudDesktopAccess(method: string, path: string): boolean { return isCloudDesktopJoin(method, path) - || (method === CLOUD_DESKTOP_CONTROL_ROUTE.method && CLOUD_DESKTOP_CONTROL_ROUTE.path.test(path)); + || (method === CLOUD_DESKTOP_CONTROL_ROUTE.method && CLOUD_DESKTOP_CONTROL_ROUTE.path.test(path)) + || (method === LOCAL_VM_SCREENSHOT_ROUTE.method && LOCAL_VM_SCREENSHOT_ROUTE.path.test(path)) + || (method === LOCAL_VM_JOIN_ROUTE.method && LOCAL_VM_JOIN_ROUTE.path.test(path)); } /** Every request the iOS app makes, and nothing else. @@ -129,6 +148,12 @@ const ALLOWED: ReadonlyArray<{ method: string; path: RegExp }> = [ CLOUD_DESKTOP_JOIN_ROUTE, CLOUD_DESKTOP_CONTROL_ROUTE, + // A picture of the Local VM — not its lifecycle, which stays on the host. + // Gated per device by the proxy like the cloud desktop above. + LOCAL_VM_SCREENSHOT_ROUTE, + // Its live desktop while a person holds the computer, relayed like the + // VPS viewer and behind the same per-device capability. + LOCAL_VM_JOIN_ROUTE, // rooms — making one, and talking in one { method: "POST", path: /^\/api\/groups$/ }, { method: "POST", path: /^\/api\/groups\/[\w-]+\/messages$/ }, diff --git a/companion/src/viewer-relay.ts b/companion/src/viewer-relay.ts index b1bd6e91b1..adcaf9c184 100644 --- a/companion/src/viewer-relay.ts +++ b/companion/src/viewer-relay.ts @@ -14,10 +14,20 @@ interface ViewerSession { origin: string; expiresAt: number; sockets: Set<{ destroy(): void }>; + /** A Local VM viewer lives only as long as its control lease holds. */ + watch?: ReturnType; } +/** Whether `controlLeaseId` still holds `botId`'s computer, asked on behalf of + * the paired device. Any failure must answer false. */ +export type ControlCheck = (deviceId: string, botId: string, controlLeaseId: string) => Promise; + const VIEWER_PATH = /^\/vps-viewer\/([A-Za-z0-9_-]{32})(\/.*)?$/; -const BOT_JOIN_PATH = /^\/api\/bots\/([\w-]+)\/computer\/join$/; +/** Both joins hand back a loopback noVNC address: a VPS through its SSH + * tunnel, and the Local VM's own published port. */ +const BOT_JOIN_PATH = /^\/api\/bots\/([\w-]+)\/(computer|local-computer)\/join$/; +const CONTROL_LEASE = /^[A-Za-z0-9_-]{16,120}$/; +const CONTROL_CHECK_MS = 3_000; const SESSION_TTL_MS = 8 * 60 * 60_000; const MAX_SESSIONS = 64; @@ -102,6 +112,13 @@ function acceptUpgrade(socket: Duplex, response: IncomingMessage): void { export class CompanionViewerRelay { readonly #sessions = new Map(); + readonly #checkControl?: ControlCheck; + readonly #checkMs: number; + + constructor(options: { checkControl?: ControlCheck; checkIntervalMs?: number } = {}) { + this.#checkControl = options.checkControl; + this.#checkMs = options.checkIntervalMs ?? CONTROL_CHECK_MS; + } #prune(): void { const now = Date.now(); @@ -116,6 +133,7 @@ export class CompanionViewerRelay { } #remove(session: ViewerSession): void { + if (session.watch) clearInterval(session.watch); this.#sessions.delete(session.id); for (const socket of session.sockets) socket.destroy(); session.sockets.clear(); @@ -137,25 +155,50 @@ export class CompanionViewerRelay { } } - rewriteJoinResponse(path: string, value: unknown, deviceId?: string): unknown { - const botId = BOT_JOIN_PATH.exec(path)?.[1]; + rewriteJoinResponse(path: string, value: unknown, deviceId?: string, controlLeaseId?: string | null): unknown { + const join = BOT_JOIN_PATH.exec(path); + const botId = join?.[1]; if (!botId || !value || typeof value !== "object" || Array.isArray(value)) return value; const body = value as Record; const viewer = safeLoopbackViewer(body.joinUrl); if (!viewer) return value; if (!deviceId) throw new Error("the paired device has no viewer identity"); + // A Local VM viewer is bound to the control lease that asked for it, and + // is closed the moment that lease stops holding the computer. Without a + // lease, or nothing to check it with, it is never handed out at all. + const localVm = join[2] === "local-computer"; + if (localVm && (!controlLeaseId || !CONTROL_LEASE.test(controlLeaseId) || !this.#checkControl)) { + throw new Error("a Local VM viewer needs a control lease"); + } this.#prune(); this.close(deviceId, botId); const id = randomBytes(24).toString("base64url"); - this.#sessions.set(id, { + const session: ViewerSession = { id, botId, deviceId, origin: viewer.origin, expiresAt: Date.now() + SESSION_TTL_MS, sockets: new Set(), - }); + }; + this.#sessions.set(id, session); + if (localVm) { + const check = this.#checkControl!; + const lease = controlLeaseId!; + let checking = false; + session.watch = setInterval(() => { + if (checking) return; + checking = true; + check(deviceId, botId, lease) + .catch(() => false) + .then((owned) => { + checking = false; + if (!owned && this.#isActive(session)) this.#remove(session); + }); + }, this.#checkMs); + session.watch.unref?.(); + } const settings = new URLSearchParams(viewer.hash.slice(1)); settings.set("path", `vps-viewer/${id}/websockify`); diff --git a/companion/test/proxy-response.test.ts b/companion/test/proxy-response.test.ts index 081385a207..d467ecd84c 100644 --- a/companion/test/proxy-response.test.ts +++ b/companion/test/proxy-response.test.ts @@ -101,11 +101,11 @@ describe("preparing a harness response for a device", () => { } }); - it("requires the host to enable cloud desktop for viewer and preview requests", async () => { + it("requires the host to enable computer access for viewer and preview requests", async () => { cloudDesktopAccess = false; try { - for (const action of ["join", "screenshot"]) { - const { status, text } = await device(`/api/bots/b1/computer/${action}`, "POST"); + for (const path of ["computer/join", "computer/screenshot", "local-computer/screenshot", "local-computer/join"]) { + const { status, text } = await device(`/api/bots/b1/${path}`, "POST"); expect(status).toBe(403); expect(text).toContain("enable it in OpenMausBot"); expect(text).toContain("Settings → Remote access"); @@ -159,6 +159,26 @@ describe("preparing a harness response for a device", () => { expect(joinUrl).not.toContain("127.0.0.1:45678"); }); + it("turns the Local VM's loopback viewer into the same device-scoped path", async () => { + respond = (res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + joinUrl: "http://127.0.0.1:45679/vnc.html#autoconnect=true&resize=scale&password=vm-secret", + })); + }; + const { status, text } = await device("/api/bots/b1/local-computer/join?controlLeaseId=phone-lease-0123456789", "POST"); + expect(status).toBe(200); + const joinUrl = String(JSON.parse(text).joinUrl); + expect(joinUrl).toMatch(/^\/vps-viewer\/[A-Za-z0-9_-]{32}\/vnc\.html#/); + expect(joinUrl).toContain("password=vm-secret"); + expect(joinUrl).not.toContain("127.0.0.1:45679"); + + // Without a control lease the address never reaches the device. + const refused = await device("/api/bots/b1/local-computer/join", "POST"); + expect(refused.status).toBe(502); + expect(refused.text).not.toContain("vm-secret"); + }); + it("never forwards a body it could not scrub", async () => { // `scrub` recurses once per level, so a deeply nested body throws // RangeError while JSON.parse handles it without complaint. That gap is diff --git a/companion/test/routes.test.ts b/companion/test/routes.test.ts index aa5d0578dc..ea7567e5bd 100644 --- a/companion/test/routes.test.ts +++ b/companion/test/routes.test.ts @@ -7,7 +7,7 @@ // and the one that quietly stopped being true once before. import { describe, expect, it } from "vitest"; -import { denyReason } from "../src/routes.ts"; +import { denyReason, isCloudDesktopAccess } from "../src/routes.ts"; const ask = (method: string, path: string, authenticated = true) => denyReason({ method, path, authenticated }); @@ -70,6 +70,8 @@ describe("what the app may do", () => { ["POST", "/api/bots/bot_123/computer/control"], ["POST", "/api/bots/bot_123/computer/screenshot"], ["POST", "/api/bots/bot_123/computer/viewer-close"], + ["POST", "/api/bots/bot_123/local-computer/screenshot"], + ["POST", "/api/bots/bot_123/local-computer/join"], ["POST", "/api/groups/room-1/messages"], ["POST", "/api/groups/room-1/interrupt"], ["DELETE", "/api/groups/room-1/queue/queue_1"], @@ -190,6 +192,24 @@ describe("what it may not", () => { expect(allowed("POST", "/api/bots/bot_123/computer/exec")).toBe(false); }); + it("previews a Local VM without reaching its lifecycle", () => { + expect(allowed("POST", "/api/bots/bot_123/local-computer/screenshot")).toBe(true); + expect(isCloudDesktopAccess("POST", "/api/bots/bot_123/local-computer/screenshot")).toBe(true); + expect(allowed("GET", "/api/bots/bot_123/local-computer/screenshot")).toBe(false); + expect(allowed("GET", "/api/bots/bot_123/local-computer")).toBe(false); + for (const action of ["run", "stop", "remove"]) { + expect(allowed("POST", `/api/bots/bot_123/local-computer/${action}`)).toBe(false); + } + expect(allowed("POST", "/api/local-computer/screenshot")).toBe(false); + }); + + it("joins a Local VM's live desktop only behind computer access", () => { + expect(allowed("POST", "/api/bots/bot_123/local-computer/join")).toBe(true); + expect(isCloudDesktopAccess("POST", "/api/bots/bot_123/local-computer/join")).toBe(true); + expect(allowed("GET", "/api/bots/bot_123/local-computer/join")).toBe(false); + expect(allowed("POST", "/api/local-computer/join")).toBe(false); + }); + it("allows only the exact encrypted credential submission verb", () => { expect(allowed("POST", "/api/bots/bot_123/secret-cards/message_1/provide")).toBe(true); expect(allowed("GET", "/api/bots/bot_123/secret-cards/message_1/provide")).toBe(false); diff --git a/companion/test/viewer-relay.test.ts b/companion/test/viewer-relay.test.ts index 7eefa20405..760ad36c17 100644 --- a/companion/test/viewer-relay.test.ts +++ b/companion/test/viewer-relay.test.ts @@ -37,6 +37,58 @@ describe("VPS companion viewer relay", () => { { joinUrl: "http://203.0.113.8:6901/vnc.html#password=stolen" }, "device-1", )).toEqual({ joinUrl: "http://203.0.113.8:6901/vnc.html#password=stolen" }); + + expect(relay.rewriteJoinResponse( + "/api/bots/bot-1/local-computer/run", + { joinUrl: "http://127.0.0.1:45679/vnc.html#password=vm-secret" }, + "device-1", + )).toEqual({ joinUrl: "http://127.0.0.1:45679/vnc.html#password=vm-secret" }); + }); + + it("binds a Local VM viewer to its control lease and closes it once that lease lets go", async () => { + const vmJoin = { joinUrl: "http://127.0.0.1:45679/vnc.html#password=vm-secret" }; + // No lease, or nothing to check it with: never handed out. + expect(() => new CompanionViewerRelay().rewriteJoinResponse( + "/api/bots/bot-1/local-computer/join", vmJoin, "device-1", "phone-lease-0123456789", + )).toThrow(/control lease/); + let owned = true; + const asked: string[] = []; + const relay = new CompanionViewerRelay({ + checkIntervalMs: 10, + checkControl: async (deviceId, botId, lease) => { + asked.push(`${deviceId}/${botId}/${lease}`); + return owned; + }, + }); + expect(() => relay.rewriteJoinResponse("/api/bots/bot-1/local-computer/join", vmJoin, "device-1", null)).toThrow(); + expect(() => relay.rewriteJoinResponse("/api/bots/bot-1/local-computer/join", vmJoin, "device-1", "short")).toThrow(); + + const vm = relay.rewriteJoinResponse( + "/api/bots/bot-1/local-computer/join", vmJoin, "device-1", "phone-lease-0123456789", + ) as { joinUrl: string }; + expect(vm.joinUrl).toMatch(/^\/vps-viewer\/[A-Za-z0-9_-]{32}\/vnc\.html#/); + expect(vm.joinUrl).not.toContain("127.0.0.1"); + const viewerPath = vm.joinUrl.split("#")[0]; + const device = { id: "device-1", cloudDesktopAccess: true }; + const status = (path: string) => new Promise((resolve) => { + const res = { + writeHead: (code: number) => { resolve(code); return res; }, + end: () => undefined, + once: () => res, + destroy: () => undefined, + headersSent: false, + }; + relay.handleHttp({ url: path, method: "POST", headers: {} } as never, res as never, device); + }); + // POST is refused by method, but only after the session is found (404 otherwise). + await new Promise((resolve) => setTimeout(resolve, 30)); + expect(asked[0]).toBe("device-1/bot-1/phone-lease-0123456789"); + expect(relay.isViewerPath(viewerPath)).toBe(true); + + owned = false; + await new Promise((resolve) => setTimeout(resolve, 40)); + // The session is gone: even a well-formed request for it is not found. + expect(await status(viewerPath)).toBe(404); }); it("pins HTTP and WebSocket traffic to the session, device, and loopback viewer", async () => { diff --git a/docs/ios-companion.md b/docs/ios-companion.md index 98cf82fbc9..f85604eeac 100644 --- a/docs/ios-companion.md +++ b/docs/ios-companion.md @@ -25,6 +25,8 @@ The first version includes: - Approvals and questions, including narrow “always allow” grants. - Resumable SSE, streamed reply text, reconnect hydration, and an opt-in live Boat computer view. The loopback-only VPS SSH viewer remains desktop-only. +- On-demand stills of a bot's Local VM, idle or working, for a phone the + computer owner has allowed computer access. - Markdown rendering and Keychain storage for the phone's pairing trust. - Secure completion of supported credential-request cards using iOS Password AutoFill and QR-pinned HPKE encryption. Apple Passwords/iCloud Keychain is @@ -279,6 +281,36 @@ Allowed in the first release: - Fetch settled screen images and opt into live screen frames. - Request a fresh interactive cloud-desktop viewer only when the computer owner has enabled that capability for this specific paired phone. +- With the same capability, fetch a still of a bot's Local VM + (`POST /api/bots/:id/local-computer/screenshot`). The app always passes the + conversation's `threadId`, so the harness answers 409 for a conversation + that is not on the Local VM. This reads a picture; it cannot start, stop, + remove the VM. Like the desktop panel's preview, each capture + counts as use of the VM, so an open computer view keeps it from being + reclaimed as idle; leaving the view or backgrounding the app stops that. +- With the same capability, drive the Local VM: take the bot's computer under + a device control lease (`POST /api/bots/:id/computer/control`), then + `POST /api/bots/:id/local-computer/join?controlLeaseId=…`. The harness answers + that only for a loopback caller, only for a conversation on the Local VM, + and only to the lease that holds the computer. The sidecar rewrites the + loopback noVNC address into its device-scoped relay path, then asks the + harness every few seconds whether that lease still holds + (`action: "check"`, read-only) and closes the relay when it does not. The + phone speaks RFB over the relay. Hand back closes the viewer and releases + the lease, and so does backgrounding the app. In shared Local VM mode the + hold pauses only this bot, as it does on the Mac. +- A phone paired with the server directly (`openmausbot serve` behind + Tailscale Serve or a tunnel, no sidecar) drives the Local VM the same way, + but the join answers it with the server's own authenticated desktop proxy + (`/api/desktop-viewer/local//websockify`), bound to its control + lease, and the VNC password; never the loopback address. The proxy re-checks + the lease and the session every few seconds and closes the socket when + either lapses; hand back closes it at once. Computer access is the pairing's + scope: Full access (`openmausbot pair`) may, chat-only (`--client`) is + answered 403 and the app shows computer access as off. +- Phone control supports shared and per-bot Local VMs. Pool mode is refused + on both connection paths until a viewer can reserve its seat against + other bots; a control hold on one bot alone cannot do that. - Send messages, interrupt bots, answer approvals/questions, and mark chats read. - Create a basic bot. diff --git a/docs/verification/README.md b/docs/verification/README.md index 54210ff07e..c0ec1a73ed 100644 --- a/docs/verification/README.md +++ b/docs/verification/README.md @@ -174,6 +174,10 @@ interrupting a newer request. The [iOS thread checks](ios-threads.md) cover the native thread tree, folder search and draft isolation using disposable simulators and an offline fixture. +The [iOS Local VM view](ios-local-vm.md) pairs a disposable simulator with an +isolated server, companion sidecar and synthetic Local VM to check on-demand +stills and the per-device computer-access gate. + The [Android stream recovery checks](android-stream-recovery.md) exercise early stream closure and fallback through disposable HTTP endpoints. diff --git a/docs/verification/ios-local-vm.md b/docs/verification/ios-local-vm.md new file mode 100644 index 0000000000..543c3b5245 --- /dev/null +++ b/docs/verification/ios-local-vm.md @@ -0,0 +1,119 @@ +# iOS Local VM view + +Launch the isolated server, synthetic Local VM and companion sidecar: + +```sh +node --experimental-strip-types scripts/verify-ios-local-vm.ts +``` + +The script starts the standard fake-engine server in a disposable home, a +synthetic `docker` that answers only inspection and the two screenshot execs, +an offline password-protected RFB desktop (`scripts/testing/fake-vnc-desktop.ts`) +published as the VM's noVNC port, and the companion sidecar pointed at that +server. It creates a bot named Vee +with `computer: "vm"`, checks that +`POST /api/bots/:id/local-computer/screenshot` returns a PNG, then prints the +sidecar address and a pairing code. The desktop is a captured Local VM session +(`scripts/testing/fixtures/local-vm-desktop.png`, an XFCE desktop with a terminal +open and nothing private on it); each capture types one more character at the +prompt, so a refresh is visible. It never reaches a +real container runtime, a VM, or the user's OpenMausBot data. Ctrl-C stops the +server and sidecar and removes the temporary data. + +The sidecar listens on all interfaces like the real one, but only a device +holding a fresh pairing code from this run can use it. + +Use a disposable simulator: + +1. Build the `OpenMausCompanion` scheme with a team and local signing, as in + the [iOS runbook](../../ios/TESTING.md), and install it on a fresh simulator. +2. Pair by opening + `openmausbot://pair?address=127.0.0.1:PORT&code=CODE` in the simulator with + the printed address and code, then tap **Connect**. +3. Open Vee, then its computer. With computer access off (the default), the view + says to turn on **Allow computer view** in Settings → Remote access. +4. Enable it from the printed control address: + `curl -X POST http://127.0.0.1:CONTROL/devices/DEVICE_ID/cloud-desktop` + (`GET /state` on the same address lists the device id). The sidecar drops + the device's connection so it reconnects with the new capability. +5. Without leaving the view, the idle VM's picture appears at the next + 30-second check, and another character appears at the prompt on each refresh + after that. Revoking + access (`DELETE` on the same address) clears the picture and brings the + notice back at the next check. +6. Set another bot's computer to `off` on the harness; its computer view keeps + the existing "only captured while it is working" message. +7. Back on Vee, tap **Take control**. The live desktop appears with a pointer + ring, and the printed events address reports one authenticated connection + and `controlHeld: true`. +8. Swipe on the trackpad: the pointer moves and `lastPointer` follows. Tap: the + desktop paints a marker where the click landed, and the phone shows it. +9. Open the keyboard and type: `typed` shows the text. +10. **Hand Back**: `controlHeld` returns to `false` and **Take control** is + offered again. Taking control and then sending the app to the background + releases it too. +11. Take control again, then release it from the harness instead + (`POST /api/bots/BOT/computer/control` with `{"action":"release"}`): within + a few seconds the sidecar's lease check fails, the relay closes, and the + phone shows the desktop as disconnected. + +The companion route and capability checks are covered by +`companion/test/routes.test.ts` and `companion/test/proxy-response.test.ts`; +the join and relay rewrite by `companion/test/viewer-relay.test.ts` and +`server/index.test.ts`; the client calls by +`ios/Tests/CompanionCoreTests/LocalVmScreenshotClientTests.swift` and +`LocalVmControlClientTests.swift`; and the VNC protocol, byte for byte, by +`RFBTests.swift`. + +## Phones paired with the server directly + +A phone paired with `openmausbot serve` itself (a headless server, reached over +Tailscale Serve or a tunnel) has no companion sidecar, so nothing rewrites the +VM's noVNC address for it. The join route answers such a phone differently: a +path on the server's own authenticated desktop proxy +(`/api/desktop-viewer/local/shared/websockify` for the shared VM; per-bot +targets likewise), bound to the phone's control lease and to the +conversation whose VM seat the join picked, plus the VNC password. The phone never sees a loopback address, and the +proxy re-checks the lease and the session every few seconds and closes the +socket when either lapses. Computer access is the pairing's scope: a Full +access pairing (`openmausbot pair`) may; a chat-only one (`--client`) is +answered 403, which the phone shows as computer access being off. + +Phone control is unavailable in pool mode, through either connection path. +A bot's control hold does not reserve a pool seat against other bots. The +join route refuses pool mode, and the desktop proxy independently refuses +lease-bound pool URLs before reading credentials or opening a socket. +Shared and per-bot desktops remain supported. Reserving a pool seat for the +whole viewer lifetime is required before enabling interactive pool viewers. + +Check it against the same fixture, talking to the printed `harness` address +rather than the sidecar. With `BOT` and `THREAD` from the fixture's output and +`LEASE` any name of 16 to 120 URL-safe characters: + +1. Pair a phone session: `POST /api/auth/pairing` with `{}` (loopback is the + owner), then `POST /api/auth/pair` with the code. Use its token as a bearer + below. Pair a second one with `{"scopes":["client"]}` for the chat-only case. +2. Chat-only: `POST /api/bots/BOT/local-computer/join?threadId=THREAD&controlLeaseId=LEASE` + and a WebSocket upgrade of + `/api/desktop-viewer/local/shared/websockify?botId=BOT&threadId=THREAD&controlLeaseId=LEASE` + both answer 403. +3. Full access, before taking control: the join answers 409 "Take control of + this computer first", and so does the proxy. +4. `POST /api/bots/BOT/computer/control` with `{"action":"take","controlLeaseId":"LEASE"}`, + then the join: 200 with `socketPath` and `password`, and no `joinUrl` or + `127.0.0.1` anywhere in the body. +5. Upgrade `/` + `socketPath` with the bearer: 101, and the first bytes are the + desktop's `RFB 003.008` greeting. The events address shows one connection + and `controlHeld: true`. +6. `POST /api/bots/BOT/computer/viewer-close` answers `{"closed":true}` and the + socket closes at once; a viewer-close for another bot leaves it open. Open it again, then release the lease + (`{"action":"release","controlLeaseId":"LEASE"}`): the socket closes within + about five seconds, and both the join and the proxy answer 409 again. + `POST /api/auth/logout` on the phone's session closes an open socket + immediately. +7. The sidecar path is unchanged: the same join from loopback, without a + bearer, still returns the raw `joinUrl` for the sidecar to rewrite. + +The proxy's lease binding is covered by `server/routes/desktop-viewer.test.ts` +and the route's answers to direct sessions by `server/index.test.ts`; the +phone's acceptance of only this proxy shape by `LocalVmControlClientTests.swift`. diff --git a/ios/App/ComputerView.swift b/ios/App/ComputerView.swift index 56cc03fd33..cc578c8464 100644 --- a/ios/App/ComputerView.swift +++ b/ios/App/ComputerView.swift @@ -10,6 +10,12 @@ // off unless this view is on screen. `watchScreen` reopens the stream asking // for them and `stopWatchingScreen` reopens it asking not to; both resume // from the cursor, so the reconnect costs nothing but a round trip. +// +// A Local VM can also be pictured while its bot is idle: this view asks the +// harness for a still every thirty seconds (every three while the bot works +// and the stream has gone quiet), the same cadence as the desktop panel. The +// Mac has to allow computer access for this phone first; until it does, the +// sidecar answers 403 and the view says where to turn it on. import SwiftUI import CompanionCore // Unconditional for the same reason as ChatView: `UIImage` is used below @@ -25,18 +31,57 @@ struct ComputerView: View { @State private var openingDesktop = false @State private var desktopURL: URL? @State private var desktopError: String? + @Environment(\.scenePhase) private var scenePhase + /// The latest on-demand Local VM still, and when it arrived. + @State private var polled: (shot: LocalVmScreenshot, at: Date)? + /// When the event stream last delivered a frame, so the newer of the + /// two pictures is the one on screen. + @State private var streamFrameAt: Date? + @State private var fetchingFirstStill = false + @State private var vmProblem: LocalVmProblem? + /// The live desktop while this phone holds the Local VM, and the lease + /// it holds it under. + @State private var control: (desktop: LocalVmDesktop, leaseId: String, client: CompanionClient)? + @State private var takingControl = false + /// A hand-back still releasing. The lease id is reused per bot and + /// computer, so a take started now would be undone when that release + /// lands; Take control waits for it. + @State private var handingBack = false + /// The take in flight, cancelled if the person leaves before it lands. + @State private var taking: Task? + @State private var controlError: String? + + private enum LocalVmProblem: Equatable { + /// The Mac has not allowed computer access for this phone. + case accessOff + /// The VM exists in this conversation but cannot be pictured now. + case unavailable(String) + } private var frame: ScreenFrame? { session.state.screens[bot.id] } + /// Whichever picture is newer: a streamed frame of a working bot, or a + /// still fetched on demand. + private var shownImageData: Data? { + if let polled, streamFrameAt.map({ $0 < polled.at }) ?? true { return polled.shot.data } + return frame?.data + } + + /// Cloud computers have their own viewer below; every other kind may be + /// the Local VM, which the harness confirms or refuses (409) per thread. + private var mayBeLocalVm: Bool { current.computer != "cloud" } + /// The bot as the stream last described it — `busy` is what tells us /// whether more frames are coming or this is the last one. - private var current: Bot { session.state.bot(bot.id) ?? bot } + /// Projected onto the thread this view was opened from, so a task thread + /// pictures its own computer, not the bot's default conversation. + private var current: Bot { session.state.bot(bot.id)?.projected(forThread: bot.threadId) ?? bot } var body: some View { ZStack { Color.black.ignoresSafeArea() - if let image = frame.flatMap(\.data).flatMap(UIImage.init(data:)) { + if let image = shownImageData.flatMap(UIImage.init(data:)) { Image(uiImage: image) .resizable() .scaledToFit() @@ -44,6 +89,20 @@ struct ComputerView: View { // letterbox. Pinch-to-zoom would be the obvious next // thing; scaledToFit is the honest starting point. .accessibilityLabel("\(current.name)'s computer") + // The last good picture stays up, but says when it could + // not be refreshed rather than passing for current. With + // access off only a streamed frame can be on screen; it + // stays, captioned, so the notice is not lost behind it. + .overlay(alignment: .bottom) { + switch vmProblem { + case .accessOff: + caption("lock.display") { Text("Computer access is off for this phone") } + case let .unavailable(reason): + caption("exclamationmark.triangle.fill") { Text("Couldn't refresh: \(reason)") } + case nil: + EmptyView() + } + } } else { waiting } @@ -61,6 +120,9 @@ struct ComputerView: View { } } .safeAreaInset(edge: .bottom) { + if polled != nil && vmProblem == nil && control == nil { + takeControlBar + } // A VPS-backed bot is "cloud" too, but the server refuses to mint // an interactive desktop for it — no button beats a dead one. An // older harness never sends cloudBackend, so nil keeps the button. @@ -119,18 +181,215 @@ struct ComputerView: View { } .onDisappear { session.stopWatchingScreen(of: bot.id) + // A take still in flight is abandoned; when it lands it hands + // the computer straight back. + taking?.cancel() + } + .onValueChange(of: frame?.png) { png in + if png != nil { streamFrameAt = Date() } + } + // Restarted when the bot starts or stops working (the cadence + // changes); stopped in the background and while this phone is + // driving the VM live. + .task(id: "\(current.busy == true)|\(scenePhase == .active)|\(control == nil)") { + guard scenePhase == .active, control == nil else { return } + await pollLocalVm() + } + .fullScreenCover(isPresented: Binding( + get: { control != nil }, + set: { if !$0 { Task { await handBack() } } } + )) { + if let control { + LocalVmControlView(botName: current.name, desktop: control.desktop) { + Task { await handBack() } + } + } + } + // Control needs the app in front: a phone that is locked or + // switched away gives the computer back rather than leaving the bot + // locked out behind a lease nobody is using. + .onValueChange(of: scenePhase) { phase in + guard phase == .background else { return } + taking?.cancel() + if control != nil { Task { await handBack() } } + } + } + + /// Take or join the Local VM: a person can then drive it from the + /// trackpad, and the bot's own computer actions are refused until Hand + /// Back. + private var takeControlBar: some View { + VStack(spacing: 8) { + if let controlError { + Text(verbatim: controlError) + .font(.footnote) + .foregroundStyle(.red) + .multilineTextAlignment(.center) + } + Button { + taking = Task { await takeControl() } + } label: { + if takingControl { + ProgressView().tint(.white).frame(maxWidth: .infinity) + } else { + Label("Take control", systemImage: "hand.raised") + .frame(maxWidth: .infinity) + } + } + .buttonStyle(.borderedProminent) + .disabled(takingControl || handingBack) + Text("The bot pauses its computer work until you hand it back.") + .font(.caption) + .foregroundStyle(Color.white.opacity(0.6)) + .multilineTextAlignment(.center) } + .padding(.horizontal, 18) + .padding(.vertical, 12) + .background(.ultraThinMaterial) } + private func takeControl() async { + guard !handingBack else { return } + takingControl = true + controlError = nil + defer { takingControl = false } + do { + let viewer = try await session.takeLocalVm(for: current) + // The person left (or the app went to the background) while this + // was in flight: give the computer straight back instead of + // opening a desktop nobody is looking at. + guard !Task.isCancelled, scenePhase == .active else { + await session.handBackDetached(bot: current, leaseId: viewer.leaseId, client: viewer.client) + return + } + let desktop = LocalVmDesktop(request: viewer.request, password: viewer.password) + desktop.start() + control = (desktop, viewer.leaseId, viewer.client) + } catch is CancellationError { + return + } catch { + if !Task.isCancelled { controlError = error.localizedDescription } + } + } + + private func handBack() async { + guard let taken = control else { return } + control = nil + taken.desktop.stop() + handingBack = true + defer { handingBack = false } + await session.handBackLocalVm(for: current, leaseId: taken.leaseId, client: taken.client) + } + + /// Fetch Local VM stills while this view is on screen. Stops on a 409 + /// saying this conversation is not on the Local VM, and on a 404 from a + /// computer too old to offer it. With computer access off it keeps asking + /// at the idle cadence, so turning it on at the Mac shows up here without + /// leaving the view. + private func pollLocalVm() async { + guard mayBeLocalVm else { return } + fetchingFirstStill = polled == nil + defer { fetchingFirstStill = false } + while !Task.isCancelled { + let busy = current.busy == true + // A working bot's frames already arrive on the stream; only fill + // in when it has gone quiet for longer than a frame interval. + let streamFresh = streamFrameAt.map { Date().timeIntervalSince($0) < 10 } ?? false + if !(busy && streamFresh) { + // Stamped when asked, so a slow capture never outranks a + // streamed frame that arrived while it was being taken. + let askedAt = Date() + do { + let shot = try await session.localVmScreenshot(for: current) + polled = (shot, askedAt) + vmProblem = nil + } catch let APIError.status(code, message) { + switch code { + case 403 where message?.contains("computer access is off") == true + || (message?.contains("admin scope") == true && session.pairedWithServer): + // Revoked or never granted (or, on a server paired + // directly, a chat-only pairing): stop showing the + // old picture. + polled = nil + vmProblem = .accessOff + case 404: + return + case 409 where message?.contains("not using the Local VM") == true: + polled = nil + vmProblem = nil + return + default: + vmProblem = .unavailable(APIError.status(code: code, message: message).localizedDescription) + } + } catch is CancellationError { + return + } catch { + if Task.isCancelled { return } + vmProblem = .unavailable(error.localizedDescription) + } + fetchingFirstStill = false + } + try? await Task.sleep(for: .seconds(busy && vmProblem == nil ? 3 : 30)) + } + } + + @ViewBuilder private var waiting: some View { + switch vmProblem { + case .accessOff where session.pairedWithServer: + notice( + systemImage: "lock.display", + title: "Computer access is off for this phone", + detail: Text("This phone was paired with chat-only access. Pair it again with Full access (openmausbot pair, without --client) to see and control the Local VM.") + ) + case .accessOff: + notice( + systemImage: "lock.display", + title: "Computer access is off for this phone", + detail: Text("Turn on Allow computer view for this phone in OpenMausBot → Settings → Remote access on your computer.") + ) + case let .unavailable(reason): + notice(systemImage: "display.trianglebadge.exclamationmark", title: "Can't show the Local VM", detail: Text(verbatim: reason)) + case nil: + streamWaiting + } + } + + private func caption(_ systemImage: String, @ViewBuilder text: () -> Text) -> some View { + Label { text() } icon: { Image(systemName: systemImage) } + .font(.system(size: 12, weight: .medium)) + .foregroundStyle(Color.white.opacity(0.85)) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(.ultraThinMaterial, in: Capsule()) + .padding(.bottom, 12) + } + + private func notice(systemImage: String, title: LocalizedStringKey, detail: Text) -> some View { + VStack(spacing: 10) { + Image(systemName: systemImage) + .font(.system(size: 28)) + .foregroundStyle(Color.white.opacity(0.7)) + Text(title) + .font(.system(size: 15, weight: .semibold)) + .foregroundStyle(Color.white.opacity(0.85)) + detail + .font(.system(size: 13)) + .foregroundStyle(Color.white.opacity(0.55)) + } + .multilineTextAlignment(.center) + .padding(.horizontal, 32) + } + + private var streamWaiting: some View { VStack(spacing: 12) { ProgressView().tint(.white) - Text(current.busy == true ? "Waiting for a frame…" : "Nothing to show yet") + Text(current.busy == true || fetchingFirstStill ? "Waiting for a frame…" : "Nothing to show yet") .font(.system(size: 15)) .foregroundStyle(Color.white.opacity(0.7)) // An idle bot is not being screenshotted at all, so this would // otherwise be an indefinite spinner with no explanation. - if current.busy != true { + if current.busy != true && !fetchingFirstStill { Text("This bot's computer is only captured while it is working.") .font(.system(size: 13)) .foregroundStyle(Color.white.opacity(0.45)) diff --git a/ios/App/LocalVmControlView.swift b/ios/App/LocalVmControlView.swift new file mode 100644 index 0000000000..73430994ba --- /dev/null +++ b/ios/App/LocalVmControlView.swift @@ -0,0 +1,415 @@ +// Driving a bot's Local VM from the phone: the live desktop on top, a +// trackpad beneath it, and a keyboard on demand. +// +// The pointer moves relatively, like a laptop trackpad, rather than jumping +// to wherever a finger lands on the picture: the desktop is drawn at a +// fraction of its size, and a fingertip covers several of its controls at +// once. One finger moves, a tap clicks, two fingers tapping right-click, a +// hold then a move drags, and two fingers moving scroll. +import CompanionCore +import SwiftUI +import UIKit + +struct LocalVmControlView: View { + let botName: String + @ObservedObject var desktop: LocalVmDesktop + let handBack: () -> Void + + @State private var typing = false + /// The keyboard's own appearance, not the button that asked for it. The + /// room the keyboard takes and the trackpad's collapse change in one + /// transaction, so the desktop between them shrinks in one motion: left + /// to SwiftUI's own keyboard avoidance, the two run on different curves, + /// the screen is briefly shorter than the picture needs, and the picture + /// narrows for a few frames before growing back. + @StateObject private var keyboard = KeyboardPresence() + /// The trackpad's hint, cut the moment the keyboard starts up and faded + /// back as the pad grows. Its own flag, not an animation on the hint: + /// an animation modifier there delays the hint's move as well as its + /// fade, and it would slide into place late. + @State private var hintShown = true + + var body: some View { + VStack(spacing: 14) { + header + screen + .frame(maxHeight: .infinity) + // The desktop is drawn as wide as the phone, so on a phone it + // needs only a couple of hundred points of height. With the + // keyboard up, a full-size trackpad is what squeezes it below + // that; a short one leaves the picture at full width and keeps a + // click within reach, which typing into a desktop needs often. + Trackpad(desktop: desktop) + .frame(height: keyboard.visible ? 96 : 230) + .overlay { + VStack(spacing: 6) { + Capsule().fill(Color.white.opacity(0.35)).frame(width: 36, height: 4) + Text("Trackpad") + .font(.system(size: 15, weight: .semibold)) + // Kept in the layout so the label does not re-centre + // around it. Cut, not faded, as the pad shrinks: a + // fading line rides up over the label while the + // layout moves. It fades back in once the pad has + // grown again. + Text("Swipe to move · Tap to click · Hold to drag") + .font(.system(size: 12)) + .foregroundStyle(Color.white.opacity(0.6)) + .opacity(hintShown ? 1 : 0) + } + .foregroundStyle(Color.white.opacity(0.85)) + .allowsHitTesting(false) + } + .accessibilityElement() + .accessibilityLabel("Trackpad") + .accessibilityHint("Swipe to move the pointer, tap to click, hold to drag") + // Holds first responder while typing; invisible. + KeyCatcher(active: $typing, desktop: desktop) + .frame(width: 1, height: 1) + .opacity(0.01) + } + .padding(.horizontal, 16) + .padding(.top, 8) + .padding(.bottom, 12 + keyboard.height) + .ignoresSafeArea(.keyboard) + .onValueChange(of: keyboard.visible) { visible in + if visible { + var cut = Transaction() + cut.disablesAnimations = true + withTransaction(cut) { hintShown = false } + return + } + Task { @MainActor in + try? await Task.sleep(for: .milliseconds(250)) + guard !keyboard.visible else { return } + withAnimation(.easeIn(duration: 0.2)) { hintShown = true } + } + } + .background(Color.black.ignoresSafeArea()) + .preferredColorScheme(.dark) + } + + private var header: some View { + HStack(spacing: 10) { + Button("Hand Back", action: handBack) + .font(.system(size: 16, weight: .medium)) + .padding(.horizontal, 16) + .frame(height: 44) + .glassCapsule() + .accessibilityHint("Gives the computer back to the bot") + + Spacer(minLength: 4) + VStack(spacing: 2) { + Text(botName) + .font(.system(size: 16, weight: .semibold)) + .lineLimit(1) + Text("Local VM") + .font(.system(size: 12)) + .foregroundStyle(.secondary) + } + Spacer(minLength: 4) + + HStack(spacing: 0) { + Button { + typing.toggle() + } label: { + Image(systemName: typing ? "keyboard.chevron.compact.down" : "keyboard") + .font(.system(size: 17, weight: .medium)) + .frame(width: 44, height: 44) + } + .accessibilityLabel(typing ? "Hide keyboard" : "Show keyboard") + + Menu { + Button("Escape") { desktop.press([RFBKey.escape]) } + Button("Tab") { desktop.press([RFBKey.tab]) } + Button("Right click") { desktop.click(.right) } + Button("Copy (Ctrl+C)") { desktop.press([RFBKey.control, 0x63]) } + Button("Paste (Ctrl+V)") { desktop.press([RFBKey.control, 0x76]) } + Button("Select all (Ctrl+A)") { desktop.press([RFBKey.control, 0x61]) } + Button("Ctrl+Alt+Del") { desktop.press([RFBKey.control, RFBKey.alt, RFBKey.delete]) } + } label: { + Image(systemName: "ellipsis") + .font(.system(size: 17, weight: .medium)) + .frame(width: 44, height: 44) + } + .accessibilityLabel("More keys") + } + .padding(.horizontal, 4) + .glassCapsule() + } + .foregroundStyle(Color.primary) + } + + private var screen: some View { + GeometryReader { proxy in + ZStack { + RoundedRectangle(cornerRadius: 18, style: .continuous) + .fill(Color.white.opacity(0.06)) + switch desktop.phase { + case .connecting: + ProgressView("Connecting to the Local VM…") + .tint(.white) + case let .failed(reason): + VStack(spacing: 8) { + Image(systemName: "display.trianglebadge.exclamationmark") + .font(.system(size: 26)) + Text("The desktop disconnected") + .font(.system(size: 15, weight: .semibold)) + Text("Control may have been taken back on the computer, or the connection dropped. Hand back, then take control again.") + .font(.system(size: 13)) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + Text(verbatim: reason) + .font(.system(size: 11)) + .foregroundStyle(.tertiary) + .multilineTextAlignment(.center) + } + .padding(24) + case .live: + if let image = desktop.image { + let fitted = fit(desktop.desktopSize, in: proxy.size) + Image(decorative: image, scale: 1) + .resizable() + .interpolation(.medium) + .frame(width: fitted.width, height: fitted.height) + .overlay(alignment: .topLeading) { + pointer + .position( + x: desktop.cursor.x / max(desktop.desktopSize.width, 1) * fitted.width, + y: desktop.cursor.y / max(desktop.desktopSize.height, 1) * fitted.height + ) + } + .accessibilityLabel("\(botName)'s Local VM") + } else { + ProgressView().tint(.white) + } + } + } + .frame(width: proxy.size.width, height: proxy.size.height) + } + } + + /// A ring rather than an arrow: it says where a click will land without + /// pretending to be the desktop's own cursor. + private var pointer: some View { + ZStack { + Circle().strokeBorder(Color.white, lineWidth: 2).frame(width: 22, height: 22) + Circle().fill(Color.white).frame(width: 5, height: 5) + } + .shadow(color: .black.opacity(0.6), radius: 2) + .allowsHitTesting(false) + } + + private func fit(_ size: CGSize, in bounds: CGSize) -> CGSize { + guard size.width > 0, size.height > 0 else { return .zero } + let scale = min(bounds.width / size.width, bounds.height / size.height) + return CGSize(width: size.width * scale, height: size.height * scale) + } +} + +/// The trackpad's gestures, in UIKit, where one- and two-finger pans and taps +/// and a hold-to-drag can coexist with explicit priorities. +private struct Trackpad: UIViewRepresentable { + let desktop: LocalVmDesktop + + func makeCoordinator() -> Coordinator { Coordinator(desktop: desktop) } + + func makeUIView(context: Context) -> UIView { + let view = UIView() + view.backgroundColor = UIColor.white.withAlphaComponent(0.1) + view.layer.cornerRadius = 22 + view.layer.cornerCurve = .continuous + view.layer.borderWidth = 0.5 + view.layer.borderColor = UIColor.white.withAlphaComponent(0.15).cgColor + let coordinator = context.coordinator + + let move = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.move(_:))) + move.maximumNumberOfTouches = 1 + let scroll = UIPanGestureRecognizer(target: coordinator, action: #selector(Coordinator.scroll(_:))) + scroll.minimumNumberOfTouches = 2 + scroll.maximumNumberOfTouches = 2 + let click = UITapGestureRecognizer(target: coordinator, action: #selector(Coordinator.click(_:))) + let rightClick = UITapGestureRecognizer(target: coordinator, action: #selector(Coordinator.rightClick(_:))) + rightClick.numberOfTouchesRequired = 2 + let drag = UILongPressGestureRecognizer(target: coordinator, action: #selector(Coordinator.drag(_:))) + drag.minimumPressDuration = 0.35 + drag.allowableMovement = 8 + + // Holding still long enough is a drag, so a pan only moves once the + // hold has failed. + move.require(toFail: drag) + for recognizer in [move, scroll, click, rightClick, drag] { + recognizer.delegate = coordinator + view.addGestureRecognizer(recognizer) + } + coordinator.trackpad = view + return view + } + + func updateUIView(_ uiView: UIView, context: Context) {} + + final class Coordinator: NSObject, UIGestureRecognizerDelegate { + let desktop: LocalVmDesktop + weak var trackpad: UIView? + private var lastMove = CGPoint.zero + private var lastDrag = CGPoint.zero + private var scrollCarry: CGFloat = 0 + + init(desktop: LocalVmDesktop) { self.desktop = desktop } + + /// Desktop pixels per trackpad point: the pad spans the desktop's + /// width in about one and a half swipes. + @MainActor private var speed: CGFloat { + let width = max(trackpad?.bounds.width ?? 1, 1) + return max(desktop.desktopSize.width / width * 0.75, 1) + } + + @MainActor @objc func move(_ recognizer: UIPanGestureRecognizer) { + let point = recognizer.translation(in: recognizer.view) + // The pan only begins once the hold-to-drag has failed, by which + // time the finger has already travelled; start from there rather + // than jumping the pointer by that distance. + if recognizer.state == .began { lastMove = point } + let delta = CGSize(width: (point.x - lastMove.x) * speed, height: (point.y - lastMove.y) * speed) + lastMove = point + desktop.move(by: delta) + } + + @MainActor @objc func scroll(_ recognizer: UIPanGestureRecognizer) { + if recognizer.state == .began { scrollCarry = 0 } + let velocity = recognizer.translation(in: recognizer.view).y + recognizer.setTranslation(.zero, in: recognizer.view) + // Natural scrolling: fingers up moves the content up (wheel down). + scrollCarry -= velocity + let notch: CGFloat = 18 + let notches = Int(scrollCarry / notch) + if notches != 0 { + scrollCarry -= CGFloat(notches) * notch + desktop.scroll(notches: notches) + } + } + + @MainActor @objc func click(_ recognizer: UITapGestureRecognizer) { + UIImpactFeedbackGenerator(style: .light).impactOccurred() + desktop.click(.left) + } + + @MainActor @objc func rightClick(_ recognizer: UITapGestureRecognizer) { + UIImpactFeedbackGenerator(style: .light).impactOccurred() + desktop.click(.right) + } + + @MainActor @objc func drag(_ recognizer: UILongPressGestureRecognizer) { + let point = recognizer.location(in: recognizer.view) + switch recognizer.state { + case .began: + UIImpactFeedbackGenerator(style: .medium).impactOccurred() + lastDrag = point + desktop.setDragging(true) + case .changed: + desktop.move(by: CGSize(width: (point.x - lastDrag.x) * speed, height: (point.y - lastDrag.y) * speed)) + lastDrag = point + default: + desktop.setDragging(false) + } + } + + func gestureRecognizer( + _ gestureRecognizer: UIGestureRecognizer, + shouldRecognizeSimultaneouslyWith other: UIGestureRecognizer + ) -> Bool { false } + } +} + +/// An invisible first responder that turns the system keyboard into key +/// events: typed characters, Return, and Backspace. +private struct KeyCatcher: UIViewRepresentable { + @Binding var active: Bool + let desktop: LocalVmDesktop + + func makeUIView(context: Context) -> KeyInputView { + let view = KeyInputView() + view.desktop = desktop + view.onResign = { context.coordinator.resigned() } + return view + } + + func updateUIView(_ view: KeyInputView, context: Context) { + context.coordinator.binding = $active + // Outside SwiftUI's update pass: a responder change made during it + // can be dropped, and then the keyboard never comes. + let active = active + DispatchQueue.main.async { + if active, !view.isFirstResponder { view.becomeFirstResponder() } + if !active, view.isFirstResponder { view.resignFirstResponder() } + } + } + + func makeCoordinator() -> Coordinator { Coordinator(binding: $active) } + + final class Coordinator { + var binding: Binding + init(binding: Binding) { self.binding = binding } + func resigned() { if binding.wrappedValue { binding.wrappedValue = false } } + } + + final class KeyInputView: UIView, UIKeyInput { + weak var desktop: LocalVmDesktop? + var onResign: (() -> Void)? + + override var canBecomeFirstResponder: Bool { true } + var hasText: Bool { true } + var autocorrectionType: UITextAutocorrectionType = .no + var autocapitalizationType: UITextAutocapitalizationType = .none + var smartQuotesType: UITextSmartQuotesType = .no + var smartDashesType: UITextSmartDashesType = .no + var spellCheckingType: UITextSpellCheckingType = .no + var keyboardType: UIKeyboardType = .asciiCapable + + func insertText(_ text: String) { + MainActor.assumeIsolated { desktop?.type(text) } + } + + func deleteBackward() { + MainActor.assumeIsolated { desktop?.press([RFBKey.backspace]) } + } + + override func resignFirstResponder() -> Bool { + let resigned = super.resignFirstResponder() + if resigned { onResign?() } + return resigned + } + } +} + +/// Whether the system keyboard is up and how much of the window it covers +/// beyond the bottom safe area, both changed in one animation that follows +/// the keyboard's own spring, so views keyed on them move in step with it. +@MainActor +private final class KeyboardPresence: ObservableObject { + @Published private(set) var visible = false + @Published private(set) var height: CGFloat = 0 + private var observers: [NSObjectProtocol] = [] + + init() { + let center = NotificationCenter.default + for (name, shown) in [(UIResponder.keyboardWillShowNotification, true), (UIResponder.keyboardWillHideNotification, false)] { + observers.append(center.addObserver(forName: name, object: nil, queue: .main) { [weak self] note in + let end = (note.userInfo?[UIResponder.keyboardFrameEndUserInfoKey] as? NSValue)?.cgRectValue ?? .zero + MainActor.assumeIsolated { + let window = UIApplication.shared.connectedScenes + .compactMap { ($0 as? UIWindowScene)?.keyWindow }.first + let covered = window.map { max(0, $0.bounds.maxY - end.minY - $0.safeAreaInsets.bottom) } ?? 0 + // UIKit slides the keyboard on this spring. + withAnimation(.interpolatingSpring(mass: 3, stiffness: 1000, damping: 500)) { + self?.visible = shown + self?.height = shown ? covered : 0 + } + } + }) + } + } + + deinit { + for observer in observers { NotificationCenter.default.removeObserver(observer) } + } +} diff --git a/ios/App/LocalVmDesktop.swift b/ios/App/LocalVmDesktop.swift new file mode 100644 index 0000000000..3f59d11f85 --- /dev/null +++ b/ios/App/LocalVmDesktop.swift @@ -0,0 +1,198 @@ +// A live connection to a bot's Local VM desktop: RFB over the sidecar's +// relayed WebSocket, and the pointer the trackpad moves. +// +// The protocol is RFBClient in CompanionCore; this owns the socket, turns the +// framebuffer into a picture, and keeps a cursor of its own, because the +// phone drives a pointer relatively, like a laptop trackpad, rather than +// tapping where it wants to click. +import CompanionCore +import CoreGraphics +import Foundation + +@MainActor +final class LocalVmDesktop: ObservableObject { + enum Phase: Equatable { + case connecting + case live + case failed(String) + } + + @Published private(set) var phase = Phase.connecting + @Published private(set) var image: CGImage? + /// Where the pointer is, in desktop pixels. + @Published private(set) var cursor = CGPoint.zero + @Published private(set) var desktopSize = CGSize.zero + + private let request: URLRequest + private let rfb: RFBClient + private var socket: URLSessionWebSocketTask? + private var receiving: Task? + private var keepAlive: Task? + private var buttons: RFBButtons = [] + + init(request: URLRequest, password: String?) { + self.request = request + rfb = RFBClient(password: password) + } + + func start() { + guard socket == nil else { return } + let socket = URLSession.shared.webSocketTask(with: request) + // A Local VM framebuffer arrives as one multi-megabyte update. + socket.maximumMessageSize = 64 << 20 + self.socket = socket + socket.resume() + receiving = Task { [weak self] in await Self.receive(from: socket, into: self) } + // A still desktop sends nothing, and an idle socket is one the + // network is free to drop. + keepAlive = Task { [weak socket] in + while !Task.isCancelled { + try? await Task.sleep(for: .seconds(15)) + socket?.sendPing { _ in } + } + } + } + + func stop() { + receiving?.cancel() + keepAlive?.cancel() + socket?.cancel(with: .goingAway, reason: nil) + socket = nil + } + + // MARK: - Input + + /// Move the pointer by a trackpad delta, already scaled to desktop pixels. + func move(by delta: CGSize) { + guard phase == .live else { return } + cursor.x = min(max(0, cursor.x + delta.width), max(0, desktopSize.width - 1)) + cursor.y = min(max(0, cursor.y + delta.height), max(0, desktopSize.height - 1)) + sendPointer() + } + + func click(_ button: RFBButtons = .left) { + guard phase == .live else { return } + rfb.pointer(x: Int(cursor.x), y: Int(cursor.y), buttons: buttons.union(button)) + rfb.pointer(x: Int(cursor.x), y: Int(cursor.y), buttons: buttons) + flush() + } + + /// Press or release the left button where the pointer is, for dragging. + func setDragging(_ dragging: Bool) { + guard phase == .live else { return } + if dragging { buttons.insert(.left) } else { buttons.remove(.left) } + sendPointer() + } + + /// One wheel notch per call; positive `notches` scrolls down. + func scroll(notches: Int) { + guard phase == .live, notches != 0 else { return } + let wheel: RFBButtons = notches > 0 ? .scrollDown : .scrollUp + for _ in 0 ..< min(abs(notches), 10) { + rfb.pointer(x: Int(cursor.x), y: Int(cursor.y), buttons: buttons.union(wheel)) + rfb.pointer(x: Int(cursor.x), y: Int(cursor.y), buttons: buttons) + } + flush() + } + + func type(_ text: String) { + guard phase == .live else { return } + for character in text { + if let keysym = RFBKey.keysym(for: character) { rfb.tap(keysym) } + } + flush() + } + + /// Press `keys` in order and release them in reverse, as a chord. + func press(_ keys: [UInt32]) { + guard phase == .live else { return } + for key in keys { rfb.key(key, down: true) } + for key in keys.reversed() { rfb.key(key, down: false) } + flush() + } + + // MARK: - Socket + + private func sendPointer() { + rfb.pointer(x: Int(cursor.x), y: Int(cursor.y), buttons: buttons) + flush() + } + + private func flush() { + let data = rfb.takeOutgoing() + guard !data.isEmpty, let socket else { return } + socket.send(.data(data)) { [weak self] error in + guard let error else { return } + Task { @MainActor in self?.fail(error) } + } + } + + /// Holds the desktop only weakly between messages, so a desktop nobody + /// keeps is released and its socket closed rather than kept alive by its + /// own read loop. + private static func receive(from socket: URLSessionWebSocketTask, into owner: LocalVmDesktop?) async { + weak var desktop = owner + do { + while !Task.isCancelled { + let message = try await socket.receive() + let data: Data + switch message { + case let .data(bytes): data = bytes + case let .string(text): data = Data(text.utf8) + @unknown default: continue + } + guard let desktop else { socket.cancel(with: .goingAway, reason: nil); return } + try desktop.handle(desktop.rfb.receive(data)) + desktop.flush() + } + } catch { + if !Task.isCancelled { desktop?.fail(error) } + } + } + + private func handle(_ events: [RFBEvent]) { + var redraw = false + for event in events { + switch event { + case let .connected(width, height, _): + desktopSize = CGSize(width: width, height: height) + cursor = CGPoint(x: width / 2, y: height / 2) + phase = .live + case let .updated(resized): + if resized { + desktopSize = CGSize(width: rfb.width, height: rfb.height) + cursor.x = min(cursor.x, max(0, desktopSize.width - 1)) + cursor.y = min(cursor.y, max(0, desktopSize.height - 1)) + } + redraw = true + case .bell, .clipboard: + break + } + } + if redraw { image = Self.picture(rfb.framebuffer, width: rfb.width, height: rfb.height) } + } + + private func fail(_ error: Error) { + guard phase != .failed(error.localizedDescription) else { return } + phase = .failed(error.localizedDescription) + stop() + } + + /// BGRX, little-endian 32-bit: exactly what the RFB client asked for. + private static func picture(_ pixels: [UInt8], width: Int, height: Int) -> CGImage? { + guard width > 0, height > 0, let provider = CGDataProvider(data: Data(pixels) as CFData) else { return nil } + return CGImage( + width: width, + height: height, + bitsPerComponent: 8, + bitsPerPixel: 32, + bytesPerRow: width * 4, + space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue), + provider: provider, + decode: nil, + shouldInterpolate: true, + intent: .defaultIntent + ) + } +} diff --git a/ios/App/Localizable.xcstrings b/ios/App/Localizable.xcstrings index 96178d3a07..8e1ef941f9 100644 --- a/ios/App/Localizable.xcstrings +++ b/ios/App/Localizable.xcstrings @@ -4784,6 +4784,276 @@ } } } + }, + "Computer access is off for this phone": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "O acesso ao computador está desativado para este celular" + } + } + } + }, + "This phone was paired with chat-only access. Pair it again with Full access (openmausbot pair, without --client) to see and control the Local VM.": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Este celular foi pareado com acesso somente ao chat. Pareie-o de novo com acesso total (openmausbot pair, sem --client) para ver e controlar a VM local." + } + } + } + }, + "Turn on Allow computer view for this phone in OpenMausBot → Settings → Remote access on your computer.": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Ative Permitir ver o computador para este celular em OpenMausBot → Configurações → Acesso remoto no seu computador." + } + } + } + }, + "Can't show the Local VM": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Não é possível mostrar a VM local" + } + } + } + }, + "Couldn't refresh: %@": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Não foi possível atualizar: %@" + } + } + } + }, + "Take control": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Assumir o controle" + } + } + } + }, + "The bot pauses its computer work until you hand it back.": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "O bot pausa o trabalho no computador até você devolver o controle." + } + } + } + }, + "Hand Back": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Devolver" + } + } + } + }, + "Gives the computer back to the bot": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Devolve o computador ao bot" + } + } + } + }, + "Local VM": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "VM local" + } + } + } + }, + "Trackpad": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Trackpad" + } + } + } + }, + "Swipe to move · Tap to click · Hold to drag": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Deslize para mover · Toque para clicar · Segure para arrastar" + } + } + } + }, + "Swipe to move the pointer, tap to click, hold to drag": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Deslize para mover o ponteiro, toque para clicar, segure para arrastar" + } + } + } + }, + "Connecting to the Local VM…": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Conectando à VM local…" + } + } + } + }, + "The desktop disconnected": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "A área de trabalho se desconectou" + } + } + } + }, + "Show keyboard": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Mostrar teclado" + } + } + } + }, + "Hide keyboard": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Ocultar teclado" + } + } + } + }, + "More keys": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Mais teclas" + } + } + } + }, + "Escape": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Esc" + } + } + } + }, + "Tab": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Tab" + } + } + } + }, + "Right click": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Clique direito" + } + } + } + }, + "Copy (Ctrl+C)": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Copiar (Ctrl+C)" + } + } + } + }, + "Paste (Ctrl+V)": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Colar (Ctrl+V)" + } + } + } + }, + "Select all (Ctrl+A)": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Selecionar tudo (Ctrl+A)" + } + } + } + }, + "Ctrl+Alt+Del": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "Ctrl+Alt+Del" + } + } + } + }, + "%@'s Local VM": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "VM local de %@" + } + } + } + }, + "Control may have been taken back on the computer, or the connection dropped. Hand back, then take control again.": { + "localizations": { + "pt-BR": { + "stringUnit": { + "state": "translated", + "value": "O controle pode ter sido retomado no computador, ou a conexão caiu. Devolva e assuma o controle de novo." + } + } + } } }, "version": "1.0" diff --git a/ios/App/Session.swift b/ios/App/Session.swift index 6d34373265..dfbce72e3d 100644 --- a/ios/App/Session.swift +++ b/ios/App/Session.swift @@ -58,6 +58,9 @@ final class Session: ObservableObject { /// `Connection.canAdminister`. Views hide owner-only controls when this /// is false rather than offer buttons the server would answer 403 to. var canAdminister: Bool { connection?.canAdminister ?? false } + /// Paired with a server directly rather than through the companion + /// sidecar: see `Connection.pairedWithServer`. + var pairedWithServer: Bool { connection?.pairedWithServer ?? false } @Published private(set) var status: Status = .unpaired /// Transient, user-facing failures from an action they just took. @Published var actionError: String? @@ -1652,6 +1655,108 @@ final class Session: ObservableObject { } } + /// Run one call against the active computer, marking the pairing + /// unauthorized when the computer says so. + private func withClient(_ call: (CompanionClient) async throws -> T) async throws -> T { + guard let client else { throw APIError.transport("This computer is offline.") } + return try await withClient(client, call: call) + } + + /// Run one call against a particular computer. Local VM control holds on + /// to the client that granted its lease, so joining the desktop and + /// handing back reach that computer even after the phone has switched + /// to another one in Settings. While that computer is still the active + /// one, the call goes down its current route: a lease must not be + /// released through an address the session has since moved away from. + private func withClient(_ client: CompanionClient, call: (CompanionClient) async throws -> T) async throws -> T { + let client = self.client?.connection.id == client.connection.id ? self.client ?? client : client + do { + return try await call(client) + } catch let error as APIError where error.isUnauthorized { + // As for screenshots: a call answered by the computer the phone + // has since switched away from must not evict the new session. + guard !Task.isCancelled, self.client?.connection.id == client.connection.id else { + throw CancellationError() + } + status = .unauthorized + throw error + } + } + + /// The control lease this phone uses for one bot on one computer. Kept + /// across launches, so a session the app never got to hand back (it was + /// killed while driving) can be taken again and released, rather than + /// leaving the bot locked behind a lease nobody remembers. + private func localVmLease(for bot: Bot, on client: CompanionClient) -> String { + let key = "localVmControlLease.\(client.connection.id).\(bot.id)" + if let saved = UserDefaults.standard.string(forKey: key) { return saved } + let lease = "phone-" + UUID().uuidString + UserDefaults.standard.set(lease, forKey: key) + return lease + } + + /// Take the bot's computer under this phone's lease and open its Local + /// VM's relayed desktop. Any failure after asking hands the computer + /// straight back, so a failed attempt never leaves the bot locked out — + /// except when someone else holds it: then there is nothing of ours to + /// release, and closing viewers could disturb theirs. + /// + /// The client that answered the take comes back with the lease: the + /// caller hands back through it, so a computer switched away from in + /// Settings is still released rather than left paused under our lease. + func takeLocalVm(for bot: Bot) async throws -> (request: URLRequest, password: String?, leaseId: String, client: CompanionClient) { + guard let client else { throw APIError.transport("This computer is offline.") } + let leaseId = localVmLease(for: bot, on: client) + var handBackOnFailure = true + do { + let state = try await withClient(client) { try await $0.computerControl(botId: bot.id, take: true, leaseId: leaseId) } + if state.held, state.owned == false { + handBackOnFailure = false + throw APIError.transport("Someone else is already controlling this computer.") + } + guard state.held else { throw APIError.transport("The computer could not be taken. Try again.") } + return try await withClient(client) { client in + let viewer = try await client.localVmViewer(botId: bot.id, threadId: bot.threadId, leaseId: leaseId) + return (try client.viewerSocketRequest(viewer), viewer.password, leaseId, client) + } + } catch { + if handBackOnFailure { await handBackDetached(bot: bot, leaseId: leaseId, client: client) } + throw error + } + } + + /// Hand back from a task of its own, so cancelling whatever asked (the + /// person left mid-take) cannot cancel the release with it. + func handBackDetached(bot: Bot, leaseId: String, client: CompanionClient) async { + await Task { await self.handBackLocalVm(for: bot, leaseId: leaseId, client: client) }.value + } + + /// Close this device's viewer and release the lease on the computer that + /// granted it, finishing even if the app is on its way to the background. + /// Best effort: releasing a lease that no longer holds anything is a + /// no-op on the harness. + func handBackLocalVm(for bot: Bot, leaseId: String, client: CompanionClient) async { + let task = UIApplication.shared.beginBackgroundTask(withName: "Hand back the Local VM") + defer { if task != .invalid { UIApplication.shared.endBackgroundTask(task) } } + _ = try? await withClient(client) { try await $0.closeViewer(botId: bot.id) } + _ = try? await withClient(client) { try await $0.computerControl(botId: bot.id, take: false, leaseId: leaseId) } + } + + func localVmScreenshot(for bot: Bot) async throws -> LocalVmScreenshot { + guard let client else { throw APIError.transport("This computer is offline.") } + do { + return try await client.localVmScreenshot(botId: bot.id, threadId: bot.threadId) + } catch let error as APIError where error.isUnauthorized { + // A poll still in flight when the phone switched computers must + // not evict the new session with the old token's 401. + guard !Task.isCancelled, self.client?.connection.id == client.connection.id else { + throw CancellationError() + } + status = .unauthorized + throw error + } + } + func markRead(_ chat: Chat) async { await perform(quietly: true) { switch chat { diff --git a/ios/README.md b/ios/README.md index 238c8a57fc..4f9c985350 100644 --- a/ios/README.md +++ b/ios/README.md @@ -181,11 +181,12 @@ here by simply not having the methods: | Read bots, rooms and transcripts | Write API keys (`PUT /api/config`) | | Send messages, make a bot or a room | Manage pairing or revoke devices | | Share selected text, links, images and documents | Browse arbitrary files on the phone or Mac | -| **Answer approvals and questions** | Drive the Local VM or this computer | +| **Answer approvals and questions** | Drive this computer, or change the Local VM's lifecycle | | Interrupt a bot, mark chats read | Reach `/api/internal/*` | | File visible bots into one sidebar section | Use general bot or room `PATCH` routes | | Fetch screen images on demand | Load the packaged desktop UI | | Open an explicitly enabled cloud desktop | Provision, sleep or run shell commands on cloud computers | +| See an explicitly enabled Local VM, idle or working, and take control of it | | Marking a chat read and remembering an approval use purpose-built server verbs. Section creation likewise uses one strict atomic batch route. The @@ -199,8 +200,31 @@ mean losing the ability to lock it out. Interactive cloud desktop access is additionally enabled per paired device and starts off. The phone asks the Mac to mint a fresh provider URL after an explicit warning, validates that it is HTTPS, opens it in an in-app Safari -sheet, and never persists it. The Local VM's loopback-only noVNC listener and -the host computer remain unreachable through the companion. +sheet, and never persists it. + +The same per-device switch (**Allow computer view** in Settings → Remote access) +lets the phone fetch a still of a bot's Local VM on demand, so the computer view +shows it even while the bot is idle: every 30 seconds while the view is open, +every 3 while the bot works and its streamed frames have gone quiet. It is a +picture only. + +With the same switch on, **Take control** under that picture drives the VM from +the phone. The phone takes the bot's computer under its own control lease — the +harness then refuses that bot's computer actions, as when someone takes control +on the Mac (in the default shared Local VM mode, other bots on the same VM are +not paused, on the phone or on the Mac) — and asks for the VM's live desktop. +The harness hands that out only to a loopback caller and only to the lease +holding the computer; the sidecar relays it to this one device the way it +already relays a VPS viewer, so the VM's noVNC port never leaves the Mac, and it +re-checks the lease every few seconds and cuts the relay as soon as it no +longer holds (released from the Mac, say). A small RFB client in +`CompanionCore` speaks VNC over that WebSocket: a trackpad moves a pointer (tap +to click, two fingers to right-click or scroll, hold to drag), and the system +keyboard types. **Hand Back**, or sending the app to the background, closes the +viewer and releases the lease. The lease is kept per bot, so if the app is +killed while driving, taking control again resumes it and Hand Back releases it. +The Local VM's lifecycle and the host computer remain unreachable through the +companion. ## Design notes diff --git a/ios/Sources/CompanionCore/Client.swift b/ios/Sources/CompanionCore/Client.swift index d880123318..65c86150d4 100644 --- a/ios/Sources/CompanionCore/Client.swift +++ b/ios/Sources/CompanionCore/Client.swift @@ -1770,6 +1770,95 @@ public struct CompanionClient: Sendable { ) } + /// A still of the bot's Local VM, whether or not it is working. The phone + /// always names the thread, so the harness answers 409 when that + /// conversation is not on the Local VM rather than picturing a computer it + /// isn't using (and, in pool mode, pictures that thread's own VM). The + /// sidecar requires the same per-device computer access as the cloud + /// desktop, and answers 403 while it is off; a server paired directly + /// answers 403 to a chat-only pairing. + public func localVmScreenshot(botId: String, threadId: String) async throws -> LocalVmScreenshot { + guard Self.validRouteID(botId), Self.validRouteID(threadId) else { throw APIError.badURL } + var request = try makeRequest( + "POST", + "/api/bots/\(botId)/local-computer/screenshot", + query: [URLQueryItem(name: "threadId", value: threadId)] + ) + // The harness execs into the VM for each capture; a busy VM can take + // longer than an ordinary call. + request.timeoutInterval = 45 + return try await send(request, as: LocalVmScreenshot.self) + } + + /// Take or hand back a bot's computer under this device's control lease. + /// While held, the harness refuses the bot's own computer actions. + @discardableResult + public func computerControl(botId: String, take: Bool, leaseId: String) async throws -> ComputerControlState { + guard Self.validRouteID(botId), Self.validRouteID(leaseId), (16...120).contains(leaseId.count) else { + throw APIError.badURL + } + return try await send( + try makeRequest( + "POST", + "/api/bots/\(botId)/computer/control", + body: ["action": take ? "take" : "release", "controlLeaseId": leaseId] + ), + as: ComputerControlState.self + ) + } + + /// The Local VM's live desktop, relayed by the sidecar or, on a phone + /// paired with the server directly, proxied by the server itself. The + /// harness grants it only to the lease that holds the computer, and the + /// relay or proxy closes as soon as that lease stops holding it. + public func localVmViewer(botId: String, threadId: String, leaseId: String) async throws -> LocalVmViewerSession { + guard Self.validRouteID(botId), Self.validRouteID(threadId), Self.validRouteID(leaseId), + (16...120).contains(leaseId.count) + else { throw APIError.badURL } + return try await send( + try makeRequest( + "POST", + "/api/bots/\(botId)/local-computer/join", + query: [ + URLQueryItem(name: "threadId", value: threadId), + URLQueryItem(name: "controlLeaseId", value: leaseId), + ], + body: [:] + ), + as: LocalVmViewerSession.self + ) + } + + /// Close this device's relayed viewers for the bot. + public func closeViewer(botId: String) async throws { + guard Self.validRouteID(botId) else { throw APIError.badURL } + // The harness takes computer mutations as JSON only; without a body + // it answers 415 and closes nothing. + try await send(try makeRequest("POST", "/api/bots/\(botId)/computer/viewer-close", body: [:])) + } + + /// The authenticated WebSocket request for a Local VM viewer, relayed or + /// proxied: same host and token as every other call, `ws` or `wss` to + /// match. + public func viewerSocketRequest(_ viewer: LocalVmViewerSession) throws -> URLRequest { + guard let base = connection.baseURL, + var components = URLComponents(url: base, resolvingAgainstBaseURL: false) + else { throw APIError.badURL } + components.scheme = components.scheme == "https" ? "wss" : "ws" + components.path = "/" + viewer.socketPath + let query = viewer.socketQuery.sorted { $0.key < $1.key }.map { URLQueryItem(name: $0.key, value: $0.value) } + components.queryItems = query.isEmpty ? nil : query + guard let url = components.url else { throw APIError.badURL } + var request = URLRequest(url: url) + request.timeoutInterval = requestTimeout + if let token { request.setValue("Bearer \(token)", forHTTPHeaderField: "Authorization") } + // websockify carries RFB in binary frames, and the sidecar relays + // that subprotocol. The server's own proxy answers without one, and + // a handshake that asked for one would be refused. + if viewer.relayed { request.setValue("binary", forHTTPHeaderField: "Sec-WebSocket-Protocol") } + return request + } + public func markRead(botId: String, threadId: String? = nil) async throws { try await send(try makeRequest("POST", "/api/bots/\(botId)/read", body: threadId.map { ["threadId": $0] })) } diff --git a/ios/Sources/CompanionCore/Models.swift b/ios/Sources/CompanionCore/Models.swift index 5567a89f1f..245292fe0b 100644 --- a/ios/Sources/CompanionCore/Models.swift +++ b/ios/Sources/CompanionCore/Models.swift @@ -890,6 +890,140 @@ public struct CompanionConnectionMetadata: Decodable, Sendable { } } +/// Who is driving a bot's computer. `owned` is present only when the request +/// named a control lease, and says whether that lease is the one holding it. +public struct ComputerControlState: Decodable, Sendable, Equatable { + public let held: Bool + public let owned: Bool? + + public init(held: Bool, owned: Bool? = nil) { + self.held = held + self.owned = owned + } +} + +/// The Local VM's live desktop, as the sidecar relays it to this device: a +/// WebSocket path that only this paired device may open, and the VNC password +/// the desktop asks for. In memory only, like `CloudDesktopSession`; the path +/// is a short-lived capability. +public struct LocalVmViewerSession: Decodable, Sendable, Equatable { + /// The WebSocket path on the paired computer, without its leading slash: + /// `vps-viewer/<32 characters>/websockify` when the companion sidecar + /// relays the desktop, `api/desktop-viewer/local//websockify` + /// when the server itself proxies it to a directly paired phone. + public let socketPath: String + /// What the server's own proxy needs to bind the socket to the control + /// lease (`botId`, `controlLeaseId`, and the `threadId` whose VM seat the + /// join picked). Empty for a sidecar relay. + public let socketQuery: [String: String] + public let password: String? + + /// Whether the companion sidecar relays this desktop. The sidecar speaks + /// websockify's `binary` subprotocol; the server's proxy negotiates none. + public var relayed: Bool { socketPath.hasPrefix("vps-viewer/") } + + private enum CodingKeys: String, CodingKey { case joinUrl, socketPath, password } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + if let raw = try container.decodeIfPresent(String.self, forKey: .socketPath) { + guard let parsed = Self.parseDirect(raw) else { + throw DecodingError.dataCorruptedError( + forKey: .socketPath, + in: container, + debugDescription: "Local VM viewer must be the paired server's own desktop proxy" + ) + } + (socketPath, socketQuery) = parsed + let password = try container.decodeIfPresent(String.self, forKey: .password) + self.password = password?.isEmpty == false ? password : nil + return + } + let raw = try container.decode(String.self, forKey: .joinUrl) + guard let parsed = Self.parse(raw) else { + throw DecodingError.dataCorruptedError( + forKey: .joinUrl, + in: container, + debugDescription: "Local VM viewer must be a relayed path on the paired computer" + ) + } + (socketPath, password) = parsed + socketQuery = [:] + } + + /// Only the sidecar's relay shape is accepted: anything with a scheme or + /// host (a loopback address that was not rewritten, or somewhere else + /// entirely) is refused rather than dialled. + static func parse(_ raw: String) -> (String, String?)? { + guard let components = URLComponents(string: raw), + components.scheme == nil, components.host == nil, + let match = raw.range(of: #"^/vps-viewer/([A-Za-z0-9_-]{32})/"#, options: .regularExpression) + else { return nil } + let id = raw[match].dropFirst("/vps-viewer/".count).dropLast() + let settings = URLComponents(string: "?" + (components.fragment ?? ""))?.queryItems ?? [] + let expected = "vps-viewer/\(id)/websockify" + let path = settings.first { $0.name == "path" }?.value ?? expected + guard path == expected else { return nil } + let password = settings.first { $0.name == "password" }?.value + return (path, password?.isEmpty == false ? password : nil) + } + + /// Only the server's own desktop proxy, for a Local VM target, carrying + /// nothing but the lease binding. Like `parse`, a scheme or host means + /// somewhere other than the paired server and is refused. + static func parseDirect(_ raw: String) -> (String, [String: String])? { + guard let components = URLComponents(string: raw), + components.scheme == nil, components.host == nil, components.fragment == nil, + components.path.range( + of: #"^api/desktop-viewer/local/(shared|bot-[a-f0-9]{64}|pool-\d+)/websockify$"#, + options: .regularExpression + ) != nil + else { return nil } + var query: [String: String] = [:] + for item in components.queryItems ?? [] { + guard ["botId", "threadId", "controlLeaseId"].contains(item.name), let value = item.value, !value.isEmpty, + query[item.name] == nil + else { return nil } + query[item.name] = value + } + guard query["botId"] != nil, query["controlLeaseId"] != nil else { return nil } + return (components.path, query) + } +} + +/// One still of a bot's Local VM, fetched on demand. The harness answers +/// with a `data:` URL; anything but a PNG or JPEG in base64 is refused rather +/// than handed to an image decoder. +public struct LocalVmScreenshot: Decodable, Sendable, Equatable { + public let data: Data + public let mime: String + + private enum CodingKeys: String, CodingKey { case image } + + public init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + let raw = try container.decode(String.self, forKey: .image) + guard let parsed = Self.parse(raw) else { + throw DecodingError.dataCorruptedError( + forKey: .image, + in: container, + debugDescription: "Local VM screenshot must be a base64 PNG or JPEG data URL" + ) + } + (data, mime) = parsed + } + + static func parse(_ raw: String) -> (Data, String)? { + for mime in ["image/png", "image/jpeg"] { + let prefix = "data:\(mime);base64," + guard raw.hasPrefix(prefix) else { continue } + guard let data = Data(base64Encoded: String(raw.dropFirst(prefix.count))), !data.isEmpty else { return nil } + return (data, mime) + } + return nil + } +} + /// A freshly minted provider viewer. It is deliberately not Codable for /// persistence: the URL is a short-lived bearer credential and belongs only /// in memory for the browser session that requested it. diff --git a/ios/Sources/CompanionCore/RFB.swift b/ios/Sources/CompanionCore/RFB.swift new file mode 100644 index 0000000000..ca12eba2ca --- /dev/null +++ b/ios/Sources/CompanionCore/RFB.swift @@ -0,0 +1,482 @@ +// The client half of RFB (the VNC protocol), as far as the phone needs it to +// look at and drive a bot's Local VM through the sidecar's viewer relay. +// +// This is the protocol only: bytes in, bytes out, and a framebuffer. The +// WebSocket that carries it lives in the app, which is what keeps this +// testable byte for byte with `swift test`. +// +// Deliberately small. It speaks 3.3, 3.7 and 3.8; offers None and VNC +// authentication; asks the server for 32-bit little-endian BGRX pixels so the +// framebuffer is already in the layout Core Graphics wants; and understands +// Raw, CopyRect and DesktopSize. Raw is what every server can send, and on the +// LAN or a tailnet it is fine. A compressed encoding is the obvious next step +// for cellular, and the place it would slot in is `rectangleLength`. +import CommonCrypto +import Foundation + +public enum RFBError: Error, Equatable, LocalizedError, Sendable { + case unsupportedVersion(String) + case noUsableSecurity([UInt8]) + case passwordRequired + case authenticationFailed(String) + case unsupportedEncoding(Int32) + case unexpectedMessage(UInt8) + case malformed(String) + + public var errorDescription: String? { + switch self { + case let .unsupportedVersion(version): return "The desktop speaks an unsupported VNC version (\(version))." + case .noUsableSecurity: return "The desktop asked for a sign-in method this app does not support." + case .passwordRequired: return "The desktop asked for a password the computer did not provide." + case let .authenticationFailed(reason): return reason.isEmpty ? "The desktop refused the connection." : reason + case let .unsupportedEncoding(encoding): return "The desktop sent an unsupported picture format (\(encoding))." + case let .unexpectedMessage(type): return "The desktop sent an unexpected message (\(type))." + case let .malformed(what): return "The desktop sent a malformed \(what)." + } + } +} + +/// X11 keysyms for the keys a phone keyboard can produce beyond printable +/// characters. +public enum RFBKey { + public static let backspace: UInt32 = 0xFF08 + public static let tab: UInt32 = 0xFF09 + public static let returnKey: UInt32 = 0xFF0D + public static let escape: UInt32 = 0xFF1B + public static let delete: UInt32 = 0xFFFF + public static let left: UInt32 = 0xFF51 + public static let up: UInt32 = 0xFF52 + public static let right: UInt32 = 0xFF53 + public static let down: UInt32 = 0xFF54 + public static let shift: UInt32 = 0xFFE1 + public static let control: UInt32 = 0xFFE3 + public static let alt: UInt32 = 0xFFE9 + public static let superKey: UInt32 = 0xFFEB + + /// The keysym for one typed character: Latin-1 maps to itself, a newline + /// is Return, and everything else uses the Unicode keysym range. + public static func keysym(for character: Character) -> UInt32? { + if character == "\n" || character == "\r\n" { return returnKey } + if character == "\t" { return tab } + guard character.unicodeScalars.count == 1, let scalar = character.unicodeScalars.first else { return nil } + let value = scalar.value + if (0x20...0x7E).contains(value) || (0xA0...0xFF).contains(value) { return value } + if value < 0x20 || value == 0x7F { return nil } + return 0x0100_0000 | value + } +} + +/// Mouse buttons in RFB's pointer mask. +public struct RFBButtons: OptionSet, Sendable, Hashable { + public let rawValue: UInt8 + public init(rawValue: UInt8) { self.rawValue = rawValue } + public static let left = RFBButtons(rawValue: 1) + public static let middle = RFBButtons(rawValue: 2) + public static let right = RFBButtons(rawValue: 4) + public static let scrollUp = RFBButtons(rawValue: 8) + public static let scrollDown = RFBButtons(rawValue: 16) + public static let scrollLeft = RFBButtons(rawValue: 32) + public static let scrollRight = RFBButtons(rawValue: 64) +} + +public enum RFBEvent: Equatable, Sendable { + /// The handshake finished; the framebuffer has this size. + case connected(width: Int, height: Int, name: String) + /// One FramebufferUpdate was applied. `resized` when its size changed. + case updated(resized: Bool) + case bell + case clipboard(String) +} + +public final class RFBClient { + private enum Phase { + case version + case securityTypes + case securityType33 + case challenge + case securityResult + case serverInit + case normal + } + + static let encodingRaw: Int32 = 0 + static let encodingCopyRect: Int32 = 1 + static let encodingDesktopSize: Int32 = -223 + /// Larger than any desktop a Local VM runs, and small enough that a + /// misbehaving server cannot make the phone allocate gigabytes. + static let maxDimension = 8192 + + private let password: String? + private var phase = Phase.version + private var minor = 8 + private var pending = Data() + private var outgoing = Data() + + public private(set) var width = 0 + public private(set) var height = 0 + public private(set) var name = "" + /// 32-bit little-endian BGRX, row-major, `width * 4` bytes per row. + public private(set) var framebuffer: [UInt8] = [] + + public var isConnected: Bool { phase == .normal } + + public init(password: String?) { + self.password = password + } + + /// Bytes to send, in order. Draining clears them. + public func takeOutgoing() -> Data { + defer { outgoing.removeAll(keepingCapacity: true) } + return outgoing + } + + /// Feed bytes from the server. Returns what happened, in order; throws + /// when the session cannot continue. + @discardableResult + public func receive(_ data: Data) throws -> [RFBEvent] { + pending.append(data) + var events: [RFBEvent] = [] + while let event = try step() { + if case .some(let happened) = event { events.append(happened) } + } + return events + } + + // MARK: - Input + + public func pointer(x: Int, y: Int, buttons: RFBButtons) { + guard isConnected else { return } + var message = Data([5, buttons.rawValue]) + message.appendUInt16(UInt16(clamping: max(0, min(x, width - 1)))) + message.appendUInt16(UInt16(clamping: max(0, min(y, height - 1)))) + outgoing.append(message) + } + + public func key(_ keysym: UInt32, down: Bool) { + guard isConnected else { return } + var message = Data([4, down ? 1 : 0, 0, 0]) + message.appendUInt32(keysym) + outgoing.append(message) + } + + /// Press and release, for typed text. + public func tap(_ keysym: UInt32) { + key(keysym, down: true) + key(keysym, down: false) + } + + public func requestUpdate(incremental: Bool = true) { + guard isConnected else { return } + var message = Data([3, incremental ? 1 : 0]) + message.appendUInt16(0) + message.appendUInt16(0) + message.appendUInt16(UInt16(clamping: width)) + message.appendUInt16(UInt16(clamping: height)) + outgoing.append(message) + } + + // MARK: - Protocol + + /// One message, if enough bytes are here. `nil` means wait for more; + /// `.some(nil)` means a message was handled with nothing to report. + private func step() throws -> RFBEvent?? { + switch phase { + case .version: + guard pending.count >= 12 else { return nil } + let text = String(decoding: pending.prefix(12), as: UTF8.self) + consume(12) + guard text.hasPrefix("RFB "), text.hasSuffix("\n"), + let major = Int(text.dropFirst(4).prefix(3)), let serverMinor = Int(text.dropFirst(8).prefix(3)), + major == 3 + else { throw RFBError.unsupportedVersion(text.trimmingCharacters(in: .whitespacesAndNewlines)) } + // 3.3, 3.7 and 3.8 are the versions there are; a higher minor + // from a newer server is answered with the newest we speak. + minor = serverMinor >= 8 ? 8 : serverMinor >= 7 ? 7 : 3 + outgoing.append(Data("RFB 003.00\(minor)\n".utf8)) + phase = minor == 3 ? .securityType33 : .securityTypes + return .some(nil) + + case .securityType33: + guard pending.count >= 4 else { return nil } + let type = pending.readUInt32(at: 0) + if type == 0 { + guard let reason = try reasonString(at: 4) else { return nil } + throw RFBError.authenticationFailed(reason) + } + consume(4) + switch type { + case 1: return try finishSecurity() + case 2: + guard password != nil else { throw RFBError.passwordRequired } + phase = .challenge + return .some(nil) + default: throw RFBError.noUsableSecurity([UInt8(truncatingIfNeeded: type)]) + } + + case .securityTypes: + guard let count = pending.first else { return nil } + if count == 0 { + guard let reason = try reasonString(at: 1) else { return nil } + throw RFBError.authenticationFailed(reason) + } + guard pending.count >= 1 + Int(count) else { return nil } + let offered = Array(pending[pending.startIndex + 1 ..< pending.startIndex + 1 + Int(count)]) + consume(1 + Int(count)) + if offered.contains(2), password != nil { + outgoing.append(Data([2])) + phase = .challenge + } else if offered.contains(1) { + outgoing.append(Data([1])) + // 3.7 sends no SecurityResult after None. + return minor == 8 ? waitForResult() : try finishSecurity() + } else if offered.contains(2) { + throw RFBError.passwordRequired + } else { + throw RFBError.noUsableSecurity(offered) + } + return .some(nil) + + case .challenge: + guard pending.count >= 16 else { return nil } + let challenge = Data(pending.prefix(16)) + consume(16) + outgoing.append(try Self.vncAuthResponse(challenge: challenge, password: password ?? "")) + return waitForResult() + + case .securityResult: + guard pending.count >= 4 else { return nil } + let result = pending.readUInt32(at: 0) + if result != 0 { + // Only 3.8 explains itself. + if minor == 8 { + guard let reason = try reasonString(at: 4) else { return nil } + throw RFBError.authenticationFailed(reason) + } + throw RFBError.authenticationFailed("") + } + consume(4) + return try finishSecurity() + + case .serverInit: + guard pending.count >= 24 else { return nil } + let nameLength = Int(pending.readUInt32(at: 20)) + guard nameLength <= 4096 else { throw RFBError.malformed("desktop name") } + guard pending.count >= 24 + nameLength else { return nil } + width = Int(pending.readUInt16(at: 0)) + height = Int(pending.readUInt16(at: 2)) + guard width <= Self.maxDimension, height <= Self.maxDimension else { throw RFBError.malformed("desktop size") } + name = String(decoding: pending.subdata(in: pending.startIndex + 24 ..< pending.startIndex + 24 + nameLength), as: UTF8.self) + consume(24 + nameLength) + framebuffer = [UInt8](repeating: 0, count: width * height * 4) + phase = .normal + sendSetPixelFormat() + sendSetEncodings() + requestUpdate(incremental: false) + return .some(.connected(width: width, height: height, name: name)) + + case .normal: + guard let type = pending.first else { return nil } + switch type { + case 0: return try framebufferUpdate() + case 1: + // SetColourMapEntries: never asked for with true colour; skip. + guard pending.count >= 6 else { return nil } + let length = 6 + Int(pending.readUInt16(at: 4)) * 6 + guard pending.count >= length else { return nil } + consume(length) + return .some(nil) + case 2: + consume(1) + return .some(.bell) + case 3: + guard pending.count >= 8 else { return nil } + let length = Int(pending.readUInt32(at: 4)) + guard length <= 1 << 20 else { throw RFBError.malformed("clipboard") } + guard pending.count >= 8 + length else { return nil } + // ServerCutText is Latin-1 by definition. + let bytes = pending.subdata(in: pending.startIndex + 8 ..< pending.startIndex + 8 + length) + let text = String(data: bytes, encoding: .isoLatin1) ?? "" + consume(8 + length) + return .some(.clipboard(text)) + default: + throw RFBError.unexpectedMessage(type) + } + } + } + + private func waitForResult() -> RFBEvent?? { + phase = .securityResult + return .some(nil) + } + + private func finishSecurity() throws -> RFBEvent?? { + // ClientInit: share the desktop, so the bot's own session survives. + outgoing.append(Data([1])) + phase = .serverInit + return .some(nil) + } + + /// A length-prefixed reason at `offset`, or nil until it has arrived. + private func reasonString(at offset: Int) throws -> String? { + guard pending.count >= offset + 4 else { return nil } + let length = Int(pending.readUInt32(at: offset)) + guard length <= 4096 else { throw RFBError.malformed("refusal") } + guard pending.count >= offset + 4 + length else { return nil } + return String(decoding: pending.subdata(in: pending.startIndex + offset + 4 ..< pending.startIndex + offset + 4 + length), as: UTF8.self) + } + + /// A whole FramebufferUpdate, applied only once every rectangle has + /// arrived, so a partial message never leaves half a picture. + private func framebufferUpdate() throws -> RFBEvent?? { + guard pending.count >= 4 else { return nil } + let count = Int(pending.readUInt16(at: 2)) + var offset = 4 + var rects: [(x: Int, y: Int, w: Int, h: Int, encoding: Int32, data: Int)] = [] + for _ in 0 ..< count { + guard pending.count >= offset + 12 else { return nil } + let x = Int(pending.readUInt16(at: offset)) + let y = Int(pending.readUInt16(at: offset + 2)) + let w = Int(pending.readUInt16(at: offset + 4)) + let h = Int(pending.readUInt16(at: offset + 6)) + let encoding = Int32(bitPattern: pending.readUInt32(at: offset + 8)) + if encoding == Self.encodingDesktopSize, w > Self.maxDimension || h > Self.maxDimension { + throw RFBError.malformed("desktop size") + } + let body = try rectangleLength(width: w, height: h, encoding: encoding) + guard pending.count >= offset + 12 + body else { return nil } + rects.append((x, y, w, h, encoding, offset + 12)) + offset += 12 + body + } + var resized = false + for rect in rects { + switch rect.encoding { + case Self.encodingDesktopSize: + width = rect.w + height = rect.h + framebuffer = [UInt8](repeating: 0, count: width * height * 4) + resized = true + case Self.encodingCopyRect: + let sourceX = Int(pending.readUInt16(at: rect.data)) + let sourceY = Int(pending.readUInt16(at: rect.data + 2)) + copyRect(x: rect.x, y: rect.y, w: rect.w, h: rect.h, fromX: sourceX, fromY: sourceY) + default: + blit(x: rect.x, y: rect.y, w: rect.w, h: rect.h, at: rect.data) + } + } + consume(offset) + // Keep the picture coming: one incremental request per update. + requestUpdate(incremental: true) + return .some(.updated(resized: resized)) + } + + private func rectangleLength(width w: Int, height h: Int, encoding: Int32) throws -> Int { + switch encoding { + case Self.encodingRaw: return w * h * 4 + case Self.encodingCopyRect: return 4 + case Self.encodingDesktopSize: return 0 + default: throw RFBError.unsupportedEncoding(encoding) + } + } + + private func blit(x: Int, y: Int, w: Int, h: Int, at offset: Int) { + guard w > 0, h > 0 else { return } + pending.withUnsafeBytes { raw in + let source = raw.baseAddress!.advanced(by: offset).assumingMemoryBound(to: UInt8.self) + framebuffer.withUnsafeMutableBufferPointer { target in + for row in 0 ..< h where y + row < height { + let columns = max(0, min(w, width - x)) + guard columns > 0 else { continue } + let from = source.advanced(by: row * w * 4) + let to = target.baseAddress!.advanced(by: ((y + row) * width + x) * 4) + to.update(from: from, count: columns * 4) + } + } + } + } + + private func copyRect(x: Int, y: Int, w: Int, h: Int, fromX: Int, fromY: Int) { + guard w > 0, h > 0, x + w <= width, y + h <= height, fromX + w <= width, fromY + h <= height else { return } + let rowBytes = w * 4 + // Overlapping regions: walk rows away from the destination, and let + // memmove handle overlap within a row. + let rows: [Int] = y > fromY ? Array((0 ..< h).reversed()) : Array(0 ..< h) + framebuffer.withUnsafeMutableBytes { bytes in + for row in rows { + let from = ((fromY + row) * width + fromX) * 4 + let to = ((y + row) * width + x) * 4 + memmove(bytes.baseAddress! + to, bytes.baseAddress! + from, rowBytes) + } + } + } + + private func sendSetPixelFormat() { + var message = Data([0, 0, 0, 0]) + // 32 bpp, depth 24, little-endian, true colour, 8 bits per channel, + // red at 16, green at 8, blue at 0: BGRX in memory. + message.append(contentsOf: [32, 24, 0, 1]) + message.appendUInt16(255) + message.appendUInt16(255) + message.appendUInt16(255) + message.append(contentsOf: [16, 8, 0, 0, 0, 0]) + outgoing.append(message) + } + + private func sendSetEncodings() { + let encodings = [Self.encodingCopyRect, Self.encodingRaw, Self.encodingDesktopSize] + var message = Data([2, 0]) + message.appendUInt16(UInt16(encodings.count)) + for encoding in encodings { message.appendUInt32(UInt32(bitPattern: encoding)) } + outgoing.append(message) + } + + /// O(1): a slice shares storage, and every read is relative to + /// `startIndex`. The next `append` compacts it. + private func consume(_ count: Int) { + pending = pending.dropFirst(count) + if pending.isEmpty { pending = Data() } + } + + /// VNC authentication: DES-encrypt the 16-byte challenge with the first + /// eight bytes of the password, each byte's bits reversed (the protocol's + /// historical quirk). + static func vncAuthResponse(challenge: Data, password: String) throws -> Data { + var key = [UInt8](repeating: 0, count: 8) + for (index, byte) in Array(password.utf8.prefix(8)).enumerated() { + var reversed: UInt8 = 0 + for bit in 0 ..< 8 where byte & (1 << bit) != 0 { reversed |= 1 << (7 - bit) } + key[index] = reversed + } + var output = [UInt8](repeating: 0, count: 16) + var written = 0 + let status = challenge.withUnsafeBytes { input in + CCCrypt( + CCOperation(kCCEncrypt), CCAlgorithm(kCCAlgorithmDES), CCOptions(kCCOptionECBMode), + key, kCCKeySizeDES, nil, + input.baseAddress, 16, + &output, 16, &written + ) + } + guard status == kCCSuccess, written == 16 else { throw RFBError.malformed("authentication challenge") } + return Data(output) + } +} + +private extension Data { + func readUInt16(at offset: Int) -> UInt16 { + let base = startIndex + offset + return UInt16(self[base]) << 8 | UInt16(self[base + 1]) + } + + func readUInt32(at offset: Int) -> UInt32 { + let base = startIndex + offset + return UInt32(self[base]) << 24 | UInt32(self[base + 1]) << 16 | UInt32(self[base + 2]) << 8 | UInt32(self[base + 3]) + } + + mutating func appendUInt16(_ value: UInt16) { + append(contentsOf: [UInt8(value >> 8), UInt8(value & 0xFF)]) + } + + mutating func appendUInt32(_ value: UInt32) { + append(contentsOf: [UInt8(value >> 24), UInt8(value >> 16 & 0xFF), UInt8(value >> 8 & 0xFF), UInt8(value & 0xFF)]) + } +} diff --git a/ios/Tests/CompanionCoreTests/LocalVmControlClientTests.swift b/ios/Tests/CompanionCoreTests/LocalVmControlClientTests.swift new file mode 100644 index 0000000000..904ba91df4 --- /dev/null +++ b/ios/Tests/CompanionCoreTests/LocalVmControlClientTests.swift @@ -0,0 +1,187 @@ +// Taking a bot's computer, joining its Local VM's relayed desktop, and the +// WebSocket request that carries it. +import XCTest +@testable import CompanionCore + +private final class LocalVmControlStub: URLProtocol { + static var capturedRequest: URLRequest? + static var capturedBody: Data? + static var statusCode = 200 + static var responseBody = Data() + + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + + override func startLoading() { + Self.capturedRequest = request + Self.capturedBody = request.httpBody ?? request.httpBodyStream.map { stream in + stream.open() + defer { stream.close() } + var data = Data() + var buffer = [UInt8](repeating: 0, count: 1_024) + while stream.hasBytesAvailable { + let count = stream.read(&buffer, maxLength: buffer.count) + if count <= 0 { break } + data.append(buffer, count: count) + } + return data + } + let response = HTTPURLResponse( + url: request.url!, + statusCode: Self.statusCode, + httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + )! + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: Self.responseBody) + client?.urlProtocolDidFinishLoading(self) + } + + override func stopLoading() {} +} + +final class LocalVmControlClientTests: XCTestCase { + private var session: URLSession! + private let relayID = String(repeating: "a", count: 32) + + override func setUp() { + super.setUp() + LocalVmControlStub.capturedRequest = nil + LocalVmControlStub.capturedBody = nil + LocalVmControlStub.statusCode = 200 + LocalVmControlStub.responseBody = Data() + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [LocalVmControlStub.self] + session = URLSession(configuration: configuration) + } + + override func tearDown() { + session?.invalidateAndCancel() + session = nil + super.tearDown() + } + + private func client(host: String = "127.0.0.1") -> CompanionClient { + CompanionClient(connection: Connection(name: "Test", host: host, port: 8810), token: "paired-token", session: session) + } + + func testTakesAndHandsBackUnderALease() async throws { + let lease = "phone-lease-0123456789" + LocalVmControlStub.responseBody = Data(#"{"held":true,"helpReason":null,"owned":true,"acquired":true}"#.utf8) + let state = try await client().computerControl(botId: "bot_1", take: true, leaseId: lease) + XCTAssertEqual(state, ComputerControlState(held: true, owned: true)) + let request = try XCTUnwrap(LocalVmControlStub.capturedRequest) + XCTAssertEqual(request.url?.path, "/api/bots/bot_1/computer/control") + XCTAssertEqual(request.value(forHTTPHeaderField: "Content-Type"), "application/json") + let body = try XCTUnwrap(JSONSerialization.jsonObject(with: XCTUnwrap(LocalVmControlStub.capturedBody)) as? [String: String]) + XCTAssertEqual(body, ["action": "take", "controlLeaseId": lease]) + + LocalVmControlStub.responseBody = Data(#"{"held":false,"helpReason":null,"released":true}"#.utf8) + _ = try await client().computerControl(botId: "bot_1", take: false, leaseId: lease) + let released = try XCTUnwrap(JSONSerialization.jsonObject(with: XCTUnwrap(LocalVmControlStub.capturedBody)) as? [String: String]) + XCTAssertEqual(released["action"], "release") + } + + func testClosesTheViewerAsAJsonMutation() async throws { + LocalVmControlStub.responseBody = Data(#"{"closed":true}"#.utf8) + try await client().closeViewer(botId: "bot_1") + let request = try XCTUnwrap(LocalVmControlStub.capturedRequest) + XCTAssertEqual(request.url?.path, "/api/bots/bot_1/computer/viewer-close") + XCTAssertEqual(request.value(forHTTPHeaderField: "Content-Type"), "application/json") + XCTAssertEqual(LocalVmControlStub.capturedBody, Data("{}".utf8)) + } + + func testRefusesALeaseTheHarnessWouldReject() async { + do { + _ = try await client().computerControl(botId: "bot_1", take: true, leaseId: "short") + XCTFail("expected a bad URL") + } catch { + guard case APIError.badURL = error else { return XCTFail("\(error)") } + } + } + + func testJoinsTheThreadsLocalVmAndBuildsItsAuthenticatedSocket() async throws { + LocalVmControlStub.responseBody = try JSONSerialization.data(withJSONObject: [ + "joinUrl": "/vps-viewer/\(relayID)/vnc.html#autoconnect=true&resize=scale&password=vm-secret&path=vps-viewer%2F\(relayID)%2Fwebsockify", + ]) + let viewer = try await client().localVmViewer(botId: "bot_1", threadId: "th-2", leaseId: "phone-lease-0123456789") + let request = try XCTUnwrap(LocalVmControlStub.capturedRequest) + XCTAssertEqual(request.url?.path, "/api/bots/bot_1/local-computer/join") + XCTAssertEqual(request.url?.query, "threadId=th-2&controlLeaseId=phone-lease-0123456789") + XCTAssertEqual(request.value(forHTTPHeaderField: "Content-Type"), "application/json") + XCTAssertEqual(viewer.socketPath, "vps-viewer/\(relayID)/websockify") + XCTAssertEqual(viewer.password, "vm-secret") + + let socket = try client().viewerSocketRequest(viewer) + XCTAssertEqual(socket.url?.absoluteString, "ws://127.0.0.1:8810/vps-viewer/\(relayID)/websockify") + XCTAssertEqual(socket.value(forHTTPHeaderField: "Authorization"), "Bearer paired-token") + XCTAssertEqual(socket.value(forHTTPHeaderField: "Sec-WebSocket-Protocol"), "binary") + } + + func testJoinsThroughTheServersOwnProxyWhenPairedDirectly() async throws { + let lease = "phone-lease-0123456789" + LocalVmControlStub.responseBody = try JSONSerialization.data(withJSONObject: [ + "socketPath": "api/desktop-viewer/local/shared/websockify?botId=bot_1&threadId=th-2&controlLeaseId=\(lease)", + "password": "vm-secret", + ]) + let viewer = try await client(host: "bot.tail0a93.ts.net").localVmViewer(botId: "bot_1", threadId: "th-2", leaseId: lease) + XCTAssertEqual(viewer.socketPath, "api/desktop-viewer/local/shared/websockify") + XCTAssertEqual(viewer.socketQuery, ["botId": "bot_1", "threadId": "th-2", "controlLeaseId": lease]) + XCTAssertEqual(viewer.password, "vm-secret") + XCTAssertFalse(viewer.relayed) + + let socket = try client(host: "bot.tail0a93.ts.net").viewerSocketRequest(viewer) + XCTAssertEqual( + socket.url?.absoluteString, + "ws://bot.tail0a93.ts.net:8810/api/desktop-viewer/local/shared/websockify?botId=bot_1&controlLeaseId=\(lease)&threadId=th-2" + ) + XCTAssertEqual(socket.value(forHTTPHeaderField: "Authorization"), "Bearer paired-token") + // The server's proxy negotiates no subprotocol; asking for one would fail the handshake. + XCTAssertNil(socket.value(forHTTPHeaderField: "Sec-WebSocket-Protocol")) + } + + func testAcceptsOnlyTheServersDesktopProxyShape() { + let lease = "phone-lease-0123456789" + let bound = "botId=bot_1&controlLeaseId=\(lease)" + for raw in [ + "http://127.0.0.1:45679/vnc.html#password=vm-secret", + "https://desktop.example/api/desktop-viewer/local/shared/websockify?\(bound)", + "//evil.example/api/desktop-viewer/local/shared/websockify?\(bound)", + "api/desktop-viewer/local/shared/websockify", + "api/desktop-viewer/local/shared/websockify?botId=bot_1", + "api/desktop-viewer/local/shared/websockify?\(bound)&host=evil.example", + "api/desktop-viewer/local/shared/websockify?\(bound)&botId=bot_2", + "api/desktop-viewer/local/shared/websockify?\(bound)&threadId=", + "api/desktop-viewer/local/127.0.0.1:22/websockify?\(bound)", + "api/desktop-viewer/vps/bot_1/websockify?\(bound)", + "api/desktop-viewer/local/shared?\(bound)", + "api/desktop-viewer/local/shared/websockify?\(bound)#password=x", + ] { + XCTAssertNil(LocalVmViewerSession.parseDirect(raw), raw) + } + let hash = String(repeating: "0", count: 64) + for target in ["shared", "bot-\(hash)", "pool-3"] { + let parsed = LocalVmViewerSession.parseDirect("api/desktop-viewer/local/\(target)/websockify?\(bound)") + XCTAssertEqual(parsed?.0, "api/desktop-viewer/local/\(target)/websockify") + XCTAssertEqual(parsed?.1, ["botId": "bot_1", "controlLeaseId": lease]) + } + XCTAssertEqual( + LocalVmViewerSession.parseDirect("api/desktop-viewer/local/pool-3/websockify?\(bound)&threadId=th-2")?.1, + ["botId": "bot_1", "threadId": "th-2", "controlLeaseId": lease] + ) + } + + func testAcceptsOnlyTheSidecarsRelayShape() { + for raw in [ + "http://127.0.0.1:45679/vnc.html#password=vm-secret", + "https://desktop.example/vps-viewer/\(relayID)/vnc.html", + "//evil.example/vps-viewer/\(relayID)/vnc.html", + "/vps-viewer/short/vnc.html#password=x", + "/vps-viewer/\(relayID)/vnc.html#path=vps-viewer%2F\(String(repeating: "b", count: 32))%2Fwebsockify", + "/other/\(relayID)/vnc.html", + ] { + XCTAssertNil(LocalVmViewerSession.parse(raw), raw) + } + XCTAssertEqual(LocalVmViewerSession.parse("/vps-viewer/\(relayID)/vnc.html")?.0, "vps-viewer/\(relayID)/websockify") + } +} diff --git a/ios/Tests/CompanionCoreTests/LocalVmScreenshotClientTests.swift b/ios/Tests/CompanionCoreTests/LocalVmScreenshotClientTests.swift new file mode 100644 index 0000000000..662a7b814e --- /dev/null +++ b/ios/Tests/CompanionCoreTests/LocalVmScreenshotClientTests.swift @@ -0,0 +1,122 @@ +// An on-demand still of a bot's Local VM: the route the phone asks, and the +// data URL it is willing to turn into an image. +import XCTest +@testable import CompanionCore + +private final class LocalVmScreenshotStub: URLProtocol { + static var capturedRequest: URLRequest? + static var statusCode = 200 + static var responseBody = Data() + + override class func canInit(with request: URLRequest) -> Bool { true } + override class func canonicalRequest(for request: URLRequest) -> URLRequest { request } + + override func startLoading() { + Self.capturedRequest = request + let response = HTTPURLResponse( + url: request.url!, + statusCode: Self.statusCode, + httpVersion: "HTTP/1.1", + headerFields: ["Content-Type": "application/json"] + )! + client?.urlProtocol(self, didReceive: response, cacheStoragePolicy: .notAllowed) + client?.urlProtocol(self, didLoad: Self.responseBody) + client?.urlProtocolDidFinishLoading(self) + } + + override func stopLoading() {} +} + +final class LocalVmScreenshotClientTests: XCTestCase { + private var session: URLSession! + private var client: CompanionClient! + private let png = Data([0x89, 0x50, 0x4E, 0x47, 0x0D, 0x0A, 0x1A, 0x0A, 1, 2, 3]) + + override func setUp() { + super.setUp() + LocalVmScreenshotStub.capturedRequest = nil + LocalVmScreenshotStub.statusCode = 200 + LocalVmScreenshotStub.responseBody = Data() + let configuration = URLSessionConfiguration.ephemeral + configuration.protocolClasses = [LocalVmScreenshotStub.self] + session = URLSession(configuration: configuration) + client = CompanionClient( + connection: Connection(name: "Test", host: "127.0.0.1", port: 8810), + token: "paired-token", + session: session + ) + } + + override func tearDown() { + session?.invalidateAndCancel() + session = nil + client = nil + super.tearDown() + } + + private func body(image: String) throws -> Data { + try JSONSerialization.data(withJSONObject: ["image": image]) + } + + func testAsksForTheThreadsLocalVmAndDecodesThePicture() async throws { + LocalVmScreenshotStub.responseBody = try body(image: "data:image/png;base64,\(png.base64EncodedString())") + + let shot = try await client.localVmScreenshot(botId: "bot_1", threadId: "th-2") + + let request = try XCTUnwrap(LocalVmScreenshotStub.capturedRequest) + XCTAssertEqual(request.httpMethod, "POST") + XCTAssertEqual(request.url?.path, "/api/bots/bot_1/local-computer/screenshot") + XCTAssertEqual(request.url?.query, "threadId=th-2") + XCTAssertEqual(request.value(forHTTPHeaderField: "Authorization"), "Bearer paired-token") + XCTAssertEqual(shot.data, png) + XCTAssertEqual(shot.mime, "image/png") + } + + func testAcceptsAJpegStill() throws { + let jpeg = Data([0xFF, 0xD8, 0xFF, 0xE0, 1, 2]) + let shot = try JSONDecoder().decode( + LocalVmScreenshot.self, + from: try body(image: "data:image/jpeg;base64,\(jpeg.base64EncodedString())") + ) + XCTAssertEqual(shot.data, jpeg) + XCTAssertEqual(shot.mime, "image/jpeg") + } + + func testRefusesAnythingButABase64PngOrJpeg() throws { + for image in [ + "data:image/svg+xml;base64,\(Data("".utf8).base64EncodedString())", + "data:text/html;base64,\(Data("".utf8).base64EncodedString())", + "data:image/png;base64,not base64!", + "data:image/png;base64,", + "https://example.com/screen.png", + png.base64EncodedString() + ] { + XCTAssertThrowsError(try JSONDecoder().decode(LocalVmScreenshot.self, from: try body(image: image)), image) + } + } + + func testSurfacesTheSidecarRefusalWhenComputerAccessIsOff() async throws { + LocalVmScreenshotStub.statusCode = 403 + LocalVmScreenshotStub.responseBody = Data(#"{"error":"computer access is off for this device"}"#.utf8) + + do { + _ = try await client.localVmScreenshot(botId: "bot_1", threadId: "th-2") + XCTFail("expected a 403") + } catch let APIError.status(code, message) { + XCTAssertEqual(code, 403) + XCTAssertEqual(message, "computer access is off for this device") + } + } + + func testRefusesIdsThatWouldChangeTheRoute() async { + for (botId, threadId) in [("../config", "th"), ("bot", "th?x=1"), ("", "th")] { + do { + _ = try await client.localVmScreenshot(botId: botId, threadId: threadId) + XCTFail("expected a bad URL for \(botId) / \(threadId)") + } catch { + guard case APIError.badURL = error else { return XCTFail("\(error)") } + } + } + XCTAssertNil(LocalVmScreenshotStub.capturedRequest) + } +} diff --git a/ios/Tests/CompanionCoreTests/RFBTests.swift b/ios/Tests/CompanionCoreTests/RFBTests.swift new file mode 100644 index 0000000000..80b9f06f4a --- /dev/null +++ b/ios/Tests/CompanionCoreTests/RFBTests.swift @@ -0,0 +1,188 @@ +// The VNC client's protocol half, byte for byte: handshake, authentication, +// the picture, and input. No sockets. +import XCTest +@testable import CompanionCore + +final class RFBTests: XCTestCase { + private func serverInit(width: UInt16, height: UInt16, name: String = "VM") -> Data { + var data = Data() + data.append(contentsOf: [UInt8(width >> 8), UInt8(width & 0xFF), UInt8(height >> 8), UInt8(height & 0xFF)]) + data.append(contentsOf: [32, 24, 0, 1, 0, 255, 0, 255, 0, 255, 16, 8, 0, 0, 0, 0]) + let bytes = Array(name.utf8) + data.append(contentsOf: [0, 0, UInt8(bytes.count >> 8), UInt8(bytes.count & 0xFF)]) + data.append(contentsOf: bytes) + return data + } + + private func u16(_ value: Int) -> [UInt8] { [UInt8(value >> 8 & 0xFF), UInt8(value & 0xFF)] } + private func s32(_ value: Int32) -> [UInt8] { + let bits = UInt32(bitPattern: value) + return [UInt8(bits >> 24), UInt8(bits >> 16 & 0xFF), UInt8(bits >> 8 & 0xFF), UInt8(bits & 0xFF)] + } + + /// A client through the 3.8 handshake with no password, framebuffer `w`×`h`. + private func connected(width: Int = 4, height: Int = 3) throws -> RFBClient { + let client = RFBClient(password: nil) + try client.receive(Data("RFB 003.008\n".utf8)) + try client.receive(Data([1, 1])) + try client.receive(Data([0, 0, 0, 0])) + let events = try client.receive(serverInit(width: UInt16(width), height: UInt16(height))) + XCTAssertEqual(events, [.connected(width: width, height: height, name: "VM")]) + _ = client.takeOutgoing() + return client + } + + func testNegotiates38WithoutAPasswordAndAsksForBGRXPixels() throws { + let client = RFBClient(password: nil) + XCTAssertEqual(try client.receive(Data("RFB 003.008\n".utf8)), []) + XCTAssertEqual(client.takeOutgoing(), Data("RFB 003.008\n".utf8)) + + try client.receive(Data([2, 1, 16])) // None and Tight offered + XCTAssertEqual(client.takeOutgoing(), Data([1])) + + try client.receive(Data([0, 0, 0, 0])) + XCTAssertEqual(client.takeOutgoing(), Data([1]), "ClientInit shares the desktop") + + let events = try client.receive(serverInit(width: 1280, height: 800, name: "Local VM")) + XCTAssertEqual(events, [.connected(width: 1280, height: 800, name: "Local VM")]) + XCTAssertTrue(client.isConnected) + XCTAssertEqual(client.framebuffer.count, 1280 * 800 * 4) + + let out = [UInt8](client.takeOutgoing()) + // SetPixelFormat: 32 bpp, depth 24, little-endian, true colour, BGRX. + XCTAssertEqual(Array(out[0 ..< 20]), [0, 0, 0, 0, 32, 24, 0, 1, 0, 255, 0, 255, 0, 255, 16, 8, 0, 0, 0, 0]) + // SetEncodings: CopyRect, Raw, DesktopSize. + XCTAssertEqual(Array(out[20 ..< 36]), [2, 0, 0, 3] + s32(1) + s32(0) + s32(-223)) + // A full, non-incremental first request. + XCTAssertEqual(Array(out[36 ..< 46]), [3, 0, 0, 0, 0, 0] + u16(1280) + u16(800)) + } + + func testAnswersAVNCPasswordChallenge() throws { + let client = RFBClient(password: "password") + try client.receive(Data("RFB 003.008\n".utf8)) + _ = client.takeOutgoing() + try client.receive(Data([2, 1, 2])) + XCTAssertEqual(client.takeOutgoing(), Data([2]), "prefers VNC auth when it has a password") + + try client.receive(Data("0123456789abcdef".utf8)) + // Independently computed: DES-ECB of the challenge under "password" + // with each key byte's bits reversed. + XCTAssertEqual(client.takeOutgoing().map { String(format: "%02x", $0) }.joined(), "5645abeb5f1e6475e8feb11beb66ea19") + + try client.receive(Data([0, 0, 0, 0])) + XCTAssertEqual(client.takeOutgoing(), Data([1])) + } + + func testReportsTheServersReasonWhenAuthenticationFails() { + let client = RFBClient(password: "wrong") + XCTAssertNoThrow(try client.receive(Data("RFB 003.008\n".utf8) + Data([1, 2]) + Data(repeating: 7, count: 16))) + let reason = Array("Authentication failed".utf8) + XCTAssertThrowsError(try client.receive(Data([0, 0, 0, 1, 0, 0, 0, UInt8(reason.count)] + reason))) { error in + XCTAssertEqual(error as? RFBError, .authenticationFailed("Authentication failed")) + } + } + + func testRefusesAPasswordServerWithoutAPassword() { + let client = RFBClient(password: nil) + XCTAssertThrowsError(try client.receive(Data("RFB 003.008\n".utf8) + Data([1, 2]))) { error in + XCTAssertEqual(error as? RFBError, .passwordRequired) + } + } + + func testSpeaks33WhereTheServerChoosesSecurity() throws { + let client = RFBClient(password: nil) + try client.receive(Data("RFB 003.003\n".utf8)) + XCTAssertEqual(client.takeOutgoing(), Data("RFB 003.003\n".utf8)) + try client.receive(Data([0, 0, 0, 1])) + XCTAssertEqual(client.takeOutgoing(), Data([1]), "None needs no result in 3.3; straight to ClientInit") + XCTAssertEqual(try client.receive(serverInit(width: 2, height: 2)), [.connected(width: 2, height: 2, name: "VM")]) + } + + func testRefusesAnUnknownProtocol() { + let client = RFBClient(password: nil) + XCTAssertThrowsError(try client.receive(Data("HTTP/1.1 200\n".utf8))) + } + + func testAppliesARawRectangleOnlyOnceItHasAllArrived() throws { + let client = try connected(width: 4, height: 3) + var update: [UInt8] = [0, 0] + u16(1) + u16(1) + u16(1) + u16(2) + u16(2) + s32(0) + update += [1, 2, 3, 0, 4, 5, 6, 0, 7, 8, 9, 0, 10, 11, 12, 0] + // Split mid-rectangle, the way WebSocket messages arrive. + XCTAssertEqual(try client.receive(Data(update.prefix(20))), []) + XCTAssertEqual(client.framebuffer, [UInt8](repeating: 0, count: 48), "a partial update draws nothing") + XCTAssertEqual(try client.receive(Data(update.dropFirst(20))), [.updated(resized: false)]) + + let row1 = Array(client.framebuffer[16 ..< 32]) + let row2 = Array(client.framebuffer[32 ..< 48]) + XCTAssertEqual(row1, [0, 0, 0, 0, 1, 2, 3, 0, 4, 5, 6, 0, 0, 0, 0, 0]) + XCTAssertEqual(row2, [0, 0, 0, 0, 7, 8, 9, 0, 10, 11, 12, 0, 0, 0, 0, 0]) + XCTAssertEqual(client.takeOutgoing(), Data([3, 1, 0, 0, 0, 0] + u16(4) + u16(3)), "asks for the next change") + } + + func testCopiesARectangleAndFollowsADesktopResize() throws { + let client = try connected(width: 2, height: 1) + try client.receive(Data([0, 0] + u16(1) + u16(0) + u16(0) + u16(1) + u16(1) + s32(0) + [9, 8, 7, 0])) + try client.receive(Data([0, 0] + u16(1) + u16(1) + u16(0) + u16(1) + u16(1) + s32(1) + u16(0) + u16(0))) + XCTAssertEqual(client.framebuffer, [9, 8, 7, 0, 9, 8, 7, 0]) + + let events = try client.receive(Data([0, 0] + u16(1) + u16(0) + u16(0) + u16(3) + u16(2) + s32(-223))) + XCTAssertEqual(events, [.updated(resized: true)]) + XCTAssertEqual(client.width, 3) + XCTAssertEqual(client.height, 2) + XCTAssertEqual(client.framebuffer.count, 3 * 2 * 4) + } + + func testRefusesADesktopTooLargeToAllocate() throws { + let client = RFBClient(password: nil) + try client.receive(Data("RFB 003.008\n".utf8) + Data([1, 1]) + Data([0, 0, 0, 0])) + XCTAssertThrowsError(try client.receive(serverInit(width: 65_000, height: 65_000))) + + let resized = try connected() + XCTAssertThrowsError(try resized.receive(Data([0, 0] + u16(1) + u16(0) + u16(0) + u16(60_000) + u16(60_000) + s32(-223)))) + } + + func testCopiesOverlappingRectanglesWithoutSmearing() throws { + let client = try connected(width: 3, height: 1) + try client.receive(Data([0, 0] + u16(1) + u16(0) + u16(0) + u16(3) + u16(1) + s32(0) + [1, 0, 0, 0, 2, 0, 0, 0, 3, 0, 0, 0])) + // Shift the first two pixels right by one: source and target overlap. + try client.receive(Data([0, 0] + u16(1) + u16(1) + u16(0) + u16(2) + u16(1) + s32(1) + u16(0) + u16(0))) + XCTAssertEqual(client.framebuffer.enumerated().filter { $0.offset % 4 == 0 }.map(\.element), [1, 1, 2]) + } + + func testRefusesAnEncodingItDidNotAskFor() throws { + let client = try connected() + XCTAssertThrowsError(try client.receive(Data([0, 0] + u16(1) + u16(0) + u16(0) + u16(1) + u16(1) + s32(7)))) { error in + XCTAssertEqual(error as? RFBError, .unsupportedEncoding(7)) + } + } + + func testReadsBellAndLatin1Clipboard() throws { + let client = try connected() + let events = try client.receive(Data([2, 3, 0, 0, 0, 0, 0, 0, 3, 0x63, 0x61, 0xE9])) + XCTAssertEqual(events, [.bell, .clipboard("caé")]) + } + + func testSendsPointerAndKeysClampedToTheDesktop() throws { + let client = try connected(width: 100, height: 50) + client.pointer(x: 150, y: -4, buttons: [.left]) + XCTAssertEqual(client.takeOutgoing(), Data([5, 1] + u16(99) + u16(0))) + client.tap(RFBKey.returnKey) + XCTAssertEqual(client.takeOutgoing(), Data([4, 1, 0, 0, 0, 0, 0xFF, 0x0D, 4, 0, 0, 0, 0, 0, 0xFF, 0x0D])) + } + + func testSendsNothingBeforeTheHandshakeFinishes() { + let client = RFBClient(password: nil) + client.pointer(x: 1, y: 1, buttons: []) + client.key(0x61, down: true) + client.requestUpdate() + XCTAssertTrue(client.takeOutgoing().isEmpty) + } + + func testMapsTypedCharactersToKeysyms() { + XCTAssertEqual(RFBKey.keysym(for: "a"), 0x61) + XCTAssertEqual(RFBKey.keysym(for: "é"), 0xE9) + XCTAssertEqual(RFBKey.keysym(for: "\n"), RFBKey.returnKey) + XCTAssertEqual(RFBKey.keysym(for: "世"), 0x0100_4E16) + XCTAssertNil(RFBKey.keysym(for: "👍🏽")) + } +} diff --git a/scripts/testing/fake-vnc-desktop.ts b/scripts/testing/fake-vnc-desktop.ts new file mode 100644 index 0000000000..0d8b087b18 --- /dev/null +++ b/scripts/testing/fake-vnc-desktop.ts @@ -0,0 +1,161 @@ +// An offline RFB 3.8 desktop with a real-sized picture and VNC +// authentication, for driving a client the way a person would: every pointer +// and key event is recorded, and each click paints a marker where it landed +// so the client's next update shows it. No host display, container daemon or +// network outside loopback is used. +import { createServer } from "node:http"; +import { EventEmitter, once } from "node:events"; +import { WebSocketServer, type WebSocket } from "ws"; + +export interface FakeVncDesktopOptions { + width?: number; + height?: number; + /** Paints the initial desktop as RGB triples, row-major. */ + paint?: (x: number, y: number) => [number, number, number]; +} + +export interface PointerEvent { x: number; y: number; buttons: number } + +export async function fakeVncDesktop(options: FakeVncDesktopOptions = {}) { + const width = options.width ?? 1280; + const height = options.height ?? 800; + const paint = options.paint ?? (() => [0x1a, 0x24, 0x36]); + // BGRX, little-endian 32-bit: what a client asking for true colour with + // red at 16, green at 8 and blue at 0 expects. + const framebuffer = Buffer.alloc(width * height * 4); + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const [r, g, b] = paint(x, y); + const at = (y * width + x) * 4; + framebuffer[at] = b; framebuffer[at + 1] = g; framebuffer[at + 2] = r; + } + } + const server = createServer((_req, res) => res.writeHead(404).end()); + const sockets = new WebSocketServer({ server, path: "/websockify" }); + const pointer: PointerEvent[] = []; + const keys: Array<{ keysym: number; down: boolean }> = []; + const authResponses: Buffer[] = []; + const events = new EventEmitter(); + let connections = 0; + + const rect = (x: number, y: number, w: number, h: number) => { + const data = Buffer.alloc(12 + w * h * 4); + data.writeUInt16BE(x, 0); data.writeUInt16BE(y, 2); data.writeUInt16BE(w, 4); data.writeUInt16BE(h, 6); + data.writeInt32BE(0, 8); + for (let row = 0; row < h; row++) { + framebuffer.copy(data, 12 + row * w * 4, ((y + row) * width + x) * 4, ((y + row) * width + x + w) * 4); + } + return data; + }; + const update = (socket: WebSocket, rects: Buffer[]) => { + const header = Buffer.alloc(4); + header.writeUInt16BE(rects.length, 2); + socket.send(Buffer.concat([header, ...rects])); + }; + + sockets.on("connection", socket => { + connections++; + let pending = Buffer.alloc(0); + let phase = 0; + let waiting = false; + let lastButtons = 0; + const dirty: Array<[number, number, number, number]> = []; + const flushDirty = () => { + if (!waiting || !dirty.length) return; + waiting = false; + update(socket, dirty.splice(0).map(([x, y, w, h]) => rect(x, y, w, h))); + }; + socket.send(Buffer.from("RFB 003.008\n")); + socket.on("message", raw => { + pending = Buffer.concat([pending, Buffer.from(raw as Buffer)]); + for (;;) { + let length: number; + if (phase === 0) length = 12; + else if (phase === 1) length = 1; + else if (phase === 2) length = 16; + else if (phase === 3) length = 1; + else { + if (!pending.length) return; + switch (pending[0]) { + case 0: length = 20; break; + case 2: if (pending.length < 4) return; length = 4 + pending.readUInt16BE(2) * 4; break; + case 3: length = 10; break; + case 4: length = 8; break; + case 5: length = 6; break; + case 6: if (pending.length < 8) return; length = 8 + pending.readUInt32BE(4); break; + default: socket.close(); return; + } + } + if (pending.length < length) return; + const message = pending.subarray(0, length); + pending = pending.subarray(length); + if (phase === 0) { + phase = 1; + socket.send(Buffer.from([1, 2])); // VNC authentication only + } else if (phase === 1) { + if (message[0] !== 2) { socket.close(); return; } + phase = 2; + socket.send(Buffer.alloc(16, 0x5a)); // the challenge + } else if (phase === 2) { + // The client's DES answer is recorded, not checked: the fixture + // proves the client authenticates, not that DES works. + authResponses.push(Buffer.from(message)); + phase = 3; + socket.send(Buffer.alloc(4)); // SecurityResult OK + } else if (phase === 3) { + phase = 4; + const name = Buffer.from("Isolated Local VM"); + const init = Buffer.alloc(24 + name.length); + init.writeUInt16BE(width, 0); init.writeUInt16BE(height, 2); + init[4] = 32; init[5] = 24; init[7] = 1; + init.writeUInt16BE(255, 8); init.writeUInt16BE(255, 10); init.writeUInt16BE(255, 12); + init[14] = 16; init[15] = 8; + init.writeUInt32BE(name.length, 20); name.copy(init, 24); + socket.send(init); + } else if (message[0] === 3) { + if (message[1] === 0) update(socket, [rect(0, 0, width, height)]); + else { waiting = true; flushDirty(); } + } else if (message[0] === 4) { + const event = { keysym: message.readUInt32BE(4), down: message[1] === 1 }; + keys.push(event); + events.emit("key", event); + } else if (message[0] === 5) { + const event = { buttons: message[1], x: message.readUInt16BE(2), y: message.readUInt16BE(4) }; + pointer.push(event); + events.emit("pointer", event); + // A left press paints a marker where it landed. + if (event.buttons & 1 && !(lastButtons & 1)) { + const size = 24; + const x0 = Math.max(0, Math.min(width - size, event.x - size / 2)); + const y0 = Math.max(0, Math.min(height - size, event.y - size / 2)); + for (let y = y0; y < y0 + size; y++) { + for (let x = x0; x < x0 + size; x++) { + const at = (y * width + x) * 4; + framebuffer[at] = 0x3f; framebuffer[at + 1] = 0x8c; framebuffer[at + 2] = 0xff; + } + } + dirty.push([x0, y0, size, size]); + flushDirty(); + } + lastButtons = event.buttons; + } + } + }); + }); + await new Promise(resolve => server.listen(0, "127.0.0.1", resolve)); + return { + port: (server.address() as { port: number }).port, + pointer, + keys, + authResponses, + connections: () => connections, + /** Typed characters, from key-down events in the printable range. */ + typed: () => keys.filter(k => k.down && k.keysym >= 0x20 && k.keysym <= 0xff).map(k => String.fromCharCode(k.keysym)).join(""), + nextPointer: () => once(events, "pointer", { signal: AbortSignal.timeout(10_000) }), + async close() { + for (const socket of sockets.clients) socket.terminate(); + await new Promise(resolve => sockets.close(() => resolve())); + await new Promise(resolve => server.close(() => resolve())); + }, + }; +} diff --git a/scripts/testing/fixtures/local-vm-desktop.png b/scripts/testing/fixtures/local-vm-desktop.png new file mode 100644 index 0000000000..0e635c2d19 Binary files /dev/null and b/scripts/testing/fixtures/local-vm-desktop.png differ diff --git a/scripts/testing/png.ts b/scripts/testing/png.ts new file mode 100644 index 0000000000..88c3b19f8c --- /dev/null +++ b/scripts/testing/png.ts @@ -0,0 +1,63 @@ +// Reads the 8-bit, non-interlaced PNGs our fixtures keep: truecolour with or +// without alpha. Enough to paint a captured desktop into a synthetic +// framebuffer without pulling in an image library. +import { inflateSync } from "node:zlib"; + +export interface DecodedPng { + width: number; + height: number; + /** RGB at (x, y); alpha is dropped. */ + pixel: (x: number, y: number) => [number, number, number]; +} + +export function decodePng(file: Buffer): DecodedPng { + if (!file.subarray(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]))) throw new Error("not a PNG"); + let width = 0, height = 0, channels = 0; + const data: Buffer[] = []; + for (let at = 8; at < file.length;) { + const length = file.readUInt32BE(at); + const type = file.toString("latin1", at + 4, at + 8); + const body = file.subarray(at + 8, at + 8 + length); + if (type === "IHDR") { + width = body.readUInt32BE(0); + height = body.readUInt32BE(4); + const [depth, colour, , , interlace] = [body[8], body[9], body[10], body[11], body[12]]; + channels = colour === 2 ? 3 : colour === 6 ? 4 : 0; + if (depth !== 8 || !channels || interlace) throw new Error("only 8-bit non-interlaced truecolour PNGs are supported"); + } else if (type === "IDAT") data.push(body); + at += 12 + length; + } + const raw = inflateSync(Buffer.concat(data)); + const stride = width * channels; + const pixels = Buffer.alloc(stride * height); + for (let y = 0, at = 0; y < height; y++) { + const filter = raw[at++]; + const row = pixels.subarray(y * stride, (y + 1) * stride); + const above = y ? pixels.subarray((y - 1) * stride, y * stride) : Buffer.alloc(stride); + raw.copy(row, 0, at, at + stride); + at += stride; + for (let i = 0; i < stride; i++) { + const left = i >= channels ? row[i - channels] : 0; + const up = above[i]; + const upLeft = i >= channels ? above[i - channels] : 0; + let predicted = 0; + if (filter === 1) predicted = left; + else if (filter === 2) predicted = up; + else if (filter === 3) predicted = (left + up) >> 1; + else if (filter === 4) { + const p = left + up - upLeft; + const pa = Math.abs(p - left), pb = Math.abs(p - up), pc = Math.abs(p - upLeft); + predicted = pa <= pb && pa <= pc ? left : pb <= pc ? up : upLeft; + } + row[i] = (row[i] + predicted) & 0xff; + } + } + return { + width, + height, + pixel: (x, y) => { + const at = y * stride + x * channels; + return [pixels[at], pixels[at + 1], pixels[at + 2]]; + }, + }; +} diff --git a/scripts/verify-ios-local-vm.ts b/scripts/verify-ios-local-vm.ts new file mode 100644 index 0000000000..c3060ea15a --- /dev/null +++ b/scripts/verify-ios-local-vm.ts @@ -0,0 +1,237 @@ +// Fake-engine server + synthetic Local VM + companion sidecar, in disposable +// homes, for checking the iOS computer view against a Local VM by hand. +// +// node --experimental-strip-types scripts/verify-ios-local-vm.ts +// +// Prints the sidecar address and a pairing code for the Simulator, then stays +// up until Ctrl-C. The synthetic `docker` answers only inspection and the two +// screenshot execs; each capture returns the captured desktop with another +// character typed at its prompt, so a refresh is visible on the phone. It never reaches a real container runtime, +// VM, or the user's OpenMausBot data. +import { spawn, type ChildProcess } from "node:child_process"; +import { mkdtempSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { deflateSync } from "node:zlib"; +import { launchVerificationServer, type VerificationServer } from "./control-omb.ts"; +import { fixtureApi } from "./testing/preview-fixture.ts"; +import { fakeVncDesktop } from "./testing/fake-vnc-desktop.ts"; +import { decodePng } from "./testing/png.ts"; +import { createServer as createHttpServer } from "node:http"; +import { BASE_IMAGE_DIGEST, CUA_DRIVER_VERSION, IMAGE, IMAGE_LAYER_VERSION } from "../server/container-computer.ts"; + +const root = fileURLToPath(new URL("..", import.meta.url)); + +/** The synthetic desktop: a captured Local VM session (an XFCE desktop with + * a terminal open), with `frame` characters typed at the prompt so each + * capture is distinct. The capture holds no account or network details. */ +const captured = decodePng(readFileSync(join(root, "scripts/testing/fixtures/local-vm-desktop.png"))); +function scene(frame: number): (x: number, y: number) => [number, number, number] { + // The prompt's block cursor, in terminal cells of 9 by 16 pixels. + const cursor = { x: 460, y: 153, w: 8, h: 16, step: 9 }; + const typed = frame % 4; + return (x, y) => { + if (y >= cursor.y && y < cursor.y + cursor.h && x >= cursor.x && x < cursor.x + cursor.step * (typed + 1)) { + const cell = Math.floor((x - cursor.x) / cursor.step); + const inGlyph = (x - cursor.x) % cursor.step < cursor.w; + if (cell === typed) return inGlyph ? [0xff, 0xff, 0xff] : [0, 0, 0]; + // Typed characters: a lower-case run in the terminal's text colour. + const ink = inGlyph && y >= cursor.y + 5 && y < cursor.y + 13 && (x - cursor.x) % cursor.step !== 3; + return ink ? [0xd0, 0xd0, 0xd0] : [0, 0, 0]; + } + return captured.pixel(x, y); + }; +} + +/** A flat RGB PNG of `scene(frame)`, so each capture is distinct. */ +function desktopPng(frame: number, width = 1280, height = 800): Buffer { + const pixel = scene(frame); + const raw = Buffer.alloc((width * 3 + 1) * height); + for (let y = 0; y < height; y++) { + const row = y * (width * 3 + 1); + raw[row] = 0; + for (let x = 0; x < width; x++) raw.set(pixel(x, y), row + 1 + x * 3); + } + const chunk = (type: string, data: Buffer) => { + const length = Buffer.alloc(4); + length.writeUInt32BE(data.length); + const body = Buffer.concat([Buffer.from(type), data]); + const crc = Buffer.alloc(4); + crc.writeUInt32BE(crc32(body)); + return Buffer.concat([length, body, crc]); + }; + const header = Buffer.alloc(13); + header.writeUInt32BE(width, 0); + header.writeUInt32BE(height, 4); + header[8] = 8; // bit depth + header[9] = 2; // truecolour + return Buffer.concat([ + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]), + chunk("IHDR", header), + chunk("IDAT", deflateSync(raw)), + chunk("IEND", Buffer.alloc(0)), + ]); +} + +function crc32(data: Buffer): number { + let crc = ~0; + for (const byte of data) { + crc ^= byte; + for (let bit = 0; bit < 8; bit++) crc = (crc >>> 1) ^ (0xedb88320 & -(crc & 1)); + } + return ~crc >>> 0; +} + +async function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = createServer(); + server.once("error", reject); + server.listen(0, "127.0.0.1", () => { + const { port } = server.address() as { port: number }; + server.close(() => resolve(port)); + }); + }); +} + +async function waitFor(url: string, child: ChildProcess): Promise { + for (let attempt = 0; attempt < 100; attempt++) { + if (child.exitCode !== null) throw new Error(`${url}: process exited (${child.exitCode})`); + try { + if ((await fetch(url)).ok) return; + } catch {} + await new Promise((resolve) => setTimeout(resolve, 200)); + } + throw new Error(`${url} did not come up`); +} + +const scratch = mkdtempSync(join(tmpdir(), "omb-ios-local-vm-")); +const bin = join(scratch, "bin"); +const frames = join(scratch, "frames"); +mkdirSync(bin); +mkdirSync(frames); +for (let frame = 0; frame < 4; frame++) { + writeFileSync(join(frames, `${frame}.b64`), desktopPng(frame).toString("base64")); +} +let fixture: VerificationServer | undefined; +let sidecar: ChildProcess | undefined; +// The VM's live desktop, behind VNC authentication like the real one. +const desktop = await fakeVncDesktop({ paint: scene(0) }); +// A signal during server startup aborts it; the launcher then stops its child +// and removes its own data directory before the launch promise settles. +const startup = new AbortController(); +let launching: Promise | undefined; +let stopping: Promise | undefined; +const stop = () => stopping ??= (async () => { + sidecar?.kill("SIGTERM"); + startup.abort(); + await launching?.catch(() => {}); + await desktop.close().catch(() => {}); + await fixture?.close().catch(() => {}); + rmSync(scratch, { recursive: true, force: true }); +})(); +process.once("SIGINT", () => void stop().then(() => process.exit(0))); +process.once("SIGTERM", () => void stop().then(() => process.exit(0))); + +try { + // Read-only inspection plus the two screenshot execs. Anything else fails, + // so no command can reach a real container runtime. + writeFileSync(join(bin, "docker"), `#!${process.execPath} +const fs = require('node:fs'); +let args = process.argv.slice(2); +if (args[0] === '-H') args = args.slice(2); +const labels = ${JSON.stringify({ "com.openmausbot.local-vm": "1", "com.openmausbot.cua-driver": CUA_DRIVER_VERSION, "com.openmausbot.cua-base": BASE_IMAGE_DIGEST, "com.openmausbot.image-layer": IMAGE_LAYER_VERSION, "com.openmausbot.workspace": "1" })}; +const imageId = 'sha256:' + 'a'.repeat(64); +const counter = ${JSON.stringify(join(scratch, "captures"))}; +let result; +if (args[0] === 'exec' && args.includes('base64')) { + const n = fs.existsSync(counter) ? Number(fs.readFileSync(counter, 'utf8')) : 0; + fs.writeFileSync(counter, String(n + 1)); + result = fs.readFileSync(${JSON.stringify(frames)} + '/' + (n % 4) + '.b64', 'utf8'); +} +else if (args[0] === 'exec') result = args.includes('--version') ? 'cua-driver ${CUA_DRIVER_VERSION}' + : args.includes('health_report') ? {schema_version:'1',overall:'ok',checks:[]} : {}; +else if (args[0] === 'info') result = 'fixture'; +else if (args[0] === 'image' && args[1] === 'inspect') result = [{Id:imageId,Config:{Labels:labels}}]; +else if (args[0] === 'inspect' && args[1] === 'openmausbot-computer') result = [{ + Config:{Image:${JSON.stringify(IMAGE)},Labels:labels,Env:['VNC_PW=fixture-password']}, + State:{Running:true},Image:imageId, + Mounts:[{Type:'bind',Source:require('node:path').join(process.env.OMB_DATA_DIR,'vm-home'),Destination:'/home/cua/workspace',RW:true}], + HostConfig:{PortBindings:{'6901/tcp':[{HostIp:'127.0.0.1',HostPort:'${desktop.port}'}]}, + Privileged:false,Memory:4294967296,MemorySwap:4294967296,NanoCpus:2000000000,PidsLimit:512, + CapDrop:['ALL'],CapAdd:['CAP_SETUID','CAP_SETGID'],IpcMode:'private',ShmSize:536870912, + CgroupnsMode:'private',SecurityOpt:[],RestartPolicy:{Name:'no',MaximumRetryCount:0}}, + NetworkSettings:{Ports:{'6901/tcp':[{HostIp:'127.0.0.1',HostPort:'${desktop.port}'}]}} +}]; +else if (args[0] === 'ps') result = ''; +else process.exit(1); +process.stdout.write(typeof result === 'string' ? result : JSON.stringify(result)); +`, { mode: 0o700 }); + + launching = launchVerificationServer(process.env, startup.signal, { + binDir: bin, host: "ssh://127.0.0.1:1", sshKey: join(scratch, "unused-key"), staticDir: join(root, "dist"), + }); + fixture = await launching; + const api = fixtureApi(fixture.info.url); + const { bot } = await api("POST", "/api/bots", { + name: "Vee", description: "Works on the Local VM.", modelSelection: { instanceId: "claude", model: "claude-sonnet-4-5" }, + }); + await api("PATCH", `/api/bots/${bot.id}`, { computer: "vm" }); + const still = await api("POST", `/api/bots/${bot.id}/local-computer/screenshot?threadId=${bot.threadId}`); + if (!String(still.image).startsWith("data:image/png;base64,")) throw new Error("fixture screenshot route did not return a PNG"); + + const harnessPort = new URL(fixture.info.url).port; + const companionPort = await freePort(); + const controlPort = await freePort(); + const companionHome = join(scratch, "companion-home"); + mkdirSync(companionHome); + sidecar = spawn(process.execPath, ["--experimental-strip-types", join(root, "companion", "src", "index.ts")], { + env: { + PATH: process.env.PATH, + HOME: companionHome, + USERPROFILE: companionHome, + OMB_PORT: harnessPort, + OMB_COMPANION_PORT: String(companionPort), + OMB_CONTROL_PORT: String(controlPort), + OMB_COMPANION_DIR: join(companionHome, "companion"), + }, + stdio: ["ignore", "inherit", "inherit"], + }); + await waitFor(`http://127.0.0.1:${controlPort}/state`, sidecar); + const { code } = await (await fetch(`http://127.0.0.1:${controlPort}/pairing`, { method: "POST" })).json() as { code: string }; + + console.log(JSON.stringify({ + harness: fixture.info.url, + companion: `127.0.0.1:${companionPort}`, + control: `http://127.0.0.1:${controlPort}`, + pairingCode: code, + bot: { id: bot.id, threadId: bot.threadId }, + log: fixture.info.logPath, + }, null, 2)); + console.log("Pair the Simulator with the address and code above. Allow computer view with:"); + console.log(` curl -X POST http://127.0.0.1:${controlPort}/devices//cloud-desktop`); + // What the phone has done to the desktop, and who holds the computer. + const eventsPort = await freePort(); + createHttpServer(async (_req, res) => { + const control = await api("GET", `/api/bots/${bot.id}/computer/control`).catch(() => null); + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ + connections: desktop.connections(), + authenticated: desktop.authResponses.length, + pointerEvents: desktop.pointer.length, + lastPointer: desktop.pointer.at(-1) ?? null, + clicks: desktop.pointer.filter((event, index, all) => event.buttons & 1 && !((all[index - 1]?.buttons ?? 0) & 1)).length, + typed: desktop.typed(), + keys: desktop.keys.length, + controlHeld: control?.held ?? null, + }, null, 2)); + }).listen(eventsPort, "127.0.0.1"); + console.log(`Desktop events and control state: http://127.0.0.1:${eventsPort}/`); + console.log("Ctrl-C stops everything and removes the temporary data."); + await new Promise(() => {}); +} catch (error) { + console.error(error); + await stop(); + process.exit(1); +} diff --git a/server/computer-control.test.ts b/server/computer-control.test.ts index 8e5d120204..404d657bb5 100644 --- a/server/computer-control.test.ts +++ b/server/computer-control.test.ts @@ -172,4 +172,15 @@ describe("computer control", () => { control.forget("ghost"); expect(changes).toEqual([]); }); + + it("reports whether a lease holds the computer without taking a free one", () => { + const control = new ComputerControl(); + expect(control.ownsLease("bot", "lease-aaaaaaaaaaaaaa")).toBe(false); + expect(control.snapshot("bot").held).toBe(false); + control.acquireLease("bot", "lease-aaaaaaaaaaaaaa"); + expect(control.ownsLease("bot", "lease-aaaaaaaaaaaaaa")).toBe(true); + expect(control.ownsLease("bot", "lease-bbbbbbbbbbbbbb")).toBe(false); + control.release("bot"); + expect(control.ownsLease("bot", "lease-aaaaaaaaaaaaaa")).toBe(false); + }); }); diff --git a/server/computer-control.ts b/server/computer-control.ts index d195fab161..2740650e4b 100644 --- a/server/computer-control.ts +++ b/server/computer-control.ts @@ -109,6 +109,13 @@ export class ComputerControl { return { snapshot: this.changed(botId), owned: true, acquired: true }; } + /** Whether this workspace lease is the one holding the wheel right now. + * Read-only: unlike `acquireLease`, a free computer is not taken. */ + ownsLease(botId: string, controlLeaseId: string): boolean { + const entry = this.entries.get(botId); + return entry?.heldSinceMs != null && entry.controlLeaseId === controlLeaseId; + } + /** The person hands the wheel back. Also settles any open help request — * the waiting bot resumes from this one state change. */ release(botId: string): ControlSnapshot { diff --git a/server/index.test.ts b/server/index.test.ts index 57acb8aeef..5df34bc299 100644 --- a/server/index.test.ts +++ b/server/index.test.ts @@ -8456,6 +8456,126 @@ describe("harness HTTP API", () => { } }); + it("hands out a Local VM's viewer only to the lease holding the computer, once the VM is ready", async () => { + const bot = (await api("POST", "/api/bots")).body.bot; + const lease = "phone-lease-0123456789"; + const join = (controlLeaseId = lease) => + api("POST", `/api/bots/${bot.id}/local-computer/join?controlLeaseId=${controlLeaseId}`, {}); + try { + expect((await api("POST", `/api/bots/${bot.id}/local-computer/join`, {})).status).toBe(400); + const unheld = await join(); + expect(unheld.status).toBe(409); + expect(unheld.body.error).toMatch(/take control/i); + + // Someone else holding the computer is not this lease holding it. + expect((await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "take" })).status).toBe(200); + expect((await join()).status).toBe(409); + expect((await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "check", controlLeaseId: lease })).body) + .toMatchObject({ held: true, owned: false }); + expect((await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "release" })).status).toBe(200); + + expect((await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "take", controlLeaseId: lease })).body) + .toMatchObject({ held: true, owned: true }); + expect((await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "check", controlLeaseId: lease })).body) + .toMatchObject({ held: true, owned: true }); + // This suite's container runtime is unavailable, so the VM is never ready. + const notReady = await join(); + expect(notReady.status).toBe(409); + expect(notReady.body.joinUrl).toBeUndefined(); + + await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "release", controlLeaseId: lease }); + // Checking never takes a free computer. + expect((await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "check", controlLeaseId: lease })).body) + .toMatchObject({ held: false, owned: false }); + expect((await api("POST", "/api/bots/no-such-bot/local-computer/join", {})).status).toBe(404); + } finally { + await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "release" }).catch(() => undefined); + await api("DELETE", `/api/bots/${bot.id}`).catch(() => undefined); + } + }); + + it("lets a phone paired directly with Full access ask for the Local VM desktop under its lease, and nobody else", async () => { + const bot = (await api("POST", "/api/bots")).body.bot; + const lease = "phone-lease-0123456789"; + const owner = await asPairedPerson("Owner's phone"); + const chatOnlyWindow = await api("POST", "/api/auth/pairing", { scopes: ["client"] }); + const chatOnlyPaired = await fetch(`${BASE}/api/auth/pair`, { + method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ code: chatOnlyWindow.body.code }), + }); + const chatOnlyToken = ((await chatOnlyPaired.json()) as any).token as string; + const asChatOnly = async (method: string, path: string, payload?: unknown) => { + const res = await fetch(`${BASE}${path}`, { + method, headers: { authorization: `Bearer ${chatOnlyToken}`, ...(payload ? { "content-type": "application/json" } : {}) }, + body: payload ? JSON.stringify(payload) : undefined, + }); + return { status: res.status, body: await res.json() as any }; + }; + const join = `/api/bots/${bot.id}/local-computer/join?controlLeaseId=${lease}`; + const proxy = `/api/desktop-viewer/local/shared?botId=${bot.id}&controlLeaseId=${lease}`; + try { + // Chat-only: no computer access at all, on every step of the way. + expect((await asChatOnly("POST", `/api/bots/${bot.id}/computer/control`, { action: "take", controlLeaseId: lease })).status).toBe(403); + expect((await asChatOnly("POST", join, {})).status).toBe(403); + expect((await asChatOnly("GET", proxy)).status).toBe(403); + expect((await asChatOnly("POST", `/api/bots/${bot.id}/computer/viewer-close`, {})).status).toBe(403); + + // Full access, but not holding the computer: no desktop. + const unheld = await owner.call("POST", join, {}); + expect(unheld.status).toBe(409); + expect(unheld.body.error).toMatch(/take control/i); + const unheldProxy = await owner.call("GET", proxy); + expect(unheldProxy.status).toBe(409); + expect(unheldProxy.body.error).toMatch(/take control/i); + + // Holding it: the join reaches the VM (which this suite cannot start) + // and the proxy is bound to this lease; neither hands out an address. + expect((await owner.call("POST", `/api/bots/${bot.id}/computer/control`, { action: "take", controlLeaseId: lease })).body) + .toMatchObject({ held: true, owned: true }); + const notReady = await owner.call("POST", join, {}); + expect(notReady.status).toBe(409); + expect(notReady.body.error).not.toMatch(/take control/i); + expect(notReady.body.joinUrl).toBeUndefined(); + expect(notReady.body.socketPath).toBeUndefined(); + const boundProxy = await owner.call("GET", proxy); + expect(boundProxy.status).toBe(409); + expect(boundProxy.body.error).not.toMatch(/take control/i); + // Another bot's lease, or a lease that is not this one, holds nothing here. + expect((await owner.call("GET", `/api/desktop-viewer/local/shared?botId=${bot.id}&controlLeaseId=phone-lease-9876543210`)).status).toBe(409); + + // Hand back: closing the viewer is allowed and idempotent, and the proxy refuses again. + expect((await owner.call("POST", `/api/bots/${bot.id}/computer/viewer-close`, {})).body).toEqual({ closed: false }); + await owner.call("POST", `/api/bots/${bot.id}/computer/control`, { action: "release", controlLeaseId: lease }); + expect((await owner.call("GET", proxy)).status).toBe(409); + } finally { + await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "release" }).catch(() => undefined); + await api("DELETE", `/api/bots/${bot.id}`).catch(() => undefined); + } + }); + + it("refuses Local VM phone joins in pool mode for direct and companion callers", async () => { + const { mode, maxInstances } = (await api("GET", "/api/config")).body.localVm; + const previous = { mode, maxInstances }; + const bot = (await api("POST", "/api/bots")).body.bot; + const owner = await asPairedPerson("Pool viewer phone"); + const lease = "phone-pool-lease-0123456789"; + try { + expect((await api("PATCH", "/api/config", { localVm: { ...previous, mode: "pool" } })).status).toBe(200); + expect((await owner.call("POST", `/api/bots/${bot.id}/computer/control`, { action: "take", controlLeaseId: lease })).body.owned).toBe(true); + const join = `/api/bots/${bot.id}/local-computer/join?controlLeaseId=${lease}`; + for (const response of [await owner.call("POST", join, {}), await api("POST", join, {})]) { + expect(response.status).toBe(409); + expect(response.body.error).toContain("pooled Local VMs"); + expect(response.body.joinUrl).toBeUndefined(); + expect(response.body.socketPath).toBeUndefined(); + expect(response.body.password).toBeUndefined(); + } + } finally { + await api("POST", `/api/bots/${bot.id}/computer/control`, { action: "release", controlLeaseId: lease }); + await api("PATCH", "/api/config", { localVm: previous }); + await api("DELETE", `/api/bots/${bot.id}`); + } + }); + it("keeps shared Local VM mode by default and resolves isolated targets per bot when enabled", async () => { const first = (await api("POST", "/api/bots")).body.bot; const second = (await api("POST", "/api/bots")).body.bot; diff --git a/server/index.ts b/server/index.ts index 75fa691cf9..4baa75c514 100644 --- a/server/index.ts +++ b/server/index.ts @@ -5345,6 +5345,17 @@ const desktopViewer = createDesktopViewer({ }); }, live: auth => auth.kind === "loopback" || sessions.isLive(auth.session.id), + // A phone paired with this server directly drives a bot's Local VM through + // this proxy under its control lease (see the local-computer join route). + // The lease must hold the bot's computer, and that computer must be the + // desktop being viewed: a lease on one bot opens no other bot's VM. + lease: (id, botId, controlLeaseId, threadId) => { + const bot = store.bot(botId); + if (!bot || viewerTargetId(localVmTargetForStatus(bot.id, threadId)) !== id) return; + const key = botComputerControlKey(bot); + const holds = () => computerControl.ownsLease(key, controlLeaseId); + return holds() ? holds : undefined; + }, }); function closeSessionStreams(sessionId: string): void { browserLive.closeForOwner(sessionId); @@ -21786,6 +21797,58 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { image: await containerComputerScreenshot(undefined, undefined, target), }); } + // The Local VM's live desktop for a phone, granted only to the control + // lease that holds this bot's computer right now. Two callers: + // + // The companion sidecar (a loopback caller) gets the VM's own noVNC + // address, VNC password included, relays it to a paired phone the owner + // has allowed computer access, keeps re-checking the lease with + // `action: "check"`, and cuts the relay when it no longer holds. + // + // A phone paired with this server directly (`openmausbot serve`, no + // sidecar) never sees that address. It gets this server's own + // authenticated desktop proxy, bound to its lease: the proxy re-checks + // the lease and the session every few seconds and closes the socket when + // either lapses. Reaching here at all took the admin scope, which a + // chat-only pairing (`openmausbot pair --client`) does not have, so + // computer access is the same explicit choice the owner's browser makes. + m = path.match(/^\/api\/bots\/([\w-]+)\/local-computer\/join$/); + if (m && method === "POST") { + if (!String(req.headers["content-type"] ?? "").toLowerCase().startsWith("application/json")) { + return json(res, 415, { error: "content-type must be application/json" }); + } + const bot = computerPreviewBot(m[1], url); + if (!bot) return json(res, 404, { error: "no such bot" }); + const threadId = url.searchParams.has("threadId") ? bot.threadId : undefined; + if (threadId && await computerPreviewSurface(bot, threadId) !== "vm") { + return json(res, 409, { error: "This conversation is not using the Local VM" }); + } + // A bot's human-control hold does not reserve a pool seat. Until a + // viewer can own that seat, another bot could drive the same desktop. + if (localVmMode(cfg) === "pool") { + return json(res, 409, { error: "Phone control is not available for pooled Local VMs. Use shared or per-bot mode in Settings → Computers." }); + } + const lease = controlLeaseIdSchema.safeParse(url.searchParams.get("controlLeaseId") ?? undefined); + if (!lease.success) return json(res, 400, { error: "controlLeaseId is required" }); + const controlBot = store.bot(bot.id); + if (!controlBot || !computerControl.ownsLease(botComputerControlKey(controlBot), lease.data)) { + return json(res, 409, { error: "Take control of this computer first" }); + } + const target = localVmTargetForStatus(bot.id, threadId); + const status = await containerComputerStatus(undefined, undefined, target); + if (!status.ready || !status.managed || status.container !== "running" || status.network !== "loopback" + || !status.viewer_url) { + return json(res, 409, { error: status.problem ?? "The Local VM is not ready" }); + } + localVmIdleFor(target).touch(); + res.setHeader("cache-control", "private, no-store"); + if (auth.kind === "loopback") return json(res, 200, { joinUrl: status.viewer_url }); + const socket = new URLSearchParams({ botId: bot.id, ...(threadId ? { threadId } : {}), controlLeaseId: lease.data }); + return json(res, 200, { + socketPath: `api/desktop-viewer/${viewerTargetId(target)}/websockify?${socket}`, + password: new URLSearchParams(new URL(status.viewer_url).hash.slice(1)).get("password"), + }); + } // identity handshake for the packaged app's port fallback: the forked // child proves it is OURS by echoing its pid (a stray dev server has @@ -23322,10 +23385,16 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { const result = computerControl.releaseLease(controlKey, controlLeaseId); return json(res, 200, { ...result.snapshot, released: result.released }); } + // Read-only: does this lease still hold the wheel? The companion + // sidecar asks this while it relays a phone's live Local VM desktop, + // and cuts the relay the moment the answer is no. + if (action === "check" && controlLeaseId) { + return json(res, 200, { ...computerControl.snapshot(controlKey), owned: computerControl.ownsLease(controlKey, controlLeaseId) }); + } if (action === "take") return json(res, 200, computerControl.take(controlKey)); if (action === "release") return json(res, 200, computerControl.release(controlKey)); if (action === "dismiss-help") return json(res, 200, computerControl.dismissHelp(controlKey)); - return json(res, 400, { error: "action must be take, release, or dismiss-help" }); + return json(res, 400, { error: "action must be take, release, check, or dismiss-help" }); } return json(res, 405, { error: "method not allowed" }); } @@ -23336,7 +23405,12 @@ const handleRequest = async (req: IncomingMessage, res: ServerResponse) => { if (!String(req.headers["content-type"] ?? "").toLowerCase().startsWith("application/json")) { return json(res, 415, { error: "content-type must be application/json" }); } - return json(res, 200, bot.cloudBackend === "vps" ? vps.closeVpsDesktopTunnel(bot.id) : { closed: false }); + // A directly paired phone's Local VM desktop runs through this server's + // own proxy under its lease on this bot; hand-back closes it here, ahead + // of the lease re-check. The session's other viewers are left alone. + const closedViewers = auth.kind === "session" ? desktopViewer.closeForOwner(auth.session.id, bot.id) : 0; + if (bot.cloudBackend === "vps") return json(res, 200, vps.closeVpsDesktopTunnel(bot.id)); + return json(res, 200, { closed: closedViewers > 0 }); } m = path.match(/^\/api\/bots\/([\w-]+)\/computer\/(provision|join|sleep|exec|screenshot|remove)$/); if (m && method === "POST") { diff --git a/server/request-auth.test.ts b/server/request-auth.test.ts index a4a3290a9b..84b2ec7b22 100644 --- a/server/request-auth.test.ts +++ b/server/request-auth.test.ts @@ -110,6 +110,7 @@ describe("scopes", () => { for (const [method, path] of [ ["POST", "/api/cli-test"], ["GET", "/api/cli-candidates"], ["GET", "/api/instances"], ["PATCH", "/api/instances/claude"], ["POST", "/api/bots/x/computer/exec"], ["POST", "/api/bots/x/computer/join"], ["POST", "/api/local-computer/run"], + ["POST", "/api/bots/x/local-computer/join"], ["POST", "/api/bots/x/local-computer/screenshot"], ["GET", "/api/computers/boxes"], ["POST", "/api/computers/boxes/bx_23456789/delete"], ["POST", "/api/webhooks"], ["POST", "/api/webhooks/w/rotate"], ["POST", "/api/bots/x/skills"], ["PATCH", "/api/bots/x/skills/s"], ["PATCH", "/api/bots/x/model"], ["PATCH", "/api/groups/g/setup"], ["POST", "/api/teams/import"], ["GET", "/api/teams/scout"], diff --git a/server/routes/desktop-viewer.test.ts b/server/routes/desktop-viewer.test.ts index 623efd166b..9fc8cc6580 100644 --- a/server/routes/desktop-viewer.test.ts +++ b/server/routes/desktop-viewer.test.ts @@ -32,6 +32,9 @@ let inspection: Promise | undefined; let now: number; let targetLookups: number; let handled: () => void; +/** Which (viewer target, bot, lease) triples hold the computer right now. */ +let leases: Set; +const leaseKey = (id: string, botId: string, lease: string) => `${id} ${botId} ${lease}`; let targets = [SHARED_LOCAL_VM_TARGET, perBotLocalVmTarget("test-bot"), poolLocalVmTarget(1)]; const peers = new Set(); const base = "/api/desktop-viewer/local/shared"; @@ -81,6 +84,7 @@ beforeEach(async () => { inspected = []; touched = []; statusOverrides = {}; inspection = undefined; seenPath = undefined; vpsTarget = undefined; targetLookups = 0; handled = () => {}; + leases = new Set(); targets = [SHARED_LOCAL_VM_TARGET, perBotLocalVmTarget("test-bot"), poolLocalVmTarget(1)]; desktop = createServer((_req, res) => res.writeHead(404).end()); desktop.on("upgrade", (req, socket) => { @@ -111,6 +115,11 @@ beforeEach(async () => { }); }, live: auth => auth.kind === "loopback" || sessions.isLive(auth.session.id), + lease: (id, botId, lease, threadId) => { + if (!leases.has(leaseKey(id, botId, lease)) || (threadId !== undefined && threadId !== "th-1")) return; + // Settled at open; afterwards only the lease is asked about. + return () => leases.has(leaseKey(id, botId, lease)); + }, }); sessions.onSessionRevoked(id => viewer.closeForOwner(id)); const handle = async (req: Parameters[0], res: Parameters[0]) => { @@ -322,3 +331,87 @@ it("uses the same authenticated proxy for VPS and releases only its own connecti await closed; expect(holds).toBe(0); }); + +// A phone paired with the server directly drives the Local VM through this +// proxy under its control lease, with no sidecar in between. +const lease = "phone-lease-0123456789"; +const bound = (query = `botId=test-bot&controlLeaseId=${lease}`) => `${base}/websockify?${query}`; + +it("refuses phone control of a pool seat even with a valid bot lease", async () => { + const target = viewerTargetId(poolLocalVmTarget(1)); + leases.add(leaseKey(target, "test-bot", lease)); + const query = `botId=test-bot&threadId=th-1&controlLeaseId=${lease}`; + for (const response of [ + await open(`/api/desktop-viewer/${target}/websockify?${query}`), + await get(`/api/desktop-viewer/${target}?${query}`), + ]) { + expect(response.status).toBe(409); + expect(JSON.stringify(response.body)).toContain("pooled Local VMs"); + } + expect(inspected).toEqual([]); +}); + +it("opens a lease-bound viewer only while that lease holds the bot's computer", async () => { + const refused = await open(bound()); + expect(refused.status).toBe(409); + expect(refused.body).toContain("Take control"); + // Nothing was inspected for a caller that does not hold the computer. + expect(inspected).toEqual([]); + + leases.add(leaseKey("local/shared", "test-bot", lease)); + const answer = await open(bound(), { authorization: `Bearer ${admin.token}`, "sec-websocket-protocol": "binary" }); + expect(answer.status).toBe(101); + const socket = answer.socket!; + const echo = new Promise(resolve => socket.once("data", resolve)); + const bytes = Buffer.from("RFB 003.008\n"); + socket.write(bytes); + expect(await echo).toEqual(bytes); + // The lease names travel no further than this proxy. + expect(seenPath).toBe("/websockify"); + expect(seenHeaders.authorization).toBeUndefined(); + + // Hand back: the lease stops holding, and the desktop closes at the next check. + const closed = new Promise(resolve => socket.once("close", () => resolve())); + leases.clear(); + await closed; + expect((await open(bound())).status).toBe(409); +}, 10_000); + +it("binds a lease to one bot's desktop and refuses malformed or half-given lease names", async () => { + leases.add(leaseKey("local/shared", "test-bot", lease)); + // The same lease, asked against another bot or another desktop, holds nothing. + expect((await open(bound(`botId=other-bot&controlLeaseId=${lease}`))).status).toBe(409); + expect((await open(`/api/desktop-viewer/${viewerTargetId(targets[1])}/websockify?botId=test-bot&controlLeaseId=${lease}`)).status).toBe(409); + for (const query of ["botId=test-bot", `controlLeaseId=${lease}`, "botId=test-bot&controlLeaseId=short", `botId=bad%20bot&controlLeaseId=${lease}`, + "threadId=th-1", `botId=test-bot&threadId=bad%20thread&controlLeaseId=${lease}`]) { + expect((await open(bound(query))).status).toBe(400); + } + // The conversation named at the join picks the seat; another conversation holds nothing here. + expect((await open(bound(`botId=test-bot&threadId=th-1&controlLeaseId=${lease}`))).status).toBe(101); + expect((await open(bound(`botId=test-bot&threadId=th-2&controlLeaseId=${lease}`))).status).toBe(409); + // The password read goes through the same binding. + expect((await get(`${base}?botId=test-bot&controlLeaseId=${lease}`)).body).toEqual({ password: "fixture-secret" }); + leases.clear(); + expect((await get(`${base}?botId=test-bot&controlLeaseId=${lease}`)).status).toBe(409); +}); + +it("keeps the session and scope checks ahead of the lease, and closes a session's viewers per bot or all at once", async () => { + leases.add(leaseKey("local/shared", "test-bot", lease)); + leases.add(leaseKey("local/shared", "other-bot", lease)); + expect((await open(bound(), { cookie: `test_session=${member.token}` })).status).toBe(403); + expect((await open(bound(), { cookie: "test_session=revoked" })).status).toBe(401); + const mine = (await open(bound())).socket!; + const other = (await open(bound(`botId=other-bot&controlLeaseId=${lease}`))).socket!; + const browser = (await open()).socket!; + // Hand-back on one bot: only that bot's lease-bound viewer closes. + const closedMine = once(mine, "close"); + expect(viewer.closeForOwner("nobody", "test-bot")).toBe(0); + expect(viewer.closeForOwner(admin.session.id, "test-bot")).toBe(1); + await closedMine; + expect(other.destroyed).toBe(false); + expect(browser.destroyed).toBe(false); + // Sign-out: everything the session had open. + const rest = Promise.all([other, browser].map(socket => once(socket, "close"))); + expect(viewer.closeForOwner(admin.session.id)).toBe(2); + await rest; +}); diff --git a/server/routes/desktop-viewer.ts b/server/routes/desktop-viewer.ts index 05c163e058..1f4cac14f3 100644 --- a/server/routes/desktop-viewer.ts +++ b/server/routes/desktop-viewer.ts @@ -12,6 +12,8 @@ const ROUTE = /^\/api\/desktop-viewer\/(local\/(?:shared|bot-[a-f0-9]{64}|pool-\ const HANDSHAKE_MS = 10_000; const RECHECK_MS = 5_000; const WS_GUID = "258EAFA5-E914-47DA-95CA-C5AB0DC85B11"; +const BOT_ID = /^[\w-]+$/; +const CONTROL_LEASE = /^[A-Za-z0-9_-]{16,120}$/; /** Providers resolve a managed loopback endpoint, never a browser-supplied URL. */ export interface DesktopConnection { @@ -32,11 +34,17 @@ export function desktopViewerUrl(target: string, threadId?: string): string { return `/desktop-viewer#${params}`; } -interface Upgrade { socket: Socket; head: Buffer; release: () => void; close: () => void; owner?: string } +interface Upgrade { socket: Socket; head: Buffer; release: () => void; close: () => void; owner?: string; botId?: string } export function createDesktopViewer(deps: { target: (id: string) => DesktopTarget | undefined; live: (auth: RequestAuth) => boolean; + /** Bind a viewer to a control lease: a phone driving the Local VM directly, + * without the companion sidecar. Answers nothing unless `controlLeaseId` + * holds `botId`'s computer right now and the target is that computer; + * otherwise a probe that says whether the lease still holds. Pool targets + * are refused before this callback because a bot hold cannot reserve a seat. */ + lease?: (id: string, botId: string, controlLeaseId: string, threadId?: string) => (() => boolean) | undefined; }) { const upgrades = new Map(); let stopped = false; @@ -77,7 +85,7 @@ export function createDesktopViewer(deps: { }); } - const route: RouteHandler = async ({ req, res, path, method, auth, json }) => { + const route: RouteHandler = async ({ req, res, url, path, method, auth, json }) => { const match = ROUTE.exec(path); if (!match) return PASS; res.setHeader("cache-control", "private, no-store"); @@ -85,10 +93,34 @@ export function createDesktopViewer(deps: { // Also enforce at this boundary; the central gate defaults these paths // to admin, like the existing Local VM status and control endpoints. if (!auth.scopes.includes("admin") || !isSameOrigin(req)) return json(res, 403, { error: "forbidden" }); + // A viewer bound to a control lease: both names or neither, well formed, + // and holding before anything is inspected. Checked again with every + // liveness pass, so handing back closes the desktop within seconds. The + // conversation, when named, picks the VM seat the join picked. + const botId = url.searchParams.get("botId"); + const controlLeaseId = url.searchParams.get("controlLeaseId"); + const threadId = url.searchParams.get("threadId") ?? undefined; + if ((botId === null) !== (controlLeaseId === null)) { + return json(res, 400, { error: "botId and controlLeaseId go together" }); + } + if (botId !== null && (!BOT_ID.test(botId) || !CONTROL_LEASE.test(controlLeaseId!) || !deps.lease + || (threadId !== undefined && !BOT_ID.test(threadId)))) { + return json(res, 400, { error: "botId, threadId or controlLeaseId is not valid" }); + } + if (botId === null && threadId !== undefined) return json(res, 400, { error: "threadId needs botId and controlLeaseId" }); + // Do not let a saved or constructed socket URL bypass the join refusal: + // a bot's control lease does not exclude other users of a pooled seat. + if (botId !== null && match[1].startsWith("local/pool-")) { + return json(res, 409, { error: "Phone control is not available for pooled Local VMs. Use shared or per-bot mode in Settings → Computers." }); + } + const bound = botId === null ? undefined : deps.lease!(match[1], botId, controlLeaseId!, threadId); + if (botId !== null && !bound) return json(res, 409, { error: "Take control of this computer first" }); + const holds = () => bound?.() ?? true; const target = deps.target(match[1]); if (!target) return json(res, 404, { error: "Desktop not found" }); const upgrade = upgrades.get(req); if (upgrade && auth.kind === "session") upgrade.owner = auth.session.id; + if (upgrade && botId !== null) upgrade.botId = botId; let connection: DesktopConnection; try { connection = await target.resolve(); } catch (error) { @@ -100,6 +132,7 @@ export function createDesktopViewer(deps: { if (res.destroyed || upgrade?.socket.destroyed) return; const live = () => deps.live(auth) && deps.target(match[1])?.key === target.key && (connection.live?.() ?? true); if (!live()) return json(res, 401, { error: "Viewer access expired" }); + if (!holds()) return json(res, 409, { error: "Take control of this computer first" }); if (!Number.isInteger(connection.port) || connection.port < 1 || connection.port > 65535) { return json(res, 409, { error: "The desktop viewer is not available." }); } @@ -144,7 +177,7 @@ export function createDesktopViewer(deps: { upstream.once("response", (answer) => { answer.resume(); fail(); }); upstream.once("upgrade", (answer, remote, remoteHead) => { const accept = createHash("sha1").update(key + WS_GUID).digest("base64"); - if (!live() || socket.destroyed || answer.headers["sec-websocket-accept"] !== accept + if (!live() || !holds() || socket.destroyed || answer.headers["sec-websocket-accept"] !== accept || answer.headers.upgrade?.toLowerCase() !== "websocket") { remote.destroy(); fail(); return; } @@ -161,7 +194,7 @@ export function createDesktopViewer(deps: { socket.pipe(remote).pipe(socket); connection.touch?.(); recheck = setInterval(() => { - if (!live()) upgrade.close(); + if (!live() || !holds()) upgrade.close(); else connection.touch?.(); }, RECHECK_MS); recheck.unref(); @@ -171,7 +204,17 @@ export function createDesktopViewer(deps: { return { route, attach, - closeForOwner: (owner: string) => { for (const upgrade of upgrades.values()) if (upgrade.owner === owner) upgrade.close(); }, + /** Close a session's viewers: all of them (sign-out, revocation), or only + * those it opened under a lease on one bot (that bot's hand-back). */ + closeForOwner: (owner: string, botId?: string) => { + let closed = 0; + for (const upgrade of upgrades.values()) { + if (upgrade.owner !== owner || (botId !== undefined && upgrade.botId !== botId)) continue; + upgrade.close(); + closed++; + } + return closed; + }, closeAll: () => { stopped = true; for (const upgrade of upgrades.values()) upgrade.close();