Add agent whiteboard: per-agent tab, MCP drawing tools, live SSE sync

CLAUDE.md

@@ -144,6 +144,7 @@ Before marking any task as done, run the following checks and fix any failures:
 - Treat `127.0.0.1:6767` as production by default; do not stop or kill the existing production server for ad-hoc testing.
 - When backend changes need local validation, use `repo_dev_up` and point validation tooling to the printed URL.
 - Only operate on production (`:6767`) when explicitly requested by the user.
+- Scoped MCP routes (`/api/mcp/:agentId`, `/api/mcp/jobs/:runId/:agentId`) 403 on the server bearer token — they accept only agent-scoped HMAC tokens or no token. Tests and local tooling must call them with no Authorization header.
 
 ## Development Database
 

apps/server/package.json

@@ -35,6 +35,7 @@
     "@types/pg": "^8.18.0",
     "@types/ws": "^8.18.1",
     "@vitest/coverage-v8": "4.1.2",
+    "tsx": "^4.23.0",
     "vite": "^6.0.0",
     "vitest": "^4.0.18"
   }

apps/server/src/db/migrations/0029_whiteboards.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS whiteboards (
+  agent_id TEXT PRIMARY KEY REFERENCES agents(id) ON DELETE CASCADE,
+  scene JSONB NOT NULL DEFAULT '{"elements": []}'::jsonb,
+  version BIGINT NOT NULL DEFAULT 1,
+  updated_by TEXT NOT NULL DEFAULT 'user',
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

apps/server/src/routes/mcp.ts

@@ -49,6 +49,8 @@ type McpRouteDeps = {
   mcpRenameSession: unknown;
   mcpShareMedia: unknown;
   mcpListMedia: unknown;
+  mcpGetWhiteboard: unknown;
+  mcpUpdateWhiteboard: unknown;
   mcpSubmitFeedback: unknown;
   mcpListPersonas: unknown;
   mcpLaunchPersona: unknown;
@@ -289,6 +291,8 @@ export async function registerMcpRoutes(
       renameSession: deps.mcpRenameSession,
       shareMedia: deps.mcpShareMedia,
       listMedia: deps.mcpListMedia,
+      getWhiteboard: deps.mcpGetWhiteboard,
+      updateWhiteboard: deps.mcpUpdateWhiteboard,
       submitFeedback: deps.mcpSubmitFeedback,
       listPersonas: deps.mcpListPersonas,
       launchPersona: deps.mcpLaunchPersona,

apps/server/src/routes/whiteboard.ts

@@ -0,0 +1,146 @@
+import path from "node:path";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+
+import type { FastifyInstance } from "fastify";
+import type { Pool } from "pg";
+
+import type { AgentManager } from "../agents/manager.js";
+import { resolveMediaDir } from "../shared/media.js";
+import {
+  EMPTY_SCENE,
+  isValidScene,
+  loadWhiteboard,
+  MAX_ELEMENTS,
+  saveWhiteboard,
+  WHITEBOARD_SNAPSHOT_FILENAME,
+} from "../shared/whiteboard-store.js";
+
+// Fastify's default JSON bodyLimit is 1 MB; real boards blow past it.
+const SCENE_BODY_LIMIT = 8 * 1024 * 1024;
+
+type WhiteboardRouteDeps = {
+  pool: Pool;
+  mediaRoot: string;
+  agentManager: AgentManager;
+  publishUiEvent: (event: unknown) => void;
+};
+
+export async function registerWhiteboardRoutes(
+  app: FastifyInstance,
+  deps: WhiteboardRouteDeps
+): Promise<void> {
+  app.get("/api/v1/agents/:id/whiteboard", async (request, reply) => {
+    const params = request.params as { id?: string };
+    const id = params.id ?? "";
+    const agent = await deps.agentManager.getAgent(id);
+    if (!agent) {
+      return reply.code(404).send({ error: "Agent not found." });
+    }
+
+    const row = await loadWhiteboard(deps.pool, id);
+    if (!row) {
+      return { scene: EMPTY_SCENE, version: 0, updatedAt: null };
+    }
+    return {
+      scene: row.scene,
+      version: Number(row.version),
+      updatedAt: row.updated_at.toISOString(),
+    };
+  });
+
+  app.put(
+    "/api/v1/agents/:id/whiteboard",
+    { bodyLimit: SCENE_BODY_LIMIT },
+    async (request, reply) => {
+      const params = request.params as { id?: string };
+      const id = params.id ?? "";
+      const agent = await deps.agentManager.getAgent(id);
+      if (!agent) {
+        return reply.code(404).send({ error: "Agent not found." });
+      }
+
+      const body = request.body as
+        | { scene?: unknown; baseVersion?: unknown }
+        | undefined;
+      if (!isValidScene(body?.scene)) {
+        return reply.code(400).send({
+          error: `scene must be an object with an elements array (max ${MAX_ELEMENTS}).`,
+        });
+      }
+      const baseVersion =
+        typeof body?.baseVersion === "number" &&
+        Number.isInteger(body.baseVersion) &&
+        body.baseVersion >= 0
+          ? body.baseVersion
+          : null;
+      if (baseVersion === null) {
+        return reply
+          .code(400)
+          .send({ error: "baseVersion must be a non-negative integer." });
+      }
+
+      const saved = await saveWhiteboard(
+        deps.pool,
+        id,
+        body.scene,
+        baseVersion,
+        "user"
+      );
+      if (!saved) {
+        const current = await loadWhiteboard(deps.pool, id);
+        return reply.code(409).send({
+          error: "Whiteboard was modified by someone else.",
+          scene: current?.scene ?? EMPTY_SCENE,
+          version: current ? Number(current.version) : 0,
+        });
+      }
+
+      deps.publishUiEvent({
+        type: "whiteboard.changed",
+        agentId: id,
+        version: saved.version,
+        source: "user",
+      });
+      return { ok: true, version: saved.version };
+    }
+  );
+
+  app.post("/api/v1/agents/:id/whiteboard/snapshot", async (request, reply) => {
+    const params = request.params as { id?: string };
+    const id = params.id ?? "";
+    const agent = await deps.agentManager.getAgent(id);
+    if (!agent) {
+      return reply.code(404).send({ error: "Agent not found." });
+    }
+
+    const data = await request.file();
+    if (!data || data.mimetype !== "image/png") {
+      return reply.code(400).send({ error: "A PNG file field is required." });
+    }
+
+    const mediaDir = resolveMediaDir(id, agent.mediaDir, deps.mediaRoot);
+    await mkdir(mediaDir, { recursive: true });
+    const buffer = await data.toBuffer();
+    await writeFile(path.join(mediaDir, WHITEBOARD_SNAPSHOT_FILENAME), buffer);
+    return { ok: true, sizeBytes: buffer.length };
+  });
+
+  // Clearing the board must also clear the snapshot, or whiteboard_get
+  // keeps pointing agents at a rendering of the erased drawing.
+  app.delete(
+    "/api/v1/agents/:id/whiteboard/snapshot",
+    async (request, reply) => {
+      const params = request.params as { id?: string };
+      const id = params.id ?? "";
+      const agent = await deps.agentManager.getAgent(id);
+      if (!agent) {
+        return reply.code(404).send({ error: "Agent not found." });
+      }
+      const mediaDir = resolveMediaDir(id, agent.mediaDir, deps.mediaRoot);
+      await rm(path.join(mediaDir, WHITEBOARD_SNAPSHOT_FILENAME), {
+        force: true,
+      });
+      return { ok: true };
+    }
+  );
+}

apps/server/src/server.ts

@@ -105,6 +105,7 @@ import { registerFeedbackRoutes } from "./routes/feedback.js";
 import { registerJobRoutes } from "./routes/jobs.js";
 import { registerTemplateRoutes } from "./routes/templates.js";
 import { registerMediaRoutes } from "./routes/media.js";
+import { registerWhiteboardRoutes } from "./routes/whiteboard.js";
 import { registerMcpRoutes } from "./routes/mcp.js";
 import { registerPersonaReviewRoutes } from "./routes/persona-reviews.js";
 import { registerPersonalityRoutes } from "./routes/personalities.js";
@@ -475,6 +476,8 @@ async function registerRoutes() {
     mcpRenameSession: mcpHandlers.renameSession,
     mcpShareMedia: mcpHandlers.shareMedia,
     mcpListMedia: mcpHandlers.listMedia,
+    mcpGetWhiteboard: mcpHandlers.getWhiteboard,
+    mcpUpdateWhiteboard: mcpHandlers.updateWhiteboard,
     mcpSubmitFeedback: mcpHandlers.submitFeedback,
     mcpListPersonas: mcpHandlers.listPersonas,
     mcpLaunchPersona: mcpHandlers.launchPersona,
@@ -577,6 +580,13 @@ async function registerRoutes() {
     publishUiEvent: (event) => uiEventBroker.publish(event as UiEvent),
   });
 
+  await registerWhiteboardRoutes(app, {
+    pool,
+    mediaRoot: config.mediaRoot,
+    agentManager,
+    publishUiEvent: (event) => uiEventBroker.publish(event as UiEvent),
+  });
+
   await registerAgentRoutes(app, {
     pool,
     appLog: app.log,

apps/server/src/server/mcp-handlers.ts

@@ -28,6 +28,7 @@ import { resolveHeadSha } from "../shared/git/worktree.js";
 import { isMediaFile, isTextFile, resolveMediaDir } from "../shared/media.js";
 import type { PublishUiEvent, SendAgentPrompt } from "./mcp-handler-types.js";
 import { createReviewHandlers } from "./mcp-review-handlers.js";
+import { createWhiteboardHandlers } from "./mcp-whiteboard-handlers.js";
 
 const AGENT_LATEST_EVENT_TYPES = [
   "working",
@@ -151,8 +152,16 @@ export function createMcpHandlers(deps: CreateMcpHandlersDeps) {
     sendAgentPrompt,
   });
 
+  const whiteboardHandlers = createWhiteboardHandlers({
+    pool,
+    mediaRoot,
+    agentManager,
+    publishUiEvent,
+  });
+
   return {
     ...reviewHandlers,
+    ...whiteboardHandlers,
 
     async upsertEvent(
       agentId: string,

apps/server/src/server/mcp-whiteboard-handlers.ts

@@ -0,0 +1,120 @@
+import path from "node:path";
+import { stat } from "node:fs/promises";
+
+import type { Pool } from "pg";
+
+import type { AgentManager } from "../agents/manager.js";
+import { resolveMediaDir } from "../shared/media.js";
+import {
+  simplifyElements,
+  type WhiteboardGetResult,
+  type WhiteboardUpdateResult,
+} from "../shared/whiteboard.js";
+import {
+  loadWhiteboard,
+  MAX_ELEMENTS,
+  saveWhiteboard,
+  WHITEBOARD_SNAPSHOT_FILENAME,
+} from "../shared/whiteboard-store.js";
+import {
+  applyWhiteboardOps,
+  type WhiteboardOp,
+} from "../shared/whiteboard-builder.js";
+import type { PublishUiEvent } from "./mcp-handler-types.js";
+
+type CreateWhiteboardHandlersDeps = {
+  pool: Pool;
+  mediaRoot: string;
+  agentManager: AgentManager;
+  publishUiEvent: PublishUiEvent;
+};
+
+export function createWhiteboardHandlers(deps: CreateWhiteboardHandlersDeps) {
+  const { pool, mediaRoot, agentManager, publishUiEvent } = deps;
+
+  return {
+    async getWhiteboard(agentId: string): Promise<WhiteboardGetResult> {
+      const agent = await agentManager.getAgent(agentId);
+      if (!agent) throw new Error("Agent not found.");
+
+      const row = await loadWhiteboard(pool, agentId);
+      const snapshotFile = path.join(
+        resolveMediaDir(agentId, agent.mediaDir, mediaRoot),
+        WHITEBOARD_SNAPSHOT_FILENAME
+      );
+      const snapshotStat = await stat(snapshotFile).catch(() => null);
+      const snapshotPath = snapshotStat?.isFile() ? snapshotFile : null;
+      return {
+        elements: row ? simplifyElements(row.scene.elements) : [],
+        version: row ? Number(row.version) : 0,
+        updatedAt: row ? row.updated_at.toISOString() : null,
+        updatedBy: row ? row.updated_by : null,
+        snapshotPath,
+        // The PNG is exported by a connected browser; agent-side edits (or a
+        // closed tab) leave it depicting an older scene.
+        snapshotStale:
+          snapshotPath !== null &&
+          row !== null &&
+          snapshotStat !== null &&
+          row.updated_at.getTime() > snapshotStat.mtime.getTime(),
+      };
+    },
+
+    async updateWhiteboard(
+      agentId: string,
+      ops: WhiteboardOp[]
+    ): Promise<WhiteboardUpdateResult> {
+      const agent = await agentManager.getAgent(agentId);
+      if (!agent) throw new Error("Agent not found.");
+
+      // The user's editor saves concurrently; retry op application on top of
+      // the fresh scene when the optimistic version check loses the race.
+      for (let attempt = 0; attempt < 3; attempt++) {
+        const row = await loadWhiteboard(pool, agentId);
+        const baseVersion = row ? Number(row.version) : 0;
+        const existing = row ? row.scene.elements : [];
+        const result = applyWhiteboardOps(existing, ops);
+
+        if (result.errors.length === ops.length) {
+          // Every op failed — nothing changed, report without saving.
+          return {
+            version: baseVersion,
+            created: [],
+            errors: result.errors,
+            warnings: result.warnings,
+            elements: simplifyElements(existing),
+          };
+        }
+        if (result.elements.length > MAX_ELEMENTS) {
+          throw new Error(`Board is full (max ${MAX_ELEMENTS} elements).`);
+        }
+
+        const saved = await saveWhiteboard(
+          pool,
+          agentId,
+          { elements: result.elements },
+          baseVersion,
+          "agent"
+        );
+        if (saved) {
+          publishUiEvent({
+            type: "whiteboard.changed",
+            agentId,
+            version: saved.version,
+            source: "agent",
+          });
+          return {
+            version: saved.version,
+            created: result.created,
+            errors: result.errors,
+            warnings: result.warnings,
+            elements: simplifyElements(result.elements),
+          };
+        }
+      }
+      throw new Error(
+        "Whiteboard is being edited concurrently; try again in a moment."
+      );
+    },
+  };
+}

apps/server/src/server/ui-events.ts

@@ -18,6 +18,12 @@ export type UiEvent =
     }
   | { type: "agent.deleted"; agentId: string }
   | { type: "media.changed"; agentId: string }
+  | {
+      type: "whiteboard.changed";
+      agentId: string;
+      version: number;
+      source: "user" | "agent";
+    }
   | { type: "media.seen"; agentId: string; keys: string[] }
   | { type: "stream.started"; agentId: string }
   | { type: "stream.stopped"; agentId: string }