Add per-agent whiteboard with Excalidraw thin relay architecture

apps/server/src/db/migrations/0030_whiteboards.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS whiteboards (
+  agent_id   TEXT PRIMARY KEY,
+  scene      JSONB NOT NULL DEFAULT '{"elements":[]}',
+  version    INTEGER NOT NULL DEFAULT 1,
+  updated_by TEXT NOT NULL DEFAULT 'user',
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

apps/server/src/routes/mcp.ts

@@ -49,6 +49,11 @@ type McpRouteDeps = {
   mcpRenameSession: unknown;
   mcpShareMedia: unknown;
   mcpListMedia: unknown;
+  mcpGetWhiteboard: unknown;
+  mcpUpdateWhiteboard: unknown;
+  mcpClearWhiteboard: unknown;
+  mcpLibraryList: unknown;
+  mcpInsertLibraryItem: unknown;
   mcpSubmitFeedback: unknown;
   mcpListPersonas: unknown;
   mcpLaunchPersona: unknown;
@@ -207,6 +212,11 @@ export async function registerMcpRoutes(
       renameSession: deps.mcpRenameSession,
       shareMedia: deps.mcpShareMedia,
       listMedia: deps.mcpListMedia,
+      getWhiteboard: deps.mcpGetWhiteboard,
+      updateWhiteboard: deps.mcpUpdateWhiteboard,
+      clearWhiteboard: deps.mcpClearWhiteboard,
+      libraryList: deps.mcpLibraryList,
+      insertLibraryItem: deps.mcpInsertLibraryItem,
       submitFeedback: deps.mcpSubmitFeedback,
       upsertPin: deps.mcpUpsertPin,
       deletePin: deps.mcpDeletePin,
@@ -295,6 +305,11 @@ export async function registerMcpRoutes(
       renameSession: deps.mcpRenameSession,
       shareMedia: deps.mcpShareMedia,
       listMedia: deps.mcpListMedia,
+      getWhiteboard: deps.mcpGetWhiteboard,
+      updateWhiteboard: deps.mcpUpdateWhiteboard,
+      clearWhiteboard: deps.mcpClearWhiteboard,
+      libraryList: deps.mcpLibraryList,
+      insertLibraryItem: deps.mcpInsertLibraryItem,
       submitFeedback: deps.mcpSubmitFeedback,
       listPersonas: deps.mcpListPersonas,
       launchPersona: deps.mcpLaunchPersona,

apps/server/src/routes/whiteboard.ts

@@ -0,0 +1,143 @@
+import path from "node:path";
+import { mkdir, rm, writeFile } from "node:fs/promises";
+
+import type { FastifyInstance } from "fastify";
+import type { Pool } from "pg";
+
+import type { AgentManager } from "../agents/manager.js";
+import { resolveMediaDir } from "../shared/media.js";
+import {
+  EMPTY_SCENE,
+  isValidScene,
+  loadWhiteboard,
+  MAX_ELEMENTS,
+  saveWhiteboard,
+  WHITEBOARD_SNAPSHOT_FILENAME,
+} from "../shared/whiteboard-store.js";
+
+const SCENE_BODY_LIMIT = 8 * 1024 * 1024;
+
+type WhiteboardRouteDeps = {
+  pool: Pool;
+  mediaRoot: string;
+  agentManager: AgentManager;
+  publishUiEvent: (event: unknown) => void;
+};
+
+export async function registerWhiteboardRoutes(
+  app: FastifyInstance,
+  deps: WhiteboardRouteDeps
+): Promise<void> {
+  app.get("/api/v1/agents/:id/whiteboard", async (request, reply) => {
+    const params = request.params as { id?: string };
+    const id = params.id ?? "";
+    const agent = await deps.agentManager.getAgent(id);
+    if (!agent) {
+      return reply.code(404).send({ error: "Agent not found." });
+    }
+
+    const row = await loadWhiteboard(deps.pool, id);
+    if (!row) {
+      return { scene: EMPTY_SCENE, version: 0, updatedAt: null };
+    }
+    return {
+      scene: row.scene,
+      version: Number(row.version),
+      updatedAt: row.updated_at.toISOString(),
+    };
+  });
+
+  app.put(
+    "/api/v1/agents/:id/whiteboard",
+    { bodyLimit: SCENE_BODY_LIMIT },
+    async (request, reply) => {
+      const params = request.params as { id?: string };
+      const id = params.id ?? "";
+      const agent = await deps.agentManager.getAgent(id);
+      if (!agent) {
+        return reply.code(404).send({ error: "Agent not found." });
+      }
+
+      const body = request.body as
+        | { scene?: unknown; baseVersion?: unknown }
+        | undefined;
+      if (!isValidScene(body?.scene)) {
+        return reply.code(400).send({
+          error: `scene must be an object with an elements array (max ${MAX_ELEMENTS}).`,
+        });
+      }
+      const baseVersion =
+        typeof body?.baseVersion === "number" &&
+        Number.isInteger(body.baseVersion) &&
+        body.baseVersion >= 0
+          ? body.baseVersion
+          : null;
+      if (baseVersion === null) {
+        return reply
+          .code(400)
+          .send({ error: "baseVersion must be a non-negative integer." });
+      }
+
+      const saved = await saveWhiteboard(
+        deps.pool,
+        id,
+        body.scene,
+        baseVersion,
+        "user"
+      );
+      if (!saved) {
+        const current = await loadWhiteboard(deps.pool, id);
+        return reply.code(409).send({
+          error: "Whiteboard was modified by someone else.",
+          scene: current?.scene ?? EMPTY_SCENE,
+          version: current ? Number(current.version) : 0,
+        });
+      }
+
+      deps.publishUiEvent({
+        type: "whiteboard.changed",
+        agentId: id,
+        version: saved.version,
+        source: "user",
+      });
+      return { ok: true, version: saved.version };
+    }
+  );
+
+  app.post("/api/v1/agents/:id/whiteboard/snapshot", async (request, reply) => {
+    const params = request.params as { id?: string };
+    const id = params.id ?? "";
+    const agent = await deps.agentManager.getAgent(id);
+    if (!agent) {
+      return reply.code(404).send({ error: "Agent not found." });
+    }
+
+    const data = await request.file();
+    if (!data || data.mimetype !== "image/png") {
+      return reply.code(400).send({ error: "A PNG file field is required." });
+    }
+
+    const mediaDir = resolveMediaDir(id, agent.mediaDir, deps.mediaRoot);
+    await mkdir(mediaDir, { recursive: true });
+    const buffer = await data.toBuffer();
+    await writeFile(path.join(mediaDir, WHITEBOARD_SNAPSHOT_FILENAME), buffer);
+    return { ok: true, sizeBytes: buffer.length };
+  });
+
+  app.delete(
+    "/api/v1/agents/:id/whiteboard/snapshot",
+    async (request, reply) => {
+      const params = request.params as { id?: string };
+      const id = params.id ?? "";
+      const agent = await deps.agentManager.getAgent(id);
+      if (!agent) {
+        return reply.code(404).send({ error: "Agent not found." });
+      }
+      const mediaDir = resolveMediaDir(id, agent.mediaDir, deps.mediaRoot);
+      await rm(path.join(mediaDir, WHITEBOARD_SNAPSHOT_FILENAME), {
+        force: true,
+      });
+      return { ok: true };
+    }
+  );
+}

apps/server/src/server.ts

@@ -1,5 +1,6 @@
 import path from "node:path";
 import os from "node:os";
+import { fileURLToPath } from "node:url";
 import {
   mkdir,
   readFile,
@@ -36,6 +37,7 @@ import { deleteSetting, getSetting, setSetting } from "./db/settings.js";
 import { runCommand } from "./shared/lib/run-command.js";
 import { shouldSkipAutomaticMacPathProbe } from "./shared/mac-path-privacy.js";
 import { mimeType, resolveMediaDir } from "./shared/media.js";
+import { LibraryCatalog } from "./shared/whiteboard-library.js";
 import { handleMcpRequest } from "./shared/mcp/server.js";
 import { readReleaseStore, writeReleaseStore } from "./release-store.js";
 import {
@@ -105,6 +107,7 @@ import { registerFeedbackRoutes } from "./routes/feedback.js";
 import { registerJobRoutes } from "./routes/jobs.js";
 import { registerTemplateRoutes } from "./routes/templates.js";
 import { registerMediaRoutes } from "./routes/media.js";
+import { registerWhiteboardRoutes } from "./routes/whiteboard.js";
 import { registerMcpRoutes } from "./routes/mcp.js";
 import { registerPersonaReviewRoutes } from "./routes/persona-reviews.js";
 import { registerPersonalityRoutes } from "./routes/personalities.js";
@@ -146,7 +149,29 @@ import { createActivityMonitor } from "./agents/activity-monitor.js";
 import { createAutoRenamePrompter } from "./agents/auto-rename-prompter.js";
 import { DiffStatsRefresher } from "./agents/diff-stats-refresher.js";
 
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 const config = loadConfig();
+let libraryCatalog: LibraryCatalog | undefined;
+try {
+  libraryCatalog = await LibraryCatalog.fromDirectory(
+    path.resolve(
+      __dirname,
+      "..",
+      "..",
+      "web",
+      "public",
+      "excalidraw",
+      "libraries"
+    )
+  );
+} catch (err) {
+  console.warn(
+    `Failed to load whiteboard library catalog, continuing without library support: ${
+      err instanceof Error ? err.message : String(err)
+    }`
+  );
+  libraryCatalog = undefined;
+}
 const app = Fastify({
   logger: true,
   ...(config.tls && { https: { cert: config.tls.cert, key: config.tls.key } }),
@@ -319,6 +344,7 @@ const mcpHandlers = createMcpHandlers({
   withStreamFlag,
   sendAgentPrompt: injectAgentPrompt,
   appLog: app.log,
+  libraryCatalog,
 });
 const jobTerminalStatuses = new Set([
   "completed",
@@ -476,6 +502,11 @@ async function registerRoutes() {
     mcpRenameSession: mcpHandlers.renameSession,
     mcpShareMedia: mcpHandlers.shareMedia,
     mcpListMedia: mcpHandlers.listMedia,
+    mcpGetWhiteboard: mcpHandlers.getWhiteboard,
+    mcpUpdateWhiteboard: mcpHandlers.updateWhiteboard,
+    mcpClearWhiteboard: mcpHandlers.clearWhiteboard,
+    mcpLibraryList: mcpHandlers.libraryList,
+    mcpInsertLibraryItem: mcpHandlers.insertLibraryItem,
     mcpSubmitFeedback: mcpHandlers.submitFeedback,
     mcpListPersonas: mcpHandlers.listPersonas,
     mcpLaunchPersona: mcpHandlers.launchPersona,
@@ -581,6 +612,13 @@ async function registerRoutes() {
     publishUiEvent: (event) => uiEventBroker.publish(event as UiEvent),
   });
 
+  await registerWhiteboardRoutes(app, {
+    pool,
+    mediaRoot: config.mediaRoot,
+    agentManager,
+    publishUiEvent: (event) => uiEventBroker.publish(event as UiEvent),
+  });
+
   await registerAgentRoutes(app, {
     pool,
     appLog: app.log,

apps/server/src/server/mcp-handlers.ts

@@ -28,6 +28,8 @@ import { resolveHeadSha } from "../shared/git/worktree.js";
 import { isMediaFile, isTextFile, resolveMediaDir } from "../shared/media.js";
 import type { PublishUiEvent, SendAgentPrompt } from "./mcp-handler-types.js";
 import { createReviewHandlers } from "./mcp-review-handlers.js";
+import { createWhiteboardHandlers } from "./mcp-whiteboard-handlers.js";
+import type { LibraryCatalog } from "../shared/whiteboard-library.js";
 
 const AGENT_LATEST_EVENT_TYPES = [
   "working",
@@ -63,6 +65,7 @@ type CreateMcpHandlersDeps = {
   ) => T & { hasStream: boolean };
   sendAgentPrompt: SendAgentPrompt;
   appLog: FastifyBaseLogger;
+  libraryCatalog?: LibraryCatalog;
 };
 
 function isAgentLatestEventType(value: unknown): value is AgentLatestEventType {
@@ -141,6 +144,7 @@ export function createMcpHandlers(deps: CreateMcpHandlersDeps) {
     withStreamFlag,
     sendAgentPrompt,
     appLog,
+    libraryCatalog,
   } = deps;
 
   const reviewHandlers = createReviewHandlers({
@@ -151,8 +155,17 @@ export function createMcpHandlers(deps: CreateMcpHandlersDeps) {
     sendAgentPrompt,
   });
 
+  const whiteboardHandlers = createWhiteboardHandlers({
+    pool,
+    mediaRoot,
+    agentManager,
+    publishUiEvent,
+    libraryCatalog,
+  });
+
   return {
     ...reviewHandlers,
+    ...whiteboardHandlers,
 
     async upsertEvent(
       agentId: string,