Add per-agent whiteboard using tldraw

apps/server/package.json

@@ -20,6 +20,8 @@
     "@fastify/rate-limit": "^10.3.0",
     "@fastify/websocket": "^11.2.0",
     "@modelcontextprotocol/sdk": "^1.27.1",
+    "@tldraw/store": "5.2.4",
+    "@tldraw/tlschema": "5.2.4",
     "bcryptjs": "^3.0.3",
     "croner": "^10.0.1",
     "dotenv": "^17.3.1",

apps/server/src/db/migrations/0030_whiteboards.sql

@@ -0,0 +1,7 @@
+CREATE TABLE IF NOT EXISTS whiteboards (
+  agent_id TEXT PRIMARY KEY REFERENCES agents(id) ON DELETE CASCADE,
+  scene JSONB NOT NULL DEFAULT '{}',
+  version INTEGER NOT NULL DEFAULT 1,
+  created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);

apps/server/src/routes/mcp.ts

@@ -72,6 +72,8 @@ type McpRouteDeps = {
   mcpJobLog: unknown;
   mcpSendMessage: unknown;
   mcpListAgentsForAgent: unknown;
+  mcpGetWhiteboard: unknown;
+  mcpUpdateWhiteboard: unknown;
   mcpMethodNotAllowed: () => unknown;
 };
 
@@ -239,6 +241,8 @@ export async function registerMcpRoutes(
       crudTools: buildCrudCallbacks(deps),
       brainStore: deps.brainStore,
       publishBrainChanged: deps.publishBrainChanged,
+      getWhiteboard: deps.mcpGetWhiteboard,
+      updateWhiteboard: deps.mcpUpdateWhiteboard,
     } as Parameters<typeof handleMcpRequest>[3]);
   });
 
@@ -329,6 +333,8 @@ export async function registerMcpRoutes(
       crudTools: buildCrudCallbacks(deps),
       brainStore: deps.brainStore,
       publishBrainChanged: deps.publishBrainChanged,
+      getWhiteboard: deps.mcpGetWhiteboard,
+      updateWhiteboard: deps.mcpUpdateWhiteboard,
     } as Parameters<typeof handleMcpRequest>[3]);
   });
 

apps/server/src/routes/whiteboard.ts

@@ -0,0 +1,134 @@
+import path from "node:path";
+import { mkdir, readFile, writeFile, unlink } from "node:fs/promises";
+
+import type { FastifyInstance } from "fastify";
+import type { Pool } from "pg";
+
+import type { AgentManager } from "../agents/manager.js";
+import { applyOps } from "../shared/whiteboard-builder.js";
+import { resolveMediaDir } from "../shared/media.js";
+import { getWhiteboard, saveWhiteboard } from "../shared/whiteboard-store.js";
+import {
+  simplifyElements,
+  type WhiteboardOp,
+  type WhiteboardScene,
+} from "../shared/whiteboard.js";
+
+type WhiteboardRouteDeps = {
+  pool: Pool;
+  mediaRoot: string;
+  agentManager: AgentManager;
+  publishUiEvent: (event: unknown) => void;
+};
+
+export async function registerWhiteboardRoutes(
+  app: FastifyInstance,
+  deps: WhiteboardRouteDeps
+): Promise<void> {
+  const { pool, mediaRoot, agentManager, publishUiEvent } = deps;
+
+  app.get("/api/v1/agents/:agentId/whiteboard", async (request, reply) => {
+    const { agentId } = request.params as { agentId: string };
+    const row = await getWhiteboard(pool, agentId);
+    const scene: WhiteboardScene = row?.scene ?? { records: [] };
+    return reply.send({
+      scene,
+      version: row?.version ?? 0,
+      elements: simplifyElements(scene.records ?? []),
+    });
+  });
+
+  app.put("/api/v1/agents/:agentId/whiteboard", async (request, reply) => {
+    const { agentId } = request.params as { agentId: string };
+    const body = request.body as {
+      scene?: WhiteboardScene;
+      ops?: WhiteboardOp[];
+    };
+
+    if (body.scene) {
+      const row = await getWhiteboard(pool, agentId);
+      const { version } = await saveWhiteboard(
+        pool,
+        agentId,
+        body.scene,
+        row?.version ?? null
+      );
+      publishUiEvent({
+        type: "whiteboard.changed",
+        agentId,
+        source: "user",
+      });
+      return reply.send({
+        version,
+        elementCount: (body.scene.records ?? []).length,
+      });
+    }
+
+    if (body.ops) {
+      const row = await getWhiteboard(pool, agentId);
+      const currentScene: WhiteboardScene = row?.scene ?? { records: [] };
+      const updated = applyOps(currentScene, body.ops);
+      const { version } = await saveWhiteboard(
+        pool,
+        agentId,
+        updated,
+        row?.version ?? null
+      );
+      publishUiEvent({
+        type: "whiteboard.changed",
+        agentId,
+        source: "user",
+      });
+      return reply.send({
+        version,
+        elementCount: updated.records.length,
+      });
+    }
+
+    return reply.code(400).send({ error: "scene or ops required" });
+  });
+
+  app.post(
+    "/api/v1/agents/:agentId/whiteboard/snapshot",
+    async (request, reply) => {
+      const { agentId } = request.params as { agentId: string };
+      const agent = await agentManager.getAgent(agentId);
+      if (!agent) {
+        return reply.code(404).send({ error: "Agent not found" });
+      }
+
+      const data = await request.file();
+      if (!data) {
+        return reply.code(400).send({ error: "No file uploaded" });
+      }
+
+      const buffer = await data.toBuffer();
+      const mediaDir = resolveMediaDir(agentId, agent.mediaDir, mediaRoot);
+      await mkdir(mediaDir, { recursive: true });
+      const snapshotPath = path.join(mediaDir, "whiteboard-snapshot.png");
+      await writeFile(snapshotPath, buffer);
+
+      return reply.send({ path: snapshotPath, sizeBytes: buffer.length });
+    }
+  );
+
+  app.delete(
+    "/api/v1/agents/:agentId/whiteboard/snapshot",
+    async (request, reply) => {
+      const { agentId } = request.params as { agentId: string };
+      const agent = await agentManager.getAgent(agentId);
+      if (!agent) {
+        return reply.code(404).send({ error: "Agent not found" });
+      }
+
+      const mediaDir = resolveMediaDir(agentId, agent.mediaDir, mediaRoot);
+      const snapshotPath = path.join(mediaDir, "whiteboard-snapshot.png");
+      try {
+        await unlink(snapshotPath);
+      } catch {
+        // file doesn't exist, that's fine
+      }
+      return reply.send({ deleted: true });
+    }
+  );
+}

apps/server/src/server.ts

@@ -110,6 +110,7 @@ import { registerPersonaReviewRoutes } from "./routes/persona-reviews.js";
 import { registerPersonalityRoutes } from "./routes/personalities.js";
 import { registerReviewRoutes } from "./routes/reviews.js";
 import { registerQuickPhraseRoutes } from "./routes/quick-phrases.js";
+import { registerWhiteboardRoutes } from "./routes/whiteboard.js";
 import { registerReleaseRoutes } from "./routes/release.js";
 import { createAutoCheckRuntime } from "./release-auto-check.js";
 import { registerStaticRoutes } from "./routes/static.js";
@@ -499,6 +500,8 @@ async function registerRoutes() {
     mcpJobFailed: mcpHandlers.jobFailed,
     mcpJobNeedsInput: mcpHandlers.jobNeedsInput,
     mcpJobLog: mcpHandlers.jobLog,
+    mcpGetWhiteboard: mcpHandlers.getWhiteboardForAgent,
+    mcpUpdateWhiteboard: mcpHandlers.updateWhiteboardForAgent,
     mcpMethodNotAllowed,
   });
 
@@ -643,6 +646,15 @@ async function registerRoutes() {
     handleAgentError,
   });
 
+  // --- Whiteboard ---
+
+  await registerWhiteboardRoutes(app, {
+    pool,
+    mediaRoot: config.mediaRoot,
+    agentManager,
+    publishUiEvent: (event) => uiEventBroker.publish(event as UiEvent),
+  });
+
   // --- Feedback ---
 
   await registerFeedbackRoutes(app, {

apps/server/src/server/mcp-handlers.ts

@@ -28,6 +28,7 @@ import { resolveHeadSha } from "../shared/git/worktree.js";
 import { isMediaFile, isTextFile, resolveMediaDir } from "../shared/media.js";
 import type { PublishUiEvent, SendAgentPrompt } from "./mcp-handler-types.js";
 import { createReviewHandlers } from "./mcp-review-handlers.js";
+import { createWhiteboardHandlers } from "./mcp-whiteboard-handlers.js";
 
 const AGENT_LATEST_EVENT_TYPES = [
   "working",
@@ -151,8 +152,16 @@ export function createMcpHandlers(deps: CreateMcpHandlersDeps) {
     sendAgentPrompt,
   });
 
+  const whiteboardHandlers = createWhiteboardHandlers({
+    pool,
+    mediaRoot,
+    agentManager,
+    publishUiEvent,
+  });
+
   return {
     ...reviewHandlers,
+    ...whiteboardHandlers,
 
     async upsertEvent(
       agentId: string,

apps/server/src/server/mcp-whiteboard-handlers.ts

@@ -0,0 +1,75 @@
+import type { Pool } from "pg";
+
+import { applyOps } from "../shared/whiteboard-builder.js";
+import { getWhiteboard, saveWhiteboard } from "../shared/whiteboard-store.js";
+import {
+  simplifyElements,
+  type WhiteboardOp,
+  type WhiteboardScene,
+  type WhiteboardGetResult,
+  type WhiteboardUpdateResult,
+} from "../shared/whiteboard.js";
+import { resolveMediaDir } from "../shared/media.js";
+import type { AgentManager } from "../agents/manager.js";
+import type { PublishUiEvent } from "./mcp-handler-types.js";
+
+type CreateWhiteboardHandlersDeps = {
+  pool: Pool;
+  mediaRoot: string;
+  agentManager: AgentManager;
+  publishUiEvent: PublishUiEvent;
+};
+
+export function createWhiteboardHandlers(deps: CreateWhiteboardHandlersDeps) {
+  const { pool, mediaRoot, agentManager, publishUiEvent } = deps;
+
+  return {
+    async getWhiteboardForAgent(agentId: string): Promise<WhiteboardGetResult> {
+      const row = await getWhiteboard(pool, agentId);
+      const scene: WhiteboardScene = row?.scene ?? { records: [] };
+      const records = scene.records ?? [];
+
+      const agent = await agentManager.getAgent(agentId);
+      let snapshotPath: string | null = null;
+      if (agent) {
+        const dir = resolveMediaDir(agentId, agent.mediaDir, mediaRoot);
+        snapshotPath = `${dir}/whiteboard-snapshot.png`;
+      }
+
+      return {
+        scene,
+        version: row?.version ?? 0,
+        elements: simplifyElements(records),
+        snapshotPath,
+      };
+    },
+
+    async updateWhiteboardForAgent(
+      agentId: string,
+      ops: WhiteboardOp[]
+    ): Promise<WhiteboardUpdateResult> {
+      const row = await getWhiteboard(pool, agentId);
+      const currentScene: WhiteboardScene = row?.scene ?? { records: [] };
+      const currentVersion = row?.version ?? null;
+
+      const updatedScene = applyOps(currentScene, ops);
+      const { version } = await saveWhiteboard(
+        pool,
+        agentId,
+        updatedScene,
+        currentVersion
+      );
+
+      publishUiEvent({
+        type: "whiteboard.changed",
+        agentId,
+        source: "agent",
+      } as never);
+
+      return {
+        version,
+        elementCount: updatedScene.records.length,
+      };
+    },
+  };
+}

apps/server/src/server/ui-events.ts

@@ -62,6 +62,11 @@ export type UiEvent =
   | {
       type: "release.cached_info_changed";
       snapshot: ReleaseInfoSnapshot | null;
+    }
+  | {
+      type: "whiteboard.changed";
+      agentId: string;
+      source: "user" | "agent";
     };
 
 export class UiEventBroker {

apps/server/src/shared/mcp/server.ts

@@ -21,6 +21,7 @@ import { registerPersonaTools } from "./persona-tools.js";
 import { registerPrTools } from "./pr-tools.js";
 import { loadRepoTools, type RepoToolParam } from "./repo-tools.js";
 import { toToolError } from "./tool-error.js";
+import { registerWhiteboardTools } from "./whiteboard-tools.js";
 
 export type McpAgent = {
   id: string;
@@ -126,6 +127,8 @@ const AGENT_TOOLS = new Set([
   "create_template",
   "update_template",
   "delete_template",
+  "whiteboard_get",
+  "whiteboard_update",
 ]);
 
 const JOB_TOOLS = new Set([
@@ -470,6 +473,18 @@ export type McpRequestContext = {
   toolScope?: "agent" | "reviewer" | "job";
   brainStore?: BrainStore;
   publishBrainChanged?: (repoRoot: string) => void;
+  getWhiteboard?: (
+    agentId: string
+  ) => Promise<{
+    scene: unknown;
+    version: number;
+    elements: unknown[];
+    snapshotPath: string | null;
+  }>;
+  updateWhiteboard?: (
+    agentId: string,
+    ops: unknown[]
+  ) => Promise<{ version: number; elementCount: number }>;
 };
 
 export async function handleMcpRequest(
@@ -605,6 +620,15 @@ async function createDispatchMcpServer(
       context.getFeedbackSummary ?? context.jobTools?.getFeedbackSummary,
   });
 
+  // ── Whiteboard tools ──────────────────────────────────────────────
+  if (context.agent && context.getWhiteboard && context.updateWhiteboard) {
+    registerWhiteboardTools(server, allowed, {
+      agentId: context.agent.id,
+      getWhiteboard: context.getWhiteboard,
+      updateWhiteboard: context.updateWhiteboard,
+    });
+  }
+
   // ── Job & template CRUD tools ─────────────────────────────────────
   if (context.crudTools) {
     registerCrudTools(server, allowed, {