From 5d4f7220819fd61370a00bdaae11de5b4c9059e5 Mon Sep 17 00:00:00 2001
From: collissimo <148455189+collissimo@users.noreply.github.com>
Date: Sat, 31 Oct 2026 17:15:26 +0100
Subject: [PATCH 1/4] feat(dictation): recordings can ask for word times on
 their own clock

A screen capture lines clicks up with what was said, so the engine can time words and maps them back across the pauses it cut. Dictation does not ask and keeps its small reply.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 apps/desktop/src/scrawl/narration.test.ts     | 101 ++++++
 apps/desktop/src/scrawl/narration.ts          | 300 ++++++++++++++++++
 apps/server/src/dictation/Dictation.ts        |   3 +-
 .../src/dictation/DictationEngine.test.ts     |  42 ++-
 apps/server/src/dictation/DictationEngine.ts  | 123 +++++--
 apps/server/src/dictation/localDictation.ts   |   9 +-
 apps/server/src/dictation/worker.ts           |  14 +-
 packages/contracts/src/dictation.ts           |  11 +
 8 files changed, 568 insertions(+), 35 deletions(-)
 create mode 100644 apps/desktop/src/scrawl/narration.test.ts
 create mode 100644 apps/desktop/src/scrawl/narration.ts

diff --git a/apps/desktop/src/scrawl/narration.test.ts b/apps/desktop/src/scrawl/narration.test.ts
new file mode 100644
index 0000000000000..e962b6d697be5
--- /dev/null
+++ b/apps/desktop/src/scrawl/narration.test.ts
@@ -0,0 +1,101 @@
+import { describe, expect, it } from "vitest";
+
+import {
+  buildNarration,
+  foldActions,
+  formatNarrationLine,
+  splitSentences,
+  type TimedWord,
+} from "./narration.ts";
+
+const say = (text: string, start = 0, wordMs = 300): Array<TimedWord> =>
+  text.split(" ").map((word, index) => ({
+    text: word,
+    t0: start + index * wordMs,
+    t1: start + (index + 1) * wordMs,
+  }));
+
+describe("splitSentences", () => {
+  it("keeps a thinking pause and a wrong full stop inside the sentence", () => {
+    const words = [
+      ...say("Ich will hier bitte", 0),
+      ...say("eine andere Ansicht. der einzelnen Tage.", 3_000),
+      ...say("Neuer Satz.", 6_000),
+    ];
+    expect(splitSentences(words).map((sentence) => sentence.words.join(" "))).toEqual([
+      "Ich will hier bitte eine andere Ansicht. der einzelnen Tage.",
+      "Neuer Satz.",
+    ]);
+  });
+
+  it("gives up on an unfinished sentence after a long silence", () => {
+    const words = [...say("Ich will hier", 0), ...say("Neuer Satz.", 10_000)];
+    expect(splitSentences(words)).toHaveLength(2);
+  });
+});
+
+describe("foldActions", () => {
+  it("folds click bursts and drops the hand's small drags", () => {
+    const actions = foldActions(
+      [
+        { kind: "click", t: 1_000, x: 0.1, y: 0.1 },
+        { kind: "click", t: 1_500, x: 0.2, y: 0.1 },
+        { kind: "click", t: 2_900, x: 0.3, y: 0.1 },
+        { kind: "click", t: 9_000, x: 0.5, y: 0.5 },
+        { kind: "drag", t: 12_000, t1: 12_200, x0: 0.5, y0: 0.5, x1: 0.51, y1: 0.5 },
+        { kind: "drag", t: 13_000, t1: 13_400, x0: 0.5, y0: 0.5, x1: 0.1, y1: 0.5 },
+      ],
+      [],
+    );
+    expect(actions).toMatchObject([
+      { kind: "clicks", count: 3, t0: 1_000, t1: 2_900 },
+      { kind: "clicks", count: 1, t0: 9_000 },
+      { kind: "drag", direction: "left" },
+    ]);
+  });
+});
+
+describe("buildNarration", () => {
+  it("never cuts a sentence at a click, and loses no word, click or mark", () => {
+    const words = say("Bitte diese Boxen hier entfernen und den Text darunter auch.", 0);
+    const lines = buildNarration({
+      words,
+      events: [
+        { kind: "click", t: 600, x: 0.1, y: 0.1 },
+        { kind: "click", t: 2_000, x: 0.4, y: 0.4 },
+      ],
+      marks: [{ id: "m1", t0: 900, t1: 1_100 }],
+      durationMs: 4_000,
+    });
+    expect(lines.map((line) => line.text)).toEqual([words.map((word) => word.text).join(" ")]);
+    expect(lines[0]!.actions.map((action) => action.kind).toSorted()).toEqual(["clicks", "mark"]);
+    expect(formatNarrationLine(lines[0]!, 0)).toBe(
+      "- #1 (0:01) [marked, clicked ×2] “Bitte diese Boxen hier entfernen und den Text darunter auch.”",
+    );
+  });
+
+  it("gives a click said over no speech its own line", () => {
+    const lines = buildNarration({
+      words: say("Hallo Welt.", 0),
+      events: [{ kind: "click", t: 20_000, x: 0.5, y: 0.5 }],
+      marks: [{ id: "m1", t0: 100, t1: 300 }],
+      durationMs: 25_000,
+    });
+    expect(formatNarrationLine(lines.at(-1)!, 1)).toBe("- #2 (0:20) clicked");
+  });
+
+  it("keeps every mark over budget, and speech ahead of the first keyframe with it", () => {
+    const marks = Array.from({ length: 9 }, (_, index) => ({
+      id: `m${index}`,
+      t0: 2_000 + index * 2_000,
+      t1: 2_100 + index * 2_000,
+    }));
+    const words = [
+      ...say("Erster Satz ohne Aktion.", 0, 100),
+      ...marks.flatMap((mark) => say("Das hier markiert.", mark.t0, 100)),
+    ];
+    const lines = buildNarration({ words, events: [], marks, durationMs: 22_000 });
+    expect(lines).toHaveLength(9);
+    expect(lines[0]!.text).toBe("Erster Satz ohne Aktion. Das hier markiert.");
+  });
+});
diff --git a/apps/desktop/src/scrawl/narration.ts b/apps/desktop/src/scrawl/narration.ts
new file mode 100644
index 0000000000000..4e60f6f8aeac2
--- /dev/null
+++ b/apps/desktop/src/scrawl/narration.ts
@@ -0,0 +1,300 @@
+// Turns timed words and screen actions into the lines an agent reads. Speech and pointing do not
+// run in step: on 160 real clicks the nearest pointing word ("hier", "das") lay anywhere from 3.3 s
+// before to 3 s after the click, and 42% of the clicks came in bursts. So a sentence is never cut
+// at an action. Whole sentences own a keyframe, and the actions said over them are listed in front.
+
+export interface TimedWord {
+  readonly t0: number;
+  readonly t1: number;
+  readonly text: string;
+}
+
+export type CaptureEvent =
+  | { readonly kind: "click"; readonly t: number; readonly x: number; readonly y: number }
+  | {
+      readonly kind: "drag";
+      readonly t: number;
+      readonly t1: number;
+      readonly x0: number;
+      readonly y0: number;
+      readonly x1: number;
+      readonly y1: number;
+    }
+  | { readonly kind: "scroll"; readonly t: number };
+
+export interface CaptureMark {
+  readonly id: string;
+  readonly t0: number;
+  readonly t1: number;
+  readonly shape?: string;
+}
+
+export type NarrationAction =
+  | {
+      readonly kind: "clicks";
+      readonly t0: number;
+      readonly t1: number;
+      readonly count: number;
+      readonly x: number;
+      readonly y: number;
+    }
+  | { readonly kind: "drag"; readonly t0: number; readonly t1: number; readonly direction: string }
+  | { readonly kind: "scroll"; readonly t0: number; readonly t1: number }
+  | { readonly kind: "mark"; readonly t0: number; readonly t1: number; readonly id: string };
+
+export interface NarrationLine {
+  readonly t0: number;
+  readonly t1: number;
+  /** When the keyframe of this line should be taken. */
+  readonly frameT: number;
+  readonly text: string;
+  readonly actions: ReadonlyArray<NarrationAction>;
+}
+
+interface Sentence {
+  t0: number;
+  t1: number;
+  words: Array<string>;
+  actions: Array<NarrationAction>;
+}
+
+// Thinking pauses run long and the speech goes on where it stopped, so a pause ends only a finished
+// sentence, or an unfinished one when it runs past this.
+const ABANDONED_GAP_MS = 5_000;
+const MAX_SENTENCE_WORDS = 60;
+const BURST_GAP_MS = 1_500;
+const SCROLL_GAP_MS = 1_500;
+// Scrawl's rule: a drag this short (as a share of the screen) is the hand moving, not an act.
+const SHORT_DRAG = 0.15;
+// An action in a speech pause goes to the sentence it is nearest to, up to this far away.
+const ATTACH_MS = 3_000;
+const SETTLE_MS = 300;
+// Scrawl's keyframe budget: each image costs a reading model about 1.5k tokens.
+const budgetFor = (durationMs: number) =>
+  Math.min(12, 6 + Math.max(0, Math.ceil((durationMs - 40_000) / 20_000)));
+const COVERAGE_MS = 35_000;
+const COVERAGE_WORDS = 100;
+
+const SCORES: Record<NarrationAction["kind"], number> = {
+  mark: 100,
+  clicks: 60,
+  drag: 50,
+  scroll: 30,
+};
+
+const endsSentence = (word: string) => /[.!?;:]["”’)\]]*$/.test(word);
+const endsClause = (word: string) => /,["”’)\]]*$/.test(word);
+
+export function splitSentences(words: ReadonlyArray<TimedWord>): Array<Sentence> {
+  const sentences: Array<Sentence> = [];
+  let open: Sentence | null = null;
+  for (const word of [...words].sort((a, b) => a.t0 - b.t0)) {
+    const text = word.text.trim();
+    if (text === "") continue;
+    if (open !== null && word.t0 - open.t1 >= ABANDONED_GAP_MS) open = null;
+    const last = sentences.at(-1);
+    // The model also ends sentences that go on: German starts a new one with a capital.
+    if (open === null && last !== undefined && word.t0 - last.t1 < ABANDONED_GAP_MS) {
+      if (/^\p{Ll}/u.test(text)) open = last;
+    }
+    if (open === null) {
+      open = { t0: word.t0, t1: word.t1, words: [text], actions: [] };
+      sentences.push(open);
+    } else {
+      open.t1 = Math.max(open.t1, word.t1);
+      open.words.push(text);
+    }
+    const tooLong = open.words.length >= MAX_SENTENCE_WORDS && endsClause(text);
+    if (endsSentence(text) || tooLong) open = null;
+  }
+  return sentences;
+}
+
+const directionOf = (dx: number, dy: number) =>
+  Math.abs(dx) >= Math.abs(dy) ? (dx < 0 ? "left" : "right") : dy < 0 ? "up" : "down";
+
+export function foldActions(
+  events: ReadonlyArray<CaptureEvent>,
+  marks: ReadonlyArray<CaptureMark>,
+): Array<NarrationAction> {
+  const actions: Array<NarrationAction> = marks.map((mark) => ({
+    kind: "mark",
+    t0: mark.t0,
+    t1: mark.t1,
+    id: mark.id,
+  }));
+  let burst: { t0: number; t1: number; count: number; x: number; y: number } | null = null;
+  let scroll: { t0: number; t1: number } | null = null;
+  const closeBurst = () => {
+    if (burst !== null) actions.push({ kind: "clicks", ...burst });
+    burst = null;
+  };
+  const closeScroll = () => {
+    if (scroll !== null) actions.push({ kind: "scroll", ...scroll });
+    scroll = null;
+  };
+  for (const event of [...events].sort((a, b) => a.t - b.t)) {
+    if (event.kind === "click") {
+      if (burst !== null && event.t - burst.t1 <= BURST_GAP_MS) {
+        burst = { ...burst, t1: event.t, count: burst.count + 1, x: event.x, y: event.y };
+      } else {
+        closeBurst();
+        burst = { t0: event.t, t1: event.t, count: 1, x: event.x, y: event.y };
+      }
+    } else if (event.kind === "scroll") {
+      if (scroll !== null && event.t - scroll.t1 <= SCROLL_GAP_MS) {
+        scroll = { ...scroll, t1: event.t };
+      } else {
+        closeScroll();
+        scroll = { t0: event.t, t1: event.t };
+      }
+    } else {
+      const dx = event.x1 - event.x0;
+      const dy = event.y1 - event.y0;
+      if (Math.hypot(dx, dy) >= SHORT_DRAG) {
+        actions.push({
+          kind: "drag",
+          t0: event.t,
+          t1: event.t1,
+          direction: directionOf(dx, dy),
+        });
+      }
+    }
+  }
+  closeBurst();
+  closeScroll();
+  return actions.sort((a, b) => a.t0 - b.t0);
+}
+
+const attach = (sentences: Array<Sentence>, actions: ReadonlyArray<NarrationAction>) => {
+  for (const action of actions) {
+    const at = (action.t0 + action.t1) / 2;
+    let best: Sentence | null = null;
+    let bestDistance = Infinity;
+    for (const sentence of sentences) {
+      const distance = Math.max(sentence.t0 - at, at - sentence.t1, 0);
+      if (distance < bestDistance) {
+        best = sentence;
+        bestDistance = distance;
+      }
+    }
+    if (best !== null && bestDistance <= ATTACH_MS) {
+      best.actions.push(action);
+    } else {
+      sentences.push({ t0: action.t0, t1: action.t1, words: [], actions: [action] });
+      sentences.sort((a, b) => a.t0 - b.t0);
+    }
+  }
+};
+
+const scoreOf = (sentence: Sentence) =>
+  sentence.actions.reduce((best, action) => Math.max(best, SCORES[action.kind]), 0);
+
+/** The sentences that own a keyframe: every mark, then the strongest actions, then long stretches. */
+const chooseFrameOwners = (sentences: ReadonlyArray<Sentence>, durationMs: number) => {
+  const budget = budgetFor(durationMs);
+  const owners = new Set<number>();
+  const ranked = sentences
+    .map((sentence, index) => ({ index, score: scoreOf(sentence) }))
+    .filter((entry) => entry.score > 0)
+    .sort((a, b) => b.score - a.score || a.index - b.index);
+  for (const { index, score } of ranked) {
+    if (score >= SCORES.mark || owners.size < budget) owners.add(index);
+  }
+  if (sentences.length > 0) {
+    let stretchStart = 0;
+    let stretchWords = 0;
+    for (let index = 0; index < sentences.length; index++) {
+      const sentence = sentences[index]!;
+      if (owners.has(index)) {
+        stretchStart = sentence.t0;
+        stretchWords = sentence.words.length;
+        continue;
+      }
+      stretchWords += sentence.words.length;
+      const long = sentence.t1 - stretchStart > COVERAGE_MS || stretchWords > COVERAGE_WORDS;
+      if (long && owners.size < budget) {
+        owners.add(index);
+        stretchStart = sentence.t0;
+        stretchWords = sentence.words.length;
+      }
+    }
+  }
+  // Speech with nothing on screen still gets one frame to be read against.
+  if (owners.size === 0 && sentences.length > 0) owners.add(0);
+  return owners;
+};
+
+const frameTimeOf = (t0: number, actions: ReadonlyArray<NarrationAction>, durationMs: number) => {
+  const marks = actions.filter((action) => action.kind === "mark");
+  const moment = marks.at(-1) ?? actions.at(-1);
+  if (moment === undefined) return t0;
+  const settle = moment.kind === "mark" ? 0 : SETTLE_MS;
+  return Math.min(moment.t1 + settle, durationMs);
+};
+
+export function buildNarration(input: {
+  readonly words: ReadonlyArray<TimedWord>;
+  readonly events: ReadonlyArray<CaptureEvent>;
+  readonly marks: ReadonlyArray<CaptureMark>;
+  readonly durationMs: number;
+}): Array<NarrationLine> {
+  const sentences = splitSentences(input.words);
+  attach(sentences, foldActions(input.events, input.marks));
+  const owners = chooseFrameOwners(sentences, input.durationMs);
+  const groups: Array<Array<Sentence>> = [];
+  let ownerSeen = false;
+  sentences.forEach((sentence, index) => {
+    const owner = owners.has(index);
+    // Speech ahead of the first keyframe belongs to it.
+    if (groups.length === 0 || (owner && ownerSeen)) groups.push([sentence]);
+    else groups.at(-1)!.push(sentence);
+    ownerSeen ||= owner;
+  });
+  return groups.map((group) => {
+    const actions = group.flatMap((sentence) => sentence.actions);
+    const t0 = group[0]!.t0;
+    return {
+      t0,
+      t1: Math.max(...group.map((sentence) => sentence.t1)),
+      frameT: frameTimeOf(t0, actions, input.durationMs),
+      text: group.flatMap((sentence) => sentence.words).join(" "),
+      actions,
+    };
+  });
+}
+
+const formatClock = (ms: number) => {
+  const seconds = Math.round(ms / 1000);
+  return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
+};
+
+const describeActions = (actions: ReadonlyArray<NarrationAction>) => {
+  let marks = 0;
+  let clicks = 0;
+  let scrolled = false;
+  const drags = new Set<string>();
+  for (const action of actions) {
+    if (action.kind === "mark") marks++;
+    else if (action.kind === "clicks") clicks += action.count;
+    else if (action.kind === "scroll") scrolled = true;
+    else drags.add(action.direction);
+  }
+  const counted = (word: string, count: number) => (count > 1 ? `${word} ×${count}` : word);
+  return [
+    marks > 0 ? counted("marked", marks) : null,
+    clicks > 0 ? counted("clicked", clicks) : null,
+    ...[...drags].map((direction) => `dragged ${direction}`),
+    scrolled ? "scrolled" : null,
+  ]
+    .filter((part) => part !== null)
+    .join(", ");
+};
+
+/** `- #2 (0:09) [clicked ×3] “…”`: the shape Scrawl pastes, which agents and skills already read. */
+export function formatNarrationLine(line: NarrationLine, index: number): string {
+  const head = `- #${index + 1} (${formatClock(line.frameT)})`;
+  const actions = describeActions(line.actions);
+  if (line.text === "") return `${head} ${actions}`.trimEnd();
+  return `${head}${actions === "" ? "" : ` [${actions}]`} “${line.text}”`;
+}
diff --git a/apps/server/src/dictation/Dictation.ts b/apps/server/src/dictation/Dictation.ts
index 4f492b8509ed5..dc2d1d979f5e2 100644
--- a/apps/server/src/dictation/Dictation.ts
+++ b/apps/server/src/dictation/Dictation.ts
@@ -97,7 +97,8 @@ const make = Effect.gen(function* () {
 
   const finish = Effect.fn("Dictation.finish")(function* (input: DictationFinishInput) {
     const result = yield* Effect.tryPromise({
-      try: () => engine.finish(input.sessionId),
+      try: () =>
+        engine.finish(input.sessionId, input.words === undefined ? {} : { words: input.words }),
       catch: toDictationError,
     });
     return finishResult(result);
diff --git a/apps/server/src/dictation/DictationEngine.test.ts b/apps/server/src/dictation/DictationEngine.test.ts
index fd612de076c66..960a40f3242c1 100644
--- a/apps/server/src/dictation/DictationEngine.test.ts
+++ b/apps/server/src/dictation/DictationEngine.test.ts
@@ -9,8 +9,10 @@ import {
   SAMPLE_RATE,
   decodePcm16,
   keepSpeech,
+  keptSpans,
   normalizeLanguage,
   splitAtPauses,
+  toRecordingMs,
   type TranscribeModule,
 } from "./DictationEngine.ts";
 import { dictationBackend } from "./localDictation.ts";
@@ -136,6 +138,16 @@ describe("audio input", () => {
     expect(kept.length).toBe(Math.round(3.4 * SAMPLE_RATE));
     expect(keepSpeech(audio, [])).toHaveLength(0);
   });
+  it("puts a word back on the recording's clock across a cut pause", () => {
+    const spans = keptSpans([
+      { startSample: SAMPLE_RATE, endSample: 2 * SAMPLE_RATE },
+      { startSample: 9 * SAMPLE_RATE, endSample: 10 * SAMPLE_RATE },
+    ]);
+    expect(toRecordingMs(500, spans)).toBe(1_500);
+    // 1 s of speech and the kept 0.4 s of the pause come before the second stretch.
+    expect(toRecordingMs(1_600, spans)).toBe(9_200);
+    expect(toRecordingMs(1_600, [])).toBe(1_600);
+  });
   it("reads the backend from the environment, auto when unset or unknown", () => {
     expect(dictationBackend(" CPU ")).toBe("cpu");
     expect(dictationBackend("vulkan")).toBe("vulkan");
@@ -159,7 +171,7 @@ class FakeModel {
   runs: Array<{ length: number; language: string | undefined }> = [];
   disposed = false;
   readonly backend = "CPU";
-  readonly capabilities = { languages: ["de", "en"] };
+  readonly capabilities = { languages: ["de", "en"], maxTimestampKind: "word" };
   static async load(path: string) {
     FakeModel.loads.push(path);
     return new FakeModel();
@@ -170,7 +182,14 @@ class FakeModel {
         this.runs.push({ length: pcm.length, language: opts.language });
         FakeModel.runLengths.push(pcm.length);
         if (FakeModel.runError !== undefined) throw FakeModel.runError;
-        return { text: "  Hallo   Welt ", aborted: false };
+        return {
+          text: "  Hallo   Welt ",
+          aborted: false,
+          words: [
+            { text: " Hallo", t0Ms: 100, t1Ms: 400 },
+            { text: " Welt", t0Ms: 1_600, t1Ms: 1_800 },
+          ],
+        };
       },
       dispose: () => undefined,
     };
@@ -230,6 +249,25 @@ describe("DictationEngine", () => {
     expect(FakeModel.loads).toEqual(["model.gguf"]);
   });
 
+  it("times words on the recording's clock only when asked to", async () => {
+    FakeModel.vadSegments = [
+      { startSample: SAMPLE_RATE, endSample: 2 * SAMPLE_RATE },
+      { startSample: 9 * SAMPLE_RATE, endSample: 10 * SAMPLE_RATE },
+    ];
+    const { engine } = makeEngine(undefined, async () => "vad.gguf");
+    engine.append("a", "de", speech(10));
+    expect(await engine.finish("a", { words: true })).toEqual({
+      text: "Hallo Welt",
+      words: [
+        { text: "Hallo", t0: 1_100, t1: 1_400 },
+        { text: "Welt", t0: 9_200, t1: 9_400 },
+      ],
+    });
+    engine.append("b", "de", speech(10));
+    expect(await engine.finish("b")).toEqual({ text: "Hallo Welt" });
+    FakeModel.vadSegments = [];
+  });
+
   it("transcribes a long recording in pieces and joins them", async () => {
     const { engine } = makeEngine();
     engine.append("long", "de", speech(40));
diff --git a/apps/server/src/dictation/DictationEngine.ts b/apps/server/src/dictation/DictationEngine.ts
index 0824790afa358..5873664aa59bc 100644
--- a/apps/server/src/dictation/DictationEngine.ts
+++ b/apps/server/src/dictation/DictationEngine.ts
@@ -47,8 +47,20 @@ export interface DictationEngineOptions {
   readonly log?: (message: string) => void;
 }
 
+export interface TranscriptWord {
+  readonly text: string;
+  /** Milliseconds from the start of the recording, pauses included. */
+  readonly t0: number;
+  readonly t1: number;
+}
+
 export type TranscriptResult =
-  | { readonly text: string; readonly reason?: undefined }
+  | {
+      readonly text: string;
+      readonly reason?: undefined;
+      /** Only when asked for and the model can time words. */
+      readonly words?: ReadonlyArray<TranscriptWord>;
+    }
   | { readonly text: ""; readonly reason: "no-speech" | "no-audio" };
 
 interface LoadedModel {
@@ -104,31 +116,56 @@ export function splitAtPauses(audio: Float32Array): ReadonlyArray<Float32Array>
   return pieces;
 }
 
-/** The audio of `segments`, with the pauses between them cut down to a short gap. */
-export function keepSpeech(
-  audio: Float32Array,
-  segments: ReadonlyArray<TranscribeCpp.VadSegment>,
-): Float32Array {
-  const parts: Array<Float32Array> = [];
+/** A stretch of the shortened audio and where it began in the recording, in samples. */
+export interface KeptSpan {
+  readonly keptStart: number;
+  readonly recordingStart: number;
+  readonly length: number;
+}
+
+export const keptSpans = (segments: ReadonlyArray<TranscribeCpp.VadSegment>): Array<KeptSpan> => {
+  const spans: Array<KeptSpan> = [];
+  let keptStart = 0;
   let previousEnd: number | undefined;
+  const add = (recordingStart: number, end: number) => {
+    spans.push({ keptStart, recordingStart, length: end - recordingStart });
+    keptStart += end - recordingStart;
+  };
   for (const { startSample, endSample } of segments) {
     if (previousEnd !== undefined) {
-      parts.push(
-        audio.subarray(previousEnd, Math.min(startSample, previousEnd + KEPT_PAUSE_SAMPLES)),
-      );
+      add(previousEnd, Math.min(startSample, previousEnd + KEPT_PAUSE_SAMPLES));
     }
-    parts.push(audio.subarray(startSample, endSample));
+    add(startSample, endSample);
     previousEnd = endSample;
   }
-  const speech = new Float32Array(parts.reduce((sum, part) => sum + part.length, 0));
-  let offset = 0;
-  for (const part of parts) {
-    speech.set(part, offset);
-    offset += part.length;
+  return spans;
+};
+
+/** The audio of `segments`, with the pauses between them cut down to a short gap. */
+export function keepSpeech(
+  audio: Float32Array,
+  segments: ReadonlyArray<TranscribeCpp.VadSegment>,
+): Float32Array {
+  const spans = keptSpans(segments);
+  const speech = new Float32Array(spans.reduce((sum, span) => sum + span.length, 0));
+  for (const span of spans) {
+    speech.set(
+      audio.subarray(span.recordingStart, span.recordingStart + span.length),
+      span.keptStart,
+    );
   }
   return speech;
 }
 
+/** Word times come back on the shortened audio; clicks and frames live on the recording's clock. */
+export function toRecordingMs(keptMs: number, spans: ReadonlyArray<KeptSpan>): number {
+  if (spans.length === 0) return keptMs;
+  const sample = (keptMs / 1000) * SAMPLE_RATE;
+  const span =
+    spans.find((candidate) => sample < candidate.keptStart + candidate.length) ?? spans.at(-1)!;
+  return Math.round(((span.recordingStart + (sample - span.keptStart)) / SAMPLE_RATE) * 1000);
+}
+
 export function decodePcm16(base64: string): Float32Array {
   const bytes = Buffer.from(base64, "base64");
   const count = Math.floor(bytes.length / 2);
@@ -194,7 +231,10 @@ export class DictationEngine {
   }
 
   /** Ends a session and transcribes all of it. */
-  async finish(sessionId: string): Promise<TranscriptResult> {
+  async finish(
+    sessionId: string,
+    options?: { readonly words?: boolean },
+  ): Promise<TranscriptResult> {
     const session = this.#sessions.get(sessionId);
     this.#sessions.delete(sessionId);
     this.#lastUsedAt = this.#now();
@@ -209,33 +249,52 @@ export class DictationEngine {
     // A preview run in flight is aborted instead of making the final run wait for it.
     this.#previewAbort?.abort();
     const started = this.#now();
-    const text = await this.#exclusive(async () => {
+    const canTimeWords = ["word", "token"].includes(loaded.model.capabilities.maxTimestampKind);
+    const wantWords = options?.words === true && canTimeWords;
+    const { text, words } = await this.#exclusive(async () => {
       const texts: Array<string> = [];
-      for (const piece of splitAtPauses(await this.#speechOnly(audio))) {
+      const words: Array<TranscriptWord> = [];
+      const { speech, spans } = await this.#speechWithSpans(audio);
+      let pieceStartMs = 0;
+      for (const piece of splitAtPauses(speech)) {
+        const collect = (result: TranscribeCpp.Transcript) => {
+          texts.push(result.text);
+          if (!wantWords) return;
+          for (const word of result.words) {
+            words.push({
+              text: word.text.trim(),
+              t0: toRecordingMs(pieceStartMs + word.t0Ms, spans),
+              t1: toRecordingMs(pieceStartMs + word.t1Ms, spans),
+            });
+          }
+        };
         try {
-          const result = await loaded.session.run(
-            piece,
-            this.#runOptions(loaded, session.language),
+          collect(
+            await loaded.session.run(piece, {
+              ...this.#runOptions(loaded, session.language),
+              ...(wantWords ? { timestamps: "word" as const } : {}),
+            }),
           );
-          texts.push(result.text);
         } catch (error) {
           const module = await this.#loadModule();
           if (error instanceof module.OutputTruncated && error.partialResult !== undefined) {
             // One piece that ran away must not cost the rest of the recording.
-            texts.push(error.partialResult.text);
+            collect(error.partialResult);
           } else if (!(error instanceof module.InputTooShort)) {
             throw error;
           }
         }
+        pieceStartMs += (piece.length / SAMPLE_RATE) * 1000;
       }
-      return texts.join(" ");
+      return { text: texts.join(" "), words };
     });
     this.#lastUsedAt = this.#now();
     const cleaned = text.split(/\s+/).filter(Boolean).join(" ");
     this.#options.log?.(
       `transcribed ${(audio.length / SAMPLE_RATE).toFixed(1)}s of audio in ${this.#now() - started}ms`,
     );
-    return cleaned ? { text: cleaned } : { text: "", reason: "no-speech" };
+    if (!cleaned) return { text: "", reason: "no-speech" };
+    return wantWords ? { text: cleaned, words } : { text: cleaned };
   }
 
   cancel(sessionId: string): void {
@@ -314,14 +373,20 @@ export class DictationEngine {
 
   /** Must run inside #exclusive: a VAD session serves one run at a time. */
   async #speechOnly(audio: Float32Array): Promise<Float32Array> {
+    return (await this.#speechWithSpans(audio)).speech;
+  }
+
+  async #speechWithSpans(
+    audio: Float32Array,
+  ): Promise<{ readonly speech: Float32Array; readonly spans: ReadonlyArray<KeptSpan> }> {
     const vad = await this.#loadVad();
-    if (vad === undefined) return audio;
+    if (vad === undefined) return { speech: audio, spans: [] };
     try {
       const { segments } = await vad.session.run(audio, VAD_OPTIONS);
-      return keepSpeech(audio, segments);
+      return { speech: keepSpeech(audio, segments), spans: keptSpans(segments) };
     } catch (error) {
       this.#options.log?.(`voice activity detection failed, keeping pauses: ${describe(error)}`);
-      return audio;
+      return { speech: audio, spans: [] };
     }
   }
 
diff --git a/apps/server/src/dictation/localDictation.ts b/apps/server/src/dictation/localDictation.ts
index 6ff0b002dc9a1..d51c67d831464 100644
--- a/apps/server/src/dictation/localDictation.ts
+++ b/apps/server/src/dictation/localDictation.ts
@@ -96,5 +96,12 @@ export function finishResult(result: TranscriptResult): DictationFinishResult {
   }
   const cleaned = splitCodeword(removeFillers(result.text), DEFAULT_STRUCTURE_CODEWORDS);
   if (cleaned.text === "") return { text: "", reason: "no-speech", structureRequested: false };
-  return { text: cleaned.text, structureRequested: cleaned.found };
+  if (result.words === undefined) return { text: cleaned.text, structureRequested: cleaned.found };
+  // Fillers go word by word: the words carry the times a capture lines its clicks up with.
+  const words = result.words.filter((word) => removeFillers(word.text) !== "");
+  return {
+    text: cleaned.text,
+    structureRequested: cleaned.found,
+    words: cleaned.found ? words.slice(0, -1) : words,
+  };
 }
diff --git a/apps/server/src/dictation/worker.ts b/apps/server/src/dictation/worker.ts
index a27fbe325dae9..76e0c4c0752b7 100644
--- a/apps/server/src/dictation/worker.ts
+++ b/apps/server/src/dictation/worker.ts
@@ -21,7 +21,12 @@ export type DictationWorkerRequest =
       readonly language: string | undefined;
       readonly pcm: string;
     }
-  | { readonly id: number; readonly method: "finish"; readonly sessionId: string }
+  | {
+      readonly id: number;
+      readonly method: "finish";
+      readonly sessionId: string;
+      readonly words?: boolean;
+    }
   | { readonly id: number; readonly method: "cancel"; readonly sessionId: string };
 
 export type DictationWorkerResponse =
@@ -50,7 +55,12 @@ export async function handleDictationWorkerRequest(
         return {
           id: request.id,
           ok: true,
-          value: finishResult(await engine.finish(request.sessionId)),
+          value: finishResult(
+            await engine.finish(
+              request.sessionId,
+              request.words === undefined ? {} : { words: request.words },
+            ),
+          ),
         };
       case "cancel":
         engine.cancel(request.sessionId);
diff --git a/packages/contracts/src/dictation.ts b/packages/contracts/src/dictation.ts
index 7270e337cf3e7..f387d3ebc66e1 100644
--- a/packages/contracts/src/dictation.ts
+++ b/packages/contracts/src/dictation.ts
@@ -30,18 +30,29 @@ export type DictationAppendResult = typeof DictationAppendResult.Type;
 
 export const DictationFinishInput = Schema.Struct({
   sessionId: DictationSessionId,
+  /** Also time each word. Only screen captures need this; dictation keeps its reply small. */
+  words: Schema.optional(Schema.Boolean),
 });
 export type DictationFinishInput = typeof DictationFinishInput.Type;
 
 export const DictationNoTextReason = Schema.Literals(["no-speech", "no-audio"]);
 export type DictationNoTextReason = typeof DictationNoTextReason.Type;
 
+export const DictationWord = Schema.Struct({
+  text: Schema.String,
+  /** Milliseconds from the start of the recording, pauses included. */
+  t0: Schema.Number,
+  t1: Schema.Number,
+});
+export type DictationWord = typeof DictationWord.Type;
+
 export const DictationFinishResult = Schema.Struct({
   /** Cleaned transcript; empty when nothing was recognized, and `reason` says why. */
   text: Schema.String,
   reason: Schema.optional(DictationNoTextReason),
   /** The speaker ended with the codeword that asks for a structured rewrite (already cut). */
   structureRequested: Schema.Boolean,
+  words: Schema.optional(Schema.Array(DictationWord)),
 });
 export type DictationFinishResult = typeof DictationFinishResult.Type;
 

From 159cc9b741470539524092c671107bb522d69df2 Mon Sep 17 00:00:00 2001
From: collissimo <148455189+collissimo@users.noreply.github.com>
Date: Sat, 31 Oct 2026 17:37:26 +0100
Subject: [PATCH 2/4] feat(scrawl): capture the screen with voice, clicks and
 ink, inside T3 Code

Alt+Shift+S or the composer button records the monitor under the mouse, also over other programs. Sentences stay whole and own their keyframes, clicks the screen did not answer read as pointing, and the bundle keeps Scrawl's shape.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 FORK.md                                       |  19 +
 apps/desktop/src/ipc/DesktopIpcHandlers.ts    |  12 +
 apps/desktop/src/ipc/channels.ts              |   5 +
 apps/desktop/src/ipc/methods/scrawl.ts        |  58 +++
 apps/desktop/src/main.ts                      |   2 +
 apps/desktop/src/preload.ts                   |   5 +
 apps/desktop/src/scrawl/DesktopScrawl.ts      | 362 +++++++++++++++
 apps/desktop/src/scrawl/inputTracker.test.ts  |  34 ++
 apps/desktop/src/scrawl/inputTracker.ts       | 117 +++++
 apps/web/src/components/chat/ChatComposer.tsx |  37 ++
 .../components/chat/ComposerScrawlButton.tsx  |  88 ++++
 apps/web/src/dictation/microphoneCapture.ts   |   3 +
 apps/web/src/dictation/useDictation.ts        |   4 +-
 .../src/scrawl/narration.test.ts              |  29 +-
 apps/{desktop => web}/src/scrawl/narration.ts |  89 ++--
 apps/web/src/scrawl/sceneChanges.test.ts      |  29 ++
 apps/web/src/scrawl/sceneChanges.ts           |  64 +++
 apps/web/src/scrawl/scrawlBundle.ts           | 413 ++++++++++++++++++
 apps/web/src/scrawl/scrawlCapture.ts          | 374 ++++++++++++++++
 packages/contracts/src/index.ts               |   1 +
 packages/contracts/src/ipc.ts                 |  13 +
 packages/contracts/src/scrawl.ts              |  79 ++++
 22 files changed, 1804 insertions(+), 33 deletions(-)
 create mode 100644 apps/desktop/src/ipc/methods/scrawl.ts
 create mode 100644 apps/desktop/src/scrawl/DesktopScrawl.ts
 create mode 100644 apps/desktop/src/scrawl/inputTracker.test.ts
 create mode 100644 apps/desktop/src/scrawl/inputTracker.ts
 create mode 100644 apps/web/src/components/chat/ComposerScrawlButton.tsx
 rename apps/{desktop => web}/src/scrawl/narration.test.ts (79%)
 rename apps/{desktop => web}/src/scrawl/narration.ts (77%)
 create mode 100644 apps/web/src/scrawl/sceneChanges.test.ts
 create mode 100644 apps/web/src/scrawl/sceneChanges.ts
 create mode 100644 apps/web/src/scrawl/scrawlBundle.ts
 create mode 100644 apps/web/src/scrawl/scrawlCapture.ts
 create mode 100644 packages/contracts/src/scrawl.ts

diff --git a/FORK.md b/FORK.md
index 54c3930ec268c..682ad250619b0 100644
--- a/FORK.md
+++ b/FORK.md
@@ -150,3 +150,22 @@ can be rebound in Settings → Keybindings ("Composer: Dictate").
   `apps/web/src/components/chat/ComposerDictationButton.tsx`, `packages/contracts/src/dictation.ts`.
   The standalone app for every other program is [collissimo/t3-voice](https://github.com/collissimo/t3-voice);
   it leaves F9 to T3 Code while T3 Code is in front.
+
+### Screen capture (Scrawl, built in)
+
+**Alt+Shift+S** anywhere, or the screen button in the composer, records the monitor under the
+mouse, also while another program is in front. Talk, click, hold **Shift** and move the mouse to
+draw red ink (it fades 5 seconds after Shift is let go). Alt+Shift+S again finishes: the text and
+the contact sheet land in the composer on screen. Esc in T3 Code discards a capture.
+
+- Desktop app only: a browser cannot see the whole screen, and mobile has no such thing. Clicks and
+  ink need Windows; elsewhere a capture has voice and video.
+- While the Scrawl app runs it holds Alt+Shift+S. T3 Code logs that and takes the shortcut within
+  15 seconds of Scrawl closing. The composer button works either way.
+- Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences
+  own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer
+  read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1.
+- Uses the dictation model above, with word times mapped back across the pauses it cuts out.
+- Bundles are in `<T3 home>/scrawl/captures` in Scrawl's format, so the Scrawl skill and the
+  `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. Video and audio are deleted after 7 days.
+- Code: `apps/desktop/src/scrawl/`, `apps/web/src/scrawl/`, `packages/contracts/src/scrawl.ts`.
diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts
index 1501a10b20366..f378757607428 100644
--- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts
+++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts
@@ -10,6 +10,13 @@ import {
   setLocalEnvironmentEnabled,
 } from "./methods/localEnvironment.ts";
 import { dictationAppend, dictationCancel, dictationFinish } from "./methods/dictation.ts";
+import {
+  scrawlAppendVideo,
+  scrawlCancel,
+  scrawlStart,
+  scrawlStop,
+  scrawlWriteFile,
+} from "./methods/scrawl.ts";
 import {
   getAdvertisedEndpoints,
   getServerExposureState,
@@ -100,6 +107,11 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
   yield* ipc.handle(setLocalEnvironmentEnabled);
   yield* ipc.handle(dictationAppend);
   yield* ipc.handle(dictationFinish);
+  yield* ipc.handle(scrawlStart);
+  yield* ipc.handle(scrawlStop);
+  yield* ipc.handle(scrawlCancel);
+  yield* ipc.handle(scrawlAppendVideo);
+  yield* ipc.handle(scrawlWriteFile);
   yield* ipc.handle(dictationCancel);
   yield* ipc.handle(getLocalEnvironmentBearerToken);
 
diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts
index 88e36c8b5aebf..58d11aa9e8e3e 100644
--- a/apps/desktop/src/ipc/channels.ts
+++ b/apps/desktop/src/ipc/channels.ts
@@ -39,6 +39,11 @@ export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environm
 export const DICTATION_APPEND_CHANNEL = "desktop:dictation-append";
 export const DICTATION_FINISH_CHANNEL = "desktop:dictation-finish";
 export const DICTATION_CANCEL_CHANNEL = "desktop:dictation-cancel";
+export const SCRAWL_START_CHANNEL = "desktop:scrawl-start";
+export const SCRAWL_STOP_CHANNEL = "desktop:scrawl-stop";
+export const SCRAWL_CANCEL_CHANNEL = "desktop:scrawl-cancel";
+export const SCRAWL_APPEND_VIDEO_CHANNEL = "desktop:scrawl-append-video";
+export const SCRAWL_WRITE_FILE_CHANNEL = "desktop:scrawl-write-file";
 export const GET_LOCAL_ENVIRONMENT_BEARER_TOKEN_CHANNEL =
   "desktop:get-local-environment-bearer-token";
 export const GET_CLIENT_SETTINGS_CHANNEL = "desktop:get-client-settings";
diff --git a/apps/desktop/src/ipc/methods/scrawl.ts b/apps/desktop/src/ipc/methods/scrawl.ts
new file mode 100644
index 0000000000000..b82618fa4642d
--- /dev/null
+++ b/apps/desktop/src/ipc/methods/scrawl.ts
@@ -0,0 +1,58 @@
+import {
+  ScrawlAppendVideoInput,
+  ScrawlCaptureRef,
+  ScrawlStartResult,
+  ScrawlStopResult,
+  ScrawlWriteFileInput,
+} from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+
+import * as DesktopScrawl from "../../scrawl/DesktopScrawl.ts";
+import * as IpcChannels from "../channels.ts";
+import { makeIpcMethod } from "../DesktopIpc.ts";
+
+export const scrawlStart = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_START_CHANNEL,
+  payload: Schema.Void,
+  result: ScrawlStartResult,
+  handler: Effect.fn("desktop.ipc.scrawl.start")(function* () {
+    return yield* (yield* DesktopScrawl.DesktopScrawl).start();
+  }),
+});
+
+export const scrawlStop = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_STOP_CHANNEL,
+  payload: ScrawlCaptureRef,
+  result: ScrawlStopResult,
+  handler: Effect.fn("desktop.ipc.scrawl.stop")(function* (input) {
+    return yield* (yield* DesktopScrawl.DesktopScrawl).stop(input);
+  }),
+});
+
+export const scrawlCancel = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_CANCEL_CHANNEL,
+  payload: ScrawlCaptureRef,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.scrawl.cancel")(function* (input) {
+    yield* (yield* DesktopScrawl.DesktopScrawl).cancel(input);
+  }),
+});
+
+export const scrawlAppendVideo = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL,
+  payload: ScrawlAppendVideoInput,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.scrawl.appendVideo")(function* (input) {
+    yield* (yield* DesktopScrawl.DesktopScrawl).appendVideo(input);
+  }),
+});
+
+export const scrawlWriteFile = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_WRITE_FILE_CHANNEL,
+  payload: ScrawlWriteFileInput,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.scrawl.writeFile")(function* (input) {
+    yield* (yield* DesktopScrawl.DesktopScrawl).writeFile(input);
+  }),
+});
diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts
index d44e48932a093..63ab8ea13516d 100644
--- a/apps/desktop/src/main.ts
+++ b/apps/desktop/src/main.ts
@@ -41,6 +41,7 @@ import * as DesktopConnectionCatalogStore from "./app/DesktopConnectionCatalogSt
 import * as DesktopClerk from "./app/DesktopClerk.ts";
 import * as DesktopCliCommand from "./app/DesktopCliCommand.ts";
 import * as DesktopDictation from "./dictation/DesktopDictation.ts";
+import * as DesktopScrawl from "./scrawl/DesktopScrawl.ts";
 import * as DesktopApplicationMenu from "./window/DesktopApplicationMenu.ts";
 import * as DesktopAssets from "./app/DesktopAssets.ts";
 import * as DesktopBackendConfiguration from "./backend/DesktopBackendConfiguration.ts";
@@ -210,6 +211,7 @@ const layerDesktopApplication = Layer.mergeAll(
   DesktopLinuxUrlHandler.layer,
   DesktopCliCommand.layer,
   DesktopDictation.layer,
+  DesktopScrawl.layer,
   layerDesktopSsh,
 ).pipe(
   Layer.provideMerge(layerDesktopSnapShot),
diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts
index 96cef2b58df54..34a785cd56d19 100644
--- a/apps/desktop/src/preload.ts
+++ b/apps/desktop/src/preload.ts
@@ -122,6 +122,11 @@ contextBridge.exposeInMainWorld("desktopBridge", {
   dictationAppend: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_APPEND_CHANNEL, input),
   dictationFinish: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_FINISH_CHANNEL, input),
   dictationCancel: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_CANCEL_CHANNEL, input),
+  scrawlStart: () => ipcRenderer.invoke(IpcChannels.SCRAWL_START_CHANNEL),
+  scrawlStop: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_STOP_CHANNEL, input),
+  scrawlCancel: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_CANCEL_CHANNEL, input),
+  scrawlAppendVideo: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL, input),
+  scrawlWriteFile: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_WRITE_FILE_CHANNEL, input),
   getClientSettings: () => ipcRenderer.invoke(IpcChannels.GET_CLIENT_SETTINGS_CHANNEL),
   setClientSettings: (settings) =>
     ipcRenderer.invoke(IpcChannels.SET_CLIENT_SETTINGS_CHANNEL, settings),
diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts
new file mode 100644
index 0000000000000..8cbf37dd81e38
--- /dev/null
+++ b/apps/desktop/src/scrawl/DesktopScrawl.ts
@@ -0,0 +1,362 @@
+// @effect-diagnostics globalTimers:off -- The cursor poll and pill clock run at native callback boundaries outside Effect fibers.
+// @effect-diagnostics globalTimersInEffect:off -- They are started here and owned by the capture, not a fiber.
+// @effect-diagnostics globalDate:off -- Input times share the wall clock with the renderer's recorder.
+// @effect-diagnostics globalDateInEffect:off -- Same clock as above.
+// @effect-diagnostics nodeBuiltinImport:off -- Bundles are plain files written outside any Effect service.
+// The desktop half of a Scrawl capture: the global shortcut, the mouse and Shift watcher, the ink
+// and the recording pill, and the bundle on disk. The renderer records and builds the bundle.
+
+import {
+  SCRAWL_INK_HOLD_MS,
+  SCRAWL_TOGGLE_ACTION,
+  type ScrawlAppendVideoInput,
+  type ScrawlCaptureRef,
+  type ScrawlStartResult,
+  type ScrawlStopResult,
+  type ScrawlWriteFileInput,
+} from "@t3tools/contracts";
+import * as Context from "effect/Context";
+import * as Effect from "effect/Effect";
+import * as Layer from "effect/Layer";
+import * as Schema from "effect/Schema";
+import * as Electron from "electron";
+import * as NodeFSP from "node:fs/promises";
+import * as NodePath from "node:path";
+
+import * as DesktopEnvironment from "../app/DesktopEnvironment.ts";
+import { loadWindowsForegroundApi } from "../electron/WindowsForeground.ts";
+import { activeWindow } from "../snapShot/ActiveWindow.ts";
+import * as DesktopWindow from "../window/DesktopWindow.ts";
+import { ScrawlInputTracker } from "./inputTracker.ts";
+
+const DEFAULT_SHORTCUT = "Alt+Shift+S";
+// Another program (the Scrawl app uses the same keys) may hold the shortcut and let go of it later.
+const SHORTCUT_RETRY_MS = 15_000;
+// Short presses must not fall between two looks. Polling needs no hook, no message pump and no
+// permission, and costs two native calls per look, only while a capture runs.
+const INPUT_POLL_MS = 10;
+const VK_LBUTTON = 0x01;
+const VK_SHIFT = 0x10;
+// Like Scrawl: frames, text and contact sheet stay, the heavy media goes.
+const MEDIA_KEPT_MS = 7 * 24 * 60 * 60 * 1000;
+const MEDIA_FILES = ["recording.mp4", "recording.webm", "audio.wav"];
+const PILL_WIDTH = 96;
+const PILL_HEIGHT = 30;
+
+export class DesktopScrawlError extends Schema.TaggedError<DesktopScrawlError>()(
+  "DesktopScrawlError",
+  { reason: Schema.String },
+) {
+  override get message(): string {
+    return this.reason;
+  }
+}
+
+export class DesktopScrawl extends Context.Service<
+  DesktopScrawl,
+  {
+    readonly start: () => Effect.Effect<ScrawlStartResult, DesktopScrawlError>;
+    readonly stop: (input: ScrawlCaptureRef) => Effect.Effect<ScrawlStopResult, DesktopScrawlError>;
+    readonly cancel: (input: ScrawlCaptureRef) => Effect.Effect<void>;
+    readonly appendVideo: (
+      input: ScrawlAppendVideoInput,
+    ) => Effect.Effect<void, DesktopScrawlError>;
+    readonly writeFile: (input: ScrawlWriteFileInput) => Effect.Effect<void, DesktopScrawlError>;
+  }
+>()("@t3tools/desktop/scrawl/DesktopScrawl") {}
+
+const INK_PAGE = `<!doctype html><html><body style="margin:0;background:transparent;overflow:hidden">
+<canvas id="c" style="position:fixed;inset:0;width:100%;height:100%"></canvas><script>
+const c=document.getElementById("c"),g=c.getContext("2d");let last=null;
+const fit=()=>{c.width=innerWidth*devicePixelRatio;c.height=innerHeight*devicePixelRatio;g.scale(devicePixelRatio,devicePixelRatio);g.strokeStyle="#ff2d2d";g.lineWidth=4;g.lineCap="round";g.lineJoin="round"};fit();
+window.inkMove=(x,y)=>{if(last){g.beginPath();g.moveTo(last[0],last[1]);g.lineTo(x,y);g.stroke()}last=[x,y]};
+window.inkLift=()=>{last=null};
+window.inkClear=()=>{last=null;g.clearRect(0,0,innerWidth,innerHeight)};
+</script></body></html>`;
+
+const PILL_PAGE = `<!doctype html><html><body style="margin:0;background:#18181b;color:#fafafa;font:600 13px system-ui;display:flex;align-items:center;justify-content:center;gap:8px;height:100vh;border-radius:999px;overflow:hidden;user-select:none">
+<span style="width:9px;height:9px;border-radius:50%;background:#ef4444"></span><span id="t">0:00</span>
+<script>window.setClock=(v)=>{document.getElementById("t").textContent=v}</script></body></html>`;
+
+const dropOldMedia = async (capturesDir: string, now: number) => {
+  const bundles = await NodeFSP.readdir(capturesDir).catch(() => [] as Array<string>);
+  for (const bundle of bundles) {
+    if (!bundle.endsWith(".scrawl")) continue;
+    const folder = NodePath.join(capturesDir, bundle);
+    const stat = await NodeFSP.stat(folder).catch(() => null);
+    if (stat === null || now - stat.mtimeMs < MEDIA_KEPT_MS) continue;
+    await Promise.all(
+      MEDIA_FILES.map((file) => NodeFSP.rm(NodePath.join(folder, file), { force: true })),
+    ).catch(() => undefined);
+  }
+};
+
+const dataUrl = (html: string) => `data:text/html;charset=utf-8,${encodeURIComponent(html)}`;
+
+const overlayOptions = (bounds: Electron.Rectangle): Electron.BrowserWindowConstructorOptions => ({
+  ...bounds,
+  transparent: true,
+  frame: false,
+  focusable: false,
+  resizable: false,
+  movable: false,
+  hasShadow: false,
+  skipTaskbar: true,
+  alwaysOnTop: true,
+  show: false,
+  webPreferences: { sandbox: true, contextIsolation: true, backgroundThrottling: false },
+});
+
+const toError = (cause: unknown) =>
+  new DesktopScrawlError({ reason: cause instanceof Error ? cause.message : String(cause) });
+
+interface RunningCapture {
+  readonly id: string;
+  readonly bundlePath: string;
+  readonly startedAtMs: number;
+  readonly tracker: ScrawlInputTracker;
+  readonly display: Electron.Display;
+  readonly ink: Electron.BrowserWindow;
+  readonly pill: Electron.BrowserWindow;
+  readonly inputTimer: ReturnType<typeof setInterval>;
+  inkTimer: ReturnType<typeof setTimeout> | null;
+  readonly clockTimer: ReturnType<typeof setInterval>;
+}
+
+const make = Effect.gen(function* () {
+  const environment = yield* DesktopEnvironment.DesktopEnvironment;
+  const desktopWindow = yield* DesktopWindow.DesktopWindow;
+  const services = yield* Effect.context<never>();
+  const capturesDir = NodePath.join(environment.baseDir, "scrawl", "captures");
+  let running: RunningCapture | null = null;
+  // Finished captures keep writing their frames and summary after the watcher stopped.
+  const bundles = new Map<string, string>();
+  let nextIndex = 0;
+
+  const ink = (capture: RunningCapture, script: string) => {
+    if (!capture.ink.isDestroyed()) void capture.ink.webContents.executeJavaScript(script, true);
+  };
+
+  /** Clicks and strokes need Windows' key state; elsewhere a capture has voice and video only. */
+  const startInputPoll = (isKeyDown: ((virtualKey: number) => boolean) | null) => {
+    let left = false;
+    let shift = false;
+    return setInterval(() => {
+      const current = running;
+      if (current === null) return;
+      const t = Date.now() - current.startedAtMs;
+      const at = Electron.screen.getCursorScreenPoint();
+      if (isKeyDown !== null) {
+        const nextLeft = isKeyDown(VK_LBUTTON);
+        const nextShift = isKeyDown(VK_SHIFT);
+        if (nextLeft && !left) current.tracker.leftDown(t, at);
+        if (!nextLeft && left) current.tracker.leftUp(t, at);
+        if (nextShift && !shift) {
+          current.tracker.shiftDown(t, at);
+          if (current.inkTimer !== null) clearTimeout(current.inkTimer);
+          current.inkTimer = null;
+        }
+        if (!nextShift && shift) {
+          current.tracker.shiftUp();
+          ink(current, "inkLift()");
+          // Fades on a clock, not on the next click, so a click's effect on the screen stays its own.
+          current.inkTimer = setTimeout(() => ink(current, "inkClear()"), SCRAWL_INK_HOLD_MS);
+        }
+        left = nextLeft;
+        shift = nextShift;
+      }
+      const point = current.tracker.moved(t, at);
+      if (point !== null) {
+        ink(
+          current,
+          `inkMove(${point.x - current.display.bounds.x},${point.y - current.display.bounds.y})`,
+        );
+      }
+    }, INPUT_POLL_MS);
+  };
+
+  const start = Effect.fn("DesktopScrawl.start")(function* () {
+    if (running !== null) {
+      return yield* new DesktopScrawlError({ reason: "A capture is already running." });
+    }
+    void dropOldMedia(capturesDir, Date.now());
+    const cursor = Electron.screen.getCursorScreenPoint();
+    const display = Electron.screen.getDisplayNearestPoint(cursor);
+    const sources = yield* Effect.tryPromise({
+      try: () =>
+        Electron.desktopCapturer.getSources({
+          types: ["screen"],
+          thumbnailSize: { width: 0, height: 0 },
+        }),
+      catch: toError,
+    });
+    const source = sources.find((entry) => entry.display_id === String(display.id)) ?? sources[0];
+    if (source === undefined) {
+      return yield* new DesktopScrawlError({ reason: "No screen to record." });
+    }
+    const front = yield* Effect.tryPromise({
+      try: () => activeWindow(environment.platform),
+      catch: toError,
+    }).pipe(Effect.orElseSucceed(() => undefined));
+    const keys =
+      environment.platform === "win32"
+        ? yield* Effect.tryPromise({ try: () => loadWindowsForegroundApi(), catch: toError })
+        : null;
+    const startedAtMs = Date.now();
+    const id = `scrawl-${startedAtMs.toString(36)}-${(nextIndex++).toString(36)}`;
+    const bundlePath = NodePath.join(capturesDir, `${id}.scrawl`);
+    yield* Effect.tryPromise({
+      try: () => NodeFSP.mkdir(NodePath.join(bundlePath, "frames"), { recursive: true }),
+      catch: toError,
+    });
+    const tracker = new ScrawlInputTracker(display.bounds);
+
+    const inkWindow = new Electron.BrowserWindow(overlayOptions(display.bounds));
+    inkWindow.setIgnoreMouseEvents(true);
+    inkWindow.setAlwaysOnTop(true, "screen-saver");
+    void inkWindow.loadURL(dataUrl(INK_PAGE));
+    inkWindow.showInactive();
+
+    const pill = new Electron.BrowserWindow(
+      overlayOptions({
+        x: display.bounds.x + Math.round((display.bounds.width - PILL_WIDTH) / 2),
+        y: display.bounds.y + 8,
+        width: PILL_WIDTH,
+        height: PILL_HEIGHT,
+      }),
+    );
+    pill.setIgnoreMouseEvents(true);
+    pill.setAlwaysOnTop(true, "screen-saver");
+    // Stays out of the recording, so it never covers what is being shown.
+    pill.setContentProtection(true);
+    void pill.loadURL(dataUrl(PILL_PAGE));
+    pill.showInactive();
+
+    const capture: RunningCapture = {
+      id,
+      bundlePath,
+      startedAtMs,
+      tracker,
+      display,
+      ink: inkWindow,
+      pill,
+      inputTimer: startInputPoll(keys === null ? null : keys.isKeyDown),
+      inkTimer: null,
+      clockTimer: setInterval(() => {
+        const current = running;
+        if (current === null || current.pill.isDestroyed()) return;
+        const seconds = Math.floor((Date.now() - current.startedAtMs) / 1000);
+        const clock = `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
+        void current.pill.webContents.executeJavaScript(`setClock("${clock}")`, true);
+      }, 1000),
+    };
+    running = capture;
+    bundles.set(id, bundlePath);
+    const scale = display.scaleFactor;
+    return {
+      id,
+      sourceId: source.id,
+      width: Math.round(display.bounds.width * scale),
+      height: Math.round(display.bounds.height * scale),
+      startedAtMs,
+      app: front?.owner.name.replace(/\.exe$/i, "") ?? "",
+      windowTitle: front?.title ?? "",
+      bundlePath,
+    } satisfies ScrawlStartResult;
+  });
+
+  const tearDown = (capture: RunningCapture) => {
+    clearInterval(capture.inputTimer);
+    if (capture.inkTimer !== null) clearTimeout(capture.inkTimer);
+    clearInterval(capture.clockTimer);
+    if (!capture.ink.isDestroyed()) capture.ink.destroy();
+    if (!capture.pill.isDestroyed()) capture.pill.destroy();
+    if (running === capture) running = null;
+  };
+
+  const stop = Effect.fn("DesktopScrawl.stop")(function* (input: ScrawlCaptureRef) {
+    if (running === null || running.id !== input.id) {
+      return yield* new DesktopScrawlError({ reason: "This capture is not running." });
+    }
+    const capture = running;
+    capture.tracker.shiftUp();
+    tearDown(capture);
+    const { events, strokes } = capture.tracker.result();
+    return { events: [...events], strokes: [...strokes] };
+  });
+
+  const cancel = Effect.fn("DesktopScrawl.cancel")(function* (input: ScrawlCaptureRef) {
+    if (running?.id === input.id) tearDown(running);
+    const bundlePath = bundles.get(input.id);
+    bundles.delete(input.id);
+    if (bundlePath !== undefined) {
+      yield* Effect.promise(() =>
+        NodeFSP.rm(bundlePath, { recursive: true, force: true }).catch(() => undefined),
+      );
+    }
+  });
+
+  const appendVideo = Effect.fn("DesktopScrawl.appendVideo")(function* (
+    input: ScrawlAppendVideoInput,
+  ) {
+    const bundlePath = bundles.get(input.id);
+    if (bundlePath === undefined) {
+      return yield* new DesktopScrawlError({ reason: "Unknown capture." });
+    }
+    yield* Effect.tryPromise({
+      try: () => NodeFSP.appendFile(NodePath.join(bundlePath, input.name), input.bytes),
+      catch: toError,
+    });
+  });
+
+  const writeFile = Effect.fn("DesktopScrawl.writeFile")(function* (input: ScrawlWriteFileInput) {
+    const bundlePath = bundles.get(input.id);
+    if (bundlePath === undefined) {
+      return yield* new DesktopScrawlError({ reason: "Unknown capture." });
+    }
+    const target = NodePath.join(bundlePath, input.path);
+    yield* Effect.tryPromise({
+      try: async () => {
+        await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true });
+        await NodeFSP.writeFile(target, input.bytes);
+      },
+      catch: toError,
+    });
+  });
+
+  yield* Effect.acquireRelease(
+    Effect.promise(() => {
+      const run = Effect.runForkWith(services);
+      const register = () =>
+        Electron.globalShortcut.register(DEFAULT_SHORTCUT, () => {
+          run(
+            desktopWindow
+              .dispatchMenuAction(SCRAWL_TOGGLE_ACTION, { reveal: false })
+              .pipe(Effect.ignore),
+          );
+        });
+      const whenReady = async () => {
+        await Electron.app.whenReady();
+        if (register()) return null;
+        run(Effect.logWarning(`scrawl: ${DEFAULT_SHORTCUT} is taken by another program`));
+        const retry = setInterval(() => {
+          if (register()) clearInterval(retry);
+        }, SHORTCUT_RETRY_MS);
+        retry.unref?.();
+        return retry;
+      };
+      return whenReady();
+    }),
+    (retry) =>
+      Effect.promise(async () => {
+        const timer = await retry;
+        if (timer !== null) clearInterval(timer);
+        Electron.globalShortcut.unregister(DEFAULT_SHORTCUT);
+        if (running !== null) tearDown(running);
+      }),
+  );
+
+  return DesktopScrawl.of({ start, stop, cancel, appendVideo, writeFile });
+});
+
+export const layer = Layer.effect(DesktopScrawl, make);
diff --git a/apps/desktop/src/scrawl/inputTracker.test.ts b/apps/desktop/src/scrawl/inputTracker.test.ts
new file mode 100644
index 0000000000000..62d7f4c2705a4
--- /dev/null
+++ b/apps/desktop/src/scrawl/inputTracker.test.ts
@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { ScrawlInputTracker } from "./inputTracker.ts";
+
+const secondDisplay = { x: 1920, y: 0, width: 1000, height: 500 };
+
+describe("ScrawlInputTracker", () => {
+  it("tells clicks from drags, on the captured display only", () => {
+    const tracker = new ScrawlInputTracker(secondDisplay);
+    tracker.leftDown(100, { x: 2420, y: 250 });
+    tracker.leftUp(150, { x: 2422, y: 251 });
+    tracker.leftDown(200, { x: 2020, y: 100 });
+    tracker.leftUp(600, { x: 2420, y: 100 });
+    tracker.leftDown(700, { x: 100, y: 100 });
+    tracker.leftUp(750, { x: 100, y: 100 });
+    expect(tracker.result().events).toEqual([
+      { kind: "click", t: 100, x: 0.5, y: 0.5 },
+      { kind: "drag", t: 200, t1: 600, x0: 0.1, y0: 0.2, x1: 0.5, y1: 0.2 },
+    ]);
+  });
+
+  it("keeps a stroke drawn with Shift, and no stroke for a typed capital", () => {
+    const tracker = new ScrawlInputTracker(secondDisplay);
+    tracker.shiftDown(0, { x: 2000, y: 100 });
+    expect(tracker.shiftUp()).toBe(false);
+    tracker.shiftDown(100, { x: 2000, y: 100 });
+    expect(tracker.moved(110, { x: 2000, y: 100 })).toBeNull();
+    tracker.moved(120, { x: 2100, y: 100 });
+    expect(tracker.shiftUp()).toBe(true);
+    expect(tracker.result().strokes).toEqual([
+      { t0: 100, t1: 120, points: [0.08, 0.2, 0.18, 0.2] },
+    ]);
+  });
+});
diff --git a/apps/desktop/src/scrawl/inputTracker.ts b/apps/desktop/src/scrawl/inputTracker.ts
new file mode 100644
index 0000000000000..37e6564e0d46b
--- /dev/null
+++ b/apps/desktop/src/scrawl/inputTracker.ts
@@ -0,0 +1,117 @@
+// Turns button and Shift changes plus cursor positions into clicks, drags and strokes.
+
+import type { ScrawlInputEvent, ScrawlStroke } from "@t3tools/contracts";
+
+export interface DisplayArea {
+  readonly x: number;
+  readonly y: number;
+  readonly width: number;
+  readonly height: number;
+}
+
+export interface CursorPoint {
+  readonly x: number;
+  readonly y: number;
+}
+
+// In screen points: a hand that moves this little while pressing meant a click.
+const CLICK_SLOP = 4;
+const MIN_STROKE_LENGTH = 6;
+
+const round4 = (value: number) => Math.round(value * 10_000) / 10_000;
+
+export class ScrawlInputTracker {
+  readonly #area: DisplayArea;
+  readonly #events: Array<ScrawlInputEvent> = [];
+  readonly #strokes: Array<ScrawlStroke> = [];
+  #press: { readonly t: number; readonly at: CursorPoint } | null = null;
+  #stroke: { readonly t0: number; readonly raw: Array<CursorPoint>; t1: number } | null = null;
+
+  constructor(area: DisplayArea) {
+    this.#area = area;
+  }
+
+  #inside(point: CursorPoint): boolean {
+    const { x, y, width, height } = this.#area;
+    return point.x >= x && point.y >= y && point.x < x + width && point.y < y + height;
+  }
+
+  #normalized(point: CursorPoint): { readonly x: number; readonly y: number } {
+    return {
+      x: round4((point.x - this.#area.x) / this.#area.width),
+      y: round4((point.y - this.#area.y) / this.#area.height),
+    };
+  }
+
+  leftDown(t: number, at: CursorPoint): void {
+    this.#press = this.#inside(at) ? { t, at } : null;
+  }
+
+  leftUp(t: number, at: CursorPoint): void {
+    const press = this.#press;
+    this.#press = null;
+    if (press === null) return;
+    const start = this.#normalized(press.at);
+    if (Math.hypot(at.x - press.at.x, at.y - press.at.y) <= CLICK_SLOP) {
+      this.#events.push({ kind: "click", t: press.t, x: start.x, y: start.y });
+      return;
+    }
+    const end = this.#normalized(at);
+    this.#events.push({
+      kind: "drag",
+      t: press.t,
+      t1: t,
+      x0: start.x,
+      y0: start.y,
+      x1: end.x,
+      y1: end.y,
+    });
+  }
+
+  shiftDown(t: number, at: CursorPoint): void {
+    this.#stroke = { t0: t, t1: t, raw: [at] };
+  }
+
+  /** Returns the new position when it extends the stroke being drawn. */
+  moved(t: number, at: CursorPoint): CursorPoint | null {
+    const stroke = this.#stroke;
+    if (stroke === null) return null;
+    const last = stroke.raw.at(-1)!;
+    if (last.x === at.x && last.y === at.y) return null;
+    stroke.raw.push(at);
+    stroke.t1 = t;
+    return at;
+  }
+
+  /** Ends the stroke. A Shift press without movement (typing a capital) leaves nothing behind. */
+  shiftUp(): boolean {
+    const stroke = this.#stroke;
+    this.#stroke = null;
+    if (stroke === null) return false;
+    let length = 0;
+    for (let index = 1; index < stroke.raw.length; index++) {
+      const previous = stroke.raw[index - 1]!;
+      const point = stroke.raw[index]!;
+      length += Math.hypot(point.x - previous.x, point.y - previous.y);
+    }
+    if (length < MIN_STROKE_LENGTH) return false;
+    const inside = stroke.raw.filter((point) => this.#inside(point));
+    if (inside.length < 2) return false;
+    this.#strokes.push({
+      t0: stroke.t0,
+      t1: stroke.t1,
+      points: inside.flatMap((point) => {
+        const { x, y } = this.#normalized(point);
+        return [x, y];
+      }),
+    });
+    return true;
+  }
+
+  result(): {
+    readonly events: ReadonlyArray<ScrawlInputEvent>;
+    readonly strokes: ReadonlyArray<ScrawlStroke>;
+  } {
+    return { events: this.#events, strokes: this.#strokes };
+  }
+}
diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx
index 2f6b73d1dd379..d40066b10aa89 100644
--- a/apps/web/src/components/chat/ChatComposer.tsx
+++ b/apps/web/src/components/chat/ChatComposer.tsx
@@ -40,6 +40,7 @@ import {
   ProviderDriverKind,
   ProviderInstanceId,
   PROVIDER_SEND_TURN_MAX_ATTACHMENTS,
+  PROVIDER_SEND_TURN_MAX_IMAGE_BYTES,
 } from "@t3tools/contracts";
 import type { EnvironmentConnectionPresentation } from "@t3tools/client-runtime/connection";
 import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime";
@@ -230,6 +231,8 @@ const isEnvironmentRequestInvalidError = Schema.is(EnvironmentRequestInvalidErro
 import { markDictatedText } from "@t3tools/client-runtime/voice-input/dictationMarker";
 import { useDictation } from "../../dictation/useDictation";
 import { ComposerDictationButton } from "./ComposerDictationButton";
+import { ComposerScrawlButton } from "./ComposerScrawlButton";
+import { cancelScrawl, type ScrawlDelivery } from "../../scrawl/scrawlCapture";
 import { DictationHistoryDialog } from "./DictationHistoryDialog";
 import { toastManager } from "../ui/toast";
 import { PlayIcon } from "lucide-react";
@@ -2460,6 +2463,34 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
         ensureLeadingBoundary: true,
       }),
   });
+  // A stable object: the capture keeps it and calls the composer's latest handlers.
+  const scrawlInsertStable = useStableCallback((text: string) =>
+    insertComposerTextStable(text, "end", { ensureLeadingBoundary: true }),
+  );
+  const scrawlAttachStable = useStableCallback(async (file: File) => {
+    const { compressImageToByteLimit, readFileAsDataUrl } =
+      await import("../../lib/imageCompression");
+    const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES);
+    if (!compressed.ok) return false;
+    const previewUrl = await readFileAsDataUrl(compressed.file);
+    return (
+      addComposerImagesToDraft([
+        {
+          type: "image",
+          id: randomUUID(),
+          name: compressed.file.name,
+          mimeType: compressed.file.type,
+          sizeBytes: compressed.file.size,
+          previewUrl,
+          file: compressed.file,
+        },
+      ]).length > 0
+    );
+  });
+  const [scrawlDelivery] = useState<ScrawlDelivery>(() => ({
+    insert: scrawlInsertStable,
+    attachImage: scrawlAttachStable,
+  }));
   const toggleDictationStable = useStableCallback(dictation.toggle);
   const cancelDictationStable = useStableCallback(dictation.cancel);
   const insertDictationFromHistoryStable = useStableCallback(dictation.insertFromHistory);
@@ -2472,6 +2503,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
       if (event.key === "Escape") {
         // The open history takes Esc for itself, so closing it cannot drop a running recording.
         if (dictationHistoryOpen) return;
+        if (cancelScrawl()) {
+          event.preventDefault();
+          event.stopPropagation();
+          return;
+        }
         // Before anything else claims Esc (it can also interrupt a running turn).
         if (cancelDictationStable()) {
           event.preventDefault();
@@ -3488,6 +3524,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps)
               >
                 <div className="me-2 flex items-center gap-0.5">
                   {composerAttachButton}
+                  <ComposerScrawlButton delivery={scrawlDelivery} />
                   <ComposerDictationButton
                     state={dictation.state}
                     onToggle={toggleDictationStable}
diff --git a/apps/web/src/components/chat/ComposerScrawlButton.tsx b/apps/web/src/components/chat/ComposerScrawlButton.tsx
new file mode 100644
index 0000000000000..61f528088dd73
--- /dev/null
+++ b/apps/web/src/components/chat/ComposerScrawlButton.tsx
@@ -0,0 +1,88 @@
+import { SCRAWL_TOGGLE_ACTION } from "@t3tools/contracts";
+import { LoaderCircleIcon, ScreenShareIcon } from "lucide-react";
+import { useEffect, useSyncExternalStore } from "react";
+
+import {
+  getScrawlState,
+  isScrawlAvailable,
+  registerScrawlDelivery,
+  subscribeScrawl,
+  toggleScrawl,
+  type ScrawlDelivery,
+} from "../../scrawl/scrawlCapture";
+import { Button } from "../ui/button";
+import { Tooltip, TooltipPopup, TooltipTrigger } from "../ui/tooltip";
+
+const NOTICE_TONES = {
+  success: "text-success",
+  warning: "text-warning",
+  error: "text-destructive",
+} as const;
+
+const SHORTCUT = "Alt+Shift+S";
+
+/** Records the screen with voice, clicks and ink. Only the desktop app can see the whole screen. */
+export function ComposerScrawlButton(props: { readonly delivery: ScrawlDelivery }) {
+  const { phase, notice } = useSyncExternalStore(subscribeScrawl, getScrawlState);
+  const { delivery } = props;
+
+  // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- offers this composer to a capture running outside React
+  useEffect(() => registerScrawlDelivery(delivery), [delivery]);
+
+  // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to the global shortcut, which arrives as a desktop menu action
+  useEffect(
+    () =>
+      window.desktopBridge?.onMenuAction((action) => {
+        if (action === SCRAWL_TOGGLE_ACTION) toggleScrawl();
+      }),
+    [],
+  );
+
+  if (!isScrawlAvailable()) return null;
+  const recording = phase === "recording";
+  const busy = phase === "starting" || phase === "building";
+  const status =
+    phase === "building"
+      ? "Building the capture"
+      : recording
+        ? `Recording this screen · ${SHORTCUT} finishes · Esc discards`
+        : null;
+  return (
+    <>
+      {status !== null || notice !== null ? (
+        <div
+          role="status"
+          aria-live="polite"
+          className="pointer-events-none absolute start-2 bottom-full z-20 mb-3 w-max max-w-[min(32rem,calc(100%-1rem))] rounded-lg border border-border bg-popover px-3 py-2 text-sm shadow-md"
+        >
+          {status !== null ? (
+            <span className={recording ? "text-destructive" : "text-warning"}>{status}</span>
+          ) : notice !== null ? (
+            <span className={NOTICE_TONES[notice.tone]}>{notice.text}</span>
+          ) : null}
+        </div>
+      ) : null}
+      <Tooltip>
+        <TooltipTrigger
+          render={
+            <Button
+              type="button"
+              variant={recording ? "destructive" : "ghost"}
+              size="icon-sm"
+              onPointerDown={(event) => event.preventDefault()}
+              onClick={toggleScrawl}
+              disabled={busy}
+              aria-label={recording ? "Finish screen capture" : "Capture the screen"}
+              aria-pressed={recording}
+            />
+          }
+        >
+          {busy ? <LoaderCircleIcon className="animate-spin" /> : <ScreenShareIcon />}
+        </TooltipTrigger>
+        <TooltipPopup>
+          {recording ? `Finish capture (${SHORTCUT})` : `Capture the screen (${SHORTCUT})`}
+        </TooltipPopup>
+      </Tooltip>
+    </>
+  );
+}
diff --git a/apps/web/src/dictation/microphoneCapture.ts b/apps/web/src/dictation/microphoneCapture.ts
index 5c76f610b8177..73514b3cb36e8 100644
--- a/apps/web/src/dictation/microphoneCapture.ts
+++ b/apps/web/src/dictation/microphoneCapture.ts
@@ -13,6 +13,8 @@ export interface MicrophoneCapture {
   /** PCM recorded since the previous call, as base64 of 16-bit little-endian samples. */
   readonly takePcm: () => string;
   readonly stop: () => void;
+  /** The raw track, for a screen recording that wants the voice too. */
+  readonly stream: MediaStream;
 }
 
 // From the app root, not the current route: /<thread>/x.js would hit the SPA fallback.
@@ -76,6 +78,7 @@ export async function startMicrophoneCapture(): Promise<MicrophoneCapture> {
   }
 
   return {
+    stream,
     takePcm: () => {
       const taken = blocks.splice(0, blocks.length);
       return encodePcm16(taken);
diff --git a/apps/web/src/dictation/useDictation.ts b/apps/web/src/dictation/useDictation.ts
index de9e12fe90548..6eae476072714 100644
--- a/apps/web/src/dictation/useDictation.ts
+++ b/apps/web/src/dictation/useDictation.ts
@@ -27,7 +27,7 @@ import { DEFAULT_DICTATION_LANGUAGE } from "./dictationLanguages";
 
 const SEND_INTERVAL_MS = 500;
 // A new key: the old one may hold a choice made when a flag still showed it, now easy to overlook.
-const LANGUAGE_KEY = "t3code:dictation-language-v2";
+export const DICTATION_LANGUAGE_KEY = "t3code:dictation-language-v2";
 
 export type DictationPhase = "idle" | "starting" | "recording" | "transcribing" | "structuring";
 
@@ -95,7 +95,7 @@ export function useDictation(input: {
   const cancelSession = useAtomCommand(serverEnvironment.dictationCancel, { reportFailure: false });
   const structure = useAtomCommand(serverEnvironment.dictationStructure, { reportFailure: false });
   const [language, setLanguage] = useLocalStorage(
-    LANGUAGE_KEY,
+    DICTATION_LANGUAGE_KEY,
     DEFAULT_DICTATION_LANGUAGE,
     Schema.String,
   );
diff --git a/apps/desktop/src/scrawl/narration.test.ts b/apps/web/src/scrawl/narration.test.ts
similarity index 79%
rename from apps/desktop/src/scrawl/narration.test.ts
rename to apps/web/src/scrawl/narration.test.ts
index e962b6d697be5..acb64dee3b179 100644
--- a/apps/desktop/src/scrawl/narration.test.ts
+++ b/apps/web/src/scrawl/narration.test.ts
@@ -1,4 +1,4 @@
-import { describe, expect, it } from "vitest";
+import { describe, expect, it } from "vite-plus/test";
 
 import {
   buildNarration,
@@ -74,6 +74,33 @@ describe("buildNarration", () => {
     );
   });
 
+  it("tells a click the screen answered from a hand that was pointing", () => {
+    const lines = buildNarration({
+      words: say("Das hier und das hier.", 0),
+      events: [
+        { kind: "click", t: 100, x: 0.1, y: 0.1 },
+        { kind: "click", t: 200, x: 0.2, y: 0.1 },
+        { kind: "click", t: 5_000, x: 0.5, y: 0.5 },
+      ],
+      marks: [],
+      durationMs: 7_000,
+      sceneChanges: [5_400],
+    });
+    expect(formatNarrationLine(lines[0]!, 0)).toContain("[pointed ×2]");
+    expect(formatNarrationLine(lines[1]!, 1)).toBe("- #2 (0:05) clicked");
+  });
+
+  it("says what a keyframe without words or actions shows", () => {
+    const lines = buildNarration({
+      words: [],
+      events: [],
+      marks: [],
+      durationMs: 3_000,
+      sceneChanges: [2_000],
+    });
+    expect(formatNarrationLine(lines[0]!, 0)).toBe("- #1 (0:02) screen changed");
+  });
+
   it("gives a click said over no speech its own line", () => {
     const lines = buildNarration({
       words: say("Hallo Welt.", 0),
diff --git a/apps/desktop/src/scrawl/narration.ts b/apps/web/src/scrawl/narration.ts
similarity index 77%
rename from apps/desktop/src/scrawl/narration.ts
rename to apps/web/src/scrawl/narration.ts
index 4e60f6f8aeac2..3cfbb57469e76 100644
--- a/apps/desktop/src/scrawl/narration.ts
+++ b/apps/web/src/scrawl/narration.ts
@@ -35,12 +35,14 @@ export type NarrationAction =
       readonly t0: number;
       readonly t1: number;
       readonly count: number;
-      readonly x: number;
-      readonly y: number;
+      readonly points: ReadonlyArray<readonly [number, number]>;
+      /** False when the screen did not react: the hand was pointing, not operating anything. */
+      readonly changed?: boolean;
     }
   | { readonly kind: "drag"; readonly t0: number; readonly t1: number; readonly direction: string }
   | { readonly kind: "scroll"; readonly t0: number; readonly t1: number }
-  | { readonly kind: "mark"; readonly t0: number; readonly t1: number; readonly id: string };
+  | { readonly kind: "mark"; readonly t0: number; readonly t1: number; readonly id: string }
+  | { readonly kind: "scene"; readonly t0: number; readonly t1: number };
 
 export interface NarrationLine {
   readonly t0: number;
@@ -63,6 +65,7 @@ interface Sentence {
 const ABANDONED_GAP_MS = 5_000;
 const MAX_SENTENCE_WORDS = 60;
 const BURST_GAP_MS = 1_500;
+const REACTION_MS = 1_000;
 const SCROLL_GAP_MS = 1_500;
 // Scrawl's rule: a drag this short (as a share of the screen) is the hand moving, not an act.
 const SHORT_DRAG = 0.15;
@@ -79,6 +82,7 @@ const SCORES: Record<NarrationAction["kind"], number> = {
   mark: 100,
   clicks: 60,
   drag: 50,
+  scene: 40,
   scroll: 30,
 };
 
@@ -123,31 +127,27 @@ export function foldActions(
     t1: mark.t1,
     id: mark.id,
   }));
-  let burst: { t0: number; t1: number; count: number; x: number; y: number } | null = null;
-  let scroll: { t0: number; t1: number } | null = null;
-  const closeBurst = () => {
-    if (burst !== null) actions.push({ kind: "clicks", ...burst });
-    burst = null;
-  };
-  const closeScroll = () => {
-    if (scroll !== null) actions.push({ kind: "scroll", ...scroll });
-    scroll = null;
-  };
+  const bursts: Array<{
+    t0: number;
+    t1: number;
+    count: number;
+    points: Array<readonly [number, number]>;
+  }> = [];
+  const scrolls: Array<{ t0: number; t1: number }> = [];
   for (const event of [...events].sort((a, b) => a.t - b.t)) {
     if (event.kind === "click") {
-      if (burst !== null && event.t - burst.t1 <= BURST_GAP_MS) {
-        burst = { ...burst, t1: event.t, count: burst.count + 1, x: event.x, y: event.y };
+      const burst = bursts.at(-1);
+      if (burst !== undefined && event.t - burst.t1 <= BURST_GAP_MS) {
+        burst.t1 = event.t;
+        burst.count++;
+        burst.points.push([event.x, event.y]);
       } else {
-        closeBurst();
-        burst = { t0: event.t, t1: event.t, count: 1, x: event.x, y: event.y };
+        bursts.push({ t0: event.t, t1: event.t, count: 1, points: [[event.x, event.y]] });
       }
     } else if (event.kind === "scroll") {
-      if (scroll !== null && event.t - scroll.t1 <= SCROLL_GAP_MS) {
-        scroll = { ...scroll, t1: event.t };
-      } else {
-        closeScroll();
-        scroll = { t0: event.t, t1: event.t };
-      }
+      const scroll = scrolls.at(-1);
+      if (scroll !== undefined && event.t - scroll.t1 <= SCROLL_GAP_MS) scroll.t1 = event.t;
+      else scrolls.push({ t0: event.t, t1: event.t });
     } else {
       const dx = event.x1 - event.x0;
       const dy = event.y1 - event.y0;
@@ -161,8 +161,8 @@ export function foldActions(
       }
     }
   }
-  closeBurst();
-  closeScroll();
+  for (const burst of bursts) actions.push({ kind: "clicks", ...burst });
+  for (const scroll of scrolls) actions.push({ kind: "scroll", ...scroll });
   return actions.sort((a, b) => a.t0 - b.t0);
 }
 
@@ -233,14 +233,39 @@ const frameTimeOf = (t0: number, actions: ReadonlyArray<NarrationAction>, durati
   return Math.min(moment.t1 + settle, durationMs);
 };
 
+/** Clicks the screen answered stay clicks; the rest are marked as pointing. */
+const withReactions = (
+  actions: ReadonlyArray<NarrationAction>,
+  sceneChanges: ReadonlyArray<number>,
+): Array<NarrationAction> => {
+  const clickTimes = actions.filter((action) => action.kind === "clicks");
+  const reacted = (action: { readonly t0: number; readonly t1: number }) =>
+    sceneChanges.some((t) => t >= action.t0 && t <= action.t1 + REACTION_MS);
+  const scenes: Array<NarrationAction> = sceneChanges
+    .filter((t) => !clickTimes.some((a) => t >= a.t0 && t <= a.t1 + REACTION_MS))
+    .map((t) => ({ kind: "scene", t0: t, t1: t }));
+  return [
+    ...actions.map((action) =>
+      action.kind === "clicks" ? { ...action, changed: reacted(action) } : action,
+    ),
+    ...scenes,
+  ].toSorted((a, b) => a.t0 - b.t0);
+};
+
 export function buildNarration(input: {
   readonly words: ReadonlyArray<TimedWord>;
   readonly events: ReadonlyArray<CaptureEvent>;
   readonly marks: ReadonlyArray<CaptureMark>;
   readonly durationMs: number;
+  /** When the screen changed; without it clicks are taken at face value. */
+  readonly sceneChanges?: ReadonlyArray<number>;
 }): Array<NarrationLine> {
   const sentences = splitSentences(input.words);
-  attach(sentences, foldActions(input.events, input.marks));
+  const folded = foldActions(input.events, input.marks);
+  attach(
+    sentences,
+    input.sceneChanges === undefined ? folded : withReactions(folded, input.sceneChanges),
+  );
   const owners = chooseFrameOwners(sentences, input.durationMs);
   const groups: Array<Array<Sentence>> = [];
   let ownerSeen = false;
@@ -272,18 +297,22 @@ const formatClock = (ms: number) => {
 const describeActions = (actions: ReadonlyArray<NarrationAction>) => {
   let marks = 0;
   let clicks = 0;
+  let pointed = 0;
   let scrolled = false;
   const drags = new Set<string>();
   for (const action of actions) {
     if (action.kind === "mark") marks++;
-    else if (action.kind === "clicks") clicks += action.count;
-    else if (action.kind === "scroll") scrolled = true;
-    else drags.add(action.direction);
+    else if (action.kind === "clicks") {
+      if (action.changed === false) pointed += action.count;
+      else clicks += action.count;
+    } else if (action.kind === "scroll") scrolled = true;
+    else if (action.kind === "drag") drags.add(action.direction);
   }
   const counted = (word: string, count: number) => (count > 1 ? `${word} ×${count}` : word);
   return [
     marks > 0 ? counted("marked", marks) : null,
     clicks > 0 ? counted("clicked", clicks) : null,
+    pointed > 0 ? counted("pointed", pointed) : null,
     ...[...drags].map((direction) => `dragged ${direction}`),
     scrolled ? "scrolled" : null,
   ]
@@ -295,6 +324,6 @@ const describeActions = (actions: ReadonlyArray<NarrationAction>) => {
 export function formatNarrationLine(line: NarrationLine, index: number): string {
   const head = `- #${index + 1} (${formatClock(line.frameT)})`;
   const actions = describeActions(line.actions);
-  if (line.text === "") return `${head} ${actions}`.trimEnd();
+  if (line.text === "") return `${head} ${actions === "" ? "screen changed" : actions}`;
   return `${head}${actions === "" ? "" : ` [${actions}]`} “${line.text}”`;
 }
diff --git a/apps/web/src/scrawl/sceneChanges.test.ts b/apps/web/src/scrawl/sceneChanges.test.ts
new file mode 100644
index 0000000000000..a627d78832cdd
--- /dev/null
+++ b/apps/web/src/scrawl/sceneChanges.test.ts
@@ -0,0 +1,29 @@
+import { describe, expect, it } from "vite-plus/test";
+
+import { findSceneChanges, THUMB_HEIGHT, THUMB_WIDTH, withoutInkChanges } from "./sceneChanges.ts";
+
+const blank = () => new Float32Array(THUMB_WIDTH * THUMB_HEIGHT);
+
+describe("withoutInkChanges", () => {
+  it("drops changes from drawing and fading ink, and keeps the rest", () => {
+    const strokes = [{ t0: 2_000, t1: 3_000 }];
+    expect(withoutInkChanges([1_000, 2_500, 3_500, 6_000, 8_000, 9_000], strokes, 5_000)).toEqual([
+      1_000, 6_000, 9_000,
+    ]);
+  });
+});
+
+describe("findSceneChanges", () => {
+  it("ignores a moving cursor and finds a real transition", () => {
+    const cursor = blank();
+    cursor.fill(1, 10, 14);
+    const page = blank().fill(0.6);
+    expect(
+      findSceneChanges([
+        { t: 0, luma: blank() },
+        { t: 750, luma: cursor },
+        { t: 1_500, luma: page },
+      ]),
+    ).toEqual([1_500]);
+  });
+});
diff --git a/apps/web/src/scrawl/sceneChanges.ts b/apps/web/src/scrawl/sceneChanges.ts
new file mode 100644
index 0000000000000..7f5035feb5258
--- /dev/null
+++ b/apps/web/src/scrawl/sceneChanges.ts
@@ -0,0 +1,64 @@
+// Finds moments the screen changed from small luma thumbnails, so nothing but a 64x36 grid is kept
+// per sample. Scrawl's rule: a moving cursor spans at most a few of the 8x6 blocks, so the score is
+// the fourth-worst block, and real transitions across at least four blocks still register.
+
+export const THUMB_WIDTH = 64;
+export const THUMB_HEIGHT = 36;
+const COLUMNS = 8;
+const ROWS = 6;
+const CHANGE_THRESHOLD = 0.03;
+
+export interface LumaSample {
+  readonly t: number;
+  readonly luma: Float32Array;
+}
+
+function sceneDifference(a: Float32Array, b: Float32Array): number {
+  const blockWidth = THUMB_WIDTH / COLUMNS;
+  const blockHeight = THUMB_HEIGHT / ROWS;
+  let global = 0;
+  const blocks: Array<number> = [];
+  for (let row = 0; row < ROWS; row++) {
+    for (let column = 0; column < COLUMNS; column++) {
+      let sum = 0;
+      for (let y = row * blockHeight; y < (row + 1) * blockHeight; y++) {
+        for (let x = column * blockWidth; x < (column + 1) * blockWidth; x++) {
+          const index = y * THUMB_WIDTH + x;
+          sum += Math.abs(a[index]! - b[index]!);
+        }
+      }
+      global += sum;
+      blocks.push(sum / (blockWidth * blockHeight));
+    }
+  }
+  blocks.sort((x, y) => y - x);
+  return Math.max(global / (THUMB_WIDTH * THUMB_HEIGHT), Math.min(1, blocks[3]! * 1.4));
+}
+
+// One sample either way: a change is only seen at the next sample after it happened.
+const INK_MARGIN_MS = 750;
+
+/** Drawing and fading ink changes the picture without the screen doing anything. */
+export function withoutInkChanges(
+  changes: ReadonlyArray<number>,
+  strokes: ReadonlyArray<{ readonly t0: number; readonly t1: number }>,
+  inkHoldMs: number,
+): Array<number> {
+  return changes.filter(
+    (t) =>
+      !strokes.some(
+        ({ t0, t1 }) =>
+          (t >= t0 - INK_MARGIN_MS && t <= t1 + INK_MARGIN_MS) ||
+          Math.abs(t - (t1 + inkHoldMs)) <= INK_MARGIN_MS,
+      ),
+  );
+}
+
+export function findSceneChanges(samples: ReadonlyArray<LumaSample>): Array<number> {
+  const changes: Array<number> = [];
+  for (let index = 1; index < samples.length; index++) {
+    const difference = sceneDifference(samples[index - 1]!.luma, samples[index]!.luma);
+    if (difference >= CHANGE_THRESHOLD) changes.push(samples[index]!.t);
+  }
+  return changes;
+}
diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts
new file mode 100644
index 0000000000000..a47af4114d75b
--- /dev/null
+++ b/apps/web/src/scrawl/scrawlBundle.ts
@@ -0,0 +1,413 @@
+// Builds what an agent reads from a finished recording: keyframes taken from the video, the contact
+// sheet, the summary text and the manifest. The folder keeps Scrawl's shape, so the Scrawl skill and
+// the `scrawl` CLI (zoom, frames, export) read these bundles as well.
+
+import {
+  SCRAWL_INK_HOLD_MS,
+  type ScrawlStartResult,
+  type ScrawlStopResult,
+  type ScrawlStroke,
+} from "@t3tools/contracts";
+
+import {
+  buildNarration,
+  formatNarrationLine,
+  type CaptureMark,
+  type NarrationLine,
+  type TimedWord,
+} from "./narration";
+import { withoutInkChanges } from "./sceneChanges";
+
+const SHEET_LONG_EDGE = 1600;
+const SHEET_GUTTER = 6;
+const STROKE_GROUP_MS = 1_500;
+const CROP_PADDING = 0.08;
+// Cropping to most of the screen only loses context.
+const MAX_CROP_AREA = 0.7;
+const MIN_CROP_SIDE = 0.35;
+
+export interface CaptureRecording {
+  readonly session: ScrawlStartResult;
+  readonly input: ScrawlStopResult;
+  readonly words: ReadonlyArray<TimedWord>;
+  readonly transcriptText: string;
+  readonly model: string;
+  readonly video: Blob;
+  readonly videoName: string;
+  /** Milliseconds between the start of the audio, which all times count from, and of the video. */
+  readonly videoOffsetMs: number;
+  /** How far the input clock runs ahead of the audio clock. */
+  readonly inputOffsetMs: number;
+  readonly durationMs: number;
+  readonly sceneChanges: ReadonlyArray<number>;
+  readonly wav: Uint8Array;
+}
+
+interface ManifestMark extends CaptureMark {
+  readonly strokes: ReadonlyArray<number>;
+  readonly boxNorm: readonly [number, number, number, number];
+  readonly shape: string;
+  readonly description: string;
+}
+
+// Scrawl stores flat [x, y, dt] triples with positions in 0..=10000.
+const STROKE_SCALE = 10_000;
+const manifestStroke = (stroke: ScrawlStroke, width: number, height: number) => {
+  const count = stroke.points.length / 2;
+  const span = stroke.t1 - stroke.t0;
+  const pts: Array<number> = [];
+  for (let index = 0; index < count; index++) {
+    pts.push(
+      Math.round(stroke.points[index * 2]! * STROKE_SCALE),
+      Math.round(stroke.points[index * 2 + 1]! * STROKE_SCALE),
+      count > 1 ? Math.round((span * index) / (count - 1)) : 0,
+    );
+  }
+  return { t0: stroke.t0, t1: stroke.t1, fw: width, fh: height, pts };
+};
+
+const KEYFRAME_REASONS: Record<NarrationLine["actions"][number]["kind"], string> = {
+  mark: "mark",
+  clicks: "click",
+  drag: "click",
+  scene: "scene_change",
+  scroll: "scene_change",
+};
+
+export interface BuiltBundle {
+  readonly text: string;
+  readonly contactSheet: Blob;
+  readonly lines: ReadonlyArray<NarrationLine>;
+}
+
+const strokesToMarks = (strokes: ReadonlyArray<ScrawlStroke>): Array<ManifestMark> => {
+  const groups: Array<{ t0: number; t1: number; strokes: Array<number> }> = [];
+  strokes.forEach((stroke, index) => {
+    const open = groups.at(-1);
+    if (open !== undefined && stroke.t0 - open.t1 <= STROKE_GROUP_MS) {
+      open.t1 = stroke.t1;
+      open.strokes.push(index);
+    } else {
+      groups.push({ t0: stroke.t0, t1: stroke.t1, strokes: [index] });
+    }
+  });
+  return groups.map((group, index) => {
+    const xs: Array<number> = [];
+    const ys: Array<number> = [];
+    for (const strokeIndex of group.strokes) {
+      strokes[strokeIndex]!.points.forEach((value, at) => (at % 2 === 0 ? xs : ys).push(value));
+    }
+    return {
+      id: `m${index + 1}`,
+      ...group,
+      boxNorm: [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)],
+      shape: "mark",
+      description: `marked (${group.strokes.length} ${group.strokes.length === 1 ? "stroke" : "strokes"})`,
+    };
+  });
+};
+
+/** Everything the hand touched, padded, or null when that is most of the screen anyway. */
+function activityBox(
+  recording: Pick<CaptureRecording, "input">,
+): {
+  readonly x: number;
+  readonly y: number;
+  readonly width: number;
+  readonly height: number;
+} | null {
+  const xs: Array<number> = [];
+  const ys: Array<number> = [];
+  for (const event of recording.input.events) {
+    if (event.kind === "click") {
+      xs.push(event.x);
+      ys.push(event.y);
+    } else {
+      xs.push(event.x0, event.x1);
+      ys.push(event.y0, event.y1);
+    }
+  }
+  for (const stroke of recording.input.strokes) {
+    stroke.points.forEach((value, index) => (index % 2 === 0 ? xs : ys).push(value));
+  }
+  if (xs.length === 0) return null;
+  const widen = (low: number, high: number) => {
+    let start = low - CROP_PADDING;
+    let end = high + CROP_PADDING;
+    if (end - start < MIN_CROP_SIDE) {
+      const middle = (start + end) / 2;
+      start = middle - MIN_CROP_SIDE / 2;
+      end = middle + MIN_CROP_SIDE / 2;
+    }
+    start = Math.max(0, start);
+    end = Math.min(1, end);
+    return [start, end - start] as const;
+  };
+  const [x, width] = widen(Math.min(...xs), Math.max(...xs));
+  const [y, height] = widen(Math.min(...ys), Math.max(...ys));
+  return width * height > MAX_CROP_AREA ? null : { x, y, width, height };
+}
+
+const seekTo = (video: HTMLVideoElement, seconds: number) =>
+  new Promise<void>((resolve, reject) => {
+    const onSeeked = () => {
+      video.removeEventListener("seeked", onSeeked);
+      video.removeEventListener("error", onError);
+      resolve();
+    };
+    const onError = () => {
+      video.removeEventListener("seeked", onSeeked);
+      video.removeEventListener("error", onError);
+      reject(new Error("The recording cannot be read."));
+    };
+    video.addEventListener("seeked", onSeeked);
+    video.addEventListener("error", onError);
+    video.currentTime = seconds;
+  });
+
+const loadVideo = async (blob: Blob) => {
+  const video = document.createElement("video");
+  video.muted = true;
+  video.preload = "auto";
+  const url = URL.createObjectURL(blob);
+  video.src = url;
+  await new Promise<void>((resolve, reject) => {
+    video.onloadedmetadata = () => resolve();
+    video.onerror = () => reject(new Error("The recording cannot be read."));
+  });
+  // A recorder's file does not state its length until the player has walked to the end.
+  if (!Number.isFinite(video.duration)) {
+    await seekTo(video, 1e7);
+  }
+  return { video, dispose: () => URL.revokeObjectURL(url) };
+};
+
+const toBlob = (canvas: HTMLCanvasElement) =>
+  new Promise<Blob>((resolve, reject) => {
+    canvas.toBlob(
+      (blob) => (blob === null ? reject(new Error("A frame cannot be encoded.")) : resolve(blob)),
+      "image/png",
+    );
+  });
+
+const drawClickRings = (
+  context: CanvasRenderingContext2D,
+  line: NarrationLine,
+  width: number,
+  height: number,
+) => {
+  context.lineWidth = Math.max(2, width / 700);
+  context.strokeStyle = "rgba(239, 68, 68, 0.85)";
+  for (const action of line.actions) {
+    if (action.kind !== "clicks") continue;
+    for (const [x, y] of action.points) {
+      context.beginPath();
+      context.arc(x * width, y * height, Math.max(10, width / 90), 0, Math.PI * 2);
+      context.stroke();
+    }
+  }
+};
+
+const drawChip = (context: CanvasRenderingContext2D, label: string, height: number) => {
+  // Blind reading tests in Scrawl: the number needs its own white square to be read as a number.
+  const size = Math.max(24, height / 18);
+  context.fillStyle = "#ffffff";
+  context.strokeStyle = "#111111";
+  context.lineWidth = 2;
+  context.beginPath();
+  context.roundRect(5, 5, size, size, 4);
+  context.fill();
+  context.stroke();
+  context.fillStyle = "#000000";
+  context.font = `700 ${Math.round(size * 0.68)}px system-ui`;
+  context.textAlign = "center";
+  context.textBaseline = "middle";
+  context.fillText(label, 5 + size / 2, 5 + size / 2 + 1);
+};
+
+/** Frames in time order in justified rows, each with its number. */
+const drawContactSheet = (frames: ReadonlyArray<HTMLCanvasElement>) => {
+  const columns = Math.max(1, Math.ceil(Math.sqrt(frames.length)));
+  const rows = Math.ceil(frames.length / columns);
+  const first = frames[0]!;
+  const aspect = first.width / first.height;
+  const landscape = aspect >= 1;
+  const cellWidth = landscape
+    ? Math.floor((SHEET_LONG_EDGE - SHEET_GUTTER * (columns + 1)) / columns)
+    : Math.floor((SHEET_LONG_EDGE / aspect - SHEET_GUTTER * (columns + 1)) / columns);
+  const cellHeight = Math.round(cellWidth / aspect);
+  const sheet = document.createElement("canvas");
+  sheet.width = columns * cellWidth + (columns + 1) * SHEET_GUTTER;
+  sheet.height = rows * cellHeight + (rows + 1) * SHEET_GUTTER;
+  const context = sheet.getContext("2d")!;
+  context.fillStyle = "#101014";
+  context.fillRect(0, 0, sheet.width, sheet.height);
+  frames.forEach((frame, index) => {
+    const x = SHEET_GUTTER + (index % columns) * (cellWidth + SHEET_GUTTER);
+    const y = SHEET_GUTTER + Math.floor(index / columns) * (cellHeight + SHEET_GUTTER);
+    context.save();
+    context.translate(x, y);
+    context.drawImage(frame, 0, 0, cellWidth, cellHeight);
+    drawChip(context, String(index + 1), cellHeight);
+    context.restore();
+  });
+  return sheet;
+};
+
+const formatDuration = (ms: number) => {
+  const seconds = Math.round(ms / 1000);
+  return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`;
+};
+
+const summaryText = (
+  recording: Pick<CaptureRecording, "session" | "durationMs">,
+  lines: ReadonlyArray<NarrationLine>,
+  cropped: boolean,
+) => {
+  const { app, windowTitle, bundlePath } = recording.session;
+  const source = [app, windowTitle ? `“${windowTitle}”` : ""].filter(Boolean).join(" ") || "Screen";
+  return [
+    `[scrawl ${formatDuration(recording.durationMs)} · ${source} · ${bundlePath}]`,
+    "Open contact-sheet.png and frames/ in that folder before changing anything: the marks and words only make sense on the images, which can show more than the text does.",
+    ...(cropped ? ["Cropped to: the activity (full frames in frames-full/)"] : []),
+    ...lines.map(formatNarrationLine),
+  ].join("\n");
+};
+
+export async function buildBundle(
+  recording: CaptureRecording,
+  write: (path: string, bytes: Uint8Array) => Promise<void>,
+): Promise<BuiltBundle> {
+  const shift = (t: number) => Math.max(0, t - recording.inputOffsetMs);
+  const events = recording.input.events.map((event) =>
+    event.kind === "click"
+      ? { ...event, t: shift(event.t) }
+      : { ...event, t: shift(event.t), t1: shift(event.t1) },
+  );
+  const strokes = recording.input.strokes.map((stroke) => ({
+    ...stroke,
+    t0: shift(stroke.t0),
+    t1: shift(stroke.t1),
+  }));
+  const marks = strokesToMarks(strokes);
+  const lines = buildNarration({
+    words: recording.words,
+    events,
+    marks,
+    durationMs: recording.durationMs,
+    sceneChanges: withoutInkChanges(recording.sceneChanges, strokes, SCRAWL_INK_HOLD_MS),
+  });
+  const box = activityBox({ input: { events, strokes } });
+
+  const { video, dispose } = await loadVideo(recording.video);
+  const frames: Array<HTMLCanvasElement> = [];
+  const fullFrames: Array<HTMLCanvasElement> = [];
+  const bytesOf = async (blob: Blob) => new Uint8Array(await blob.arrayBuffer());
+  try {
+    for (const [index, line] of lines.entries()) {
+      const seconds = Math.max(0, (line.frameT - recording.videoOffsetMs) / 1000);
+      await seekTo(video, Math.min(seconds, Math.max(0, video.duration - 0.05)));
+      const full = document.createElement("canvas");
+      full.width = video.videoWidth;
+      full.height = video.videoHeight;
+      const context = full.getContext("2d")!;
+      context.drawImage(video, 0, 0);
+      drawClickRings(context, line, full.width, full.height);
+      const name = `frame-${String(index + 1).padStart(2, "0")}.png`;
+      if (box === null) {
+        await write(`frames/${name}`, await bytesOf(await toBlob(full)));
+        frames.push(full);
+        continue;
+      }
+      await write(`frames-full/${name}`, await bytesOf(await toBlob(full)));
+      fullFrames.push(full);
+      const cropped = document.createElement("canvas");
+      cropped.width = Math.round(box.width * full.width);
+      cropped.height = Math.round(box.height * full.height);
+      cropped
+        .getContext("2d")!
+        .drawImage(full, -Math.round(box.x * full.width), -Math.round(box.y * full.height));
+      await write(`frames/${name}`, await bytesOf(await toBlob(cropped)));
+      frames.push(cropped);
+    }
+  } finally {
+    dispose();
+  }
+
+  const sheet = drawContactSheet(frames);
+  const contactSheet = await toBlob(sheet);
+  await write("contact-sheet.png", await bytesOf(contactSheet));
+  if (fullFrames.length > 0) {
+    await write(
+      "contact-sheet-full.png",
+      await bytesOf(await toBlob(drawContactSheet(fullFrames))),
+    );
+  }
+  await write("audio.wav", recording.wav);
+
+  const text = summaryText(recording, lines, box !== null);
+  const encoder = new TextEncoder();
+  await write("summary.md", encoder.encode(`${text}\n`));
+  const { width, height } = recording.session;
+  const manifest = {
+    version: 1,
+    id: recording.session.id,
+    createdAtMs: Date.now(),
+    durationMs: recording.durationMs,
+    target: {
+      mode: "display",
+      app: recording.session.app,
+      windowTitle: recording.session.windowTitle,
+      displayId: 0,
+      frame: { x: 0, y: 0, width, height, scale: 1 },
+    },
+    transcript: {
+      source: recording.model,
+      model: recording.model,
+      text: recording.transcriptText,
+      segments: [],
+      words: recording.words,
+    },
+    strokes: strokes.map((stroke) => manifestStroke(stroke, width, height)),
+    marks,
+    events: events.map((event) => (event.kind === "click" ? { ...event, button: "left" } : event)),
+    keyframes: lines.map((line, index) => {
+      const main = line.actions.find((action) => action.kind === "mark") ?? line.actions.at(-1);
+      return {
+        index: index + 1,
+        t: Math.round(line.frameT),
+        reason: main === undefined ? "speech" : KEYFRAME_REASONS[main.kind],
+        file: `frames/frame-${String(index + 1).padStart(2, "0")}.png`,
+        event: main?.kind ?? "speech",
+        said: line.text,
+      };
+    }),
+    contactSheet: "contact-sheet.png",
+    video: recording.videoName,
+    videoOffsetMs: recording.videoOffsetMs,
+    audio: "audio.wav",
+    windows: [],
+    cursor: [],
+    ...(box === null
+      ? {}
+      : {
+          autocrop: {
+            label: "the activity",
+            fullFrames: "frames-full",
+            fullContactSheet: "contact-sheet-full.png",
+            crops: lines.map((_line, index) => ({
+              index: index + 1,
+              x: Math.round(box.x * width),
+              y: Math.round(box.y * height),
+              width: Math.round(box.width * width),
+              height: Math.round(box.height * height),
+              kind: "focus",
+              reason: "around the activity",
+              apps: [recording.session.app],
+            })),
+          },
+        }),
+  };
+  await write("scrawl.json", encoder.encode(JSON.stringify(manifest)));
+  return { text, contactSheet, lines };
+}
diff --git a/apps/web/src/scrawl/scrawlCapture.ts b/apps/web/src/scrawl/scrawlCapture.ts
new file mode 100644
index 0000000000000..0ae199d4c78cc
--- /dev/null
+++ b/apps/web/src/scrawl/scrawlCapture.ts
@@ -0,0 +1,374 @@
+// One screen capture at a time, kept outside React: the shortcut works from any other program, and
+// switching threads while recording must not end it. Whichever composer is on screen at the end
+// receives the result.
+
+import type { DesktopBridge, ScrawlStartResult } from "@t3tools/contracts";
+import * as Schema from "effect/Schema";
+
+import { startMicrophoneCapture, type MicrophoneCapture } from "../dictation/microphoneCapture";
+import { DEFAULT_DICTATION_LANGUAGE } from "../dictation/dictationLanguages";
+import { DICTATION_LANGUAGE_KEY } from "../dictation/useDictation";
+import { getLocalStorageItem } from "../hooks/useLocalStorage";
+import { randomUUID } from "../lib/utils";
+import { buildBundle } from "./scrawlBundle";
+import { findSceneChanges, THUMB_HEIGHT, THUMB_WIDTH, type LumaSample } from "./sceneChanges";
+
+const SEND_INTERVAL_MS = 500;
+const SAMPLE_INTERVAL_MS = 750;
+const VIDEO_TIMESLICE_MS = 2_000;
+// Keyframes come out of this video, so small text has to survive it.
+const VIDEO_BITS_PER_SECOND = 6_000_000;
+const MAX_FRAME_RATE = 15;
+
+export type ScrawlPhase = "idle" | "starting" | "recording" | "building";
+
+export interface ScrawlState {
+  readonly phase: ScrawlPhase;
+  readonly notice: { readonly text: string; readonly tone: "error" | "warning" | "success" } | null;
+}
+
+export interface ScrawlDelivery {
+  readonly insert: (text: string) => boolean;
+  readonly attachImage: (file: File) => Promise<boolean>;
+}
+
+interface ImageCaptureLike {
+  grabFrame(): Promise<ImageBitmap>;
+}
+declare const ImageCapture: new (track: MediaStreamTrack) => ImageCaptureLike;
+
+interface Running {
+  readonly session: ScrawlStartResult;
+  readonly sessionId: string;
+  readonly screen: MediaStream;
+  readonly mic: MicrophoneCapture;
+  readonly recorder: MediaRecorder;
+  readonly videoName: string;
+  readonly chunks: Array<Blob>;
+  readonly samples: Array<LumaSample>;
+  readonly pcm: Array<string>;
+  readonly micStartedAt: number;
+  readonly language: string;
+  readonly pcmTimer: ReturnType<typeof setInterval>;
+  readonly sampleTimer: ReturnType<typeof setInterval>;
+  videoStartedAt: number;
+  writes: Promise<void>;
+  sending: Promise<void>;
+}
+
+const IDLE: ScrawlState = { phase: "idle", notice: null };
+
+let state: ScrawlState = IDLE;
+let running: Running | null = null;
+let delivery: ScrawlDelivery | null = null;
+let noticeTimer: ReturnType<typeof setTimeout> | null = null;
+const listeners = new Set<() => void>();
+
+const setState = (next: ScrawlState) => {
+  state = next;
+  for (const listener of listeners) listener();
+};
+
+const notify = (notice: NonNullable<ScrawlState["notice"]>) => {
+  if (noticeTimer !== null) clearTimeout(noticeTimer);
+  setState({ phase: "idle", notice });
+  noticeTimer = setTimeout(() => {
+    noticeTimer = null;
+    if (state.notice === notice) setState({ ...state, notice: null });
+  }, 8_000);
+};
+
+const describe = (error: unknown) => (error instanceof Error ? error.message : String(error));
+
+export const subscribeScrawl = (listener: () => void) => {
+  listeners.add(listener);
+  return () => listeners.delete(listener);
+};
+export const getScrawlState = () => state;
+
+/** The composer on screen takes the result; the latest to mount wins, as the visible one did. */
+export const registerScrawlDelivery = (target: ScrawlDelivery) => {
+  delivery = target;
+  return () => {
+    if (delivery === target) delivery = null;
+  };
+};
+
+const scrawlBridge = (): DesktopBridge | null =>
+  window.desktopBridge?.scrawlStart === undefined ? null : window.desktopBridge;
+
+export const isScrawlAvailable = () => scrawlBridge() !== null;
+
+const openScreen = (sourceId: string) =>
+  navigator.mediaDevices.getUserMedia({
+    audio: false,
+    video: {
+      mandatory: {
+        chromeMediaSource: "desktop",
+        chromeMediaSourceId: sourceId,
+        maxFrameRate: MAX_FRAME_RATE,
+      },
+    },
+  } as unknown as MediaStreamConstraints);
+
+const chooseVideoFormat = () =>
+  MediaRecorder.isTypeSupported("video/mp4;codecs=avc1.640032")
+    ? { mimeType: "video/mp4;codecs=avc1.640032", name: "recording.mp4" }
+    : { mimeType: "video/webm;codecs=vp9", name: "recording.webm" };
+
+/** Keeps one small grey thumbnail per sample; nothing at full size stays in memory. */
+const startSampling = (track: MediaStreamTrack, startedAt: number, samples: Array<LumaSample>) => {
+  const grabber = new ImageCapture(track);
+  const canvas = document.createElement("canvas");
+  canvas.width = THUMB_WIDTH;
+  canvas.height = THUMB_HEIGHT;
+  const context = canvas.getContext("2d", { willReadFrequently: true })!;
+  let busy = false;
+  return setInterval(() => {
+    if (busy) return;
+    busy = true;
+    void grabber
+      .grabFrame()
+      .then((frame) => {
+        const t = Date.now() - startedAt;
+        context.drawImage(frame, 0, 0, THUMB_WIDTH, THUMB_HEIGHT);
+        frame.close();
+        const { data } = context.getImageData(0, 0, THUMB_WIDTH, THUMB_HEIGHT);
+        const luma = new Float32Array(THUMB_WIDTH * THUMB_HEIGHT);
+        for (let index = 0; index < luma.length; index++) {
+          const offset = index * 4;
+          luma[index] =
+            (0.2126 * data[offset]! + 0.7152 * data[offset + 1]! + 0.0722 * data[offset + 2]!) /
+            255;
+        }
+        samples.push({ t, luma });
+      })
+      // A frame missed while the track starts or ends only thins the samples.
+      .catch(() => undefined)
+      .finally(() => {
+        busy = false;
+      });
+  }, SAMPLE_INTERVAL_MS);
+};
+
+const decodeBase64 = (value: string) => {
+  const binary = atob(value);
+  const bytes = new Uint8Array(binary.length);
+  for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index);
+  return bytes;
+};
+
+/** 16 kHz mono 16-bit, as the dictation pipeline records it. */
+const encodeWav = (pieces: ReadonlyArray<Uint8Array>) => {
+  const length = pieces.reduce((sum, piece) => sum + piece.length, 0);
+  const wav = new Uint8Array(44 + length);
+  const view = new DataView(wav.buffer);
+  const ascii = (offset: number, text: string) => {
+    for (let index = 0; index < text.length; index++) {
+      view.setUint8(offset + index, text.charCodeAt(index));
+    }
+  };
+  ascii(0, "RIFF");
+  view.setUint32(4, 36 + length, true);
+  ascii(8, "WAVEfmt ");
+  view.setUint32(16, 16, true);
+  view.setUint16(20, 1, true);
+  view.setUint16(22, 1, true);
+  view.setUint32(24, 16_000, true);
+  view.setUint32(28, 32_000, true);
+  view.setUint16(32, 2, true);
+  view.setUint16(34, 16, true);
+  ascii(36, "data");
+  view.setUint32(40, length, true);
+  let offset = 44;
+  for (const piece of pieces) {
+    wav.set(piece, offset);
+    offset += piece.length;
+  }
+  return wav;
+};
+
+const sendPcm = (bridge: DesktopBridge, capture: Running, pcm: string) => {
+  capture.pcm.push(pcm);
+  capture.sending = capture.sending.then(async () => {
+    // The recording goes on without words when the speech model is missing; the end says so.
+    await bridge
+      .dictationAppend?.({ sessionId: capture.sessionId, language: capture.language, pcm })
+      .catch(() => undefined);
+  });
+};
+
+const start = async (bridge: DesktopBridge) => {
+  setState({ phase: "starting", notice: null });
+  let session: ScrawlStartResult | null = null;
+  let screen: MediaStream | null = null;
+  let mic: MicrophoneCapture | null = null;
+  try {
+    session = await bridge.scrawlStart!();
+    screen = await openScreen(session.sourceId);
+    mic = await startMicrophoneCapture();
+    const micStartedAt = Date.now();
+    const format = chooseVideoFormat();
+    const recorder = new MediaRecorder(
+      new MediaStream([...screen.getVideoTracks(), ...mic.stream.getAudioTracks()]),
+      { mimeType: format.mimeType, videoBitsPerSecond: VIDEO_BITS_PER_SECOND },
+    );
+    const samples: Array<LumaSample> = [];
+    const capture: Running = {
+      session,
+      sessionId: randomUUID(),
+      screen,
+      mic,
+      recorder,
+      videoName: format.name,
+      chunks: [],
+      samples,
+      pcm: [],
+      micStartedAt,
+      language:
+        getLocalStorageItem(DICTATION_LANGUAGE_KEY, Schema.String) ?? DEFAULT_DICTATION_LANGUAGE,
+      pcmTimer: setInterval(() => {
+        if (running !== null) sendPcm(bridge, running, running.mic.takePcm());
+      }, SEND_INTERVAL_MS),
+      sampleTimer: startSampling(screen.getVideoTracks()[0]!, micStartedAt, samples),
+      videoStartedAt: micStartedAt,
+      writes: Promise.resolve(),
+      sending: Promise.resolve(),
+    };
+    recorder.onstart = () => {
+      capture.videoStartedAt = Date.now();
+    };
+    recorder.ondataavailable = (event) => {
+      if (event.data.size === 0) return;
+      capture.chunks.push(event.data);
+      // Written as it comes, so a crash keeps the video recorded so far.
+      capture.writes = capture.writes.then(async () => {
+        await bridge.scrawlAppendVideo?.({
+          id: capture.session.id,
+          name: capture.videoName,
+          bytes: new Uint8Array(await event.data.arrayBuffer()),
+        });
+      });
+    };
+    recorder.start(VIDEO_TIMESLICE_MS);
+    running = capture;
+    setState({ phase: "recording", notice: null });
+  } catch (error) {
+    for (const track of screen?.getTracks() ?? []) track.stop();
+    mic?.stop();
+    if (session !== null) await bridge.scrawlCancel?.({ id: session.id }).catch(() => undefined);
+    notify({ text: `Recording cannot start: ${describe(error)}`, tone: "error" });
+  }
+};
+
+const release = (capture: Running) => {
+  clearInterval(capture.pcmTimer);
+  clearInterval(capture.sampleTimer);
+  for (const track of capture.screen.getTracks()) track.stop();
+  capture.mic.stop();
+};
+
+const stopRecorder = (recorder: MediaRecorder) =>
+  new Promise<void>((resolve) => {
+    if (recorder.state === "inactive") {
+      resolve();
+      return;
+    }
+    recorder.onstop = () => resolve();
+    recorder.stop();
+  });
+
+const deliver = async (text: string, contactSheet: Blob, bundlePath: string) => {
+  if (delivery === null || !delivery.insert(`${text}\n`)) {
+    // The clipboard refuses a window that is not in front, and this one usually is not.
+    const copied = await navigator.clipboard?.writeText(text).then(
+      () => true,
+      () => false,
+    );
+    notify({
+      text: copied
+        ? "No composer can take the capture · its text is on the clipboard"
+        : `No composer can take the capture · it is saved in ${bundlePath}`,
+      tone: "warning",
+    });
+    return;
+  }
+  const attached = await delivery.attachImage(
+    new File([contactSheet], "contact-sheet.png", { type: "image/png" }),
+  );
+  notify(
+    attached
+      ? { text: "Capture added to the composer", tone: "success" }
+      : { text: "Capture text added · the contact sheet could not be attached", tone: "warning" },
+  );
+};
+
+const finish = async (bridge: DesktopBridge, capture: Running) => {
+  running = null;
+  setState({ phase: "building", notice: null });
+  try {
+    const input = await bridge.scrawlStop!({ id: capture.session.id });
+    const durationMs = Date.now() - capture.micStartedAt;
+    await stopRecorder(capture.recorder);
+    sendPcm(bridge, capture, capture.mic.takePcm());
+    release(capture);
+    await capture.sending;
+    await capture.writes;
+    const transcript = await bridge.dictationFinish?.({
+      sessionId: capture.sessionId,
+      words: true,
+    });
+    const words = transcript?.ok ? (transcript.value.words ?? []) : [];
+    const transcriptText = transcript?.ok ? transcript.value.text : "";
+    const built = await buildBundle(
+      {
+        session: capture.session,
+        input,
+        words,
+        transcriptText,
+        model: "parakeet",
+        video: new Blob(capture.chunks, { type: capture.recorder.mimeType }),
+        videoName: capture.videoName,
+        videoOffsetMs: capture.videoStartedAt - capture.micStartedAt,
+        inputOffsetMs: capture.micStartedAt - capture.session.startedAtMs,
+        durationMs,
+        sceneChanges: findSceneChanges(capture.samples),
+        wav: encodeWav(capture.pcm.map(decodeBase64)),
+      },
+      (path, bytes) => bridge.scrawlWriteFile!({ id: capture.session.id, path, bytes }),
+    );
+    await deliver(built.text, built.contactSheet, capture.session.bundlePath);
+    if (transcript !== undefined && !transcript.ok) {
+      notify({ text: `Capture has no transcript: ${transcript.reason}`, tone: "warning" });
+    }
+  } catch (error) {
+    release(capture);
+    notify({
+      text: `The capture could not be built: ${describe(error)}. Its video is in ${capture.session.bundlePath}`,
+      tone: "error",
+    });
+  }
+};
+
+/** Starts a capture, or finishes the running one. */
+export const toggleScrawl = () => {
+  const bridge = scrawlBridge();
+  if (bridge === null) return;
+  if (state.phase === "idle") void start(bridge);
+  else if (state.phase === "recording" && running !== null) void finish(bridge, running);
+};
+
+/** Drops a running capture and its files. Returns whether there was one. */
+export const cancelScrawl = (): boolean => {
+  const bridge = scrawlBridge();
+  const capture = running;
+  if (bridge === null || capture === null) return false;
+  running = null;
+  capture.recorder.ondataavailable = null;
+  capture.recorder.stop();
+  release(capture);
+  void bridge.dictationCancel?.({ sessionId: capture.sessionId }).catch(() => undefined);
+  void bridge.scrawlCancel?.({ id: capture.session.id }).catch(() => undefined);
+  setState(IDLE);
+  return true;
+};
diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts
index 4a3e8df253d9d..3b6f13bc22916 100644
--- a/packages/contracts/src/index.ts
+++ b/packages/contracts/src/index.ts
@@ -1,4 +1,5 @@
 export * from "./baseSchemas.ts";
+export * from "./scrawl.ts";
 export * from "./assistantCitations.ts";
 export * from "./composerContext.ts";
 export * from "./composerContextClipboard.ts";
diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts
index bf5f6730c5a06..f286eb1e9f7e6 100644
--- a/packages/contracts/src/ipc.ts
+++ b/packages/contracts/src/ipc.ts
@@ -19,6 +19,13 @@ import type {
   DictationFinishInput,
   DictationFinishResult,
 } from "./dictation.ts";
+import type {
+  ScrawlAppendVideoInput,
+  ScrawlCaptureRef,
+  ScrawlStartResult,
+  ScrawlStopResult,
+  ScrawlWriteFileInput,
+} from "./scrawl.ts";
 import { ExecutionEnvironmentDescriptor } from "./environment.ts";
 import { type ClientSettings, type QuitConfirmationMode, SnapShotShortcut } from "./settings.ts";
 import type { EditorId } from "./editor.ts";
@@ -1051,6 +1058,12 @@ export interface DesktopBridge {
     input: DictationFinishInput,
   ) => Promise<DesktopDictationOutcome<DictationFinishResult>>;
   dictationCancel?: (input: DictationCancelInput) => Promise<void>;
+  /** Screen captures with voice, clicks and ink; only the desktop app can see the whole screen. */
+  scrawlStart?: () => Promise<ScrawlStartResult>;
+  scrawlStop?: (input: ScrawlCaptureRef) => Promise<ScrawlStopResult>;
+  scrawlCancel?: (input: ScrawlCaptureRef) => Promise<void>;
+  scrawlAppendVideo?: (input: ScrawlAppendVideoInput) => Promise<void>;
+  scrawlWriteFile?: (input: ScrawlWriteFileInput) => Promise<void>;
   getLocalEnvironmentBearerToken: () => Promise<string>;
   getClientSettings: () => Promise<ClientSettings | null>;
   setClientSettings: (settings: ClientSettings) => Promise<void>;
diff --git a/packages/contracts/src/scrawl.ts b/packages/contracts/src/scrawl.ts
new file mode 100644
index 0000000000000..665833f19d28b
--- /dev/null
+++ b/packages/contracts/src/scrawl.ts
@@ -0,0 +1,79 @@
+import * as Schema from "effect/Schema";
+
+// Scrawl: a screen capture with voice, clicks and red ink, packed into a bundle for agents. The
+// desktop app watches the mouse and draws the ink; the renderer records and builds the bundle.
+
+export const ScrawlCaptureId = Schema.String.check(Schema.isPattern(/^scrawl-[a-z0-9-]{1,64}$/));
+export type ScrawlCaptureId = typeof ScrawlCaptureId.Type;
+
+export const ScrawlStartResult = Schema.Struct({
+  id: ScrawlCaptureId,
+  /** For getUserMedia's chromeMediaSourceId. */
+  sourceId: Schema.String,
+  width: Schema.Number,
+  height: Schema.Number,
+  /** Wall clock of the moment event times count from. */
+  startedAtMs: Schema.Number,
+  app: Schema.String,
+  windowTitle: Schema.String,
+  bundlePath: Schema.String,
+});
+export type ScrawlStartResult = typeof ScrawlStartResult.Type;
+
+/** Positions are fractions of the captured display, from its top left. */
+export const ScrawlInputEvent = Schema.Union([
+  Schema.Struct({
+    kind: Schema.Literal("click"),
+    t: Schema.Number,
+    x: Schema.Number,
+    y: Schema.Number,
+  }),
+  Schema.Struct({
+    kind: Schema.Literal("drag"),
+    t: Schema.Number,
+    t1: Schema.Number,
+    x0: Schema.Number,
+    y0: Schema.Number,
+    x1: Schema.Number,
+    y1: Schema.Number,
+  }),
+]);
+export type ScrawlInputEvent = typeof ScrawlInputEvent.Type;
+
+export const ScrawlStroke = Schema.Struct({
+  t0: Schema.Number,
+  t1: Schema.Number,
+  /** x, y, x, y, ... */
+  points: Schema.Array(Schema.Number),
+});
+export type ScrawlStroke = typeof ScrawlStroke.Type;
+
+export const ScrawlStopResult = Schema.Struct({
+  events: Schema.Array(ScrawlInputEvent),
+  strokes: Schema.Array(ScrawlStroke),
+});
+export type ScrawlStopResult = typeof ScrawlStopResult.Type;
+
+export const ScrawlCaptureRef = Schema.Struct({ id: ScrawlCaptureId });
+export type ScrawlCaptureRef = typeof ScrawlCaptureRef.Type;
+
+export const ScrawlAppendVideoInput = Schema.Struct({
+  id: ScrawlCaptureId,
+  name: Schema.String,
+  bytes: Schema.Uint8Array,
+});
+export type ScrawlAppendVideoInput = typeof ScrawlAppendVideoInput.Type;
+
+export const ScrawlWriteFileInput = Schema.Struct({
+  id: ScrawlCaptureId,
+  /** Relative to the bundle, such as frames/frame-01.png. */
+  path: Schema.String.check(Schema.isPattern(/^[\w-]+(?:\/[\w.-]+|\.[\w.-]+)$/)),
+  bytes: Schema.Uint8Array,
+});
+export type ScrawlWriteFileInput = typeof ScrawlWriteFileInput.Type;
+
+/** How long ink stays after Shift is let go. Its fading is not the screen answering a click. */
+export const SCRAWL_INK_HOLD_MS = 5_000;
+
+/** The menu action the global shortcut sends to the renderer. */
+export const SCRAWL_TOGGLE_ACTION = "scrawl-toggle";

From 0180795fb78d4b20172f126f9af7809dd2a54407 Mon Sep 17 00:00:00 2001
From: collissimo <148455189+collissimo@users.noreply.github.com>
Date: Sat, 31 Oct 2026 17:39:17 +0100
Subject: [PATCH 3/4] style(scrawl): format

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 apps/web/src/scrawl/scrawlBundle.ts | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts
index a47af4114d75b..f90df615ecd55 100644
--- a/apps/web/src/scrawl/scrawlBundle.ts
+++ b/apps/web/src/scrawl/scrawlBundle.ts
@@ -108,9 +108,7 @@ const strokesToMarks = (strokes: ReadonlyArray<ScrawlStroke>): Array<ManifestMar
 };
 
 /** Everything the hand touched, padded, or null when that is most of the screen anyway. */
-function activityBox(
-  recording: Pick<CaptureRecording, "input">,
-): {
+function activityBox(recording: Pick<CaptureRecording, "input">): {
   readonly x: number;
   readonly y: number;
   readonly width: number;

From 777e85b37948b820d277b64589e3884fc402f1c5 Mon Sep 17 00:00:00 2001
From: collissimo <148455189+collissimo@users.noreply.github.com>
Date: Sat, 31 Oct 2026 17:49:42 +0100
Subject: [PATCH 4/4] fix(scrawl): silent captures build, untimed speech stays,
 a lost renderer frees the overlays

The recording pill moves to the corner because it shows in the recording.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
---
 FORK.md                                  |  2 ++
 apps/desktop/src/scrawl/DesktopScrawl.ts | 24 +++++++++++++-----------
 apps/web/src/scrawl/narration.test.ts    | 18 ++++++++++++++++++
 apps/web/src/scrawl/narration.ts         | 18 +++++++++++++++++-
 apps/web/src/scrawl/scrawlCapture.ts     | 10 +++++++++-
 5 files changed, 59 insertions(+), 13 deletions(-)

diff --git a/FORK.md b/FORK.md
index 682ad250619b0..cc27d2eb494b2 100644
--- a/FORK.md
+++ b/FORK.md
@@ -166,6 +166,8 @@ the contact sheet land in the composer on screen. Esc in T3 Code discards a capt
   own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer
   read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1.
 - Uses the dictation model above, with word times mapped back across the pauses it cuts out.
+- The recording pill in the corner shows in the video and keyframes; `setContentProtection` does
+  not keep it out on Windows 10 22H2.
 - Bundles are in `<T3 home>/scrawl/captures` in Scrawl's format, so the Scrawl skill and the
   `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. Video and audio are deleted after 7 days.
 - Code: `apps/desktop/src/scrawl/`, `apps/web/src/scrawl/`, `packages/contracts/src/scrawl.ts`.
diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts
index 8cbf37dd81e38..f63fe2e36f6c7 100644
--- a/apps/desktop/src/scrawl/DesktopScrawl.ts
+++ b/apps/desktop/src/scrawl/DesktopScrawl.ts
@@ -74,7 +74,7 @@ window.inkLift=()=>{last=null};
 window.inkClear=()=>{last=null;g.clearRect(0,0,innerWidth,innerHeight)};
 </script></body></html>`;
 
-const PILL_PAGE = `<!doctype html><html><body style="margin:0;background:#18181b;color:#fafafa;font:600 13px system-ui;display:flex;align-items:center;justify-content:center;gap:8px;height:100vh;border-radius:999px;overflow:hidden;user-select:none">
+const PILL_PAGE = `<!doctype html><html><body style="margin:0;background:#18181b;color:#fafafa;font:600 13px system-ui;display:flex;align-items:center;justify-content:center;gap:8px;height:100vh;overflow:hidden;user-select:none">
 <span style="width:9px;height:9px;border-radius:50%;background:#ef4444"></span><span id="t">0:00</span>
 <script>window.setClock=(v)=>{document.getElementById("t").textContent=v}</script></body></html>`;
 
@@ -176,9 +176,9 @@ const make = Effect.gen(function* () {
   };
 
   const start = Effect.fn("DesktopScrawl.start")(function* () {
-    if (running !== null) {
-      return yield* new DesktopScrawlError({ reason: "A capture is already running." });
-    }
+    // The renderer owns the capture and only starts when it has none, so one left here lost its
+    // renderer to a reload. Without this its ink and pill would stay on screen for good.
+    if (running !== null) tearDown(running);
     void dropOldMedia(capturesDir, Date.now());
     const cursor = Electron.screen.getCursorScreenPoint();
     const display = Electron.screen.getDisplayNearestPoint(cursor);
@@ -217,18 +217,20 @@ const make = Effect.gen(function* () {
     void inkWindow.loadURL(dataUrl(INK_PAGE));
     inkWindow.showInactive();
 
-    const pill = new Electron.BrowserWindow(
-      overlayOptions({
-        x: display.bounds.x + Math.round((display.bounds.width - PILL_WIDTH) / 2),
-        y: display.bounds.y + 8,
+    // In the corner, because it shows in the recording: on Windows 10 22H2 neither a transparent nor
+    // an opaque window stayed out of it with setContentProtection.
+    const pill = new Electron.BrowserWindow({
+      ...overlayOptions({
+        x: display.workArea.x + display.workArea.width - PILL_WIDTH - 8,
+        y: display.workArea.y + display.workArea.height - PILL_HEIGHT - 8,
         width: PILL_WIDTH,
         height: PILL_HEIGHT,
       }),
-    );
+      transparent: false,
+      backgroundColor: "#18181b",
+    });
     pill.setIgnoreMouseEvents(true);
     pill.setAlwaysOnTop(true, "screen-saver");
-    // Stays out of the recording, so it never covers what is being shown.
-    pill.setContentProtection(true);
     void pill.loadURL(dataUrl(PILL_PAGE));
     pill.showInactive();
 
diff --git a/apps/web/src/scrawl/narration.test.ts b/apps/web/src/scrawl/narration.test.ts
index acb64dee3b179..2c82b0b07cf7b 100644
--- a/apps/web/src/scrawl/narration.test.ts
+++ b/apps/web/src/scrawl/narration.test.ts
@@ -5,6 +5,7 @@ import {
   foldActions,
   formatNarrationLine,
   splitSentences,
+  wordsOrWhole,
   type TimedWord,
 } from "./narration.ts";
 
@@ -101,6 +102,23 @@ describe("buildNarration", () => {
     expect(formatNarrationLine(lines[0]!, 0)).toBe("- #1 (0:02) screen changed");
   });
 
+  it("still has a keyframe when nothing was said or done", () => {
+    const lines = buildNarration({ words: [], events: [], marks: [], durationMs: 4_000 });
+    expect(lines.map((line) => formatNarrationLine(line, 0))).toEqual([
+      "- #1 (0:02) nothing said or done",
+    ]);
+  });
+
+  it("keeps the speech when the model could not time its words", () => {
+    const lines = buildNarration({
+      words: wordsOrWhole([], "Das hier bitte weg. Und das auch.", 8_000),
+      events: [{ kind: "click", t: 2_000, x: 0.5, y: 0.5 }],
+      marks: [],
+      durationMs: 8_000,
+    });
+    expect(lines.map((line) => line.text)).toEqual(["Das hier bitte weg. Und das auch."]);
+  });
+
   it("gives a click said over no speech its own line", () => {
     const lines = buildNarration({
       words: say("Hallo Welt.", 0),
diff --git a/apps/web/src/scrawl/narration.ts b/apps/web/src/scrawl/narration.ts
index 3cfbb57469e76..a9b67540af783 100644
--- a/apps/web/src/scrawl/narration.ts
+++ b/apps/web/src/scrawl/narration.ts
@@ -89,6 +89,14 @@ const SCORES: Record<NarrationAction["kind"], number> = {
 const endsSentence = (word: string) => /[.!?;:]["”’)\]]*$/.test(word);
 const endsClause = (word: string) => /,["”’)\]]*$/.test(word);
 
+/** A model that cannot time words still said them: the speech stays whole on one line. */
+export const wordsOrWhole = (
+  words: ReadonlyArray<TimedWord>,
+  text: string,
+  durationMs: number,
+): ReadonlyArray<TimedWord> =>
+  words.length > 0 || text === "" ? words : [{ text, t0: 0, t1: durationMs }];
+
 export function splitSentences(words: ReadonlyArray<TimedWord>): Array<Sentence> {
   const sentences: Array<Sentence> = [];
   let open: Sentence | null = null;
@@ -266,6 +274,11 @@ export function buildNarration(input: {
     sentences,
     input.sceneChanges === undefined ? folded : withReactions(folded, input.sceneChanges),
   );
+  // Nothing said, done or changed: the picture is still what the capture shows.
+  if (sentences.length === 0) {
+    const frameT = input.durationMs / 2;
+    return [{ t0: 0, t1: input.durationMs, frameT, text: "", actions: [] }];
+  }
   const owners = chooseFrameOwners(sentences, input.durationMs);
   const groups: Array<Array<Sentence>> = [];
   let ownerSeen = false;
@@ -324,6 +337,9 @@ const describeActions = (actions: ReadonlyArray<NarrationAction>) => {
 export function formatNarrationLine(line: NarrationLine, index: number): string {
   const head = `- #${index + 1} (${formatClock(line.frameT)})`;
   const actions = describeActions(line.actions);
-  if (line.text === "") return `${head} ${actions === "" ? "screen changed" : actions}`;
+  if (line.text === "") {
+    const shown = line.actions.length > 0 ? "screen changed" : "nothing said or done";
+    return `${head} ${actions === "" ? shown : actions}`;
+  }
   return `${head}${actions === "" ? "" : ` [${actions}]`} “${line.text}”`;
 }
diff --git a/apps/web/src/scrawl/scrawlCapture.ts b/apps/web/src/scrawl/scrawlCapture.ts
index 0ae199d4c78cc..f431c3b7cb04d 100644
--- a/apps/web/src/scrawl/scrawlCapture.ts
+++ b/apps/web/src/scrawl/scrawlCapture.ts
@@ -10,6 +10,7 @@ import { DEFAULT_DICTATION_LANGUAGE } from "../dictation/dictationLanguages";
 import { DICTATION_LANGUAGE_KEY } from "../dictation/useDictation";
 import { getLocalStorageItem } from "../hooks/useLocalStorage";
 import { randomUUID } from "../lib/utils";
+import { wordsOrWhole } from "./narration";
 import { buildBundle } from "./scrawlBundle";
 import { findSceneChanges, THUMB_HEIGHT, THUMB_WIDTH, type LumaSample } from "./sceneChanges";
 
@@ -261,7 +262,10 @@ const start = async (bridge: DesktopBridge) => {
   }
 };
 
+const released = new WeakSet<Running>();
 const release = (capture: Running) => {
+  if (released.has(capture)) return;
+  released.add(capture);
   clearInterval(capture.pcmTimer);
   clearInterval(capture.sampleTimer);
   for (const track of capture.screen.getTracks()) track.stop();
@@ -318,8 +322,12 @@ const finish = async (bridge: DesktopBridge, capture: Running) => {
       sessionId: capture.sessionId,
       words: true,
     });
-    const words = transcript?.ok ? (transcript.value.words ?? []) : [];
     const transcriptText = transcript?.ok ? transcript.value.text : "";
+    const words = wordsOrWhole(
+      transcript?.ok ? (transcript.value.words ?? []) : [],
+      transcriptText,
+      durationMs,
+    );
     const built = await buildBundle(
       {
         session: capture.session,