feat(scrawl): capture the screen with voice, clicks and ink, inside T3 Code

FORK.md

@@ -150,3 +150,24 @@ can be rebound in Settings → Keybindings ("Composer: Dictate").
   `apps/web/src/components/chat/ComposerDictationButton.tsx`, `packages/contracts/src/dictation.ts`.
   The standalone app for every other program is [collissimo/t3-voice](https://github.com/collissimo/t3-voice);
   it leaves F9 to T3 Code while T3 Code is in front.
+
+### Screen capture (Scrawl, built in)
+
+**Alt+Shift+S** anywhere, or the screen button in the composer, records the monitor under the
+mouse, also while another program is in front. Talk, click, hold **Shift** and move the mouse to
+draw red ink (it fades 5 seconds after Shift is let go). Alt+Shift+S again finishes: the text and
+the contact sheet land in the composer on screen. Esc in T3 Code discards a capture.
+
+- Desktop app only: a browser cannot see the whole screen, and mobile has no such thing. Clicks and
+  ink need Windows; elsewhere a capture has voice and video.
+- While the Scrawl app runs it holds Alt+Shift+S. T3 Code logs that and takes the shortcut within
+  15 seconds of Scrawl closing. The composer button works either way.
+- Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences
+  own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer
+  read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1.
+- Uses the dictation model above, with word times mapped back across the pauses it cuts out.
+- The recording pill in the corner shows in the video and keyframes; `setContentProtection` does
+  not keep it out on Windows 10 22H2.
+- Bundles are in `<T3 home>/scrawl/captures` in Scrawl's format, so the Scrawl skill and the
+  `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. Video and audio are deleted after 7 days.
+- Code: `apps/desktop/src/scrawl/`, `apps/web/src/scrawl/`, `packages/contracts/src/scrawl.ts`.

apps/desktop/src/ipc/DesktopIpcHandlers.ts

@@ -10,6 +10,13 @@ import {
   setLocalEnvironmentEnabled,
 } from "./methods/localEnvironment.ts";
 import { dictationAppend, dictationCancel, dictationFinish } from "./methods/dictation.ts";
+import {
+  scrawlAppendVideo,
+  scrawlCancel,
+  scrawlStart,
+  scrawlStop,
+  scrawlWriteFile,
+} from "./methods/scrawl.ts";
 import {
   getAdvertisedEndpoints,
   getServerExposureState,
@@ -100,6 +107,11 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers"
   yield* ipc.handle(setLocalEnvironmentEnabled);
   yield* ipc.handle(dictationAppend);
   yield* ipc.handle(dictationFinish);
+  yield* ipc.handle(scrawlStart);
+  yield* ipc.handle(scrawlStop);
+  yield* ipc.handle(scrawlCancel);
+  yield* ipc.handle(scrawlAppendVideo);
+  yield* ipc.handle(scrawlWriteFile);
   yield* ipc.handle(dictationCancel);
   yield* ipc.handle(getLocalEnvironmentBearerToken);
 

apps/desktop/src/ipc/channels.ts

@@ -39,6 +39,11 @@ export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environm
 export const DICTATION_APPEND_CHANNEL = "desktop:dictation-append";
 export const DICTATION_FINISH_CHANNEL = "desktop:dictation-finish";
 export const DICTATION_CANCEL_CHANNEL = "desktop:dictation-cancel";
+export const SCRAWL_START_CHANNEL = "desktop:scrawl-start";
+export const SCRAWL_STOP_CHANNEL = "desktop:scrawl-stop";
+export const SCRAWL_CANCEL_CHANNEL = "desktop:scrawl-cancel";
+export const SCRAWL_APPEND_VIDEO_CHANNEL = "desktop:scrawl-append-video";
+export const SCRAWL_WRITE_FILE_CHANNEL = "desktop:scrawl-write-file";
 export const GET_LOCAL_ENVIRONMENT_BEARER_TOKEN_CHANNEL =
   "desktop:get-local-environment-bearer-token";
 export const GET_CLIENT_SETTINGS_CHANNEL = "desktop:get-client-settings";

apps/desktop/src/ipc/methods/scrawl.ts

@@ -0,0 +1,58 @@
+import {
+  ScrawlAppendVideoInput,
+  ScrawlCaptureRef,
+  ScrawlStartResult,
+  ScrawlStopResult,
+  ScrawlWriteFileInput,
+} from "@t3tools/contracts";
+import * as Effect from "effect/Effect";
+import * as Schema from "effect/Schema";
+
+import * as DesktopScrawl from "../../scrawl/DesktopScrawl.ts";
+import * as IpcChannels from "../channels.ts";
+import { makeIpcMethod } from "../DesktopIpc.ts";
+
+export const scrawlStart = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_START_CHANNEL,
+  payload: Schema.Void,
+  result: ScrawlStartResult,
+  handler: Effect.fn("desktop.ipc.scrawl.start")(function* () {
+    return yield* (yield* DesktopScrawl.DesktopScrawl).start();
+  }),
+});
+
+export const scrawlStop = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_STOP_CHANNEL,
+  payload: ScrawlCaptureRef,
+  result: ScrawlStopResult,
+  handler: Effect.fn("desktop.ipc.scrawl.stop")(function* (input) {
+    return yield* (yield* DesktopScrawl.DesktopScrawl).stop(input);
+  }),
+});
+
+export const scrawlCancel = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_CANCEL_CHANNEL,
+  payload: ScrawlCaptureRef,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.scrawl.cancel")(function* (input) {
+    yield* (yield* DesktopScrawl.DesktopScrawl).cancel(input);
+  }),
+});
+
+export const scrawlAppendVideo = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL,
+  payload: ScrawlAppendVideoInput,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.scrawl.appendVideo")(function* (input) {
+    yield* (yield* DesktopScrawl.DesktopScrawl).appendVideo(input);
+  }),
+});
+
+export const scrawlWriteFile = makeIpcMethod({
+  channel: IpcChannels.SCRAWL_WRITE_FILE_CHANNEL,
+  payload: ScrawlWriteFileInput,
+  result: Schema.Void,
+  handler: Effect.fn("desktop.ipc.scrawl.writeFile")(function* (input) {
+    yield* (yield* DesktopScrawl.DesktopScrawl).writeFile(input);
+  }),
+});

apps/desktop/src/main.ts

@@ -41,6 +41,7 @@ import * as DesktopConnectionCatalogStore from "./app/DesktopConnectionCatalogSt
 import * as DesktopClerk from "./app/DesktopClerk.ts";
 import * as DesktopCliCommand from "./app/DesktopCliCommand.ts";
 import * as DesktopDictation from "./dictation/DesktopDictation.ts";
+import * as DesktopScrawl from "./scrawl/DesktopScrawl.ts";
 import * as DesktopApplicationMenu from "./window/DesktopApplicationMenu.ts";
 import * as DesktopAssets from "./app/DesktopAssets.ts";
 import * as DesktopBackendConfiguration from "./backend/DesktopBackendConfiguration.ts";
@@ -210,6 +211,7 @@ const layerDesktopApplication = Layer.mergeAll(
   DesktopLinuxUrlHandler.layer,
   DesktopCliCommand.layer,
   DesktopDictation.layer,
+  DesktopScrawl.layer,
   layerDesktopSsh,
 ).pipe(
   Layer.provideMerge(layerDesktopSnapShot),

apps/desktop/src/preload.ts

@@ -122,6 +122,11 @@ contextBridge.exposeInMainWorld("desktopBridge", {
   dictationAppend: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_APPEND_CHANNEL, input),
   dictationFinish: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_FINISH_CHANNEL, input),
   dictationCancel: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_CANCEL_CHANNEL, input),
+  scrawlStart: () => ipcRenderer.invoke(IpcChannels.SCRAWL_START_CHANNEL),
+  scrawlStop: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_STOP_CHANNEL, input),
+  scrawlCancel: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_CANCEL_CHANNEL, input),
+  scrawlAppendVideo: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL, input),
+  scrawlWriteFile: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_WRITE_FILE_CHANNEL, input),
   getClientSettings: () => ipcRenderer.invoke(IpcChannels.GET_CLIENT_SETTINGS_CHANNEL),
   setClientSettings: (settings) =>
     ipcRenderer.invoke(IpcChannels.SET_CLIENT_SETTINGS_CHANNEL, settings),