diff --git a/FORK.md b/FORK.md index 54c3930ec268c..cc27d2eb494b2 100644 --- a/FORK.md +++ b/FORK.md @@ -150,3 +150,24 @@ can be rebound in Settings → Keybindings ("Composer: Dictate"). `apps/web/src/components/chat/ComposerDictationButton.tsx`, `packages/contracts/src/dictation.ts`. The standalone app for every other program is [collissimo/t3-voice](https://github.com/collissimo/t3-voice); it leaves F9 to T3 Code while T3 Code is in front. + +### Screen capture (Scrawl, built in) + +**Alt+Shift+S** anywhere, or the screen button in the composer, records the monitor under the +mouse, also while another program is in front. Talk, click, hold **Shift** and move the mouse to +draw red ink (it fades 5 seconds after Shift is let go). Alt+Shift+S again finishes: the text and +the contact sheet land in the composer on screen. Esc in T3 Code discards a capture. + +- Desktop app only: a browser cannot see the whole screen, and mobile has no such thing. Clicks and + ink need Windows; elsewhere a capture has voice and video. +- While the Scrawl app runs it holds Alt+Shift+S. T3 Code logs that and takes the shortcut within + 15 seconds of Scrawl closing. The composer button works either way. +- Speech and pointing do not run in step, so sentences are never cut at a click: whole sentences + own a keyframe, click bursts fold into one (`clicked ×3`), and clicks the screen did not answer + read as `pointed`. On 32 real captures this took cuts through sentences from 59 of 127 lines to 1. +- Uses the dictation model above, with word times mapped back across the pauses it cuts out. +- The recording pill in the corner shows in the video and keyframes; `setContentProtection` does + not keep it out on Windows 10 22H2. +- Bundles are in `/scrawl/captures` in Scrawl's format, so the Scrawl skill and the + `scrawl` CLI (`inspect`, `zoom`, `frames`) read them. Video and audio are deleted after 7 days. +- Code: `apps/desktop/src/scrawl/`, `apps/web/src/scrawl/`, `packages/contracts/src/scrawl.ts`. diff --git a/apps/desktop/src/ipc/DesktopIpcHandlers.ts b/apps/desktop/src/ipc/DesktopIpcHandlers.ts index 1501a10b20366..f378757607428 100644 --- a/apps/desktop/src/ipc/DesktopIpcHandlers.ts +++ b/apps/desktop/src/ipc/DesktopIpcHandlers.ts @@ -10,6 +10,13 @@ import { setLocalEnvironmentEnabled, } from "./methods/localEnvironment.ts"; import { dictationAppend, dictationCancel, dictationFinish } from "./methods/dictation.ts"; +import { + scrawlAppendVideo, + scrawlCancel, + scrawlStart, + scrawlStop, + scrawlWriteFile, +} from "./methods/scrawl.ts"; import { getAdvertisedEndpoints, getServerExposureState, @@ -100,6 +107,11 @@ export const installDesktopIpcHandlers = Effect.fn("desktop.ipc.installHandlers" yield* ipc.handle(setLocalEnvironmentEnabled); yield* ipc.handle(dictationAppend); yield* ipc.handle(dictationFinish); + yield* ipc.handle(scrawlStart); + yield* ipc.handle(scrawlStop); + yield* ipc.handle(scrawlCancel); + yield* ipc.handle(scrawlAppendVideo); + yield* ipc.handle(scrawlWriteFile); yield* ipc.handle(dictationCancel); yield* ipc.handle(getLocalEnvironmentBearerToken); diff --git a/apps/desktop/src/ipc/channels.ts b/apps/desktop/src/ipc/channels.ts index 88e36c8b5aebf..58d11aa9e8e3e 100644 --- a/apps/desktop/src/ipc/channels.ts +++ b/apps/desktop/src/ipc/channels.ts @@ -39,6 +39,11 @@ export const SET_LOCAL_ENVIRONMENT_ENABLED_CHANNEL = "desktop:set-local-environm export const DICTATION_APPEND_CHANNEL = "desktop:dictation-append"; export const DICTATION_FINISH_CHANNEL = "desktop:dictation-finish"; export const DICTATION_CANCEL_CHANNEL = "desktop:dictation-cancel"; +export const SCRAWL_START_CHANNEL = "desktop:scrawl-start"; +export const SCRAWL_STOP_CHANNEL = "desktop:scrawl-stop"; +export const SCRAWL_CANCEL_CHANNEL = "desktop:scrawl-cancel"; +export const SCRAWL_APPEND_VIDEO_CHANNEL = "desktop:scrawl-append-video"; +export const SCRAWL_WRITE_FILE_CHANNEL = "desktop:scrawl-write-file"; export const GET_LOCAL_ENVIRONMENT_BEARER_TOKEN_CHANNEL = "desktop:get-local-environment-bearer-token"; export const GET_CLIENT_SETTINGS_CHANNEL = "desktop:get-client-settings"; diff --git a/apps/desktop/src/ipc/methods/scrawl.ts b/apps/desktop/src/ipc/methods/scrawl.ts new file mode 100644 index 0000000000000..b82618fa4642d --- /dev/null +++ b/apps/desktop/src/ipc/methods/scrawl.ts @@ -0,0 +1,58 @@ +import { + ScrawlAppendVideoInput, + ScrawlCaptureRef, + ScrawlStartResult, + ScrawlStopResult, + ScrawlWriteFileInput, +} from "@t3tools/contracts"; +import * as Effect from "effect/Effect"; +import * as Schema from "effect/Schema"; + +import * as DesktopScrawl from "../../scrawl/DesktopScrawl.ts"; +import * as IpcChannels from "../channels.ts"; +import { makeIpcMethod } from "../DesktopIpc.ts"; + +export const scrawlStart = makeIpcMethod({ + channel: IpcChannels.SCRAWL_START_CHANNEL, + payload: Schema.Void, + result: ScrawlStartResult, + handler: Effect.fn("desktop.ipc.scrawl.start")(function* () { + return yield* (yield* DesktopScrawl.DesktopScrawl).start(); + }), +}); + +export const scrawlStop = makeIpcMethod({ + channel: IpcChannels.SCRAWL_STOP_CHANNEL, + payload: ScrawlCaptureRef, + result: ScrawlStopResult, + handler: Effect.fn("desktop.ipc.scrawl.stop")(function* (input) { + return yield* (yield* DesktopScrawl.DesktopScrawl).stop(input); + }), +}); + +export const scrawlCancel = makeIpcMethod({ + channel: IpcChannels.SCRAWL_CANCEL_CHANNEL, + payload: ScrawlCaptureRef, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.cancel")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).cancel(input); + }), +}); + +export const scrawlAppendVideo = makeIpcMethod({ + channel: IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL, + payload: ScrawlAppendVideoInput, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.appendVideo")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).appendVideo(input); + }), +}); + +export const scrawlWriteFile = makeIpcMethod({ + channel: IpcChannels.SCRAWL_WRITE_FILE_CHANNEL, + payload: ScrawlWriteFileInput, + result: Schema.Void, + handler: Effect.fn("desktop.ipc.scrawl.writeFile")(function* (input) { + yield* (yield* DesktopScrawl.DesktopScrawl).writeFile(input); + }), +}); diff --git a/apps/desktop/src/main.ts b/apps/desktop/src/main.ts index d44e48932a093..63ab8ea13516d 100644 --- a/apps/desktop/src/main.ts +++ b/apps/desktop/src/main.ts @@ -41,6 +41,7 @@ import * as DesktopConnectionCatalogStore from "./app/DesktopConnectionCatalogSt import * as DesktopClerk from "./app/DesktopClerk.ts"; import * as DesktopCliCommand from "./app/DesktopCliCommand.ts"; import * as DesktopDictation from "./dictation/DesktopDictation.ts"; +import * as DesktopScrawl from "./scrawl/DesktopScrawl.ts"; import * as DesktopApplicationMenu from "./window/DesktopApplicationMenu.ts"; import * as DesktopAssets from "./app/DesktopAssets.ts"; import * as DesktopBackendConfiguration from "./backend/DesktopBackendConfiguration.ts"; @@ -210,6 +211,7 @@ const layerDesktopApplication = Layer.mergeAll( DesktopLinuxUrlHandler.layer, DesktopCliCommand.layer, DesktopDictation.layer, + DesktopScrawl.layer, layerDesktopSsh, ).pipe( Layer.provideMerge(layerDesktopSnapShot), diff --git a/apps/desktop/src/preload.ts b/apps/desktop/src/preload.ts index 96cef2b58df54..34a785cd56d19 100644 --- a/apps/desktop/src/preload.ts +++ b/apps/desktop/src/preload.ts @@ -122,6 +122,11 @@ contextBridge.exposeInMainWorld("desktopBridge", { dictationAppend: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_APPEND_CHANNEL, input), dictationFinish: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_FINISH_CHANNEL, input), dictationCancel: (input) => ipcRenderer.invoke(IpcChannels.DICTATION_CANCEL_CHANNEL, input), + scrawlStart: () => ipcRenderer.invoke(IpcChannels.SCRAWL_START_CHANNEL), + scrawlStop: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_STOP_CHANNEL, input), + scrawlCancel: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_CANCEL_CHANNEL, input), + scrawlAppendVideo: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_APPEND_VIDEO_CHANNEL, input), + scrawlWriteFile: (input) => ipcRenderer.invoke(IpcChannels.SCRAWL_WRITE_FILE_CHANNEL, input), getClientSettings: () => ipcRenderer.invoke(IpcChannels.GET_CLIENT_SETTINGS_CHANNEL), setClientSettings: (settings) => ipcRenderer.invoke(IpcChannels.SET_CLIENT_SETTINGS_CHANNEL, settings), diff --git a/apps/desktop/src/scrawl/DesktopScrawl.ts b/apps/desktop/src/scrawl/DesktopScrawl.ts new file mode 100644 index 0000000000000..f63fe2e36f6c7 --- /dev/null +++ b/apps/desktop/src/scrawl/DesktopScrawl.ts @@ -0,0 +1,364 @@ +// @effect-diagnostics globalTimers:off -- The cursor poll and pill clock run at native callback boundaries outside Effect fibers. +// @effect-diagnostics globalTimersInEffect:off -- They are started here and owned by the capture, not a fiber. +// @effect-diagnostics globalDate:off -- Input times share the wall clock with the renderer's recorder. +// @effect-diagnostics globalDateInEffect:off -- Same clock as above. +// @effect-diagnostics nodeBuiltinImport:off -- Bundles are plain files written outside any Effect service. +// The desktop half of a Scrawl capture: the global shortcut, the mouse and Shift watcher, the ink +// and the recording pill, and the bundle on disk. The renderer records and builds the bundle. + +import { + SCRAWL_INK_HOLD_MS, + SCRAWL_TOGGLE_ACTION, + type ScrawlAppendVideoInput, + type ScrawlCaptureRef, + type ScrawlStartResult, + type ScrawlStopResult, + type ScrawlWriteFileInput, +} from "@t3tools/contracts"; +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Schema from "effect/Schema"; +import * as Electron from "electron"; +import * as NodeFSP from "node:fs/promises"; +import * as NodePath from "node:path"; + +import * as DesktopEnvironment from "../app/DesktopEnvironment.ts"; +import { loadWindowsForegroundApi } from "../electron/WindowsForeground.ts"; +import { activeWindow } from "../snapShot/ActiveWindow.ts"; +import * as DesktopWindow from "../window/DesktopWindow.ts"; +import { ScrawlInputTracker } from "./inputTracker.ts"; + +const DEFAULT_SHORTCUT = "Alt+Shift+S"; +// Another program (the Scrawl app uses the same keys) may hold the shortcut and let go of it later. +const SHORTCUT_RETRY_MS = 15_000; +// Short presses must not fall between two looks. Polling needs no hook, no message pump and no +// permission, and costs two native calls per look, only while a capture runs. +const INPUT_POLL_MS = 10; +const VK_LBUTTON = 0x01; +const VK_SHIFT = 0x10; +// Like Scrawl: frames, text and contact sheet stay, the heavy media goes. +const MEDIA_KEPT_MS = 7 * 24 * 60 * 60 * 1000; +const MEDIA_FILES = ["recording.mp4", "recording.webm", "audio.wav"]; +const PILL_WIDTH = 96; +const PILL_HEIGHT = 30; + +export class DesktopScrawlError extends Schema.TaggedError()( + "DesktopScrawlError", + { reason: Schema.String }, +) { + override get message(): string { + return this.reason; + } +} + +export class DesktopScrawl extends Context.Service< + DesktopScrawl, + { + readonly start: () => Effect.Effect; + readonly stop: (input: ScrawlCaptureRef) => Effect.Effect; + readonly cancel: (input: ScrawlCaptureRef) => Effect.Effect; + readonly appendVideo: ( + input: ScrawlAppendVideoInput, + ) => Effect.Effect; + readonly writeFile: (input: ScrawlWriteFileInput) => Effect.Effect; + } +>()("@t3tools/desktop/scrawl/DesktopScrawl") {} + +const INK_PAGE = ` +`; + +const PILL_PAGE = ` +0:00 +`; + +const dropOldMedia = async (capturesDir: string, now: number) => { + const bundles = await NodeFSP.readdir(capturesDir).catch(() => [] as Array); + for (const bundle of bundles) { + if (!bundle.endsWith(".scrawl")) continue; + const folder = NodePath.join(capturesDir, bundle); + const stat = await NodeFSP.stat(folder).catch(() => null); + if (stat === null || now - stat.mtimeMs < MEDIA_KEPT_MS) continue; + await Promise.all( + MEDIA_FILES.map((file) => NodeFSP.rm(NodePath.join(folder, file), { force: true })), + ).catch(() => undefined); + } +}; + +const dataUrl = (html: string) => `data:text/html;charset=utf-8,${encodeURIComponent(html)}`; + +const overlayOptions = (bounds: Electron.Rectangle): Electron.BrowserWindowConstructorOptions => ({ + ...bounds, + transparent: true, + frame: false, + focusable: false, + resizable: false, + movable: false, + hasShadow: false, + skipTaskbar: true, + alwaysOnTop: true, + show: false, + webPreferences: { sandbox: true, contextIsolation: true, backgroundThrottling: false }, +}); + +const toError = (cause: unknown) => + new DesktopScrawlError({ reason: cause instanceof Error ? cause.message : String(cause) }); + +interface RunningCapture { + readonly id: string; + readonly bundlePath: string; + readonly startedAtMs: number; + readonly tracker: ScrawlInputTracker; + readonly display: Electron.Display; + readonly ink: Electron.BrowserWindow; + readonly pill: Electron.BrowserWindow; + readonly inputTimer: ReturnType; + inkTimer: ReturnType | null; + readonly clockTimer: ReturnType; +} + +const make = Effect.gen(function* () { + const environment = yield* DesktopEnvironment.DesktopEnvironment; + const desktopWindow = yield* DesktopWindow.DesktopWindow; + const services = yield* Effect.context(); + const capturesDir = NodePath.join(environment.baseDir, "scrawl", "captures"); + let running: RunningCapture | null = null; + // Finished captures keep writing their frames and summary after the watcher stopped. + const bundles = new Map(); + let nextIndex = 0; + + const ink = (capture: RunningCapture, script: string) => { + if (!capture.ink.isDestroyed()) void capture.ink.webContents.executeJavaScript(script, true); + }; + + /** Clicks and strokes need Windows' key state; elsewhere a capture has voice and video only. */ + const startInputPoll = (isKeyDown: ((virtualKey: number) => boolean) | null) => { + let left = false; + let shift = false; + return setInterval(() => { + const current = running; + if (current === null) return; + const t = Date.now() - current.startedAtMs; + const at = Electron.screen.getCursorScreenPoint(); + if (isKeyDown !== null) { + const nextLeft = isKeyDown(VK_LBUTTON); + const nextShift = isKeyDown(VK_SHIFT); + if (nextLeft && !left) current.tracker.leftDown(t, at); + if (!nextLeft && left) current.tracker.leftUp(t, at); + if (nextShift && !shift) { + current.tracker.shiftDown(t, at); + if (current.inkTimer !== null) clearTimeout(current.inkTimer); + current.inkTimer = null; + } + if (!nextShift && shift) { + current.tracker.shiftUp(); + ink(current, "inkLift()"); + // Fades on a clock, not on the next click, so a click's effect on the screen stays its own. + current.inkTimer = setTimeout(() => ink(current, "inkClear()"), SCRAWL_INK_HOLD_MS); + } + left = nextLeft; + shift = nextShift; + } + const point = current.tracker.moved(t, at); + if (point !== null) { + ink( + current, + `inkMove(${point.x - current.display.bounds.x},${point.y - current.display.bounds.y})`, + ); + } + }, INPUT_POLL_MS); + }; + + const start = Effect.fn("DesktopScrawl.start")(function* () { + // The renderer owns the capture and only starts when it has none, so one left here lost its + // renderer to a reload. Without this its ink and pill would stay on screen for good. + if (running !== null) tearDown(running); + void dropOldMedia(capturesDir, Date.now()); + const cursor = Electron.screen.getCursorScreenPoint(); + const display = Electron.screen.getDisplayNearestPoint(cursor); + const sources = yield* Effect.tryPromise({ + try: () => + Electron.desktopCapturer.getSources({ + types: ["screen"], + thumbnailSize: { width: 0, height: 0 }, + }), + catch: toError, + }); + const source = sources.find((entry) => entry.display_id === String(display.id)) ?? sources[0]; + if (source === undefined) { + return yield* new DesktopScrawlError({ reason: "No screen to record." }); + } + const front = yield* Effect.tryPromise({ + try: () => activeWindow(environment.platform), + catch: toError, + }).pipe(Effect.orElseSucceed(() => undefined)); + const keys = + environment.platform === "win32" + ? yield* Effect.tryPromise({ try: () => loadWindowsForegroundApi(), catch: toError }) + : null; + const startedAtMs = Date.now(); + const id = `scrawl-${startedAtMs.toString(36)}-${(nextIndex++).toString(36)}`; + const bundlePath = NodePath.join(capturesDir, `${id}.scrawl`); + yield* Effect.tryPromise({ + try: () => NodeFSP.mkdir(NodePath.join(bundlePath, "frames"), { recursive: true }), + catch: toError, + }); + const tracker = new ScrawlInputTracker(display.bounds); + + const inkWindow = new Electron.BrowserWindow(overlayOptions(display.bounds)); + inkWindow.setIgnoreMouseEvents(true); + inkWindow.setAlwaysOnTop(true, "screen-saver"); + void inkWindow.loadURL(dataUrl(INK_PAGE)); + inkWindow.showInactive(); + + // In the corner, because it shows in the recording: on Windows 10 22H2 neither a transparent nor + // an opaque window stayed out of it with setContentProtection. + const pill = new Electron.BrowserWindow({ + ...overlayOptions({ + x: display.workArea.x + display.workArea.width - PILL_WIDTH - 8, + y: display.workArea.y + display.workArea.height - PILL_HEIGHT - 8, + width: PILL_WIDTH, + height: PILL_HEIGHT, + }), + transparent: false, + backgroundColor: "#18181b", + }); + pill.setIgnoreMouseEvents(true); + pill.setAlwaysOnTop(true, "screen-saver"); + void pill.loadURL(dataUrl(PILL_PAGE)); + pill.showInactive(); + + const capture: RunningCapture = { + id, + bundlePath, + startedAtMs, + tracker, + display, + ink: inkWindow, + pill, + inputTimer: startInputPoll(keys === null ? null : keys.isKeyDown), + inkTimer: null, + clockTimer: setInterval(() => { + const current = running; + if (current === null || current.pill.isDestroyed()) return; + const seconds = Math.floor((Date.now() - current.startedAtMs) / 1000); + const clock = `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; + void current.pill.webContents.executeJavaScript(`setClock("${clock}")`, true); + }, 1000), + }; + running = capture; + bundles.set(id, bundlePath); + const scale = display.scaleFactor; + return { + id, + sourceId: source.id, + width: Math.round(display.bounds.width * scale), + height: Math.round(display.bounds.height * scale), + startedAtMs, + app: front?.owner.name.replace(/\.exe$/i, "") ?? "", + windowTitle: front?.title ?? "", + bundlePath, + } satisfies ScrawlStartResult; + }); + + const tearDown = (capture: RunningCapture) => { + clearInterval(capture.inputTimer); + if (capture.inkTimer !== null) clearTimeout(capture.inkTimer); + clearInterval(capture.clockTimer); + if (!capture.ink.isDestroyed()) capture.ink.destroy(); + if (!capture.pill.isDestroyed()) capture.pill.destroy(); + if (running === capture) running = null; + }; + + const stop = Effect.fn("DesktopScrawl.stop")(function* (input: ScrawlCaptureRef) { + if (running === null || running.id !== input.id) { + return yield* new DesktopScrawlError({ reason: "This capture is not running." }); + } + const capture = running; + capture.tracker.shiftUp(); + tearDown(capture); + const { events, strokes } = capture.tracker.result(); + return { events: [...events], strokes: [...strokes] }; + }); + + const cancel = Effect.fn("DesktopScrawl.cancel")(function* (input: ScrawlCaptureRef) { + if (running?.id === input.id) tearDown(running); + const bundlePath = bundles.get(input.id); + bundles.delete(input.id); + if (bundlePath !== undefined) { + yield* Effect.promise(() => + NodeFSP.rm(bundlePath, { recursive: true, force: true }).catch(() => undefined), + ); + } + }); + + const appendVideo = Effect.fn("DesktopScrawl.appendVideo")(function* ( + input: ScrawlAppendVideoInput, + ) { + const bundlePath = bundles.get(input.id); + if (bundlePath === undefined) { + return yield* new DesktopScrawlError({ reason: "Unknown capture." }); + } + yield* Effect.tryPromise({ + try: () => NodeFSP.appendFile(NodePath.join(bundlePath, input.name), input.bytes), + catch: toError, + }); + }); + + const writeFile = Effect.fn("DesktopScrawl.writeFile")(function* (input: ScrawlWriteFileInput) { + const bundlePath = bundles.get(input.id); + if (bundlePath === undefined) { + return yield* new DesktopScrawlError({ reason: "Unknown capture." }); + } + const target = NodePath.join(bundlePath, input.path); + yield* Effect.tryPromise({ + try: async () => { + await NodeFSP.mkdir(NodePath.dirname(target), { recursive: true }); + await NodeFSP.writeFile(target, input.bytes); + }, + catch: toError, + }); + }); + + yield* Effect.acquireRelease( + Effect.promise(() => { + const run = Effect.runForkWith(services); + const register = () => + Electron.globalShortcut.register(DEFAULT_SHORTCUT, () => { + run( + desktopWindow + .dispatchMenuAction(SCRAWL_TOGGLE_ACTION, { reveal: false }) + .pipe(Effect.ignore), + ); + }); + const whenReady = async () => { + await Electron.app.whenReady(); + if (register()) return null; + run(Effect.logWarning(`scrawl: ${DEFAULT_SHORTCUT} is taken by another program`)); + const retry = setInterval(() => { + if (register()) clearInterval(retry); + }, SHORTCUT_RETRY_MS); + retry.unref?.(); + return retry; + }; + return whenReady(); + }), + (retry) => + Effect.promise(async () => { + const timer = await retry; + if (timer !== null) clearInterval(timer); + Electron.globalShortcut.unregister(DEFAULT_SHORTCUT); + if (running !== null) tearDown(running); + }), + ); + + return DesktopScrawl.of({ start, stop, cancel, appendVideo, writeFile }); +}); + +export const layer = Layer.effect(DesktopScrawl, make); diff --git a/apps/desktop/src/scrawl/inputTracker.test.ts b/apps/desktop/src/scrawl/inputTracker.test.ts new file mode 100644 index 0000000000000..62d7f4c2705a4 --- /dev/null +++ b/apps/desktop/src/scrawl/inputTracker.test.ts @@ -0,0 +1,34 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { ScrawlInputTracker } from "./inputTracker.ts"; + +const secondDisplay = { x: 1920, y: 0, width: 1000, height: 500 }; + +describe("ScrawlInputTracker", () => { + it("tells clicks from drags, on the captured display only", () => { + const tracker = new ScrawlInputTracker(secondDisplay); + tracker.leftDown(100, { x: 2420, y: 250 }); + tracker.leftUp(150, { x: 2422, y: 251 }); + tracker.leftDown(200, { x: 2020, y: 100 }); + tracker.leftUp(600, { x: 2420, y: 100 }); + tracker.leftDown(700, { x: 100, y: 100 }); + tracker.leftUp(750, { x: 100, y: 100 }); + expect(tracker.result().events).toEqual([ + { kind: "click", t: 100, x: 0.5, y: 0.5 }, + { kind: "drag", t: 200, t1: 600, x0: 0.1, y0: 0.2, x1: 0.5, y1: 0.2 }, + ]); + }); + + it("keeps a stroke drawn with Shift, and no stroke for a typed capital", () => { + const tracker = new ScrawlInputTracker(secondDisplay); + tracker.shiftDown(0, { x: 2000, y: 100 }); + expect(tracker.shiftUp()).toBe(false); + tracker.shiftDown(100, { x: 2000, y: 100 }); + expect(tracker.moved(110, { x: 2000, y: 100 })).toBeNull(); + tracker.moved(120, { x: 2100, y: 100 }); + expect(tracker.shiftUp()).toBe(true); + expect(tracker.result().strokes).toEqual([ + { t0: 100, t1: 120, points: [0.08, 0.2, 0.18, 0.2] }, + ]); + }); +}); diff --git a/apps/desktop/src/scrawl/inputTracker.ts b/apps/desktop/src/scrawl/inputTracker.ts new file mode 100644 index 0000000000000..37e6564e0d46b --- /dev/null +++ b/apps/desktop/src/scrawl/inputTracker.ts @@ -0,0 +1,117 @@ +// Turns button and Shift changes plus cursor positions into clicks, drags and strokes. + +import type { ScrawlInputEvent, ScrawlStroke } from "@t3tools/contracts"; + +export interface DisplayArea { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} + +export interface CursorPoint { + readonly x: number; + readonly y: number; +} + +// In screen points: a hand that moves this little while pressing meant a click. +const CLICK_SLOP = 4; +const MIN_STROKE_LENGTH = 6; + +const round4 = (value: number) => Math.round(value * 10_000) / 10_000; + +export class ScrawlInputTracker { + readonly #area: DisplayArea; + readonly #events: Array = []; + readonly #strokes: Array = []; + #press: { readonly t: number; readonly at: CursorPoint } | null = null; + #stroke: { readonly t0: number; readonly raw: Array; t1: number } | null = null; + + constructor(area: DisplayArea) { + this.#area = area; + } + + #inside(point: CursorPoint): boolean { + const { x, y, width, height } = this.#area; + return point.x >= x && point.y >= y && point.x < x + width && point.y < y + height; + } + + #normalized(point: CursorPoint): { readonly x: number; readonly y: number } { + return { + x: round4((point.x - this.#area.x) / this.#area.width), + y: round4((point.y - this.#area.y) / this.#area.height), + }; + } + + leftDown(t: number, at: CursorPoint): void { + this.#press = this.#inside(at) ? { t, at } : null; + } + + leftUp(t: number, at: CursorPoint): void { + const press = this.#press; + this.#press = null; + if (press === null) return; + const start = this.#normalized(press.at); + if (Math.hypot(at.x - press.at.x, at.y - press.at.y) <= CLICK_SLOP) { + this.#events.push({ kind: "click", t: press.t, x: start.x, y: start.y }); + return; + } + const end = this.#normalized(at); + this.#events.push({ + kind: "drag", + t: press.t, + t1: t, + x0: start.x, + y0: start.y, + x1: end.x, + y1: end.y, + }); + } + + shiftDown(t: number, at: CursorPoint): void { + this.#stroke = { t0: t, t1: t, raw: [at] }; + } + + /** Returns the new position when it extends the stroke being drawn. */ + moved(t: number, at: CursorPoint): CursorPoint | null { + const stroke = this.#stroke; + if (stroke === null) return null; + const last = stroke.raw.at(-1)!; + if (last.x === at.x && last.y === at.y) return null; + stroke.raw.push(at); + stroke.t1 = t; + return at; + } + + /** Ends the stroke. A Shift press without movement (typing a capital) leaves nothing behind. */ + shiftUp(): boolean { + const stroke = this.#stroke; + this.#stroke = null; + if (stroke === null) return false; + let length = 0; + for (let index = 1; index < stroke.raw.length; index++) { + const previous = stroke.raw[index - 1]!; + const point = stroke.raw[index]!; + length += Math.hypot(point.x - previous.x, point.y - previous.y); + } + if (length < MIN_STROKE_LENGTH) return false; + const inside = stroke.raw.filter((point) => this.#inside(point)); + if (inside.length < 2) return false; + this.#strokes.push({ + t0: stroke.t0, + t1: stroke.t1, + points: inside.flatMap((point) => { + const { x, y } = this.#normalized(point); + return [x, y]; + }), + }); + return true; + } + + result(): { + readonly events: ReadonlyArray; + readonly strokes: ReadonlyArray; + } { + return { events: this.#events, strokes: this.#strokes }; + } +} diff --git a/apps/server/src/dictation/Dictation.ts b/apps/server/src/dictation/Dictation.ts index 4f492b8509ed5..dc2d1d979f5e2 100644 --- a/apps/server/src/dictation/Dictation.ts +++ b/apps/server/src/dictation/Dictation.ts @@ -97,7 +97,8 @@ const make = Effect.gen(function* () { const finish = Effect.fn("Dictation.finish")(function* (input: DictationFinishInput) { const result = yield* Effect.tryPromise({ - try: () => engine.finish(input.sessionId), + try: () => + engine.finish(input.sessionId, input.words === undefined ? {} : { words: input.words }), catch: toDictationError, }); return finishResult(result); diff --git a/apps/server/src/dictation/DictationEngine.test.ts b/apps/server/src/dictation/DictationEngine.test.ts index fd612de076c66..960a40f3242c1 100644 --- a/apps/server/src/dictation/DictationEngine.test.ts +++ b/apps/server/src/dictation/DictationEngine.test.ts @@ -9,8 +9,10 @@ import { SAMPLE_RATE, decodePcm16, keepSpeech, + keptSpans, normalizeLanguage, splitAtPauses, + toRecordingMs, type TranscribeModule, } from "./DictationEngine.ts"; import { dictationBackend } from "./localDictation.ts"; @@ -136,6 +138,16 @@ describe("audio input", () => { expect(kept.length).toBe(Math.round(3.4 * SAMPLE_RATE)); expect(keepSpeech(audio, [])).toHaveLength(0); }); + it("puts a word back on the recording's clock across a cut pause", () => { + const spans = keptSpans([ + { startSample: SAMPLE_RATE, endSample: 2 * SAMPLE_RATE }, + { startSample: 9 * SAMPLE_RATE, endSample: 10 * SAMPLE_RATE }, + ]); + expect(toRecordingMs(500, spans)).toBe(1_500); + // 1 s of speech and the kept 0.4 s of the pause come before the second stretch. + expect(toRecordingMs(1_600, spans)).toBe(9_200); + expect(toRecordingMs(1_600, [])).toBe(1_600); + }); it("reads the backend from the environment, auto when unset or unknown", () => { expect(dictationBackend(" CPU ")).toBe("cpu"); expect(dictationBackend("vulkan")).toBe("vulkan"); @@ -159,7 +171,7 @@ class FakeModel { runs: Array<{ length: number; language: string | undefined }> = []; disposed = false; readonly backend = "CPU"; - readonly capabilities = { languages: ["de", "en"] }; + readonly capabilities = { languages: ["de", "en"], maxTimestampKind: "word" }; static async load(path: string) { FakeModel.loads.push(path); return new FakeModel(); @@ -170,7 +182,14 @@ class FakeModel { this.runs.push({ length: pcm.length, language: opts.language }); FakeModel.runLengths.push(pcm.length); if (FakeModel.runError !== undefined) throw FakeModel.runError; - return { text: " Hallo Welt ", aborted: false }; + return { + text: " Hallo Welt ", + aborted: false, + words: [ + { text: " Hallo", t0Ms: 100, t1Ms: 400 }, + { text: " Welt", t0Ms: 1_600, t1Ms: 1_800 }, + ], + }; }, dispose: () => undefined, }; @@ -230,6 +249,25 @@ describe("DictationEngine", () => { expect(FakeModel.loads).toEqual(["model.gguf"]); }); + it("times words on the recording's clock only when asked to", async () => { + FakeModel.vadSegments = [ + { startSample: SAMPLE_RATE, endSample: 2 * SAMPLE_RATE }, + { startSample: 9 * SAMPLE_RATE, endSample: 10 * SAMPLE_RATE }, + ]; + const { engine } = makeEngine(undefined, async () => "vad.gguf"); + engine.append("a", "de", speech(10)); + expect(await engine.finish("a", { words: true })).toEqual({ + text: "Hallo Welt", + words: [ + { text: "Hallo", t0: 1_100, t1: 1_400 }, + { text: "Welt", t0: 9_200, t1: 9_400 }, + ], + }); + engine.append("b", "de", speech(10)); + expect(await engine.finish("b")).toEqual({ text: "Hallo Welt" }); + FakeModel.vadSegments = []; + }); + it("transcribes a long recording in pieces and joins them", async () => { const { engine } = makeEngine(); engine.append("long", "de", speech(40)); diff --git a/apps/server/src/dictation/DictationEngine.ts b/apps/server/src/dictation/DictationEngine.ts index 0824790afa358..5873664aa59bc 100644 --- a/apps/server/src/dictation/DictationEngine.ts +++ b/apps/server/src/dictation/DictationEngine.ts @@ -47,8 +47,20 @@ export interface DictationEngineOptions { readonly log?: (message: string) => void; } +export interface TranscriptWord { + readonly text: string; + /** Milliseconds from the start of the recording, pauses included. */ + readonly t0: number; + readonly t1: number; +} + export type TranscriptResult = - | { readonly text: string; readonly reason?: undefined } + | { + readonly text: string; + readonly reason?: undefined; + /** Only when asked for and the model can time words. */ + readonly words?: ReadonlyArray; + } | { readonly text: ""; readonly reason: "no-speech" | "no-audio" }; interface LoadedModel { @@ -104,31 +116,56 @@ export function splitAtPauses(audio: Float32Array): ReadonlyArray return pieces; } -/** The audio of `segments`, with the pauses between them cut down to a short gap. */ -export function keepSpeech( - audio: Float32Array, - segments: ReadonlyArray, -): Float32Array { - const parts: Array = []; +/** A stretch of the shortened audio and where it began in the recording, in samples. */ +export interface KeptSpan { + readonly keptStart: number; + readonly recordingStart: number; + readonly length: number; +} + +export const keptSpans = (segments: ReadonlyArray): Array => { + const spans: Array = []; + let keptStart = 0; let previousEnd: number | undefined; + const add = (recordingStart: number, end: number) => { + spans.push({ keptStart, recordingStart, length: end - recordingStart }); + keptStart += end - recordingStart; + }; for (const { startSample, endSample } of segments) { if (previousEnd !== undefined) { - parts.push( - audio.subarray(previousEnd, Math.min(startSample, previousEnd + KEPT_PAUSE_SAMPLES)), - ); + add(previousEnd, Math.min(startSample, previousEnd + KEPT_PAUSE_SAMPLES)); } - parts.push(audio.subarray(startSample, endSample)); + add(startSample, endSample); previousEnd = endSample; } - const speech = new Float32Array(parts.reduce((sum, part) => sum + part.length, 0)); - let offset = 0; - for (const part of parts) { - speech.set(part, offset); - offset += part.length; + return spans; +}; + +/** The audio of `segments`, with the pauses between them cut down to a short gap. */ +export function keepSpeech( + audio: Float32Array, + segments: ReadonlyArray, +): Float32Array { + const spans = keptSpans(segments); + const speech = new Float32Array(spans.reduce((sum, span) => sum + span.length, 0)); + for (const span of spans) { + speech.set( + audio.subarray(span.recordingStart, span.recordingStart + span.length), + span.keptStart, + ); } return speech; } +/** Word times come back on the shortened audio; clicks and frames live on the recording's clock. */ +export function toRecordingMs(keptMs: number, spans: ReadonlyArray): number { + if (spans.length === 0) return keptMs; + const sample = (keptMs / 1000) * SAMPLE_RATE; + const span = + spans.find((candidate) => sample < candidate.keptStart + candidate.length) ?? spans.at(-1)!; + return Math.round(((span.recordingStart + (sample - span.keptStart)) / SAMPLE_RATE) * 1000); +} + export function decodePcm16(base64: string): Float32Array { const bytes = Buffer.from(base64, "base64"); const count = Math.floor(bytes.length / 2); @@ -194,7 +231,10 @@ export class DictationEngine { } /** Ends a session and transcribes all of it. */ - async finish(sessionId: string): Promise { + async finish( + sessionId: string, + options?: { readonly words?: boolean }, + ): Promise { const session = this.#sessions.get(sessionId); this.#sessions.delete(sessionId); this.#lastUsedAt = this.#now(); @@ -209,33 +249,52 @@ export class DictationEngine { // A preview run in flight is aborted instead of making the final run wait for it. this.#previewAbort?.abort(); const started = this.#now(); - const text = await this.#exclusive(async () => { + const canTimeWords = ["word", "token"].includes(loaded.model.capabilities.maxTimestampKind); + const wantWords = options?.words === true && canTimeWords; + const { text, words } = await this.#exclusive(async () => { const texts: Array = []; - for (const piece of splitAtPauses(await this.#speechOnly(audio))) { + const words: Array = []; + const { speech, spans } = await this.#speechWithSpans(audio); + let pieceStartMs = 0; + for (const piece of splitAtPauses(speech)) { + const collect = (result: TranscribeCpp.Transcript) => { + texts.push(result.text); + if (!wantWords) return; + for (const word of result.words) { + words.push({ + text: word.text.trim(), + t0: toRecordingMs(pieceStartMs + word.t0Ms, spans), + t1: toRecordingMs(pieceStartMs + word.t1Ms, spans), + }); + } + }; try { - const result = await loaded.session.run( - piece, - this.#runOptions(loaded, session.language), + collect( + await loaded.session.run(piece, { + ...this.#runOptions(loaded, session.language), + ...(wantWords ? { timestamps: "word" as const } : {}), + }), ); - texts.push(result.text); } catch (error) { const module = await this.#loadModule(); if (error instanceof module.OutputTruncated && error.partialResult !== undefined) { // One piece that ran away must not cost the rest of the recording. - texts.push(error.partialResult.text); + collect(error.partialResult); } else if (!(error instanceof module.InputTooShort)) { throw error; } } + pieceStartMs += (piece.length / SAMPLE_RATE) * 1000; } - return texts.join(" "); + return { text: texts.join(" "), words }; }); this.#lastUsedAt = this.#now(); const cleaned = text.split(/\s+/).filter(Boolean).join(" "); this.#options.log?.( `transcribed ${(audio.length / SAMPLE_RATE).toFixed(1)}s of audio in ${this.#now() - started}ms`, ); - return cleaned ? { text: cleaned } : { text: "", reason: "no-speech" }; + if (!cleaned) return { text: "", reason: "no-speech" }; + return wantWords ? { text: cleaned, words } : { text: cleaned }; } cancel(sessionId: string): void { @@ -314,14 +373,20 @@ export class DictationEngine { /** Must run inside #exclusive: a VAD session serves one run at a time. */ async #speechOnly(audio: Float32Array): Promise { + return (await this.#speechWithSpans(audio)).speech; + } + + async #speechWithSpans( + audio: Float32Array, + ): Promise<{ readonly speech: Float32Array; readonly spans: ReadonlyArray }> { const vad = await this.#loadVad(); - if (vad === undefined) return audio; + if (vad === undefined) return { speech: audio, spans: [] }; try { const { segments } = await vad.session.run(audio, VAD_OPTIONS); - return keepSpeech(audio, segments); + return { speech: keepSpeech(audio, segments), spans: keptSpans(segments) }; } catch (error) { this.#options.log?.(`voice activity detection failed, keeping pauses: ${describe(error)}`); - return audio; + return { speech: audio, spans: [] }; } } diff --git a/apps/server/src/dictation/localDictation.ts b/apps/server/src/dictation/localDictation.ts index 6ff0b002dc9a1..d51c67d831464 100644 --- a/apps/server/src/dictation/localDictation.ts +++ b/apps/server/src/dictation/localDictation.ts @@ -96,5 +96,12 @@ export function finishResult(result: TranscriptResult): DictationFinishResult { } const cleaned = splitCodeword(removeFillers(result.text), DEFAULT_STRUCTURE_CODEWORDS); if (cleaned.text === "") return { text: "", reason: "no-speech", structureRequested: false }; - return { text: cleaned.text, structureRequested: cleaned.found }; + if (result.words === undefined) return { text: cleaned.text, structureRequested: cleaned.found }; + // Fillers go word by word: the words carry the times a capture lines its clicks up with. + const words = result.words.filter((word) => removeFillers(word.text) !== ""); + return { + text: cleaned.text, + structureRequested: cleaned.found, + words: cleaned.found ? words.slice(0, -1) : words, + }; } diff --git a/apps/server/src/dictation/worker.ts b/apps/server/src/dictation/worker.ts index a27fbe325dae9..76e0c4c0752b7 100644 --- a/apps/server/src/dictation/worker.ts +++ b/apps/server/src/dictation/worker.ts @@ -21,7 +21,12 @@ export type DictationWorkerRequest = readonly language: string | undefined; readonly pcm: string; } - | { readonly id: number; readonly method: "finish"; readonly sessionId: string } + | { + readonly id: number; + readonly method: "finish"; + readonly sessionId: string; + readonly words?: boolean; + } | { readonly id: number; readonly method: "cancel"; readonly sessionId: string }; export type DictationWorkerResponse = @@ -50,7 +55,12 @@ export async function handleDictationWorkerRequest( return { id: request.id, ok: true, - value: finishResult(await engine.finish(request.sessionId)), + value: finishResult( + await engine.finish( + request.sessionId, + request.words === undefined ? {} : { words: request.words }, + ), + ), }; case "cancel": engine.cancel(request.sessionId); diff --git a/apps/web/src/components/chat/ChatComposer.tsx b/apps/web/src/components/chat/ChatComposer.tsx index 2f6b73d1dd379..d40066b10aa89 100644 --- a/apps/web/src/components/chat/ChatComposer.tsx +++ b/apps/web/src/components/chat/ChatComposer.tsx @@ -40,6 +40,7 @@ import { ProviderDriverKind, ProviderInstanceId, PROVIDER_SEND_TURN_MAX_ATTACHMENTS, + PROVIDER_SEND_TURN_MAX_IMAGE_BYTES, } from "@t3tools/contracts"; import type { EnvironmentConnectionPresentation } from "@t3tools/client-runtime/connection"; import { settlePromise, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; @@ -230,6 +231,8 @@ const isEnvironmentRequestInvalidError = Schema.is(EnvironmentRequestInvalidErro import { markDictatedText } from "@t3tools/client-runtime/voice-input/dictationMarker"; import { useDictation } from "../../dictation/useDictation"; import { ComposerDictationButton } from "./ComposerDictationButton"; +import { ComposerScrawlButton } from "./ComposerScrawlButton"; +import { cancelScrawl, type ScrawlDelivery } from "../../scrawl/scrawlCapture"; import { DictationHistoryDialog } from "./DictationHistoryDialog"; import { toastManager } from "../ui/toast"; import { PlayIcon } from "lucide-react"; @@ -2460,6 +2463,34 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) ensureLeadingBoundary: true, }), }); + // A stable object: the capture keeps it and calls the composer's latest handlers. + const scrawlInsertStable = useStableCallback((text: string) => + insertComposerTextStable(text, "end", { ensureLeadingBoundary: true }), + ); + const scrawlAttachStable = useStableCallback(async (file: File) => { + const { compressImageToByteLimit, readFileAsDataUrl } = + await import("../../lib/imageCompression"); + const compressed = await compressImageToByteLimit(file, PROVIDER_SEND_TURN_MAX_IMAGE_BYTES); + if (!compressed.ok) return false; + const previewUrl = await readFileAsDataUrl(compressed.file); + return ( + addComposerImagesToDraft([ + { + type: "image", + id: randomUUID(), + name: compressed.file.name, + mimeType: compressed.file.type, + sizeBytes: compressed.file.size, + previewUrl, + file: compressed.file, + }, + ]).length > 0 + ); + }); + const [scrawlDelivery] = useState(() => ({ + insert: scrawlInsertStable, + attachImage: scrawlAttachStable, + })); const toggleDictationStable = useStableCallback(dictation.toggle); const cancelDictationStable = useStableCallback(dictation.cancel); const insertDictationFromHistoryStable = useStableCallback(dictation.insertFromHistory); @@ -2472,6 +2503,11 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) if (event.key === "Escape") { // The open history takes Esc for itself, so closing it cannot drop a running recording. if (dictationHistoryOpen) return; + if (cancelScrawl()) { + event.preventDefault(); + event.stopPropagation(); + return; + } // Before anything else claims Esc (it can also interrupt a running turn). if (cancelDictationStable()) { event.preventDefault(); @@ -3488,6 +3524,7 @@ export const ChatComposer = memo(function ChatComposer(props: ChatComposerProps) >
{composerAttachButton} + registerScrawlDelivery(delivery), [delivery]); + + // oxlint-disable-next-line t3code/no-unreasoned-state-sync -- subscribes to the global shortcut, which arrives as a desktop menu action + useEffect( + () => + window.desktopBridge?.onMenuAction((action) => { + if (action === SCRAWL_TOGGLE_ACTION) toggleScrawl(); + }), + [], + ); + + if (!isScrawlAvailable()) return null; + const recording = phase === "recording"; + const busy = phase === "starting" || phase === "building"; + const status = + phase === "building" + ? "Building the capture" + : recording + ? `Recording this screen · ${SHORTCUT} finishes · Esc discards` + : null; + return ( + <> + {status !== null || notice !== null ? ( +
+ {status !== null ? ( + {status} + ) : notice !== null ? ( + {notice.text} + ) : null} +
+ ) : null} + + event.preventDefault()} + onClick={toggleScrawl} + disabled={busy} + aria-label={recording ? "Finish screen capture" : "Capture the screen"} + aria-pressed={recording} + /> + } + > + {busy ? : } + + + {recording ? `Finish capture (${SHORTCUT})` : `Capture the screen (${SHORTCUT})`} + + + + ); +} diff --git a/apps/web/src/dictation/microphoneCapture.ts b/apps/web/src/dictation/microphoneCapture.ts index 5c76f610b8177..73514b3cb36e8 100644 --- a/apps/web/src/dictation/microphoneCapture.ts +++ b/apps/web/src/dictation/microphoneCapture.ts @@ -13,6 +13,8 @@ export interface MicrophoneCapture { /** PCM recorded since the previous call, as base64 of 16-bit little-endian samples. */ readonly takePcm: () => string; readonly stop: () => void; + /** The raw track, for a screen recording that wants the voice too. */ + readonly stream: MediaStream; } // From the app root, not the current route: //x.js would hit the SPA fallback. @@ -76,6 +78,7 @@ export async function startMicrophoneCapture(): Promise { } return { + stream, takePcm: () => { const taken = blocks.splice(0, blocks.length); return encodePcm16(taken); diff --git a/apps/web/src/dictation/useDictation.ts b/apps/web/src/dictation/useDictation.ts index de9e12fe90548..6eae476072714 100644 --- a/apps/web/src/dictation/useDictation.ts +++ b/apps/web/src/dictation/useDictation.ts @@ -27,7 +27,7 @@ import { DEFAULT_DICTATION_LANGUAGE } from "./dictationLanguages"; const SEND_INTERVAL_MS = 500; // A new key: the old one may hold a choice made when a flag still showed it, now easy to overlook. -const LANGUAGE_KEY = "t3code:dictation-language-v2"; +export const DICTATION_LANGUAGE_KEY = "t3code:dictation-language-v2"; export type DictationPhase = "idle" | "starting" | "recording" | "transcribing" | "structuring"; @@ -95,7 +95,7 @@ export function useDictation(input: { const cancelSession = useAtomCommand(serverEnvironment.dictationCancel, { reportFailure: false }); const structure = useAtomCommand(serverEnvironment.dictationStructure, { reportFailure: false }); const [language, setLanguage] = useLocalStorage( - LANGUAGE_KEY, + DICTATION_LANGUAGE_KEY, DEFAULT_DICTATION_LANGUAGE, Schema.String, ); diff --git a/apps/web/src/scrawl/narration.test.ts b/apps/web/src/scrawl/narration.test.ts new file mode 100644 index 0000000000000..2c82b0b07cf7b --- /dev/null +++ b/apps/web/src/scrawl/narration.test.ts @@ -0,0 +1,146 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { + buildNarration, + foldActions, + formatNarrationLine, + splitSentences, + wordsOrWhole, + type TimedWord, +} from "./narration.ts"; + +const say = (text: string, start = 0, wordMs = 300): Array => + text.split(" ").map((word, index) => ({ + text: word, + t0: start + index * wordMs, + t1: start + (index + 1) * wordMs, + })); + +describe("splitSentences", () => { + it("keeps a thinking pause and a wrong full stop inside the sentence", () => { + const words = [ + ...say("Ich will hier bitte", 0), + ...say("eine andere Ansicht. der einzelnen Tage.", 3_000), + ...say("Neuer Satz.", 6_000), + ]; + expect(splitSentences(words).map((sentence) => sentence.words.join(" "))).toEqual([ + "Ich will hier bitte eine andere Ansicht. der einzelnen Tage.", + "Neuer Satz.", + ]); + }); + + it("gives up on an unfinished sentence after a long silence", () => { + const words = [...say("Ich will hier", 0), ...say("Neuer Satz.", 10_000)]; + expect(splitSentences(words)).toHaveLength(2); + }); +}); + +describe("foldActions", () => { + it("folds click bursts and drops the hand's small drags", () => { + const actions = foldActions( + [ + { kind: "click", t: 1_000, x: 0.1, y: 0.1 }, + { kind: "click", t: 1_500, x: 0.2, y: 0.1 }, + { kind: "click", t: 2_900, x: 0.3, y: 0.1 }, + { kind: "click", t: 9_000, x: 0.5, y: 0.5 }, + { kind: "drag", t: 12_000, t1: 12_200, x0: 0.5, y0: 0.5, x1: 0.51, y1: 0.5 }, + { kind: "drag", t: 13_000, t1: 13_400, x0: 0.5, y0: 0.5, x1: 0.1, y1: 0.5 }, + ], + [], + ); + expect(actions).toMatchObject([ + { kind: "clicks", count: 3, t0: 1_000, t1: 2_900 }, + { kind: "clicks", count: 1, t0: 9_000 }, + { kind: "drag", direction: "left" }, + ]); + }); +}); + +describe("buildNarration", () => { + it("never cuts a sentence at a click, and loses no word, click or mark", () => { + const words = say("Bitte diese Boxen hier entfernen und den Text darunter auch.", 0); + const lines = buildNarration({ + words, + events: [ + { kind: "click", t: 600, x: 0.1, y: 0.1 }, + { kind: "click", t: 2_000, x: 0.4, y: 0.4 }, + ], + marks: [{ id: "m1", t0: 900, t1: 1_100 }], + durationMs: 4_000, + }); + expect(lines.map((line) => line.text)).toEqual([words.map((word) => word.text).join(" ")]); + expect(lines[0]!.actions.map((action) => action.kind).toSorted()).toEqual(["clicks", "mark"]); + expect(formatNarrationLine(lines[0]!, 0)).toBe( + "- #1 (0:01) [marked, clicked ×2] “Bitte diese Boxen hier entfernen und den Text darunter auch.”", + ); + }); + + it("tells a click the screen answered from a hand that was pointing", () => { + const lines = buildNarration({ + words: say("Das hier und das hier.", 0), + events: [ + { kind: "click", t: 100, x: 0.1, y: 0.1 }, + { kind: "click", t: 200, x: 0.2, y: 0.1 }, + { kind: "click", t: 5_000, x: 0.5, y: 0.5 }, + ], + marks: [], + durationMs: 7_000, + sceneChanges: [5_400], + }); + expect(formatNarrationLine(lines[0]!, 0)).toContain("[pointed ×2]"); + expect(formatNarrationLine(lines[1]!, 1)).toBe("- #2 (0:05) clicked"); + }); + + it("says what a keyframe without words or actions shows", () => { + const lines = buildNarration({ + words: [], + events: [], + marks: [], + durationMs: 3_000, + sceneChanges: [2_000], + }); + expect(formatNarrationLine(lines[0]!, 0)).toBe("- #1 (0:02) screen changed"); + }); + + it("still has a keyframe when nothing was said or done", () => { + const lines = buildNarration({ words: [], events: [], marks: [], durationMs: 4_000 }); + expect(lines.map((line) => formatNarrationLine(line, 0))).toEqual([ + "- #1 (0:02) nothing said or done", + ]); + }); + + it("keeps the speech when the model could not time its words", () => { + const lines = buildNarration({ + words: wordsOrWhole([], "Das hier bitte weg. Und das auch.", 8_000), + events: [{ kind: "click", t: 2_000, x: 0.5, y: 0.5 }], + marks: [], + durationMs: 8_000, + }); + expect(lines.map((line) => line.text)).toEqual(["Das hier bitte weg. Und das auch."]); + }); + + it("gives a click said over no speech its own line", () => { + const lines = buildNarration({ + words: say("Hallo Welt.", 0), + events: [{ kind: "click", t: 20_000, x: 0.5, y: 0.5 }], + marks: [{ id: "m1", t0: 100, t1: 300 }], + durationMs: 25_000, + }); + expect(formatNarrationLine(lines.at(-1)!, 1)).toBe("- #2 (0:20) clicked"); + }); + + it("keeps every mark over budget, and speech ahead of the first keyframe with it", () => { + const marks = Array.from({ length: 9 }, (_, index) => ({ + id: `m${index}`, + t0: 2_000 + index * 2_000, + t1: 2_100 + index * 2_000, + })); + const words = [ + ...say("Erster Satz ohne Aktion.", 0, 100), + ...marks.flatMap((mark) => say("Das hier markiert.", mark.t0, 100)), + ]; + const lines = buildNarration({ words, events: [], marks, durationMs: 22_000 }); + expect(lines).toHaveLength(9); + expect(lines[0]!.text).toBe("Erster Satz ohne Aktion. Das hier markiert."); + }); +}); diff --git a/apps/web/src/scrawl/narration.ts b/apps/web/src/scrawl/narration.ts new file mode 100644 index 0000000000000..a9b67540af783 --- /dev/null +++ b/apps/web/src/scrawl/narration.ts @@ -0,0 +1,345 @@ +// Turns timed words and screen actions into the lines an agent reads. Speech and pointing do not +// run in step: on 160 real clicks the nearest pointing word ("hier", "das") lay anywhere from 3.3 s +// before to 3 s after the click, and 42% of the clicks came in bursts. So a sentence is never cut +// at an action. Whole sentences own a keyframe, and the actions said over them are listed in front. + +export interface TimedWord { + readonly t0: number; + readonly t1: number; + readonly text: string; +} + +export type CaptureEvent = + | { readonly kind: "click"; readonly t: number; readonly x: number; readonly y: number } + | { + readonly kind: "drag"; + readonly t: number; + readonly t1: number; + readonly x0: number; + readonly y0: number; + readonly x1: number; + readonly y1: number; + } + | { readonly kind: "scroll"; readonly t: number }; + +export interface CaptureMark { + readonly id: string; + readonly t0: number; + readonly t1: number; + readonly shape?: string; +} + +export type NarrationAction = + | { + readonly kind: "clicks"; + readonly t0: number; + readonly t1: number; + readonly count: number; + readonly points: ReadonlyArray; + /** False when the screen did not react: the hand was pointing, not operating anything. */ + readonly changed?: boolean; + } + | { readonly kind: "drag"; readonly t0: number; readonly t1: number; readonly direction: string } + | { readonly kind: "scroll"; readonly t0: number; readonly t1: number } + | { readonly kind: "mark"; readonly t0: number; readonly t1: number; readonly id: string } + | { readonly kind: "scene"; readonly t0: number; readonly t1: number }; + +export interface NarrationLine { + readonly t0: number; + readonly t1: number; + /** When the keyframe of this line should be taken. */ + readonly frameT: number; + readonly text: string; + readonly actions: ReadonlyArray; +} + +interface Sentence { + t0: number; + t1: number; + words: Array; + actions: Array; +} + +// Thinking pauses run long and the speech goes on where it stopped, so a pause ends only a finished +// sentence, or an unfinished one when it runs past this. +const ABANDONED_GAP_MS = 5_000; +const MAX_SENTENCE_WORDS = 60; +const BURST_GAP_MS = 1_500; +const REACTION_MS = 1_000; +const SCROLL_GAP_MS = 1_500; +// Scrawl's rule: a drag this short (as a share of the screen) is the hand moving, not an act. +const SHORT_DRAG = 0.15; +// An action in a speech pause goes to the sentence it is nearest to, up to this far away. +const ATTACH_MS = 3_000; +const SETTLE_MS = 300; +// Scrawl's keyframe budget: each image costs a reading model about 1.5k tokens. +const budgetFor = (durationMs: number) => + Math.min(12, 6 + Math.max(0, Math.ceil((durationMs - 40_000) / 20_000))); +const COVERAGE_MS = 35_000; +const COVERAGE_WORDS = 100; + +const SCORES: Record = { + mark: 100, + clicks: 60, + drag: 50, + scene: 40, + scroll: 30, +}; + +const endsSentence = (word: string) => /[.!?;:]["”’)\]]*$/.test(word); +const endsClause = (word: string) => /,["”’)\]]*$/.test(word); + +/** A model that cannot time words still said them: the speech stays whole on one line. */ +export const wordsOrWhole = ( + words: ReadonlyArray, + text: string, + durationMs: number, +): ReadonlyArray => + words.length > 0 || text === "" ? words : [{ text, t0: 0, t1: durationMs }]; + +export function splitSentences(words: ReadonlyArray): Array { + const sentences: Array = []; + let open: Sentence | null = null; + for (const word of [...words].sort((a, b) => a.t0 - b.t0)) { + const text = word.text.trim(); + if (text === "") continue; + if (open !== null && word.t0 - open.t1 >= ABANDONED_GAP_MS) open = null; + const last = sentences.at(-1); + // The model also ends sentences that go on: German starts a new one with a capital. + if (open === null && last !== undefined && word.t0 - last.t1 < ABANDONED_GAP_MS) { + if (/^\p{Ll}/u.test(text)) open = last; + } + if (open === null) { + open = { t0: word.t0, t1: word.t1, words: [text], actions: [] }; + sentences.push(open); + } else { + open.t1 = Math.max(open.t1, word.t1); + open.words.push(text); + } + const tooLong = open.words.length >= MAX_SENTENCE_WORDS && endsClause(text); + if (endsSentence(text) || tooLong) open = null; + } + return sentences; +} + +const directionOf = (dx: number, dy: number) => + Math.abs(dx) >= Math.abs(dy) ? (dx < 0 ? "left" : "right") : dy < 0 ? "up" : "down"; + +export function foldActions( + events: ReadonlyArray, + marks: ReadonlyArray, +): Array { + const actions: Array = marks.map((mark) => ({ + kind: "mark", + t0: mark.t0, + t1: mark.t1, + id: mark.id, + })); + const bursts: Array<{ + t0: number; + t1: number; + count: number; + points: Array; + }> = []; + const scrolls: Array<{ t0: number; t1: number }> = []; + for (const event of [...events].sort((a, b) => a.t - b.t)) { + if (event.kind === "click") { + const burst = bursts.at(-1); + if (burst !== undefined && event.t - burst.t1 <= BURST_GAP_MS) { + burst.t1 = event.t; + burst.count++; + burst.points.push([event.x, event.y]); + } else { + bursts.push({ t0: event.t, t1: event.t, count: 1, points: [[event.x, event.y]] }); + } + } else if (event.kind === "scroll") { + const scroll = scrolls.at(-1); + if (scroll !== undefined && event.t - scroll.t1 <= SCROLL_GAP_MS) scroll.t1 = event.t; + else scrolls.push({ t0: event.t, t1: event.t }); + } else { + const dx = event.x1 - event.x0; + const dy = event.y1 - event.y0; + if (Math.hypot(dx, dy) >= SHORT_DRAG) { + actions.push({ + kind: "drag", + t0: event.t, + t1: event.t1, + direction: directionOf(dx, dy), + }); + } + } + } + for (const burst of bursts) actions.push({ kind: "clicks", ...burst }); + for (const scroll of scrolls) actions.push({ kind: "scroll", ...scroll }); + return actions.sort((a, b) => a.t0 - b.t0); +} + +const attach = (sentences: Array, actions: ReadonlyArray) => { + for (const action of actions) { + const at = (action.t0 + action.t1) / 2; + let best: Sentence | null = null; + let bestDistance = Infinity; + for (const sentence of sentences) { + const distance = Math.max(sentence.t0 - at, at - sentence.t1, 0); + if (distance < bestDistance) { + best = sentence; + bestDistance = distance; + } + } + if (best !== null && bestDistance <= ATTACH_MS) { + best.actions.push(action); + } else { + sentences.push({ t0: action.t0, t1: action.t1, words: [], actions: [action] }); + sentences.sort((a, b) => a.t0 - b.t0); + } + } +}; + +const scoreOf = (sentence: Sentence) => + sentence.actions.reduce((best, action) => Math.max(best, SCORES[action.kind]), 0); + +/** The sentences that own a keyframe: every mark, then the strongest actions, then long stretches. */ +const chooseFrameOwners = (sentences: ReadonlyArray, durationMs: number) => { + const budget = budgetFor(durationMs); + const owners = new Set(); + const ranked = sentences + .map((sentence, index) => ({ index, score: scoreOf(sentence) })) + .filter((entry) => entry.score > 0) + .sort((a, b) => b.score - a.score || a.index - b.index); + for (const { index, score } of ranked) { + if (score >= SCORES.mark || owners.size < budget) owners.add(index); + } + if (sentences.length > 0) { + let stretchStart = 0; + let stretchWords = 0; + for (let index = 0; index < sentences.length; index++) { + const sentence = sentences[index]!; + if (owners.has(index)) { + stretchStart = sentence.t0; + stretchWords = sentence.words.length; + continue; + } + stretchWords += sentence.words.length; + const long = sentence.t1 - stretchStart > COVERAGE_MS || stretchWords > COVERAGE_WORDS; + if (long && owners.size < budget) { + owners.add(index); + stretchStart = sentence.t0; + stretchWords = sentence.words.length; + } + } + } + // Speech with nothing on screen still gets one frame to be read against. + if (owners.size === 0 && sentences.length > 0) owners.add(0); + return owners; +}; + +const frameTimeOf = (t0: number, actions: ReadonlyArray, durationMs: number) => { + const marks = actions.filter((action) => action.kind === "mark"); + const moment = marks.at(-1) ?? actions.at(-1); + if (moment === undefined) return t0; + const settle = moment.kind === "mark" ? 0 : SETTLE_MS; + return Math.min(moment.t1 + settle, durationMs); +}; + +/** Clicks the screen answered stay clicks; the rest are marked as pointing. */ +const withReactions = ( + actions: ReadonlyArray, + sceneChanges: ReadonlyArray, +): Array => { + const clickTimes = actions.filter((action) => action.kind === "clicks"); + const reacted = (action: { readonly t0: number; readonly t1: number }) => + sceneChanges.some((t) => t >= action.t0 && t <= action.t1 + REACTION_MS); + const scenes: Array = sceneChanges + .filter((t) => !clickTimes.some((a) => t >= a.t0 && t <= a.t1 + REACTION_MS)) + .map((t) => ({ kind: "scene", t0: t, t1: t })); + return [ + ...actions.map((action) => + action.kind === "clicks" ? { ...action, changed: reacted(action) } : action, + ), + ...scenes, + ].toSorted((a, b) => a.t0 - b.t0); +}; + +export function buildNarration(input: { + readonly words: ReadonlyArray; + readonly events: ReadonlyArray; + readonly marks: ReadonlyArray; + readonly durationMs: number; + /** When the screen changed; without it clicks are taken at face value. */ + readonly sceneChanges?: ReadonlyArray; +}): Array { + const sentences = splitSentences(input.words); + const folded = foldActions(input.events, input.marks); + attach( + sentences, + input.sceneChanges === undefined ? folded : withReactions(folded, input.sceneChanges), + ); + // Nothing said, done or changed: the picture is still what the capture shows. + if (sentences.length === 0) { + const frameT = input.durationMs / 2; + return [{ t0: 0, t1: input.durationMs, frameT, text: "", actions: [] }]; + } + const owners = chooseFrameOwners(sentences, input.durationMs); + const groups: Array> = []; + let ownerSeen = false; + sentences.forEach((sentence, index) => { + const owner = owners.has(index); + // Speech ahead of the first keyframe belongs to it. + if (groups.length === 0 || (owner && ownerSeen)) groups.push([sentence]); + else groups.at(-1)!.push(sentence); + ownerSeen ||= owner; + }); + return groups.map((group) => { + const actions = group.flatMap((sentence) => sentence.actions); + const t0 = group[0]!.t0; + return { + t0, + t1: Math.max(...group.map((sentence) => sentence.t1)), + frameT: frameTimeOf(t0, actions, input.durationMs), + text: group.flatMap((sentence) => sentence.words).join(" "), + actions, + }; + }); +} + +const formatClock = (ms: number) => { + const seconds = Math.round(ms / 1000); + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; +}; + +const describeActions = (actions: ReadonlyArray) => { + let marks = 0; + let clicks = 0; + let pointed = 0; + let scrolled = false; + const drags = new Set(); + for (const action of actions) { + if (action.kind === "mark") marks++; + else if (action.kind === "clicks") { + if (action.changed === false) pointed += action.count; + else clicks += action.count; + } else if (action.kind === "scroll") scrolled = true; + else if (action.kind === "drag") drags.add(action.direction); + } + const counted = (word: string, count: number) => (count > 1 ? `${word} ×${count}` : word); + return [ + marks > 0 ? counted("marked", marks) : null, + clicks > 0 ? counted("clicked", clicks) : null, + pointed > 0 ? counted("pointed", pointed) : null, + ...[...drags].map((direction) => `dragged ${direction}`), + scrolled ? "scrolled" : null, + ] + .filter((part) => part !== null) + .join(", "); +}; + +/** `- #2 (0:09) [clicked ×3] “…”`: the shape Scrawl pastes, which agents and skills already read. */ +export function formatNarrationLine(line: NarrationLine, index: number): string { + const head = `- #${index + 1} (${formatClock(line.frameT)})`; + const actions = describeActions(line.actions); + if (line.text === "") { + const shown = line.actions.length > 0 ? "screen changed" : "nothing said or done"; + return `${head} ${actions === "" ? shown : actions}`; + } + return `${head}${actions === "" ? "" : ` [${actions}]`} “${line.text}”`; +} diff --git a/apps/web/src/scrawl/sceneChanges.test.ts b/apps/web/src/scrawl/sceneChanges.test.ts new file mode 100644 index 0000000000000..a627d78832cdd --- /dev/null +++ b/apps/web/src/scrawl/sceneChanges.test.ts @@ -0,0 +1,29 @@ +import { describe, expect, it } from "vite-plus/test"; + +import { findSceneChanges, THUMB_HEIGHT, THUMB_WIDTH, withoutInkChanges } from "./sceneChanges.ts"; + +const blank = () => new Float32Array(THUMB_WIDTH * THUMB_HEIGHT); + +describe("withoutInkChanges", () => { + it("drops changes from drawing and fading ink, and keeps the rest", () => { + const strokes = [{ t0: 2_000, t1: 3_000 }]; + expect(withoutInkChanges([1_000, 2_500, 3_500, 6_000, 8_000, 9_000], strokes, 5_000)).toEqual([ + 1_000, 6_000, 9_000, + ]); + }); +}); + +describe("findSceneChanges", () => { + it("ignores a moving cursor and finds a real transition", () => { + const cursor = blank(); + cursor.fill(1, 10, 14); + const page = blank().fill(0.6); + expect( + findSceneChanges([ + { t: 0, luma: blank() }, + { t: 750, luma: cursor }, + { t: 1_500, luma: page }, + ]), + ).toEqual([1_500]); + }); +}); diff --git a/apps/web/src/scrawl/sceneChanges.ts b/apps/web/src/scrawl/sceneChanges.ts new file mode 100644 index 0000000000000..7f5035feb5258 --- /dev/null +++ b/apps/web/src/scrawl/sceneChanges.ts @@ -0,0 +1,64 @@ +// Finds moments the screen changed from small luma thumbnails, so nothing but a 64x36 grid is kept +// per sample. Scrawl's rule: a moving cursor spans at most a few of the 8x6 blocks, so the score is +// the fourth-worst block, and real transitions across at least four blocks still register. + +export const THUMB_WIDTH = 64; +export const THUMB_HEIGHT = 36; +const COLUMNS = 8; +const ROWS = 6; +const CHANGE_THRESHOLD = 0.03; + +export interface LumaSample { + readonly t: number; + readonly luma: Float32Array; +} + +function sceneDifference(a: Float32Array, b: Float32Array): number { + const blockWidth = THUMB_WIDTH / COLUMNS; + const blockHeight = THUMB_HEIGHT / ROWS; + let global = 0; + const blocks: Array = []; + for (let row = 0; row < ROWS; row++) { + for (let column = 0; column < COLUMNS; column++) { + let sum = 0; + for (let y = row * blockHeight; y < (row + 1) * blockHeight; y++) { + for (let x = column * blockWidth; x < (column + 1) * blockWidth; x++) { + const index = y * THUMB_WIDTH + x; + sum += Math.abs(a[index]! - b[index]!); + } + } + global += sum; + blocks.push(sum / (blockWidth * blockHeight)); + } + } + blocks.sort((x, y) => y - x); + return Math.max(global / (THUMB_WIDTH * THUMB_HEIGHT), Math.min(1, blocks[3]! * 1.4)); +} + +// One sample either way: a change is only seen at the next sample after it happened. +const INK_MARGIN_MS = 750; + +/** Drawing and fading ink changes the picture without the screen doing anything. */ +export function withoutInkChanges( + changes: ReadonlyArray, + strokes: ReadonlyArray<{ readonly t0: number; readonly t1: number }>, + inkHoldMs: number, +): Array { + return changes.filter( + (t) => + !strokes.some( + ({ t0, t1 }) => + (t >= t0 - INK_MARGIN_MS && t <= t1 + INK_MARGIN_MS) || + Math.abs(t - (t1 + inkHoldMs)) <= INK_MARGIN_MS, + ), + ); +} + +export function findSceneChanges(samples: ReadonlyArray): Array { + const changes: Array = []; + for (let index = 1; index < samples.length; index++) { + const difference = sceneDifference(samples[index - 1]!.luma, samples[index]!.luma); + if (difference >= CHANGE_THRESHOLD) changes.push(samples[index]!.t); + } + return changes; +} diff --git a/apps/web/src/scrawl/scrawlBundle.ts b/apps/web/src/scrawl/scrawlBundle.ts new file mode 100644 index 0000000000000..f90df615ecd55 --- /dev/null +++ b/apps/web/src/scrawl/scrawlBundle.ts @@ -0,0 +1,411 @@ +// Builds what an agent reads from a finished recording: keyframes taken from the video, the contact +// sheet, the summary text and the manifest. The folder keeps Scrawl's shape, so the Scrawl skill and +// the `scrawl` CLI (zoom, frames, export) read these bundles as well. + +import { + SCRAWL_INK_HOLD_MS, + type ScrawlStartResult, + type ScrawlStopResult, + type ScrawlStroke, +} from "@t3tools/contracts"; + +import { + buildNarration, + formatNarrationLine, + type CaptureMark, + type NarrationLine, + type TimedWord, +} from "./narration"; +import { withoutInkChanges } from "./sceneChanges"; + +const SHEET_LONG_EDGE = 1600; +const SHEET_GUTTER = 6; +const STROKE_GROUP_MS = 1_500; +const CROP_PADDING = 0.08; +// Cropping to most of the screen only loses context. +const MAX_CROP_AREA = 0.7; +const MIN_CROP_SIDE = 0.35; + +export interface CaptureRecording { + readonly session: ScrawlStartResult; + readonly input: ScrawlStopResult; + readonly words: ReadonlyArray; + readonly transcriptText: string; + readonly model: string; + readonly video: Blob; + readonly videoName: string; + /** Milliseconds between the start of the audio, which all times count from, and of the video. */ + readonly videoOffsetMs: number; + /** How far the input clock runs ahead of the audio clock. */ + readonly inputOffsetMs: number; + readonly durationMs: number; + readonly sceneChanges: ReadonlyArray; + readonly wav: Uint8Array; +} + +interface ManifestMark extends CaptureMark { + readonly strokes: ReadonlyArray; + readonly boxNorm: readonly [number, number, number, number]; + readonly shape: string; + readonly description: string; +} + +// Scrawl stores flat [x, y, dt] triples with positions in 0..=10000. +const STROKE_SCALE = 10_000; +const manifestStroke = (stroke: ScrawlStroke, width: number, height: number) => { + const count = stroke.points.length / 2; + const span = stroke.t1 - stroke.t0; + const pts: Array = []; + for (let index = 0; index < count; index++) { + pts.push( + Math.round(stroke.points[index * 2]! * STROKE_SCALE), + Math.round(stroke.points[index * 2 + 1]! * STROKE_SCALE), + count > 1 ? Math.round((span * index) / (count - 1)) : 0, + ); + } + return { t0: stroke.t0, t1: stroke.t1, fw: width, fh: height, pts }; +}; + +const KEYFRAME_REASONS: Record = { + mark: "mark", + clicks: "click", + drag: "click", + scene: "scene_change", + scroll: "scene_change", +}; + +export interface BuiltBundle { + readonly text: string; + readonly contactSheet: Blob; + readonly lines: ReadonlyArray; +} + +const strokesToMarks = (strokes: ReadonlyArray): Array => { + const groups: Array<{ t0: number; t1: number; strokes: Array }> = []; + strokes.forEach((stroke, index) => { + const open = groups.at(-1); + if (open !== undefined && stroke.t0 - open.t1 <= STROKE_GROUP_MS) { + open.t1 = stroke.t1; + open.strokes.push(index); + } else { + groups.push({ t0: stroke.t0, t1: stroke.t1, strokes: [index] }); + } + }); + return groups.map((group, index) => { + const xs: Array = []; + const ys: Array = []; + for (const strokeIndex of group.strokes) { + strokes[strokeIndex]!.points.forEach((value, at) => (at % 2 === 0 ? xs : ys).push(value)); + } + return { + id: `m${index + 1}`, + ...group, + boxNorm: [Math.min(...xs), Math.min(...ys), Math.max(...xs), Math.max(...ys)], + shape: "mark", + description: `marked (${group.strokes.length} ${group.strokes.length === 1 ? "stroke" : "strokes"})`, + }; + }); +}; + +/** Everything the hand touched, padded, or null when that is most of the screen anyway. */ +function activityBox(recording: Pick): { + readonly x: number; + readonly y: number; + readonly width: number; + readonly height: number; +} | null { + const xs: Array = []; + const ys: Array = []; + for (const event of recording.input.events) { + if (event.kind === "click") { + xs.push(event.x); + ys.push(event.y); + } else { + xs.push(event.x0, event.x1); + ys.push(event.y0, event.y1); + } + } + for (const stroke of recording.input.strokes) { + stroke.points.forEach((value, index) => (index % 2 === 0 ? xs : ys).push(value)); + } + if (xs.length === 0) return null; + const widen = (low: number, high: number) => { + let start = low - CROP_PADDING; + let end = high + CROP_PADDING; + if (end - start < MIN_CROP_SIDE) { + const middle = (start + end) / 2; + start = middle - MIN_CROP_SIDE / 2; + end = middle + MIN_CROP_SIDE / 2; + } + start = Math.max(0, start); + end = Math.min(1, end); + return [start, end - start] as const; + }; + const [x, width] = widen(Math.min(...xs), Math.max(...xs)); + const [y, height] = widen(Math.min(...ys), Math.max(...ys)); + return width * height > MAX_CROP_AREA ? null : { x, y, width, height }; +} + +const seekTo = (video: HTMLVideoElement, seconds: number) => + new Promise((resolve, reject) => { + const onSeeked = () => { + video.removeEventListener("seeked", onSeeked); + video.removeEventListener("error", onError); + resolve(); + }; + const onError = () => { + video.removeEventListener("seeked", onSeeked); + video.removeEventListener("error", onError); + reject(new Error("The recording cannot be read.")); + }; + video.addEventListener("seeked", onSeeked); + video.addEventListener("error", onError); + video.currentTime = seconds; + }); + +const loadVideo = async (blob: Blob) => { + const video = document.createElement("video"); + video.muted = true; + video.preload = "auto"; + const url = URL.createObjectURL(blob); + video.src = url; + await new Promise((resolve, reject) => { + video.onloadedmetadata = () => resolve(); + video.onerror = () => reject(new Error("The recording cannot be read.")); + }); + // A recorder's file does not state its length until the player has walked to the end. + if (!Number.isFinite(video.duration)) { + await seekTo(video, 1e7); + } + return { video, dispose: () => URL.revokeObjectURL(url) }; +}; + +const toBlob = (canvas: HTMLCanvasElement) => + new Promise((resolve, reject) => { + canvas.toBlob( + (blob) => (blob === null ? reject(new Error("A frame cannot be encoded.")) : resolve(blob)), + "image/png", + ); + }); + +const drawClickRings = ( + context: CanvasRenderingContext2D, + line: NarrationLine, + width: number, + height: number, +) => { + context.lineWidth = Math.max(2, width / 700); + context.strokeStyle = "rgba(239, 68, 68, 0.85)"; + for (const action of line.actions) { + if (action.kind !== "clicks") continue; + for (const [x, y] of action.points) { + context.beginPath(); + context.arc(x * width, y * height, Math.max(10, width / 90), 0, Math.PI * 2); + context.stroke(); + } + } +}; + +const drawChip = (context: CanvasRenderingContext2D, label: string, height: number) => { + // Blind reading tests in Scrawl: the number needs its own white square to be read as a number. + const size = Math.max(24, height / 18); + context.fillStyle = "#ffffff"; + context.strokeStyle = "#111111"; + context.lineWidth = 2; + context.beginPath(); + context.roundRect(5, 5, size, size, 4); + context.fill(); + context.stroke(); + context.fillStyle = "#000000"; + context.font = `700 ${Math.round(size * 0.68)}px system-ui`; + context.textAlign = "center"; + context.textBaseline = "middle"; + context.fillText(label, 5 + size / 2, 5 + size / 2 + 1); +}; + +/** Frames in time order in justified rows, each with its number. */ +const drawContactSheet = (frames: ReadonlyArray) => { + const columns = Math.max(1, Math.ceil(Math.sqrt(frames.length))); + const rows = Math.ceil(frames.length / columns); + const first = frames[0]!; + const aspect = first.width / first.height; + const landscape = aspect >= 1; + const cellWidth = landscape + ? Math.floor((SHEET_LONG_EDGE - SHEET_GUTTER * (columns + 1)) / columns) + : Math.floor((SHEET_LONG_EDGE / aspect - SHEET_GUTTER * (columns + 1)) / columns); + const cellHeight = Math.round(cellWidth / aspect); + const sheet = document.createElement("canvas"); + sheet.width = columns * cellWidth + (columns + 1) * SHEET_GUTTER; + sheet.height = rows * cellHeight + (rows + 1) * SHEET_GUTTER; + const context = sheet.getContext("2d")!; + context.fillStyle = "#101014"; + context.fillRect(0, 0, sheet.width, sheet.height); + frames.forEach((frame, index) => { + const x = SHEET_GUTTER + (index % columns) * (cellWidth + SHEET_GUTTER); + const y = SHEET_GUTTER + Math.floor(index / columns) * (cellHeight + SHEET_GUTTER); + context.save(); + context.translate(x, y); + context.drawImage(frame, 0, 0, cellWidth, cellHeight); + drawChip(context, String(index + 1), cellHeight); + context.restore(); + }); + return sheet; +}; + +const formatDuration = (ms: number) => { + const seconds = Math.round(ms / 1000); + return `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, "0")}`; +}; + +const summaryText = ( + recording: Pick, + lines: ReadonlyArray, + cropped: boolean, +) => { + const { app, windowTitle, bundlePath } = recording.session; + const source = [app, windowTitle ? `“${windowTitle}”` : ""].filter(Boolean).join(" ") || "Screen"; + return [ + `[scrawl ${formatDuration(recording.durationMs)} · ${source} · ${bundlePath}]`, + "Open contact-sheet.png and frames/ in that folder before changing anything: the marks and words only make sense on the images, which can show more than the text does.", + ...(cropped ? ["Cropped to: the activity (full frames in frames-full/)"] : []), + ...lines.map(formatNarrationLine), + ].join("\n"); +}; + +export async function buildBundle( + recording: CaptureRecording, + write: (path: string, bytes: Uint8Array) => Promise, +): Promise { + const shift = (t: number) => Math.max(0, t - recording.inputOffsetMs); + const events = recording.input.events.map((event) => + event.kind === "click" + ? { ...event, t: shift(event.t) } + : { ...event, t: shift(event.t), t1: shift(event.t1) }, + ); + const strokes = recording.input.strokes.map((stroke) => ({ + ...stroke, + t0: shift(stroke.t0), + t1: shift(stroke.t1), + })); + const marks = strokesToMarks(strokes); + const lines = buildNarration({ + words: recording.words, + events, + marks, + durationMs: recording.durationMs, + sceneChanges: withoutInkChanges(recording.sceneChanges, strokes, SCRAWL_INK_HOLD_MS), + }); + const box = activityBox({ input: { events, strokes } }); + + const { video, dispose } = await loadVideo(recording.video); + const frames: Array = []; + const fullFrames: Array = []; + const bytesOf = async (blob: Blob) => new Uint8Array(await blob.arrayBuffer()); + try { + for (const [index, line] of lines.entries()) { + const seconds = Math.max(0, (line.frameT - recording.videoOffsetMs) / 1000); + await seekTo(video, Math.min(seconds, Math.max(0, video.duration - 0.05))); + const full = document.createElement("canvas"); + full.width = video.videoWidth; + full.height = video.videoHeight; + const context = full.getContext("2d")!; + context.drawImage(video, 0, 0); + drawClickRings(context, line, full.width, full.height); + const name = `frame-${String(index + 1).padStart(2, "0")}.png`; + if (box === null) { + await write(`frames/${name}`, await bytesOf(await toBlob(full))); + frames.push(full); + continue; + } + await write(`frames-full/${name}`, await bytesOf(await toBlob(full))); + fullFrames.push(full); + const cropped = document.createElement("canvas"); + cropped.width = Math.round(box.width * full.width); + cropped.height = Math.round(box.height * full.height); + cropped + .getContext("2d")! + .drawImage(full, -Math.round(box.x * full.width), -Math.round(box.y * full.height)); + await write(`frames/${name}`, await bytesOf(await toBlob(cropped))); + frames.push(cropped); + } + } finally { + dispose(); + } + + const sheet = drawContactSheet(frames); + const contactSheet = await toBlob(sheet); + await write("contact-sheet.png", await bytesOf(contactSheet)); + if (fullFrames.length > 0) { + await write( + "contact-sheet-full.png", + await bytesOf(await toBlob(drawContactSheet(fullFrames))), + ); + } + await write("audio.wav", recording.wav); + + const text = summaryText(recording, lines, box !== null); + const encoder = new TextEncoder(); + await write("summary.md", encoder.encode(`${text}\n`)); + const { width, height } = recording.session; + const manifest = { + version: 1, + id: recording.session.id, + createdAtMs: Date.now(), + durationMs: recording.durationMs, + target: { + mode: "display", + app: recording.session.app, + windowTitle: recording.session.windowTitle, + displayId: 0, + frame: { x: 0, y: 0, width, height, scale: 1 }, + }, + transcript: { + source: recording.model, + model: recording.model, + text: recording.transcriptText, + segments: [], + words: recording.words, + }, + strokes: strokes.map((stroke) => manifestStroke(stroke, width, height)), + marks, + events: events.map((event) => (event.kind === "click" ? { ...event, button: "left" } : event)), + keyframes: lines.map((line, index) => { + const main = line.actions.find((action) => action.kind === "mark") ?? line.actions.at(-1); + return { + index: index + 1, + t: Math.round(line.frameT), + reason: main === undefined ? "speech" : KEYFRAME_REASONS[main.kind], + file: `frames/frame-${String(index + 1).padStart(2, "0")}.png`, + event: main?.kind ?? "speech", + said: line.text, + }; + }), + contactSheet: "contact-sheet.png", + video: recording.videoName, + videoOffsetMs: recording.videoOffsetMs, + audio: "audio.wav", + windows: [], + cursor: [], + ...(box === null + ? {} + : { + autocrop: { + label: "the activity", + fullFrames: "frames-full", + fullContactSheet: "contact-sheet-full.png", + crops: lines.map((_line, index) => ({ + index: index + 1, + x: Math.round(box.x * width), + y: Math.round(box.y * height), + width: Math.round(box.width * width), + height: Math.round(box.height * height), + kind: "focus", + reason: "around the activity", + apps: [recording.session.app], + })), + }, + }), + }; + await write("scrawl.json", encoder.encode(JSON.stringify(manifest))); + return { text, contactSheet, lines }; +} diff --git a/apps/web/src/scrawl/scrawlCapture.ts b/apps/web/src/scrawl/scrawlCapture.ts new file mode 100644 index 0000000000000..f431c3b7cb04d --- /dev/null +++ b/apps/web/src/scrawl/scrawlCapture.ts @@ -0,0 +1,382 @@ +// One screen capture at a time, kept outside React: the shortcut works from any other program, and +// switching threads while recording must not end it. Whichever composer is on screen at the end +// receives the result. + +import type { DesktopBridge, ScrawlStartResult } from "@t3tools/contracts"; +import * as Schema from "effect/Schema"; + +import { startMicrophoneCapture, type MicrophoneCapture } from "../dictation/microphoneCapture"; +import { DEFAULT_DICTATION_LANGUAGE } from "../dictation/dictationLanguages"; +import { DICTATION_LANGUAGE_KEY } from "../dictation/useDictation"; +import { getLocalStorageItem } from "../hooks/useLocalStorage"; +import { randomUUID } from "../lib/utils"; +import { wordsOrWhole } from "./narration"; +import { buildBundle } from "./scrawlBundle"; +import { findSceneChanges, THUMB_HEIGHT, THUMB_WIDTH, type LumaSample } from "./sceneChanges"; + +const SEND_INTERVAL_MS = 500; +const SAMPLE_INTERVAL_MS = 750; +const VIDEO_TIMESLICE_MS = 2_000; +// Keyframes come out of this video, so small text has to survive it. +const VIDEO_BITS_PER_SECOND = 6_000_000; +const MAX_FRAME_RATE = 15; + +export type ScrawlPhase = "idle" | "starting" | "recording" | "building"; + +export interface ScrawlState { + readonly phase: ScrawlPhase; + readonly notice: { readonly text: string; readonly tone: "error" | "warning" | "success" } | null; +} + +export interface ScrawlDelivery { + readonly insert: (text: string) => boolean; + readonly attachImage: (file: File) => Promise; +} + +interface ImageCaptureLike { + grabFrame(): Promise; +} +declare const ImageCapture: new (track: MediaStreamTrack) => ImageCaptureLike; + +interface Running { + readonly session: ScrawlStartResult; + readonly sessionId: string; + readonly screen: MediaStream; + readonly mic: MicrophoneCapture; + readonly recorder: MediaRecorder; + readonly videoName: string; + readonly chunks: Array; + readonly samples: Array; + readonly pcm: Array; + readonly micStartedAt: number; + readonly language: string; + readonly pcmTimer: ReturnType; + readonly sampleTimer: ReturnType; + videoStartedAt: number; + writes: Promise; + sending: Promise; +} + +const IDLE: ScrawlState = { phase: "idle", notice: null }; + +let state: ScrawlState = IDLE; +let running: Running | null = null; +let delivery: ScrawlDelivery | null = null; +let noticeTimer: ReturnType | null = null; +const listeners = new Set<() => void>(); + +const setState = (next: ScrawlState) => { + state = next; + for (const listener of listeners) listener(); +}; + +const notify = (notice: NonNullable) => { + if (noticeTimer !== null) clearTimeout(noticeTimer); + setState({ phase: "idle", notice }); + noticeTimer = setTimeout(() => { + noticeTimer = null; + if (state.notice === notice) setState({ ...state, notice: null }); + }, 8_000); +}; + +const describe = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +export const subscribeScrawl = (listener: () => void) => { + listeners.add(listener); + return () => listeners.delete(listener); +}; +export const getScrawlState = () => state; + +/** The composer on screen takes the result; the latest to mount wins, as the visible one did. */ +export const registerScrawlDelivery = (target: ScrawlDelivery) => { + delivery = target; + return () => { + if (delivery === target) delivery = null; + }; +}; + +const scrawlBridge = (): DesktopBridge | null => + window.desktopBridge?.scrawlStart === undefined ? null : window.desktopBridge; + +export const isScrawlAvailable = () => scrawlBridge() !== null; + +const openScreen = (sourceId: string) => + navigator.mediaDevices.getUserMedia({ + audio: false, + video: { + mandatory: { + chromeMediaSource: "desktop", + chromeMediaSourceId: sourceId, + maxFrameRate: MAX_FRAME_RATE, + }, + }, + } as unknown as MediaStreamConstraints); + +const chooseVideoFormat = () => + MediaRecorder.isTypeSupported("video/mp4;codecs=avc1.640032") + ? { mimeType: "video/mp4;codecs=avc1.640032", name: "recording.mp4" } + : { mimeType: "video/webm;codecs=vp9", name: "recording.webm" }; + +/** Keeps one small grey thumbnail per sample; nothing at full size stays in memory. */ +const startSampling = (track: MediaStreamTrack, startedAt: number, samples: Array) => { + const grabber = new ImageCapture(track); + const canvas = document.createElement("canvas"); + canvas.width = THUMB_WIDTH; + canvas.height = THUMB_HEIGHT; + const context = canvas.getContext("2d", { willReadFrequently: true })!; + let busy = false; + return setInterval(() => { + if (busy) return; + busy = true; + void grabber + .grabFrame() + .then((frame) => { + const t = Date.now() - startedAt; + context.drawImage(frame, 0, 0, THUMB_WIDTH, THUMB_HEIGHT); + frame.close(); + const { data } = context.getImageData(0, 0, THUMB_WIDTH, THUMB_HEIGHT); + const luma = new Float32Array(THUMB_WIDTH * THUMB_HEIGHT); + for (let index = 0; index < luma.length; index++) { + const offset = index * 4; + luma[index] = + (0.2126 * data[offset]! + 0.7152 * data[offset + 1]! + 0.0722 * data[offset + 2]!) / + 255; + } + samples.push({ t, luma }); + }) + // A frame missed while the track starts or ends only thins the samples. + .catch(() => undefined) + .finally(() => { + busy = false; + }); + }, SAMPLE_INTERVAL_MS); +}; + +const decodeBase64 = (value: string) => { + const binary = atob(value); + const bytes = new Uint8Array(binary.length); + for (let index = 0; index < binary.length; index++) bytes[index] = binary.charCodeAt(index); + return bytes; +}; + +/** 16 kHz mono 16-bit, as the dictation pipeline records it. */ +const encodeWav = (pieces: ReadonlyArray) => { + const length = pieces.reduce((sum, piece) => sum + piece.length, 0); + const wav = new Uint8Array(44 + length); + const view = new DataView(wav.buffer); + const ascii = (offset: number, text: string) => { + for (let index = 0; index < text.length; index++) { + view.setUint8(offset + index, text.charCodeAt(index)); + } + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + length, true); + ascii(8, "WAVEfmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); + view.setUint16(22, 1, true); + view.setUint32(24, 16_000, true); + view.setUint32(28, 32_000, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, length, true); + let offset = 44; + for (const piece of pieces) { + wav.set(piece, offset); + offset += piece.length; + } + return wav; +}; + +const sendPcm = (bridge: DesktopBridge, capture: Running, pcm: string) => { + capture.pcm.push(pcm); + capture.sending = capture.sending.then(async () => { + // The recording goes on without words when the speech model is missing; the end says so. + await bridge + .dictationAppend?.({ sessionId: capture.sessionId, language: capture.language, pcm }) + .catch(() => undefined); + }); +}; + +const start = async (bridge: DesktopBridge) => { + setState({ phase: "starting", notice: null }); + let session: ScrawlStartResult | null = null; + let screen: MediaStream | null = null; + let mic: MicrophoneCapture | null = null; + try { + session = await bridge.scrawlStart!(); + screen = await openScreen(session.sourceId); + mic = await startMicrophoneCapture(); + const micStartedAt = Date.now(); + const format = chooseVideoFormat(); + const recorder = new MediaRecorder( + new MediaStream([...screen.getVideoTracks(), ...mic.stream.getAudioTracks()]), + { mimeType: format.mimeType, videoBitsPerSecond: VIDEO_BITS_PER_SECOND }, + ); + const samples: Array = []; + const capture: Running = { + session, + sessionId: randomUUID(), + screen, + mic, + recorder, + videoName: format.name, + chunks: [], + samples, + pcm: [], + micStartedAt, + language: + getLocalStorageItem(DICTATION_LANGUAGE_KEY, Schema.String) ?? DEFAULT_DICTATION_LANGUAGE, + pcmTimer: setInterval(() => { + if (running !== null) sendPcm(bridge, running, running.mic.takePcm()); + }, SEND_INTERVAL_MS), + sampleTimer: startSampling(screen.getVideoTracks()[0]!, micStartedAt, samples), + videoStartedAt: micStartedAt, + writes: Promise.resolve(), + sending: Promise.resolve(), + }; + recorder.onstart = () => { + capture.videoStartedAt = Date.now(); + }; + recorder.ondataavailable = (event) => { + if (event.data.size === 0) return; + capture.chunks.push(event.data); + // Written as it comes, so a crash keeps the video recorded so far. + capture.writes = capture.writes.then(async () => { + await bridge.scrawlAppendVideo?.({ + id: capture.session.id, + name: capture.videoName, + bytes: new Uint8Array(await event.data.arrayBuffer()), + }); + }); + }; + recorder.start(VIDEO_TIMESLICE_MS); + running = capture; + setState({ phase: "recording", notice: null }); + } catch (error) { + for (const track of screen?.getTracks() ?? []) track.stop(); + mic?.stop(); + if (session !== null) await bridge.scrawlCancel?.({ id: session.id }).catch(() => undefined); + notify({ text: `Recording cannot start: ${describe(error)}`, tone: "error" }); + } +}; + +const released = new WeakSet(); +const release = (capture: Running) => { + if (released.has(capture)) return; + released.add(capture); + clearInterval(capture.pcmTimer); + clearInterval(capture.sampleTimer); + for (const track of capture.screen.getTracks()) track.stop(); + capture.mic.stop(); +}; + +const stopRecorder = (recorder: MediaRecorder) => + new Promise((resolve) => { + if (recorder.state === "inactive") { + resolve(); + return; + } + recorder.onstop = () => resolve(); + recorder.stop(); + }); + +const deliver = async (text: string, contactSheet: Blob, bundlePath: string) => { + if (delivery === null || !delivery.insert(`${text}\n`)) { + // The clipboard refuses a window that is not in front, and this one usually is not. + const copied = await navigator.clipboard?.writeText(text).then( + () => true, + () => false, + ); + notify({ + text: copied + ? "No composer can take the capture · its text is on the clipboard" + : `No composer can take the capture · it is saved in ${bundlePath}`, + tone: "warning", + }); + return; + } + const attached = await delivery.attachImage( + new File([contactSheet], "contact-sheet.png", { type: "image/png" }), + ); + notify( + attached + ? { text: "Capture added to the composer", tone: "success" } + : { text: "Capture text added · the contact sheet could not be attached", tone: "warning" }, + ); +}; + +const finish = async (bridge: DesktopBridge, capture: Running) => { + running = null; + setState({ phase: "building", notice: null }); + try { + const input = await bridge.scrawlStop!({ id: capture.session.id }); + const durationMs = Date.now() - capture.micStartedAt; + await stopRecorder(capture.recorder); + sendPcm(bridge, capture, capture.mic.takePcm()); + release(capture); + await capture.sending; + await capture.writes; + const transcript = await bridge.dictationFinish?.({ + sessionId: capture.sessionId, + words: true, + }); + const transcriptText = transcript?.ok ? transcript.value.text : ""; + const words = wordsOrWhole( + transcript?.ok ? (transcript.value.words ?? []) : [], + transcriptText, + durationMs, + ); + const built = await buildBundle( + { + session: capture.session, + input, + words, + transcriptText, + model: "parakeet", + video: new Blob(capture.chunks, { type: capture.recorder.mimeType }), + videoName: capture.videoName, + videoOffsetMs: capture.videoStartedAt - capture.micStartedAt, + inputOffsetMs: capture.micStartedAt - capture.session.startedAtMs, + durationMs, + sceneChanges: findSceneChanges(capture.samples), + wav: encodeWav(capture.pcm.map(decodeBase64)), + }, + (path, bytes) => bridge.scrawlWriteFile!({ id: capture.session.id, path, bytes }), + ); + await deliver(built.text, built.contactSheet, capture.session.bundlePath); + if (transcript !== undefined && !transcript.ok) { + notify({ text: `Capture has no transcript: ${transcript.reason}`, tone: "warning" }); + } + } catch (error) { + release(capture); + notify({ + text: `The capture could not be built: ${describe(error)}. Its video is in ${capture.session.bundlePath}`, + tone: "error", + }); + } +}; + +/** Starts a capture, or finishes the running one. */ +export const toggleScrawl = () => { + const bridge = scrawlBridge(); + if (bridge === null) return; + if (state.phase === "idle") void start(bridge); + else if (state.phase === "recording" && running !== null) void finish(bridge, running); +}; + +/** Drops a running capture and its files. Returns whether there was one. */ +export const cancelScrawl = (): boolean => { + const bridge = scrawlBridge(); + const capture = running; + if (bridge === null || capture === null) return false; + running = null; + capture.recorder.ondataavailable = null; + capture.recorder.stop(); + release(capture); + void bridge.dictationCancel?.({ sessionId: capture.sessionId }).catch(() => undefined); + void bridge.scrawlCancel?.({ id: capture.session.id }).catch(() => undefined); + setState(IDLE); + return true; +}; diff --git a/packages/contracts/src/dictation.ts b/packages/contracts/src/dictation.ts index 7270e337cf3e7..f387d3ebc66e1 100644 --- a/packages/contracts/src/dictation.ts +++ b/packages/contracts/src/dictation.ts @@ -30,18 +30,29 @@ export type DictationAppendResult = typeof DictationAppendResult.Type; export const DictationFinishInput = Schema.Struct({ sessionId: DictationSessionId, + /** Also time each word. Only screen captures need this; dictation keeps its reply small. */ + words: Schema.optional(Schema.Boolean), }); export type DictationFinishInput = typeof DictationFinishInput.Type; export const DictationNoTextReason = Schema.Literals(["no-speech", "no-audio"]); export type DictationNoTextReason = typeof DictationNoTextReason.Type; +export const DictationWord = Schema.Struct({ + text: Schema.String, + /** Milliseconds from the start of the recording, pauses included. */ + t0: Schema.Number, + t1: Schema.Number, +}); +export type DictationWord = typeof DictationWord.Type; + export const DictationFinishResult = Schema.Struct({ /** Cleaned transcript; empty when nothing was recognized, and `reason` says why. */ text: Schema.String, reason: Schema.optional(DictationNoTextReason), /** The speaker ended with the codeword that asks for a structured rewrite (already cut). */ structureRequested: Schema.Boolean, + words: Schema.optional(Schema.Array(DictationWord)), }); export type DictationFinishResult = typeof DictationFinishResult.Type; diff --git a/packages/contracts/src/index.ts b/packages/contracts/src/index.ts index 4a3e8df253d9d..3b6f13bc22916 100644 --- a/packages/contracts/src/index.ts +++ b/packages/contracts/src/index.ts @@ -1,4 +1,5 @@ export * from "./baseSchemas.ts"; +export * from "./scrawl.ts"; export * from "./assistantCitations.ts"; export * from "./composerContext.ts"; export * from "./composerContextClipboard.ts"; diff --git a/packages/contracts/src/ipc.ts b/packages/contracts/src/ipc.ts index bf5f6730c5a06..f286eb1e9f7e6 100644 --- a/packages/contracts/src/ipc.ts +++ b/packages/contracts/src/ipc.ts @@ -19,6 +19,13 @@ import type { DictationFinishInput, DictationFinishResult, } from "./dictation.ts"; +import type { + ScrawlAppendVideoInput, + ScrawlCaptureRef, + ScrawlStartResult, + ScrawlStopResult, + ScrawlWriteFileInput, +} from "./scrawl.ts"; import { ExecutionEnvironmentDescriptor } from "./environment.ts"; import { type ClientSettings, type QuitConfirmationMode, SnapShotShortcut } from "./settings.ts"; import type { EditorId } from "./editor.ts"; @@ -1051,6 +1058,12 @@ export interface DesktopBridge { input: DictationFinishInput, ) => Promise>; dictationCancel?: (input: DictationCancelInput) => Promise; + /** Screen captures with voice, clicks and ink; only the desktop app can see the whole screen. */ + scrawlStart?: () => Promise; + scrawlStop?: (input: ScrawlCaptureRef) => Promise; + scrawlCancel?: (input: ScrawlCaptureRef) => Promise; + scrawlAppendVideo?: (input: ScrawlAppendVideoInput) => Promise; + scrawlWriteFile?: (input: ScrawlWriteFileInput) => Promise; getLocalEnvironmentBearerToken: () => Promise; getClientSettings: () => Promise; setClientSettings: (settings: ClientSettings) => Promise; diff --git a/packages/contracts/src/scrawl.ts b/packages/contracts/src/scrawl.ts new file mode 100644 index 0000000000000..665833f19d28b --- /dev/null +++ b/packages/contracts/src/scrawl.ts @@ -0,0 +1,79 @@ +import * as Schema from "effect/Schema"; + +// Scrawl: a screen capture with voice, clicks and red ink, packed into a bundle for agents. The +// desktop app watches the mouse and draws the ink; the renderer records and builds the bundle. + +export const ScrawlCaptureId = Schema.String.check(Schema.isPattern(/^scrawl-[a-z0-9-]{1,64}$/)); +export type ScrawlCaptureId = typeof ScrawlCaptureId.Type; + +export const ScrawlStartResult = Schema.Struct({ + id: ScrawlCaptureId, + /** For getUserMedia's chromeMediaSourceId. */ + sourceId: Schema.String, + width: Schema.Number, + height: Schema.Number, + /** Wall clock of the moment event times count from. */ + startedAtMs: Schema.Number, + app: Schema.String, + windowTitle: Schema.String, + bundlePath: Schema.String, +}); +export type ScrawlStartResult = typeof ScrawlStartResult.Type; + +/** Positions are fractions of the captured display, from its top left. */ +export const ScrawlInputEvent = Schema.Union([ + Schema.Struct({ + kind: Schema.Literal("click"), + t: Schema.Number, + x: Schema.Number, + y: Schema.Number, + }), + Schema.Struct({ + kind: Schema.Literal("drag"), + t: Schema.Number, + t1: Schema.Number, + x0: Schema.Number, + y0: Schema.Number, + x1: Schema.Number, + y1: Schema.Number, + }), +]); +export type ScrawlInputEvent = typeof ScrawlInputEvent.Type; + +export const ScrawlStroke = Schema.Struct({ + t0: Schema.Number, + t1: Schema.Number, + /** x, y, x, y, ... */ + points: Schema.Array(Schema.Number), +}); +export type ScrawlStroke = typeof ScrawlStroke.Type; + +export const ScrawlStopResult = Schema.Struct({ + events: Schema.Array(ScrawlInputEvent), + strokes: Schema.Array(ScrawlStroke), +}); +export type ScrawlStopResult = typeof ScrawlStopResult.Type; + +export const ScrawlCaptureRef = Schema.Struct({ id: ScrawlCaptureId }); +export type ScrawlCaptureRef = typeof ScrawlCaptureRef.Type; + +export const ScrawlAppendVideoInput = Schema.Struct({ + id: ScrawlCaptureId, + name: Schema.String, + bytes: Schema.Uint8Array, +}); +export type ScrawlAppendVideoInput = typeof ScrawlAppendVideoInput.Type; + +export const ScrawlWriteFileInput = Schema.Struct({ + id: ScrawlCaptureId, + /** Relative to the bundle, such as frames/frame-01.png. */ + path: Schema.String.check(Schema.isPattern(/^[\w-]+(?:\/[\w.-]+|\.[\w.-]+)$/)), + bytes: Schema.Uint8Array, +}); +export type ScrawlWriteFileInput = typeof ScrawlWriteFileInput.Type; + +/** How long ink stays after Shift is let go. Its fading is not the screen answering a click. */ +export const SCRAWL_INK_HOLD_MS = 5_000; + +/** The menu action the global shortcut sends to the renderer. */ +export const SCRAWL_TOGGLE_ACTION = "scrawl-toggle";