diff --git a/src/components/voice/diagram.tsx b/src/components/voice/diagram.tsx new file mode 100644 index 0000000..fc83a19 --- /dev/null +++ b/src/components/voice/diagram.tsx @@ -0,0 +1,492 @@ +import { type Accessor, createSignal, createUniqueId, For, onCleanup, onMount, Show } from "solid-js"; +import { FLIGHT, FRAME, fill, has, head, type Loss, type Packet, type Segment, type Word } from "./sim"; + +// Every drawn element can be swapped for a hand-drawn image (PNG/SVG under /public). +// Sizes are in viewBox units, where the diagram is 600 wide; draw at 4x for crisp output. +// An array of images cycles at FPS, like hand-drawn animation. Frames follow the clock, so scrubbing works. +export type Sprite = string | string[]; + +export interface Art { + kraken?: Sprite; // 28x28, chases the edge of the packet loss + packet?: Sprite; // 8x8, a packet in flight + retransmit?: Sprite; // 8x8, a retransmitted packet in flight + lost?: Sprite; // 10x10, where a lost packet died + you?: Sprite; // 36x36, replaces the "you" lane label + ai?: Sprite; // 36x36, replaces the "AI" lane label + backdrop?: string; // 600 x 176 per panel, drawn under everything + overlay?: string; // 600 x 176 per panel, drawn over everything + font?: string; // CSS font-family for all text; the page has to load it + wobble?: boolean; // roughen the generated shapes so they sit next to hand-drawn art +} + +const FPS = 8; + +export interface Track { + who: "you" | "ai"; + words: Word[]; + segments: Segment[]; + note?: (t: number) => string | undefined; +} + +export interface Panel { + title: string; + subtitle: string; + top: Track; + bottom: Track; + packets: Packet[]; + loss: Loss; + result: { at: number; text: string; good: boolean }; +} + +export interface Props { + axis: number; // media seconds shown on the x-axis + end: number; // wall-clock seconds until the animation settles + panels: Panel[]; + legend: { have: string; done: string }; + label: string; // accessible description of the whole animation + art?: Art; +} + +const HOLD = 2.5; // seconds to linger on the final frame before looping +const SPEEDS = [0.5, 1, 0.25]; + +const W = 600; +const GUTTER = 44; +const TRACK_W = W - GUTTER - 8; +const TRACK_H = 26; +const PANEL_H = 176; +const TOP_Y = 28; +const BOTTOM_Y = 118; + +const COLOR = { + done: "#22c55e", + have: "rgba(34, 197, 94, 0.28)", + missing: "#f59e0b", + lost: "#ef4444", + track: "#1e293b", + text: "#f1f5f9", + dim: "#64748b", + title: "#cbd5e1", +}; + +export function Diagram(props: Props) { + const [t, setT] = createSignal(0); + const [playing, setPlaying] = createSignal(false); + const [speed, setSpeed] = createSignal(SPEEDS[0]); + const now = () => Math.min(t(), props.end); + const id = createUniqueId(); + + let root!: HTMLDivElement; + + onMount(() => { + if (matchMedia("(prefers-reduced-motion: reduce)").matches) { + setT(props.end); + return; + } + + let last = performance.now(); + let frame = 0; + const tick = (time: number) => { + const dt = (time - last) / 1000; + last = time; + if (playing()) { + const next = t() + dt * speed(); + setT(next > props.end + HOLD ? 0 : next); + } + frame = requestAnimationFrame(tick); + }; + frame = requestAnimationFrame(tick); + + // Only run while on screen, so the reader doesn't miss the start. + const observer = new IntersectionObserver(([entry]) => setPlaying(entry.isIntersecting), { threshold: 0.5 }); + observer.observe(root); + + onCleanup(() => { + cancelAnimationFrame(frame); + observer.disconnect(); + }); + }); + + const x = (ts: number) => GUTTER + (ts / props.axis) * TRACK_W; + + return ( +
+ + + + + + + + + + + + + {(panel, i) => ( + + + + )} + + + +
+ + { + setPlaying(false); + setT(e.currentTarget.valueAsNumber); + }} + aria-label="Time" + /> + + {now().toFixed(1)}s +
+ +
+ {props.legend.done} + {props.legend.have} + waiting for retransmit + lost forever +
+
+ ); +} + +function Swatch(props: { fill?: string; stroke?: string; children: string }) { + return ( + + + {props.children} + + ); +} + +interface ViewProps { + t: Accessor; + x: (ts: number) => number; + art: Art; + hatch: string; + wobble?: string; +} + +function SpriteView(props: { + sprite: Sprite; + t: Accessor; + x: number; + y: number; + size: number; + opacity?: number; +}) { + const href = () => { + const s = props.sprite; + return Array.isArray(s) ? s[Math.floor(props.t() * FPS) % s.length] : s; + }; + return ( + + ); +} + +function PanelView(props: ViewProps & { panel: Panel }) { + const p = props.panel; + return ( + <> + {(href) => } + + {p.title} + + {p.subtitle} + + + + + + + + = p.result.at ? 1 : 0} + style={{ transition: "opacity 0.3s" }} + > + {p.result.text} + + + {(href) => } + + ); +} + +function TrackView(props: ViewProps & { track: Track; y: number; notes: "above" | "below" }) { + const w = () => props.x(FRAME) - props.x(0); + const pos = () => head(props.track.segments, props.t()); + const note = () => props.track.note?.(props.t()); + + return ( + + + {props.track.who === "ai" ? "AI" : "you"} + + } + > + {(sprite) => } + + + + + {(s) => } + + + + + {(word) => } + + + 0}> + + + + + {note()} + + + + ); +} + +function SegmentView(props: { s: Segment; t: Accessor; x: number; y: number; w: number; hatch: string }) { + const lost = () => has(props.s.lost, props.t()); + const have = () => has(props.s.have, props.t()); + const f = () => fill(props.s, props.t()); + const missing = () => has(props.s.missing, props.t()) && !have() && !lost(); + + return ( + + + + + 0 && !lost()}> + + + + + + + + + + ); +} + +function WordView(props: { + word: Word; + segments: Segment[]; + t: Accessor; + x: (ts: number) => number; + y: number; +}) { + const overlap = props.segments.filter((s) => s.ts < props.word.end && s.ts + FRAME > props.word.start); + const lost = () => overlap.some((s) => has(s.lost, props.t())); + const done = () => overlap.every((s) => fill(s, props.t()) >= 1); + + const x1 = props.x(props.word.start); + const x2 = props.x(props.word.end); + const mid = props.y + TRACK_H / 2; + + return ( + + + {props.word.text} + + + + + + ); +} + +function Network(props: ViewProps & { packets: Packet[]; loss: Loss }) { + const y0 = TOP_Y + TRACK_H; + const y1 = BOTTOM_Y; + const mid = (y0 + y1) / 2; + + // The kraken eats every packet sent while it's around; shade the slice it has chewed so far. + const lost = props.packets.filter((p) => p.lost); + const eaten = () => lost.filter((p) => p.sent <= props.t()); + const kraken = () => props.t() >= props.loss.start && props.t() < props.loss.end + 0.3 && eaten().length > 0; + const from = () => props.x(lost[0]?.ts ?? 0); + const to = () => props.x((eaten().at(-1)?.ts ?? 0) + FRAME); + + return ( + + + + {/* Chase the chewing edge, unless that would fall off the right side. */} + W ? from() - 24 : to() + 4} y={mid + 6} font-size="18"> + 🐙 + + } + > + {(sprite) => ( + W ? from() - 16 : to() + 16} y={mid} size={28} /> + )} + + + + {(p) => } + + + ); +} + +function PacketView(props: { p: Packet; t: Accessor; art: Art; x: number; y0: number; y1: number }) { + const lerp = (k: number) => props.y0 + (props.y1 - props.y0) * k; + const first = () => (props.t() - props.p.sent) / FLIGHT; + const rtx = () => (props.p.rtx === undefined ? -1 : (props.t() - props.p.rtx) / FLIGHT); + + // Lost packets die halfway, where the kraken lives. + const flying = () => first() >= 0 && first() <= (props.p.lost ? 0.5 : 1); + const dead = () => props.p.lost && first() > 0.5 && first() < 3; + const fade = () => 1 - (first() - 0.5) / 2.5; + + return ( + + + } + > + {(sprite) => } + + + + + × + + } + > + {(sprite) => ( + + )} + + + = 0 && rtx() <= 1}> + } + > + {(sprite) => } + + + + ); +} diff --git a/src/components/voice/sim.ts b/src/components/voice/sim.ts new file mode 100644 index 0000000..e5344b6 --- /dev/null +++ b/src/components/voice/sim.ts @@ -0,0 +1,84 @@ +// A tiny deterministic network model for the voice AI diagrams. +// Everything is a pure function of wall-clock time `t` so the animation can be scrubbed. + +// Seconds of audio per packet. Opus uses 20ms; 100ms keeps the dots countable. +export const FRAME = 0.1; + +// One-way network delay, exaggerated so packets are visible in flight. +export const FLIGHT = 0.25; + +export interface Word { + text: string; + start: number; + end: number; +} + +// A packet-sized slice of a track, keyed by media timestamp. +// Each field is the wall-clock time the slice enters that state. +export interface Segment { + ts: number; + have?: number; // available but not consumed (light fill) + start?: number; // consumption starts (solid fill grows) + done?: number; // consumption finishes + missing?: number; // known to be missing, waiting on a retransmit + lost?: number; // given up on +} + +export interface Packet { + ts: number; + sent: number; + lost: boolean; + rtx?: number; // retransmit time, if the transport bothers +} + +// A window of wall-clock time where every packet sent is lost. +export interface Loss { + start: number; + end: number; +} + +export function frames(duration: number): number[] { + const count = Math.round(duration / FRAME); + return Array.from({ length: count }, (_, i) => round(i * FRAME)); +} + +export function transmit(ts: number[], send: (ts: number) => number, loss: Loss, retransmit: boolean): Packet[] { + let retransmits = 0; + return ts.map((ts) => { + const sent = send(ts); + const lost = sent >= loss.start && sent < loss.end; + // Retransmits queue up behind the loss and go out in a quick burst once it clears. + const rtx = lost && retransmit ? loss.end + 0.02 * retransmits++ : undefined; + return { ts, sent, lost, rtx }; + }); +} + +export function arrival(p: Packet): number | undefined { + if (!p.lost) return p.sent + FLIGHT; + if (p.rtx !== undefined) return p.rtx + FLIGHT; + return undefined; +} + +export function fill(s: Segment, t: number): number { + if (s.start === undefined || s.done === undefined || t < s.start) return 0; + return Math.min(1, (t - s.start) / Math.max(1e-6, s.done - s.start)); +} + +export function has(at: number | undefined, t: number): boolean { + return at !== undefined && t >= at; +} + +// The media timestamp consumption has reached. +export function head(segments: Segment[], t: number): number { + let pos = 0; + for (const s of segments) { + if (has(s.lost, t)) pos = Math.max(pos, s.ts + FRAME); + const f = fill(s, t); + if (f > 0) pos = Math.max(pos, s.ts + FRAME * f); + } + return pos; +} + +function round(n: number): number { + return Math.round(n * 1000) / 1000; +} diff --git a/src/components/voice/stt.tsx b/src/components/voice/stt.tsx new file mode 100644 index 0000000..b634cb7 --- /dev/null +++ b/src/components/voice/stt.tsx @@ -0,0 +1,80 @@ +// Speech-to-text: a human talks at human speed, the AI listens over a lossy network. +import { type Art, Diagram, type Panel } from "./diagram"; +import { arrival, FLIGHT, FRAME, frames, has, head, type Segment, transmit } from "./sim"; + +const WORDS = [ + { text: "What", start: 0.1, end: 0.5 }, + { text: "is", start: 0.6, end: 0.8 }, + { text: "le", start: 0.9, end: 1.1 }, + { text: "capital", start: 1.3, end: 2.0 }, + { text: "of", start: 2.1, end: 2.3 }, + { text: "France?", start: 2.5, end: 3.3 }, +]; + +const AXIS = 3.4; +const LOSS = { start: 1.3, end: 2.1 }; + +// How much faster than real-time the model can chew through buffered audio. +// Real models are much faster; 4x keeps the catch-up visible. +const CATCHUP = 4; + +const TS = frames(AXIS); +const SPOKEN: Segment[] = TS.map((ts) => ({ ts, start: ts, done: ts + FRAME })); + +// A packet leaves once its frame has been spoken. +const send = (ts: number) => ts + FRAME; + +function panel(retransmit: boolean): Panel { + const packets = transmit(TS, send, LOSS, retransmit); + + // The model consumes audio in order, as soon as it's available. + let prev = 0; + const heard: Segment[] = packets.map((p) => { + const arrive = arrival(p); + // WebRTC skips the hole once the next packet shows up. + if (arrive === undefined) return { ts: p.ts, lost: p.sent + FRAME + FLIGHT }; + + const start = Math.max(arrive, prev); + prev = start + FRAME / CATCHUP; + const missing = p.lost ? p.sent + FRAME + FLIGHT : undefined; + return { ts: p.ts, have: arrive, start, done: prev, missing }; + }); + + const finished = Math.max(...heard.map((s) => s.done ?? s.lost ?? 0)); + + // How far behind the speaker the model is, ignoring the usual network delay. + const note = (t: number) => { + const behind = head(SPOKEN, t) - head(heard, t) - FLIGHT - FRAME; + if (behind < 0.1) return undefined; + const waiting = heard.some((s) => has(s.missing, t) && !has(s.have, t)); + return `${waiting ? "waiting" : `catching up at ${CATCHUP}×`} · ${behind.toFixed(1)}s behind`; + }; + + return { + title: retransmit ? "MoQ" : "WebRTC", + subtitle: retransmit ? "retransmits for up to 5s" : "gives up on late audio", + top: { who: "you", words: WORDS, segments: SPOKEN }, + bottom: { who: "ai", words: WORDS, segments: heard, note }, + packets, + loss: LOSS, + result: retransmit + ? { at: finished + 0.3, text: "🤖 “Paris.”", good: true } + : { at: finished + 0.3, text: "🤖 “Sorry, you broke up. Can you repeat that?”", good: false }, + }; +} + +const PANELS = [panel(false), panel(true)]; +const END = Math.max(...PANELS.map((p) => p.result.at)) + 0.5; + +export default function SpeechToText(props: { art?: Art }) { + return ( + + ); +} diff --git a/src/components/voice/tts.tsx b/src/components/voice/tts.tsx new file mode 100644 index 0000000..02fef66 --- /dev/null +++ b/src/components/voice/tts.tsx @@ -0,0 +1,94 @@ +// Text-to-speech: the AI talks faster than real-time, the human listens at human speed. +import { type Art, Diagram, type Panel } from "./diagram"; +import { arrival, FLIGHT, FRAME, frames, has, head, type Segment, transmit } from "./sim"; + +const WORDS = [ + { text: "Paris.", start: 0.1, end: 0.6 }, + { text: "Obviously.", start: 0.9, end: 1.6 }, + { text: "Did", start: 2.0, end: 2.2 }, + { text: "you", start: 2.3, end: 2.5 }, + { text: "even", start: 2.6, end: 2.9 }, + { text: "try", start: 3.0, end: 3.2 }, + { text: "googling", start: 3.3, end: 3.8 }, + { text: "it?", start: 3.9, end: 4.2 }, +]; + +const AXIS = 4.3; +const LOSS = { start: 1.0, end: 1.8 }; + +const GENERATE = 10; // the model speaks 10x faster than real-time +const THROUGHPUT = 3; // MoQ sends as fast as the (congested) network allows +const DELAY = 0.5; // both players start playback after the same delay + +const TS = frames(AXIS); +const generated = (ts: number) => (ts + FRAME) / GENERATE; + +function panel(moq: boolean): Panel { + // WebRTC is a real-time transport: audio goes out at human speed, no matter how fast it was made. + const send = moq ? (ts: number) => Math.max(generated(ts), ts / THROUGHPUT) : (ts: number) => ts + FRAME; + const packets = transmit(TS, send, LOSS, moq); + const pace = moq ? FRAME / THROUGHPUT : FRAME; + + const spoken: Segment[] = packets.map((p) => ({ + ts: p.ts, + have: generated(p.ts), + start: p.sent, + done: p.sent + pace, + })); + + // The player plays each frame at its timestamp, if it showed up in time. + const heard: Segment[] = packets.map((p) => { + const arrive = arrival(p); + const play = p.ts + DELAY; + if (arrive === undefined || arrive > play) return { ts: p.ts, lost: play }; + const missing = p.lost ? p.sent + FRAME + FLIGHT : undefined; + return { ts: p.ts, have: arrive, start: play, done: play + FRAME, missing }; + }); + + const queued = (t: number) => { + const ahead = Math.min(t * GENERATE, AXIS) - head(spoken, t); + return ahead > 0.3 ? `${ahead.toFixed(1)}s generated, waiting for real-time` : undefined; + }; + + const buffered = (t: number) => { + const playhead = head(heard, t); + if (playhead <= 0 || playhead >= AXIS) return undefined; + let edge = playhead; + for (const s of heard) { + if (s.ts + FRAME <= playhead) continue; + if (!has(s.have, t)) break; + edge = s.ts + FRAME; + } + return `${(edge - playhead).toFixed(1)}s buffered`; + }; + + const finished = AXIS + DELAY; + + return { + title: moq ? "MoQ" : "WebRTC", + subtitle: moq ? "sends everything ASAP, the player buffers" : "sends at human speed, gives up on late audio", + top: { who: "ai", words: WORDS, segments: spoken, note: moq ? undefined : queued }, + bottom: { who: "you", words: WORDS, segments: heard, note: buffered }, + packets, + loss: LOSS, + result: moq + ? { at: finished + 0.2, text: "🧑 “Rude, but fair.”", good: true } + : { at: finished + 0.2, text: "🧑 “Sorry, you broke up. Can you repeat that?”", good: false }, + }; +} + +const PANELS = [panel(false), panel(true)]; +const END = AXIS + DELAY + 0.7; + +export default function TextToSpeech(props: { art?: Art }) { + return ( + + ); +}