Voice diagrams: aligned loss, click-to-seek

src/components/voice/diagram.tsx

@@ -1,9 +1,9 @@
 import { type Accessor, createSignal, createUniqueId, For, onCleanup, onMount, Show } from "solid-js";
-import { FLIGHT, FRAME, fill, has, head, type Loss, type Packet, type Segment, type Word } from "./sim";
+import { FLIGHT, FRAME, fill, has, head, type Loss, type Packet, reach, type Segment, type Word } from "./sim";
 
 // Every drawn element can be swapped for a hand-drawn image (PNG/SVG under /public).
 // Sizes are in viewBox units, where the diagram is 600 wide; draw at 4x for crisp output.
-// An array of images cycles at FPS, like hand-drawn animation. Frames follow the clock, so scrubbing works.
+// An array of images cycles at FPS, like hand-drawn animation. Frames follow the clock.
 export type Sprite = string | string[];
 
 export interface Art {
@@ -48,7 +48,8 @@ export interface Props {
 }
 
 const HOLD = 2.5; // seconds to linger on the final frame before looping
-const SPEEDS = [0.5, 1, 0.25];
+const SPEED = 0.5; // half speed, so the packets are easy to follow
+const STEP = 0.1; // seconds per arrow key
 
 const W = 600;
 const GUTTER = 44;
@@ -72,7 +73,6 @@ const COLOR = {
 export function Diagram(props: Props) {
 	const [t, setT] = createSignal(0);
 	const [playing, setPlaying] = createSignal(false);
-	const [speed, setSpeed] = createSignal(SPEEDS[0]);
 	const now = () => Math.min(t(), props.end);
 	const id = createUniqueId();
 
@@ -90,7 +90,7 @@ export function Diagram(props: Props) {
 			const dt = (time - last) / 1000;
 			last = time;
 			if (playing()) {
-				const next = t() + dt * speed();
+				const next = t() + dt * SPEED;
 				setT(next > props.end + HOLD ? 0 : next);
 			}
 			frame = requestAnimationFrame(tick);
@@ -109,46 +109,77 @@ export function Diagram(props: Props) {
 
 	const x = (ts: number) => GUTTER + (ts / props.axis) * TRACK_W;
 
+	// The keyboard path for seeking: arrows step, Home/End jump. Clicking a track is the mouse path.
+	const key = (e: KeyboardEvent) => {
+		const target = {
+			ArrowLeft: now() - STEP,
+			ArrowDown: now() - STEP,
+			ArrowRight: now() + STEP,
+			ArrowUp: now() + STEP,
+			Home: 0,
+			End: props.end,
+		}[e.key];
+		if (target === undefined) return;
+		e.preventDefault();
+		setPlaying(false);
+		setT(Math.max(0, Math.min(props.end, target)));
+	};
+
 	return (
 		<div ref={root} class="not-prose my-8 select-none">
-			<svg
-				viewBox={`0 0 ${W} ${PANEL_H * props.panels.length}`}
-				class="w-full"
-				font-size="12"
-				font-family={props.art?.font}
-				role="img"
+			<div
+				class="rounded focus-visible:outline focus-visible:outline-2 focus-visible:outline-green-500"
+				role="slider"
+				tabIndex={0}
 				aria-label={props.label}
+				aria-valuemin={0}
+				aria-valuemax={props.end}
+				aria-valuenow={Math.round(now() * 10) / 10}
+				aria-valuetext={`${now().toFixed(1)} seconds`}
+				onKeyDown={key}
 			>
-				<defs>
-					<pattern id={`${id}-lost`} width="6" height="6" patternUnits="userSpaceOnUse" patternTransform="rotate(45)">
-						<rect width="6" height="6" fill="rgba(239, 68, 68, 0.25)" />
-						<rect width="2" height="6" fill={COLOR.lost} />
-					</pattern>
-					<filter id={`${id}-wobble`}>
-						<feTurbulence type="fractalNoise" baseFrequency="0.06" numOctaves="2" seed="3" />
-						<feDisplacementMap in="SourceGraphic" scale="3" xChannelSelector="R" yChannelSelector="G" />
-					</filter>
-				</defs>
-				<For each={props.panels}>
-					{(panel, i) => (
-						<g transform={`translate(0 ${i() * PANEL_H})`}>
-							<PanelView
-								panel={panel}
-								t={now}
-								x={x}
-								art={props.art ?? {}}
-								hatch={`url(#${id}-lost)`}
-								wobble={props.art?.wobble ? `url(#${id}-wobble)` : undefined}
-							/>
-						</g>
-					)}
-				</For>
-			</svg>
+				<svg
+					viewBox={`0 0 ${W} ${PANEL_H * props.panels.length}`}
+					class="block w-full"
+					font-size="12"
+					font-family={props.art?.font}
+					aria-hidden="true"
+				>
+					<defs>
+						<pattern id={`${id}-lost`} width="6" height="6" patternUnits="userSpaceOnUse" patternTransform="rotate(45)">
+							<rect width="6" height="6" fill="rgba(239, 68, 68, 0.25)" />
+							<rect width="2" height="6" fill={COLOR.lost} />
+						</pattern>
+						<filter id={`${id}-wobble`}>
+							<feTurbulence type="fractalNoise" baseFrequency="0.06" numOctaves="2" seed="3" />
+							<feDisplacementMap in="SourceGraphic" scale="3" xChannelSelector="R" yChannelSelector="G" />
+						</filter>
+					</defs>
+					<For each={props.panels}>
+						{(panel, i) => (
+							<g transform={`translate(0 ${i() * PANEL_H})`}>
+								<PanelView
+									panel={panel}
+									t={now}
+									x={x}
+									seek={(frac, segments) => {
+										const at = reach(segments, frac * props.axis);
+										if (at !== undefined) setT(at);
+									}}
+									art={props.art ?? {}}
+									hatch={`url(#${id}-lost)`}
+									wobble={props.art?.wobble ? `url(#${id}-wobble)` : undefined}
+								/>
+							</g>
+						)}
+					</For>
+				</svg>
+			</div>
 
-			<div class="mt-2 flex items-center gap-3 text-sm text-slate-400">
+			<div class="mt-2 flex flex-wrap items-center gap-x-4 gap-y-1 text-xs text-slate-400">
 				<button
 					type="button"
-					class="w-8 rounded bg-slate-800 py-1 text-slate-200 hover:bg-slate-700"
+					class="w-7 rounded bg-slate-800 py-0.5 text-slate-200 hover:bg-slate-700"
 					onClick={() => {
 						if (t() >= props.end) setT(0);
 						setPlaying(!playing());
@@ -157,31 +188,6 @@ export function Diagram(props: Props) {
 				>
 					{playing() ? "❚❚" : "▶"}
 				</button>
-				<input
-					type="range"
-					class="flex-1 accent-green-500"
-					min="0"
-					max={props.end}
-					step="0.01"
-					value={now()}
-					onInput={(e) => {
-						setPlaying(false);
-						setT(e.currentTarget.valueAsNumber);
-					}}
-					aria-label="Time"
-				/>
-				<button
-					type="button"
-					class="w-12 rounded bg-slate-800 py-1 tabular-nums text-slate-200 hover:bg-slate-700"
-					onClick={() => setSpeed(SPEEDS[(SPEEDS.indexOf(speed()) + 1) % SPEEDS.length])}
-					aria-label="Playback speed"
-				>
-					{speed()}×
-				</button>
-				<span class="w-10 text-right tabular-nums">{now().toFixed(1)}s</span>
-			</div>
-
-			<div class="mt-2 flex flex-wrap gap-x-4 gap-y-1 text-xs text-slate-400">
 				<Swatch fill={COLOR.done}>{props.legend.done}</Swatch>
 				<Swatch fill={COLOR.have}>{props.legend.have}</Swatch>
 				<Swatch stroke={COLOR.missing}>waiting for retransmit</Swatch>
@@ -214,6 +220,7 @@ function Swatch(props: { fill?: string; stroke?: string; children: string }) {
 interface ViewProps {
 	t: Accessor<number>;
 	x: (ts: number) => number;
+	seek: (frac: number, segments: Segment[]) => void; // jump to when a track reaches `frac` of the way along
 	art: Art;
 	hatch: string;
 	wobble?: string;
@@ -271,7 +278,9 @@ function PanelView(props: ViewProps & { panel: Panel }) {
 				{p.result.text}
 			</text>
 
-			<Show when={props.art.overlay}>{(href) => <image href={href()} width={W} height={PANEL_H} />}</Show>
+			<Show when={props.art.overlay}>
+				{(href) => <image href={href()} width={W} height={PANEL_H} pointer-events="none" />}
+			</Show>
 		</>
 	);
 }
@@ -314,6 +323,20 @@ function TrackView(props: ViewProps & { track: Track; y: number; notes: "above"
 					stroke-width="2"
 				/>
 			</Show>
+			{/* Click a track to jump to when it reaches that point. */}
+			{/* biome-ignore lint/a11y/noStaticElementInteractions: the mouse path; the <svg> slider handles the keyboard. */}
+			<rect
+				x={GUTTER}
+				y={props.y}
+				width={TRACK_W}
+				height={TRACK_H}
+				fill="transparent"
+				class="cursor-pointer"
+				onClick={(e) => {
+					const r = e.currentTarget.getBoundingClientRect();
+					props.seek((e.clientX - r.left) / r.width, props.track.segments);
+				}}
+			/>
 			<Show when={note()}>
 				<text
 					x={props.x(pos()) + 6}

src/components/voice/sim.ts

@@ -1,5 +1,5 @@
 // A tiny deterministic network model for the voice AI diagrams.
-// Everything is a pure function of wall-clock time `t` so the animation can be scrubbed.
+// Everything is a pure function of wall-clock time `t` so the animation is deterministic.
 
 // Seconds of audio per packet. Opus uses 20ms; 100ms keeps the dots countable.
 export const FRAME = 0.1;
@@ -79,6 +79,14 @@ export function head(segments: Segment[], t: number): number {
 	return pos;
 }
 
+// When consumption reaches media timestamp `ts`, the inverse of head().
+export function reach(segments: Segment[], ts: number): number | undefined {
+	const s = segments.find((s) => ts < s.ts + FRAME);
+	if (s?.lost !== undefined) return s.lost;
+	if (s?.start === undefined || s.done === undefined) return undefined;
+	return s.start + ((s.done - s.start) * Math.max(0, ts - s.ts)) / FRAME;
+}
+
 function round(n: number): number {
 	return Math.round(n * 1000) / 1000;
 }

src/components/voice/stt.tsx

@@ -12,7 +12,7 @@ const WORDS = [
 ];
 
 const AXIS = 3.4;
-const LOSS = { start: 1.3, end: 2.1 };
+const LOSS = { start: 1.3, end: 2.4 };
 
 // How much faster than real-time the model can chew through buffered audio.
 // Real models are much faster; 4x keeps the catch-up visible.

src/components/voice/tts.tsx

@@ -14,7 +14,8 @@ const WORDS = [
 ];
 
 const AXIS = 4.3;
-const LOSS = { start: 1.0, end: 1.8 };
+// The same stretch of audio is lost on both transports, by media timestamp, so they lose the same words.
+const LOST = { start: 0.7, end: 2.5 };
 
 const GENERATE = 10; // the model speaks 10x faster than real-time
 const THROUGHPUT = 3; // MoQ sends as fast as the (congested) network allows
@@ -26,7 +27,9 @@ const generated = (ts: number) => (ts + FRAME) / GENERATE;
 function panel(moq: boolean): Panel {
 	// WebRTC is a real-time transport: audio goes out at human speed, no matter how fast it was made.
 	const send = moq ? (ts: number) => Math.max(generated(ts), ts / THROUGHPUT) : (ts: number) => ts + FRAME;
-	const packets = transmit(TS, send, LOSS, moq);
+	// Sending is monotonic, so the outage is the wall-clock window those timestamps went out in.
+	const loss = { start: send(LOST.start), end: send(LOST.end) };
+	const packets = transmit(TS, send, loss, moq);
 	const pace = moq ? FRAME / THROUGHPUT : FRAME;
 
 	const spoken: Segment[] = packets.map((p) => ({
@@ -70,7 +73,7 @@ function panel(moq: boolean): Panel {
 		top: { who: "ai", words: WORDS, segments: spoken, note: moq ? undefined : queued },
 		bottom: { who: "you", words: WORDS, segments: heard, note: buffered },
 		packets,
-		loss: LOSS,
+		loss,
 		result: moq
 			? { at: finished + 0.2, text: "🧑 “Rude, but fair.”", good: true }
 			: { at: finished + 0.2, text: "🧑 “Sorry, you broke up. Can you repeat that?”", good: false },