quest(audio-jitter-target): Audio jitter target

demo/web/src/index.ts

@@ -119,9 +119,6 @@ function createTile(name: string): WatchTile {
 	const watch = document.createElement("moq-watch") as MoqWatch;
 	watch.name = name;
 	watch.muted = true; // unmuted only while active (see below)
-	// Default to a fixed 100ms jitter buffer (instead of adaptive "auto") so
-	// the delay visualization has something to show. Drag it in the panel.
-	watch.setAttribute("delay", "100ms");
 	const canvas = document.createElement("canvas");
 	canvas.style.cssText = "width: 100%; height: auto;";
 	watch.appendChild(canvas);

demo/web/src/viz.ts

@@ -307,7 +307,7 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement
 		const timestamp = watch.sync.now() as number | undefined;
 		const stalled = watch.video.out.stalled.peek();
 		drawRanges(video.canvas, watch.video.out.buffered.peek(), timestamp, stalled);
-		drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, false);
+		drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, watch.audio.out.stalled.peek());
 		parent.animate(draw);
 	};
 	parent.animate(draw);

doc/bin/cli.md

@@ -82,18 +82,25 @@ MKV uses the same flag to cap clusters, which otherwise follow video GOPs.
 
 ```bash
 moq --connect https://relay.example.com/anon --broadcast my-stream.hang play
-moq ... play --delay 500ms          # trade latency for a jittery link
+moq ... play --delay 500ms          # fix the delay instead of measuring it
 ```
 
 Decodes H.264, H.265, and AV1 video and Opus, PCM, and AAC-LC audio using
 the platform hardware decoder where available. `--video-name` and
 `--audio-name` pick a rendition.
 
-Playback runs on a clock it owns. `--delay` (default 100 ms) is how far it
-trails the live edge, which is both the jitter a late frame may absorb and the
-point past which a stalled group is skipped. The speaker holds the delay, with a
-50 ms floor under it, and the picture is scheduled against where the speaker
-actually is. While video owns the clock, a frame arriving earlier than predicted
+Playback runs on a clock it owns. `--delay` is how far it trails the live
+edge: the jitter a late frame may absorb. The default, `auto`, measures how
+unevenly audio arrives and sizes the speaker's buffer to match, using the same
+[algorithm](/concept/audio-jitter) as the browser player, so a publisher that
+flushes 100 ms at a time gets a buffer deep enough to play through the next
+flush. It waits up to 2 s on a stalled group before skipping it, since a budget
+any shorter would hide the very lateness it measures; a broadcast with no audio
+has nothing to measure, so video trails by 100 ms and skips past that. A duration fixes the delay
+instead, and doubles as the point past which a stalled group is skipped. The
+speaker holds the delay, with a 50 ms floor under it: it pads back up to the
+delay after running dry and skips back down onto it after a burst. The picture
+is scheduled against where the speaker actually is. While video owns the clock, a frame arriving earlier than predicted
 pulls playback forward, so a late start catches up to live instead of staying
 behind it. Once the speaker owns the clock, video follows the speaker instead.
 

doc/concept/audio-jitter.md

@@ -37,7 +37,27 @@ frame:
 | Native | `container::Consumer::read()`, awaited by `moq-audio`'s decode consumer |
 
 Both sit below the decoder and above nothing: no reordering, no gap handling, no
-age budget. A frame dropped by the relay is never observed, which is correct.
+age budget.
+
+Natively that holds only as far as the caller lets it. The container consumer
+reads groups in sequence order and applies its age budget inside `read()`, so a
+group skipped for age is never observed, and a frame queued behind a stalled
+group is observed when the stall clears. A caller that estimates keeps the budget
+at least as long as the estimate's ceiling, which is why `moq play` waits 2 s on
+a stalled group under `--delay auto`. The observation is also only as honest as
+the caller is prompt: `read()` is timed when it resolves, so a player drains it
+as packets come and lets the playback buffer hold the target, rather than
+pacing reads to the speaker.
+
+A frame dropped by the relay is never observed, which is correct only while the
+drop is not the receiver's own doing. The subscription's max age drops groups
+before any container sees them, at the relay and in the receiver's own transport
+subscriber, so a subscription cut to the target hides every frame later than
+the target: the estimate creeps up one bucket at a time while the frames it
+should have measured go unplayed. In automatic mode both implementations
+subscribe with at least the ceiling, 2 s: `moq play --delay auto` through its
+age budget, and the browser through the audio subscription alone, since its
+container consumer observes each frame before applying the local budget.
 A frame the container will later discard *is* observed, which is also correct:
 it arrived, and when it arrived is the measurement.
 
@@ -218,10 +238,8 @@ carry a number the payload already states.
 
 ### What is not in the frame duration
 
-The browser's AudioWorklet renders in fixed 128-sample blocks, and
-`js/watch/src/audio/config.ts` currently adds that block to the per-codec frame
-duration, so 48 kHz Opus reads as 23 ms rather than 20 ms. **That block is not
-part of the target.** It is a property of the render backend, not of the
+The browser's AudioWorklet renders in fixed 128-sample blocks. **That block is
+not part of the target.** It is a property of the render backend, not of the
 network, and native has no worklet at all. If one language's step carried it and
 the other's did not, the two would produce different target series from the same
 trace and the corpus could not hold.
@@ -268,9 +286,6 @@ point strictly contains the publisher's. The floor earns its place by being
 correct *immediately*, before the estimator has seen a full flush cycle, which
 is the one thing a measurement cannot be.
 
-Note that `js/watch` today adds the two rather than taking the maximum, which
-over-buffers a bursty publisher by its own flush span.
-
 ## Rise, fall, startup, and the ceiling
 
 There is **no separate rise or fall limiter**. The histogram is the only

doc/concept/audio-jitter/corpus.test.ts

@@ -2,7 +2,7 @@
 // has to catch is the corpus drifting from the algorithm it claims to encode: a checked-in
 // file edited by hand to make a failing implementation pass would otherwise go unnoticed.
 //
-// Grading an implementation is each language's own test suite, since neither exists yet.
+// Grading an implementation is each language's own test suite, e.g. js/hang's jitter.test.ts.
 
 import { describe, expect, test } from "bun:test";
 import { readFileSync } from "node:fs";

doc/lib/js/watch.md

@@ -30,7 +30,7 @@ in sync at the latency you ask for.
 | --- | --- |
 | `url`, `name` | Relay URL (with `?jwt=` if needed) and broadcast name. |
 | `paused`, `muted`, `volume` | The usual player controls, mirrored as reactive properties. |
-| `delay` | How far playback trails the live edge: `"auto"` (derived from RTT, the default), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. |
+| `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames actually arrive; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. |
 | `buffer` | Future-dated media held beyond the live edge before playback skips ahead, e.g. `"30s"`. Defaults to none. |
 | `captions` | The caption track to show, or absent for off. `el.text.out.available` lists the renditions for a picker. |
 | `visible` | Only subscribe to video while the element is on screen: a margin (`"20%"` default, `"200px"`), `"always"`, or `"never"`. |

js/hang/src/container/consumer.test.ts

@@ -515,6 +515,25 @@ test("Consumer skips groups via PTS-span when over the max age", async () => {
 	consumer.close();
 });
 
+test("Consumer measures arrivals as they land", async () => {
+	const track = new Track.Producer("test");
+	const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("audio"), maxAge: 5000 as Time.Milli });
+
+	// Nothing observed yet: the cold-start estimate.
+	expect(consumer.spread.peek()).toBe(100 as Time.Milli);
+
+	// Groups arriving exactly as fast as their media, across two resample intervals. Nothing is
+	// late, so the evidence pulls the estimate down from the cold start.
+	for (let i = 0; i < 3; i++) {
+		writeGroupWithLegacyFrames(track, i, [(i * 520_000) as Time.Micro]);
+		await settle(520);
+	}
+	expect(consumer.spread.peek()).toBeLessThan(100 as Time.Milli);
+
+	track.close();
+	consumer.close();
+});
+
 // --- Ordering ---
 
 test("Consumer delivers groups in sequence order regardless of arrival order", async () => {

js/hang/src/container/consumer.ts

@@ -3,6 +3,7 @@ import * as Moq from "@moq/net";
 import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq/signals";
 
 import type { Format } from "./format";
+import { Jitter } from "./jitter";
 import type { BufferedRanges, Frame } from "./types";
 
 /** Options for constructing a {@link Consumer}. */
@@ -100,6 +101,16 @@ export class Consumer {
 	/** The time ranges currently buffered and ready to play. */
 	readonly buffered: Getter<BufferedRanges> = this.#buffered;
 
+	#jitter = new Jitter();
+	#spread = new Signal<Time.Milli>(this.#jitter.measured);
+	/**
+	 * How much buffer late arrivals need, measured as frames come off the transport.
+	 *
+	 * The measured term of doc/concept/audio-jitter.md, excluding the codec frame and any advertised
+	 * floor. Starts at the cold-start estimate before any frame arrives.
+	 */
+	readonly spread: Getter<Time.Milli> = this.#spread;
+
 	#signals = new Effect();
 	#closed = new Once<Error | null>();
 
@@ -172,11 +183,19 @@ export class Consumer {
 				const next = await group.consumer.readFrame();
 				if (!next) break;
 				group.empty = false;
+				const arrival = Moq.Time.Milli.now();
 
 				const decoded = this.#format.decode(next.payload);
 
 				for (const sample of decoded) {
 					const marker = this.#format.end?.(sample) !== undefined;
+
+					// Observed here, before any gap handling or age budget: a target derived from
+					// what survives the budget would only ever confirm the budget it was cut to.
+					if (!marker) {
+						this.#jitter.observe(arrival, Moq.Time.Milli.fromMicro(sample.timestamp));
+						this.#spread.set(this.#jitter.measured);
+					}
 					const frame: Frame = {
 						payload: sample.payload,
 						timestamp: sample.timestamp,

js/hang/src/container/jitter.test.ts

@@ -0,0 +1,95 @@
+import { describe, expect, it } from "bun:test";
+import { readdirSync, readFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+import { fileURLToPath } from "node:url";
+import type { Time } from "@moq/net";
+import { Jitter } from "./jitter";
+
+// The conformance corpus beside doc/concept/audio-jitter.md, read directly so this is graded
+// against the document rather than against another implementation.
+const CORPUS = join(dirname(fileURLToPath(import.meta.url)), "../../../../doc/concept/audio-jitter");
+
+interface Case {
+	name: string;
+	frame: number;
+	advertised?: number;
+	delay?: number;
+	arrival: number[];
+	media: number[];
+	target: [number, number][];
+}
+
+const cases: Case[] = readdirSync(CORPUS)
+	.filter((file) => file.endsWith(".json"))
+	.map((file) => JSON.parse(readFileSync(join(CORPUS, file), "utf8")));
+
+// The composition the document specifies on top of the measured term. `@moq/watch` owns it in
+// production (`audio/latency.ts`); it is repeated here so the whole target series is checked.
+function target(jitter: Jitter, c: Case): number {
+	if (c.delay !== undefined) return c.delay;
+	return Math.max(jitter.measured, c.advertised ?? 0) + c.frame;
+}
+
+describe("audio jitter conformance", () => {
+	it("reads the whole corpus", () => {
+		expect(cases.map((c) => c.name).sort()).toEqual([
+			"advertised",
+			"buildup",
+			"fixed",
+			"flush",
+			"idle",
+			"paced",
+			"reorder",
+			"spike",
+			"tunein",
+		]);
+	});
+
+	for (const c of cases) {
+		it(c.name, () => {
+			const jitter = new Jitter();
+			const expected = new Map(c.target);
+			let current: number | undefined;
+			for (let i = 0; i < c.arrival.length; i++) {
+				jitter.observe(c.arrival[i] as Time.Milli, c.media[i] as Time.Milli);
+				const value = target(jitter, c);
+				const listed = expected.get(i);
+				if (listed !== undefined) {
+					expect(value).toBeCloseTo(listed, 6);
+				} else if (current !== undefined) {
+					// Unlisted frames hold the previous target.
+					expect(value).toBeCloseTo(current, 6);
+				}
+				current = value;
+			}
+		});
+	}
+});
+
+describe("Jitter", () => {
+	it("starts at the prior's 95th percentile", () => {
+		expect(new Jitter().measured).toBe(100 as Time.Milli);
+	});
+
+	it("is not moved by a stale frame ahead of the live edge", () => {
+		// The #3517 regression: one frame from a stale group, then the live edge 14.5s of media later,
+		// then evenly paced audio with no real jitter. The jump must not read as a delay or a frame.
+		const jitter = new Jitter();
+		jitter.observe(0 as Time.Milli, 0 as Time.Milli);
+		for (let i = 0; i < 500; i++) {
+			jitter.observe((10 + i * 20) as Time.Milli, (14_500 + i * 20) as Time.Milli);
+		}
+		expect(jitter.measured).toBe(20 as Time.Milli);
+	});
+
+	it("does not follow the absolute delay", () => {
+		// Only the spread of arrivals matters, not how far away the sender is.
+		const near = new Jitter();
+		const far = new Jitter();
+		for (let i = 0; i < 500; i++) {
+			near.observe((10 + i * 20) as Time.Milli, (i * 20) as Time.Milli);
+			far.observe((2000 + i * 20) as Time.Milli, (i * 20) as Time.Milli);
+		}
+		expect(far.measured).toBe(near.measured);
+	});
+});