diff --git a/Cargo.lock b/Cargo.lock index 0c59a825ce..2420ea9cb6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4182,6 +4182,8 @@ dependencies = [ "rand 0.10.3", "ringbuf", "rubato", + "serde", + "serde_json", "sonora", "symphonia-codec-aac", "symphonia-core", diff --git a/demo/web/src/index.ts b/demo/web/src/index.ts index 346ff50913..768c56424f 100644 --- a/demo/web/src/index.ts +++ b/demo/web/src/index.ts @@ -119,9 +119,6 @@ function createTile(name: string): WatchTile { const watch = document.createElement("moq-watch") as MoqWatch; watch.name = name; watch.muted = true; // unmuted only while active (see below) - // Default to a fixed 100ms jitter buffer (instead of adaptive "auto") so - // the delay visualization has something to show. Drag it in the panel. - watch.setAttribute("delay", "100ms"); const canvas = document.createElement("canvas"); canvas.style.cssText = "width: 100%; height: auto;"; watch.appendChild(canvas); diff --git a/demo/web/src/viz.ts b/demo/web/src/viz.ts index abe1a5a496..827b83eda4 100644 --- a/demo/web/src/viz.ts +++ b/demo/web/src/viz.ts @@ -307,7 +307,7 @@ export function bufferBars(parent: Signals.Effect, watch: MoqWatch): HTMLElement const timestamp = watch.sync.now() as number | undefined; const stalled = watch.video.out.stalled.peek(); drawRanges(video.canvas, watch.video.out.buffered.peek(), timestamp, stalled); - drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, false); + drawRanges(audio.canvas, watch.audio.out.buffered.peek(), timestamp, watch.audio.out.stalled.peek()); parent.animate(draw); }; parent.animate(draw); diff --git a/doc/bin/cli.md b/doc/bin/cli.md index c64dfd088f..179045eee0 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -153,7 +153,7 @@ MKV uses the same flag to cap clusters, which otherwise follow video GOPs. ```bash moq --connect https://relay.example.com/anon --broadcast my-stream.hang play -moq ... play --delay 500ms # trade latency for a jittery link +moq ... play --delay 500ms # fix the delay instead of measuring it ``` Decodes H.264, H.265, and AV1 video using the platform hardware decoder where @@ -162,11 +162,18 @@ log names the decoder each track opened. `--video-name` and `--audio-name` pick a rendition. HE-AAC signaled only in band (implicit SBR, as over MPEG-TS) plays as its half-rate AAC-LC core. -Playback runs on a clock it owns. `--delay` (default 100 ms) is how far it -trails the live edge, which is both the jitter a late frame may absorb and the -point past which a stalled group is skipped. The speaker holds the delay, with a -50 ms floor under it, and the picture is scheduled against where the speaker -actually is. While video owns the clock, a frame arriving earlier than predicted +Playback runs on a clock it owns. `--delay` is how far it trails the live +edge: the jitter a late frame may absorb. The default, `auto`, measures how +unevenly audio arrives and sizes the speaker's buffer to match, using the same +[algorithm](/concept/audio-jitter) as the browser player, so a publisher that +flushes 100 ms at a time gets a buffer deep enough to play through the next +flush. It waits up to 2 s on a stalled group before skipping it, since a budget +any shorter would hide the very lateness it measures; a broadcast with no audio +has nothing to measure, so video trails by 100 ms and skips past that. A duration fixes the delay +instead, and doubles as the point past which a stalled group is skipped. The +speaker holds the delay, with a 50 ms floor under it: it pads back up to the +delay after running dry and skips back down onto it after a burst. The picture +is scheduled against where the speaker actually is. While video owns the clock, a frame arriving earlier than predicted pulls playback forward, so a late start catches up to live instead of staying behind it. Once the speaker owns the clock, video follows the speaker instead. diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index f300399fba..9aac9465b0 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -37,7 +37,30 @@ frame: | Native | `container::Consumer::read()`, awaited by `moq-audio`'s decode consumer | Both sit below the decoder and above nothing: no reordering, no gap handling, no -age budget. A frame dropped by the relay is never observed, which is correct. +age budget. + +Natively that holds only as far as the caller lets it. The container consumer +reads groups in sequence order and applies its age budget inside `read()`, so a +group skipped for age is never observed, and a frame queued behind a stalled +group is observed when the stall clears. A caller that estimates keeps the budget +at least as long as the estimate's ceiling, which is why `moq play` waits 2 s on +a stalled group under `--delay auto`. The observation is also only as honest as +the caller is prompt: `read()` is timed when it resolves, so a player drains it +as packets come and lets the playback buffer hold the target, rather than +pacing reads to the speaker. + +A frame dropped by the relay is never observed, which is correct only while the +drop is not the receiver's own doing. The subscription's max age drops groups +before any container sees them, at the relay and in the receiver's own transport +subscriber, so a subscription cut to the target hides every frame later than +the target: the estimate creeps up one bucket at a time while the frames it +should have measured go unplayed. In automatic mode both implementations +subscribe with at least the ceiling, 2 s: `moq play --delay auto` through its +age budget, and the browser through the audio subscription alone, since its +container consumer observes each frame before applying the local budget. +Browser video keeps the shared budget, because a longer one fetches whole stale +groups, so a broadcast with no audio measures only the lateness that budget +lets through. A frame the container will later discard *is* observed, which is also correct: it arrived, and when it arrived is the measurement. @@ -218,10 +241,8 @@ carry a number the payload already states. ### What is not in the frame duration -The browser's AudioWorklet renders in fixed 128-sample blocks, and -`js/watch/src/audio/config.ts` currently adds that block to the per-codec frame -duration, so 48 kHz Opus reads as 23 ms rather than 20 ms. **That block is not -part of the target.** It is a property of the render backend, not of the +The browser's AudioWorklet renders in fixed 128-sample blocks. **That block is +not part of the target.** It is a property of the render backend, not of the network, and native has no worklet at all. If one language's step carried it and the other's did not, the two would produce different target series from the same trace and the corpus could not hold. @@ -268,9 +289,6 @@ point strictly contains the publisher's. The floor earns its place by being correct *immediately*, before the estimator has seen a full flush cycle, which is the one thing a measurement cannot be. -Note that `js/watch` today adds the two rather than taking the maximum, which -over-buffers a bursty publisher by its own flush span. - ## Across renditions Every track plays from one clock, anchored on the earliest lateness any track diff --git a/doc/concept/audio-jitter/corpus.test.ts b/doc/concept/audio-jitter/corpus.test.ts index dd24b2b887..93f7a19e36 100644 --- a/doc/concept/audio-jitter/corpus.test.ts +++ b/doc/concept/audio-jitter/corpus.test.ts @@ -2,7 +2,7 @@ // has to catch is the corpus drifting from the algorithm it claims to encode: a checked-in // file edited by hand to make a failing implementation pass would otherwise go unnoticed. // -// Grading an implementation is each language's own test suite, since neither exists yet. +// Grading an implementation is each language's own test suite, e.g. js/hang's jitter.test.ts. import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index fb64b51549..718cccf3a8 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -30,7 +30,7 @@ in sync at the latency you ask for. | --- | --- | | `url`, `name` | Relay URL (with `?jwt=` if needed) and broadcast name. | | `paused`, `muted`, `volume` | The usual player controls, mirrored as reactive properties. | -| `delay` | How far playback trails the live edge: `"auto"` (derived from RTT, the default), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. | +| `delay` | How far playback trails the live edge: `"auto"` (the default, sized from how late frames actually arrive; see [audio jitter](/concept/audio-jitter)), a duration like `"300ms"`, or `"instant"` to paint frames as they decode with no pacing at all. | | `buffer` | Future-dated media held beyond the live edge before playback skips ahead, e.g. `"30s"`. Defaults to none. | | `captions` | The caption track to show, or absent for off. `el.text.out.available` lists the renditions for a picker. | | `visible` | Only subscribe to video while the element is on screen: a margin (`"20%"` default, `"200px"`), `"always"`, or `"never"`. | diff --git a/doc/setup/upgrade.md b/doc/setup/upgrade.md index 88ee9c0c3a..75f0dc45d9 100644 --- a/doc/setup/upgrade.md +++ b/doc/setup/upgrade.md @@ -37,6 +37,13 @@ These land with the next breaking release, not the 2026-09-23 train. In TypeScript, `Track.Producer`'s `used` and `unused()` are `producer.demand().used` and `.unused()`, and `Allocator.reserve` takes `producer.demand()`, replacing the `Bandwidth.Demand` interface. +- **The `"auto"` delay is measured, not derived from RTT** (#4162). It is sized + from how late frames arrive (see [audio jitter](/concept/audio-jitter)) in + `@moq/watch` and `moq play`, which now defaults `--delay` to `auto` instead of + `100ms`. A numeric `@moq/watch` delay is taken literally instead of having + the rendition's own delay added on top. `Sync.out.jitter` now always + equals `Sync.out.delay`, and `"auto"` with no decoder registered resolves to + 0 rather than 100 ms. - **moq-mux has no clock translators.** `clock::Anchor`, `clock::Lane`, and `SourceMap` (#4667) are gone, along with the importers' `live()`. Publish the source's own timestamps and let the catalog clock map them to wall time; diff --git a/js/hang/src/container/consumer.test.ts b/js/hang/src/container/consumer.test.ts index b66bc837c8..8c624f2976 100644 --- a/js/hang/src/container/consumer.test.ts +++ b/js/hang/src/container/consumer.test.ts @@ -645,6 +645,25 @@ test("Consumer skips groups via PTS-span when over the max age", async () => { consumer.close(); }); +test("Consumer measures arrivals as they land", async () => { + const track = new Track.Producer("test"); + const consumer = new Consumer(track.subscribe(), { format: new LegacyFormat("audio"), maxAge: 5000 as Time.Milli }); + + // Nothing observed yet: the cold-start estimate. + expect(consumer.spread.peek()).toBe(100 as Time.Milli); + + // Groups arriving exactly as fast as their media, across two resample intervals. Nothing is + // late, so the evidence pulls the estimate down from the cold start. + for (let i = 0; i < 3; i++) { + writeGroupWithLegacyFrames(track, i, [(i * 520_000) as Time.Micro]); + await settle(520); + } + expect(consumer.spread.peek()).toBeLessThan(100 as Time.Milli); + + track.close(); + consumer.close(); +}); + // --- Ordering --- test("Consumer delivers groups in sequence order regardless of arrival order", async () => { diff --git a/js/hang/src/container/consumer.ts b/js/hang/src/container/consumer.ts index 533bb1eda9..7d81c24edd 100644 --- a/js/hang/src/container/consumer.ts +++ b/js/hang/src/container/consumer.ts @@ -3,6 +3,7 @@ import * as Moq from "@moq/net"; import { Effect, type Getter, type GetterInit, getter, Once, Signal } from "@moq/signals"; import type { Format } from "./format"; +import { Jitter } from "./jitter"; import type { BufferedRanges, Frame } from "./types"; /** Options for constructing a {@link Consumer}. */ @@ -102,6 +103,16 @@ export class Consumer { /** The time ranges currently buffered and ready to play. */ readonly buffered: Getter = this.#buffered; + #jitter = new Jitter(); + #spread = new Signal(this.#jitter.measured); + /** + * How much buffer late arrivals need, measured as frames come off the transport. + * + * The measured term of doc/concept/audio-jitter.md, excluding the codec frame and any advertised + * floor. Starts at the cold-start estimate before any frame arrives. + */ + readonly spread: Getter = this.#spread; + #signals = new Effect(); #closed = new Once(); @@ -174,11 +185,19 @@ export class Consumer { const next = await group.consumer.readFrame(); if (!next) break; group.empty = false; + const arrival = Moq.Time.Milli.now(); const decoded = this.#format.decode(next.payload); for (const sample of decoded) { const marker = this.#format.end?.(sample) !== undefined; + + // Observed here, before any gap handling or age budget: a target derived from + // what survives the budget would only ever confirm the budget it was cut to. + if (!marker) { + this.#jitter.observe(arrival, Moq.Time.Milli.fromMicro(sample.timestamp)); + this.#spread.set(this.#jitter.measured); + } const frame: Frame = { payload: sample.payload, timestamp: sample.timestamp, diff --git a/js/hang/src/container/jitter.test.ts b/js/hang/src/container/jitter.test.ts new file mode 100644 index 0000000000..4eb28c140a --- /dev/null +++ b/js/hang/src/container/jitter.test.ts @@ -0,0 +1,95 @@ +import { describe, expect, it } from "bun:test"; +import { readdirSync, readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { Time } from "@moq/net"; +import { Jitter } from "./jitter"; + +// The conformance corpus beside doc/concept/audio-jitter.md, read directly so this is graded +// against the document rather than against another implementation. +const CORPUS = join(dirname(fileURLToPath(import.meta.url)), "../../../../doc/concept/audio-jitter"); + +interface Case { + name: string; + frame: number; + advertised?: number; + delay?: number; + arrival: number[]; + media: number[]; + target: [number, number][]; +} + +const cases: Case[] = readdirSync(CORPUS) + .filter((file) => file.endsWith(".json")) + .map((file) => JSON.parse(readFileSync(join(CORPUS, file), "utf8"))); + +// The composition the document specifies on top of the measured term. `@moq/watch` owns it in +// production (`audio/latency.ts`); it is repeated here so the whole target series is checked. +function target(jitter: Jitter, c: Case): number { + if (c.delay !== undefined) return c.delay; + return Math.max(jitter.measured, c.advertised ?? 0) + c.frame; +} + +describe("audio jitter conformance", () => { + it("reads the whole corpus", () => { + expect(cases.map((c) => c.name).sort()).toEqual([ + "advertised", + "buildup", + "fixed", + "flush", + "idle", + "paced", + "reorder", + "spike", + "tunein", + ]); + }); + + for (const c of cases) { + it(c.name, () => { + const jitter = new Jitter(); + const expected = new Map(c.target); + let current: number | undefined; + for (let i = 0; i < c.arrival.length; i++) { + jitter.observe(c.arrival[i] as Time.Milli, c.media[i] as Time.Milli); + const value = target(jitter, c); + const listed = expected.get(i); + if (listed !== undefined) { + expect(value).toBeCloseTo(listed, 6); + } else if (current !== undefined) { + // Unlisted frames hold the previous target. + expect(value).toBeCloseTo(current, 6); + } + current = value; + } + }); + } +}); + +describe("Jitter", () => { + it("starts at the prior's 95th percentile", () => { + expect(new Jitter().measured).toBe(100 as Time.Milli); + }); + + it("is not moved by a stale frame ahead of the live edge", () => { + // The #3517 regression: one frame from a stale group, then the live edge 14.5s of media later, + // then evenly paced audio with no real jitter. The jump must not read as a delay or a frame. + const jitter = new Jitter(); + jitter.observe(0 as Time.Milli, 0 as Time.Milli); + for (let i = 0; i < 500; i++) { + jitter.observe((10 + i * 20) as Time.Milli, (14_500 + i * 20) as Time.Milli); + } + expect(jitter.measured).toBe(20 as Time.Milli); + }); + + it("does not follow the absolute delay", () => { + // Only the spread of arrivals matters, not how far away the sender is. + const near = new Jitter(); + const far = new Jitter(); + for (let i = 0; i < 500; i++) { + near.observe((10 + i * 20) as Time.Milli, (i * 20) as Time.Milli); + far.observe((2000 + i * 20) as Time.Milli, (i * 20) as Time.Milli); + } + expect(far.measured).toBe(near.measured); + }); +}); diff --git a/js/hang/src/container/jitter.ts b/js/hang/src/container/jitter.ts new file mode 100644 index 0000000000..6cbb9d0a70 --- /dev/null +++ b/js/hang/src/container/jitter.ts @@ -0,0 +1,125 @@ +import type { Time } from "@moq/net"; + +// Implements the estimator in doc/concept/audio-jitter.md, which is normative. The design and +// every constant come from WebRTC NetEq's underrun optimizer; this is written from the document, +// not from NetEq's source. + +/** Milliseconds of media time the reference search looks back over. */ +const WINDOW = 2000; + +/** Milliseconds of delay per histogram bucket. */ +const BUCKET = 20; + +/** Histogram buckets, so the measured term saturates at `BUCKET * BUCKETS`. */ +const BUCKETS = 100; + +/** The quantile of the delay distribution the estimate is read at. */ +const QUANTILE = 0.95; + +/** Weight retained per folded observation, a ~20s half-life at one fold per `RESAMPLE`. */ +const FORGET = 0.983; + +/** Milliseconds of arrival time per resampled observation. */ +const RESAMPLE = 500; + +/** Holds the forget factor below `FORGET` early on, so evidence overwrites the prior. */ +const RAMP = 2; + +/** Idle intervals folded at once, purely to bound the work. */ +const IDLE_MAX = 1024; + +/** The cold-start distribution, which is also what an idle interval relaxes back to. */ +const PRIOR = Float64Array.from({ length: BUCKETS }, (_, i) => 0.5 ** (i + 1)); + +/** + * How much buffer late arrivals need, measured from arrival timing alone. + * + * Each frame's delay is measured against the fastest frame in the last `WINDOW` of media time, the + * largest delay per `RESAMPLE` interval lands in a decaying histogram, and the estimate is its 95th + * percentile. Excludes the codec frame and any advertised floor, which the caller composes on top. + */ +export class Jitter { + /** Arrivals still inside the reference window, in media order. */ + #history: { arrival: number; media: number }[] = []; + /** The largest media timestamp observed, so a reordered frame is recognized. */ + #newest = Number.NEGATIVE_INFINITY; + + /** The arrival resample intervals are counted from, unset until the first observation. */ + #origin?: number; + /** The interval currently accumulating, counted from `#origin`. */ + #interval = 0; + /** The largest delay seen in that interval. */ + #peak = 0; + + #histogram = Float64Array.from(PRIOR); + /** Observations folded in, which drives the cold-start ramp. */ + #count = 0; + + /** Fold one frame in: `arrival` on the receiver's monotonic clock, `media` its timestamp. */ + observe(arrival: Time.Milli, media: Time.Milli): void { + // A reordered frame is excluded rather than measured: the media-time gap back to the frame + // that overtook it would otherwise be added to the delay. + if (media <= this.#newest) return; + this.#newest = media; + + // Pruning by media time is what makes a timestamp jump self-correcting: everything before it + // leaves the window at once and the new frame becomes its own reference. + let stale = 0; + while (stale < this.#history.length && this.#history[stale].media < media - WINDOW) stale++; + if (stale > 0) this.#history.splice(0, stale); + this.#history.push({ arrival, media }); + + // Delay against the fastest frame in the window, not the previous one, so a queue that + // fills a little per frame reads as the build-up it is. + let reference = this.#history[0]; + for (const entry of this.#history) { + if (entry.arrival - entry.media < reference.arrival - reference.media) reference = entry; + } + const delay = Math.max(0, arrival - reference.arrival - (media - reference.media)); + + if (this.#origin === undefined) { + this.#origin = arrival; + this.#peak = delay; + return; + } + + const interval = Math.floor((arrival - this.#origin) / RESAMPLE); + if (interval <= this.#interval) { + this.#peak = Math.max(this.#peak, delay); + return; + } + + this.#fold(Math.min(BUCKETS - 1, Math.floor(this.#peak / BUCKET))); + + // An interval with no arrival relaxes the estimate toward the prior at the same rate. + const idle = Math.min(interval - this.#interval - 1, IDLE_MAX); + for (let i = 0; i < idle; i++) this.#fold(undefined); + + this.#interval = interval; + this.#peak = delay; + } + + /** The measured term: the upper edge of the bucket holding the 95th percentile. */ + get measured(): Time.Milli { + let cumulative = 0; + for (let i = 0; i < BUCKETS; i++) { + cumulative += this.#histogram[i]; + if (cumulative > QUANTILE) return ((i + 1) * BUCKET) as Time.Milli; + } + return (BUCKETS * BUCKET) as Time.Milli; + } + + // `h = f*h + (1-f)*x`, where x is all weight on the observed bucket, or the prior when the + // interval held no arrival. The total stays at 1 without renormalizing. + #fold(bucket: number | undefined): void { + this.#count += 1; + const forget = Math.min(FORGET, this.#count / (this.#count + RAMP)); + const fresh = 1 - forget; + + for (let i = 0; i < BUCKETS; i++) { + this.#histogram[i] *= forget; + if (bucket === undefined) this.#histogram[i] += fresh * PRIOR[i]; + } + if (bucket !== undefined) this.#histogram[bucket] += fresh; + } +} diff --git a/js/hang/src/util/opus.test.ts b/js/hang/src/util/opus.test.ts index d86001567b..50f054aa87 100644 --- a/js/hang/src/util/opus.test.ts +++ b/js/hang/src/util/opus.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from "bun:test"; -import { pickRate, preSkip, SAMPLE_RATES, supportsRate, toDOps } from "./opus"; +import { packetSamples, pickRate, preSkip, SAMPLE_RATES, supportsRate, toDOps } from "./opus"; describe("pickRate", () => { // Matches the pick_opus_rate tests in rs/moq-audio/src/codec.rs. @@ -80,3 +80,33 @@ describe("preSkip", () => { expect(preSkip(dops)).toBe(312); }); }); + +describe("packetSamples", () => { + // Matches packet_samples_reads_toc in rs/moq-mux/src/codec/opus/mod.rs. + it("reads the TOC byte", () => { + // config 16 (CELT NB 2.5 ms), code 0 (1 frame). + expect(packetSamples(new Uint8Array([16 << 3]))).toBe(120); + // config 3 (SILK NB 60 ms), code 0. + expect(packetSamples(new Uint8Array([3 << 3]))).toBe(2880); + // config 1 (SILK NB 20 ms), code 1 (2 frames). + expect(packetSamples(new Uint8Array([(1 << 3) | 1]))).toBe(1920); + // config 1, code 3 with 4 frames. + expect(packetSamples(new Uint8Array([(1 << 3) | 3, 4]))).toBe(3840); + // config 31 (CELT FB 20 ms), the libopus real-time default. + expect(packetSamples(new Uint8Array([31 << 3]))).toBe(960); + }); + + it("is undefined for a truncated packet", () => { + expect(packetSamples(new Uint8Array([]))).toBeUndefined(); + expect(packetSamples(new Uint8Array([(1 << 3) | 3]))).toBeUndefined(); + }); + + it("is undefined outside 1 frame and 120 ms", () => { + // Code 3 with zero frames. + expect(packetSamples(new Uint8Array([(1 << 3) | 3, 0]))).toBeUndefined(); + // Code 3 with 63 frames of 60 ms. + expect(packetSamples(new Uint8Array([(3 << 3) | 3, 63]))).toBeUndefined(); + // Code 3 with 2 frames of 60 ms is the 120 ms maximum. + expect(packetSamples(new Uint8Array([(3 << 3) | 3, 2]))).toBe(5760); + }); +}); diff --git a/js/hang/src/util/opus.ts b/js/hang/src/util/opus.ts index 76461e4d49..28ec3aba4d 100644 --- a/js/hang/src/util/opus.ts +++ b/js/hang/src/util/opus.ts @@ -83,3 +83,43 @@ export function toDOps(description: Uint8Array): Uint8Array { output[10] = input.getUint8(10); return output; } + +/** + * Number of 48 kHz samples in an Opus packet, read from its TOC byte (RFC 6716 §3.1). + * + * Mirrors `packet_samples` in rs/moq-mux. Opus timing is always reckoned at 48 kHz regardless of the + * encoder's internal bandwidth. Undefined for an empty packet, a code-3 packet missing its + * frame-count byte, or a packet outside the 1 frame and 120 ms RFC 6716 allows. + */ +export function packetSamples(packet: Uint8Array): number | undefined { + const toc = packet.at(0); + if (toc === undefined) return undefined; + + let frames: number; + const code = toc & 0b11; + if (code === 0) frames = 1; + else if (code !== 3) frames = 2; + else { + // Code 3: the frame count is the low 6 bits of the following byte. + const count = packet.at(1); + if (count === undefined) return undefined; + frames = count & 0b11_1111; + } + + const samples = configSamples(toc >> 3) * frames; + if (frames === 0 || samples > MAX_PACKET_SAMPLES) return undefined; + return samples; +} + +// RFC 6716 §3.2.5: a packet holds at most 120 ms of audio. +const MAX_PACKET_SAMPLES = 5_760; + +// 48 kHz samples per frame for a TOC config index (0..=31), per RFC 6716 Table 1. +function configSamples(config: number): number { + // SILK NB/MB/WB: 10, 20, 40, 60 ms. + if (config < 12) return [480, 960, 1920, 2880][config % 4]; + // Hybrid SWB/FB: 10, 20 ms. + if (config < 16) return [480, 960][config % 2]; + // CELT NB/WB/SWB/FB: 2.5, 5, 10, 20 ms. + return [120, 240, 480, 960][config % 4]; +} diff --git a/js/moq-boy/src/game.ts b/js/moq-boy/src/game.ts index 9d2a03d39d..05caf1071a 100644 --- a/js/moq-boy/src/game.ts +++ b/js/moq-boy/src/game.ts @@ -136,10 +136,7 @@ export class Game { }); this.#signals.cleanup(() => this.audioSource.close()); - this.sync = new Watch.Sync({ - delay: this.delay, - probe: connection.probe, - }); + this.sync = new Watch.Sync({ delay: this.delay }); this.#signals.cleanup(() => this.sync.close()); this.#signals.run(this.#runPixelBudget.bind(this)); diff --git a/js/watch/src/audio/buffer.ts b/js/watch/src/audio/buffer.ts index 528ed93608..6e54ababf1 100644 --- a/js/watch/src/audio/buffer.ts +++ b/js/watch/src/audio/buffer.ts @@ -21,7 +21,7 @@ class Backpressure { this.#headroom = headroom; } - // Move the gate as the floor changes (e.g. "real-time" jitter tracking RTT). + // Move the gate as the floor changes (e.g. "auto" tracking the measured arrival spread). setHeadroom(headroom: Time.Micro): void { this.#headroom = headroom; } @@ -100,6 +100,9 @@ export interface AudioBuffer { /** Whether the buffer is stalled (waiting to fill). */ readonly stalled: Getter; + /** How many times the ring has run dry mid-playback, cumulative. */ + readonly underruns: Getter; + /** Release any resources (event listeners, intervals, etc.). */ close(): void; } @@ -148,6 +151,9 @@ class SharedAudioBuffer implements AudioBuffer { readonly #stalled = new Signal(true); readonly stalled: Getter = this.#stalled; + readonly #underruns = new Signal(0); + readonly underruns: Getter = this.#underruns; + #backpressure: Backpressure; #signals = new Effect(); @@ -174,6 +180,7 @@ class SharedAudioBuffer implements AudioBuffer { const stalled = this.#ring.stalled; this.#timestamp.set(this.#ring.timestamp); this.#stalled.set(stalled); + this.#underruns.set(this.#ring.underruns); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (stalled) this.#backpressure.flush(); @@ -212,7 +219,10 @@ class SharedAudioBuffer implements AudioBuffer { } wait(timestamp: Time.Micro): Promise { - // Stalled = still filling the floor (bootstrap or underflow): let frames through to refill. + // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an + // explicit reset): let frames through so the ring refills to the floor. This is the single + // re-buffer path; the ring un-stalls itself on the insert that reaches the floor, so the + // gate closes again within one frame rather than draining the whole lookahead. if (this.#ring.stalled) return Promise.resolve(); return this.#backpressure.wait(timestamp, this.#ring.timestamp); } @@ -235,6 +245,9 @@ class PostAudioBuffer implements AudioBuffer { readonly #stalled = new Signal(true); readonly stalled: Getter = this.#stalled; + readonly #underruns = new Signal(0); + readonly underruns: Getter = this.#underruns; + // Backpressure runs off the playhead the worklet reports in its state messages. #backpressure: Backpressure; @@ -257,6 +270,7 @@ class PostAudioBuffer implements AudioBuffer { if (data?.type === "state") { this.#timestamp.set(data.timestamp); this.#stalled.set(data.stalled); + this.#underruns.set(data.underruns); // While stalled the playhead is parked, so release the decode loop to refill the floor; // once playing, hold it to ~the floor ahead. if (data.stalled) this.#backpressure.flush(); @@ -297,7 +311,8 @@ class PostAudioBuffer implements AudioBuffer { } wait(timestamp: Time.Micro): Promise { - // Stalled = still filling the floor (bootstrap or underflow): let frames through to refill. + // Stalled = still filling the floor (bootstrap, an underrun the reader re-stalled on, or an + // explicit reset): let frames through so the ring refills to the floor. See SharedAudioBuffer. if (this.#stalled.peek()) return Promise.resolve(); // Uses the worklet-reported playhead, which lags by a state-message interval; the floor's // headroom covers that. The worklet still drops the oldest if a frame slips through. diff --git a/js/watch/src/audio/config.test.ts b/js/watch/src/audio/config.test.ts index ca5dee3a23..6922de15d6 100644 --- a/js/watch/src/audio/config.test.ts +++ b/js/watch/src/audio/config.test.ts @@ -2,7 +2,7 @@ import { expect, test } from "bun:test"; import * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; import { Effect, Signal } from "@moq/signals"; -import { decoderConfig, playbackIdentity, playbackJitter } from "./config"; +import { decoderConfig, frameDuration, packetDuration, playbackIdentity } from "./config"; const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); @@ -60,26 +60,27 @@ test("routing and decoder inputs change the playback identity", () => { expect(playbackIdentity(config({ numberOfChannels: 1 }))).not.toEqual(base); }); -test("an advertised jitter of zero falls back to the codec frame duration", () => { - // 48kHz stereo Opus: 20ms frames plus the 128 sample worklet quantum (3ms). - const floor = playbackJitter(config()); - expect(floor).toBe(Time.Milli(23)); - expect(playbackJitter(config({ jitter: 0 }))).toBe(floor); - expect(playbackJitter(config({ jitter: 60 }))).toBe(Time.Milli(63)); +test("AAC and MP3 frame durations follow their codec frame sizes", () => { + expect(frameDuration(config({ codec: "mp4a.40.2", sampleRate: 48000 }))).toBeCloseTo(21.333, 3); + expect(frameDuration(config({ codec: "mp4a.40.5", sampleRate: 48000 }))).toBeCloseTo(42.667, 3); + expect(frameDuration(config({ codec: "mp3", sampleRate: 48000 }))).toBe(Time.Milli(24)); + expect(frameDuration(config({ codec: "mp3", sampleRate: 24000 }))).toBe(Time.Milli(24)); }); -test("a rendition's delay adds to its jitter", () => { - expect(playbackJitter(config({ delay: 200 }))).toBe(Time.Milli(223)); - expect(playbackJitter(config({ delay: 200, jitter: 60 }))).toBe(Time.Milli(263)); +test("Opus and unknown codecs have no constant frame duration", () => { + // Opus states its duration per packet, in the TOC byte. + expect(frameDuration(config())).toBeUndefined(); + expect(frameDuration(config({ codec: "flac" }))).toBeUndefined(); }); -test("AAC and MP3 jitter follows their codec frame sizes", () => { - expect(playbackJitter(config({ codec: "mp4a.40.2", sampleRate: 48000 }))).toBe(Time.Milli(25)); - expect(playbackJitter(config({ codec: "mp4a.40.2", sampleRate: 24000 }))).toBe(Time.Milli(49)); - expect(playbackJitter(config({ codec: "mp3", sampleRate: 48000 }))).toBe(Time.Milli(27)); - expect(playbackJitter(config({ codec: "mp3", sampleRate: 24000 }))).toBe(Time.Milli(30)); -}); - -test("an unknown codec without advertised jitter only reserves the worklet quantum", () => { - expect(playbackJitter(config({ codec: "flac", jitter: 0 }))).toBe(Time.Milli(3)); +test("an implicit CMAF duration falls through to the Opus TOC", () => { + // TOC 0x78: config 15 (hybrid fullband, 20 ms), one frame. + const frame = { + timestamp: Time.Micro(0), + keyframe: true, + payload: new Uint8Array([0x78, 0]), + duration: Time.Micro(0), + }; + expect(packetDuration("opus", frame)).toBe(Time.Milli(20)); + expect(packetDuration("mp4a.40.2", frame)).toBeUndefined(); }); diff --git a/js/watch/src/audio/config.ts b/js/watch/src/audio/config.ts index 2665ce83d8..a24fed8ff8 100644 --- a/js/watch/src/audio/config.ts +++ b/js/watch/src/audio/config.ts @@ -1,10 +1,11 @@ import type * as Catalog from "@moq/hang/catalog"; +import type * as Container from "@moq/hang/container"; +import * as Util from "@moq/hang/util"; import { Time } from "@moq/net"; -// AudioWorklet always renders in 128-sample quanta. -const WORKLET_QUANTUM = 128; -const OPUS_FRAME_DURATION_MS = 20; +const OPUS_RATE = 48_000; const AAC_LC_FRAME_SAMPLES = 1024; +const AAC_HE_FRAME_SAMPLES = 2048; const MP3_MPEG1_FRAME_SAMPLES = 1152; const MP3_MPEG2_FRAME_SAMPLES = 576; const MP3_MPEG1_MIN_SAMPLE_RATE = 32000; @@ -41,38 +42,36 @@ export function playbackIdentity(config: Catalog.AudioConfig): PlaybackIdentity } /** - * The sync buffer a rendition needs, in milliseconds: its catalog `delay` behind the broadcast's - * earliest rendition plus its own jitter. + * A frame's duration as a constant over the catalog's sample rate, for codecs that fix it. + * + * Undefined for Opus, whose packets state their own duration (see `Util.Opus.packetSamples`), and + * for codecs with no constant. Never learned from observed timestamps. */ -export function playbackJitter(config: Catalog.AudioConfig): Time.Milli { - // A publisher advertising 0 is claiming frames are never delayed, which no encoder can do, so - // fall back to the codec's frame duration the same way an absent field does. - const codecJitter = (config.jitter || defaultJitter(config)) ?? 0; - - // Add the worklet render quantum so the ring buffer has margin between frame arrivals. - const overhead = Math.ceil((WORKLET_QUANTUM / config.sampleRate) * 1000); - return Time.Milli((config.delay ?? 0) + codecJitter + overhead); +export function frameDuration(config: Pick): Time.Milli | undefined { + const samples = frameSamples(config); + if (samples === undefined) return undefined; + return Time.Milli((samples * 1000) / config.sampleRate); } -// Estimate the minimum jitter (frame duration) based on the audio codec. -// TODO these are defaults; the actual frame duration depends on encoder config. -function defaultJitter(config: Catalog.AudioConfig): number | undefined { - if (config.codec.startsWith("opus")) { - // Opus supports 2.5–60ms but 20ms is the real-time default. - return OPUS_FRAME_DURATION_MS; - } - - if (config.codec.startsWith("mp4a")) { - // 1024 samples for LC-AAC; HE-AAC/AAC-LD use different sizes. - return Math.ceil((AAC_LC_FRAME_SAMPLES / config.sampleRate) * 1000); - } +/** + * A frame's own duration, when it states one: the container's per-sample duration (CMAF), else the + * duration an Opus packet declares in its TOC byte (RFC 6716 §3.1), always reckoned at 48 kHz. + * CMAF reports an implicit duration as zero, which states nothing. + */ +export function packetDuration(codec: string, frame: Container.Frame): Time.Milli | undefined { + if (frame.duration) return Time.Milli.fromMicro(frame.duration); + if (!codec.startsWith("opus")) return undefined; + const samples = Util.Opus.packetSamples(frame.payload); + return samples === undefined ? undefined : Time.Milli((samples * 1000) / OPUS_RATE); +} +function frameSamples(config: Pick): number | undefined { + // HE-AAC (v1 and v2) doubles the output rate of its AAC-LC core. + if (config.codec === "mp4a.40.5" || config.codec === "mp4a.40.29") return AAC_HE_FRAME_SAMPLES; + if (config.codec.startsWith("mp4a")) return AAC_LC_FRAME_SAMPLES; if (config.codec === "mp3") { - // 1152 samples per frame for MPEG-1 Layer III; MPEG-2/2.5 use 576. - const samples = - config.sampleRate >= MP3_MPEG1_MIN_SAMPLE_RATE ? MP3_MPEG1_FRAME_SAMPLES : MP3_MPEG2_FRAME_SAMPLES; - return Math.ceil((samples / config.sampleRate) * 1000); + // MPEG-1 Layer III has 1152 samples per frame; MPEG-2/2.5 have 576. + return config.sampleRate >= MP3_MPEG1_MIN_SAMPLE_RATE ? MP3_MPEG1_FRAME_SAMPLES : MP3_MPEG2_FRAME_SAMPLES; } - return undefined; } diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index 11aad68392..2c5ecddb7d 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -19,9 +19,16 @@ import { nextMedia, subscribeMedia } from "../media"; import type { Sync } from "../sync"; import { type AudioBuffer, createAudioBuffer } from "./buffer"; -import { type DecoderConfig, decoderConfig, type PlaybackIdentity, playbackIdentity } from "./config"; +import { + type DecoderConfig, + decoderConfig, + frameDuration, + type PlaybackIdentity, + packetDuration, + playbackIdentity, +} from "./config"; import { Handover } from "./handover"; -import { ringSamples } from "./latency"; +import { AUTO_MAX_AGE, ringSamples, target } from "./latency"; // A blob: URL, or a hosted file when assets() is set; see vite-plugin-worklet. import RenderWorklet from "./render-worklet.ts?worklet"; import type { Source } from "./source"; @@ -59,6 +66,9 @@ type DecoderOutput = { // Whether the audio buffer is stalled (waiting to fill) stalled: Signal; + // How many times the ring ran dry mid-playback, so the UI can show that the target is too low. + underruns: Signal; + // Combined buffered ranges (network jitter + decode buffer) buffered: Signal; }; @@ -86,6 +96,7 @@ export class Decoder { stats: new Signal(undefined), timestamp: new Signal(undefined), stalled: new Signal(true), + underruns: new Signal(0), buffered: new Signal([]), }; readonly out = readonlys(this.#out); @@ -105,6 +116,15 @@ export class Decoder { // Ordered discontinuity and endpoint state from the container consumer. #terminal = new Terminal(); + // The container consumer's arrival estimate, unset while nothing is subscribed. + #measured = new Signal(undefined); + + // The subscription's max age: the shared budget, raised to the estimator's ceiling in "auto". + #subscribeMaxAge = new Signal(Time.Milli.zero); + + // The codec's frame duration: the catalog constant, refined by each frame's own duration. + #frame = new Signal(undefined); + // Which subscription the ring's buffered samples came from. See #runDecoder. #handover = new Handover(); @@ -124,7 +144,19 @@ export class Decoder { this.source = props.source; this.sync = props.sync; - this.#signals.cleanup(this.sync.register(this.source.out.jitter)); + // The "auto" playout target this track needs, per doc/concept/audio-jitter.md. + const playout = this.#signals.computed((effect) => { + const measured = effect.get(this.#measured); + if (measured === undefined) return undefined; + const delay = effect.get(this.source.out.config)?.delay; + return target({ + measured, + advertised: effect.get(this.source.out.jitter), + frame: effect.get(this.#frame), + delay: delay !== undefined ? Time.Milli(delay) : undefined, + }); + }); + this.#signals.cleanup(this.sync.register(playout)); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -134,12 +166,22 @@ export class Decoder { return config ? decoderConfig(config) : undefined; }); + this.#signals.run(this.#runSubscribeMaxAge.bind(this)); this.#signals.run(this.#runWorklet.bind(this)); this.#signals.run(this.#runEnabled.bind(this)); this.#signals.run(this.#runLatency.bind(this)); this.#signals.run(this.#runDecoder.bind(this)); } + // A group the relay expires is never observed, so a subscription cut to the target would cap the + // estimate at the target it already holds. The container consumer keeps the shared budget as its + // local skip, since it observes each frame before applying it. See doc/concept/audio-jitter.md. + #runSubscribeMaxAge(effect: Effect): void { + const maxAge = effect.get(this.sync.out.maxAge); + const auto = effect.get(this.sync.in.delay) === "auto"; + this.#subscribeMaxAge.set(auto ? Time.Milli.max(maxAge, AUTO_MAX_AGE) : maxAge); + } + #runWorklet(effect: Effect): void { // It takes a second or so to initialize the AudioContext/AudioWorklet, so do it even if disabled. // This is less efficient for video-only playback but makes muting/unmuting instant. @@ -213,6 +255,9 @@ export class Decoder { effect.run((inner) => { this.#out.stalled.set(inner.get(ring.stalled)); }); + effect.run((inner) => { + this.#out.underruns.set(inner.get(ring.underruns)); + }); effect.set(this.#out.root, worklet); }); @@ -245,7 +290,8 @@ export class Decoder { if (!ring) return; // A rise parks playback until the ring refills to the new floor, which is what keeps audio in - // step with video. A catalog's jitter estimate can rise a millisecond at a time. + // step with video. The measured target moves a bucket at a time, and a rise the buffered + // slack already covers costs no silence at all. const delay = effect.get(this.sync.out.delay); ring.setLatency(ringSamples(ring.rate, delay)); } @@ -265,6 +311,7 @@ export class Decoder { if (!identity) return; const config = identity.decoder; + this.#frame.set(frameDuration(config)); // Honor a per-rendition `broadcast` override: subscribe on the resolved source // broadcast instead of the catalog's own broadcast. @@ -282,7 +329,7 @@ export class Decoder { broadcast: active, track, priority: Catalog.PRIORITY.audio, - maxAge: this.sync.out.maxAge, + maxAge: this.#subscribeMaxAge, }); if (!sub) return; @@ -314,6 +361,11 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Feed the arrival estimate into the playout target. Cleared on teardown so a departed track + // stops holding the buffer open. + effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); + effect.cleanup(() => this.#measured.set(undefined)); + effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); if (!loaded) return; // cancelled @@ -369,6 +421,9 @@ export class Decoder { const timestamp = Time.Milli.fromMicro(frame.timestamp as Time.Micro); this.sync.received(timestamp, "audio"); + const duration = packetDuration(config.codec, frame); + if (duration !== undefined) this.#frame.set(duration); + this.#out.stats.update((stats) => ({ bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); @@ -421,6 +476,11 @@ export class Decoder { this.#out.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Feed the arrival estimate into the playout target. Cleared on teardown so a departed track + // stops holding the buffer open. + effect.run((inner) => this.#measured.set(inner.get(consumer.spread))); + effect.cleanup(() => this.#measured.set(undefined)); + effect.spawn(async () => { const loaded = await Util.Libav.polyfill(); if (!loaded) return; // cancelled @@ -458,6 +518,9 @@ export class Decoder { const timestamp = Time.Milli.fromMicro(frame.timestamp); this.sync.received(timestamp, "audio"); + const duration = packetDuration(config.codec, frame); + if (duration !== undefined) this.#frame.set(duration); + this.#out.stats.update((stats) => ({ bytesReceived: (stats?.bytesReceived ?? 0) + frame.payload.byteLength, })); diff --git a/js/watch/src/audio/latency.test.ts b/js/watch/src/audio/latency.test.ts index e691821045..c3b02a7962 100644 --- a/js/watch/src/audio/latency.test.ts +++ b/js/watch/src/audio/latency.test.ts @@ -1,9 +1,28 @@ import { describe, expect, it } from "bun:test"; import type { Time } from "@moq/net"; -import { ringSamples } from "./latency"; +import { ringSamples, target } from "./latency"; const ms = (value: number) => value as Time.Milli; +describe("target", () => { + it("adds one frame to the measured term", () => { + expect(target({ measured: ms(40), frame: ms(20) })).toBe(ms(60)); + }); + + it("floors the measured term at the advertised span instead of adding it", () => { + expect(target({ measured: ms(40), advertised: ms(200), frame: ms(20) })).toBe(ms(220)); + expect(target({ measured: ms(300), advertised: ms(200), frame: ms(20) })).toBe(ms(320)); + }); + + it("adds the catalog delay on top, since the measurement cancels a cross-track offset", () => { + expect(target({ measured: ms(40), advertised: ms(60), frame: ms(20), delay: ms(200) })).toBe(ms(280)); + }); + + it("is the measured term alone when nothing else is known", () => { + expect(target({ measured: ms(100) })).toBe(ms(100)); + }); +}); + describe("ringSamples", () => { // `delay="instant"` reports a zero buffer. Passed through, the ring rejects it and the // worklet is left with no backend that any later resize can revive. diff --git a/js/watch/src/audio/latency.ts b/js/watch/src/audio/latency.ts index 1885d28fd9..1284f94088 100644 --- a/js/watch/src/audio/latency.ts +++ b/js/watch/src/audio/latency.ts @@ -1,5 +1,39 @@ import { Time } from "@moq/net"; +/** The terms of the audio playout target, all in milliseconds. */ +export interface Target { + /** The arrival estimate from the container consumer. */ + measured: Time.Milli; + /** The flush span the rendition advertises, if any. */ + advertised?: Time.Milli; + /** The codec's frame duration, if known. */ + frame?: Time.Milli; + /** The catalog `delay`: how far this rendition trails the broadcast's earliest one, if any. */ + delay?: Time.Milli; +} + +/** + * The "auto" playout target: the measured term floored by the advertised span, plus one frame, plus + * the rendition's catalog delay. + * + * The advertised span is a floor rather than an addend, because the receiver's measurement already + * contains the publisher's flush delay. The catalog delay is an addend, because the measurement is + * taken against the track's own fastest frame and cancels any offset between tracks. See + * doc/concept/audio-jitter.md. + */ +export function target(props: Target): Time.Milli { + const floored = Time.Milli.max(props.measured, props.advertised ?? Time.Milli.zero); + const own = Time.Milli.add(floored, props.frame ?? Time.Milli.zero); + return Time.Milli.add(own, props.delay ?? Time.Milli.zero); +} + +/** + * The least subscription max age an "auto" track asks for: the estimator's ceiling. + * + * The measured term saturates at 100 buckets of 20 ms, so a frame later than this adds nothing. + */ +export const AUTO_MAX_AGE = 2000 as Time.Milli; + // An AudioWorkletProcessor renders in fixed 128-sample quanta, so a ring shallower than one can // never be read from. const RENDER_QUANTUM = 128; diff --git a/js/watch/src/audio/playout.test.ts b/js/watch/src/audio/playout.test.ts new file mode 100644 index 0000000000..4722bcbb5a --- /dev/null +++ b/js/watch/src/audio/playout.test.ts @@ -0,0 +1,271 @@ +import { describe, expect, it, spyOn } from "bun:test"; +import * as Container from "@moq/hang/container"; +import { Group, Time, Track, Varint } from "@moq/net"; +import { AUTO_MAX_AGE, target } from "./latency"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; + +// Replay recorded-shape arrival traces through both rings and count what a listener would hear: +// quanta rendered short (an underrun) and samples the ring threw away (a skip). Both rings are +// driven through the same harness, since the postMessage fallback is the path every page without +// cross-origin isolation takes. Each ring is sized from the target the estimator settles on, measured +// through the transport subscription and container consumer the decoder reads. + +const RATE = 48000; +const QUANTUM = 128; // an AudioWorklet render quantum +const CHUNK_MS = 20; // one Opus frame +const CHUNK = (RATE * CHUNK_MS) / 1000; + +/** Deterministic PRNG, so a trace is the same on every machine. */ +function rng(seed: number): () => number { + let a = seed >>> 0; + return () => { + a = (a + 0x6d2b79f5) >>> 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} + +/** One frame of a trace: when it was captured, and the wall time it reached us. */ +interface Arrival { + media: number; + arrival: number; +} + +/** + * A sender emitting a frame every `CHUNK_MS`, holding `burst - 1` of them back and flushing the + * run at once, plus `spread` ms of per-flush network jitter. `burst` of 1 is an evenly paced + * sender; anything above it is the PES packing an importer produces. + */ +function trace(frames: number, burst: number, spread: number, seed = 7): Arrival[] { + const rand = rng(seed); + const out: Arrival[] = []; + let held: number[] = []; + for (let i = 0; i < frames; i++) { + const media = i * CHUNK_MS; + held.push(media); + if (held.length < burst) continue; + // Every frame keeps its own capture time; the whole run shares one arrival. + const arrival = media + 50 + rand() * spread; + for (const m of held) out.push({ media: m, arrival }); + held = []; + } + return out; +} + +/** A legacy container frame: the media timestamp, then a payload the consumer never decodes. */ +function encode(media: Time.Micro): Uint8Array { + const timestamp = Varint.encode(media); + const frame = new Uint8Array(timestamp.byteLength + 1); + frame.set(timestamp, 0); + return frame; +} + +interface Measured { + /** The playout target after each distinct arrival instant. */ + targets: Time.Milli[]; + /** Frames that reached the consumer's reader rather than being dropped for age. */ + delivered: number; +} + +/** + * Feed `t` through the path the decoder measures on: a transport subscription into a container + * consumer, on a stubbed monotonic clock, one group per frame. After each arrival `budget` turns the + * current target into the subscription's max age, the way the decoder does. + */ +async function measure(t: Arrival[], budget: (target: Time.Milli) => Time.Milli): Promise { + let clock = 0; + const now = spyOn(performance, "now").mockImplementation(() => clock); + const track = new Track.Producer("audio"); + const subscriber = track.subscribe({ maxAge: budget(Time.Milli.zero) }); + const consumer = new Container.Consumer(subscriber, { + format: new Container.Legacy.Format("audio"), + maxAge: AUTO_MAX_AGE, + }); + + const result: Measured = { targets: [], delivered: 0 }; + const reader = (async () => { + for (;;) { + const next = await consumer.next(); + if (!next) return; + if (next.frame) result.delivered++; + } + })(); + + try { + let next = 0; + while (next < t.length) { + clock = t[next].arrival; + for (; next < t.length && t[next].arrival === clock; next++) { + const media = Time.Micro.fromMilli(t[next].media as Time.Milli); + const group = new Group.Producer(next); + group.writeFrame({ payload: encode(media), timestamp: Time.Timestamp.fromMicros(media) }); + group.close(); + track.writeGroup(group); + } + + // A macrotask boundary, so the group readers have observed everything written above. + await new Promise((resolve) => setImmediate(resolve)); + + const current = target({ measured: consumer.spread.peek(), frame: CHUNK_MS as Time.Milli }); + result.targets.push(current); + subscriber.update({ maxAge: budget(current) }); + } + } finally { + track.close(); + consumer.close(); + await reader; + now.mockRestore(); + } + + return result; +} + +/** The target the estimator settles on by the end of `t`, given the subscription "auto" asks for. */ +async function settled(t: Arrival[]): Promise { + const { targets } = await measure(t, (target) => Time.Milli.max(target, AUTO_MAX_AGE)); + return targets[targets.length - 1]; +} + +/** The two rings behind one interface, since the harness drives them identically. */ +interface Ring { + insert(timestamp: Time.Micro, data: Float32Array[]): void; + read(output: Float32Array[]): number; + readonly length: number; +} + +function shared(latencyMs: number): Ring { + const samples = Math.ceil((RATE * latencyMs) / 1000); + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(RATE, samples * 2), RATE)); + ring.setLatency(samples); + return ring; +} + +function post(latencyMs: number): Ring { + const ring = new AudioRingBuffer({ rate: RATE, channels: 1, latency: latencyMs as Time.Milli }); + return { + insert: (timestamp, data) => ring.write(timestamp, data), + read: (output) => ring.read(output), + get length() { + return ring.length; + }, + }; +} + +interface Result { + /** Quanta rendered short of a full block after the warmup, i.e. audible underruns. */ + underruns: number; + /** Samples inserted after the warmup that were never played: skipped or overflow-dropped. */ + skipped: number; +} + +/** + * Play `t` through `ring` in real time: insert every frame the moment it arrives, and pull one + * render quantum every quantum's worth of wall time, which is what the AudioWorklet does. + */ +function replay(ring: Ring, t: Arrival[], warmupMs: number): Result { + const step = (QUANTUM / RATE) * 1000; + const output = [new Float32Array(QUANTUM)]; + const end = t[t.length - 1].arrival; + + let next = 0; + let played = 0; + let inserted = 0; + let short = 0; + let started = false; + let mark: { played: number; inserted: number; buffered: number; short: number } | undefined; + + for (let now = 0; now < end; now += step) { + while (next < t.length && t[next].arrival <= now) { + ring.insert(Time.Micro.fromMilli(t[next].media as Time.Milli), [new Float32Array(CHUNK).fill(0.5)]); + inserted += CHUNK; + next++; + } + + const count = ring.read(output); + played += count; + if (started && count < QUANTUM) short++; + if (count > 0) started = true; + + if (now >= warmupMs && !mark) mark = { played, inserted, buffered: ring.length, short }; + } + + const from = mark ?? { played, inserted, buffered: ring.length, short }; + return { + underruns: short - from.short, + skipped: inserted - from.inserted - (played - from.played) - (ring.length - from.buffered), + }; +} + +const RINGS: Array<[string, (latencyMs: number) => Ring]> = [ + ["shared", shared], + ["post", post], +]; + +describe.each(RINGS)("%s ring replay", (_name, build) => { + it("plays an evenly paced sender without underruns or skips", async () => { + const t = trace(600, 1, 10); + expect(replay(build(await settled(t)), t, 2000)).toEqual({ underruns: 0, skipped: 0 }); + }); + + it("plays a bursty sender at the measured target", async () => { + // Three frames flushed at once, which is what an importer packing PES payloads produces. + const t = trace(600, 3, 5); + const measured = await settled(t); + expect(measured).toBeGreaterThan(2 * CHUNK_MS); + expect(replay(build(measured), t, 2000)).toEqual({ underruns: 0, skipped: 0 }); + }); + + it("keeps a five frame flush span playing", async () => { + // A hundred milliseconds of arrivals landing at once. The target covers the trough and the + // slack above it absorbs the peak, so the flush plays instead of being cut down to the + // target on arrival. + const t = trace(600, 5, 5); + expect(replay(build(await settled(t)), t, 2000)).toEqual({ underruns: 0, skipped: 0 }); + }); + + it("underruns constantly when the target ignores the arrival spread", () => { + // 46ms is what the round-trip formula produced on the connection this was measured on: it + // describes the network and says nothing about a sender that flushes five frames at once. + const t = trace(600, 5, 5); + const result = replay(build(46), t, 2000); + expect(result.underruns).toBeGreaterThan(100); + expect(result.skipped).toBeGreaterThan(10 * CHUNK); + }); +}); + +describe("subscription budget", () => { + // A minute of evenly paced audio, long enough for the startup ramp to hand over to the steady + // forget factor and the target to settle at its floor, then six seconds of 100ms flushes. + const PACED = 3000; + const t: Arrival[] = []; + for (let i = 0; i < PACED; i++) t.push({ media: i * CHUNK_MS, arrival: i * CHUNK_MS + 50 }); + t.push( + ...trace(300, 6, 0).map(({ media, arrival }) => ({ + media: media + PACED * CHUNK_MS, + arrival: arrival + PACED * CHUNK_MS, + })), + ); + + // The targets across the first twenty flushes, 2.4s of them. + function onset(targets: Time.Milli[]): Time.Milli[] { + return targets.slice(PACED, PACED + 20); + } + + it("reaches a flush that starts after the target settled", async () => { + const { targets, delivered } = await measure(t, (target) => Time.Milli.max(target, AUTO_MAX_AGE)); + expect(targets[PACED - 1]).toBe((2 * CHUNK_MS) as Time.Milli); + expect(delivered).toBe(t.length); + // Every flush lands whole, so its 100ms reaches the 95th percentile within a few intervals. + expect(Math.max(...onset(targets))).toBeGreaterThanOrEqual(120); + }); + + it("drops the flush it should measure when the budget follows the target", async () => { + // The transport skips a group older than the subscription's max age, before the consumer + // can observe it, so the estimate only ever sees what the target it already holds allows. + const { targets, delivered } = await measure(t, (target) => target); + expect(delivered).toBeLessThan(t.length); + expect(Math.max(...onset(targets))).toBeLessThan(120); + }); +}); diff --git a/js/watch/src/audio/render-worklet.ts b/js/watch/src/audio/render-worklet.ts index 723dfcfa99..51bf02ad02 100644 --- a/js/watch/src/audio/render-worklet.ts +++ b/js/watch/src/audio/render-worklet.ts @@ -2,11 +2,17 @@ import type { Message, State } from "./render"; import { AudioRingBuffer } from "./ring-buffer"; import { SharedRingBuffer } from "./shared-ring-buffer"; +// Samples to fade over when playback stops or resumes short of a full quantum. A shortfall +// otherwise steps straight to silence and back, which is an audible click on every underrun. +const RAMP = 64; + class Render extends AudioWorkletProcessor { // Set after init, depending on which path the main thread chose. #backend?: SharedRingBuffer | AudioRingBuffer; #underflow = 0; #stateCounter = 0; + // Whether the previous quantum ended short, so the next one fades back in. + #short = false; constructor() { super(); @@ -18,10 +24,12 @@ class Render extends AudioWorkletProcessor { const previous = this.#backend instanceof SharedRingBuffer ? this.#backend : undefined; this.#backend = new SharedRingBuffer(msg, previous); this.#underflow = 0; + this.#short = false; } else if (msg.type === "init-post") { console.log("[audio-worklet] init-post: using postMessage path"); this.#backend = new AudioRingBuffer(msg); this.#underflow = 0; + this.#short = false; } else if (msg.type === "data") { // Only meaningful in post mode. if (this.#backend instanceof AudioRingBuffer) this.#backend.write(msg.timestamp, msg.data); @@ -44,10 +52,30 @@ class Render extends AudioWorkletProcessor { const samplesRead = backend?.read(output) ?? 0; if (samplesRead < output[0].length) { + // Fade the tail of what we did read down to the silence that follows it. + for (const channel of output) { + const ramp = Math.min(RAMP, samplesRead); + for (let i = 0; i < ramp; i++) { + channel[samplesRead - ramp + i] *= 1 - (i + 1) / ramp; + } + } this.#underflow += output[0].length - samplesRead; - } else if (this.#underflow > 0 && backend) { - console.debug(`audio underflow: ${Math.round((1000 * this.#underflow) / backend.rate)}ms`); - this.#underflow = 0; + this.#short = true; + } else { + if (this.#short) { + // Fade back in from the silence the shortfall left behind. + const ramp = Math.min(RAMP, samplesRead); + for (const channel of output) { + for (let i = 0; i < ramp; i++) { + channel[i] *= (i + 1) / ramp; + } + } + this.#short = false; + } + if (this.#underflow > 0 && backend) { + console.debug(`audio underflow: ${Math.round((1000 * this.#underflow) / backend.rate)}ms`); + this.#underflow = 0; + } } // In post mode the main thread can't read worklet state directly, so we @@ -61,6 +89,7 @@ class Render extends AudioWorkletProcessor { type: "state", timestamp: backend.timestamp, stalled: backend.stalled, + underruns: backend.underruns, }; this.port.postMessage(state); } diff --git a/js/watch/src/audio/render.ts b/js/watch/src/audio/render.ts index e0c8252468..19da8f0842 100644 --- a/js/watch/src/audio/render.ts +++ b/js/watch/src/audio/render.ts @@ -53,4 +53,6 @@ export interface State { type: "state"; timestamp: Time.Micro; stalled: boolean; + // How many times the ring has run dry mid-playback, cumulative. + underruns: number; } diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index 3abcc74208..bd98dfe8f2 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -1,9 +1,10 @@ import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; import type * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; -import { type Arrival, type Options, QUANTUM, replay, target } from "./replay"; +import { type Arrival, type Options, QUANTUM, replay } from "./replay"; const RATE = 48000; +const CONFIG = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; /** Twenty ms frames for `seconds`, one per group, each arriving `lateness(i)` ms after it was captured. */ function paced(seconds: number, lateness: (i: number) => number): Arrival[] { @@ -13,13 +14,16 @@ function paced(seconds: number, lateness: (i: number) => number): Arrival[] { /** How long {@link paced} observes `seconds` of frames: the frames, plus one more. */ const observed = (seconds: number) => seconds * 1000 + 20; -/** Quanta that came back short or empty while playing, once the ring had first played. */ +/** + * Quanta that came back short or empty once the ring had first played. A ring that runs dry + * re-stalls until it refills, and that refill is silence too. + */ async function underruns(trace: Arrival[], options: Options): Promise { let started = false; let short = 0; - for await (const { output, stalled } of replay(trace, options)) { + for await (const { output } of replay(trace, options)) { const filled = output.findLastIndex((v) => v !== 0) + 1; - if (started && !stalled && filled < QUANTUM) short++; + if (started && filled < QUANTUM) short++; if (filled > 0) started = true; } return short; @@ -37,7 +41,7 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { expect( await underruns( paced(10, () => 30), - { ring, rate: RATE, delay: 100, duration: observed(10) }, + { ring, rate: RATE, config: CONFIG, delay: Time.Milli(100), duration: observed(10) }, ), ).toBe(0); }); @@ -45,8 +49,18 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { it("underruns when a flush span outlasts the target, and not when it is covered", async () => { // Five frames held and flushed at once: 80 ms of arrival spread. const trace = paced(10, (i) => 30 + (4 - (i % 5)) * 20); - expect(await underruns(trace, { ring, rate: RATE, delay: 40, duration: observed(10) })).toBeGreaterThan(50); - expect(await underruns(trace, { ring, rate: RATE, delay: 150, duration: observed(10) })).toBe(0); + expect( + await underruns(trace, { ring, rate: RATE, config: CONFIG, delay: Time.Milli(40), duration: observed(10) }), + ).toBeGreaterThan(50); + expect( + await underruns(trace, { + ring, + rate: RATE, + config: CONFIG, + delay: Time.Milli(150), + duration: observed(10), + }), + ).toBe(0); }); it("holds newer groups behind a missing one until the max age gives up on it", async () => { @@ -54,7 +68,15 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { // The consumer delivers in group order, so the ring runs dry for longer than the missing frame // while the groups behind it wait. Writing arrivals straight into the ring plays straight through. const frame = Math.ceil((20 / 1000) * (RATE / QUANTUM)); - expect(await underruns(trace, { ring, rate: RATE, delay: 100, duration: observed(10) })).toBeGreaterThan(frame); + expect( + await underruns(trace, { + ring, + rate: RATE, + config: CONFIG, + delay: Time.Milli(100), + duration: observed(10), + }), + ).toBeGreaterThan(frame); expect(warn).toHaveBeenCalled(); }); @@ -63,7 +85,15 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { const trace = paced(10, () => 30) .map((arrival, i) => ({ ...arrival, group: Math.floor(i / 5) })) .filter((_, i) => i !== 252); - expect(await underruns(trace, { ring, rate: RATE, delay: 100, duration: observed(10) })).toBeGreaterThan(0); + expect( + await underruns(trace, { + ring, + rate: RATE, + config: CONFIG, + delay: Time.Milli(100), + duration: observed(10), + }), + ).toBeGreaterThan(0); }); it("renders through the end of the observation, past the last arrival", async () => { @@ -74,7 +104,8 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { { ring, rate: RATE, - delay: 100, + config: CONFIG, + delay: Time.Milli(100), duration: 15_000, }, )) { @@ -86,15 +117,20 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { }); }); -describe("target", () => { - const config = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; - - it("sizes auto from the RTT and the rendition's jitter", async () => { - // 1.25 x 40 ms of RTT, plus the Opus frame and a render quantum. - expect(await target({ delay: "auto", config, rtt: 40 })).toBe(Time.Milli(50 + 20 + 3)); - }); +describe.each(["shared", "post"] as const)("%s ring at auto", (ring) => { + /** The delay the replay resolved once `seconds` of {@link paced} frames played out. */ + async function settled(seconds: number, lateness: (i: number) => number): Promise { + let delay = 0; + const options: Options = { ring, rate: RATE, config: CONFIG, delay: "auto", duration: observed(seconds) }; + for await (const quantum of replay(paced(seconds, lateness), options)) delay = quantum.delay; + return delay; + } - it("adds the rendition's jitter to a fixed delay", async () => { - expect(await target({ delay: Time.Milli(250), config })).toBe(Time.Milli(250 + 20 + 3)); + // A minute, so the estimator's startup ramp hands over to its steady forget factor. + it("sizes the target from the arrival spread, not a round trip", async () => { + // Evenly paced, the target is the estimator's 20 ms floor plus one frame. + expect(await settled(60, () => 30)).toBe(40); + // Five frames flushed at once: 80 ms of spread to cover, plus one frame. + expect(await settled(60, (i) => 30 + (4 - (i % 5)) * 20)).toBeGreaterThanOrEqual(100); }); }); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index 6102d61c10..a184938d2c 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -5,9 +5,10 @@ * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max age the delay * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder * does. One render quantum is read every quantum's worth of that same clock, which is what the - * AudioWorklet does. {@link target} resolves the delay with a real {@link Sync} fed the recorded - * catalog config and connection RTT, and the ring is sized from it the way the decoder sizes it, so - * a change to any of them moves what a replay hears. Decoding is taken as instant and sample exact. + * AudioWorklet does. A real {@link Sync} resolves the delay from the playout target the decoder + * registers, measured by that same consumer from the recorded catalog config, and the ring follows + * it the way the decoder resizes it, so a change to any of them moves what a replay hears. Decoding + * is taken as instant and sample exact. * * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test * can assert exact counts. @@ -17,10 +18,10 @@ import type * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import { Group, Time, Track } from "@moq/net"; -import { Signal } from "@moq/signals"; +import { Effect } from "@moq/signals"; import { type Delay, Sync } from "../sync"; -import { playbackJitter } from "./config"; -import { ringSamples } from "./latency"; +import { frameDuration } from "./config"; +import { ringSamples, target } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; import { Terminal } from "./terminal"; @@ -47,48 +48,16 @@ export interface Arrival { group: number; } -/** What the player resolves its delay from. */ -export interface Target { - /** The configured delay, as the element takes it. "instant" plays no audio, so it has no target. */ - delay: Exclude; - /** The rendition as the catalog described it: its advertised jitter. */ - config: Catalog.AudioConfig; - /** The connection's round trip, in ms, which "auto" sizes from. Unset is no PROBE. */ - rtt?: number; -} - -/** The delay a player resolves for `input`, in ms, from a real {@link Sync}. */ -export async function target(input: Target): Promise { - const sync = new Sync({ - delay: input.delay, - probe: input.rtt === undefined ? undefined : { rtt: Time.Milli(input.rtt) }, - }); - const jitter = playbackJitter(input.config); - const unregister = sync.register(new Signal(jitter)); - try { - // Sync's effects run on microtasks. The delay has settled once it holds this rendition's jitter - // on top of the resolved jitter, the only rendition registered. - while (sync.out.delay.peek() !== Time.Milli.add(sync.out.jitter.peek(), jitter)) { - await sync.out.delay.changed(); - } - return sync.out.delay.peek(); - } finally { - unregister(); - sync.close(); - } -} - /** What a replay plays through. */ export interface Options { /** `shared` is the SharedArrayBuffer ring a cross-origin isolated page runs; `post` is the rest. */ ring: "shared" | "post"; /** The sample rate, in Hz. */ rate: number; - /** - * The resolved delay, in ms. See {@link target}. It sizes the ring and is the consumer's max age, - * which `Sync` sets to the delay plus a lookahead a live viewer does not configure. - */ - delay: number; + /** The configured delay, as the element takes it. "instant" plays no audio, so it has no replay. */ + delay: Exclude; + /** The rendition as the catalog described it: its advertised jitter, delay, and codec. */ + config: Catalog.AudioConfig; /** * How long the trace observed, from its first arrival, in ms. Rendering runs to here, so an * outage after the last arrival is heard rather than cut off. @@ -106,6 +75,8 @@ export interface Quantum { stalled: boolean; /** The playhead after this quantum, on the media clock, in ms. */ timestamp: number; + /** The delay `Sync` resolved as of this quantum, in ms: what sizes the ring. */ + delay: number; } /** The two rings behind the calls the decoder and the worklet make on them. */ @@ -113,25 +84,43 @@ interface Ring { insert(timestamp: Time.Micro, data: Float32Array[]): void; read(output: Float32Array[]): number; reset(): void; + setLatency(samples: number): void; readonly stalled: boolean; readonly timestamp: Time.Micro; } -/** The shared ring, sized the way `SharedAudioBuffer` sizes it. */ +/** The shared ring, sized and grown the way `SharedAudioBuffer` sizes and grows it. */ function shared(rate: number, latency: number): Ring { - const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(rate, latency * 2), rate)); + let ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(rate, latency * 2), rate)); ring.setLatency(latency); - return ring; + return { + insert: (timestamp, data) => ring.insert(timestamp, data), + read: (output) => ring.read(output), + reset: () => ring.reset(), + setLatency: (samples) => { + ring.setLatency(samples); + if (ring.capacity < samples * 1.5) ring = ring.resize(Math.max(rate, samples * 2)); + }, + get stalled() { + return ring.stalled; + }, + get timestamp() { + return ring.timestamp; + }, + }; } +/** Samples as the milliseconds `PostAudioBuffer` posts to the worklet. */ +const millis = (rate: number, samples: number) => Time.Milli.fromSecond((samples / rate) as Time.Second); + /** The postMessage ring, sized the way `PostAudioBuffer` asks the worklet to size it. */ function post(rate: number, latency: number): Ring { - const ms = Time.Milli.fromSecond((latency / rate) as Time.Second); - const ring = new AudioRingBuffer({ rate, channels: 1, latency: ms }); + const ring = new AudioRingBuffer({ rate, channels: 1, latency: millis(rate, latency) }); return { insert: (timestamp, data) => ring.write(timestamp, data), read: (output) => ring.read(output), reset: () => ring.reset(), + setLatency: (samples) => ring.resize(millis(rate, samples)), get stalled() { return ring.stalled; }, @@ -151,7 +140,7 @@ const micros = (ms: number) => Math.round(ms * 1000) as Time.Micro; * frames tile the timeline exactly despite rounding. Any wider spacing is a frame that never * arrived, which stays missing audio rather than stretching the frame before it. */ -function frameSamples(trace: Arrival[], rate: number): Map { +function frameSamples(trace: Arrival[], rate: number): { spans: Map; typical: number } { const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b); const sample = (ms: number) => Math.round((ms * rate) / 1000); const gaps = starts.slice(1).map((next, i) => sample(next) - sample(starts[i])); @@ -164,7 +153,7 @@ function frameSamples(trace: Arrival[], rate: number): Map { const tiles = gap !== undefined && Math.abs(gap - typical) * 2 <= typical; spans.set(micros(start), tiles ? gap : typical); }); - return spans; + return { spans, typical }; } /** @@ -190,9 +179,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const end = first.at + options.duration; if (end < last.at) throw new Error(`an arrival at ${last.at} ms is past the ${options.duration} ms observed`); - const { rate } = options; - const samples = frameSamples(trace, rate); - const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, Time.Milli(options.delay))); + const { rate, config } = options; + const { spans: samples, typical } = frameSamples(trace, rate); // Where each group's stream finishes. const ends = new Map(); @@ -200,12 +188,31 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato ends.set(arrival.group, i); }); + const sync = new Sync({ delay: options.delay }); const track = new Track.Producer("audio"); const consumer = new Container.Consumer(track.subscribe({ maxAge: WIRE_MAX_AGE }), { format: new Container.Legacy.Format("audio"), - maxAge: Time.Milli(options.delay), + maxAge: sync.out.maxAge, }); + // The decoder's "auto" target. A recorded frame has no payload to read an Opus duration from, + // so a codec without a constant one takes the trace's typical spacing. + const signals = new Effect(); + const frame = frameDuration(config) ?? Time.Milli((typical * 1000) / rate); + const playout = signals.computed((effect) => + target({ + measured: effect.get(consumer.spread), + advertised: config.jitter !== undefined ? Time.Milli(config.jitter) : undefined, + frame, + delay: config.delay !== undefined ? Time.Milli(config.delay) : undefined, + }), + ); + signals.cleanup(sync.register(playout)); + await settle(); + + let delay = sync.out.delay.peek(); + const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, delay)); + // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. const terminal = new Terminal(); let failure: { error: unknown } | undefined; @@ -230,13 +237,26 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const output = new Float32Array(QUANTUM); let next = 0; + // The consumer stamps each arrival on the monotonic clock, which the estimator measures, so it + // reads the trace's clock instead until the replay finishes. + let clock = first.at; + const monotonic = performance.now; + performance.now = () => clock; + try { for (let n = 1; first.at + (n - 1) * step <= end; n++) { // The quantum ending at `now` renders once everything that arrived by then is delivered. const now = first.at + n * step; - const written = next; + let stamped = next; while (next < trace.length && trace[next].at <= now) { const arrival = trace[next]; + // The consumer stamps a frame as it reads it, so the clock moves on only once + // everything that arrived at the previous instant has been read. + if (arrival.at !== clock) { + if (next > stamped) await settle(); + stamped = next; + clock = arrival.at; + } let group = groups.get(arrival.group); if (!group) { group = new Group.Producer(arrival.group); @@ -254,16 +274,26 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato } next++; } - if (next > written) await settle(); + if (next > stamped) await settle(); if (failure) throw failure.error; + // The decoder resizes the ring as the resolved delay moves, which only an arrival does. + const resolved = sync.out.delay.peek(); + if (resolved !== delay) { + delay = resolved; + ring.setLatency(ringSamples(rate, delay)); + } + output.fill(0); ring.read([output]); - yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp) }; + yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp), delay }; } } finally { consumer.close(); track.close(); await decode; + signals.close(); + sync.close(); + performance.now = monotonic; } } diff --git a/js/watch/src/audio/ring-buffer.test.ts b/js/watch/src/audio/ring-buffer.test.ts index a53d2111e1..b91022e623 100644 --- a/js/watch/src/audio/ring-buffer.test.ts +++ b/js/watch/src/audio/ring-buffer.test.ts @@ -35,7 +35,9 @@ describe("initialization", () => { it("should initialize with valid parameters", () => { const buffer = new AudioRingBuffer({ rate: 48000, channels: 2, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(4800); // 48000 * 0.1 + // Capacity is twice the target: the ring has to be able to hold the slack the overflow band + // tolerates, or every chunk arriving on a full ring would drop the oldest samples. + expect(buffer.capacity).toBe(9600); // 48000 * 0.1 * 2 expect(buffer.length).toBe(0); }); @@ -281,9 +283,10 @@ describe("stall behavior", () => { describe("ring buffer wrapping", () => { it("should wrap around when buffer is full", () => { + // 100 sample target, so 200 samples of capacity. const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - // Fill the buffer + // Fill to the target write(buffer, 0 as Time.Milli, 100, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); @@ -297,22 +300,23 @@ describe("ring buffer wrapping", () => { // Now we have 100 samples available (50-149) expect(buffer.length).toBe(100); - // Write 50 more samples at timestamp 150, this will wrap around + // One chunk above the target is tolerated rather than dropped (50-199). write(buffer, 150 as Time.Milli, 50, { channels: 1, value: 3.0 }); + expect(buffer.length).toBe(150); - // Should still have 100 samples (buffer is at capacity) + // Past the band: drop back to the target, wrapping abs 200-249 onto slots 0-49. + write(buffer, 200 as Time.Milli, 50, { channels: 1, value: 4.0 }); expect(buffer.length).toBe(100); - // Read all 100 samples + // Read all 100 samples: abs 150-199 = 3.0, abs 200-249 = 4.0. const output2 = read(buffer, 100, 1); expect(output2[0].length).toBe(100); - // First 50 should be 2.0, next 50 should be 3.0 for (let i = 0; i < 50; i++) { - expect(output2[0][i]).toBe(2.0); + expect(output2[0][i]).toBe(3.0); } for (let i = 50; i < 100; i++) { - expect(output2[0][i]).toBe(3.0); + expect(output2[0][i]).toBe(4.0); } }); }); @@ -390,7 +394,7 @@ describe("edge cases", () => { describe("resize", () => { it("should resize to a larger buffer", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); // Write 50 samples write(buffer, 0 as Time.Milli, 50, { channels: 1, value: 1.0 }); @@ -399,26 +403,26 @@ describe("resize", () => { // Resize to larger buffer (200ms = 200 samples) buffer.resize(200 as Time.Milli); - expect(buffer.capacity).toBe(200); + expect(buffer.capacity).toBe(400); expect(buffer.length).toBe(50); // Samples preserved expect(buffer.stalled).toBe(true); // Should trigger stall }); it("should resize to a smaller buffer and keep the most recent samples", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); // Write 80 samples: first 40 with value 1.0, next 40 with value 2.0 write(buffer, 0 as Time.Milli, 40, { channels: 1, value: 1.0 }); write(buffer, 40 as Time.Milli, 40, { channels: 1, value: 2.0 }); expect(buffer.length).toBe(80); - // Resize to smaller buffer (50ms = 50 samples) - // Should keep the most recent 50 samples (samples 30-79) - buffer.resize(50 as Time.Milli); + // Resize to a 30 sample target (60 samples of capacity). + // Should keep the most recent 60 samples (samples 20-79) + buffer.resize(30 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); // Truncated to new capacity + expect(buffer.capacity).toBe(60); + expect(buffer.length).toBe(60); // Truncated to new capacity expect(buffer.stalled).toBe(false); // Already covers the new floor }); @@ -434,7 +438,7 @@ describe("resize", () => { // Should still not be stalled (no-op) expect(buffer.stalled).toBe(false); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); }); it("should throw on zero latency", () => { @@ -456,8 +460,8 @@ describe("resize", () => { // Resize to smaller buffer buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(60); expect(buffer.stalled).toBe(false); }); @@ -468,7 +472,7 @@ describe("resize", () => { // Resize empty buffer buffer.resize(200 as Time.Milli); - expect(buffer.capacity).toBe(200); + expect(buffer.capacity).toBe(400); expect(buffer.length).toBe(0); expect(buffer.stalled).toBe(true); }); @@ -488,10 +492,10 @@ describe("resize", () => { write(buffer, 100 as Time.Milli, 30, { channels: 1, value: 2.0 }); expect(buffer.length).toBe(70); // 130 - 60 - // Resize to 50 samples - should keep most recent 50 (samples 80-129) + // Resize to a 50 sample target - 100 of capacity, so all 70 survive (samples 60-129) buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(70); expect(buffer.stalled).toBe(false); }); @@ -505,21 +509,26 @@ describe("resize", () => { buffer.resize(50 as Time.Milli); expect(buffer.stalled).toBe(false); - // The overflow will discard preserved samples and advance readIndex + // Nothing is dropped: the ring has room for the preserved samples plus a chunk above the + // target. write(buffer, 50 as Time.Milli, 50, { channels: 1, value: 2.0 }); expect(buffer.stalled).toBe(false); + expect(buffer.length).toBe(100); - // Read should return the new data (value 2.0) - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); + // Read plays the preserved samples first, then the new ones. + const output = read(buffer, 100, 1); + expect(output[0].length).toBe(100); for (let i = 0; i < 50; i++) { + expect(output[0][i]).toBe(1.0); + } + for (let i = 50; i < 100; i++) { expect(output[0][i]).toBe(2.0); } }); it("should preserve samples correctly when resizing larger then filling", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 50 as Time.Milli }); - expect(buffer.capacity).toBe(50); + expect(buffer.capacity).toBe(100); // Write 30 samples write(buffer, 0 as Time.Milli, 30, { channels: 1, value: 1.0 }); @@ -527,7 +536,7 @@ describe("resize", () => { // Resize to larger buffer (100ms = 100 samples) buffer.resize(100 as Time.Milli); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); expect(buffer.length).toBe(30); // All samples preserved expect(buffer.stalled).toBe(true); @@ -568,26 +577,26 @@ describe("resize", () => { // abs 110-119 → slots 10-19, value 3.0. write(buffer, 110 as Time.Milli, 10, { channels: 1, value: 3.0 }); - // Preserved range after resize: most-recent 50 samples = abs 70..119. - // copyStart = 120 - 50 = 70, and 70 % 50 = 20 (non-zero → triggers the bug). + // Preserved range after resize: all 100 buffered samples = abs 20..119. + // copyStart = 120 - 100 = 20, and 20 % 100 = 20 (non-zero → triggers the bug). buffer.resize(50 as Time.Milli); - expect(buffer.capacity).toBe(50); - expect(buffer.length).toBe(50); + expect(buffer.capacity).toBe(100); + expect(buffer.length).toBe(100); expect(buffer.stalled).toBe(false); // Read the preserved samples. Expected layout by absolute index: - // abs 70..99 = 1.0 (30 samples, from the initial fill) + // abs 20..99 = 1.0 (80 samples, from the initial fill) // abs 100..109 = 2.0 (10 samples) // abs 110..119 = 3.0 (10 samples) - const output = read(buffer, 50, 1); - expect(output[0].length).toBe(50); - for (let i = 0; i < 30; i++) { + const output = read(buffer, 100, 1); + expect(output[0].length).toBe(100); + for (let i = 0; i < 80; i++) { expect(output[0][i]).toBe(1.0); } - for (let i = 30; i < 40; i++) { + for (let i = 80; i < 90; i++) { expect(output[0][i]).toBe(2.0); } - for (let i = 40; i < 50; i++) { + for (let i = 90; i < 100; i++) { expect(output[0][i]).toBe(3.0); } }); @@ -606,16 +615,16 @@ describe("resize", () => { // Wrap the writer over slots 0-29 with value 2.0 (abs 100-129). write(buffer, 100 as Time.Milli, 30, { channels: 1, value: 2.0 }); - // Resize smaller. Preserved = last 50 samples = abs 80..129. + // Resize smaller. All 90 buffered samples fit the new 100 sample capacity: abs 40..129. buffer.resize(50 as Time.Milli); - expect(buffer.length).toBe(50); + expect(buffer.length).toBe(90); expect(buffer.stalled).toBe(false); // Write the next 10 samples at their real timestamp. This should append, // not create a gap or be discarded as "too old". write(buffer, 130 as Time.Milli, 10, { channels: 1, value: 3.0 }); - // Buffer capacity is 50, so the oldest 10 samples (abs 80..89) drop out. + // That put the ring past the band, so it drops back to the 50 sample target. // Remaining: abs 90..99 = 1.0 (10), abs 100..129 = 2.0 (30), abs 130..139 = 3.0 (10). expect(buffer.length).toBe(50); @@ -666,7 +675,7 @@ describe("live anchoring", () => { // with zeros, un-stalled on that overflow, and left the playhead a floor before the frame. it("anchors a live ring to the first frame instead of gap-filling from zero", () => { const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 100 as Time.Milli }); - expect(buffer.capacity).toBe(100); + expect(buffer.capacity).toBe(200); write(buffer, 2000 as Time.Milli, 40, { channels: 1, value: 0.5 }); // Still short of the floor, so nothing plays yet and the ring holds only real samples. @@ -684,6 +693,63 @@ describe("live anchoring", () => { }); }); +describe("re-buffer", () => { + it("re-stalls and refills to the target after running dry", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + // Drain it. + expect(read(buffer, 40, 1)[0].length).toBe(40); + expect(buffer.length).toBe(0); + + // The next read finds nothing and parks playback rather than handing the worklet a quantum + // of silence and trying again on the next one. + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.stalled).toBe(true); + expect(buffer.underruns).toBe(1); + + // A single chunk is not the target, so playback stays parked. + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + + // Reaching the target resumes it, and nothing buffered in the meantime was lost. + write(buffer, 60 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 40, 1)[0].length).toBe(40); + }); + + it("counts a quantum it could only partly fill as an underrun", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + + write(buffer, 0 as Time.Milli, 20, { channels: 1, value: 1.0 }); + write(buffer, 20 as Time.Milli, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 30, 1)[0].length).toBe(30); + + // Ten samples remain against a quantum of twenty: the rest of the quantum is silence, which + // is an underrun whether or not a chunk lands before the next read. It is not a stall: a + // refill would cost the whole target to cover a gap shorter than one quantum. + expect(read(buffer, 20, 1)[0].length).toBe(10); + expect(buffer.stalled).toBe(false); + expect(buffer.underruns).toBe(1); + + // A chunk landing before the next read keeps playback going. + write(buffer, 40 as Time.Milli, 20, { channels: 1, value: 2.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + expect(buffer.underruns).toBe(1); + }); + + it("does not count an underrun while parked", () => { + const buffer = new AudioRingBuffer({ rate: 1000, channels: 1, latency: 40 as Time.Milli }); + read(buffer, 20, 1); + read(buffer, 20, 1); + expect(buffer.underruns).toBe(0); + }); +}); + describe("buffered mode", () => { function createBuffered(latency: number) { return new AudioRingBuffer({ rate: 1000, channels: 1, latency: latency as Time.Milli, buffered: true }); diff --git a/js/watch/src/audio/ring-buffer.ts b/js/watch/src/audio/ring-buffer.ts index 38387a13a3..3034ced5e5 100644 --- a/js/watch/src/audio/ring-buffer.ts +++ b/js/watch/src/audio/ring-buffer.ts @@ -8,6 +8,13 @@ export class AudioRingBuffer { readonly rate: number; readonly channels: number; #stalled = true; + #underruns = 0; + + // Samples in the most recent write, i.e. one decoded chunk. The overflow band tolerates this + // much above the target before dropping, so a ring sitting exactly on the target doesn't drop + // audio the moment the next chunk lands. The most recent write rather than a running maximum: + // one oversized decode would otherwise widen the band for the life of the ring. + #chunk = 0; // Buffered mode: play through everything buffered without skipping ahead. readonly #buffered: boolean; @@ -33,9 +40,11 @@ export class AudioRingBuffer { this.channels = props.channels; this.#buffered = props.buffered ?? false; - // The ring holds the latency floor as PCM. Buffered mode gives it headroom above the floor - // so the backpressure-paced decode loop (on the main thread) doesn't overflow-drop; the rest - // of the lookahead stays encoded upstream. + // The ring holds the latency floor as PCM, with headroom above it. Sizing capacity to the + // floor exactly makes the ring physically incapable of holding the slack the overflow band + // wants, so every chunk that arrives while the ring is on target drops the oldest samples. + // In buffered mode the headroom is also what keeps the backpressure-paced decode loop (on + // the main thread) from overflow-dropping; the rest of the lookahead stays encoded upstream. const capacity = this.#capacityFor(this.#latencySamples); this.#buffer = []; @@ -45,13 +54,18 @@ export class AudioRingBuffer { } #capacityFor(latencySamples: number): number { - return this.#buffered ? latencySamples * 2 : latencySamples; + return latencySamples * 2; } get stalled(): boolean { return this.#stalled; } + /** How many times the reader has run dry mid-playback. */ + get underruns(): number { + return this.#underruns; + } + get timestamp(): Time.Micro { return Time.Micro.fromSecond((this.#readIndex / this.rate) as Time.Second); } @@ -141,13 +155,18 @@ export class AudioRingBuffer { } const end = start + samples; - - // Check if we need to discard old samples to prevent overflow - const overflow = end - this.#readIndex - this.#buffer[0].length; - if (overflow >= 0) { - // Discard old samples and exit stalled mode - this.#stalled = false; - this.#readIndex += overflow; + this.#chunk = data[0].length; + + // Bound the ring. While playing, drop the oldest once it holds a whole chunk more than the + // target and land back on the target: frames arrive one chunk at a time, so a ring sitting + // exactly on the target is a chunk above it the moment the next one lands, and dropping on + // that overshoot discards audio on every single write. While stalled the reader is not + // consuming, so only the hard capacity applies; the band would throw away the very audio the + // refill is accumulating. Buffered mode plays through everything, so it is capacity-bound too. + const playing = !this.#stalled && !this.#buffered; + const limit = playing ? Math.min(this.#latencySamples + this.#chunk, this.capacity) : this.capacity; + if (end - this.#readIndex > limit) { + this.#readIndex = end - (playing ? Math.min(this.#latencySamples, this.capacity) : this.capacity); } // Fill gaps with zeros if there's a discontinuity @@ -185,9 +204,10 @@ export class AudioRingBuffer { this.#writeIndex = end; } - // Start playback once we've buffered the latency target. In buffered mode the cap - // is large, so we usually un-stall here rather than via the overflow path above. - if (this.#buffered && this.length >= this.#latencySamples) { + // Start playback once we've buffered the latency target. This is the only way out of a + // stall, so an underrun mid-playback refills to the target before resuming rather than + // playing the next chunk on an empty cushion. + if (this.length >= this.#latencySamples) { this.#stalled = false; } } @@ -218,7 +238,12 @@ export class AudioRingBuffer { if (this.#stalled) return 0; const samples = Math.min(this.#writeIndex - this.#readIndex, output[0].length); - if (samples === 0) return 0; + if (samples <= 0) { + // Ran dry mid-playback: re-stall so write() refills to the target before resuming. + this.#stalled = true; + this.#underruns++; + return 0; + } for (let channel = 0; channel < this.channels; channel++) { const dst = output[channel]; @@ -233,6 +258,12 @@ export class AudioRingBuffer { } this.#readIndex += samples; + + // A short quantum ends in silence, so it counts as an underrun even if a chunk lands before + // the next read. It does not park playback: the shortfall is under one quantum, and a + // refill would spend the whole target as silence to cover it. + if (samples < output[0].length) this.#underruns++; + return samples; } } diff --git a/js/watch/src/audio/shared-ring-buffer.test.ts b/js/watch/src/audio/shared-ring-buffer.test.ts index 72e3a621a4..972f15090e 100644 --- a/js/watch/src/audio/shared-ring-buffer.test.ts +++ b/js/watch/src/audio/shared-ring-buffer.test.ts @@ -32,6 +32,24 @@ function insert( buffer.insert(Time.Micro.fromMilli(timestampMs as Time.Milli), data); } +/** + * Insert `samples` as a run of `chunk`-sized inserts starting at `timestampMs`. + * + * The skip band is one chunk wide, so a test that wants a skip has to arrive the way a decoder + * does: many small chunks, not one big one. The rate is 1000 in these tests, so a sample is a ms. + */ +function insertChunks( + buffer: SharedRingBuffer, + timestampMs: number, + samples: number, + chunk: number, + opts?: { channels?: number; value?: number }, +): void { + for (let i = 0; i < samples; i += chunk) { + insert(buffer, timestampMs + i, Math.min(chunk, samples - i), opts); + } +} + function read(buffer: SharedRingBuffer, samples: number, channelCount?: number): Float32Array[] { const ch = channelCount ?? buffer.channels; const output: Float32Array[] = []; @@ -52,7 +70,7 @@ describe("initialization", () => { expect(init.capacity).toBe(128); expect(init.rate).toBe(1000); expect(init.samples.byteLength).toBe(2 * 128 * 4); // 2 channels * 128 samples * Float32 - expect(init.control.byteLength).toBe(4 * 4); // 4 control slots * Int32 + expect(init.control.byteLength).toBe(6 * 4); // 6 control slots * Int32 expect(init.state.byteLength).toBe(8); // packed epoch + read cursor }); @@ -407,11 +425,11 @@ describe("overflow", () => { }); describe("latency skip", () => { - it("should skip READ when buffered exceeds LATENCY", () => { + it("should skip READ when buffered exceeds LATENCY plus a chunk", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 20 }); - // Fill 60 samples — exceeds LATENCY of 20 - insert(buffer, 0, 60, { channels: 1, value: 1.0 }); + // Fill 60 samples in 10-sample chunks — a whole chunk past the 20 sample LATENCY. + insertChunks(buffer, 0, 60, 10, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); // Read should skip ahead to maintain LATENCY distance from WRITE @@ -421,6 +439,18 @@ describe("latency skip", () => { expect(output[0].length).toBe(20); }); + it("should tolerate one chunk above LATENCY", () => { + // A ring sitting on the target is a chunk above it the moment the next chunk lands, so + // skipping on that overshoot would discard audio on every single insert. + const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 20 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + const output = read(buffer, 128, 1); + expect(output[0].length).toBe(40); + }); + it("should not skip when buffered is within LATENCY", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); @@ -458,8 +488,8 @@ describe("setLatency", () => { it("should dynamically change latency affecting skip behavior", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); - // Fill 80 samples - insert(buffer, 0, 80, { channels: 1, value: 1.0 }); + // Fill 80 samples, in chunks so the skip band stays narrow. + insertChunks(buffer, 0, 80, 10, { channels: 1, value: 1.0 }); expect(buffer.stalled).toBe(false); // With LATENCY=50, reading should skip to 30 (80-50) @@ -467,7 +497,7 @@ describe("setLatency", () => { expect(output1[0].length).toBe(50); // Write more - insert(buffer, 80, 80, { channels: 1, value: 2.0 }); + insertChunks(buffer, 80, 80, 10, { channels: 1, value: 2.0 }); // Change latency to 20 buffer.setLatency(20); @@ -557,8 +587,8 @@ describe("setLatency", () => { buffer.setLatency(110); expect(buffer.stalled).toBe(false); - // The skip lands on the new floor: the last 110 samples. - expect(read(buffer, 1, 1)[0][0]).toBe(2.0); + // 150 buffered sits within a chunk of the new floor, so nothing is skipped either. + expect(read(buffer, 1, 1)[0][0]).toBe(1.0); }); it("resumes a refill once a shallower floor is already covered", () => { @@ -576,6 +606,61 @@ describe("setLatency", () => { }); }); +describe("re-buffer", () => { + it("re-stalls and refills to the target after running dry", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(buffer.stalled).toBe(false); + + // Drain it. + expect(read(buffer, 40, 1)[0].length).toBe(40); + expect(buffer.length).toBe(0); + + // The next read finds nothing and parks playback rather than handing the worklet a quantum + // of silence and trying again on the next one. + expect(read(buffer, 20, 1)[0].length).toBe(0); + expect(buffer.stalled).toBe(true); + expect(buffer.underruns).toBe(1); + + // A single chunk is not the target, so playback stays parked. + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(true); + expect(read(buffer, 20, 1)[0].length).toBe(0); + + // Reaching the target resumes it, and nothing buffered in the meantime was lost. + insert(buffer, 60, 20, { channels: 1, value: 2.0 }); + expect(buffer.stalled).toBe(false); + expect(read(buffer, 40, 1)[0].length).toBe(40); + }); + + it("counts a quantum it could only partly fill as an underrun", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + + insertChunks(buffer, 0, 40, 20, { channels: 1, value: 1.0 }); + expect(read(buffer, 30, 1)[0].length).toBe(30); + + // Ten samples remain against a quantum of twenty: the rest of the quantum is silence, which + // is an underrun whether or not a chunk lands before the next read. It is not a stall: a + // refill would cost the whole target to cover a gap shorter than one quantum. + expect(read(buffer, 20, 1)[0].length).toBe(10); + expect(buffer.stalled).toBe(false); + expect(buffer.underruns).toBe(1); + + // A chunk landing before the next read keeps playback going. + insert(buffer, 40, 20, { channels: 1, value: 2.0 }); + expect(read(buffer, 20, 1)[0].length).toBe(20); + expect(buffer.underruns).toBe(1); + }); + + it("does not count an underrun while parked", () => { + const buffer = create({ rate: 1000, channels: 1, capacity: 256, latency: 40 }); + read(buffer, 20, 1); + read(buffer, 20, 1); + expect(buffer.underruns).toBe(0); + }); +}); + describe("stalled getter", () => { it("should reflect STALLED flag", () => { const buffer = create({ rate: 1000, channels: 1, capacity: 100, latency: 50 }); diff --git a/js/watch/src/audio/shared-ring-buffer.ts b/js/watch/src/audio/shared-ring-buffer.ts index f4c0ba2a4f..b6640fdfd1 100644 --- a/js/watch/src/audio/shared-ring-buffer.ts +++ b/js/watch/src/audio/shared-ring-buffer.ts @@ -3,10 +3,24 @@ import { Time } from "@moq/net"; // Control array slot indices. The playhead is not here: see `state`. const WRITE = 0; const LATENCY = 1; +/** + * Whether playback is held while the ring refills. The one slot both threads write. + * + * The writer clears it once the ring holds the target, the reader raises it when the ring runs + * dry. Neither needs a compare-exchange. The reader only raises it having observed an empty ring, + * which means it already drained everything the writer published, and the writer only clears it + * having published enough to cover the target. A raise landing after a clear costs one quantum of + * silence and the next insert undoes it, since every insert re-checks. A clear landing after a + * raise is correct on its face: the samples are there. + */ const STALLED = 2; // Timeline identity changes only on re-anchor, independently of the packed mutation epoch. const TIMELINE = 3; -const CONTROL_SLOTS = 4; +// The size of the most recent insert, i.e. one decoded chunk. See `read`. +const CHUNK = 4; +// How many times the reader ran dry mid-playback. Diagnostics only. +const UNDERRUN = 5; +const CONTROL_SLOTS = 6; /** * The playhead and its mutation epoch, packed into one 64-bit word: epoch in the high @@ -297,6 +311,15 @@ export class SharedRingBuffer { } } + // Publish the chunk size for the reader's skip band before the samples it describes become + // visible. The two stores are not one transaction, so the reader can land between them: + // this way it sees the wider band with the old cursor, which only makes it skip less, where + // the other order would have it measure a longer span against a narrower band and cut audio + // that is not actually late. The most recent insert, not a running maximum: one oversized + // decode would otherwise widen the band for the life of the ring, and a publisher that + // changes its frame duration is described within one insert. + Atomics.store(this.#control, CHUNK, originalLength); + // Advance WRITE (only forward) Atomics.store(this.#control, WRITE, i32Max(Atomics.load(this.#control, WRITE), end)); @@ -333,17 +356,29 @@ export class SharedRingBuffer { const write = Atomics.load(this.#control, WRITE); const latency = Atomics.load(this.#control, LATENCY); - // Latency skip: if buffered data exceeds LATENCY, skip ahead. + // Latency skip: skip ahead only once the ring holds a whole chunk more than the target, + // landing back on the target. Frames arrive one chunk at a time, so a ring sitting exactly + // on the target is a chunk above it the moment the next one lands; skipping on that + // overshoot discards audio on every single insert. The chunk of slack is the difference + // between tolerating normal arrival and cutting it. // CAS ensures we never step backward relative to a concurrent writer advance. // Disabled in buffered mode, where we deliberately play through the whole buffer. const buffered = (write - read) | 0; - if (!this.buffered && latency > 0 && buffered > latency) { + const slack = Atomics.load(this.#control, CHUNK); + if (!this.buffered && latency > 0 && buffered > ((latency + slack) | 0)) { const skipTo = (write - latency) | 0; if (((skipTo - read) | 0) > 0) read = skipTo; } const available = (write - read) | 0; const count = Math.min(available, output[0].length); + if (available <= 0) { + // Ran dry mid-playback. Re-stall so `insert` refills to the target before playback + // resumes: the alternative is playing the next chunk on an empty cushion, which + // underruns again on the very next quantum. See STALLED for the write discipline. + Atomics.store(this.#control, STALLED, 1); + Atomics.add(this.#control, UNDERRUN, 1); + } if (count <= 0) { // A latency skip still has to be published, and still only if nothing moved. if (((read - readOf(state)) | 0) > 0) { @@ -372,6 +407,11 @@ export class SharedRingBuffer { return 0; } + // A short quantum ends in silence, so it counts as an underrun even if a chunk lands before + // the next read. It does not park playback: the shortfall is under one quantum, and a + // refill would spend the whole target as silence to cover it. + if (count < output[0].length) Atomics.add(this.#control, UNDERRUN, 1); + return count; } @@ -481,6 +521,8 @@ export class SharedRingBuffer { Atomics.store(dst.#control, WRITE, write); Atomics.store(dst.#control, LATENCY, latency); Atomics.store(dst.#control, STALLED, stalled); + Atomics.store(dst.#control, CHUNK, Atomics.load(this.#control, CHUNK)); + Atomics.store(dst.#control, UNDERRUN, Atomics.load(this.#control, UNDERRUN)); // Carry the unwrapped playhead over, rebased onto dst's READ. Fold the same `read` // snapshot the copy used so both sides agree on one observation; `copyStart` is at or @@ -521,6 +563,11 @@ export class SharedRingBuffer { return Atomics.load(this.#control, STALLED) === 1; } + /** How many times the reader has run dry mid-playback. */ + get underruns(): number { + return Atomics.load(this.#control, UNDERRUN); + } + /** * Number of buffered samples (WRITE - READ). * diff --git a/js/watch/src/audio/source.ts b/js/watch/src/audio/source.ts index 8503e10a74..4791946583 100644 --- a/js/watch/src/audio/source.ts +++ b/js/watch/src/audio/source.ts @@ -1,8 +1,7 @@ import type * as Catalog from "@moq/hang/catalog"; -import type * as Moq from "@moq/net"; +import * as Moq from "@moq/net"; import { Effect, type Getter, getter, type Inputs, type Readonlys, readonlys, Signal } from "@moq/signals"; import type { Broadcast } from "../broadcast"; -import { playbackJitter } from "./config"; export type Target = { // Optional manual override for the selected rendition name. @@ -33,7 +32,7 @@ type SourceOutput = { track: Signal; config: Signal; - // The per-rendition jitter (ms) to add to the sync buffer. Wired into Sync by the parent. + // The jitter the selected rendition advertises (ms): a floor on the decoder's playout target. jitter: Signal; }; @@ -124,7 +123,8 @@ export class Source { effect.set(this.#out.track, selected.track); effect.set(this.#out.config, selected.config); - effect.set(this.#out.jitter, playbackJitter(selected.config)); + const jitter = selected.config.jitter; + effect.set(this.#out.jitter, jitter !== undefined ? Moq.Time.Milli(jitter) : undefined); } /** diff --git a/js/watch/src/element.ts b/js/watch/src/element.ts index f9d1eb2657..17aef0a2c9 100644 --- a/js/watch/src/element.ts +++ b/js/watch/src/element.ts @@ -457,7 +457,7 @@ export default class MoqWatch extends HTMLElement { /** * How far playback trails the live edge, in milliseconds. See {@link Delay}. * - * `"auto"` (the default) sizes the jitter buffer from the connection RTT. `"instant"` drops the + * `"auto"` (the default) sizes the jitter buffer from how late frames arrive. `"instant"` drops the * clock instead: video paints the moment it decodes and audio is disabled. */ get delay(): Delay { diff --git a/js/watch/src/player.ts b/js/watch/src/player.ts index 997f04c819..45979dfa7a 100644 --- a/js/watch/src/player.ts +++ b/js/watch/src/player.ts @@ -22,7 +22,7 @@ export type PlayerInput = { catalogFormat: Getter; /** Catalog supplied when the format is `manual`. */ catalog: Getter; - /** Connection probe used for rendition selection and delay. */ + /** Connection probe used for rendition selection. */ probe: Getter; /** Canvas to paint video into. */ canvas: Getter; @@ -117,7 +117,7 @@ export class Player { this.text = new Text.Source({ broadcast: this.broadcast, target: this.in.captions }); this.#signals.cleanup(() => this.text.close()); - this.sync = new Sync({ delay: this.in.delay, buffer: this.in.buffer, probe: this.in.probe }); + this.sync = new Sync({ delay: this.in.delay, buffer: this.in.buffer }); this.#signals.cleanup(() => this.sync.close()); this.video = new Video.Decoder({ source: videoSource, sync: this.sync, enabled: this.#videoEnabled }); diff --git a/js/watch/src/sync.test.ts b/js/watch/src/sync.test.ts index c095fdcdc3..0f29d2b899 100644 --- a/js/watch/src/sync.test.ts +++ b/js/watch/src/sync.test.ts @@ -1,10 +1,7 @@ import { describe, expect, it } from "bun:test"; -import type * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; import { Signal } from "@moq/signals"; -import { playbackJitter } from "./audio/config"; import { type Delay, Sync } from "./sync"; -import { renditionJitter } from "./video/playhead"; // Effects in @moq/signals flush on a microtask, so let pending updates drain before asserting. const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); @@ -66,54 +63,60 @@ describe("delay and buffer", () => { expect(sync.out.maxAge.peek()).toBe(0 as Time.Milli); sync.close(); }); +}); - it("includes registered decoder jitter until the decoder unregisters", async () => { - const media = new Signal(20 as Time.Milli); - const sync = new Sync({ delay: 100 as Time.Milli }); - const unregister = sync.register(media); - await flush(); - expect(sync.out.delay.peek()).toBe(120 as Time.Milli); - - media.set(80 as Time.Milli); - await flush(); - expect(sync.out.delay.peek()).toBe(180 as Time.Milli); - - unregister(); +describe("auto delay", () => { + it("holds nothing until a decoder registers", async () => { + const sync = new Sync(); await flush(); - expect(sync.out.delay.peek()).toBe(100 as Time.Milli); + expect(sync.out.delay.peek()).toBe(0 as Time.Milli); sync.close(); }); - it("holds the largest delay plus jitter among subscribed renditions", async () => { - const audio = { codec: "opus", container: { kind: "legacy" }, sampleRate: 48000, numberOfChannels: 2 }; - const video = { codec: "avc1.640028", container: { kind: "legacy" }, jitter: 34, delay: 200 }; - const sync = new Sync({ delay: 100 as Time.Milli }); - sync.register(new Signal(playbackJitter(audio as Catalog.AudioConfig))); - const unsubscribe = sync.register(new Signal(renditionJitter(video as Catalog.VideoConfig))); + it("follows the deepest registered target until it unregisters", async () => { + const audio = new Signal(120 as Time.Milli); + const video = new Signal(40 as Time.Milli); + const sync = new Sync(); + const unregisterAudio = sync.register(audio); + sync.register(video); await flush(); - // 100 ms of network jitter over the video's 200 ms delay and 34 ms spread. - expect(sync.out.delay.peek()).toBe(334 as Time.Milli); + expect(sync.out.delay.peek()).toBe(120 as Time.Milli); + expect(sync.out.jitter.peek()).toBe(120 as Time.Milli); + + // A publisher flushing 250ms at once needs 250ms of buffer, whatever the round trip is. + audio.set(270 as Time.Milli); + await flush(); + expect(sync.out.delay.peek()).toBe(270 as Time.Milli); - // Dropping the slow rendition lowers latency to the audio's own 20 ms frame and 3 ms quantum. - unsubscribe(); + unregisterAudio(); await flush(); - expect(sync.out.delay.peek()).toBe(123 as Time.Milli); + expect(sync.out.delay.peek()).toBe(40 as Time.Milli); sync.close(); }); - it("unregisters duplicate jitter inputs independently", async () => { + it("unregisters duplicate targets independently", async () => { const media = new Signal(20 as Time.Milli); - const sync = new Sync({ delay: 100 as Time.Milli }); + const sync = new Sync(); const unregisterFirst = sync.register(media); const unregisterSecond = sync.register(media); unregisterFirst(); await flush(); - expect(sync.out.delay.peek()).toBe(120 as Time.Milli); + expect(sync.out.delay.peek()).toBe(20 as Time.Milli); unregisterSecond(); await flush(); + expect(sync.out.delay.peek()).toBe(0 as Time.Milli); + sync.close(); + }); + + it("takes a fixed delay literally, ignoring the measured targets", async () => { + const media = new Signal(250 as Time.Milli); + const sync = new Sync({ delay: 100 as Time.Milli }); + sync.register(media); + await flush(); expect(sync.out.delay.peek()).toBe(100 as Time.Milli); + expect(sync.out.jitter.peek()).toBe(100 as Time.Milli); sync.close(); }); }); diff --git a/js/watch/src/sync.ts b/js/watch/src/sync.ts index c6eff3d25d..9f3990f776 100644 --- a/js/watch/src/sync.ts +++ b/js/watch/src/sync.ts @@ -14,7 +14,9 @@ import { /** * How far playback trails the live edge. * - * `"auto"` (the default) sizes the jitter buffer from the connection RTT; a `Time.Milli` fixes it. + * `"auto"` (the default) holds the largest playout target the registered decoders measure from how + * late their frames arrive (see doc/concept/audio-jitter.md). A `Time.Milli` fixes the delay at + * exactly that number and disables adaptation. * * `"instant"` drops the buffer and the pacing together: nothing is held, and {@link Sync.wait} * returns without sleeping, so a frame presents as soon as it exists. It also overrides @@ -23,9 +25,6 @@ import { */ export type Delay = "instant" | "auto" | Time.Milli; -const MIN_JITTER = Time.Milli(20); -const FALLBACK_JITTER = Time.Milli(100); - export type SyncInput = { /** How far playback trails the live edge. See {@link Delay}. */ delay: Getter; @@ -40,10 +39,7 @@ export type SyncInput = { */ buffer: Getter; - /** - * The connection's PROBE estimates, whose RTT drives "auto" jitter. Usually wired - * from a `Connection`'s `probe`. - */ + /** @internal */ probe: Getter; }; @@ -56,12 +52,11 @@ type SyncOutput = { // This will keep being updated as we catch up to the live playhead then will be relatively static. reference: Signal; - // The resolved delay from the live edge to the playhead: jitter + max(audio, video). + // The resolved delay from the live edge to the playhead: the configured number, or in "auto" the + // largest registered playout target. Zero for "instant". delay: Signal; - // The jitter component of `delay` (always numeric). - // In "auto" mode this is updated automatically from RTT. - // When the delay is a number, jitter equals that number. + // The jitter buffer, always equal to `delay`. jitter: Signal; // The media timestamp of the most recently received frame. @@ -84,7 +79,7 @@ export class Sync { instant: new Signal(false), reference: new Signal(undefined), delay: new Signal(Time.Milli.zero), - jitter: new Signal(FALLBACK_JITTER), + jitter: new Signal(Time.Milli.zero), timestamp: new Signal(undefined), buffered: new Signal(false), maxAge: new Signal(Time.Milli.zero), @@ -94,10 +89,7 @@ export class Sync { // Per-label late-frame tracking: accumulate count and max lateness, flush on recovery. #late = new Map(); - // Minimum RTT seen, used as the baseline for jitter calculation. - // Avoids inflating jitter due to bufferbloat. - #minRtt: number | undefined; - #media = new Signal<{ jitter: Getter }[]>([]); + #media = new Signal<{ target: Getter }[]>([]); #signals = new Effect(); @@ -109,14 +101,13 @@ export class Sync { }; this.#signals.run(this.#runInstant.bind(this)); - this.#signals.run(this.#runJitter.bind(this)); this.#signals.run(this.#runDelay.bind(this)); this.#signals.run(this.#runMaxAge.bind(this)); } - /** Include a decoder's rendition delay in the shared playback clock until disposed. */ - register(jitter: Getter): Dispose { - const registered = { jitter }; + /** Hold a decoder's "auto" playout target in the shared playback clock until disposed. */ + register(target: Getter): Dispose { + const registered = { target }; this.#media.update((media) => [...media, registered]); return () => this.#media.update((media) => media.filter((candidate) => candidate !== registered)); } @@ -135,51 +126,21 @@ export class Sync { this.#out.maxAge.set(Time.Milli.add(delay, buffer)); } - #runJitter(effect: Effect): void { - const delay = effect.get(this.in.delay); - - if (delay === "instant") { - // Holds nothing at all. - this.#minRtt = undefined; - this.#out.jitter.set(Time.Milli.zero); - return; - } - - if (typeof delay === "number") { - // Fixed mode: the configured delay is the jitter. - this.#minRtt = undefined; - this.#out.jitter.set(delay); - return; - } - - // "auto" mode: compute jitter from the connection's RTT estimate. - const rtt = effect.get(this.in.probe)?.rtt; - if (rtt !== undefined) { - // Track minimum RTT as baseline, ignoring bufferbloat. - this.#minRtt = this.#minRtt !== undefined ? Math.min(this.#minRtt, rtt) : rtt; - - // Buffer enough for a retransmit (1 RTT for ACK + retransmit). - const jitter = Time.Milli(Math.max(MIN_JITTER, this.#minRtt * 1.25)); - this.#out.jitter.set(jitter); - return; - } - - // No RTT available: fall back to static default. - this.#minRtt = undefined; - this.#out.jitter.set(FALLBACK_JITTER); - } - + // A fixed delay is taken literally and "auto" holds the deepest track's target, so every track + // plays from one clock. See doc/concept/audio-jitter.md for how a target is measured. #runDelay(effect: Effect): void { - const jitter = effect.get(this.#out.jitter); - let media = Time.Milli.zero; - for (const registered of effect.get(this.#media)) { - media = Time.Milli.max(media, effect.get(registered.jitter) ?? Time.Milli.zero); + const mode = effect.get(this.in.delay); + + let delay = Time.Milli.zero; + if (typeof mode === "number") { + delay = mode; + } else if (mode === "auto") { + for (const registered of effect.get(this.#media)) { + delay = Time.Milli.max(delay, effect.get(registered.target) ?? Time.Milli.zero); + } } - // A zero delay still holds the rendition's own delay, which is a frame interval at 60fps. - // "instant" holds nothing at all. - const instant = effect.get(this.#out.instant); - const delay = instant ? Time.Milli.zero : Time.Milli.add(media, jitter); + this.#out.jitter.set(delay); this.#out.delay.set(delay); } diff --git a/js/watch/src/ui/components/buffering-indicator.ts b/js/watch/src/ui/components/buffering-indicator.ts index 03581cdd4f..4a6e6b4632 100644 --- a/js/watch/src/ui/components/buffering-indicator.ts +++ b/js/watch/src/ui/components/buffering-indicator.ts @@ -9,7 +9,15 @@ export function bufferingIndicator(parent: Effect, watch: MoqWatch): HTMLElement container.appendChild(spinner); parent.run((effect) => { - const buffering = effect.get(watch.video.out.stalled); + // Audio re-buffers on its own once the ring runs dry, which is just as much a stall as a + // video one. Gate it on audio actually being downloaded: the ring reports stalled from + // construction and drains once the download stops, so a video-only broadcast or a muted + // player would otherwise show the spinner forever. "instant" never downloads audio at all. + const audio = + effect.get(watch.audio.in.enabled) && + effect.get(watch.audio.source.out.config) !== undefined && + effect.get(watch.controls.delay) !== "instant"; + const buffering = effect.get(watch.video.out.stalled) || (audio && effect.get(watch.audio.out.stalled)); const status = effect.get(watch.broadcast.out.status); const online = status === "loading" || status === "live"; const unsupported = effect.get(watch.video.source.out.error) === "unsupported"; diff --git a/js/watch/src/ui/components/latency.ts b/js/watch/src/ui/components/latency.ts index 57428989c6..43d5c00fa4 100644 --- a/js/watch/src/ui/components/latency.ts +++ b/js/watch/src/ui/components/latency.ts @@ -63,7 +63,7 @@ export function latencyTab(parent: Effect, watch: MoqWatch): HTMLElement { const hint = DOM.create( "div", { className: "tab-hint" }, - "A larger delay smooths over network jitter at the cost of latency. Auto tracks the connection RTT. Drag the timeline to fine-tune.", + "A larger delay smooths over network jitter at the cost of latency. Auto follows how late frames actually arrive. Drag the timeline to fine-tune.", ); container.append(chips, timeline, readout, hint); diff --git a/js/watch/src/ui/stats.ts b/js/watch/src/ui/stats.ts index bbb18199ba..2e80e25f86 100644 --- a/js/watch/src/ui/stats.ts +++ b/js/watch/src/ui/stats.ts @@ -86,6 +86,7 @@ export function statsTab(parent: Effect, watch: MoqWatch): HTMLElement { const aRate2 = line(audioCard.grid, "Sample rate"); const aChannels = line(audioCard.grid, "Channels"); const aBitrate = line(audioCard.grid, "Bitrate"); + const aUnderruns = line(audioCard.grid, "Underruns"); track(parent, audioCard, { catalog: watch.audio.source.out.catalog, flag: watch.controls.muted, @@ -137,6 +138,10 @@ export function statsTab(parent: Effect, watch: MoqWatch): HTMLElement { aBitrate.textContent = aBitrate2 !== undefined ? formatBitrate(aBitrate2) : "—"; if (aStats) aPrev = { bytes: aStats.bytesReceived, when: now }; + // A non-zero count means the target is below what arrivals actually need. + const aUnder = watch.audio.out.underruns.peek(); + aUnderruns.textContent = watch.audio.out.stalled.peek() ? `${aUnder} (buffering)` : `${aUnder}`; + // Network. "Estimated max" is the congestion controller / PROBE estimate; // "Actual" is the goodput we measure from the video + audio byte counters. const probe = watch.connection.probe.peek(); diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 2d91fdb0d1..56134653c2 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -96,6 +96,8 @@ export class Decoder { // subscribed to. One value so a later source that reuses the track name still notifies: // the name alone would compare equal, and the jitter effect would keep the previous catalog. #pending = new Signal<{ track: string; catalog: Getter } | undefined>(undefined); + // The active track's arrival estimate from its container consumer. + #spread = new Signal(undefined); readonly #identity: Computed; #signals = new Effect(); @@ -115,7 +117,19 @@ export class Decoder { this.source = props.source; this.sync = props.sync; - this.#signals.cleanup(this.sync.register(this.out.jitter)); + // "auto" holds the arrival estimate plus the active rendition's catalog `delay`, floored at the + // catalog requirement. The estimate cancels any offset between tracks, so the delay is added + // back. The audio frame term has no video counterpart: the rendition jitter already covers a + // frame interval. + const target = this.#signals.computed((effect) => { + const jitter = effect.get(this.out.jitter); + const spread = effect.get(this.#spread); + if (spread === undefined) return jitter; + const active = effect.get(this.#active); + const delay = active && effect.get(active.catalog)?.video?.renditions?.[active.track]?.delay; + return Time.Milli.max(Time.Milli.add(spread, Time.Milli(delay ?? 0)), jitter ?? Time.Milli.zero); + }); + this.#signals.cleanup(this.sync.register(target)); this.#identity = this.#signals.computed((effect) => { const config = effect.get(this.source.out.config); return config ? playbackIdentity(config) : undefined; @@ -213,6 +227,7 @@ export class Decoder { if (!active) { // Clear stale data when disabled (e.g. paused or not visible). this.#out.buffered.set([]); + this.#spread.set(undefined); return; } @@ -230,6 +245,7 @@ export class Decoder { }); }); effect.proxy(this.#out.buffered, active.buffered); + effect.proxy(this.#spread, active.spread); } #runDisplay(effect: Effect): void { @@ -304,6 +320,9 @@ class DecoderTrack { // Network jitter + decode buffer. buffered = new Signal([]); + // How late frames arrive relative to the earliest one, from the container consumer. + spread = new Signal(undefined); + // Decoded frames waiting to be rendered. #buffered = new Signal([]); @@ -418,6 +437,9 @@ class DecoderTrack { this.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the arrival estimate for the "auto" target. + effect.run((inner) => this.spread.set(inner.get(consumer.spread))); + decoder.configure({ codec: this.config.codec, description: this.config.description ? Util.Hex.toBytes(this.config.description) : undefined, @@ -500,6 +522,9 @@ class DecoderTrack { this.buffered.update(() => Container.mergeBufferedRanges(network, decode)); }); + // Publish the arrival estimate for the "auto" target. + effect.run((inner) => this.spread.set(inner.get(consumer.spread))); + // Configure decoder with description from catalog decoder.configure({ codec: this.config.codec, diff --git a/quest/m0/audio-jitter-target/README.md b/quest/m0/audio-jitter-target/README.md index 604b520987..ee6bb64f90 100644 --- a/quest/m0/audio-jitter-target/README.md +++ b/quest/m0/audio-jitter-target/README.md @@ -19,64 +19,22 @@ clock](/quest/m1/av-clock.md). ## Plan -Two quests: one implementation per language against the written algorithm. -The two implementations are independent and may run in parallel. Both are -additive and target `main`: the native knob is a new field on a -`#[non_exhaustive]` struct, and the browser estimator is a new module plus a -new `spread` observation. - -Decided for landing: the line merges to `main` with a changelog -note for two behavior changes treated as fixes. `@moq/watch` `Sync` takes a -numeric delay literally instead of adding the rendition delay on top -([#3954](https://github.com/moq-dev/moq/pull/3954)), and `moq play --delay` -defaults to `auto` instead of `100ms` -([#3967](https://github.com/moq-dev/moq/pull/3967)). The old additive delay -was wrong, and both compile unchanged for existing callers. The line branch is -about 200 commits behind `main` with conflicts in `js/watch/src/sync.ts` and -`rs/moq-cli`; merge `main` in (never rebase the shared branch) before -finishing the watch quest. The raw #3477 traces are gone, so record fresh -traces with the audio quality harness in `test/audio-quality/` instead of -asking the reporter; they replace the #3477 traces wherever the quests name them. - -The algorithm is written down at `doc/concept/audio-jitter.md`, with a -conformance corpus beside it that both implementations will read. - -Neither `main` nor `release` has a measured estimator yet. `js/watch/src/sync.ts:159` -still computes `max(MIN_JITTER, minRtt * 1.25)` from the connection's PROBE, -and `js/watch/src/audio/latency.ts` still exists. `sync.ts` also adds the -advertised jitter to that term, where the document settles on a maximum. - -The prior art from PR #3517 (closed 2026-09-10, branch deleted 2026-09-30) -landed on this line's branch through -[#3954](https://github.com/moq-dev/moq/pull/3954), so the watch quest works -there. It deletes the RTT term -(`MIN_JITTER`, `FALLBACK_JITTER`, `#minRtt`, and the `probe` input are gone -from `sync.ts`; `latency.ts` survives, minus `reanchorFloor`) and plumbs a -per-track arrival `spread` through `Container.Consumer`, measured at container -frame arrival and before the age budget can skip a group, which is the right -observation point. Its estimator, `js/hang/src/container/jitter.ts`, is a -decaying histogram of 5 ms buckets, 200 of them so the percentile saturates at -one second, read at the 95th percentile plus one frame, with the arrival -minimum expiring over two 30 s windows, `reanchor()` on a discontinuity, and -the step down bounded to one frame per second. `jitter.test.ts` and -`js/watch/src/audio/replay.test.ts` cover it. What it gets wrong is the extra -frame, learned from the first gap between observed timestamps, and a rise that -is immediate and unclamped, so a tune-in across a stale group sets the target -to seconds. - -Note that `sync.ts` has since been refactored on `main` to a `register(jitter)` -list, which is one of the conflicts merging `main` in resolves. - -Native has no jitter buffer at all. `rs/moq-audio`'s `decode::Options` -(`rs/moq-audio/src/decode/consumer.rs`) carries `max_age`, how far -playback may drift from the live edge before skipping a stalled group, and -`start`, where to begin on a track that already holds groups. Nothing pads the -buffer against uneven arrivals. +Both implementations are on `main` (#4162): `js/hang/src/container/jitter.ts` +and `rs/moq-audio`'s `jitter.rs` follow `doc/concept/audio-jitter.md` and pass +its conformance corpus, `js/watch` `Sync` holds the deepest track's target in +`"auto"`, and `moq play --delay` defaults to `auto`. Both are the default for +every viewer, so `"auto"` shipped ahead of the Chrome and Safari proof the +Watch quest still owes; the maintainer accepted that, and the re-measured +replay budgets, on 2026-10-04. + +What the line still owes once Watch lands: its recorded trace replayed through +the native decode path too, asserting the same target series the browser's +`playout.test.ts` does. Grading native playback against the harness budgets is +[Audio quality native](/quest/m1/audio-quality-native.md). ## Required -- [Watch](/quest/m0/audio-jitter-target/watch.md) - js/watch and js/hang bring the #3954 estimator into conformance -- [Native](/quest/m0/audio-jitter-target/native.md) - rs/moq-audio grows a measured jitter buffer from the same algorithm +- [Watch](/quest/m0/audio-jitter-target/watch.md) - the browser's measured target, proven on Chrome and Safari against the public relay ## Closes diff --git a/quest/m0/audio-jitter-target/native.md b/quest/m0/audio-jitter-target/native.md deleted file mode 100644 index 11dfd4de36..0000000000 --- a/quest/m0/audio-jitter-target/native.md +++ /dev/null @@ -1,52 +0,0 @@ -# [M] rs/moq-audio: a measured jitter buffer, not just an upper bound - -## Goal - -Native playback holds a jitter buffer sized by the algorithm in -`doc/concept/audio-jitter.md`, and passes the same conformance corpus as the -browser on the same trace. `moq play` on a path with uneven -arrivals plays through a flush without underrunning, where today it has nothing -to absorb one. - -Boundaries: no time-stretching and no loss concealment, matching the browser. -Video keeps its own path. - -## Plan - -`rs/moq-audio` has no jitter buffer at all. `decode::Options` -(`rs/moq-audio/src/decode/consumer.rs`) carries `max_age`, how far -playback may drift from the live edge before skipping a stalled group, applied -to the subscription and clamped to the track's retention, and `start`, where -to begin on a track that already holds groups; `decode::Config` selects the -backend only. Neither adds latency: the consumer skips only when newer data is -already that far ahead. - -- Measure arrivals at the same point the browser does, on the container - consumer before the age budget can skip a group, so both languages estimate - from the same observation. That is `container::Consumer::read()`, awaited at - `rs/moq-audio/src/decode/consumer.rs:224`, which is one-to-one with arrivals - and reads no clock today. -- Add the knob to `decode::Options`, additive on the `#[non_exhaustive]` struct, - so it targets `main`. The name is settled: `delay`, matching the browser, - while `max_age` remains the live-edge skip budget. Do not reopen it. -- An explicit `Some(d)` fixes the receiver's jitter target and disables - adaptation, matching the browser's numeric delay setting. `None` selects - the shared automatic estimator. Publisher-declared buffering remains a - separate contribution, floored rather than added, with the same composition in - both implementations. Refuse an explicit target that is negative or beyond the - track's retention rather than silently changing it. -- The estimator itself is internal. Only the config field and whatever the - playback path needs to report its current target become public, and each one - is argued for rather than exposed by default. -- `rs/moq-cli` `play` gets the matching knob, and `doc/bin/cli.md` follows. Check - the examples against `--help`. -- The playback sink holds the target the way the browser's rings do: slack above - it, land on it when skipping, re-stall when dry so the next insert refills to - the target rather than playing on an empty cushion. -- Tests: the conformance corpus at `doc/concept/audio-jitter/*.json`, read - directly from Rust, plus a decode-path test that replays the same trimmed - arrival trace the browser replays and asserts the same target series. -- The frame duration comes from the codec. `opus::packet_samples` already - parses the TOC byte in `rs/moq-mux/src/codec/opus/mod.rs`, but it is - `pub(crate)`, so reaching it means widening its visibility or moving it. - diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index c5c27846eb..fcf17b2523 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -1,106 +1,58 @@ -# [M] js/watch: the playout target follows the spec, proven in a real browser +# [S] js/watch: prove the measured playout target in a real browser ## Goal -`js/watch` computes its auto target with the algorithm in -`doc/concept/audio-jitter.md` and passes the conformance corpus beside it. The -"Real-time" preset plays clean audio on a LAN and against the -public relay: zero underruns after convergence and no skip-aheads in steady -state, on both the isolated and the postMessage ring paths, confirmed by a -manual run on Chrome and Safari with a real microphone and 40 ms or more of -added RTT. +The "Real-time" preset plays clean audio on a LAN and against the public relay: +zero underruns after convergence and no skip-aheads in steady state, on both the +isolated and the postMessage ring paths, confirmed by a manual run on Chrome and +Safari with a real microphone and 40 ms or more of added RTT. -Boundaries: the ring's slack, re-stall and underrun counter are mechanical and -already correct on the #3517 branch; this quest changes the estimator above -them. +Boundaries: the estimator already conforms to `doc/concept/audio-jitter.md` and +passes the corpus in `js/hang/src/container/jitter.test.ts`. This quest measures +it, and fixes only what the measurement shows. ## Plan -Start from `origin/quest/m0/3477-watch-auto-latency`, the branch of the closed -PR #3517. Rebase it onto the base branch first, and -expect conflicts: `sync.ts` on `main` has since replaced the `audio`/`video` -inputs with a `register(jitter)` list. Nothing on `main` or `release` has any of -it: `js/watch/src/sync.ts:159` still sizes auto from `minRtt * 1.25`. - -What the branch already has, and this quest keeps: - -- The RTT term is gone. `MIN_JITTER`, `FALLBACK_JITTER` and `#minRtt` are gone - from `sync.ts` and auto is the larger of the per-track spreads. - `js/watch/src/audio/latency.ts` survives, minus `reanchorFloor`; `ringSamples` - and `RENDER_QUANTUM` stay, and `RENDER_QUANTUM` belongs to the ring's slack - rather than to the target. -- The measurement point: `Container.Consumer` observes each frame at arrival, - before the age budget can skip a group, and exposes it as a `spread` getter. -- The estimator in `js/hang/src/container/jitter.ts`: a histogram of 5 ms - buckets, 200 of them, decayed per arrival by `0.9993`, read at the 95th - percentile and clamped to `BUCKETS * BUCKET` (one second) and to the largest - spread seen; the arrival minimum expires over two 30 s windows; `reanchor()` - drops the baseline on a discontinuity; the step down is bounded to one frame - per second. `jitter.test.ts` covers it. -- The ring work: one chunk of slack above the target, landing back on the - target when skipping, re-stalling on empty, and the underrun counter reaching - the stats panel and the buffering indicator. `replay.test.ts` drives both - rings from a recorded arrival trace. - -What remains is conformance to `doc/concept/audio-jitter.md`. The -document's estimator is a different enough shape that rewriting `jitter.ts` -against it is likely cheaper than patching the branch's: - -- The "plus one frame" term. `jitter.ts` learns it as the running minimum of - positive gaps between consecutive observed timestamps, so the first gap sets - it with nothing to validate against, `#publish` raises the target to - `percentile + step` immediately, and `reanchor()` clears `#latest` but not the - learned spacing. That is the 14.56 s reading, and it still reproduces on the - branch head. Write the regression first: two frames whose media timestamps are - far apart, then paced audio with zero real jitter, asserting the target stays - near the frame duration. Then take the frame duration from the codec, which - for Opus means parsing the TOC byte at the container layer, since `js/` has no - parser today and `js/watch/src/audio/config.ts` guesses 20 ms. -- Delay measured against the fastest frame in a 2 s media-time window, not - against a 30 s arrival minimum. Pruning by media time is what makes a - timestamp jump self-correcting, so `reanchor()` goes away. -- The histogram: 20 ms buckets, 100 of them, the underrun forget factor decayed - once per resampled 500 ms maximum rather than per arrival, the cold-start - prior, and the startup ramp. No rise or fall limiter; the document explains - why the branch needed one and this does not. -- Reordered arrivals excluded from the reference and the histogram. -- The advertised jitter floors the target instead of being added to it. - `sync.ts` sums them today, which over-buffers a bursty publisher by its own - flush span. -- Run the conformance corpus in `jitter.test.ts` alongside the existing cases. - `doc/concept/audio-jitter/*.json` is checked in and read directly. -- Check what the viewer's saved preset does on load. The element defaults - `delay` to `"auto"` and nothing in `demo/` overrides it, yet a fresh session - came up on the 100 ms chip, so something is restoring or overriding it. Pin - that down: a stored preference silently winning over the default is its own - bug, and it also means auto gets far less real exposure than it looks like. -- Replay recorded traces rather than synthetic ones of the same shape. - `test/audio-quality/traces/` holds fresh ones (the #3477 traces are gone), - and `js/watch/src/audio/replay.ts` already plays a trace through - `Container.Consumer` and both rings on a simulated clock at the delay `Sync` - resolves. Replay them in `replay.test.ts`, move the replay's target onto the - estimator (its 100 ms fallback covers the public relay's missing PROBE RTT - until then), and tighten the harness's exact replay budgets with it. The - checked-in `test/audio-quality/budgets.json` was recorded locally (#4426); - re-record it from the nightly runner's first runs on `main` before - tightening, since nightly only runs `main`'s code. The replay sizes every - frame as the trace's median spacing, which a trace missing most of its - frames defeats; once the container knows the codec's frame duration, record - it in the trace and replay that instead. +`playout.test.ts` measures synthetic traces through a transport subscription +into `Container.Consumer` on a stubbed clock and sizes both rings from the +target that settles. In `"auto"` the audio subscription asks for at least the +estimator's 2 s ceiling, as native does; a subscription cut to the target hid +every frame later than it. `demo/web` sets no delay and `js/watch` stores none, +so a fresh tile runs auto. + +The harness's `js/watch/src/audio/replay.ts` plays the recorded traces in +`test/audio-quality/traces/` (the #3477 traces are gone) through +`Container.Consumer` and both rings on a simulated clock, and resolves `"auto"` +from the same estimator through a real `Sync`, resizing the ring as it moves. +Its exact budgets were re-measured when `main` merged in: underruns fell to at +most 2 a minute on every trace, but a ring that runs dry now re-stalls and +refills to the target, so `stalled_share` and `silence_share` rose (up to 3% on +`relay-bbb` and `relay-mic`), and `relay-mic` converges from a 420 ms p95 over +about 22 s with skip-aheads on the way down. + +- Decide whether those re-stall refills are what the goal wants, using the + replay rows as the measurement. Tighten the replay budgets with any fix. + Re-record the browser rows' `test/audio-quality/budgets.json` from the + nightly runner's first runs on `main` before tightening them, since they were + recorded locally (#4426) and nightly only runs `main`'s code. +- The replay sizes every frame as the trace's median spacing, which a trace + missing most of its frames defeats; record the codec's frame duration in the + trace and replay that instead. - Manual run against the public relay on Chrome and Safari, the two rows the - issue measured. Re-record the `relay-mic` trace in the same run with a real - microphone (`just test audio-quality-record`); the checked-in one used - Chromium's fake capture device. Measure the publisher's audio encoder input-to-output lag in - the same run using the reporter's instrumented harness; #3518 fixed the known - cause, so the 7.35 s lag and the 88 to 275 ms/s drift the issue reported - stand unconfirmed. If drift survives, the suspects are `writeFrame` opening a - group per audio frame under WebTransport stream credit and the main-thread - task queue delivering encoder output. Turn that into its own quest rather - than fixing it here. - -The branch also replaces `probe` in `SyncInput` with per-track `audioSpread` -and `videoSpread` inputs, which breaks the published `@moq/watch` type. This -quest lands on `main`, so it adds the spread inputs beside `probe` and stops -reading `probe`; removing it is part of the `SyncInput` reshape in -[Plan: A/V clock](/quest/m1/av-clock.md). Land the estimator so -that quest can adopt it without a second estimator change. + issue measured, with a real microphone and 40 ms or more of added RTT. Watch + the stats panel's audio underrun counter and the latency tab's auto readout, + on both the isolated and the postMessage ring paths. Leave the publisher's + DTX off (the voice default since #4808) so the run measures network jitter + rather than DTX drift. Re-record the + `relay-mic` trace in the same run (`just test audio-quality-record`); the + checked-in one used Chromium's fake capture device. Measure the publisher's + audio encoder input-to-output lag in the same run using the reporter's + instrumented harness; #3518 fixed the known cause, so the 7.35 s lag and the + 88 to 275 ms/s drift the issue reported stand unconfirmed. If drift survives, + the suspects are `writeFrame` opening a group per audio frame under + WebTransport stream credit and the main-thread task queue delivering encoder + output. Turn that into its own quest rather than fixing it here. + +## Related + +- [A/V clock](/quest/m1/av-clock.md) - reshapes `SyncInput` around the per-track target this line produces diff --git a/quest/m1/av-clock.md b/quest/m1/av-clock.md index 46bdab9aeb..9b364ddb67 100644 --- a/quest/m1/av-clock.md +++ b/quest/m1/av-clock.md @@ -20,10 +20,12 @@ spread, and one is nominated as the clock source. `SyncInput` (`js/watch/src/sync.ts`, today `delay`, `buffer`, and `probe`) breaks once, and a third track joins without another pair of inputs; `Sync.register` already keeps one jitter entry per track and grows into the handles. The -measured spread per track comes from the [Audio jitter -target](/quest/m0/audio-jitter-target/README.md) line, whose watch branch -replaces `probe` with per-track spread inputs that this quest folds into the -handles. `SyncInput` is a published `@moq/watch` shape, so this is a break. +measured target per track comes from the [Audio jitter +target](/quest/m0/audio-jitter-target/README.md) line: each decoder registers +its own target through `Sync.register`, and `probe` stays in `SyncInput` unread +so that line lands on `main`. This quest folds the registrations into the +handles and drops `probe`. `SyncInput` is a published `@moq/watch` shape, so +this is a break. Recommendations for the implementation: @@ -53,7 +55,7 @@ Recommendations for the implementation: ## Required -- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator this sits on, and the per-track spread inputs this shape carries +- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator this sits on, and the per-track targets this shape carries ## Related diff --git a/rs/moq-audio/Cargo.toml b/rs/moq-audio/Cargo.toml index 8339af8870..063eb87e84 100644 --- a/rs/moq-audio/Cargo.toml +++ b/rs/moq-audio/Cargo.toml @@ -137,4 +137,7 @@ objc2-foundation = { workspace = true, optional = true } objc2-screen-capture-kit = { workspace = true, features = ["SCStream", "SCShareableContent"], optional = true } [dev-dependencies] +# Reads the jitter conformance corpus at doc/concept/audio-jitter/. +serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index cefdb75670..cdb10d71c3 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -1,10 +1,12 @@ //! Subscribe to an encoded audio track and emit raw PCM. use std::collections::VecDeque; +use std::time::{Duration, Instant}; use bytes::Bytes; use super::decoder::{Config, Decoder}; +use crate::jitter::{self, Target}; use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Format, Frame, Layout}; @@ -43,6 +45,13 @@ pub struct Options { pub max_age: std::time::Duration, /// Initial cached-group policy. pub start: Start, + /// A fixed playout delay, or `None` to estimate it from arrival timing. + /// + /// The jitter buffer a player holds ahead of the playhead, reported by + /// [`Consumer::delay`]. Distinct from [`max_age`](Self::max_age), which is + /// the live-edge skip budget and adds no buffering. A fixed delay beyond the + /// track's retention is refused. + pub delay: Option, } impl Options { @@ -94,6 +103,10 @@ pub struct Consumer { terminal_start: Option, /// Last container playhead generation applied to timeline state. discontinuity: u64, + /// The playout delay, fixed or estimated from arrivals. + target: Target, + /// Where arrival times are measured from. + origin: Instant, } struct ActivitySpan { @@ -158,9 +171,17 @@ impl Consumer { subscriber.set_groups(live_edge..); } let track = subscriber; - let max_age = options - .max_age - .min(track.info().max_age.unwrap_or(std::time::Duration::MAX)); + let retention = track.info().max_age; + let max_age = options.max_age.min(retention.unwrap_or(Duration::MAX)); + // Holding more than the publisher keeps would wait on media it has already + // discarded, so the caller asked for something impossible. + if let (Some(delay), Some(retention)) = (options.delay, retention) + && delay > retention + { + return Err(Error::Unsupported(format!( + "playout delay {delay:?} exceeds the track's retention of {retention:?}" + ))); + } // The catalog says how the track is framed, and it is not always the legacy // wire: `moq import fmp4` publishes CMAF. Reading a moof+mdat fragment as a // varint timestamp plus a payload decodes to garbage rather than failing. @@ -168,6 +189,7 @@ impl Consumer { let track = moq_mux::container::Consumer::new(track, container); Ok(Self { + target: Target::new(options.delay, catalog.jitter), decoder, track, resampler, @@ -186,6 +208,7 @@ impl Consumer { end: None, terminal_start: None, discontinuity: 0, + origin: now(), }) } @@ -204,6 +227,17 @@ impl Consumer { self.max_age } + /// The playout delay to hold ahead of the playhead right now: the fixed + /// [`Options::delay`], or the estimate from every frame read so far. + /// + /// The estimate measures when [`read`](Self::read) hands each packet over, so + /// it is only as honest as the caller is prompt: read as soon as the previous + /// frame is handled and let the playback buffer absorb the jitter, rather than + /// pacing reads to the speaker. + pub fn delay(&self) -> Duration { + Duration::from_secs_f64(self.target.millis() / 1000.0) + } + /// Sample rate samples are actually delivered at, which is /// [`Output::sample_rate`] resolved against the catalog. pub fn sample_rate(&self) -> u32 { @@ -239,6 +273,13 @@ impl Consumer { return self.flush(); }; + // Every packet the container hands over is an arrival, including one the + // decoder is about to refuse, so it is observed before anything can fail. + let arrival = jitter::millis(now().duration_since(self.origin)); + let timestamp = mux_frame.timestamp; + let media = timestamp.value() as f64 * 1000.0 / timestamp.scale().as_u64() as f64; + self.target.observe(arrival, media); + if let Some(end) = self.track.end() && self.end != Some(end) { @@ -273,6 +314,12 @@ impl Consumer { self.delay_trimmed += trimmed; let activity = decoded.activity; let mut decoded = decoded.samples; + // The packet's duration as the codec states it, never read off the + // timestamps: those carry every tune-in and hole along with the spacing. + let covered = decoded.len() / self.decoder.layout().channels() as usize + trimmed; + if covered > 0 { + self.target.set_frame(covered as f64 * 1000.0 / rate as f64); + } if let Some(end) = self.end { let terminal_start = *self .terminal_start @@ -468,6 +515,16 @@ impl Consumer { } } +/// The receiver's monotonic clock, which a test can pin to replay a recorded +/// arrival trace without waiting it out. +fn now() -> Instant { + #[cfg(test)] + if let Some(now) = tests::CLOCK.get() { + return now; + } + Instant::now() +} + /// Whether `timestamp` fails to continue `expected`, leaving a hole (or an /// overlap) rather than the next packet in line. /// @@ -1165,6 +1222,167 @@ mod tests { ); } + thread_local! { + /// A pinned receiver clock for [`now`], so a test replays an arrival trace + /// instantly. Thread-local because a `tokio::test` runs on one thread. + pub(super) static CLOCK: std::cell::Cell> = const { std::cell::Cell::new(None) }; + } + + /// Replay a corpus trace through the whole decode path, one PCM packet per group + /// written at its recorded arrival and read straight back, and return the + /// consumer's delay after each packet in milliseconds. + /// + /// PCM rather than Opus only for speed: the frame term comes from the decoded + /// sample count either way, and an unoptimized Opus codec takes minutes over + /// the whole corpus. [`opus_states_its_own_frame_duration`] covers Opus. + async fn replay(case: &crate::jitter::tests::Case) -> Vec { + use crate::jitter::tests::duration; + + let rate = 48_000; + let samples = (case.frame * rate as f64 / 1000.0) as usize; + let mut catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, rate, 1); + catalog.jitter = case.advertised.map(duration); + + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + + let origin = Instant::now(); + CLOCK.set(Some(origin)); + let mut consumer = Consumer::new( + &subscriber, + &catalog, + "audio", + Options { + delay: case.delay.map(duration), + ..Options::new() + }, + ) + .await + .unwrap(); + + let payload: Bytes = vec![0u8; samples * size_of::()].into(); + let mut series = Vec::with_capacity(case.arrival.len()); + for (&arrival, &media) in case.arrival.iter().zip(&case.media) { + CLOCK.set(Some(origin + duration(arrival))); + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(media as u64 * 1000).unwrap(), + duration: None, + payload: payload.clone(), + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + + consumer.read().await.unwrap().expect("one decoded frame per packet"); + series.push(crate::jitter::millis(consumer.delay())); + } + CLOCK.set(None); + series + } + + /// The decode path measures what the corpus says it should: arrival when the + /// container hands a packet over, media from its timestamp, and the frame term + /// from what the codec decoded. `reorder` is left to the estimator's own test, + /// since the container refuses a timestamp that runs backwards. + #[tokio::test] + async fn the_decode_path_conforms_to_the_corpus() { + for name in [ + "advertised", + "buildup", + "fixed", + "flush", + "idle", + "paced", + "spike", + "tunein", + ] { + let case = crate::jitter::tests::case(name); + let expected = case.series(); + for (index, actual) in replay(&case).await.into_iter().enumerate() { + // A Duration is whole nanoseconds, so the round trip through one costs up + // to a nanosecond on top of the corpus tolerance. + assert!( + (actual - expected[index]).abs() <= 1e-5, + "{name}: packet {index} delays {actual} ms, expected {}", + expected[index] + ); + } + } + } + + /// The frame term is the packet's full duration, pre-skip included: Opus trims + /// codec delay off the first packet, and reading the frame off what came out + /// would shrink the term by exactly that much. + #[tokio::test] + async fn opus_states_its_own_frame_duration() { + let mut encoder = Encoder::new(&Settings::new(48_000, Layout::Mono)).unwrap(); + let catalog = encoder.catalog(); + + let broadcast = moq_net::broadcast::Info::new().produce(); + let track = broadcast + .create_track("audio", hang::container::track_info(hang::catalog::PRIORITY.audio)) + .unwrap(); + let subscriber = broadcast.consume(); + let mut producer = moq_mux::container::Producer::new( + track, + moq_mux::catalog::hang::Container::Legacy(moq_mux::container::Kind::Audio), + ); + let mut consumer = Consumer::new(&subscriber, &catalog, "audio", Options::new()) + .await + .unwrap(); + + producer + .write(moq_mux::container::Frame { + timestamp: Timestamp::from_micros(0).unwrap(), + duration: None, + payload: encoder.encode(&vec![0.25f32; encoder.frame_size()]).unwrap().payload, + keyframe: true, + }) + .unwrap(); + producer.cut(None).unwrap(); + + let frame = consumer.read().await.unwrap().expect("decoded frame"); + assert!( + frame.data.len() / size_of::() < encoder.frame_size(), + "pre-skip is trimmed" + ); + // The cold-start prior reads 100 ms, plus one 20 ms packet. + assert_eq!(consumer.delay(), std::time::Duration::from_millis(120)); + } + + #[tokio::test] + async fn a_delay_beyond_retention_is_refused() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let info = hang::container::track_info(hang::catalog::PRIORITY.audio) + .with_max_age(std::time::Duration::from_millis(100)); + let _track = broadcast.create_track("audio", info).unwrap(); + let subscriber = broadcast.consume(); + let catalog = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Pcm, 48_000, 1); + + let options = |delay| Options { + delay: Some(std::time::Duration::from_millis(delay)), + ..Options::new() + }; + let err = Consumer::new(&subscriber, &catalog, "audio", options(101)) + .await + .err() + .expect("a delay the publisher cannot hold"); + assert!(matches!(err, Error::Unsupported(_)), "{err}"); + + let consumer = Consumer::new(&subscriber, &catalog, "audio", options(100)) + .await + .unwrap(); + assert_eq!(consumer.delay(), std::time::Duration::from_millis(100)); + } + #[tokio::test] async fn max_age_is_clamped_to_publisher_retention() { let broadcast = moq_net::broadcast::Info::new().produce(); diff --git a/rs/moq-audio/src/jitter.rs b/rs/moq-audio/src/jitter.rs new file mode 100644 index 0000000000..6fa61dd0d5 --- /dev/null +++ b/rs/moq-audio/src/jitter.rs @@ -0,0 +1,333 @@ +//! The audio playout target, sized from arrival timing. +//! +//! Implements the algorithm in `doc/concept/audio-jitter.md`, which is normative: +//! the browser's `@moq/hang` implements the same page, and the conformance corpus +//! beside it grades both. The design is WebRTC NetEq's underrun estimator +//! (`underrun_optimizer`, `histogram`, and `packet_arrival_history`), implemented +//! from that description rather than from its source. +//! +//! Every value is `f64` milliseconds, evaluated in the order the page writes it, +//! so both languages produce bit-identical targets from the same trace. + +use std::collections::VecDeque; +use std::time::Duration; + +/// Milliseconds of media time the reference search looks back over. +const WINDOW: f64 = 2000.0; + +/// Milliseconds of delay per histogram bucket. +const BUCKET: f64 = 20.0; + +/// Histogram buckets, so the measured term saturates at `BUCKET * BUCKETS`. +const BUCKETS: usize = 100; + +/// The quantile of the delay distribution the target is read at. +const QUANTILE: f64 = 0.95; + +/// Weight retained per folded observation. +const FORGET: f64 = 0.983; + +/// Milliseconds of arrival time per resampled observation. +const RESAMPLE: f64 = 500.0; + +/// Holds the forget factor below [`FORGET`] for the first observations. +const RAMP: f64 = 2.0; + +/// Idle intervals folded at once, purely to bound the work. +const IDLE_MAX: f64 = 1024.0; + +/// The playout target, fixed or estimated. +pub(crate) enum Target { + /// An explicit delay, taken literally: no floor, no frame term, no estimate. + Fixed(f64), + /// Measured from arrivals. + Auto(Box), +} + +impl Target { + /// A fixed target for `Some`, otherwise the estimator floored at `advertised`. + pub(crate) fn new(delay: Option, advertised: Option) -> Self { + match delay { + Some(delay) => Self::Fixed(millis(delay)), + None => Self::Auto(Box::new(Estimator::new(advertised.map_or(0.0, millis)))), + } + } + + /// Fold one frame's arrival into the estimate. A fixed target ignores it. + pub(crate) fn observe(&mut self, arrival: f64, media: f64) { + if let Self::Auto(estimator) = self { + estimator.observe(arrival, media); + } + } + + /// Set the codec's frame duration, the term added on top of the measurement. + pub(crate) fn set_frame(&mut self, frame: f64) { + if let Self::Auto(estimator) = self { + estimator.frame = frame; + } + } + + /// The target in milliseconds. + pub(crate) fn millis(&self) -> f64 { + match self { + Self::Fixed(delay) => *delay, + Self::Auto(estimator) => estimator.target(), + } + } +} + +/// A duration in `f64` milliseconds, exact for any whole number of nanoseconds +/// below 2^53, which dividing the seconds by a thousandth would not be. +pub(crate) fn millis(duration: Duration) -> f64 { + duration.as_nanos() as f64 / 1_000_000.0 +} + +/// The cold-start distribution, which is also what an idle receiver relaxes back to. +fn prior(i: usize) -> f64 { + 0.5f64.powi(i as i32 + 1) +} + +/// One arrival still inside the reference window. +struct Entry { + arrival: f64, + media: f64, +} + +/// Estimates the playout target from arrival timing alone. +pub(crate) struct Estimator { + /// The codec's frame duration, never learned from observed timestamps. + frame: f64, + /// The publisher's advertised flush span, a floor on the measured term. + advertised: f64, + + /// Arrivals still inside the reference window, in media order. + history: VecDeque, + /// The largest media timestamp observed, so a reordered frame is recognized. + newest: f64, + + /// The arrival resample intervals are counted from, once one has been seen. + origin: Option, + /// The interval currently accumulating, counted from `origin`. + interval: f64, + /// The largest delay seen in that interval. + peak: f64, + + histogram: [f64; BUCKETS], + /// Observations folded in, which drives the cold-start ramp. + count: u64, +} + +impl Estimator { + fn new(advertised: f64) -> Self { + Self { + frame: 0.0, + advertised, + history: VecDeque::new(), + newest: f64::NEG_INFINITY, + origin: None, + interval: 0.0, + peak: 0.0, + histogram: std::array::from_fn(prior), + count: 0, + } + } + + fn observe(&mut self, arrival: f64, media: f64) { + // A reordered arrival is excluded rather than measured: the media-time gap back + // to the frame that overtook it would otherwise be added to the delay. + if media <= self.newest { + return; + } + self.newest = media; + + // Pruning by media time rather than by arrival is what makes a timestamp jump + // self-correcting: everything before it leaves the window at once. + while self.history.front().is_some_and(|entry| entry.media < media - WINDOW) { + self.history.pop_front(); + } + self.history.push_back(Entry { arrival, media }); + + // The fastest frame still in the window, so delay is measured against the + // best-case path instead of against the previous frame. + let mut reference = &self.history[0]; + for entry in &self.history { + if entry.arrival - entry.media < reference.arrival - reference.media { + reference = entry; + } + } + + let delay = f64::max(0.0, arrival - reference.arrival - (media - reference.media)); + + let Some(origin) = self.origin else { + self.origin = Some(arrival); + self.peak = delay; + return; + }; + + let interval = ((arrival - origin) / RESAMPLE).floor(); + if interval <= self.interval { + self.peak = self.peak.max(delay); + return; + } + + self.fold(Some(((self.peak / BUCKET).floor() as usize).min(BUCKETS - 1))); + + let idle = f64::min(interval - self.interval - 1.0, IDLE_MAX); + for _ in 0..idle as u32 { + self.fold(None); + } + + self.interval = interval; + self.peak = delay; + } + + fn target(&self) -> f64 { + let mut cumulative = 0.0; + let mut measured = BUCKETS as f64 * BUCKET; + for (i, weight) in self.histogram.iter().enumerate() { + cumulative += weight; + if cumulative > QUANTILE { + measured = (i + 1) as f64 * BUCKET; + break; + } + } + + measured.max(self.advertised) + self.frame + } + + // One observation is `h = f*h + (1-f)*x`, where x is the bucket that was observed, + // or the prior when the interval held no arrival at all. An idle interval therefore + // relaxes the estimate back toward cold start at the same time constant, rather than + // freezing it or renormalizing the decay away. + fn fold(&mut self, bucket: Option) { + self.count += 1; + let count = self.count as f64; + let forget = f64::min(FORGET, count / (count + RAMP)); + let fresh = 1.0 - forget; + + for (i, weight) in self.histogram.iter_mut().enumerate() { + *weight *= forget; + if bucket.is_none() { + *weight += fresh * prior(i); + } + } + if let Some(bucket) = bucket { + self.histogram[bucket] += fresh; + } + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + + /// One case of the conformance corpus at `doc/concept/audio-jitter/`. + #[derive(serde::Deserialize)] + pub(crate) struct Case { + pub frame: f64, + pub advertised: Option, + pub delay: Option, + pub arrival: Vec, + pub media: Vec, + pub target: Vec<(usize, f64)>, + } + + impl Case { + /// The expected target after each frame, expanded from the change list. + pub fn series(&self) -> Vec { + let mut changes = self.target.iter().peekable(); + let mut current = f64::NAN; + (0..self.arrival.len()) + .map(|index| { + if let Some(&&(at, value)) = changes.peek() + && at == index + { + current = value; + changes.next(); + } + current + }) + .collect() + } + } + + /// A corpus case by name, read directly from the checked-in file so neither + /// implementation is graded against the other. + pub(crate) fn case(name: &str) -> Case { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../doc/concept/audio-jitter") + .join(format!("{name}.json")); + let json = std::fs::read_to_string(&path).unwrap_or_else(|err| panic!("read {}: {err}", path.display())); + serde_json::from_str(&json).unwrap() + } + + pub(crate) fn duration(millis: f64) -> Duration { + Duration::from_nanos((millis * 1_000_000.0) as u64) + } + + /// Every case the document lists. Named rather than globbed, so a case that + /// goes missing fails here instead of silently shrinking the suite. + const CORPUS: [&str; 9] = [ + "advertised", + "buildup", + "fixed", + "flush", + "idle", + "paced", + "reorder", + "spike", + "tunein", + ]; + + #[test] + fn conforms_to_the_corpus() { + for name in CORPUS { + let case = case(name); + let mut target = Target::new(case.delay.map(duration), case.advertised.map(duration)); + target.set_frame(case.frame); + + let series = case.series(); + for (index, (&arrival, &media)) in case.arrival.iter().zip(&case.media).enumerate() { + target.observe(arrival, media); + let actual = target.millis(); + assert!( + (actual - series[index]).abs() <= 1e-6, + "{name}: frame {index} targets {actual}, expected {}", + series[index] + ); + } + } + } + + /// The #3517 regression, without the corpus: a stale frame then paced audio far + /// ahead of it must not carry the media-time gap into the target. + #[test] + fn a_timestamp_jump_is_not_a_delay() { + let mut target = Target::new(None, None); + target.set_frame(20.0); + target.observe(0.0, 0.0); + for i in 0..500 { + target.observe(10.0 + i as f64 * 20.0, 14_500.0 + i as f64 * 20.0); + } + assert_eq!(target.millis(), 40.0); + } + + #[test] + fn an_advertised_span_floors_rather_than_adds() { + let target = Target::new(None, Some(Duration::from_millis(200))); + // The prior reads 100 ms, under the advertised 200 ms. + assert_eq!(target.millis(), 200.0); + } + + #[test] + fn the_measured_term_is_bounded_by_the_histogram() { + let mut target = Target::new(None, None); + target.set_frame(20.0); + // A frame every 20 ms of media arriving a whole second apart: every interval + // sees a delay far past the last bucket. + for i in 0..200 { + target.observe(i as f64 * 1000.0, i as f64 * 20.0); + } + assert_eq!(target.millis(), BUCKETS as f64 * BUCKET + 20.0); + } +} diff --git a/rs/moq-audio/src/lib.rs b/rs/moq-audio/src/lib.rs index 8dc3c97d42..076e439167 100644 --- a/rs/moq-audio/src/lib.rs +++ b/rs/moq-audio/src/lib.rs @@ -52,6 +52,7 @@ mod activity; mod error; mod format; mod frame; +mod jitter; mod layout; mod opus; mod pcm; diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 4379abc173..21d64ba63b 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -1910,7 +1910,7 @@ mod tests { let Command::Play(play) = &cli.stages[0] else { panic!("expected play") }; - assert_eq!(play.delay, Duration::from_millis(100).into()); + assert_eq!(play.delay.to_string(), "auto"); assert_eq!(play.select.video_name.as_deref(), Some("hd")); assert!(cli.moq.validate().is_ok()); assert!(play.validate().is_ok()); diff --git a/rs/moq-cli/src/play/args.rs b/rs/moq-cli/src/play/args.rs index b4094a2246..767a6c9263 100644 --- a/rs/moq-cli/src/play/args.rs +++ b/rs/moq-cli/src/play/args.rs @@ -1,5 +1,7 @@ //! The `play` verb's command-line surface. +use std::fmt; +use std::str::FromStr; use std::time::Duration; use moq_mux::catalog::CatalogFormat; @@ -13,6 +15,47 @@ use crate::subscribe::{CatalogFormatArg, SelectArgs}; /// below pins the two together in a build that has both. const DELAY_MAX: Duration = Duration::from_secs(10); +/// How long `auto` waits on a stalled group before skipping it. +/// +/// The estimate only sees what the container hands over, and a skipped group is +/// never handed over, so a budget under the estimate's ceiling would cap the +/// target at the budget it was cut to. Two seconds is that ceiling, the most +/// lateness the estimate can represent. +const AUTO_MAX_AGE: Duration = Duration::from_secs(2); + +/// How far video alone trails the live edge under `auto`. The estimate is +/// audio's; video follows the speaker whenever there is one. +const AUTO_VIDEO_DELAY: Duration = Duration::from_millis(100); + +/// How far playback trails the live edge. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Delay { + /// Sized from how unevenly audio arrives, and following it as that changes. + Auto, + /// Held at exactly this. + Fixed(Duration), +} + +impl FromStr for Delay { + type Err = humantime::DurationError; + + fn from_str(value: &str) -> Result { + match value { + "auto" => Ok(Self::Auto), + value => humantime::parse_duration(value).map(Self::Fixed), + } + } +} + +impl fmt::Display for Delay { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Auto => f.write_str("auto"), + Self::Fixed(delay) => humantime::format_duration(*delay).fmt(f), + } + } +} + /// Play one MoQ broadcast through a native window and speaker. #[derive(usage::Args, Clone)] #[usage(unknown_flags = "error", args_override_self = false)] @@ -21,15 +64,17 @@ pub struct Args { #[usage(long, value_enum)] pub catalog_format: Option, - /// How far playback trails the live edge. + /// How far playback trails the live edge: `auto`, or a fixed duration. /// /// The playout delay: every frame is presented this long after the live edge, which is - /// how late one may arrive and still make its slot. It doubles as the staleness budget, - /// since nothing older than the playhead is worth presenting. The speaker holds the - /// delay, with a 50ms floor under it, so a smaller value than that does not reach the - /// picture either. - #[usage(long, default = "100ms")] - pub delay: crate::duration::Duration, + /// how late one may arrive and still make its slot. `auto` measures how unevenly audio + /// arrives and follows it, so a publisher that flushes in bursts gets a buffer deep + /// enough to play through them. A duration fixes the delay instead, and doubles as the + /// staleness budget, since nothing older than the playhead is worth presenting. The + /// speaker holds the delay, with a 50ms floor under it, so a smaller value than that + /// does not reach the picture either. + #[usage(long, default = "auto")] + pub delay: Delay, /// Rendition selection by track name or codec. #[usage(flatten)] @@ -44,6 +89,24 @@ impl Args { .unwrap_or_default() } + /// The fixed playout delay, or `None` to estimate it. + pub(super) fn fixed_delay(&self) -> Option { + match self.delay { + Delay::Auto => None, + Delay::Fixed(delay) => Some(delay), + } + } + + /// How long a stalled group is waited on before it is skipped. + pub(super) fn max_age(&self) -> Duration { + self.fixed_delay().unwrap_or(AUTO_MAX_AGE) + } + + /// How far video trails the live edge while no speaker is setting the pace. + pub(super) fn video_delay(&self) -> Duration { + self.fixed_delay().unwrap_or(AUTO_VIDEO_DELAY) + } + /// Reject a codec the local decoders can't open. /// /// The selection flags are shared with the stdout exports, which pass bytes @@ -60,7 +123,7 @@ impl Args { // The delay is the speaker's ring depth, so a value it cannot hold is // refused here rather than after the pipeline has opened a device. anyhow::ensure!( - self.delay.into_std() <= DELAY_MAX, + self.fixed_delay().is_none_or(|delay| delay <= DELAY_MAX), "--delay must be at most {DELAY_MAX:?}; it is the depth the speaker buffers" ); Ok(()) @@ -95,15 +158,26 @@ mod tests { assert!(parse(&["--video-codec", "vp8"]).validate().is_err()); } - /// The delay is the playout offset and the staleness budget at once, so its - /// default has to be one a live stream can actually present against. + /// A fixed delay is the playout offset and the staleness budget at once. #[test] - fn the_delay_sets_the_staleness_budget() { - assert_eq!(parse(&[]).delay.into_std(), std::time::Duration::from_millis(100)); - assert_eq!( - parse(&["--delay", "500ms"]).delay.into_std(), - std::time::Duration::from_millis(500) - ); + fn a_fixed_delay_sets_the_staleness_budget() { + let args = parse(&["--delay", "500ms"]); + assert_eq!(args.fixed_delay(), Some(Duration::from_millis(500))); + assert_eq!(args.max_age(), Duration::from_millis(500)); + assert_eq!(args.video_delay(), Duration::from_millis(500)); + } + + /// The default measures the delay, and waits on a stalled group as long as the + /// estimate can see, so the budget never caps what it measures. + #[test] + fn the_delay_defaults_to_the_estimate() { + let args = parse(&[]); + assert_eq!(args.delay, Delay::Auto); + assert_eq!(args.fixed_delay(), None); + assert_eq!(args.max_age(), AUTO_MAX_AGE); + assert_eq!(parse(&["--delay", "auto"]).delay, Delay::Auto); + assert_eq!(Delay::Auto.to_string(), "auto"); + assert_eq!(Delay::Fixed(Duration::from_millis(250)).to_string(), "250ms"); } /// The playout budget has one spelling because it always controls both @@ -123,6 +197,7 @@ mod tests { let err = parse(&["--delay", "11s"]).validate().unwrap_err().to_string(); assert!(err.contains("--delay must be at most"), "{err}"); parse(&["--delay", "10s"]).validate().unwrap(); + parse(&["--delay", "auto"]).validate().unwrap(); // The bound has to be the sink's own, which only a build carrying the sink // can say. diff --git a/rs/moq-cli/src/play/media.rs b/rs/moq-cli/src/play/media.rs index e2ed82ddbb..31a3c943bc 100644 --- a/rs/moq-cli/src/play/media.rs +++ b/rs/moq-cli/src/play/media.rs @@ -16,7 +16,7 @@ use super::args::Args; use super::output::{Output, Sink, Speaker}; use super::playback::{Kind, Playback, joined}; use super::source::subscribe; -use super::timeline::{AudioTimeline, Presentation, timestamp}; +use super::timeline::{AudioTimeline, Presentation, fit, timestamp}; use super::video::Video; use super::window::Event; @@ -25,20 +25,21 @@ use super::window::Event; /// network jitter, and one with no depth at all can never be read from. const AUDIO_BUFFER_MIN: Duration = Duration::from_millis(50); -/// How much audio is handed to the speaker per write. +/// How far the speaker may run over its target before a write skips back onto +/// it. /// -/// The playout delay lives in the sink, so every byte written past that depth -/// overshoots it, and writing in slices keeps the overshoot under one slice -/// however long a PCM frame is (an Opus packet caps at 120 ms, but a PCM one is -/// only required to be sample-aligned). Pacing between the slices is also what -/// stops `Sink::write`, which never blocks and drops whatever won't fit, from -/// losing the tail of a burst. -const AUDIO_CHUNK: Duration = Duration::from_millis(20); - -/// How much longer than the speaker could possibly hold to wait for it to -/// drain. A device that never opens reports its queue as full forever, and a -/// truncated tail beats hanging on the way out, but the budget has to cover the -/// ring the delay asked for or every finite track loses its last `delay`. +/// The level moves by a device period each time the speaker pulls and by a +/// packet each time one lands, so a margin under that would skip on ordinary +/// cadence. Anything over it is latency nobody asked for, since a live stream +/// arrives as fast as it plays and never drains the excess on its own. +const AUDIO_SLACK: Duration = Duration::from_millis(20); + +/// Silence written per slice when padding, so the buffer stays a fixed size. +const AUDIO_SILENCE: Duration = Duration::from_millis(20); + +/// How much longer than the speaker holds to wait for it to drain. A device that +/// never opens reports its queue as full forever, and a truncated tail beats +/// hanging on the way out. const AUDIO_DRAIN_GRACE: Duration = Duration::from_secs(1); /// Everything the media task needs to fill the window and the speaker. @@ -72,11 +73,6 @@ impl Media { .await .context("failed to subscribe to the catalog")?; let mut catalogs = catalog.select(self.args.select.selection(None)); - // The floored depth, not the raw delay: the speaker holds at least - // AUDIO_BUFFER_MIN whatever was asked for, so a smaller budget would skip a - // group the playhead could still have reached, and would size the hole - // fill in `play_audio` to a playhead that does not exist. - let depth = self.args.delay.into_std().max(AUDIO_BUFFER_MIN); let mut tasks = tokio::task::JoinSet::new(); let mut playback = Playback::default(); // Shared by an audio rendition and the retired tails still playing beside @@ -113,7 +109,7 @@ impl Media { // the other, a rendition switch costs that delay in silence, so the // tail plays out while the replacement fills. if let Some(sink) = sink { - tails.spawn(drain(sink, depth)); + tails.spawn(drain(sink)); } kind }); @@ -173,7 +169,15 @@ impl Media { continue; } }; - let max_age = self.args.delay.into_std(); + // Nothing older than the playhead is worth presenting, so the delay + // doubles as the staleness budget on the wire. With no speaker to + // follow, the playhead is video's own, so waiting on the audio + // estimate's budget would only freeze the picture. + let max_age = if snapshot.audio.renditions.is_empty() { + self.args.video_delay() + } else { + self.args.max_age() + }; let opened = async { let decoder = moq_video::decode::Sink::open(&config, &Default::default()).await?; let track = rendition.track(&name)?; @@ -228,7 +232,12 @@ impl Media { }; let mut decode = moq_audio::decode::Options::new(); decode.start = moq_audio::decode::Start::Latest; - decode.max_age = depth; + // Floored: the speaker holds at least AUDIO_BUFFER_MIN whatever was + // asked for, so a smaller budget would skip a group the playhead could + // still have reached, and would size the hole fill below to a playhead + // that does not exist. + decode.max_age = self.args.max_age().max(AUDIO_BUFFER_MIN); + decode.delay = self.args.fixed_delay(); // The sink and the frame-duration math below both assume f32, // so ask for it rather than inheriting the decoder default. decode.output.format = moq_audio::Format::F32; @@ -241,7 +250,7 @@ impl Media { let audio = AudioPlayback { speaker: speaker.clone().expect("opened above"), presentation: self.presentation.clone(), - depth, + latency: self.args.fixed_delay().unwrap_or_default().max(AUDIO_BUFFER_MIN), changed: self.drained.clone(), output: self.output.clone(), }; @@ -272,7 +281,8 @@ struct AudioPlayback { changed: Arc, speaker: O::Speaker, presentation: Arc>, - depth: Duration, + /// The depth the sink opens on, which also sizes its ring. + latency: Duration, output: O, } @@ -286,16 +296,17 @@ async fn play_audio( changed, speaker, presentation, - depth, + latency, output, } = playback; - // `depth` is how much the speaker holds: the playout delay, floored, and the - // same value the decoder's age budget was built from. The delay lives in the - // sink rather than in the throttle, since a sample handed over now sounds - // that much later and waiting for the delayed instant before writing would - // take it twice. The window schedules video against where the speaker - // actually is, which keeps the two together. + // The playout delay is the consumer's to size, from how unevenly packets arrive, + // and the sink is where it lives: a sample handed over now sounds that much + // later. That estimate times each packet when `read` hands it over, so this + // loop never waits on the speaker between reads. It writes everything as it + // comes and lets `fit` hold the sink on the target instead. The window + // schedules video against where the speaker actually is, which keeps the two + // together whatever the target does. let sample_rate = consumer.sample_rate(); let layout = consumer.layout(); let channels = layout.channels(); @@ -303,20 +314,26 @@ async fn play_audio( input.format = moq_audio::Format::F32; input.sample_rate = sample_rate; input.layout = layout; - input.latency = depth; + // The floor the sink opens on and pads an underflow back to, and what sizes + // its ring, so a fixed delay has to be it or the ring could not hold it. An + // estimated target moves, so it gets the floor, and `fit` steers the sink + // onto the target on every write. + input.latency = latency; let mut sink = speaker.sink(input.clone())?; + let mut dry = true; // One sample across every channel, the unit a write has to stay aligned to. let stride = channels as usize * size_of::(); - let chunk = ((AUDIO_CHUNK.as_secs_f64() * sample_rate as f64) as usize * stride).max(stride); + let samples = |duration: Duration| (duration.as_secs_f64() * sample_rate as f64).round() as u64; + let slack = samples(AUDIO_SLACK); + let silence = vec![0u8; samples(AUDIO_SILENCE).max(1) as usize * stride]; // The longest hole worth playing through, in samples. A hole this player would // rather sit through is one it is already willing to buffer, which is what the // decoder's latency budget says: anything longer is what that budget chose to // skip, so playing it as silence would hand back the delay the skip avoided. // Past it the sink skips the hole and the clock re-anchors, as it does today. - let fill_max = (consumer.max_age().as_secs_f64() * sample_rate as f64) as u64; - let silence = vec![0u8; chunk]; + let fill_max = samples(consumer.max_age()); let mut timeline = AudioTimeline::default(); @@ -344,9 +361,9 @@ async fn play_audio( }; dropping = false; - let samples = frame.data.len() / size_of::() / channels as usize; + let length = frame.data.len() / stride; let start = timestamp(frame.timestamp); - let timing = timeline.push(start, samples, sample_rate, fill_max); + let timing = timeline.push(start, length, sample_rate, fill_max); // A rewind or a hole too large to fill starts a new playback sink. The old // sink has no media clock, so its buffered audio cannot be carried across a @@ -355,35 +372,34 @@ async fn play_audio( drop(sink); presentation.lock().unwrap().restarted(); sink = speaker.sink(input.clone())?; + dry = true; } // A hole in the media is a hole in the audio, not a splice. Handing the next // frame straight to the speaker shortens the track by the missing duration, // which leaves it running ahead of media time until the clock below - // re-anchors, taking the video with it. Play the hole instead. - if timing.silence > 0 { - let mut remaining = usize::try_from(timing.silence) - .unwrap_or(usize::MAX / stride) - .saturating_mul(stride); - while remaining > 0 { - if let Some(excess) = sink.buffered().checked_sub(depth) { - tokio::time::sleep(excess).await; - } - let part = remaining.min(silence.len()); - sink.write(&silence[..part])?; - remaining -= part; - } - } - - for part in frame.data.chunks(chunk) { - // Let the speaker drain back to the target depth before topping it up. - // This is what paces the whole task: the device drains in real time, so - // the writes end up on the media clock and the sink holds the delay. - if let Some(excess) = sink.buffered().checked_sub(depth) { - tokio::time::sleep(excess).await; - } - sink.write(part)?; + // re-anchors, taking the video with it. So the hole goes in as silence ahead + // of the frame, and the pair is fitted to the target as one write. + let buffered = samples(sink.buffered()); + dry |= buffered == 0; + // Capped at the age budget: audio older than it is skipped rather than held, + // so a deeper target could never fill. The advertised floor needs the cap, + // being a number the publisher declared about itself, unbounded. The budget + // is also what the sink's ring was sized to hold. + let target = samples(consumer.delay().min(consumer.max_age()).max(AUDIO_BUFFER_MIN)); + let fit = fit(dry, buffered, target, slack, timing.silence + length as u64); + dry = false; + + // Playback drops stay on the live timeline; retrying them would add latency, + // and the sink already reports them in its logs. + let mut quiet = fit.pad + timing.silence.saturating_sub(fit.skip); + while quiet > 0 { + let part = (quiet as usize * stride).min(silence.len()); + let _ = sink.write(&silence[..part])?; + quiet -= (part / stride) as u64; } + let skip = fit.skip.saturating_sub(timing.silence) as usize * stride; + let _ = sink.write(&frame.data[skip.min(frame.data.len())..])?; // Anchor the playout clock on where the speaker has actually reached, which // is the only half of the pipeline that cannot skip ahead. A move has to @@ -403,12 +419,11 @@ async fn play_audio( } /// Play out what a retired sink still holds, instead of cutting the tail off -/// by dropping it. `latency` is the depth the sink was opened with. -async fn drain(sink: impl Sink, latency: Duration) { - // A write tops the ring up to its latency and then adds a chunk, so that sum - // is the deepest it can be when the track ends, and draining it takes exactly - // that long in real time. - let _ = tokio::time::timeout(latency + AUDIO_CHUNK + AUDIO_DRAIN_GRACE, sink.finish()).await; +/// by dropping it. +async fn drain(sink: impl Sink) { + // The estimated target moves, so the depth the sink was opened with says + // nothing about what it holds now: read it at retirement instead. + let _ = tokio::time::timeout(sink.buffered() + AUDIO_DRAIN_GRACE, sink.finish()).await; } #[cfg(test)] @@ -418,6 +433,7 @@ mod tests { use moq_mux::catalog::hang::Container; use super::*; + use crate::play::args::Delay; use crate::play::fake::Recorder; const SAMPLE_RATE: u32 = 48_000; @@ -463,7 +479,7 @@ mod tests { broadcast: "room".to_string(), args: Args { catalog_format: None, - delay: delay.into(), + delay: Delay::Fixed(delay), select: Default::default(), }, video: Default::default(), diff --git a/rs/moq-cli/src/play/timeline.rs b/rs/moq-cli/src/play/timeline.rs index 5082120f6c..53f5e41ece 100644 --- a/rs/moq-cli/src/play/timeline.rs +++ b/rs/moq-cli/src/play/timeline.rs @@ -194,6 +194,40 @@ impl AudioTimeline { } } +/// How to hand `incoming` samples to a speaker already holding `buffered`, so +/// it sits on `target`: the jitter buffer the browser's audio rings hold, in +/// sample frames. +/// +/// A `dry` ring, one that ran out or has only just opened, re-stalls: it pads up +/// to the target before playing on, rather than playing the next arrival on an +/// empty cushion. A ring more than +/// `slack` over the target lands back on it by skipping the oldest of what is +/// arriving. The slack is what a device period and a write's own length move the +/// level by, so ordinary cadence never skips. +pub(super) fn fit(dry: bool, buffered: u64, target: u64, slack: u64, incoming: u64) -> Fit { + let pad = if dry { + target.saturating_sub(buffered + incoming) + } else { + 0 + }; + let level = buffered + pad + incoming; + let skip = if level > target + slack { + (level - target).min(incoming) + } else { + 0 + }; + Fit { pad, skip } +} + +/// What [`fit`] decided for one write. +#[derive(Debug, PartialEq, Eq)] +pub(super) struct Fit { + /// Silence to write first. + pub(super) pad: u64, + /// Samples to drop from the front of the write. + pub(super) skip: u64, +} + #[cfg(test)] mod tests { use super::*; @@ -401,4 +435,34 @@ mod tests { assert!(!next.reset_sink); assert_eq!(next.silence, 0); } + + /// A dry ring refills to the target before the next arrival plays. + #[test] + fn a_dry_ring_restalls_to_the_target() { + assert_eq!(fit(true, 0, 4_800, 960, 960), Fit { pad: 3_840, skip: 0 }); + // A fresh sink starts on a little silence of its own, which counts. + assert_eq!(fit(true, 2_400, 4_800, 960, 960), Fit { pad: 1_440, skip: 0 }); + // A write longer than the target needs no padding, only the landing. + assert_eq!(fit(true, 0, 960, 960, 4_800), Fit { pad: 0, skip: 3_840 }); + } + + /// Ordinary cadence, a write on a ring already at the target, stays within + /// the slack and loses nothing. + #[test] + fn cadence_within_the_slack_plays_everything() { + assert_eq!(fit(false, 4_000, 4_800, 960, 960), Fit { pad: 0, skip: 0 }); + assert_eq!(fit(false, 4_800, 4_800, 960, 960), Fit { pad: 0, skip: 0 }); + // Running low is not running dry: nothing is padded until the ring is out. + assert_eq!(fit(false, 480, 4_800, 960, 960), Fit { pad: 0, skip: 0 }); + } + + /// A burst past the slack lands back on the target, and a ring already + /// over it drops the whole write. + #[test] + fn a_burst_lands_on_the_target() { + assert_eq!(fit(false, 4_800, 4_800, 960, 1_920), Fit { pad: 0, skip: 1_920 }); + assert_eq!(fit(false, 4_000, 4_800, 960, 4_800), Fit { pad: 0, skip: 4_000 }); + // A target that fell is reached by skipping, not by waiting to drain. + assert_eq!(fit(false, 9_600, 4_800, 960, 960), Fit { pad: 0, skip: 960 }); + } } diff --git a/rs/moq-cli/src/play/video.rs b/rs/moq-cli/src/play/video.rs index 9469c73fe5..79092ec387 100644 --- a/rs/moq-cli/src/play/video.rs +++ b/rs/moq-cli/src/play/video.rs @@ -225,7 +225,11 @@ impl Video { #[cfg(test)] mod tests { - use super::super::{args::Args, fake::Recorder, media::Media}; + use super::super::{ + args::{Args, Delay}, + fake::Recorder, + media::Media, + }; use super::*; use hang::moq_net; @@ -276,7 +280,7 @@ mod tests { origin: moq_tokio::origin::spawn().consume(), broadcast: "room".into(), args: Args { - delay: delay.into(), + delay: Delay::Fixed(delay), catalog_format: None, select: Default::default(), }, diff --git a/rs/moq-cli/src/play/window.rs b/rs/moq-cli/src/play/window.rs index 11ef44bbe3..850b3774f4 100644 --- a/rs/moq-cli/src/play/window.rs +++ b/rs/moq-cli/src/play/window.rs @@ -58,7 +58,7 @@ pub fn run( .context("failed to create the playback event loop")?; let proxy = event_loop.create_proxy(); let video = Arc::new(Mutex::new(VecDeque::new())); - let presentation = Arc::new(Mutex::new(Presentation::new(args.delay.into_std()))); + let presentation = Arc::new(Mutex::new(Presentation::new(args.video_delay()))); // Signals the decoder that the presenter took a frame, so it can hand over // the next one instead of dropping it. let drained = Arc::new(tokio::sync::Notify::new()); diff --git a/rs/moq-mux/src/codec/opus/mod.rs b/rs/moq-mux/src/codec/opus/mod.rs index 3565a11761..283b128ef0 100644 --- a/rs/moq-mux/src/codec/opus/mod.rs +++ b/rs/moq-mux/src/codec/opus/mod.rs @@ -178,8 +178,8 @@ impl Config { /// /// MPEG-TS aggregates several Opus packets into one PES, so the importer advances each /// packet's timestamp by this. Opus timing is always reckoned at 48 kHz regardless of the -/// encoder's internal bandwidth. Returns `None` for an empty packet or a code-3 packet -/// missing its frame-count byte. +/// encoder's internal bandwidth. Returns `None` for an empty packet, a code-3 packet +/// missing its frame-count byte, or a packet outside the 1 frame and 120 ms RFC 6716 allows. pub(crate) fn packet_samples(packet: &[u8]) -> Option { let toc = *packet.first()?; let frames = match toc & 0b11 { @@ -188,7 +188,9 @@ pub(crate) fn packet_samples(packet: &[u8]) -> Option { // Code 3: the frame count is the low 6 bits of the following byte. _ => (packet.get(1)? & 0b0011_1111) as u32, }; - Some(config_samples(toc >> 3) * frames) + let samples = config_samples(toc >> 3) * frames; + // RFC 6716 §3.2.5: a packet holds at most 120 ms of audio. + (frames > 0 && samples <= 5_760).then_some(samples) } /// 48 kHz samples per frame for an Opus TOC config index (0..=31), per RFC 6716 Table 1. @@ -226,6 +228,11 @@ mod tests { // config 1, code 3 with 4 frames -> 3840. assert_eq!(packet_samples(&[(1 << 3) | 3, 4]), Some(3840)); assert_eq!(packet_samples(&[]), None); + // Code 3 with zero frames, and with 63 frames of 60 ms. + assert_eq!(packet_samples(&[(1 << 3) | 3, 0]), None); + assert_eq!(packet_samples(&[(3 << 3) | 3, 63]), None); + // Two 60 ms frames is the 120 ms maximum. + assert_eq!(packet_samples(&[(3 << 3) | 3, 2]), Some(5760)); } #[test] diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 7e6633392a..d9f0164e81 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -87,8 +87,9 @@ on in nightly so a failing run preserves the inputs for diagnosis. To replay a capture, put its arrivals and final environment configuration into the version 2 `Trace` below, subtract the first arrival's `at` from every arrival and from the last sample's -`at` for the duration, and supply the observed minimum RTT. Replay holds that RTT and configuration -constant; compare the raw samples when either changed during the live run. The observer only +`at` for the duration, and supply the observed minimum RTT as a diagnostic. Replay measures the +target from the arrivals and holds the configuration constant; compare the raw samples when it +changed during the live run. The observer only records frames delivered under the player's maximum age, so a replay cannot recover groups the transport already discarded. @@ -112,9 +113,9 @@ on the delay, so the replay decides those again. `replay.ts` plays a trace through the player's own `Container.Consumer` and rings on a simulated clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a -real `Sync` resolves for the recorded catalog and round trip, and reads each quantum through the -tap's classifier. The consumer makes the player's group ordering, max age skips, and discontinuity -resets again at that delay; a group's stream is taken to finish with its last recorded frame. Every +real `Sync` resolves from the playout target that consumer measures, and reads each quantum through +the tap's classifier. The consumer makes the player's group ordering, max age skips, and +discontinuity resets again at that delay, and the ring follows the delay as it moves; a group's stream is taken to finish with its last recorded frame. Every frame is the trace's median spacing long, so a frame that never arrived stays missing audio, and rendering runs to the end of the observation, so an outage after the last arrival is heard. It writes the same samples a page does, so `analyze.ts` reduces both alike, and the shaper, transport, diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 3682697b25..955559dbde 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. Updated 2026-09-29: mild and wide now use shared order-preserving Gaussian jitter; bursty and step are new. Those 16 rows were re-recorded for 60 s each, seed 7, on the same shared Linux workstation with concurrent bounded builds. Their ceilings use twice the measurement rounded upward to two significant figures and the same floors above; target_ms_p95 adds one 20 ms bucket. Other rows retain their previous measurements. These remain current-player regression ceilings, not target audio quality; re-record on the nightly runner before tightening them. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-10-01, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in.", + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. Updated 2026-09-29: mild and wide now use shared order-preserving Gaussian jitter; bursty and step are new. Those 16 rows were re-recorded for 60 s each, seed 7, on the same shared Linux workstation with concurrent bounded builds. Their ceilings use twice the measurement rounded upward to two significant figures and the same floors above; target_ms_p95 adds one 20 ms bucket. Other rows retain their previous measurements. These remain current-player regression ceilings, not target audio quality; re-record on the nightly runner before tightening them. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-10-01, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in. Updated 2026-10-04: the replay rows now resolve \"auto\" from the measured arrival estimator instead of the round trip, and were re-measured exactly on the audio-jitter-target line; underruns fell to at most 2 a minute, while a run dry now re-stalls and refills to the target, which shows up as stalled and silent share.", "rows": [ { "runtime": "chromium", @@ -343,13 +343,13 @@ "rate": 44100, "profile": "local-aac", "ring": "isolated", - "underruns_per_min": 102.9, - "underrun_ms_per_min": 290.8, + "underruns_per_min": 0, + "underrun_ms_per_min": 0, "skip_aheads_per_min": 2, - "discarded_ms_per_min": 93.6, - "stalled_share": 0, - "silence_share": 0.005, - "target_ms_p95": 186 + "discarded_ms_per_min": 81.1, + "stalled_share": 0.025, + "silence_share": 0.025, + "target_ms_p95": 203.2 }, { "runtime": "replay", @@ -357,13 +357,13 @@ "rate": 44100, "profile": "local-aac", "ring": "plain", - "underruns_per_min": 102.9, - "underrun_ms_per_min": 290.8, - "skip_aheads_per_min": 2, - "discarded_ms_per_min": 93.6, - "stalled_share": 0, - "silence_share": 0.005, - "target_ms_p95": 186 + "underruns_per_min": 2, + "underrun_ms_per_min": 4.9, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0.017, + "silence_share": 0.015, + "target_ms_p95": 203.2 }, { "runtime": "replay", @@ -371,13 +371,13 @@ "rate": 44100, "profile": "relay-bbb", "ring": "isolated", - "underruns_per_min": 0, - "underrun_ms_per_min": 0, + "underruns_per_min": 2, + "underrun_ms_per_min": 0.5, "skip_aheads_per_min": 0, "discarded_ms_per_min": 0, - "stalled_share": 0, - "silence_share": 0, - "target_ms_p95": 289 + "stalled_share": 0.033, + "silence_share": 0.024, + "target_ms_p95": 223.2 }, { "runtime": "replay", @@ -387,11 +387,11 @@ "ring": "plain", "underruns_per_min": 0, "underrun_ms_per_min": 0, - "skip_aheads_per_min": 0, - "discarded_ms_per_min": 0, - "stalled_share": 0, - "silence_share": 0, - "target_ms_p95": 289 + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 87.9, + "stalled_share": 0.017, + "silence_share": 0.017, + "target_ms_p95": 223.2 }, { "runtime": "replay", @@ -399,13 +399,13 @@ "rate": 48000, "profile": "relay-mic", "ring": "isolated", - "underruns_per_min": 225.9, - "underrun_ms_per_min": 588.2, - "skip_aheads_per_min": 0, - "discarded_ms_per_min": 0, - "stalled_share": 0, - "silence_share": 0.01, - "target_ms_p95": 123 + "underruns_per_min": 2, + "underrun_ms_per_min": 2.7, + "skip_aheads_per_min": 4, + "discarded_ms_per_min": 510.8, + "stalled_share": 0.017, + "silence_share": 0.027, + "target_ms_p95": 420 }, { "runtime": "replay", @@ -413,13 +413,13 @@ "rate": 48000, "profile": "relay-mic", "ring": "plain", - "underruns_per_min": 225.9, - "underrun_ms_per_min": 588.2, - "skip_aheads_per_min": 0, - "discarded_ms_per_min": 0, - "stalled_share": 0, - "silence_share": 0.01, - "target_ms_p95": 123 + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 4, + "discarded_ms_per_min": 564.6, + "stalled_share": 0.017, + "silence_share": 0.027, + "target_ms_p95": 420 } ] } diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index 9b07265862..fde4eb1943 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -3,7 +3,7 @@ * `driver.ts` writes a browser row, so `analyze.ts` reduces both alike. * * The player half is `js/watch/src/audio/replay.ts`: the real container consumer and rings on a - * simulated clock, at the delay a real `Sync` resolves for the recorded catalog and RTT. This half + * simulated clock, at the delay a real `Sync` resolves from the playout target it measures. This half * reads its quanta through the same classifier the page's output tap runs, and samples the ring * every 250 ms of simulated time, as the probe samples a page. Every profile is the trace's name at * the "auto" delay a viewer gets by default. @@ -88,16 +88,11 @@ if (values.list) { const out = values.out as string; // Loaded only to play: listing runs before `run.sh` has installed the workspace the player needs. -const { replay, target } = await import("../../../../js/watch/src/audio/replay.ts"); +const { replay } = await import("../../../../js/watch/src/audio/replay.ts"); for (const { row, trace } of rows) { const tag = rowKey(row); const rate = row.rate; - const delay = await target({ - delay: "auto", - config: trace.config as unknown as Catalog.AudioConfig, - rtt: trace.rtt ?? undefined, - }); const ledger = new Ledger(rate, SILENCE_RMS); const samples: Sample[] = []; @@ -114,14 +109,14 @@ for (const { row, trace } of rows) { }; const arrivals = trace.arrivals.map(([at, timestamp, group]) => ({ at, timestamp, group })); - const sample = (quantum: { at: number; timestamp: number; stalled: boolean }) => { + const sample = (quantum: { at: number; timestamp: number; stalled: boolean; delay: number }) => { const gaps = ledger.take(); samples.push({ at: quantum.at, render: quantum.at, timestamp: quantum.timestamp, stalled: quantum.stalled, - delay, + delay: quantum.delay, quanta: ledger.counts.quanta, quiet: ledger.counts.quiet, gaps: gaps.length > 0 ? gaps : undefined, @@ -133,10 +128,11 @@ for (const { row, trace } of rows) { const quanta = replay(arrivals, { ring: row.ring === "isolated" ? "shared" : "post", rate, - delay, + delay: "auto", + config: trace.config as unknown as Catalog.AudioConfig, duration: trace.duration, }); - let last: { at: number; timestamp: number; stalled: boolean } | undefined; + let last: { at: number; timestamp: number; stalled: boolean; delay: number } | undefined; for await (const quantum of quanta) { // The render clock is the simulated one: the trace's `at`, which starts at zero. ledger.add(frame, quantum.output.length, classify([quantum.output])); @@ -145,7 +141,7 @@ for (const { row, trace } of rows) { stalled = quantum.stalled; stalls.push({ at: quantum.at, stalled }); } - last = { at: quantum.at, timestamp: quantum.timestamp, stalled: quantum.stalled }; + last = { at: quantum.at, timestamp: quantum.timestamp, stalled: quantum.stalled, delay: quantum.delay }; if (quantum.at >= next) { next += SAMPLE_INTERVAL_MS; @@ -167,5 +163,5 @@ for (const { row, trace } of rows) { }; await Bun.write(join(out, `${tag}.ndjson`), `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids: [] }, null, 1)); - console.log(`${tag}: ${samples.length} samples at a ${delay} ms target`); + console.log(`${tag}: ${samples.length} samples, settling at a ${Math.round(last?.delay ?? 0)} ms target`); }