From 36bc3b78e4cd6d7d1424f0ae7563c52c7492bf02 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 10:57:45 -0700 Subject: [PATCH 1/6] quest(audio-quality-harness/traces): claim Co-Authored-By: Claude Opus 5.5 From 59b94d5e77612e1a3b395b0248ae4d1b4971df8b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 11:27:33 -0700 Subject: [PATCH 2/6] test(audio-quality): record real arrival traces and grade their replay Records three arrival traces in headless Chromium (the public relay's bbb.hang, a local relay's AAC tone, and a browser publisher through the public relay), stamping each frame as it comes off its group's stream. The harness replays them through the player's real rings on a simulated clock at the delay Sync resolves, and grades them as exact replay rows. Co-Authored-By: Claude Opus 5.5 --- bun.lock | 4 + js/watch/src/audio/replay.test.ts | 54 + js/watch/src/audio/replay.ts | 168 ++ quest/m0/audio-jitter-target/watch.md | 15 +- quest/m0/audio-quality-harness/README.md | 11 +- quest/m0/audio-quality-harness/traces.md | 33 - test/audio-quality/README.md | 55 +- test/audio-quality/budgets.json | 86 +- test/audio-quality/clients/js/mic.html | 10 + test/audio-quality/clients/js/package.json | 4 + test/audio-quality/clients/js/record.ts | 140 ++ test/audio-quality/clients/js/recorder.html | 10 + test/audio-quality/clients/js/replay.ts | 136 ++ test/audio-quality/clients/js/src/analyze.ts | 5 +- test/audio-quality/clients/js/src/mic.ts | 27 + test/audio-quality/clients/js/src/recorder.ts | 130 ++ test/audio-quality/clients/js/src/schema.ts | 52 +- test/audio-quality/clients/js/tsconfig.json | 11 +- test/audio-quality/clients/js/vite.config.ts | 11 +- test/audio-quality/record.sh | 107 + test/audio-quality/run.sh | 48 +- test/audio-quality/traces/local-aac.json | 1566 +++++++++++++ test/audio-quality/traces/relay-bbb.json | 1954 +++++++++++++++++ test/audio-quality/traces/relay-mic.json | 1763 +++++++++++++++ test/justfile | 8 + 25 files changed, 6341 insertions(+), 67 deletions(-) create mode 100644 js/watch/src/audio/replay.test.ts create mode 100644 js/watch/src/audio/replay.ts delete mode 100644 quest/m0/audio-quality-harness/traces.md create mode 100644 test/audio-quality/clients/js/mic.html create mode 100644 test/audio-quality/clients/js/record.ts create mode 100644 test/audio-quality/clients/js/recorder.html create mode 100644 test/audio-quality/clients/js/replay.ts create mode 100644 test/audio-quality/clients/js/src/mic.ts create mode 100644 test/audio-quality/clients/js/src/recorder.ts create mode 100755 test/audio-quality/record.sh create mode 100644 test/audio-quality/traces/local-aac.json create mode 100644 test/audio-quality/traces/relay-bbb.json create mode 100644 test/audio-quality/traces/relay-mic.json diff --git a/bun.lock b/bun.lock index 7ce6980b9c..1968391f8d 100644 --- a/bun.lock +++ b/bun.lock @@ -346,6 +346,10 @@ "test/audio-quality/clients/js": { "name": "@moq/audio-quality", "dependencies": { + "@moq/hang": "workspace:*", + "@moq/net": "workspace:*", + "@moq/publish": "workspace:*", + "@moq/signals": "workspace:*", "@moq/watch": "workspace:*", }, "devDependencies": { diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts new file mode 100644 index 0000000000..109019ffea --- /dev/null +++ b/js/watch/src/audio/replay.test.ts @@ -0,0 +1,54 @@ +import { describe, expect, it } from "bun:test"; +import type * as Catalog from "@moq/hang/catalog"; +import { Time } from "@moq/net"; +import { type Arrival, type Options, QUANTUM, replay, target } from "./replay"; + +const RATE = 48000; + +/** Twenty ms frames for `seconds`, each arriving `lateness(i)` ms after it was captured. */ +function paced(seconds: number, lateness: (i: number) => number): Arrival[] { + return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20 })); +} + +/** Quanta that came back short or empty while playing, once the ring had first played. */ +function underruns(trace: Arrival[], options: Options): number { + let started = false; + let short = 0; + for (const { output, stalled } of replay(trace, options)) { + const filled = output.findLastIndex((v) => v !== 0) + 1; + if (started && !stalled && filled < QUANTUM) short++; + if (filled > 0) started = true; + } + return short; +} + +describe.each(["shared", "post"] as const)("%s ring", (ring) => { + it("plays an evenly paced sender without a gap", () => { + expect( + underruns( + paced(10, () => 30), + { ring, rate: RATE, delay: 100 }, + ), + ).toBe(0); + }); + + it("underruns when a flush span outlasts the target, and not when it is covered", () => { + // Five frames held and flushed at once: 80 ms of arrival spread. + const trace = paced(10, (i) => 30 + (4 - (i % 5)) * 20); + expect(underruns(trace, { ring, rate: RATE, delay: 40 })).toBeGreaterThan(50); + expect(underruns(trace, { ring, rate: RATE, delay: 150 })).toBe(0); + }); +}); + +describe("target", () => { + const config = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; + + it("sizes auto from the RTT and the rendition's jitter", async () => { + // 1.25 x 40 ms of RTT, plus the Opus frame and a render quantum. + expect(await target({ delay: "auto", config, rtt: 40 })).toBe(Time.Milli(50 + 20 + 3)); + }); + + it("adds the rendition's jitter to a fixed delay", async () => { + expect(await target({ delay: Time.Milli(250), config })).toBe(Time.Milli(250 + 20 + 3)); + }); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts new file mode 100644 index 0000000000..c314a57a28 --- /dev/null +++ b/js/watch/src/audio/replay.ts @@ -0,0 +1,168 @@ +/** + * Replay a recorded arrival trace through the player's real rings on a simulated clock. + * + * Every frame is inserted at the instant the trace says it reached the container consumer, and one + * render quantum is read every quantum's worth of that same clock, which is what the AudioWorklet + * does. {@link target} resolves the delay with a real {@link Sync} fed the recorded catalog config + * and connection RTT, and the ring is sized from it the way the decoder sizes it, so a change to + * either moves what a replay hears. Decoding is taken as instant and sample exact. + * + * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test + * can assert exact counts. + * + * @internal Test support, not part of the player. + */ +import type * as Catalog from "@moq/hang/catalog"; +import { Time } from "@moq/net"; +import { Signal } from "@moq/signals"; +import { type Delay, Sync } from "../sync"; +import { playbackJitter } from "./config"; +import { ringSamples } from "./latency"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; + +/** An AudioWorklet render quantum, in frames. */ +export const QUANTUM = 128; + +/** A media gap longer than this is missing audio, not one long frame. */ +const MAX_FRAME_MS = 100; + +/** One frame reaching the container consumer. */ +export interface Arrival { + /** When it arrived, on the viewer's monotonic clock, in ms. */ + at: number; + /** Its media timestamp, in ms. */ + timestamp: number; +} + +/** What the player resolves its delay from. */ +export interface Target { + /** The configured delay, as the element takes it. */ + delay: Delay; + /** The rendition as the catalog described it: its advertised jitter. */ + config: Catalog.AudioConfig; + /** The connection's round trip, in ms, which "auto" sizes from. Unset is no PROBE. */ + rtt?: number; +} + +/** The delay a player resolves for `input`, in ms, from a real {@link Sync}. */ +export async function target(input: Target): Promise { + const sync = new Sync({ + delay: input.delay, + probe: input.rtt === undefined ? undefined : { rtt: Time.Milli(input.rtt) }, + }); + const unregister = sync.register(new Signal(playbackJitter(input.config))); + try { + // Effects flush on a microtask, and a timer runs after all of them. + await new Promise((resolve) => setTimeout(resolve, 0)); + return sync.out.delay.peek(); + } finally { + unregister(); + sync.close(); + } +} + +/** What a replay plays through. */ +export interface Options { + /** `shared` is the SharedArrayBuffer ring a cross-origin isolated page runs; `post` is the rest. */ + ring: "shared" | "post"; + /** The sample rate, in Hz. */ + rate: number; + /** The resolved delay the ring is sized to, in ms. See {@link target}. */ + delay: number; +} + +/** One render quantum, as the worklet produced it. */ +export interface Quantum { + /** When it finished rendering, on the trace's clock, in ms. */ + at: number; + /** The quantum: what the ring supplied at the front, zeros after it. Reused between yields. */ + output: Float32Array; + /** Whether the ring was stalled after this quantum. */ + stalled: boolean; + /** The playhead after this quantum, on the media clock, in ms. */ + timestamp: number; +} + +/** The two rings behind the calls the decoder and the worklet make on them. */ +interface Ring { + insert(timestamp: Time.Micro, data: Float32Array[]): void; + read(output: Float32Array[]): number; + readonly stalled: boolean; + readonly timestamp: Time.Micro; +} + +/** The shared ring, sized the way `SharedAudioBuffer` sizes it. */ +function shared(rate: number, latency: number): Ring { + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(rate, latency * 2), rate)); + ring.setLatency(latency); + return ring; +} + +/** The postMessage ring, sized the way `PostAudioBuffer` asks the worklet to size it. */ +function post(rate: number, latency: number): Ring { + const ms = Time.Milli.fromSecond((latency / rate) as Time.Second); + const ring = new AudioRingBuffer({ rate, channels: 1, latency: ms }); + return { + insert: (timestamp, data) => ring.write(timestamp, data), + read: (output) => ring.read(output), + get stalled() { + return ring.stalled; + }, + get timestamp() { + return ring.timestamp; + }, + }; +} + +/** + * Samples each frame carries: up to where the next one starts, so consecutive frames tile the + * timeline exactly. A frame followed by a hole, or by nothing, gets the trace's typical frame. + */ +function frameSamples(trace: Arrival[], rate: number): number[] { + const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b); + const sample = (ms: number) => Math.round((ms * rate) / 1000); + const spans = new Map(); + for (let i = 0; i < starts.length - 1; i++) { + if (starts[i + 1] - starts[i] < MAX_FRAME_MS) spans.set(starts[i], sample(starts[i + 1]) - sample(starts[i])); + } + const sorted = [...spans.values()].sort((a, b) => a - b); + const typical = sorted[Math.floor(sorted.length / 2)]; + if (typical === undefined) throw new Error("a trace needs two frames less than 100 ms apart"); + return trace.map((a) => spans.get(a.timestamp) ?? typical); +} + +/** + * Play `trace` in simulated real time, yielding every render quantum from the first arrival to the + * last. + * + * The audio is a constant, never zero, so what the ring did not supply is exactly the zeros at the + * end of a quantum, which is how the harness's output tap reads a real page. + */ +export function* replay(trace: Arrival[], options: Options): Generator { + const first = trace[0]; + const last = trace.at(-1); + if (!first || !last) return; + + const { rate } = options; + const samples = frameSamples(trace, rate); + const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, Time.Milli(options.delay))); + + const step = (QUANTUM / rate) * 1000; + const output = new Float32Array(QUANTUM); + let next = 0; + + for (let n = 1; first.at + (n - 1) * step <= last.at; n++) { + // The quantum ending at `now` renders once everything that arrived by then is in. + const now = first.at + n * step; + while (next < trace.length && trace[next].at <= now) { + const { timestamp } = trace[next]; + ring.insert(Time.Micro.fromMilli(timestamp as Time.Milli), [new Float32Array(samples[next]).fill(0.5)]); + next++; + } + + output.fill(0); + ring.read([output]); + yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp) }; + } +} diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index 75d43a4203..2b04c4c010 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -74,11 +74,12 @@ against it is likely cheaper than patching the branch's: came up on the 100 ms chip, so something is restoring or overriding it. Pin that down: a stored preference silently winning over the default is its own bug, and it also means auto gets far less real exposure than it looks like. -- Replay recorded traces rather than synthetic ones of the same shape. The - #3477 traces are gone, so record fresh ones with the [audio quality - harness](/quest/m0/audio-quality-harness/traces.md) (decided with the - maintainer instead of asking the reporter). Trim a copy into the repository - and replay it through both rings in `replay.test.ts`. +- Replay recorded traces rather than synthetic ones of the same shape. + `test/audio-quality/traces/` holds fresh ones (the #3477 traces are gone), + and `js/watch/src/audio/replay.ts` already plays a trace through both rings + on a simulated clock at the delay `Sync` resolves. Replay them in + `replay.test.ts`, move the replay's target onto the estimator, and tighten + the harness's exact replay budgets with it. - Manual run against the public relay on Chrome and Safari, the two rows the issue measured. Measure the publisher's audio encoder input-to-output lag in the same run using the reporter's instrumented harness; #3518 fixed the known @@ -95,10 +96,6 @@ reading `probe`; removing it is part of the `SyncInput` reshape in [Plan: A/V clock](/quest/m0/plan-av-clock.md). Land the estimator so that quest can adopt it without a second estimator change. -## Required - -- [Traces](/quest/m0/audio-quality-harness/traces.md) - records the arrival traces this quest replays - ## Related - [Plan: A/V clock](/quest/m0/plan-av-clock.md) - reshapes `SyncInput` around the per-track spread this quest produces diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index dfc3b6150c..9656a4a84a 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -25,11 +25,11 @@ rather than merely both passing. The browser lane has landed as `test/audio-quality/`, upstreamed from the reporter's fork on #3477 (`fperex/moq`, branch `debug-findings-solution`) on -its own, without the fork's player, estimator, or shaper changes. That leaves -the two things that needed them: real arrival traces, which the [jitter -target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays, and the -bursty and step profiles, which need shaper features the fork has and `main` -does not. +its own, without the fork's player, estimator, or shaper changes. Real arrival +traces have landed beside it in `test/audio-quality/traces/`, graded as replay +rows and replayed by the [jitter target's watch +quest](/quest/m0/audio-jitter-target/watch.md). That leaves the bursty and step +profiles, which need shaper features the fork has and `main` does not. Jitter comes from the seeded userspace UDP shaper the transport drills run under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake @@ -54,7 +54,6 @@ quest](/quest/m0/audio-jitter-target/watch.md). ## Required -- [Traces](/quest/m0/audio-quality-harness/traces.md) - real arrival traces recorded, checked in, and graded as replay rows - [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/traces.md b/quest/m0/audio-quality-harness/traces.md deleted file mode 100644 index ce0b749962..0000000000 --- a/quest/m0/audio-quality-harness/traces.md +++ /dev/null @@ -1,33 +0,0 @@ -# [M] Real arrival traces are recorded, checked in, and graded - -## Goal - -A trimmed fixture of real audio arrival traces lives in the repository, and the -audio quality harness grades a replay of it next to its synthetic profiles. The -[jitter target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays -the same fixture through both rings in `replay.test.ts`. - -## Plan - -The #3477 traces are gone, so record fresh ones: a viewer on the public relay -playing `bbb.hang` (the AAC flush-span shape the issue measured), a local relay -as the shallow control, and a browser microphone published through the public -relay. Stamp each audio frame's arrival where the container consumer receives -it, with its media timestamp and group, on the viewer's monotonic clock (see -`test/audio-quality/clients/js/src/schema.ts`). - -The browser lane reads only public signals, and no public signal carries a -per-frame arrival, so decide where the recorder lives: a debug hook in -`@moq/watch` that the harness page enables, or a native subscriber (`moq` -CLI) that records the same path's arrivals. The reporter's fork (`fperex/moq`, -branch `debug-findings-solution`) recorded from a debug sampler in the player -and replayed through `js/watch/src/audio/replay.ts` on a simulated clock; -that replay module is the likely shape for the grading side, and landing it is -a player change this quest owns. - -Replay rows are deterministic, so their budgets are exactly what was measured, -with no headroom. - -## Related - -- [Latency ledger](/quest/m2/latency-ledger.md) - would promote an arrival stamp into public API diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 242660731f..3ebb3f483d 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -12,11 +12,13 @@ just test audio-quality --profiles mild --codecs opus --duration 20 just test audio-quality --rings plain # the production path only just test audio-quality --out ~/runs/after # keep the run directory just test audio-quality --enforce # fail on the budgets +just test audio-quality --profiles relay-bbb,local-aac,relay-mic # the replays only, in seconds ``` -The matrix is codec x profile x ring: 16 rows, about 20 minutes. `--profiles`, `--rings`, and -`--codecs` take comma-separated lists; `--seed` replays a given impairment. The table prints either -way, and only `--enforce` (which the nightly job passes) turns a breach into a failed run. +The matrix is codec x profile x ring: 16 browser rows, about 20 minutes, plus 6 [replay +rows](#traces) of recorded arrivals. `--profiles`, `--rings`, and `--codecs` take comma-separated +lists; `--seed` replays a given impairment. The table prints either way, and only `--enforce` (which +the nightly job passes) turns a breach into a failed run. ## Rows @@ -66,6 +68,41 @@ dials, so the driver answers that fetch with the relay's own hash (the certifica hash, so a different port needs nothing more). With nothing listening on TCP there, the WebSocket fallback cannot connect: the session is WebTransport through the shaper, or the row is void. +## Traces + +The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, each graded as a +`replay` row whose profile is the file's name: + +| Trace | Recorded from | +| --- | --- | +| `relay-bbb` | the public relay's `bbb.hang`: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured | +| `local-aac` | this harness's AAC publisher through a local relay: the shallow control | +| `relay-mic` | `` in headless Chromium, publishing its fake capture device through the public relay | + +A trace is a [`Trace`](clients/js/src/schema.ts): the catalog's audio rendition, the smallest +PROBE round trip the session saw, and every frame's `[at, timestamp, group]`, with `at` on the +viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, the way +`Container.Consumer` receives it, using only public `@moq/net` and `@moq/hang` in the same Chromium +over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend +on the delay, so the replay decides those again. + +`replay.ts` plays a trace through the player's own rings on a simulated clock +([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a real +`Sync` resolves for the recorded catalog and round trip, and reads each quantum through the tap's +classifier. It writes the same samples a page does, so `analyze.ts` reduces both alike, and the +shaper, transport, and clock checks have nothing to void. Decoding is taken as instant. A replay is +deterministic, so its budgets are exactly what it measured. + +```bash +just test audio-quality-record # all three, 35 s each, needs the network +just test audio-quality-record --lanes relay-bbb +``` + +Recording is by hand: the public relay and a browser publisher are not something a nightly run +should depend on. Re-measure the replay budgets after re-recording. The recorder asks the relay for +ten seconds of backlog so a late group is kept, which makes a trace's tune-in heavier than a +player's; the warmup excludes it from the grade. + ## The metric contract [`clients/js/src/schema.ts`](clients/js/src/schema.ts) is the contract. The page emits `Sample`s, @@ -130,7 +167,8 @@ holds by construction. ## Void rules -A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. +A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. A +replay row crosses no shaper, so the `shaper` void does not apply to it. | Void | Why | | --- | --- | @@ -155,8 +193,10 @@ for a week: each process's log, the shaper's counters, every row's raw samples a ```text run.sh builds, starts the relay and publishers, runs each row behind the shaper +record.sh records traces/, by hand relay.toml anonymous relay, self-signed cert budgets.json a ceiling per metric per row +traces/ recorded arrivals, one replay profile each clients/js/ index.html src/page.ts the watch element, driven from the query string src/probe.ts samples its public signals every 250 ms @@ -164,6 +204,9 @@ clients/js/ src/schema.ts the metric contract src/analyze.ts samples to a summary, unit tested driver.ts one row in headless Chromium, and its void checks + recorder.html mic.html the recorder and the microphone publisher (src/recorder.ts, src/mic.ts) + record.ts one trace in headless Chromium + replay.ts the replay rows, written as a page's samples analyze.ts grade.ts the summary per row, then the table and the verdict ``` @@ -179,5 +222,5 @@ the profiles are the ones the shaper on `main` can express. ## Not covered Safari and Firefox, video, and perceptual scoring: the grade is glitches and latency, not an opinion -about how it sounds. Recorded arrival traces are -[their own quest](../../quest/m0/audio-quality-harness/traces.md). +about how it sounds. The `relay-mic` trace captures Chromium's fake device rather than a real +microphone, so it carries a browser publisher's encode and send cadence but not a sound card's. diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 0cdde6b25c..9d8407e985 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs.", + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-09-29, with no headroom: any regression fails them, and an improvement should be tightened in.", "rows": [ { "runtime": "chromium", @@ -224,6 +224,90 @@ "stalled_share": 0.49, "silence_share": 0.75, "target_ms_p95": 82 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "local-aac", + "ring": "isolated", + "underruns_per_min": 101.7, + "underrun_ms_per_min": 287.3, + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 94.4, + "stalled_share": 0, + "silence_share": 0.004, + "target_ms_p95": 186 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "local-aac", + "ring": "plain", + "underruns_per_min": 101.7, + "underrun_ms_per_min": 287.3, + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 94.4, + "stalled_share": 0, + "silence_share": 0.004, + "target_ms_p95": 186 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "relay-bbb", + "ring": "isolated", + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0, + "target_ms_p95": 289 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "relay-bbb", + "ring": "plain", + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0, + "target_ms_p95": 289 + }, + { + "runtime": "replay", + "codec": "opus", + "rate": 48000, + "profile": "relay-mic", + "ring": "isolated", + "underruns_per_min": 227.8, + "underrun_ms_per_min": 593.2, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0.01, + "target_ms_p95": 123 + }, + { + "runtime": "replay", + "codec": "opus", + "rate": 48000, + "profile": "relay-mic", + "ring": "plain", + "underruns_per_min": 227.8, + "underrun_ms_per_min": 593.2, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0.01, + "target_ms_p95": 123 } ] } diff --git a/test/audio-quality/clients/js/mic.html b/test/audio-quality/clients/js/mic.html new file mode 100644 index 0000000000..43c0cb2d81 --- /dev/null +++ b/test/audio-quality/clients/js/mic.html @@ -0,0 +1,10 @@ + + + + + moq audio quality microphone + + + + + diff --git a/test/audio-quality/clients/js/package.json b/test/audio-quality/clients/js/package.json index 9534e140ed..dd2918b166 100644 --- a/test/audio-quality/clients/js/package.json +++ b/test/audio-quality/clients/js/package.json @@ -7,6 +7,10 @@ "test": "bun test --only-failures" }, "dependencies": { + "@moq/hang": "workspace:*", + "@moq/net": "workspace:*", + "@moq/publish": "workspace:*", + "@moq/signals": "workspace:*", "@moq/watch": "workspace:*" }, "devDependencies": { diff --git a/test/audio-quality/clients/js/record.ts b/test/audio-quality/clients/js/record.ts new file mode 100644 index 0000000000..b374c905f9 --- /dev/null +++ b/test/audio-quality/clients/js/record.ts @@ -0,0 +1,140 @@ +/** + * Records one arrival trace in headless Chromium, and writes it as a checked-in `Trace`. + * + * The recorder page subscribes to the broadcast's audio and stamps every frame the container consumer + * hands over. With `--mic`, a second page first publishes Chromium's fake capture device through + * `` to the same relay and broadcast, so the trace is a browser publisher's cadence. + * + * bun record.ts --page dist --url https://cdn.moq.dev/demo --broadcast bbb.hang --duration 35 \ + * --source "..." --description "..." --out ../../traces/relay-bbb.json + * + * The recording starts at the first arrival, tune-in burst included, since a player tunes in too. + * + * @module + */ +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { launch, sleep } from "../../../interop/clients/js/harness.ts"; +import type { Arrival, Trace } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + page: { type: "string" }, + url: { type: "string" }, + broadcast: { type: "string" }, + duration: { type: "string", default: "35" }, + source: { type: "string" }, + description: { type: "string" }, + mic: { type: "boolean", default: false }, + out: { type: "string" }, + }, +}); + +const { page: pageDir, url, broadcast, source, description, out } = values; +const durationMs = Number.parseFloat(values.duration) * 1000; +if (!pageDir || !url || !broadcast || !source || !description || !out || !(durationMs > 0)) { + console.error( + "usage: record.ts --page DIR --url U --broadcast B --source S --description D --out FILE [--duration S] [--mic]", + ); + process.exit(2); +} + +/** How long the session and the first arrival get before the recording is abandoned. */ +const STARTUP_MS = 30_000; + +// Cross-origin isolated, for the 5 us `performance.now()` a non-isolated page coarsens to 100 us. +const root = resolve(pageDir); +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const file = Bun.file(join(root, new URL(req.url).pathname)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { + headers: { "cross-origin-opener-policy": "same-origin", "cross-origin-embedder-policy": "require-corp" }, + }); + }, +}); +const pageUrl = (file: string) => `http://127.0.0.1:${server.port}/${file}?${new URLSearchParams({ url, broadcast })}`; + +/** Poll the page until `ready` returns a value, or throw naming what never happened. */ +async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { + const deadline = Date.now() + STARTUP_MS; + while (Date.now() < deadline) { + const error = await page.evaluate(() => globalThis.recorder?.error()); + if (error) throw new Error(`recorder: ${error}`); + const value = await page.evaluate(ready); + if (value !== undefined && value !== null) return value; + await sleep(200); + } + throw new Error(`${what} not within ${STARTUP_MS / 1000}s`); +} + +const browser = await launch(["--use-fake-device-for-media-stream", "--use-fake-ui-for-media-stream"]); +let failed = false; +try { + const context = await browser.newContext({ permissions: ["microphone", "camera"] }); + if (values.mic) { + const publisher = await context.newPage(); + publisher.on("pageerror", (error) => console.error(`[mic error] ${error.message}`)); + await publisher.goto(pageUrl("mic.html"), { waitUntil: "load" }); + // Subscribing before the element has an audio rendition finds no catalog to read. + const deadline = Date.now() + STARTUP_MS; + while (!(await publisher.evaluate(() => document.querySelector("moq-publish")?.audio.out.catalog.peek()))) { + if (Date.now() > deadline) throw new Error(`the microphone never published within ${STARTUP_MS / 1000}s`); + await sleep(200); + } + await sleep(2000); + } + + const page = await context.newPage(); + page.on("pageerror", (error) => console.error(`[recorder error] ${error.message}`)); + page.on("console", (msg) => { + if (msg.type() === "error" || msg.type() === "warning") console.error(`[recorder ${msg.type()}] ${msg.text()}`); + }); + await page.goto(pageUrl("recorder.html"), { waitUntil: "load" }); + + const info = await waitFor(page, "an audio subscription", () => globalThis.recorder.info()); + const arrivals: Arrival[] = []; + await waitFor(page, "a first arrival", () => { + const drained = globalThis.recorder.drain(); + return drained.length > 0 ? drained : undefined; + }).then((first) => arrivals.push(...first)); + + const start = arrivals[0][0]; + while ((arrivals.at(-1)?.[0] ?? start) < start + durationMs) { + await sleep(1000); + const error = await page.evaluate(() => globalThis.recorder.error()); + if (error) throw new Error(`recorder: ${error}`); + arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); + } + // The RTT is read last: the PROBE minimum only falls as the session runs. + const rtt = await page.evaluate(() => globalThis.recorder.info()?.rtt ?? null); + + const trimmed = arrivals.filter(([at]) => at - start <= durationMs); + const round = (x: number, places: number) => Math.round(x * 10 ** places) / 10 ** places; + const trace: Trace = { + version: 1, + source, + description, + transport: info.transport, + rtt: rtt === null ? null : round(rtt, 1), + config: info.config as Trace["config"], + arrivals: trimmed.map(([at, timestamp, group]) => [round(at - start, 2), round(timestamp, 3), group]), + }; + // One arrival per line, so the file reads as a table. + const { arrivals: rows, ...header } = trace; + const head = JSON.stringify(header, null, "\t").slice(0, -2); + await Bun.write(out, `${head},\n\t"arrivals": [\n${rows.map((r) => `\t\t${JSON.stringify(r)}`).join(",\n")}\n\t]\n}\n`); + console.log( + `${out}: ${trace.arrivals.length} arrivals over ${round(durationMs / 1000, 1)} s, ${trace.config.codec} at ${trace.config.sampleRate} Hz, ${trace.transport}, rtt ${trace.rtt} ms`, + ); +} catch (err) { + failed = true; + console.error(err instanceof Error ? err.message : String(err)); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} +process.exit(failed ? 1 : 0); diff --git a/test/audio-quality/clients/js/recorder.html b/test/audio-quality/clients/js/recorder.html new file mode 100644 index 0000000000..f3185f8b36 --- /dev/null +++ b/test/audio-quality/clients/js/recorder.html @@ -0,0 +1,10 @@ + + + + + moq audio quality recorder + + + + + diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts new file mode 100644 index 0000000000..328143751c --- /dev/null +++ b/test/audio-quality/clients/js/replay.ts @@ -0,0 +1,136 @@ +/** + * Replays the recorded traces through the player's rings, and writes each row the way `driver.ts` + * writes a browser row, so `analyze.ts` reduces both alike. + * + * The player half is `js/watch/src/audio/replay.ts`: the real rings on a simulated clock, sized to + * the delay a real `Sync` resolves for the recorded catalog and RTT. This half reads its quanta + * through the same classifier the page's output tap runs, and samples the ring every 250 ms of + * simulated time, as the probe samples a page. Every profile is the trace's name at the "auto" + * delay a viewer gets by default. + * + * bun replay.ts --traces ../../traces --profiles relay-bbb --rings plain --codecs aac --list + * bun replay.ts --traces ../../traces --out + * + * `--list` prints the row tags; otherwise it writes `.ndjson` and `.page.json` per row. + * + * @module + */ +import { readdirSync, readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { parseArgs } from "node:util"; +import type * as Catalog from "@moq/hang/catalog"; +import { classify, Ledger } from "./src/quantum.ts"; +import { + type Environment, + type Ring, + type Row, + rowKey, + SAMPLE_INTERVAL_MS, + type Sample, + SILENCE_RMS, + type Stall, + type Trace, + traceCodec, +} from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + traces: { type: "string" }, + profiles: { type: "string" }, + rings: { type: "string", default: "isolated,plain" }, + codecs: { type: "string", default: "opus,aac" }, + out: { type: "string" }, + list: { type: "boolean", default: false }, + }, +}); +if (!values.traces || (!values.list && !values.out)) { + console.error("usage: replay.ts --traces DIR [--profiles a,b] [--rings r] [--codecs c] (--list | --out DIR)"); + process.exit(2); +} +const dir = values.traces; +const split = (list: string) => list.split(",").filter((s) => s !== ""); + +const traces = readdirSync(dir) + .filter((f) => f.endsWith(".json")) + .sort() + .map((f) => ({ profile: basename(f, ".json"), trace: JSON.parse(readFileSync(join(dir, f), "utf8")) as Trace })); +const profiles = values.profiles === undefined ? undefined : split(values.profiles); +const rings = split(values.rings) as Ring[]; +const codecs = split(values.codecs); + +const rows: { row: Row; trace: Trace }[] = []; +for (const { profile, trace } of traces) { + if (trace.version !== 1) throw new Error(`${profile}: trace version ${trace.version}, expected 1`); + const codec = traceCodec(trace); + if (profiles && !profiles.includes(profile)) continue; + if (!codecs.includes(codec)) continue; + for (const ring of rings) { + rows.push({ row: { runtime: "replay", codec, rate: trace.config.sampleRate, profile, ring }, trace }); + } +} + +if (values.list) { + for (const { row } of rows) console.log(rowKey(row)); + process.exit(0); +} +const out = values.out as string; + +// Loaded only to play: listing runs before `run.sh` has installed the workspace the player needs. +const { replay, target } = await import("../../../../js/watch/src/audio/replay.ts"); + +for (const { row, trace } of rows) { + const tag = rowKey(row); + const rate = row.rate; + const delay = await target({ + delay: "auto", + config: trace.config as unknown as Catalog.AudioConfig, + rtt: trace.rtt ?? undefined, + }); + + const ledger = new Ledger(rate, SILENCE_RMS); + const samples: Sample[] = []; + let stalls: Stall[] = []; + let stalled = true; + let frame = 0; + let next = SAMPLE_INTERVAL_MS; + + const arrivals = trace.arrivals.map(([at, timestamp]) => ({ at, timestamp })); + for (const quantum of replay(arrivals, { ring: row.ring === "isolated" ? "shared" : "post", rate, delay })) { + // The render clock is the simulated one: the trace's `at`, which starts at zero. + ledger.add(frame, quantum.output.length, classify([quantum.output])); + frame += quantum.output.length; + if (quantum.stalled !== stalled) { + stalled = quantum.stalled; + stalls.push({ at: quantum.at, stalled }); + } + + if (quantum.at >= next) { + next += SAMPLE_INTERVAL_MS; + const gaps = ledger.take(); + samples.push({ + at: quantum.at, + render: quantum.at, + timestamp: quantum.timestamp, + stalled: quantum.stalled, + delay, + quanta: ledger.counts.quanta, + quiet: ledger.counts.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: stalls.length > 0 ? stalls : undefined, + }); + stalls = []; + } + } + + const environment: Environment = { + crossOriginIsolated: row.ring === "isolated", + transport: trace.transport, + codec: trace.config.codec, + rate, + jitter: trace.config.jitter, + contextRate: rate, + }; + await Bun.write(join(out, `${tag}.ndjson`), `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); + await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes: [], voids: [] }, null, 1)); + console.log(`${tag}: ${samples.length} samples at a ${delay} ms target`); +} diff --git a/test/audio-quality/clients/js/src/analyze.ts b/test/audio-quality/clients/js/src/analyze.ts index 9ee7d6c62c..84d398b6bd 100644 --- a/test/audio-quality/clients/js/src/analyze.ts +++ b/test/audio-quality/clients/js/src/analyze.ts @@ -142,8 +142,11 @@ export function analyze(input: Input): Summary { const notes = [...input.notes]; // ── the shaper's verdict ──────────────────────────────────────────────── + // A replay row never crosses one: its path is the recording. const shaper = input.shaper; - if (!shaper) { + if (row.runtime === "replay") { + // Nothing to check. + } else if (!shaper) { voids.push({ assertion: "shaper", detail: "left no report, so nothing says the path was shaped" }); } else if (shaper.status !== 0) { voids.push({ assertion: "shaper", detail: `exited ${shaper.status}: the profile never acted as configured` }); diff --git a/test/audio-quality/clients/js/src/mic.ts b/test/audio-quality/clients/js/src/mic.ts new file mode 100644 index 0000000000..ae536caa9c --- /dev/null +++ b/test/audio-quality/clients/js/src/mic.ts @@ -0,0 +1,27 @@ +/** + * The publisher page for the microphone trace: the demo's `` capturing audio only. + * + * Whatever device Chromium offers is captured through the element's real path (getUserMedia, the + * capture worklet, WebCodecs Opus), so the trace carries a browser publisher's own cadence. Under + * `record.ts` that device is Chromium's fake one, a tone. + * + * ?url=https://cdn.moq.dev/anon&broadcast=aq-mic-1234.hang + * + * @module + */ +import "@moq/publish/element"; + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const publish = document.createElement("moq-publish"); +publish.setAttribute("url", required("url")); +publish.setAttribute("name", required("broadcast")); +publish.setAttribute("source", "camera"); +publish.setAttribute("invisible", ""); +publish.setAttribute("preview", "none"); +document.body.appendChild(publish); diff --git a/test/audio-quality/clients/js/src/recorder.ts b/test/audio-quality/clients/js/src/recorder.ts new file mode 100644 index 0000000000..c0494c5cc1 --- /dev/null +++ b/test/audio-quality/clients/js/src/recorder.ts @@ -0,0 +1,130 @@ +/** + * The recorder page: subscribes to a broadcast's audio and stamps each frame as it comes off its + * group's stream, which is where the player's container consumer receives it. + * + * Built only on the public `@moq/net` and `@moq/hang` surface, so it records the path a viewer's + * player takes (a WebTransport session in Chromium, every group on its own stream) without a debug + * hook in the player. It reads groups the way `Container.Consumer` does, concurrently and decoded + * with the same format, but stamps them before the consumer's in-order delivery: a group the + * consumer holds back waiting for a missing one, or skips for age, is a decision the replay makes + * again at its own delay, and baking one in would censor the trace. + * + * ?url=https://cdn.moq.dev/demo&broadcast=bbb.hang + * + * `record.ts` drains it through `globalThis.recorder`. + * + * @module + */ +import * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import * as Moq from "@moq/net"; +import type { Arrival } from "./schema.ts"; + +/** How far back the relay may serve a group. Long, so a late group is recorded rather than dropped. */ +const MAX_AGE = Moq.Time.Milli(10_000); + +/** What the recorder has, until the driver drains it. */ +export type Recorder = { + /** Arrivals since the last drain, `at` on this page's `performance.now()`. */ + drain(): Arrival[]; + /** The session and rendition, once subscribed. */ + info(): { transport: string; rtt: number | null; config: Catalog.AudioConfig } | undefined; + /** The failure that stopped recording, if one did. */ + error(): string | undefined; +}; + +declare global { + var recorder: Recorder; +} + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const arrivals: Arrival[] = []; +let info: ReturnType; +let failure: string | undefined; +const fail = (err: unknown) => { + failure ??= err instanceof Error ? err.message : String(err); + console.error("recorder:", failure); +}; + +globalThis.recorder = { + drain: () => arrivals.splice(0, arrivals.length), + info: () => info, + error: () => failure, +}; + +/** The container format the player's decoder would pick for `config`. */ +function format(config: Catalog.AudioConfig): Container.Format { + const container = config.container; + if (container.kind === "legacy") return new Container.Legacy.Format(config); + if (container.kind === "loc") return new Container.Loc.Format("audio"); + if (container.kind === "cmaf") { + const init = Uint8Array.from(atob(container.init), (c) => c.charCodeAt(0)); + return new Container.Cmaf.Format(Container.Cmaf.decodeInitSegment(init)); + } + throw new Error(`unsupported container: ${JSON.stringify(container)}`); +} + +/** Stamp every frame of one group as it is read. Markers carry no media, so they are not arrivals. */ +async function readGroup(group: Moq.Group.Consumer, decoder: Container.Format): Promise { + for (;;) { + const next = await group.readFrame(); + const at = performance.now(); + if (!next) return; + for (const frame of decoder.decode(next.payload)) { + if (decoder.end?.(frame) !== undefined) continue; + arrivals.push([at, frame.timestamp / 1000, group.sequence]); + } + } +} + +async function record(): Promise { + const origin = new Moq.Origin.Producer(); + const connection = await Moq.Connection.connect({ url: new URL(required("url")), consume: origin }); + + let rtt: number | null = null; + connection.probe.subscribe((probe) => { + if (probe.rtt !== undefined) rtt = rtt === null ? probe.rtt : Math.min(rtt, probe.rtt); + }); + + const request = origin.request(Moq.Path.from(required("broadcast"))); + let broadcast = request.active.peek(); + while (!broadcast) { + await request.active.changed(); + broadcast = request.active.peek(); + } + + let root: Catalog.Root | undefined; + for await (const update of Catalog.watch(broadcast)) { + if (Object.keys(update.audio?.renditions ?? {}).length > 0) { + root = update; + break; + } + } + // Every recorded broadcast carries one audio rendition, so the first is the one a player picks. + const [name, config] = Object.entries(root?.audio?.renditions ?? {})[0] ?? []; + if (!name || !config) throw new Error("the catalog ended without an audio rendition"); + + const decoder = format(config); + const track = broadcast.track(name).subscribe({ priority: Catalog.PRIORITY.audio, maxAge: MAX_AGE }); + info = { + transport: connection.transport, + get rtt() { + return rtt; + }, + config, + }; + + for (;;) { + const group = await track.recvGroup(); + if (!group) throw new Error("the track ended"); + readGroup(group, decoder).catch(fail); + } +} + +record().catch(fail); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index 3e36798d1b..ca17583f6c 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -239,6 +239,12 @@ export type Ring = "isolated" | "plain"; /** The audio codecs the matrix publishes. */ export type Codec = "opus" | "aac"; +/** + * Where a row played: a headless browser over the shaper, or a {@link Trace} replayed through the + * player's rings on a simulated clock, whose profile is the trace's name. + */ +export type Runtime = "chromium" | "replay"; + /** * One matrix cell. * @@ -247,7 +253,7 @@ export type Codec = "opus" | "aac"; * threshold. */ export type Row = { - runtime: "chromium"; + runtime: Runtime; codec: Codec; /** Sample rate in Hz. */ rate: number; @@ -266,7 +272,7 @@ export function parseRow(key: string): Row { const profile = parts.slice(3, -1).join("-"); const hz = Number.parseInt(rate ?? "", 10); if ( - runtime !== "chromium" || + (runtime !== "chromium" && runtime !== "replay") || (codec !== "opus" && codec !== "aac") || (ring !== "isolated" && ring !== "plain") || !Number.isFinite(hz) || @@ -277,6 +283,48 @@ export function parseRow(key: string): Row { return { runtime, codec, rate: hz, profile, ring }; } +// ── what the recorder keeps ───────────────────────────────────────────────── + +/** + * One audio frame reaching the viewer's container consumer: when, on the `viewer` clock; its + * timestamp, on the `media` clock; and the group that carried it. + */ +export type Arrival = [at: Ms, timestamp: Ms, group: number]; + +/** A recorded arrival trace, as checked in under `traces/`. The file name is its profile. */ +export type Trace = { + /** Bumped when a field's meaning changes. */ + version: 1; + /** Where it was recorded: the relay, the broadcast, its publisher, the date. */ + source: string; + /** The shape it carries, and why it is kept. */ + description: string; + /** The transport the session negotiated. */ + transport: string; + /** The smallest round trip the connection's PROBE reported, which "auto" sizes from. */ + rtt: Ms | null; + /** The audio rendition, as the catalog advertised it. */ + config: { + codec: string; + sampleRate: number; + numberOfChannels: number; + container: { kind: string }; + jitter?: number; + delay?: number; + [key: string]: unknown; + }; + /** Every arrival in order, `at` counted from the first. */ + arrivals: Arrival[]; +}; + +/** The codec a trace's rendition is, as a row names it. */ +export function traceCodec(trace: Trace): Codec { + const codec = trace.config.codec; + if (codec === "opus") return "opus"; + if (codec.startsWith("mp4a")) return "aac"; + throw new Error(`no row codec for ${codec}`); +} + // ── what the page emits ───────────────────────────────────────────────────── /** A run of consecutive quanta the ring did not fill, as the output tap saw it. */ diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json index 5a536d16f4..b61d3dce46 100644 --- a/test/audio-quality/clients/js/tsconfig.json +++ b/test/audio-quality/clients/js/tsconfig.json @@ -3,5 +3,14 @@ "compilerOptions": { "types": ["audioworklet", "bun", "vite/client"] }, - "include": ["analyze.ts", "driver.ts", "grade.ts", "src", "vite.config.ts", "../../../../js/common/worklet.d.ts"] + "include": [ + "analyze.ts", + "driver.ts", + "grade.ts", + "record.ts", + "replay.ts", + "src", + "vite.config.ts", + "../../../../js/common/worklet.d.ts" + ] } diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts index c971d84052..68040577ab 100644 --- a/test/audio-quality/clients/js/vite.config.ts +++ b/test/audio-quality/clients/js/vite.config.ts @@ -11,9 +11,16 @@ import { defineConfig } from "vite"; import { workletInline } from "../../../../js/common/vite-plugin-worklet"; -/** esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. */ +/** + * esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. Three pages: the player + * under measurement, and the recorder and microphone publisher that `record.ts` drives. + */ export default defineConfig({ base: "./", plugins: [workletInline()], - build: { target: "esnext", outDir: "dist" }, + build: { + target: "esnext", + outDir: "dist", + rollupOptions: { input: ["index.html", "recorder.html", "mic.html"] }, + }, }); diff --git a/test/audio-quality/record.sh b/test/audio-quality/record.sh new file mode 100755 index 0000000000..2211436969 --- /dev/null +++ b/test/audio-quality/record.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Records the arrival traces under traces/, which the replay rows grade. See README.md. +# +# Three lanes, each --duration seconds from the first arrival: +# relay-bbb the public relay's Big Buck Bunny, the AAC flush-span shape #3477 measured +# local-aac an AAC tone through a local relay, the shallow control +# relay-mic a browser publishing Chromium's fake capture device through the public relay +# +# Not part of a run: re-record by hand when the path or the publishers change, and re-measure the +# replay budgets after. +set -euo pipefail + +AQ_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$AQ_DIR/../.." && pwd) +CLIENT="$AQ_DIR/clients/js" + +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$AQ_DIR/../lib/harness.sh" + +RERUN="just test audio-quality-record$(harness_argv "$@")" + +ALL_LANES=(relay-bbb local-aac relay-mic) +LANES=("${ALL_LANES[@]}") +DURATION=35 +PUBLIC="https://cdn.moq.dev" + +while [[ $# -gt 0 ]]; do + case "$1" in + --lanes) + IFS=, read -r -a LANES <<<"${2:?--lanes requires a value}" + shift 2 + ;; + --duration) + DURATION="${2:?--duration requires a value}" + shift 2 + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done +for lane in "${LANES[@]}"; do + [[ " ${ALL_LANES[*]} " == *" $lane "* ]] || { + echo "error: unknown lane '$lane' (known: ${ALL_LANES[*]})" >&2 + exit 2 + } +done + +harness_begin audio-quality-record "$RERUN" + +echo "building the pages..." +( + cd "$CLIENT" + bun install --frozen-lockfile + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir +) + +DATE=$(date -u +%Y-%m-%d) +record() { + local lane="$1" + shift + echo "" + echo "── $lane ──" + bun "$CLIENT/record.ts" --page "$HARNESS_RUN/page" --duration "$DURATION" \ + --out "$AQ_DIR/traces/$lane.json" "$@" +} + +for lane in "${LANES[@]}"; do + case "$lane" in + relay-bbb) + record "$lane" --url "$PUBLIC/demo" --broadcast bbb.hang \ + --source "$PUBLIC/demo bbb.hang, the public demo's MPEG-TS import, recorded $DATE" \ + --description "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured." + ;; + local-aac) + (cd "$WORKSPACE" && cargo build --locked -p moq-relay -p moq-cli) + target="${CARGO_TARGET_DIR:-$WORKSPACE/target}/debug" + harness_port relay + url="http://127.0.0.1:$HARNESS_PORT" + sed "s/4443/$HARNESS_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" + harness_spawn relay "$HARNESS_RUN/relay.log" "$target/moq-relay" "$HARNESS_RUN/relay.toml" + harness_ready "$url/certificate.sha256" 30 "$HARNESS_PID" || { + echo "relay never became ready" >&2 + exit 1 + } + # The harness's own AAC publisher, so the control is the shape its rows play. + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$target/moq" --connect "$url" --broadcast tone-aac.hang import ts + } + harness_spawn pub-aac "$HARNESS_RUN/pub-aac.log" publish + record "$lane" --url "$url" --broadcast tone-aac.hang \ + --source "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded $DATE" \ + --description "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer." + ;; + relay-mic) + broadcast="aq-mic-$RANDOM$RANDOM.hang" + record "$lane" --mic --url "$PUBLIC/anon" --broadcast "$broadcast" \ + --source "$PUBLIC/anon, in headless Chromium capturing its fake audio device, recorded $DATE" \ + --description "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between." + ;; + esac +done diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index c10544597e..e8f47dfe03 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -19,7 +19,13 @@ ALL_PROFILES=(near-zero mild wide fixed-250) ALL_RINGS=(isolated plain) ALL_CODECS=(opus aac) -PROFILES=("${ALL_PROFILES[@]}") +# Every recorded trace is a profile too, replayed through the rings instead of played in a browser. +ALL_TRACES=() +for trace in "$AQ_DIR"/traces/*.json; do + ALL_TRACES+=("$(basename "$trace" .json)") +done + +PROFILES=("${ALL_PROFILES[@]}" "${ALL_TRACES[@]}") RINGS=("${ALL_RINGS[@]}") CODECS=("${ALL_CODECS[@]}") DURATION=60 @@ -111,7 +117,7 @@ valid() { echo "error: unknown $name '$want' (known: $*)" >&2 exit 2 } -for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}"; done +for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}" "${ALL_TRACES[@]}"; done for r in "${RINGS[@]}"; do valid "$r" ring "${ALL_RINGS[@]}"; done for c in "${CODECS[@]}"; do valid "$c" codec "${ALL_CODECS[@]}"; done @@ -147,17 +153,33 @@ shaper_of() { } ROWS=() -for codec in "${CODECS[@]}"; do - for profile in "${PROFILES[@]}"; do +TRACES=() +for profile in "${PROFILES[@]}"; do + if [[ " ${ALL_TRACES[*]} " == *" $profile "* ]]; then + TRACES+=("$profile") + continue + fi + for codec in "${CODECS[@]}"; do for ring in "${RINGS[@]}"; do ROWS+=("chromium-$codec-$(rate_of "$codec")-$profile-$ring") done done done +# A trace's codec and rate are its recording's, so replay.ts names those rows. +joined() { + local IFS=, + echo "$*" +} +replay=(bun "$CLIENT/replay.ts" --traces "$AQ_DIR/traces" --profiles "$(joined "${TRACES[@]}")") +replay+=(--rings "$(joined "${RINGS[@]}")" --codecs "$(joined "${CODECS[@]}")") +listed=$("${replay[@]}" --list) +mapfile -t REPLAYS <<<"$listed" +[[ -n "$listed" ]] || REPLAYS=() + if [[ $LIST -eq 1 ]]; then - printf '%s\n' "${ROWS[@]}" - echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED" >&2 + printf '%s\n' "${ROWS[@]}" "${REPLAYS[@]}" + echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED; ${#REPLAYS[@]} replays of their recorded trace" >&2 exit 0 fi @@ -286,6 +308,20 @@ for tag in "${ROWS[@]}"; do } done +# ── replays ───────────────────────────────────────────────────────────────── +# Simulated time, so the whole set takes seconds, and deterministic, so the budgets are exact. +if [[ ${#REPLAYS[@]} -gt 0 ]]; then + echo "" + echo "replaying ${#REPLAYS[@]} rows" + "${replay[@]}" --out "$HARNESS_RUN" || failed=1 + for tag in "${REPLAYS[@]}"; do + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } + done +fi + # ── grade ─────────────────────────────────────────────────────────────────── echo "" grade=(bun "$CLIENT/grade.ts" --run "$HARNESS_RUN" --budgets "$AQ_DIR/budgets.json") diff --git a/test/audio-quality/traces/local-aac.json b/test/audio-quality/traces/local-aac.json new file mode 100644 index 0000000000..807d609bd7 --- /dev/null +++ b/test/audio-quality/traces/local-aac.json @@ -0,0 +1,1566 @@ +{ + "version": 1, + "source": "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded 2026-09-29", + "description": "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer.", + "transport": "webtransport", + "rtt": 1, + "config": { + "codec": "mp4a.40.2", + "container": { + "kind": "legacy" + }, + "description": "1210", + "sampleRate": 44100, + "numberOfChannels": 2, + "jitter": 163 + }, + "arrivals": [ + [0,10016.133,0], + [2.55,10039.344,1], + [5.71,10062.555,2], + [5.98,10085.766,3], + [6.4,10108.977,4], + [6.61,10132.188,5], + [6.84,10155.4,6], + [7.09,10178.677,7], + [7.59,10201.888,8], + [7.73,10225.1,9], + [7.87,10248.311,10], + [8.02,10271.522,11], + [8.17,10294.733,12], + [8.34,10317.944,13], + [9.43,10341.211,14], + [9.59,10364.422,15], + [9.75,10387.633,16], + [10.38,10410.844,17], + [10.53,10434.055,18], + [10.69,10457.266,19], + [10.84,10480.477,20], + [10.96,10503.755,21], + [11.11,10526.966,22], + [11.26,10550.177,23], + [11.4,10573.388,24], + [11.55,10596.6,25], + [11.72,10619.811,26], + [11.9,10643.022,27], + [12.08,10666.288,28], + [12.26,10689.5,29], + [12.45,10712.711,30], + [12.66,10735.922,31], + [12.83,10759.133,32], + [13.02,10782.344,33], + [13.2,10805.555,34], + [14.59,10828.833,35], + [14.84,10852.044,36], + [15.16,10875.255,37], + [15.72,10968.1,41], + [15.88,10944.888,40], + [16.06,10921.677,39], + [16.2,10898.466,38], + [123.89,11014.588,43], + [124.13,10991.377,42], + [124.91,11130.644,48], + [125.08,11107.433,47], + [125.19,11084.222,46], + [125.34,11061.011,45], + [125.43,11037.8,44], + [287.9,11200.333,51], + [288.18,11177.122,50], + [288.3,11153.911,49], + [288.42,11293.177,55], + [288.64,11269.966,54], + [288.75,11246.755,53], + [288.89,11223.544,52], + [450.27,11432.511,61], + [450.48,11409.3,60], + [450.59,11386.088,59], + [450.71,11362.877,58], + [450.93,11339.666,57], + [451.07,11316.455,56], + [451.23,11455.722,62], + [613.15,11595.044,68], + [613.37,11571.833,67], + [613.47,11548.622,66], + [613.56,11525.411,65], + [613.67,11502.2,64], + [613.87,11478.988,63], + [613.97,11618.255,69], + [774.29,11757.588,75], + [774.5,11734.377,74], + [774.59,11711.166,73], + [774.69,11687.955,72], + [774.88,11664.744,71], + [774.97,11641.533,70], + [775.08,11780.8,76], + [937.57,11920.122,82], + [937.76,11896.911,81], + [937.87,11873.7,80], + [937.98,11850.488,79], + [938.17,11827.277,78], + [938.29,11804.066,77], + [938.4,11943.333,83], + [1100.28,12036.244,87], + [1100.49,12013.033,86], + [1100.59,11989.822,85], + [1100.7,11966.611,84], + [1100.87,12105.877,90], + [1100.98,12082.666,89], + [1101.08,12059.455,88], + [1262.9,12152.366,92], + [1263.13,12129.155,91], + [1263.3,12268.422,97], + [1263.43,12245.211,96], + [1263.64,12222,95], + [1263.74,12198.788,94], + [1263.88,12175.577,93], + [1426.26,12407.744,103], + [1426.47,12384.533,102], + [1426.59,12361.322,101], + [1426.73,12338.111,100], + [1426.95,12314.9,99], + [1427.06,12291.688,98], + [1427.2,12430.955,104], + [1587.95,12570.288,110], + [1588.2,12547.077,109], + [1588.31,12523.866,108], + [1588.43,12500.655,107], + [1588.64,12477.444,106], + [1588.73,12454.233,105], + [1588.84,12593.5,111], + [1751.58,12732.822,117], + [1751.77,12709.611,116], + [1751.88,12686.4,115], + [1751.99,12663.188,114], + [1752.17,12639.977,113], + [1752.3,12616.766,112], + [1752.43,12756.033,118], + [1916.67,12895.366,124], + [1917,12872.155,123], + [1917.16,12848.944,122], + [1917.41,12825.733,121], + [1917.55,12802.522,120], + [1917.67,12779.311,119], + [1917.81,12918.577,125], + [2077.07,13057.9,131], + [2077.3,13034.688,130], + [2077.46,13011.477,129], + [2077.61,12988.266,128], + [2077.91,12965.055,127], + [2078.04,12941.844,126], + [2078.14,13081.111,132], + [2238.87,13220.444,138], + [2239.24,13197.233,137], + [2239.38,13174.022,136], + [2239.53,13150.811,135], + [2239.66,13127.6,134], + [2239.93,13104.388,133], + [2240.06,13243.655,139], + [2400.81,13382.988,145], + [2401.13,13359.777,144], + [2401.28,13336.566,143], + [2401.42,13313.355,142], + [2401.56,13290.144,141], + [2401.9,13266.933,140], + [2402.04,13406.2,146], + [2564.49,13545.522,152], + [2564.8,13522.311,151], + [2564.92,13499.1,150], + [2565.06,13475.888,149], + [2565.16,13452.677,148], + [2565.24,13429.466,147], + [2565.32,13568.733,153], + [2726.35,13708.066,159], + [2726.66,13684.855,158], + [2726.82,13661.644,157], + [2727.09,13638.433,156], + [2727.2,13615.222,155], + [2727.32,13592.011,154], + [2727.44,13731.277,160], + [2888.53,13870.6,166], + [2888.76,13847.388,165], + [2888.87,13824.177,164], + [2889.11,13800.966,163], + [2889.22,13777.755,162], + [2889.43,13754.544,161], + [2889.61,13893.811,167], + [3051.61,14033.144,173], + [3051.87,14009.933,172], + [3052.02,13986.722,171], + [3052.18,13963.511,170], + [3052.33,13940.3,169], + [3052.68,13917.088,168], + [3052.82,14056.355,174], + [3213.91,14079.622,175], + [3214.18,14218.888,181], + [3215.8,14172.466,179], + [3216.03,14149.255,178], + [3216.22,14126.044,177], + [3216.36,14102.833,176], + [3216.53,14195.677,180], + [3375.85,14335.011,186], + [3376.06,14311.8,185], + [3376.16,14288.588,184], + [3376.38,14265.377,183], + [3376.49,14242.166,182], + [3376.59,14381.433,188], + [3376.72,14358.222,187], + [3538.87,14520.766,194], + [3539.21,14497.555,193], + [3539.38,14474.344,192], + [3539.52,14451.133,191], + [3539.74,14427.922,190], + [3539.85,14404.711,189], + [3539.98,14543.977,195], + [3711.34,14706.511,202], + [3711.56,14683.3,201], + [3711.88,14660.088,200], + [3712.04,14636.877,199], + [3712.79,14613.666,198], + [3713,14590.455,197], + [3713.21,14567.244,196], + [3868.88,14845.844,208], + [3869.2,14822.633,207], + [3869.32,14799.422,206], + [3869.43,14776.211,205], + [3869.61,14753,204], + [3869.75,14729.788,203], + [3869.95,14869.055,209], + [4026.26,15008.377,215], + [4026.49,14985.166,214], + [4026.61,14961.955,213], + [4026.74,14938.744,212], + [4026.94,14915.533,211], + [4027.06,14892.322,210], + [4027.19,15031.588,216], + [4188.73,15170.922,222], + [4188.97,15147.711,221], + [4189.08,15124.5,220], + [4189.25,15101.288,219], + [4189.35,15078.077,218], + [4189.48,15054.866,217], + [4189.62,15194.133,223], + [4351.39,15333.455,229], + [4351.65,15310.244,228], + [4351.78,15287.033,227], + [4351.9,15263.822,226], + [4352.15,15240.611,225], + [4352.27,15217.4,224], + [4352.42,15356.666,230], + [4514.23,15496,236], + [4514.45,15472.788,235], + [4514.59,15449.577,234], + [4514.71,15426.366,233], + [4514.9,15403.155,232], + [4515,15379.944,231], + [4515.14,15519.211,237], + [4686.54,15658.544,243], + [4686.87,15635.333,242], + [4688.84,15588.911,240], + [4689.05,15565.7,239], + [4689.37,15542.488,238], + [4689.54,15681.755,244], + [4689.68,15612.122,241], + [4839.79,15821.077,250], + [4840.04,15797.866,249], + [4840.2,15774.655,248], + [4840.39,15751.444,247], + [4840.51,15728.233,246], + [4840.62,15705.022,245], + [4840.78,15844.288,251], + [5001.26,15983.622,257], + [5001.71,15960.411,256], + [5002.02,15937.2,255], + [5002.21,15913.988,254], + [5002.41,15890.777,253], + [5002.74,15867.566,252], + [5002.88,16006.833,258], + [5165.08,16146.155,264], + [5165.3,16122.944,263], + [5165.42,16099.733,262], + [5165.52,16076.522,261], + [5165.75,16053.311,260], + [5165.85,16030.1,259], + [5166.02,16169.366,265], + [5333.68,16192.644,266], + [5333.97,16308.7,271], + [5334.26,16285.488,270], + [5334.39,16262.277,269], + [5334.52,16239.066,268], + [5334.66,16215.855,267], + [5334.84,16331.911,272], + [5493.92,16471.233,278], + [5494.18,16448.022,277], + [5494.28,16424.811,276], + [5494.38,16401.6,275], + [5494.57,16378.388,274], + [5494.67,16355.177,273], + [5494.78,16494.444,279], + [5657.09,16633.777,285], + [5657.29,16610.566,284], + [5657.39,16587.355,283], + [5657.47,16564.144,282], + [5657.56,16540.933,281], + [5657.77,16517.722,280], + [5657.87,16656.988,286], + [5813.98,16796.322,292], + [5814.21,16773.111,291], + [5814.34,16749.9,290], + [5814.44,16726.688,289], + [5814.59,16703.477,288], + [5814.69,16680.266,287], + [5814.92,16819.533,293], + [5977.89,16958.855,299], + [5978.19,16935.644,298], + [5978.3,16912.433,297], + [5978.47,16889.222,296], + [5978.57,16866.011,295], + [5978.66,16842.8,294], + [5978.87,16982.066,300], + [6139.22,17121.4,306], + [6139.41,17098.188,305], + [6139.52,17074.977,304], + [6139.68,17051.766,303], + [6139.77,17028.555,302], + [6139.87,17005.344,301], + [6139.99,17144.611,307], + [6311.7,17283.933,313], + [6312.03,17260.722,312], + [6312.19,17237.511,311], + [6312.34,17214.3,310], + [6312.49,17191.088,309], + [6312.84,17167.877,308], + [6312.99,17307.144,314], + [6464.69,17446.477,320], + [6464.96,17423.266,319], + [6465.11,17400.055,318], + [6465.21,17376.844,317], + [6465.29,17353.633,316], + [6465.38,17330.422,315], + [6465.6,17469.688,321], + [6627.69,17609.011,327], + [6628.22,17585.8,326], + [6628.41,17562.588,325], + [6628.55,17539.377,324], + [6628.95,17516.166,323], + [6629.06,17492.955,322], + [6629.37,17632.222,328], + [6788.63,17771.555,334], + [6788.84,17748.344,333], + [6788.94,17725.133,332], + [6789.06,17701.922,331], + [6789.26,17678.711,330], + [6789.34,17655.5,329], + [6789.45,17794.766,335], + [6952.27,17934.1,341], + [6952.49,17910.888,340], + [6952.62,17887.677,339], + [6952.79,17864.466,338], + [6952.89,17841.255,337], + [6953,17818.044,336], + [6953.21,17957.311,342], + [7114.68,18096.633,348], + [7114.92,18073.422,347], + [7115.09,18050.211,346], + [7115.21,18027,345], + [7115.46,18003.788,344], + [7115.57,17980.577,343], + [7115.69,18119.844,349], + [7277.22,18259.177,355], + [7277.54,18235.966,354], + [7277.66,18212.755,353], + [7277.77,18189.544,352], + [7277.89,18166.333,351], + [7278.14,18143.122,350], + [7278.24,18282.388,356], + [7438.99,18421.711,362], + [7439.21,18398.5,361], + [7439.32,18375.288,360], + [7439.45,18352.077,359], + [7439.68,18328.866,358], + [7439.79,18305.655,357], + [7439.89,18444.922,363], + [7601.84,18584.255,369], + [7602.06,18561.044,368], + [7602.18,18537.833,367], + [7602.29,18514.622,366], + [7602.56,18491.411,365], + [7602.71,18468.2,364], + [7602.88,18607.466,370], + [7764.61,18746.788,376], + [7764.84,18723.577,375], + [7764.95,18700.366,374], + [7765.13,18677.155,373], + [7765.24,18653.944,372], + [7765.34,18630.733,371], + [7765.49,18770,377], + [7926.66,18909.333,383], + [7926.85,18886.122,382], + [7926.96,18862.911,381], + [7927.13,18839.7,380], + [7927.21,18816.488,379], + [7927.31,18793.277,378], + [7927.41,18932.544,384], + [8089.77,19071.877,390], + [8090.01,19048.666,389], + [8090.12,19025.455,388], + [8090.34,19002.244,387], + [8090.45,18979.033,386], + [8090.59,18955.822,385], + [8090.78,19095.088,391], + [8251.55,19234.411,397], + [8251.75,19211.2,396], + [8251.92,19187.988,395], + [8252.09,19164.777,394], + [8252.38,19141.566,393], + [8252.48,19118.355,392], + [8252.59,19257.622,398], + [8414.65,19396.955,404], + [8414.87,19373.744,403], + [8414.99,19350.533,402], + [8415.09,19327.322,401], + [8415.28,19304.111,400], + [8415.36,19280.9,399], + [8415.47,19420.166,405], + [8577.91,19559.488,411], + [8578.23,19536.277,410], + [8578.38,19513.066,409], + [8578.75,19489.855,408], + [8578.89,19466.644,407], + [8579.06,19443.433,406], + [8579.18,19582.7,412], + [8739.86,19722.033,418], + [8740.09,19698.822,417], + [8740.21,19675.611,416], + [8740.31,19652.4,415], + [8740.57,19629.188,414], + [8740.67,19605.977,413], + [8740.8,19745.244,419], + [8904.17,19884.566,425], + [8904.5,19861.355,424], + [8904.68,19838.144,423], + [8905,19814.933,422], + [8905.15,19791.722,421], + [8905.31,19768.511,420], + [8905.49,19907.777,426], + [9064.8,20047.111,432], + [9065.11,20023.9,431], + [9065.23,20000.688,430], + [9065.39,19977.477,429], + [9065.62,19954.266,428], + [9065.72,19931.055,427], + [9065.83,20070.322,433], + [9228,20209.655,439], + [9228.27,20186.444,438], + [9228.38,20163.233,437], + [9228.48,20140.022,436], + [9228.59,20116.811,435], + [9228.87,20093.6,434], + [9229.3,20232.866,440], + [9390.18,20372.188,446], + [9390.56,20348.977,445], + [9390.71,20325.766,444], + [9390.84,20302.555,443], + [9390.99,20279.344,442], + [9391.12,20256.133,441], + [9391.31,20395.4,447], + [9570.51,20511.522,452], + [9570.64,20534.733,453], + [9570.9,20488.311,451], + [9571.07,20465.1,450], + [9571.23,20441.888,449], + [9571.41,20418.677,448], + [9572.07,20557.944,454], + [9728.41,20697.266,460], + [9733.17,20674.055,459], + [9733.53,20650.844,458], + [9733.84,20627.633,457], + [9736.38,20604.422,456], + [9736.55,20581.211,455], + [9736.67,20720.477,461], + [9877.29,20859.811,467], + [9877.49,20836.6,466], + [9877.58,20813.388,465], + [9877.76,20790.177,464], + [9877.84,20766.966,463], + [9877.94,20743.755,462], + [9878.05,20883.022,468], + [10039.92,21022.344,474], + [10040.41,20999.133,473], + [10040.62,20975.922,472], + [10040.83,20952.711,471], + [10040.94,20929.5,470], + [10041.05,20906.288,469], + [10041.14,21045.555,475], + [10202.67,21184.888,481], + [10202.87,21161.677,480], + [10203.01,21138.466,479], + [10203.15,21115.255,478], + [10203.37,21092.044,477], + [10203.48,21068.833,476], + [10203.59,21208.1,482], + [10364.37,21347.433,488], + [10364.59,21324.222,487], + [10364.69,21301.011,486], + [10364.84,21277.8,485], + [10364.92,21254.588,484], + [10365.06,21231.377,483], + [10365.15,21370.644,489], + [10536.68,21509.966,495], + [10536.93,21486.755,494], + [10537.08,21463.544,493], + [10537.21,21440.333,492], + [10537.47,21393.911,490], + [10537.66,21533.177,496], + [10537.77,21417.122,491], + [10718.08,21579.666,498], + [10718.25,21672.511,502], + [10720.29,21649.3,501], + [10720.46,21626.088,500], + [10720.62,21602.877,499], + [10720.88,21556.455,497], + [10721.45,21695.722,503], + [10852.5,21835.044,509], + [10852.69,21811.833,508], + [10852.81,21788.622,507], + [10852.96,21765.411,506], + [10853.17,21742.2,505], + [10853.52,21718.988,504], + [10853.64,21858.255,510], + [11014.78,21997.588,516], + [11015.06,21974.377,515], + [11015.16,21951.166,514], + [11015.26,21927.955,513], + [11015.44,21904.744,512], + [11015.53,21881.533,511], + [11015.63,22020.8,517], + [11180.89,22136.911,522], + [11181.07,22160.122,523], + [11181.74,22113.7,521], + [11182.13,22090.488,520], + [11182.3,22067.277,519], + [11182.46,22044.066,518], + [11182.58,22183.333,524], + [11341.58,22322.666,530], + [11341.82,22299.455,529], + [11341.94,22276.244,528], + [11342.07,22253.033,527], + [11342.34,22229.822,526], + [11342.45,22206.611,525], + [11342.57,22345.877,531], + [11504.42,22485.211,537], + [11504.62,22462,536], + [11504.7,22438.788,535], + [11504.85,22415.577,534], + [11504.94,22392.366,533], + [11505.06,22369.155,532], + [11505.24,22508.422,538], + [11665.44,22647.744,544], + [11665.6,22624.533,543], + [11665.68,22601.322,542], + [11665.83,22578.111,541], + [11665.91,22554.9,540], + [11666.02,22531.688,539], + [11666.23,22670.955,545], + [11828.72,22810.288,551], + [11829.11,22787.077,550], + [11829.23,22763.866,549], + [11829.3,22740.655,548], + [11829.71,22717.444,547], + [11829.81,22694.233,546], + [11829.91,22833.5,552], + [11990.61,22972.822,558], + [11990.8,22949.611,557], + [11990.89,22926.4,556], + [11991.04,22903.188,555], + [11991.11,22879.977,554], + [11991.2,22856.766,553], + [11991.28,22996.033,559], + [12155.63,23135.366,565], + [12155.83,23112.155,564], + [12155.94,23088.944,563], + [12156.07,23065.733,562], + [12156.3,23042.522,561], + [12156.39,23019.311,560], + [12156.55,23158.577,566], + [12317.01,23297.9,572], + [12317.17,23274.688,571], + [12317.28,23251.477,570], + [12317.41,23228.266,569], + [12317.46,23205.055,568], + [12317.53,23181.844,567], + [12317.64,23321.111,573], + [12477.48,23460.444,579], + [12477.66,23437.233,578], + [12477.78,23414.022,577], + [12477.87,23390.811,576], + [12477.99,23367.6,575], + [12478.07,23344.388,574], + [12478.2,23483.655,580], + [12641.84,23622.988,586], + [12642.02,23599.777,585], + [12642.11,23576.566,584], + [12642.24,23553.355,583], + [12642.31,23530.144,582], + [12642.39,23506.933,581], + [12642.57,23646.2,587], + [12803.44,23785.522,593], + [12803.62,23762.311,592], + [12803.71,23739.1,591], + [12803.8,23715.888,590], + [12803.95,23692.677,589], + [12804.04,23669.466,588], + [12804.16,23808.733,594], + [12968.53,23878.433,597], + [12968.66,23948.066,600], + [12968.89,23924.855,599], + [12969.03,23901.644,598], + [12969.2,23855.222,596], + [12969.39,23832.011,595], + [12969.71,23971.277,601], + [13128.76,24110.6,607], + [13128.92,24087.388,606], + [13129.01,24064.177,605], + [13129.19,24040.966,604], + [13129.28,24017.755,603], + [13129.36,23994.544,602], + [13129.53,24133.811,608], + [13291.69,24273.144,614], + [13291.9,24249.933,613], + [13292.02,24226.722,612], + [13292.14,24203.511,611], + [13292.34,24180.3,610], + [13292.46,24157.088,609], + [13292.57,24296.355,615], + [13456.24,24435.677,621], + [13456.44,24412.466,620], + [13456.56,24389.255,619], + [13456.67,24366.044,618], + [13456.78,24342.833,617], + [13456.87,24319.622,616], + [13456.99,24458.888,622], + [13622.09,24505.377,624], + [13622.38,24482.166,623], + [13623.46,24598.222,628], + [13624.14,24575.011,627], + [13625.59,24551.8,626], + [13625.77,24528.588,625], + [13627.41,24621.433,629], + [13870.43,24760.766,635], + [13873.91,24737.555,634], + [13876.49,24714.344,633], + [13883.19,24691.133,632], + [13889,24667.922,631], + [13891.52,24644.711,630], + [13892.02,24783.977,636], + [13992.79,24923.3,642], + [13993.03,24900.088,641], + [13993.18,24876.877,640], + [13993.26,24853.666,639], + [13993.39,24830.455,638], + [13993.73,24807.244,637], + [13994.1,24946.511,643], + [14174.97,25016.211,646], + [14175.28,25085.844,649], + [14175.79,25062.633,648], + [14175.91,25039.422,647], + [14176.04,24993,645], + [14176.17,24969.788,644], + [14176.27,25109.055,650], + [14286.82,25248.377,656], + [14292.13,25225.166,655], + [14294,25178.744,653], + [14294.68,25155.533,652], + [14295.73,25132.322,651], + [14296.09,25271.588,657], + [14296.46,25201.955,654], + [14472.09,25318.077,659], + [14472.22,25294.866,658], + [14472.45,25434.133,664], + [14472.66,25410.922,663], + [14472.85,25387.711,662], + [14472.94,25364.5,661], + [14473.1,25341.288,660], + [14674.91,25573.455,670], + [14675.71,25550.244,669], + [14676.58,25503.822,667], + [14680.92,25480.611,666], + [14681.54,25457.4,665], + [14687.88,25596.666,671], + [14688.29,25527.033,668], + [14776.93,25736,677], + [14777.53,25712.788,676], + [14777.66,25689.577,675], + [14777.8,25666.366,674], + [14777.91,25643.155,673], + [14778.21,25619.944,672], + [14778.88,25759.211,678], + [14942.41,25898.544,684], + [14942.61,25875.333,683], + [14942.82,25852.122,682], + [14943,25828.911,681], + [14943.36,25805.7,680], + [14943.5,25782.488,679], + [14944.51,25921.755,685], + [15084.12,26061.077,691], + [15084.39,26037.866,690], + [15085.33,26014.655,689], + [15086.9,25991.444,688], + [15087.67,25968.233,687], + [15088.27,25945.022,686], + [15089.31,26084.288,692], + [15247.24,26223.622,698], + [15247.94,26200.411,697], + [15248.07,26177.2,696], + [15248.27,26153.988,695], + [15248.36,26130.777,694], + [15248.45,26107.566,693], + [15248.55,26246.833,699], + [15411.58,26386.155,705], + [15412.06,26362.944,704], + [15412.23,26339.733,703], + [15412.52,26316.522,702], + [15412.7,26293.311,701], + [15412.89,26270.1,700], + [15413.07,26409.366,706], + [15567.84,26548.7,712], + [15568.09,26525.488,711], + [15568.2,26502.277,710], + [15568.37,26479.066,709], + [15568.47,26455.855,708], + [15568.61,26432.644,707], + [15568.72,26571.911,713], + [15735.29,26711.233,719], + [15735.6,26688.022,718], + [15735.74,26664.811,717], + [15735.88,26641.6,716], + [15736.18,26618.388,715], + [15736.31,26595.177,714], + [15736.46,26734.444,720], + [15895.29,26873.777,726], + [15895.63,26850.566,725], + [15895.79,26827.355,724], + [15895.94,26804.144,723], + [15896.26,26780.933,722], + [15896.38,26757.722,721], + [15896.51,26896.988,727], + [16059.43,27036.322,733], + [16059.7,27013.111,732], + [16059.83,26989.9,731], + [16059.97,26966.688,730], + [16060.11,26943.477,729], + [16060.26,26920.266,728], + [16060.4,27059.533,734], + [16219.19,27198.855,740], + [16219.51,27175.644,739], + [16219.69,27152.433,738], + [16219.97,27129.222,737], + [16220.06,27106.011,736], + [16220.15,27082.8,735], + [16220.24,27222.066,741], + [16382.51,27361.4,747], + [16382.77,27338.188,746], + [16382.9,27314.977,745], + [16383.09,27291.766,744], + [16383.17,27268.555,743], + [16383.25,27245.344,742], + [16383.33,27384.611,748], + [16545.33,27523.933,754], + [16545.69,27500.722,753], + [16545.83,27477.511,752], + [16546.09,27454.3,751], + [16546.22,27431.088,750], + [16546.35,27407.877,749], + [16546.46,27547.144,755], + [16707.81,27686.477,761], + [16708.14,27663.266,760], + [16708.28,27640.055,759], + [16708.45,27616.844,758], + [16708.53,27593.633,757], + [16708.59,27570.422,756], + [16708.71,27709.688,762], + [16868,27849.011,768], + [16868.21,27825.8,767], + [16868.3,27802.588,766], + [16868.5,27779.377,765], + [16868.61,27756.166,764], + [16868.73,27732.955,763], + [16868.87,27872.222,769], + [17029.84,28011.555,775], + [17030.07,27988.344,774], + [17030.18,27965.133,773], + [17030.28,27941.922,772], + [17030.5,27918.711,771], + [17030.61,27895.5,770], + [17030.73,28034.766,776], + [17200.69,28174.1,782], + [17200.89,28150.888,781], + [17200.97,28127.677,780], + [17201.05,28104.466,779], + [17201.2,28081.255,778], + [17201.28,28058.044,777], + [17201.36,28197.311,783], + [17356.81,28336.633,789], + [17357.26,28313.422,788], + [17357.44,28290.211,787], + [17357.6,28267,786], + [17357.78,28243.788,785], + [17358.17,28220.577,784], + [17358.29,28359.844,790], + [17519.52,28499.177,796], + [17519.78,28475.966,795], + [17519.89,28452.755,794], + [17520.02,28429.544,793], + [17520.41,28406.333,792], + [17520.69,28383.122,791], + [17520.87,28522.388,797], + [17684,28661.711,803], + [17684.66,28638.5,802], + [17685.2,28615.288,801], + [17685.51,28592.077,800], + [17685.77,28568.866,799], + [17686,28545.655,798], + [17686.4,28684.922,804], + [17842.95,28824.255,810], + [17843.22,28801.044,809], + [17843.34,28777.833,808], + [17843.52,28754.622,807], + [17843.62,28731.411,806], + [17843.7,28708.2,805], + [17843.86,28847.466,811], + [18012.8,28986.788,817], + [18012.99,28963.577,816], + [18013.08,28940.366,815], + [18013.21,28917.155,814], + [18013.28,28893.944,813], + [18013.35,28870.733,812], + [18013.58,29010,818], + [18167.89,29149.333,824], + [18168.08,29126.122,823], + [18168.16,29102.911,822], + [18168.27,29079.7,821], + [18168.34,29056.488,820], + [18168.41,29033.277,819], + [18168.55,29172.544,825], + [18331.13,29311.877,831], + [18331.34,29288.666,830], + [18331.46,29265.455,829], + [18331.57,29242.244,828], + [18331.64,29219.033,827], + [18331.8,29195.822,826], + [18331.88,29335.088,832], + [18497.56,29474.411,838], + [18497.9,29451.2,837], + [18498.08,29427.988,836], + [18498.38,29404.777,835], + [18498.5,29381.566,834], + [18498.63,29358.355,833], + [18498.85,29497.622,839], + [18655.23,29636.955,845], + [18655.43,29613.744,844], + [18655.53,29590.533,843], + [18655.62,29567.322,842], + [18655.79,29544.111,841], + [18655.88,29520.9,840], + [18655.99,29660.166,846], + [18822.2,29799.488,852], + [18822.63,29776.277,851], + [18822.8,29753.066,850], + [18823.05,29729.855,849], + [18823.24,29706.644,848], + [18823.83,29683.433,847], + [18824,29822.7,853], + [18988.77,29962.033,859], + [18989.25,29938.822,858], + [18989.48,29915.611,857], + [18989.64,29892.4,856], + [18989.97,29869.188,855], + [18990.09,29845.977,854], + [18990.24,29985.244,860], + [19152.68,30124.566,866], + [19152.82,30101.355,865], + [19152.89,30078.144,864], + [19152.96,30054.933,863], + [19153.05,30031.722,862], + [19153.33,30008.511,861], + [19153.46,30147.777,867], + [19334.36,30171.055,868], + [19337.72,30310.322,874], + [19352.35,30287.111,873], + [19352.91,30263.9,872], + [19353.25,30240.688,871], + [19357.88,30217.477,870], + [19364.11,30194.266,869], + [19469.63,30426.444,879], + [19469.95,30403.233,878], + [19470.11,30380.022,877], + [19470.37,30356.811,876], + [19470.47,30333.6,875], + [19470.69,30472.866,881], + [19470.87,30449.655,880], + [19631.34,30612.188,887], + [19631.51,30588.977,886], + [19631.59,30565.766,885], + [19631.71,30542.555,884], + [19631.79,30519.344,883], + [19631.84,30496.133,882], + [19631.93,30635.4,888], + [19792.62,30774.733,894], + [19792.8,30751.522,893], + [19792.9,30728.311,892], + [19792.99,30705.1,891], + [19793.06,30681.888,890], + [19793.22,30658.677,889], + [19793.3,30797.944,895], + [19954.65,30937.266,901], + [19954.84,30914.055,900], + [19954.93,30890.844,899], + [19955.09,30867.633,898], + [19955.17,30844.422,897], + [19955.26,30821.211,896], + [19955.44,30960.477,902], + [20118.29,31099.811,908], + [20118.48,31076.6,907], + [20118.57,31053.388,906], + [20118.74,31030.177,905], + [20118.81,31006.966,904], + [20118.89,30983.755,903], + [20119,31123.022,909], + [20280.22,31262.344,915], + [20280.41,31239.133,914], + [20280.5,31215.922,913], + [20280.66,31192.711,912], + [20280.74,31169.5,911], + [20280.81,31146.288,910], + [20280.91,31285.555,916], + [20442.9,31424.888,922], + [20443.1,31401.677,921], + [20443.2,31378.466,920], + [20443.33,31355.255,919], + [20443.41,31332.044,918], + [20443.48,31308.833,917], + [20443.57,31448.1,923], + [20605.07,31587.433,929], + [20605.24,31564.222,928], + [20605.32,31541.011,927], + [20605.45,31517.8,926], + [20605.52,31494.588,925], + [20605.59,31471.377,924], + [20605.69,31610.644,930], + [20767.74,31749.966,936], + [20767.92,31726.755,935], + [20768.01,31703.544,934], + [20768.16,31680.333,933], + [20768.24,31657.122,932], + [20768.32,31633.911,931], + [20768.41,31773.177,937], + [20930.12,31912.511,943], + [20930.3,31889.3,942], + [20930.38,31866.088,941], + [20930.53,31842.877,940], + [20930.6,31819.666,939], + [20930.68,31796.455,938], + [20930.77,31935.722,944], + [21093.23,32075.044,950], + [21093.4,32051.833,949], + [21093.49,32028.622,948], + [21093.64,32005.411,947], + [21093.72,31982.2,946], + [21093.8,31958.988,945], + [21093.95,32098.255,951], + [21254.99,32237.588,957], + [21255.18,32214.377,956], + [21255.27,32191.166,955], + [21255.41,32167.955,954], + [21255.5,32144.744,953], + [21255.58,32121.533,952], + [21255.67,32260.8,958], + [21417.68,32400.122,964], + [21417.87,32376.911,963], + [21418,32353.7,962], + [21418.09,32330.488,961], + [21418.23,32307.277,960], + [21418.31,32284.066,959], + [21418.41,32423.333,965], + [21580.61,32562.666,971], + [21580.8,32539.455,970], + [21580.94,32516.244,969], + [21581.04,32493.033,968], + [21581.13,32469.822,967], + [21581.31,32446.611,966], + [21581.39,32585.877,972], + [21743.16,32725.211,978], + [21743.34,32702,977], + [21743.43,32678.788,976], + [21743.59,32655.577,975], + [21743.67,32632.366,974], + [21743.74,32609.155,973], + [21743.84,32748.422,979], + [21905.34,32887.744,985], + [21905.53,32864.533,984], + [21905.63,32841.322,983], + [21905.77,32818.111,982], + [21905.87,32794.9,981], + [21905.98,32771.688,980], + [21906.08,32910.955,986], + [22067.64,33050.288,992], + [22067.83,33027.077,991], + [22067.91,33003.866,990], + [22068.01,32980.655,989], + [22068.18,32957.444,988], + [22068.3,32934.233,987], + [22068.39,33073.5,993], + [22231.3,33212.822,999], + [22231.52,33189.611,998], + [22231.76,33166.4,997], + [22231.98,33143.188,996], + [22232.07,33119.977,995], + [22232.17,33096.766,994], + [22232.28,33236.033,1000], + [22395.19,33375.366,1006], + [22395.45,33352.155,1005], + [22395.63,33328.944,1004], + [22395.81,33305.733,1003], + [22395.98,33282.522,1002], + [22396.12,33259.311,1001], + [22396.27,33398.577,1007], + [22556.88,33537.9,1013], + [22557.18,33514.688,1012], + [22557.32,33491.477,1011], + [22557.45,33468.266,1010], + [22557.73,33445.055,1009], + [22557.9,33421.844,1008], + [22558.04,33561.111,1014], + [22720.22,33700.444,1020], + [22720.47,33677.233,1019], + [22720.67,33654.022,1018], + [22720.77,33630.811,1017], + [22720.98,33607.6,1016], + [22721.08,33584.388,1015], + [22721.19,33723.655,1021], + [22882.47,33862.988,1027], + [22882.74,33839.777,1026], + [22882.95,33816.566,1025], + [22883.05,33793.355,1024], + [22883.15,33770.144,1023], + [22883.34,33746.933,1022], + [22883.45,33886.2,1028], + [23044.69,34025.522,1034], + [23044.92,34002.311,1033], + [23045.09,33979.1,1032], + [23045.19,33955.888,1031], + [23045.34,33932.677,1030], + [23045.43,33909.466,1029], + [23045.53,34048.733,1035], + [23207.07,34188.066,1041], + [23207.29,34164.855,1040], + [23207.4,34141.644,1039], + [23207.55,34118.433,1038], + [23207.79,34095.222,1037], + [23207.9,34072.011,1036], + [23208.06,34211.277,1042], + [23370,34350.6,1048], + [23370.25,34327.388,1047], + [23370.36,34304.177,1046], + [23370.46,34280.966,1045], + [23370.7,34257.755,1044], + [23370.79,34234.544,1043], + [23370.91,34373.811,1049], + [23532.84,34513.144,1055], + [23533.06,34489.933,1054], + [23533.17,34466.722,1053], + [23533.29,34443.511,1052], + [23533.51,34420.3,1051], + [23533.6,34397.088,1050], + [23533.71,34536.355,1056], + [23696.22,34675.677,1062], + [23696.48,34652.466,1061], + [23696.58,34629.255,1060], + [23696.73,34606.044,1059], + [23696.83,34582.833,1058], + [23696.91,34559.622,1057], + [23697.07,34698.888,1063], + [23858.15,34838.222,1069], + [23858.39,34815.011,1068], + [23858.64,34791.8,1067], + [23858.84,34768.588,1066], + [23858.97,34745.377,1065], + [23859.07,34722.166,1064], + [23859.18,34861.433,1070], + [24019.44,35000.766,1076], + [24019.67,34977.555,1075], + [24019.8,34954.344,1074], + [24019.87,34931.133,1073], + [24019.95,34907.922,1072], + [24020.12,34884.711,1071], + [24020.19,35023.977,1077], + [24181.11,35163.3,1083], + [24181.28,35140.088,1082], + [24181.35,35116.877,1081], + [24181.49,35093.666,1080], + [24181.56,35070.455,1079], + [24181.63,35047.244,1078], + [24181.71,35186.511,1084], + [24345.36,35325.844,1090], + [24345.66,35302.633,1089], + [24345.77,35279.422,1088], + [24345.95,35256.211,1087], + [24346.05,35233,1086], + [24346.14,35209.788,1085], + [24346.26,35349.055,1091], + [24509.21,35488.377,1097], + [24509.52,35465.166,1096], + [24509.64,35441.955,1095], + [24509.77,35418.744,1094], + [24510.03,35395.533,1093], + [24510.14,35372.322,1092], + [24510.27,35511.588,1098], + [24673.12,35650.922,1104], + [24673.44,35627.711,1103], + [24673.58,35604.5,1102], + [24673.71,35581.288,1101], + [24673.96,35558.077,1100], + [24674.08,35534.866,1099], + [24674.22,35674.133,1105], + [24833.74,35813.455,1111], + [24833.99,35790.244,1110], + [24834.12,35767.033,1109], + [24834.21,35743.822,1108], + [24834.37,35720.611,1107], + [24834.46,35697.4,1106], + [24834.61,35836.666,1112], + [24997.09,35976,1118], + [24997.33,35952.788,1117], + [24997.47,35929.577,1116], + [24997.56,35906.366,1115], + [24997.68,35883.155,1114], + [24997.86,35859.944,1113], + [24997.99,35999.211,1119], + [25159.71,36138.544,1125], + [25159.88,36115.333,1124], + [25160,36092.122,1123], + [25160.08,36068.911,1122], + [25160.2,36045.7,1121], + [25160.27,36022.488,1120], + [25160.4,36161.755,1126], + [25322.74,36301.077,1132], + [25322.97,36277.866,1131], + [25323.07,36254.655,1130], + [25323.18,36231.444,1129], + [25323.3,36208.233,1128], + [25323.42,36185.022,1127], + [25323.55,36324.288,1133], + [25486.23,36463.622,1139], + [25486.51,36440.411,1138], + [25486.61,36417.2,1137], + [25486.78,36393.988,1136], + [25486.88,36370.777,1135], + [25487,36347.566,1134], + [25487.21,36486.833,1140], + [25645.47,36626.155,1146], + [25645.64,36602.944,1145], + [25645.75,36579.733,1144], + [25645.88,36556.522,1143], + [25646.01,36533.311,1142], + [25646.13,36510.1,1141], + [25646.51,36649.366,1147], + [25817.49,36695.855,1149], + [25817.7,36672.644,1148], + [25818.43,36742.277,1151], + [25818.57,36811.911,1154], + [25818.83,36788.7,1153], + [25819.08,36765.488,1152], + [25819.39,36719.066,1150], + [25981.76,36974.444,1161], + [25982.05,36951.233,1160], + [25982.15,36904.811,1158], + [25982.3,36881.6,1157], + [25982.38,36858.388,1156], + [25982.46,36835.177,1155], + [25982.62,36928.022,1159], + [26132.89,37113.777,1167], + [26133.16,37090.566,1166], + [26133.27,37067.355,1165], + [26133.47,37044.144,1164], + [26133.58,37020.933,1163], + [26133.69,36997.722,1162], + [26133.81,37136.988,1168], + [26296.13,37276.322,1174], + [26296.4,37253.111,1173], + [26296.51,37229.9,1172], + [26296.62,37206.688,1171], + [26296.87,37183.477,1170], + [26296.99,37160.266,1169], + [26297.11,37299.533,1175], + [26457.47,37438.855,1181], + [26457.68,37415.644,1180], + [26457.78,37392.433,1179], + [26457.98,37369.222,1178], + [26458.08,37346.011,1177], + [26458.21,37322.8,1176], + [26458.33,37462.066,1182], + [26620.12,37601.4,1188], + [26620.36,37578.188,1187], + [26620.46,37554.977,1186], + [26620.62,37531.766,1185], + [26620.72,37508.555,1184], + [26620.82,37485.344,1183], + [26620.94,37624.611,1189], + [26783.36,37763.933,1195], + [26783.56,37740.722,1194], + [26783.66,37717.511,1193], + [26783.84,37694.3,1192], + [26783.94,37671.088,1191], + [26784.03,37647.877,1190], + [26784.21,37787.144,1196], + [26945.3,37926.477,1202], + [26945.52,37903.266,1201], + [26945.62,37880.055,1200], + [26945.81,37856.844,1199], + [26945.91,37833.633,1198], + [26946.07,37810.422,1197], + [26946.2,37949.688,1203], + [27109.08,38089.011,1209], + [27109.31,38065.8,1208], + [27109.42,38042.588,1207], + [27109.61,38019.377,1206], + [27109.72,37996.166,1205], + [27109.84,37972.955,1204], + [27109.98,38112.222,1210], + [27271.98,38251.555,1216], + [27272.19,38228.344,1215], + [27272.29,38205.133,1214], + [27272.42,38181.922,1213], + [27272.63,38158.711,1212], + [27272.74,38135.5,1211], + [27272.86,38274.766,1217], + [27435.38,38414.1,1223], + [27435.63,38390.888,1222], + [27435.74,38367.677,1221], + [27435.93,38344.466,1220], + [27436.03,38321.255,1219], + [27436.12,38298.044,1218], + [27436.39,38437.311,1224], + [27596.35,38576.633,1230], + [27596.62,38553.422,1229], + [27596.73,38530.211,1228], + [27596.93,38507,1227], + [27597.08,38483.788,1226], + [27597.31,38460.577,1225], + [27597.45,38599.844,1231], + [27757.81,38739.177,1237], + [27758.02,38715.966,1236], + [27758.12,38692.755,1235], + [27758.21,38669.544,1234], + [27758.39,38646.333,1233], + [27758.48,38623.122,1232], + [27758.59,38762.388,1238], + [27922.07,38901.711,1244], + [27922.25,38878.5,1243], + [27922.34,38855.288,1242], + [27922.42,38832.077,1241], + [27922.54,38808.866,1240], + [27922.63,38785.655,1239], + [27922.72,38924.922,1245], + [28084.02,39064.255,1251], + [28084.23,39041.044,1250], + [28084.3,39017.833,1249], + [28084.42,38994.622,1248], + [28084.49,38971.411,1247], + [28084.59,38948.2,1246], + [28084.84,39087.466,1252], + [28247.27,39226.788,1258], + [28247.5,39203.577,1257], + [28247.61,39180.366,1256], + [28247.8,39157.155,1255], + [28247.91,39133.944,1254], + [28248.03,39110.733,1253], + [28248.24,39250,1259], + [28409.79,39389.333,1265], + [28410.01,39366.122,1264], + [28410.16,39342.911,1263], + [28410.26,39319.7,1262], + [28410.4,39296.488,1261], + [28410.73,39273.277,1260], + [28410.86,39412.544,1266], + [28573.39,39551.877,1272], + [28573.67,39528.666,1271], + [28573.79,39505.455,1270], + [28573.94,39482.244,1269], + [28574.06,39459.033,1268], + [28574.15,39435.822,1267], + [28574.31,39575.088,1273], + [28734.46,39714.411,1279], + [28734.79,39691.2,1278], + [28734.94,39667.988,1277], + [28735.17,39644.777,1276], + [28735.27,39621.566,1275], + [28735.38,39598.355,1274], + [28735.52,39737.622,1280], + [28897.75,39876.955,1286], + [28898.07,39853.744,1285], + [28898.43,39830.533,1284], + [28898.65,39807.322,1283], + [28898.75,39784.111,1282], + [28898.86,39760.9,1281], + [28898.99,39900.166,1287], + [29060.62,40039.488,1293], + [29060.87,40016.277,1292], + [29061.02,39993.066,1291], + [29061.29,39969.855,1290], + [29061.41,39946.644,1289], + [29061.57,39923.433,1288], + [29061.84,40062.7,1294], + [29221.44,40202.033,1300], + [29221.7,40178.822,1299], + [29221.82,40155.611,1298], + [29221.94,40132.4,1297], + [29222.23,40109.188,1296], + [29222.37,40085.977,1295], + [29222.51,40225.244,1301], + [29383.83,40364.566,1307], + [29385.58,40318.144,1305], + [29385.79,40294.933,1304], + [29385.92,40271.722,1303], + [29386.06,40248.511,1302], + [29386.19,40387.777,1308], + [29386.32,40341.355,1306], + [29546.27,40527.111,1314], + [29546.56,40503.9,1313], + [29547.11,40480.688,1312], + [29547.25,40457.477,1311], + [29547.36,40434.266,1310], + [29547.47,40411.055,1309], + [29547.58,40550.322,1315], + [29712.46,40689.655,1321], + [29712.74,40666.444,1320], + [29712.88,40643.233,1319], + [29713.11,40620.022,1318], + [29713.22,40596.811,1317], + [29713.32,40573.6,1316], + [29713.46,40712.866,1322], + [29872.6,40852.188,1328], + [29872.84,40828.977,1327], + [29872.94,40805.766,1326], + [29873.14,40782.555,1325], + [29873.22,40759.344,1324], + [29873.3,40736.133,1323], + [29873.45,40875.4,1329], + [30037.37,41014.733,1335], + [30037.59,40991.522,1334], + [30037.72,40968.311,1333], + [30037.91,40945.1,1332], + [30038.05,40921.888,1331], + [30038.17,40898.677,1330], + [30038.29,41037.944,1336], + [30209.54,41177.266,1342], + [30211.82,41154.055,1341], + [30212.55,41130.844,1340], + [30212.65,41107.633,1339], + [30217.43,41061.211,1337], + [30217.86,41200.477,1343], + [30218.12,41084.422,1338], + [30367.11,41339.811,1349], + [30367.3,41316.6,1348], + [30367.41,41293.388,1347], + [30367.54,41270.177,1346], + [30367.74,41246.966,1345], + [30367.82,41223.755,1344], + [30367.91,41363.022,1350], + [30524.24,41502.344,1356], + [30524.54,41479.133,1355], + [30524.66,41455.922,1354], + [30524.8,41432.711,1353], + [30525.17,41409.5,1352], + [30525.33,41386.288,1351], + [30525.48,41525.555,1357], + [30695.69,41664.888,1363], + [30696.02,41641.677,1362], + [30696.24,41618.466,1361], + [30696.45,41595.255,1360], + [30696.62,41572.044,1359], + [30696.88,41548.833,1358], + [30700.84,41688.1,1364], + [30850.04,41827.433,1370], + [30851.28,41804.222,1369], + [30851.44,41781.011,1368], + [30851.76,41757.8,1367], + [30851.93,41734.588,1366], + [30852.12,41711.377,1365], + [30852.3,41850.644,1371], + [31012.69,41989.966,1377], + [31013.13,42013.177,1378], + [31013.35,41966.755,1376], + [31013.53,41943.544,1375], + [31013.7,41920.333,1374], + [31013.87,41897.122,1373], + [31014.05,41873.911,1372], + [31176.13,42152.511,1384], + [31176.5,42129.3,1383], + [31176.63,42106.088,1382], + [31176.81,42082.877,1381], + [31176.94,42059.666,1380], + [31177.1,42036.455,1379], + [31177.27,42175.722,1385], + [31338.66,42315.044,1391], + [31339.11,42291.833,1390], + [31339.29,42268.622,1389], + [31339.54,42245.411,1388], + [31339.72,42222.2,1387], + [31339.91,42198.988,1386], + [31340.18,42338.255,1392], + [31497.75,42477.588,1398], + [31498,42454.377,1397], + [31498.13,42431.166,1396], + [31498.34,42407.955,1395], + [31498.45,42384.744,1394], + [31498.58,42361.533,1393], + [31498.7,42500.8,1399], + [31665.88,42640.122,1405], + [31666.22,42616.911,1404], + [31666.42,42593.7,1403], + [31666.61,42570.488,1402], + [31666.78,42547.277,1401], + [31666.99,42524.066,1400], + [31667.24,42663.333,1406], + [31826.52,42733.033,1409], + [31826.67,42802.666,1412], + [31826.91,42779.455,1411], + [31827.05,42756.244,1410], + [31827.19,42709.822,1408], + [31827.29,42686.611,1407], + [31827.42,42825.877,1413], + [31989.84,42965.211,1419], + [31991.38,42942,1418], + [31991.5,42918.788,1417], + [31991.62,42895.577,1416], + [31991.71,42872.366,1415], + [31992.28,42849.155,1414], + [31992.43,42988.422,1420], + [32146.34,43127.744,1426], + [32146.57,43104.533,1425], + [32146.67,43081.322,1424], + [32146.78,43058.111,1423], + [32146.91,43034.9,1422], + [32147.16,43011.688,1421], + [32147.26,43150.955,1427], + [32309.4,43290.288,1433], + [32309.62,43174.233,1428], + [32309.74,43313.5,1434], + [32309.84,43267.077,1432], + [32309.95,43243.866,1431], + [32310.09,43220.655,1430], + [32310.22,43197.444,1429], + [32472.45,43452.822,1440], + [32472.7,43429.611,1439], + [32472.81,43406.4,1438], + [32472.99,43383.188,1437], + [32473.14,43359.977,1436], + [32473.24,43336.766,1435], + [32473.36,43476.033,1441], + [32635.04,43615.366,1447], + [32635.3,43592.155,1446], + [32635.5,43568.944,1445], + [32635.63,43545.733,1444], + [32635.9,43522.522,1443], + [32636.04,43499.311,1442], + [32636.19,43638.577,1448], + [32798.25,43777.9,1454], + [32798.51,43754.688,1453], + [32798.62,43731.477,1452], + [32798.74,43708.266,1451], + [32798.98,43685.055,1450], + [32799.09,43661.844,1449], + [32799.22,43801.111,1455], + [32962.33,43940.444,1461], + [32963.18,43917.233,1460], + [32963.36,43894.022,1459], + [32963.49,43870.811,1458], + [32963.61,43847.6,1457], + [32963.74,43824.388,1456], + [32963.87,43963.655,1462], + [33125.55,43986.933,1463], + [33125.63,44102.988,1468], + [33125.81,44079.777,1467], + [33125.88,44056.566,1466], + [33125.97,44033.355,1465], + [33126.06,44010.144,1464], + [33126.15,44126.2,1469], + [33287.84,44265.522,1475], + [33288.12,44242.311,1474], + [33288.22,44219.1,1473], + [33288.32,44195.888,1472], + [33288.51,44172.677,1471], + [33288.6,44149.466,1470], + [33288.75,44288.733,1476], + [33453.51,44428.066,1482], + [33454.39,44404.855,1481], + [33454.56,44381.644,1480], + [33454.68,44358.433,1479], + [33454.88,44335.222,1478], + [33454.98,44312.011,1477], + [33455.11,44451.277,1483], + [33613.18,44590.6,1489], + [33614.42,44567.388,1488], + [33614.71,44544.177,1487], + [33614.88,44520.966,1486], + [33615.02,44497.755,1485], + [33615.18,44474.544,1484], + [33615.34,44613.811,1490], + [33775.06,44776.355,1497], + [33775.2,44753.144,1496], + [33775.44,44729.933,1495], + [33775.57,44706.722,1494], + [33775.69,44683.511,1493], + [33775.8,44660.3,1492], + [33775.94,44637.088,1491], + [33938.08,44915.677,1503], + [33938.3,44892.466,1502], + [33938.39,44869.255,1501], + [33938.55,44846.044,1500], + [33938.64,44822.833,1499], + [33938.79,44799.622,1498], + [33938.94,44938.888,1504], + [34098.71,45078.222,1510], + [34098.93,45055.011,1509], + [34099.04,45031.8,1508], + [34099.24,45008.588,1507], + [34099.34,44985.377,1506], + [34099.45,44962.166,1505], + [34099.57,45101.433,1511], + [34260.46,45240.766,1517], + [34260.69,45217.555,1516], + [34260.81,45194.344,1515], + [34260.93,45171.133,1514], + [34261.16,45147.922,1513], + [34261.26,45124.711,1512], + [34261.39,45263.977,1518], + [34425.51,45287.244,1519], + [34425.7,45403.3,1524], + [34426.04,45380.088,1523], + [34426.2,45356.877,1522], + [34426.35,45333.666,1521], + [34426.5,45310.455,1520], + [34426.71,45426.511,1525], + [34589.27,45565.844,1531], + [34589.57,45542.633,1530], + [34591.72,45519.422,1529], + [34591.97,45496.211,1528], + [34592.09,45473,1527], + [34592.22,45449.788,1526], + [34592.38,45589.055,1532], + [34747.99,45728.377,1538], + [34748.24,45705.166,1537], + [34748.34,45681.955,1536], + [34748.51,45658.744,1535], + [34748.6,45635.533,1534], + [34748.7,45612.322,1533], + [34748.89,45751.588,1539], + [34910.41,45890.922,1545], + [34910.63,45867.711,1544], + [34910.74,45844.5,1543], + [34910.93,45821.288,1542], + [34911.05,45798.077,1541], + [34911.16,45774.866,1540], + [34911.31,45914.133,1546] + ] +} diff --git a/test/audio-quality/traces/relay-bbb.json b/test/audio-quality/traces/relay-bbb.json new file mode 100644 index 0000000000..b18b143ab6 --- /dev/null +++ b/test/audio-quality/traces/relay-bbb.json @@ -0,0 +1,1954 @@ +{ + "version": 1, + "source": "https://cdn.moq.dev/demo bbb.hang, the public demo's MPEG-TS import, recorded 2026-09-29", + "description": "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured.", + "transport": "webtransport", + "rtt": null, + "config": { + "codec": "mp4a.40.2", + "container": { + "kind": "legacy" + }, + "description": "1210", + "sampleRate": 44100, + "numberOfChannels": 2, + "bitrate": 139613, + "jitter": 186 + }, + "arrivals": [ + [0,234535,9642], + [2.98,240014.922,9878], + [9.78,240038.133,9879], + [42.28,240061.344,9880], + [42.65,234558.211,9643], + [42.87,240084.555,9881], + [43.17,240107.766,9882], + [43.41,240131.044,9883], + [43.69,240154.255,9884], + [43.97,240177.466,9885], + [44.19,240200.677,9886], + [44.4,234581.422,9644], + [44.65,240223.888,9887], + [44.85,240247.1,9888], + [45.08,240270.311,9889], + [46.07,240293.588,9890], + [46.25,240316.8,9891], + [46.46,240340.011,9892], + [46.65,240363.222,9893], + [47.12,240386.433,9894], + [47.32,234604.633,9645], + [47.78,240409.644,9895], + [47.99,240432.855,9896], + [48.33,234627.911,9646], + [48.59,240456.122,9897], + [48.81,240479.333,9898], + [49,240502.544,9899], + [49.2,234651.122,9647], + [49.41,240525.755,9900], + [49.6,240548.966,9901], + [49.81,234674.333,9648], + [50.03,240572.177,9902], + [50.27,240595.388,9903], + [50.55,234697.544,9649], + [51.48,240618.666,9904], + [51.71,235394.133,9679], + [51.88,234720.755,9650], + [52.03,234883.3,9657], + [52.18,234743.966,9651], + [52.38,234767.177,9652], + [52.59,234326.033,9633], + [52.85,234790.455,9653], + [53.07,234349.244,9634], + [53.31,234372.455,9635], + [53.54,234813.666,9654], + [53.88,234395.666,9636], + [54.16,234418.877,9637], + [57.75,234442.155,9638], + [58.49,234465.366,9639], + [58.78,238900.355,9830], + [58.93,234860.088,9656], + [59.06,234488.577,9640], + [59.42,234511.788,9641], + [59.59,234836.877,9655], + [59.74,238969.988,9833], + [59.91,238993.266,9834], + [60.05,239039.688,9836], + [60.19,239062.9,9837], + [60.34,239086.111,9838], + [60.47,239109.322,9839], + [60.63,239132.533,9840], + [60.83,239155.811,9841], + [60.97,239179.022,9842], + [61.11,239202.233,9843], + [61.25,239225.444,9844], + [61.38,239248.655,9845], + [61.52,239271.866,9846], + [61.66,239504.1,9856], + [61.81,239527.311,9857], + [61.97,239550.522,9858], + [62.11,239573.733,9859], + [62.34,239596.944,9860], + [62.52,239620.155,9861], + [62.71,239295.077,9847], + [63.2,239318.344,9848], + [63.37,239341.555,9849], + [63.53,239643.422,9862], + [63.68,239666.633,9863], + [63.9,239689.844,9864], + [64.05,239364.766,9850], + [64.75,239387.977,9851], + [64.98,239411.188,9852], + [65.18,239713.055,9865], + [65.37,239736.266,9866], + [65.57,239759.477,9867], + [65.76,239782.688,9868], + [66.32,239805.966,9869], + [66.56,239829.177,9870], + [66.77,239434.4,9853], + [67,239457.611,9854], + [67.21,239480.888,9855], + [67.43,239852.388,9871], + [67.65,239875.6,9872], + [67.88,239898.811,9873], + [68.09,239922.022,9874], + [68.34,239945.233,9875], + [68.76,239968.5,9876], + [68.99,239991.711,9877], + [69.18,238761.033,9824], + [69.35,238714.611,9822], + [69.53,238737.822,9823], + [69.72,238784.244,9825], + [69.92,238807.455,9826], + [70.13,238830.722,9827], + [70.5,238853.933,9828], + [70.71,238877.144,9829], + [77.44,238923.566,9831], + [77.63,238946.777,9832], + [77.8,239016.477,9835], + [77.98,238644.911,9819], + [78.14,238668.188,9820], + [78.3,238691.4,9821], + [78.45,238598.488,9817], + [78.6,238621.7,9818], + [78.76,237971.544,9790], + [78.94,237994.755,9791], + [79.12,238018.033,9792], + [79.31,238041.244,9793], + [79.52,238064.455,9794], + [79.71,238087.666,9795], + [79.88,238110.877,9796], + [80.04,238134.088,9797], + [80.19,238157.3,9798], + [80.34,238180.566,9799], + [80.52,238203.777,9800], + [80.69,238226.988,9801], + [80.88,238250.2,9802], + [81.05,238273.411,9803], + [81.25,238296.622,9804], + [81.41,238319.833,9805], + [81.63,238343.111,9806], + [81.81,238366.322,9807], + [81.98,238389.533,9808], + [82.22,238412.744,9809], + [82.38,238435.955,9810], + [82.54,238459.166,9811], + [82.77,238482.377,9812], + [82.99,238505.644,9813], + [83.16,238528.855,9814], + [83.37,238552.066,9815], + [83.56,238575.277,9816], + [83.72,237483.933,9769], + [83.96,237182.066,9756], + [84.13,237205.277,9757], + [84.63,237228.555,9758], + [84.8,237251.766,9759], + [85.95,237274.977,9760], + [86.24,237298.188,9761], + [86.44,237321.4,9762], + [86.63,237344.611,9763], + [86.9,237367.822,9764], + [87.11,237391.088,9765], + [87.3,237414.3,9766], + [87.47,236044.288,9707], + [87.69,236067.5,9708], + [87.88,237437.511,9767], + [88.07,237507.144,9770], + [88.27,237530.355,9771], + [88.46,237553.633,9772], + [88.63,237576.844,9773], + [88.85,236090.777,9709], + [89.02,236113.988,9710], + [89.19,236137.2,9711], + [89.37,236160.411,9712], + [89.55,237600.055,9774], + [89.72,236183.622,9713], + [89.9,236206.833,9714], + [90.1,236230.044,9715], + [90.5,236276.522,9717], + [90.72,236299.733,9718], + [91.03,236346.155,9720], + [91.19,237623.266,9775], + [91.42,236392.577,9722], + [91.57,236415.855,9723], + [91.8,236439.066,9724], + [91.95,236462.277,9725], + [92.09,236485.488,9726], + [92.32,237692.944,9778], + [92.5,236508.7,9727], + [92.82,235231.588,9672], + [93.02,235812.111,9697], + [93.29,236555.122,9729], + [93.46,236578.388,9730], + [93.78,236624.811,9732], + [94,236648.022,9733], + [94.2,236671.233,9734], + [94.49,237739.366,9780], + [94.66,237762.577,9781], + [95.03,236694.444,9735], + [95.28,237809,9783], + [95.5,236717.655,9736], + [95.75,237832.211,9784], + [95.97,236740.933,9737], + [96.32,236787.355,9739], + [96.57,236810.566,9740], + [96.97,236856.988,9742], + [97.19,237855.488,9785], + [97.4,236880.2,9743], + [97.57,236903.466,9744], + [97.82,236926.677,9745], + [98.03,236949.888,9746], + [98.27,236973.1,9747], + [98.5,236996.311,9748], + [98.82,237019.522,9749], + [99.15,237042.733,9750], + [99.31,237066.011,9751], + [99.5,237089.222,9752], + [99.68,237112.433,9753], + [99.86,237135.644,9754], + [100.03,237158.855,9755], + [100.2,237878.7,9786], + [100.39,237901.911,9787], + [102.17,237925.122,9788], + [102.47,237948.333,9789], + [102.69,237460.722,9768], + [102.9,235254.8,9673], + [103.09,235278.077,9674], + [103.92,235301.288,9675], + [104.1,235324.5,9676], + [104.32,235347.711,9677], + [104.5,235370.922,9678], + [104.65,235161.955,9669], + [104.8,235185.166,9670], + [104.97,235208.377,9671], + [105.12,235858.533,9699], + [105.29,235881.744,9700], + [105.46,235904.955,9701], + [105.61,235928.233,9702], + [105.79,235951.444,9703], + [105.97,235974.655,9704], + [106.14,235997.866,9705], + [106.34,236021.077,9706], + [106.56,234906.511,9658], + [106.76,234929.722,9659], + [106.95,234953,9660], + [107.12,234976.211,9661], + [107.34,234999.422,9662], + [107.52,235022.633,9663], + [107.71,235045.844,9664], + [107.93,235069.055,9665], + [108.14,235092.266,9666], + [108.32,235115.533,9667], + [108.54,235138.744,9668], + [109.17,235417.344,9680], + [109.4,235440.611,9681], + [109.59,235463.822,9682], + [109.8,240897.255,9916], + [110.02,240920.466,9917], + [110.24,240943.744,9918], + [110.45,240966.955,9919], + [110.68,240990.166,9920], + [110.9,241013.377,9921], + [111.12,241036.588,9922], + [111.34,241059.8,9923], + [111.58,241083.011,9924], + [111.85,241106.277,9925], + [112.1,241129.488,9926], + [112.36,241152.7,9927], + [115.86,235487.033,9683], + [116.5,235510.244,9684], + [116.87,235533.455,9685], + [117.19,235556.666,9686], + [117.55,235579.877,9687], + [117.84,235603.155,9688], + [118.08,235626.366,9689], + [118.32,235649.577,9690], + [118.58,235672.788,9691], + [118.82,235696,9692], + [119.11,235719.211,9693], + [119.44,235742.422,9694], + [119.7,235765.688,9695], + [120.07,235788.9,9696], + [120.42,241175.911,9928], + [120.69,243474.677,10027], + [121.3,242824.522,9999], + [121.55,243242.511,10017], + [121.81,242847.733,10000], + [122.07,242801.311,9998], + [122.36,242870.944,10001], + [122.63,243265.722,10018], + [122.96,243219.3,10016], + [123.26,243288.933,10019], + [123.53,242894.222,10002], + [123.88,243312.144,10020], + [124.18,243335.355,10021], + [124.49,242917.433,10003], + [125.22,243358.566,10022], + [125.51,242522.655,9986], + [125.78,242940.644,10004], + [126.09,242545.866,9987], + [126.36,242569.144,9988], + [126.65,242592.355,9989], + [127.34,242615.566,9990], + [128.12,242638.777,9991], + [128.82,242661.988,9992], + [129.38,243381.833,10023], + [129.97,242685.2,9993], + [131.38,242708.411,9994], + [132.19,242731.677,9995], + [132.6,242754.888,9996], + [132.97,242778.1,9997], + [133.36,242499.444,9985], + [133.72,242476.233,9984], + [134.14,242453.022,9983], + [134.47,242429.811,9982], + [134.77,242406.6,9981], + [135.22,242963.855,10005], + [135.52,242987.066,10006], + [136.13,243405.044,10024], + [136.43,243428.255,10025], + [136.72,243010.277,10007], + [137.05,243033.488,10008], + [137.41,243451.466,10026], + [137.75,243079.966,10010], + [138.07,243103.177,10011], + [138.36,243497.888,10028], + [138.66,243521.1,10029], + [139,243126.388,10012], + [139.59,243149.6,10013], + [139.93,243544.377,10030], + [140.28,243172.811,10014], + [140.62,240641.877,9905], + [140.94,240665.088,9906], + [141.25,240688.3,9907], + [141.59,240711.511,9908], + [141.92,240734.722,9909], + [142.25,240757.933,9910], + [142.38,237785.788,9782], + [143.09,237716.155,9779], + [143.4,237669.688,9777], + [143.68,237646.477,9776], + [143.97,236833.777,9741], + [144.26,236764.144,9738], + [144.55,236601.6,9731], + [144.83,236531.911,9728], + [145.11,236369.366,9721], + [145.84,236322.944,9719], + [146.35,236253.311,9716], + [146.74,235835.322,9698], + [147.43,240781.2,9911], + [147.84,240804.411,9912], + [148.72,240827.622,9913], + [149.34,240850.833,9914], + [150.47,243056.755,10009], + [151.16,242127.944,9969], + [151.94,242151.155,9970], + [152.75,242174.366,9971], + [153.65,242197.577,9972], + [154.65,242220.788,9973], + [155.68,242244.055,9974], + [156.55,242267.266,9975], + [157.66,242290.477,9976], + [158.09,242313.688,9977], + [158.54,242336.9,9978], + [159.02,242360.111,9979], + [159.44,242383.322,9980], + [159.85,241918.977,9960], + [160.5,241942.188,9961], + [161.22,241965.4,9962], + [161.93,241988.611,9963], + [162.73,242011.822,9964], + [163.45,242035.033,9965], + [164.22,242058.244,9966], + [164.95,242081.522,9967], + [165.68,242104.733,9968], + [166.67,241779.655,9954], + [167.23,241802.866,9955], + [167.68,241826.077,9956], + [168.12,241849.288,9957], + [168.65,241872.5,9958], + [169.09,241895.711,9959], + [169.49,244171.266,10057], + [169.9,241570.633,9945], + [170.39,241593.9,9946], + [171.01,241617.111,9947], + [171.49,241640.322,9948], + [171.91,241663.533,9949], + [172.32,241408.088,9938], + [172.88,243776.555,10040], + [173.72,241431.366,9939], + [174.18,241454.577,9940], + [174.6,243799.766,10041], + [175.03,243822.977,10042], + [175.45,241477.788,9941], + [175.89,243846.188,10043], + [176.34,243869.455,10044], + [176.76,241501,9942], + [178.37,243892.666,10045], + [178.8,243915.877,10046], + [179.25,241524.211,9943], + [179.69,243939.088,10047], + [180.12,243962.3,10048], + [181.03,243985.511,10049], + [181.56,244008.722,10050], + [182,244032,10051], + [182.44,244055.211,10052], + [182.93,244078.422,10053], + [183.38,244101.633,10054], + [183.84,241547.422,9944], + [184.41,244124.844,10055], + [184.87,243753.344,10039], + [185.34,241384.877,9937], + [185.82,243730.133,10038], + [186.28,243706.922,10037], + [187.33,241361.666,9936], + [228.22,241338.455,9935], + [228.43,243683.644,10036], + [228.61,243660.433,10035], + [228.84,241315.244,9934], + [229.06,241292.033,9933], + [229.27,243637.222,10034], + [229.5,243614.011,10033], + [229.69,241268.822,9932], + [229.88,244148.055,10056], + [230.08,241686.744,9950], + [230.25,241709.955,9951], + [230.42,241733.166,9952], + [230.59,241756.444,9953], + [230.77,244287.377,10062], + [230.94,244310.588,10063], + [231.12,244333.8,10064], + [231.28,243196.022,10015], + [231.78,243567.588,10031], + [231.98,241199.122,9929], + [232.15,241222.333,9930], + [232.32,243590.8,10032], + [232.5,241245.544,9931], + [232.69,244194.533,10058], + [232.9,244217.744,10059], + [233.09,244240.955,10060], + [233.3,244264.166,10061], + [233.5,244449.922,10069], + [233.69,244473.133,10070], + [233.88,244496.344,10071], + [234.07,244357.077,10065], + [234.27,244380.288,10066], + [234.47,244403.5,10067], + [234.7,244426.711,10068], + [235.54,240874.044,9915], + [309.69,244542.822,10073], + [309.93,244566.033,10074], + [310.03,244589.244,10075], + [310.13,244612.455,10076], + [310.31,244635.666,10077], + [310.41,244658.877,10078], + [310.59,244519.611,10072], + [502.34,244705.366,10080], + [502.54,244728.577,10081], + [502.63,244751.788,10082], + [502.79,244775,10083], + [502.88,244798.211,10084], + [502.97,244821.422,10085], + [503.07,244682.155,10079], + [638.06,244867.911,10087], + [638.23,244891.122,10088], + [638.32,244960.755,10091], + [638.47,244914.333,10089], + [638.58,244937.544,10090], + [638.66,244844.7,10086], + [638.77,244983.966,10092], + [818.46,245030.444,10094], + [818.66,245053.655,10095], + [818.75,245076.866,10096], + [818.87,245100.077,10097], + [819.04,245123.288,10098], + [819.12,245146.5,10099], + [819.22,245007.233,10093], + [957.28,245192.988,10101], + [957.49,245216.2,10102], + [957.57,245239.411,10103], + [957.75,245262.622,10104], + [957.86,245285.833,10105], + [958.05,245309.044,10106], + [958.16,245169.777,10100], + [1107.13,245355.522,10108], + [1107.32,245378.733,10109], + [1107.44,245401.944,10110], + [1107.6,245425.155,10111], + [1107.71,245448.366,10112], + [1107.84,245471.577,10113], + [1108.03,245332.311,10107], + [1261.86,245518.066,10115], + [1262.11,245541.277,10116], + [1262.25,245564.488,10117], + [1262.41,245587.7,10118], + [1262.5,245610.911,10119], + [1262.6,245634.122,10120], + [1262.76,245494.855,10114], + [1424.35,245750.233,10125], + [1424.53,245773.444,10126], + [1424.63,245796.655,10127], + [1424.72,245657.388,10121], + [1424.91,245680.6,10122], + [1424.99,245703.811,10123], + [1425.09,245727.022,10124], + [1588.68,245843.144,10129], + [1588.91,245866.355,10130], + [1589.03,245889.566,10131], + [1589.19,245912.777,10132], + [1589.28,245935.988,10133], + [1589.37,245959.2,10134], + [1589.47,245819.933,10128], + [1753.91,246005.688,10136], + [1754.11,246028.9,10137], + [1754.21,246052.111,10138], + [1754.37,246075.322,10139], + [1754.46,246098.533,10140], + [1754.57,246121.744,10141], + [1754.71,245982.477,10135], + [1921.03,246168.222,10143], + [1921.34,246191.433,10144], + [1921.44,246214.644,10145], + [1921.66,246237.855,10146], + [1921.78,246261.066,10147], + [1921.9,246284.277,10148], + [1922.1,246145.011,10142], + [2087.59,246330.766,10150], + [2087.85,246353.977,10151], + [2087.97,246377.188,10152], + [2088.12,246400.4,10153], + [2088.21,246423.611,10154], + [2088.3,246446.822,10155], + [2088.38,246307.555,10149], + [2255.17,246493.3,10157], + [2255.32,246516.511,10158], + [2255.41,246586.144,10161], + [2255.54,246539.722,10159], + [2255.63,246562.933,10160], + [2255.71,246470.088,10156], + [2255.9,246609.355,10162], + [2424.94,246655.844,10164], + [2425.25,246679.055,10165], + [2425.35,246702.266,10166], + [2425.51,246725.477,10167], + [2425.59,246748.688,10168], + [2425.66,246771.9,10169], + [2425.73,246632.633,10163], + [2575.38,246818.377,10171], + [2575.52,246841.588,10172], + [2575.59,246864.8,10173], + [2575.72,246888.011,10174], + [2575.78,246911.222,10175], + [2575.88,246934.433,10176], + [2575.95,246795.166,10170], + [2749.89,246957.711,10177], + [2750.14,246980.922,10178], + [2750.25,247004.133,10179], + [2750.44,247027.344,10180], + [2750.56,247050.555,10181], + [2750.66,247073.766,10182], + [2750.78,247096.977,10183], + [2900.56,247120.255,10184], + [2900.73,247143.466,10185], + [2900.84,247166.677,10186], + [2900.99,247189.888,10187], + [2901.07,247213.1,10188], + [2901.15,247236.311,10189], + [2901.31,247259.522,10190], + [3071.44,247282.788,10191], + [3071.62,247306,10192], + [3071.7,247329.211,10193], + [3071.87,247352.422,10194], + [3071.95,247375.633,10195], + [3072.03,247398.844,10196], + [3072.17,247422.055,10197], + [3212.59,247468.544,10199], + [3212.79,247445.333,10198], + [3212.9,247514.966,10201], + [3213.05,247491.755,10200], + [3213.15,247538.177,10202], + [3213.24,247561.388,10203], + [3213.38,247584.6,10204], + [3378.91,247631.077,10206], + [3379.09,247654.288,10207], + [3379.18,247607.866,10205], + [3379.3,247677.5,10208], + [3379.4,247723.922,10210], + [3379.48,247747.133,10211], + [3379.6,247700.711,10209], + [3543.59,247770.411,10212], + [3544.84,247886.466,10217], + [3544.94,247909.677,10218], + [3545.05,247793.622,10213], + [3545.15,247816.833,10214], + [3545.24,247840.044,10215], + [3545.31,247863.255,10216], + [3716.75,248072.211,10225], + [3717.96,248049,10224], + [3718.09,247932.944,10219], + [3718.38,248025.788,10223], + [3718.49,248002.577,10222], + [3718.6,247979.366,10221], + [3718.68,247956.155,10220], + [3878.51,248118.7,10227], + [3878.66,248141.911,10228], + [3878.72,248165.122,10229], + [3878.87,248188.333,10230], + [3878.94,248211.544,10231], + [3879,248234.755,10232], + [3879.09,248095.488,10226], + [4024.78,248258.033,10233], + [4024.93,248397.3,10239], + [4025.03,248281.244,10234], + [4025.12,248304.455,10235], + [4025.22,248374.088,10238], + [4025.29,248327.666,10236], + [4025.37,248350.877,10237], + [4186.69,248420.566,10240], + [4186.87,248443.777,10241], + [4186.95,248466.988,10242], + [4187.18,248490.2,10243], + [4187.28,248513.411,10244], + [4187.41,248536.622,10245], + [4187.53,248559.833,10246], + [4347.91,248583.111,10247], + [4348.73,248606.322,10248], + [4348.86,248629.533,10249], + [4348.99,248652.744,10250], + [4349.12,248675.955,10251], + [4349.21,248699.166,10252], + [4349.28,248722.377,10253], + [4532.82,248768.855,10255], + [4533,248792.066,10256], + [4533.07,248815.277,10257], + [4533.21,248838.488,10258], + [4533.27,248861.7,10259], + [4533.4,248884.911,10260], + [4533.52,248745.644,10254], + [4692.91,248908.188,10261], + [4693.07,249047.455,10267], + [4693.18,248931.4,10262], + [4693.27,248954.611,10263], + [4693.34,249024.244,10266], + [4693.5,248977.822,10264], + [4693.58,249001.033,10265], + [4866.71,249070.722,10268], + [4866.88,249093.933,10269], + [4866.96,249117.144,10270], + [4867.09,249140.355,10271], + [4867.16,249163.566,10272], + [4867.24,249186.777,10273], + [4867.31,249209.988,10274], + [5018.38,249326.111,10279], + [5018.5,249349.322,10280], + [5018.59,249372.533,10281], + [5018.65,249233.266,10275], + [5018.75,249302.9,10278], + [5018.82,249256.477,10276], + [5018.9,249279.688,10277], + [5184.16,249395.811,10282], + [5184.35,249419.022,10283], + [5184.44,249442.233,10284], + [5184.53,249465.444,10285], + [5184.59,249488.655,10286], + [5184.72,249511.866,10287], + [5184.79,249535.077,10288], + [5338.05,249558.344,10289], + [5338.18,249581.555,10290], + [5338.29,249604.766,10291], + [5338.36,249627.977,10292], + [5338.5,249651.188,10293], + [5338.57,249674.4,10294], + [5338.65,249697.611,10295], + [5503.97,249720.888,10296], + [5504.18,249744.1,10297], + [5504.28,249767.311,10298], + [5504.46,249790.522,10299], + [5504.56,249813.733,10300], + [5504.65,249836.944,10301], + [5504.84,249860.155,10302], + [5650.21,249883.422,10303], + [5650.35,249906.633,10304], + [5650.45,249929.844,10305], + [5650.53,249953.055,10306], + [5650.65,249976.266,10307], + [5650.72,249999.477,10308], + [5650.8,250022.688,10309], + [5812.98,250045.966,10310], + [5813.17,250069.177,10311], + [5813.27,250092.388,10312], + [5813.4,250115.6,10313], + [5813.47,250138.811,10314], + [5813.55,250162.022,10315], + [5813.69,250185.233,10316], + [5974.33,250208.5,10317], + [5974.48,250347.766,10323], + [5974.59,250324.555,10322], + [5974.67,250301.344,10321], + [5974.8,250254.922,10319], + [5974.93,250231.711,10318], + [5975.03,250278.133,10320], + [6137.69,250371.044,10324], + [6137.88,250394.255,10325], + [6137.97,250417.466,10326], + [6138.11,250440.677,10327], + [6138.19,250510.311,10330], + [6138.28,250487.1,10329], + [6138.43,250463.888,10328], + [6304.4,250533.588,10331], + [6304.56,250556.8,10332], + [6304.64,250580.011,10333], + [6304.77,250603.222,10334], + [6304.9,250626.433,10335], + [6304.99,250649.644,10336], + [6305.13,250672.855,10337], + [6462.38,250696.122,10338], + [6463.72,250719.333,10339], + [6463.87,250742.544,10340], + [6463.96,250765.755,10341], + [6464.09,250788.966,10342], + [6464.17,250812.177,10343], + [6464.27,250835.388,10344], + [6629,250997.933,10351], + [6629.24,250881.877,10346], + [6629.38,250905.088,10347], + [6629.53,250974.722,10350], + [6629.8,250928.3,10348], + [6629.94,250951.511,10349], + [6630.08,250858.666,10345], + [6795.28,251021.2,10352], + [6795.44,251044.411,10353], + [6795.52,251067.622,10354], + [6795.66,251090.833,10355], + [6795.75,251114.044,10356], + [6795.85,251137.255,10357], + [6795.94,251160.466,10358], + [6962.14,251253.377,10362], + [6962.27,251276.588,10363], + [6962.33,251299.8,10364], + [6962.44,251323.011,10365], + [6962.5,251206.955,10360], + [6962.56,251230.166,10361], + [6962.68,251183.744,10359], + [7128.44,251346.277,10366], + [7128.59,251485.544,10372], + [7128.66,251462.333,10371], + [7128.79,251439.122,10370], + [7128.87,251369.488,10367], + [7128.95,251392.7,10368], + [7129.03,251415.911,10369], + [7294.6,251508.822,10373], + [7294.77,251532.033,10374], + [7294.87,251601.666,10377], + [7295,251555.244,10375], + [7295.12,251578.455,10376], + [7295.2,251624.877,10378], + [7295.28,251648.088,10379], + [7437.72,251694.577,10381], + [7437.94,251717.788,10382], + [7438.03,251741,10383], + [7438.17,251764.211,10384], + [7438.24,251787.422,10385], + [7438.32,251810.633,10386], + [7438.39,251671.366,10380], + [7602.92,251833.9,10387], + [7603.62,251973.166,10393], + [7604.08,251857.111,10388], + [7604.31,251880.322,10389], + [7604.6,251949.955,10392], + [7604.86,251903.533,10390], + [7605.18,251926.744,10391], + [7771.25,251996.444,10394], + [7771.68,252019.655,10395], + [7771.87,252042.866,10396], + [7772.03,252066.077,10397], + [7772.31,252089.288,10398], + [7772.43,252112.5,10399], + [7772.57,252135.711,10400], + [7933.8,252158.977,10401], + [7933.97,252182.188,10402], + [7934.03,252205.4,10403], + [7934.16,252228.611,10404], + [7934.22,252251.822,10405], + [7934.28,252275.033,10406], + [7934.36,252298.244,10407], + [8092.4,252344.733,10409], + [8092.54,252367.944,10410], + [8092.61,252391.155,10411], + [8092.75,252414.366,10412], + [8092.82,252437.577,10413], + [8093.09,252460.788,10414], + [8093.21,252321.522,10408], + [8254.12,252600.111,10420], + [8254.28,252623.322,10421], + [8254.37,252484.055,10415], + [8254.54,252530.477,10417], + [8254.62,252553.688,10418], + [8267.91,252576.9,10419], + [8268.05,252507.266,10416], + [8466.57,252785.866,10428], + [8466.72,252646.6,10422], + [8466.8,252669.811,10423], + [8466.92,252739.444,10426], + [8466.99,252762.655,10427], + [8467.06,252693.022,10424], + [8467.19,252716.233,10425], + [8586.91,252832.355,10430], + [8587.04,252855.566,10431], + [8587.1,252878.777,10432], + [8587.17,252901.988,10433], + [8587.31,252925.2,10434], + [8587.37,252948.411,10435], + [8587.46,252809.144,10429], + [8760.22,253064.522,10440], + [8760.37,253110.944,10442], + [8760.44,252994.888,10437], + [8760.55,253018.1,10438], + [8760.63,253087.733,10441], + [8760.69,253041.311,10439], + [8760.77,252971.677,10436], + [8967.66,253203.855,10446], + [8967.84,253227.066,10447], + [8968,253250.277,10448], + [8968.13,253134.222,10443], + [8968.23,253296.7,10450], + [8968.4,253157.433,10444], + [8968.49,253180.644,10445], + [8968.58,253273.488,10449], + [9092.01,253459.244,10457], + [9111.41,253412.822,10455], + [9111.57,253319.977,10451], + [9111.65,253343.188,10452], + [9111.78,253366.4,10453], + [9111.85,253389.611,10454], + [9111.93,253436.033,10456], + [9255.93,253482.522,10458], + [9256.09,253528.944,10460], + [9256.16,253552.155,10461], + [9256.34,253575.366,10462], + [9256.41,253598.577,10463], + [9256.49,253621.788,10464], + [9256.62,253505.733,10459], + [9421.78,253668.266,10466], + [9421.96,253691.477,10467], + [9422.04,253714.688,10468], + [9422.12,253737.9,10469], + [9422.31,253761.111,10470], + [9422.4,253784.322,10471], + [9422.5,253645.055,10465], + [9588.17,253830.811,10473], + [9588.44,253854.022,10474], + [9588.53,253877.233,10475], + [9588.81,253900.444,10476], + [9589.1,253923.655,10477], + [9589.18,253946.866,10478], + [9589.26,253807.6,10472], + [9754.88,254062.977,10483], + [9755.11,254086.188,10484], + [9755.25,254109.4,10485], + [9755.38,253993.344,10480], + [9755.45,254016.555,10481], + [9755.53,253970.133,10479], + [9755.6,254039.766,10482], + [9937.53,254225.522,10490], + [9937.72,254132.677,10486], + [9937.82,254155.888,10487], + [9937.94,254179.1,10488], + [9938.07,254248.733,10491], + [9938.15,254271.944,10492], + [9938.22,254202.311,10489], + [10093.97,254434.488,10499], + [10094.12,254295.222,10493], + [10094.19,254318.433,10494], + [10094.3,254341.644,10495], + [10094.36,254364.855,10496], + [10094.43,254388.066,10497], + [10094.5,254411.277,10498], + [10226.07,254480.966,10501], + [10226.2,254504.177,10502], + [10226.3,254527.388,10503], + [10226.35,254550.6,10504], + [10226.47,254573.811,10505], + [10226.53,254597.022,10506], + [10226.6,254457.755,10500], + [10410.06,254643.511,10508], + [10410.21,254666.722,10509], + [10410.28,254736.355,10512], + [10410.41,254689.933,10510], + [10410.47,254713.144,10511], + [10410.55,254620.3,10507], + [10410.62,254759.566,10513], + [10563.9,254806.044,10515], + [10564.03,254829.255,10516], + [10564.1,254852.466,10517], + [10564.22,254875.677,10518], + [10564.29,254898.888,10519], + [10564.35,254922.1,10520], + [10564.49,254782.833,10514], + [10729.24,254968.588,10522], + [10729.44,254991.8,10523], + [10729.63,255015.011,10524], + [10729.77,255038.222,10525], + [10729.97,255084.644,10527], + [10730.2,254945.377,10521], + [10730.31,255061.433,10526], + [10888.53,255131.122,10529], + [10888.82,255154.333,10530], + [10888.93,255177.544,10531], + [10889,255200.755,10532], + [10889.13,255223.966,10533], + [10889.19,255247.177,10534], + [10889.43,255107.911,10528], + [11084.73,255293.666,10536], + [11084.97,255316.877,10537], + [11085.09,255340.088,10538], + [11085.3,255363.3,10539], + [11085.43,255386.511,10540], + [11085.55,255409.722,10541], + [11085.69,255270.455,10535], + [11220.5,255456.211,10543], + [11220.65,255479.422,10544], + [11220.75,255502.633,10545], + [11220.84,255525.844,10546], + [11220.97,255549.055,10547], + [11221.06,255572.266,10548], + [11221.16,255433,10542], + [11379.03,255618.744,10550], + [11379.19,255641.955,10551], + [11379.27,255665.166,10552], + [11379.39,255688.377,10553], + [11379.47,255711.588,10554], + [11379.55,255734.8,10555], + [11379.63,255595.533,10549], + [11524.88,255781.288,10557], + [11525.08,255804.5,10558], + [11525.2,255827.711,10559], + [11525.31,255850.922,10560], + [11525.52,255874.133,10561], + [11525.61,255897.344,10562], + [11525.7,255758.077,10556], + [11688,255943.822,10564], + [11688.25,255967.033,10565], + [11688.36,255990.244,10566], + [11688.55,256013.455,10567], + [11688.65,256036.666,10568], + [11688.76,256059.877,10569], + [11688.88,255920.611,10563], + [11849.95,256106.366,10571], + [11850.16,256129.577,10572], + [11850.28,256152.788,10573], + [11850.42,256176,10574], + [11850.5,256199.211,10575], + [11850.56,256222.422,10576], + [11850.65,256083.155,10570], + [12012.25,256268.9,10578], + [12012.43,256292.111,10579], + [12012.53,256315.322,10580], + [12012.69,256338.533,10581], + [12012.77,256361.744,10582], + [12012.89,256384.955,10583], + [12012.99,256245.688,10577], + [12204.19,256431.444,10585], + [12204.41,256454.655,10586], + [12204.49,256477.866,10587], + [12204.68,256501.077,10588], + [12204.78,256524.288,10589], + [12204.88,256547.5,10590], + [12204.98,256408.233,10584], + [12340.82,256593.988,10592], + [12341.63,256617.2,10593], + [12341.77,256686.833,10596], + [12341.88,256640.411,10594], + [12342,256663.622,10595], + [12342.08,256570.777,10591], + [12342.17,256710.044,10597], + [12512.62,256849.366,10603], + [12512.78,256872.577,10604], + [12516.18,256733.311,10598], + [12516.31,256756.522,10599], + [12516.43,256779.733,10600], + [12516.5,256802.944,10601], + [12516.63,256826.155,10602], + [12671.85,256965.488,10608], + [12672.04,256988.7,10609], + [12672.41,257011.911,10610], + [12672.53,257035.122,10611], + [12672.63,256895.855,10605], + [12672.72,256919.066,10606], + [12672.96,256942.277,10607], + [12840.08,257081.6,10613], + [12840.27,257104.811,10614], + [12840.38,257128.022,10615], + [12840.52,257151.233,10616], + [12840.59,257174.444,10617], + [12840.66,257197.655,10618], + [12840.75,257058.388,10612], + [13003.86,257360.2,10625], + [13004.03,257313.777,10623], + [13004.16,257336.988,10624], + [13004.24,257290.566,10622], + [13004.41,257267.355,10621], + [13004.5,257244.144,10620], + [13004.57,257220.933,10619], + [13150.1,257522.733,10632], + [13150.26,257383.466,10626], + [13150.37,257406.677,10627], + [13150.46,257429.888,10628], + [13150.56,257453.1,10629], + [13150.64,257476.311,10630], + [13150.72,257499.522,10631], + [13313.7,257685.277,10639], + [13314,257569.222,10634], + [13314.09,257592.433,10635], + [13314.16,257615.644,10636], + [13314.27,257638.855,10637], + [13314.37,257662.066,10638], + [13314.47,257546.011,10633], + [13476.18,257731.766,10641], + [13476.34,257754.977,10642], + [13476.44,257778.188,10643], + [13476.52,257801.4,10644], + [13476.63,257824.611,10645], + [13476.71,257847.822,10646], + [13476.78,257708.555,10640], + [13638.15,257894.3,10648], + [13638.41,257917.511,10649], + [13638.58,257987.144,10652], + [13638.87,257940.722,10650], + [13639.47,257963.933,10651], + [13639.6,257871.088,10647], + [13640.33,258010.355,10653], + [13801.11,258056.844,10655], + [13801.31,258080.055,10656], + [13801.58,258103.266,10657], + [13801.71,258126.477,10658], + [13801.85,258149.688,10659], + [13802.07,258172.9,10660], + [13802.21,258033.633,10654], + [13964.66,258312.222,10666], + [13964.8,258335.433,10667], + [13964.89,258289.011,10665], + [13965.01,258196.166,10661], + [13965.08,258219.377,10662], + [13965.15,258242.588,10663], + [13965.28,258265.8,10664], + [14133.5,258358.711,10668], + [14133.65,258405.133,10670], + [14133.72,258428.344,10671], + [14133.85,258451.555,10672], + [14133.93,258474.766,10673], + [14134,258497.977,10674], + [14134.07,258381.922,10669], + [14318.44,258544.455,10676], + [14318.69,258567.666,10677], + [14318.85,258590.877,10678], + [14318.93,258614.088,10679], + [14319,258637.3,10680], + [14319.19,258660.511,10681], + [14319.31,258521.244,10675], + [14491.59,258707,10683], + [14491.75,258730.211,10684], + [14491.84,258753.422,10685], + [14491.97,258776.633,10686], + [14492.04,258799.844,10687], + [14492.11,258846.266,10689], + [14492.25,258823.055,10688], + [14492.31,258683.788,10682], + [14677.09,258869.544,10690], + [14677.25,258892.755,10691], + [14677.33,258915.966,10692], + [14677.46,258939.177,10693], + [14677.53,258962.388,10694], + [14677.6,258985.6,10695], + [14820.59,259055.288,10698], + [14820.78,259078.5,10699], + [14820.94,259101.711,10700], + [14821.09,259124.922,10701], + [14821.16,259148.133,10702], + [14821.25,259008.866,10696], + [14821.33,259032.077,10697], + [14964.22,259287.466,10708], + [14964.37,259310.677,10709], + [14964.44,259171.411,10703], + [14964.56,259194.622,10704], + [14964.63,259217.833,10705], + [14964.7,259241.044,10706], + [14964.96,259264.255,10707], + [15100.84,259333.944,10710], + [15101.09,259357.155,10711], + [15101.2,259380.366,10712], + [15101.35,259403.577,10713], + [15101.45,259426.788,10714], + [15101.59,259450,10715], + [15101.71,259473.211,10716], + [15269.97,259519.7,10718], + [15270.12,259542.911,10719], + [15270.22,259566.122,10720], + [15270.29,259589.333,10721], + [15270.35,259612.544,10722], + [15270.49,259635.755,10723], + [15270.56,259496.488,10717], + [15425.1,259682.233,10725], + [15425.25,259705.444,10726], + [15425.33,259728.655,10727], + [15425.46,259751.866,10728], + [15425.52,259775.077,10729], + [15425.61,259798.288,10730], + [15425.69,259659.022,10724], + [15590.53,259844.777,10732], + [15590.66,259867.988,10733], + [15590.77,259891.2,10734], + [15590.88,259914.411,10735], + [15590.94,259937.622,10736], + [15591.06,259960.833,10737], + [15591.19,259821.566,10731], + [15755.92,260007.322,10739], + [15756.06,260030.533,10740], + [15756.16,260053.744,10741], + [15756.25,260076.955,10742], + [15756.34,260100.166,10743], + [15756.44,260123.377,10744], + [15756.56,259984.111,10738], + [15920.88,260146.644,10745], + [15921,260239.488,10749], + [15921.11,260262.7,10750], + [15921.22,260285.911,10751], + [15921.38,260216.277,10748], + [15921.54,260169.855,10746], + [15921.63,260193.066,10747], + [16087.75,260378.822,10755], + [16087.94,260425.244,10757], + [16088.03,260448.455,10758], + [16088.15,260309.188,10752], + [16088.22,260332.4,10753], + [16088.29,260355.611,10754], + [16088.37,260402.033,10756], + [16254.79,260494.933,10760], + [16254.97,260518.144,10761], + [16255.08,260541.355,10762], + [16255.18,260564.566,10763], + [16255.28,260587.777,10764], + [16255.41,260610.988,10765], + [16255.52,260471.722,10759], + [16421.4,260657.477,10767], + [16421.72,260680.688,10768], + [16421.94,260703.9,10769], + [16422.03,260727.111,10770], + [16422.11,260750.322,10771], + [16422.19,260773.533,10772], + [16422.37,260634.266,10766], + [16563.5,260820.011,10774], + [16563.69,260843.222,10775], + [16563.78,260866.433,10776], + [16563.89,260889.644,10777], + [16564.07,260912.855,10778], + [16564.16,260936.066,10779], + [16564.25,260796.8,10773], + [16725.87,260982.555,10781], + [16726.04,261005.766,10782], + [16726.13,261028.977,10783], + [16726.21,261052.188,10784], + [16726.35,261075.4,10785], + [16726.43,261098.611,10786], + [16726.53,260959.344,10780], + [16889.66,261145.1,10788], + [16889.81,261168.311,10789], + [16889.88,261191.522,10790], + [16889.96,261214.733,10791], + [16890.03,261237.944,10792], + [16890.1,261261.155,10793], + [16890.19,261121.888,10787], + [17051.54,261307.633,10795], + [17051.7,261330.844,10796], + [17051.78,261354.055,10797], + [17051.93,261377.266,10798], + [17052.01,261400.477,10799], + [17052.08,261423.688,10800], + [17052.25,261284.422,10794], + [17221.22,261493.388,10803], + [17221.41,261516.6,10804], + [17221.48,261539.811,10805], + [17221.62,261563.022,10806], + [17221.69,261586.233,10807], + [17221.76,261446.966,10801], + [17221.92,261470.177,10802], + [17379.66,261655.922,10810], + [17379.85,261679.133,10811], + [17379.97,261725.555,10813], + [17380.05,261748.766,10814], + [17380.19,261609.5,10808], + [17380.25,261632.711,10809], + [17380.33,261702.344,10812], + [17546.72,261795.255,10816], + [17546.99,261818.466,10817], + [17547.14,261841.677,10818], + [17547.27,261864.888,10819], + [17547.5,261888.1,10820], + [17547.6,261911.311,10821], + [17547.74,261772.044,10815], + [17711.03,261934.577,10822], + [17712.21,262027.422,10826], + [17712.38,262050.633,10827], + [17712.55,262073.844,10828], + [17712.66,261957.788,10823], + [17712.81,261981,10824], + [17712.91,262004.211,10825], + [17890.29,262213.177,10834], + [17890.45,262097.122,10829], + [17890.56,262120.333,10830], + [17890.63,262143.544,10831], + [17890.76,262166.755,10832], + [17890.84,262236.388,10835], + [17890.91,262189.966,10833], + [18045.28,262352.511,10840], + [18045.44,262375.722,10841], + [18045.55,262398.933,10842], + [18045.63,262282.877,10837], + [18045.75,262306.088,10838], + [18045.84,262259.666,10836], + [18045.97,262329.3,10839], + [18240.02,262515.044,10847], + [18240.16,262422.2,10843], + [18240.22,262445.411,10844], + [18240.35,262468.622,10845], + [18240.42,262538.255,10848], + [18240.5,262561.466,10849], + [18240.63,262491.833,10846], + [18352.12,262607.955,10851], + [18352.27,262631.166,10852], + [18352.35,262654.377,10853], + [18352.47,262677.588,10854], + [18352.55,262700.8,10855], + [18352.61,262724.011,10856], + [18352.69,262584.744,10850], + [18515.84,262793.7,10859], + [18516,262816.911,10860], + [18516.09,262840.122,10861], + [18516.22,262863.333,10862], + [18516.31,262886.544,10863], + [18516.38,262747.277,10857], + [18516.52,262770.488,10858], + [18676.4,263049.088,10870], + [18676.53,262909.822,10864], + [18676.63,262933.033,10865], + [18676.71,262956.244,10866], + [18676.85,262979.455,10867], + [18676.93,263002.666,10868], + [18677.04,263025.877,10869], + [18848.16,263165.2,10875], + [18848.31,263188.411,10876], + [18848.4,263211.622,10877], + [18848.52,263095.566,10872], + [18848.59,263118.777,10873], + [18848.66,263072.355,10871], + [18848.77,263141.988,10874], + [19026.36,263304.533,10881], + [19026.53,263234.9,10878], + [19026.62,263258.111,10879], + [19026.74,263327.744,10882], + [19026.82,263350.955,10883], + [19026.9,263281.322,10880], + [19027.04,263374.166,10884], + [19178.5,263467.077,10888], + [19178.71,263513.5,10890], + [19178.81,263536.711,10891], + [19179.01,263397.444,10885], + [19179.13,263420.655,10886], + [19179.25,263443.866,10887], + [19179.59,263490.288,10889], + [19355.32,263583.188,10893], + [19355.51,263606.4,10894], + [19355.6,263559.977,10892], + [19355.74,263629.611,10895], + [19355.82,263652.822,10896], + [19355.9,263676.033,10897], + [19355.97,263699.244,10898], + [19504.23,263722.522,10899], + [19505.66,263745.733,10900], + [19505.82,263768.944,10901], + [19505.95,263792.155,10902], + [19506.03,263815.366,10903], + [19506.17,263838.577,10904], + [19506.25,263861.788,10905], + [19672.07,263954.688,10909], + [19672.25,263977.9,10910], + [19672.4,264001.111,10911], + [19672.5,263885.055,10906], + [19672.6,263908.266,10907], + [19672.76,263931.477,10908], + [19672.84,264024.322,10912], + [19815.57,264070.811,10914], + [19815.74,264094.022,10915], + [19815.85,264117.233,10916], + [19815.94,264140.444,10917], + [19816.06,264163.655,10918], + [19816.16,264186.866,10919], + [19816.24,264047.6,10913], + [20050.35,264233.344,10921], + [20050.51,264256.555,10922], + [20050.59,264326.188,10925], + [20050.72,264279.766,10923], + [20050.8,264302.977,10924], + [20050.91,264210.133,10920], + [20051.05,264349.4,10926], + [20151.41,264465.522,10931], + [20151.57,264488.733,10932], + [20151.66,264511.944,10933], + [20151.78,264372.677,10927], + [20151.87,264395.888,10928], + [20151.95,264419.1,10929], + [20152.09,264442.311,10930], + [20307.34,264535.222,10934], + [20307.53,264558.433,10935], + [20307.68,264581.644,10936], + [20307.78,264604.855,10937], + [20307.94,264628.066,10938], + [20308.02,264651.277,10939], + [20308.1,264674.488,10940], + [20466.32,264720.966,10942], + [20466.5,264744.177,10943], + [20466.58,264813.811,10946], + [20466.72,264767.388,10944], + [20466.8,264790.6,10945], + [20466.89,264697.755,10941], + [20467.06,264837.022,10947], + [20641.56,264883.511,10949], + [20641.76,264906.722,10950], + [20641.88,264929.933,10951], + [20642.07,264953.144,10952], + [20642.19,264976.355,10953], + [20642.31,264999.566,10954], + [20642.42,264860.3,10948], + [20796,265138.888,10960], + [20796.17,265162.1,10961], + [20796.27,265115.677,10959], + [20796.44,265022.833,10955], + [20796.53,265046.044,10956], + [20796.63,265069.255,10957], + [20796.74,265092.466,10958], + [20980.12,265231.8,10964], + [20980.27,265208.588,10963], + [20980.37,265255.011,10965], + [20980.44,265278.222,10966], + [20980.6,265301.433,10967], + [20980.68,265324.644,10968], + [20980.73,265185.377,10962], + [21133.86,265347.911,10969], + [21134,265394.333,10971], + [21134.09,265417.544,10972], + [21134.25,265440.755,10973], + [21134.34,265463.966,10974], + [21134.44,265487.177,10975], + [21134.66,265371.122,10970], + [21301.55,265533.666,10977], + [21301.71,265556.877,10978], + [21301.82,265580.088,10979], + [21301.9,265603.3,10980], + [21302.02,265626.511,10981], + [21302.09,265649.722,10982], + [21302.22,265510.455,10976], + [21440.19,265696.211,10984], + [21440.37,265719.422,10985], + [21440.49,265742.633,10986], + [21440.57,265765.844,10987], + [21440.71,265789.055,10988], + [21440.78,265812.266,10989], + [21440.92,265673,10983], + [21602.06,265974.8,10996], + [21602.28,265951.588,10995], + [21602.4,265928.377,10994], + [21602.49,265881.955,10992], + [21602.69,265905.166,10993], + [21602.77,265858.744,10991], + [21602.87,265835.533,10990], + [21764.78,266067.711,11000], + [21764.96,266090.922,11001], + [21765.05,266114.133,11002], + [21765.21,266137.344,11003], + [21765.28,265998.077,10997], + [21765.37,266044.5,10999], + [21765.46,266021.288,10998], + [21927.49,266253.455,11008], + [21927.66,266230.244,11007], + [21927.77,266276.666,11009], + [21927.85,266299.877,11010], + [21927.96,266183.822,11005], + [21928.03,266207.033,11006], + [21928.1,266160.611,11004], + [22090.88,266416,11015], + [22091.03,266439.211,11016], + [22091.11,266462.422,11017], + [22091.24,266346.366,11012], + [22091.3,266369.577,11013], + [22091.38,266323.155,11011], + [22091.53,266392.788,11014], + [22321.65,266508.9,11019], + [22321.82,266532.111,11020], + [22321.94,266578.533,11022], + [22322.11,266601.744,11023], + [22322.19,266624.955,11024], + [22322.25,266485.688,11018], + [22322.4,266555.322,11021], + [22433.75,266741.077,11029], + [22433.99,266764.288,11030], + [22434.12,266787.5,11031], + [22437.23,266671.444,11026], + [22437.45,266694.655,11027], + [22437.55,266648.233,11025], + [22437.66,266717.866,11028], + [22615.91,266903.622,11036], + [22616.09,266810.777,11032], + [22616.18,266833.988,11033], + [22616.31,266857.2,11034], + [22616.39,266926.833,11037], + [22616.47,266950.044,11038], + [22616.62,266880.411,11035], + [22763.72,267019.733,11041], + [22763.91,266973.311,11039], + [22764.05,267042.944,11042], + [22764.13,267066.155,11043], + [22764.27,267089.366,11044], + [22764.34,267112.577,11045], + [22764.43,267135.788,11046], + [22764.53,266996.522,11040], + [22925.46,267159.077,11047], + [22925.62,267182.288,11048], + [22925.69,267205.5,11049], + [22925.87,267228.711,11050], + [22925.94,267251.922,11051], + [22926.02,267275.133,11052], + [22926.1,267298.344,11053], + [23088.63,267344.822,11055], + [23088.78,267368.033,11056], + [23088.87,267391.244,11057], + [23089.03,267414.455,11058], + [23089.11,267437.666,11059], + [23089.19,267460.877,11060], + [23089.25,267321.611,11054], + [23253.25,267623.422,11067], + [23253.4,267553.788,11064], + [23253.51,267577,11065], + [23253.59,267600.211,11066], + [23253.67,267530.577,11063], + [23253.82,267507.366,11062], + [23253.91,267484.155,11061], + [23421.5,267669.9,11069], + [23421.66,267693.111,11070], + [23421.74,267716.322,11071], + [23421.88,267739.533,11072], + [23422.03,267785.955,11074], + [23422.13,267762.744,11073], + [23422.25,267646.688,11068], + [23587.5,267809.233,11075], + [23587.66,267832.444,11076], + [23587.77,267855.655,11077], + [23587.97,267878.866,11078], + [23588.06,267902.077,11079], + [23588.16,267925.288,11080], + [23588.29,267948.5,11081], + [23754.12,267994.977,11083], + [23754.28,268018.188,11084], + [23754.35,268041.4,11085], + [23754.44,268064.611,11086], + [23754.59,268087.822,11087], + [23754.68,268111.033,11088], + [23754.88,267971.766,11082], + [23922.3,268157.522,11090], + [23922.44,268180.733,11091], + [23922.55,268203.944,11092], + [23922.63,268227.155,11093], + [23922.77,268250.366,11094], + [23922.84,268273.577,11095], + [23923,268134.311,11089], + [24063.78,268320.066,11097], + [24063.96,268343.277,11098], + [24064.09,268366.488,11099], + [24064.19,268389.7,11100], + [24064.34,268412.911,11101], + [24064.41,268436.122,11102], + [24064.5,268296.855,11096], + [24225.27,268529.022,11106], + [24225.45,268552.233,11107], + [24225.55,268575.444,11108], + [24225.68,268598.655,11109], + [24225.75,268459.388,11103], + [24225.84,268482.6,11104], + [24225.93,268505.811,11105], + [24390.13,268737.988,11115], + [24390.29,268621.933,11110], + [24390.37,268645.144,11111], + [24390.53,268668.355,11112], + [24390.6,268761.2,11116], + [24390.68,268691.566,11113], + [24390.75,268714.777,11114], + [24566.88,268807.677,11118], + [24567.05,268830.888,11119], + [24567.14,268854.1,11120], + [24567.27,268877.311,11121], + [24567.37,268900.522,11122], + [24567.44,268923.733,11123], + [24567.6,268784.466,11117], + [24738.47,268970.222,11125], + [24738.63,268993.433,11126], + [24738.71,269016.644,11127], + [24738.85,269039.855,11128], + [24738.92,269063.066,11129], + [24739,269086.277,11130], + [24739.08,268947.011,11124], + [24879.15,269202.388,11135], + [24879.31,269225.6,11136], + [24879.41,269109.544,11131], + [24879.55,269132.755,11132], + [24879.63,269155.966,11133], + [24879.7,269179.177,11134], + [24879.79,269248.811,11137], + [25054.02,269272.088,11138], + [25055.04,269295.3,11139], + [25055.16,269318.511,11140], + [25055.26,269341.722,11141], + [25055.33,269364.933,11142], + [25055.6,269388.144,11143], + [25055.67,269411.355,11144], + [25212.62,269573.9,11151], + [25212.79,269434.633,11145], + [25212.9,269457.844,11146], + [25213.06,269481.055,11147], + [25213.14,269504.266,11148], + [25213.23,269527.477,11149], + [25213.39,269550.688,11150], + [25380.99,269597.166,11152], + [25381.31,269620.377,11153], + [25381.4,269643.588,11154], + [25381.55,269666.8,11155], + [25381.62,269690.011,11156], + [25381.71,269713.222,11157], + [25381.8,269736.433,11158], + [25548.94,269875.766,11164], + [25549.1,269898.977,11165], + [25549.19,269852.555,11163], + [25549.37,269759.711,11159], + [25549.44,269782.922,11160], + [25549.62,269806.133,11161], + [25549.71,269829.344,11162], + [25713.68,269922.244,11166], + [25714.61,269991.877,11169], + [25714.74,269945.455,11167], + [25714.86,269968.666,11168], + [25714.94,270015.088,11170], + [25715.03,270038.3,11171], + [25715.19,270061.511,11172], + [25715.26,270084.722,11173], + [25888.14,270177.644,11177], + [25888.28,270200.855,11178], + [25888.37,270224.066,11179], + [25888.51,270108.011,11174], + [25888.61,270131.222,11175], + [25888.7,270154.433,11176], + [25888.79,270247.277,11180], + [26047.55,270409.811,11187], + [26047.72,270363.388,11185], + [26047.82,270386.6,11186], + [26047.99,270316.966,11183], + [26048.07,270293.755,11182], + [26048.17,270340.177,11184], + [26048.3,270270.544,11181], + [26214.11,270479.511,11190], + [26214.21,270525.933,11192], + [26214.34,270433.088,11188], + [26214.43,270456.3,11189], + [26214.55,270502.722,11191], + [26214.65,270549.144,11193], + [26214.76,270572.355,11194], + [26380.43,270618.833,11196], + [26380.67,270642.044,11197], + [26380.78,270665.255,11198], + [26380.91,270688.466,11199], + [26381.18,270711.677,11200], + [26381.27,270734.888,11201], + [26381.35,270595.622,11195], + [26524.57,270781.377,11203], + [26524.77,270804.588,11204], + [26524.87,270827.8,11205], + [26524.97,270851.011,11206], + [26525.16,270874.222,11207], + [26525.23,270897.433,11208], + [26525.32,270758.166,11202], + [26687.77,270943.911,11210], + [26687.95,270967.122,11211], + [26688.07,270990.333,11212], + [26688.15,271013.544,11213], + [26688.28,271036.755,11214], + [26688.35,271059.966,11215], + [26688.47,270920.7,11209], + [26850.96,271106.455,11217], + [26851.13,271129.666,11218], + [26851.22,271152.877,11219], + [26851.34,271176.088,11220], + [26851.42,271199.3,11221], + [26851.49,271222.511,11222], + [26851.66,271083.244,11216], + [27012.06,271269,11224], + [27012.22,271292.211,11225], + [27012.3,271315.422,11226], + [27012.43,271338.633,11227], + [27012.5,271361.844,11228], + [27012.58,271385.055,11229], + [27012.72,271245.788,11223], + [27174.88,271454.744,11232], + [27175.05,271477.955,11233], + [27175.15,271501.166,11234], + [27175.28,271524.377,11235], + [27175.36,271408.322,11230], + [27175.44,271431.533,11231], + [27175.52,271547.588,11236], + [27340.27,271594.077,11238], + [27340.44,271617.288,11239], + [27340.62,271640.5,11240], + [27340.73,271663.711,11241], + [27340.9,271686.922,11242], + [27340.97,271710.133,11243], + [27341.07,271570.866,11237], + [27513.14,271756.611,11245], + [27513.37,271779.822,11246], + [27513.49,271803.033,11247], + [27513.67,271826.244,11248], + [27513.75,271849.455,11249], + [27513.84,271872.666,11250], + [27513.94,271733.4,11244], + [27701.13,271942.366,11253], + [27701.32,271965.577,11254], + [27701.43,271988.788,11255], + [27701.53,272012,11256], + [27701.75,272035.211,11257], + [27701.86,271895.944,11251], + [27701.99,271919.155,11252], + [27849.3,272128.111,11261], + [27849.49,272151.322,11262], + [27849.59,272174.533,11263], + [27849.8,272058.477,11258], + [27849.91,272081.688,11259], + [27849.98,272197.744,11264], + [27850.06,272104.9,11260], + [28004.41,272313.866,11269], + [28004.6,272221.022,11265], + [28004.69,272244.233,11266], + [28004.83,272267.444,11267], + [28004.91,272337.077,11270], + [28004.97,272360.288,11271], + [28005.05,272290.655,11268], + [28150.41,272383.566,11272], + [28150.57,272406.777,11273], + [28150.66,272429.988,11274], + [28150.8,272453.2,11275], + [28150.88,272476.411,11276], + [28150.96,272522.833,11278], + [28151.05,272499.622,11277], + [28339.67,272592.522,11281], + [28339.83,272615.733,11282], + [28339.91,272638.944,11283], + [28340.04,272662.155,11284], + [28340.12,272685.366,11285], + [28340.19,272546.1,11279], + [28340.28,272708.577,11286], + [28340.44,272569.311,11280], + [28502.85,272731.866,11287], + [28504.1,272755.077,11288], + [28504.23,272778.288,11289], + [28504.31,272801.5,11290], + [28504.44,272824.711,11291], + [28504.53,272847.922,11292], + [28504.63,272871.133,11293], + [28671.16,272940.822,11296], + [28671.43,272964.033,11297], + [28671.54,272987.244,11298], + [28671.69,273010.455,11299], + [28671.76,273033.666,11300], + [28671.87,272894.4,11294], + [28671.97,272917.611,11295], + [28840.35,273080.155,11302], + [28840.53,273103.366,11303], + [28840.63,273126.577,11304], + [28840.77,273149.788,11305], + [28840.85,273173,11306], + [28840.94,273196.211,11307], + [28841.01,273056.944,11301], + [29004.41,273312.322,11312], + [29004.55,273335.533,11313], + [29004.62,273358.744,11314], + [29004.74,273242.688,11309], + [29004.82,273265.9,11310], + [29004.91,273219.477,11308], + [29005.05,273289.111,11311], + [29165.59,273474.866,11319], + [29165.75,273382.022,11315], + [29165.85,273405.233,11316], + [29166,273428.444,11317], + [29166.08,273498.077,11320], + [29166.16,273521.288,11321], + [29166.25,273451.655,11318], + [29311.41,273660.611,11327], + [29311.63,273683.822,11328], + [29311.76,273544.555,11322], + [29311.88,273614.188,11325], + [29312.07,273637.4,11326], + [29312.14,273567.766,11323], + [29312.22,273590.977,11324], + [29474.61,273776.733,11332], + [29474.77,273846.366,11335], + [29474.9,273707.1,11329], + [29475.06,273730.311,11330], + [29475.13,273799.944,11333], + [29475.21,273823.155,11334], + [29475.3,273753.522,11331], + [29636.09,273892.855,11337], + [29636.25,273916.066,11338], + [29636.34,273939.277,11339], + [29636.47,273962.488,11340], + [29636.55,273985.7,11341], + [29636.63,274008.911,11342], + [29636.72,273869.644,11336], + [29799.95,274055.388,11344], + [29800.1,274078.6,11345], + [29800.19,274101.811,11346], + [29800.31,274125.022,11347], + [29800.38,274148.233,11348], + [29800.46,274171.444,11349], + [29800.6,274032.177,11343], + [29964.7,274217.933,11351], + [29964.85,274241.144,11352], + [29964.94,274264.355,11353], + [29965.06,274287.566,11354], + [29965.15,274310.777,11355], + [29965.22,274333.988,11356], + [29965.36,274194.722,11350], + [30130.13,274380.466,11358], + [30130.31,274403.677,11359], + [30130.39,274426.888,11360], + [30130.47,274450.1,11361], + [30130.65,274473.311,11362], + [30130.73,274496.522,11363], + [30130.87,274357.255,11357], + [30295.18,274612.644,11368], + [30295.38,274659.066,11370], + [30295.5,274635.855,11369], + [30295.69,274543.011,11365], + [30295.84,274566.222,11366], + [30295.97,274519.8,11364], + [30296.09,274589.433,11367], + [30462.77,274682.333,11371], + [30463.07,274705.544,11372], + [30463.24,274775.177,11375], + [30463.35,274728.755,11373], + [30463.71,274798.388,11376], + [30463.84,274821.6,11377], + [30463.98,274751.966,11374], + [30679.69,274868.088,11379], + [30679.93,274891.3,11380], + [30680.08,274914.511,11381], + [30680.27,274937.722,11382], + [30680.4,274960.933,11383], + [30680.5,274984.144,11384], + [30680.62,274844.877,11378], + [30774.94,275030.633,11386], + [30775.11,275053.844,11387], + [30775.21,275077.055,11388], + [30775.45,275100.266,11389], + [30775.63,275123.477,11390], + [30775.75,275146.688,11391], + [30775.87,275007.422,11385], + [30950.08,275286.011,11397], + [30950.25,275309.222,11398], + [30950.35,275169.955,11392], + [30950.5,275193.166,11393], + [30950.59,275216.377,11394], + [30950.69,275239.588,11395], + [30950.84,275262.8,11396], + [31111.47,275332.5,11399], + [31111.65,275355.711,11400], + [31111.74,275378.922,11401], + [31111.83,275402.133,11402], + [31112,275425.344,11403], + [31112.07,275448.555,11404], + [31112.16,275471.766,11405], + [31261.41,275518.244,11407], + [31261.55,275541.455,11408], + [31261.63,275564.666,11409], + [31261.75,275587.877,11410], + [31261.84,275611.088,11411], + [31261.92,275634.3,11412], + [31262.01,275495.033,11406], + [31424.69,275680.788,11414], + [31424.87,275704,11415], + [31424.95,275727.211,11416], + [31425.07,275750.422,11417], + [31425.15,275773.633,11418], + [31425.24,275796.844,11419], + [31425.31,275657.577,11413], + [31589.23,275843.322,11421], + [31589.38,275866.533,11422], + [31589.46,275936.166,11425], + [31589.6,275889.744,11423], + [31589.69,275912.955,11424], + [31589.77,275820.111,11420], + [31589.93,275959.377,11426], + [31756.16,276005.866,11428], + [31756.33,276029.077,11429], + [31756.42,276052.288,11430], + [31756.59,276075.5,11431], + [31756.66,276098.711,11432], + [31756.75,276121.922,11433], + [31756.85,275982.655,11427], + [31921.65,276168.411,11435], + [31921.8,276191.622,11436], + [31921.88,276214.833,11437], + [31922,276238.044,11438], + [31922.07,276261.255,11439], + [31922.16,276284.466,11440], + [31922.29,276145.2,11434], + [32092.05,276330.944,11442], + [32092.16,276354.155,11443], + [32092.24,276377.366,11444], + [32092.36,276400.577,11445], + [32092.43,276423.788,11446], + [32092.49,276447,11447], + [32092.61,276307.733,11441], + [32265.74,276493.488,11449], + [32265.89,276516.7,11450], + [32265.97,276539.911,11451], + [32266.09,276563.122,11452], + [32266.16,276586.333,11453], + [32266.22,276609.544,11454], + [32266.35,276470.277,11448], + [32399.35,276725.655,11459], + [32399.48,276748.866,11460], + [32399.55,276772.077,11461], + [32399.66,276656.022,11456], + [32399.74,276679.233,11457], + [32399.84,276632.811,11455], + [32400,276702.444,11458], + [32587.97,276795.355,11462], + [32589.35,276888.2,11466], + [32589.47,276911.411,11467], + [32589.57,276818.566,11463], + [32589.69,276841.777,11464], + [32589.77,276864.988,11465], + [32589.91,276934.622,11468], + [32589.99,276957.833,11469], + [32753,277004.322,11471], + [32753.18,277027.533,11472], + [32753.28,277050.744,11473], + [32753.43,277073.955,11474], + [32753.52,277097.166,11475], + [32753.6,277120.377,11476], + [32753.7,276981.111,11470], + [32920.49,277190.077,11479], + [32920.74,277213.288,11480], + [32920.86,277236.5,11481], + [32921.03,277259.711,11482], + [32921.12,277282.922,11483], + [32921.22,277143.655,11477], + [32921.34,277166.866,11478], + [33088.34,277306.188,11484], + [33088.57,277329.4,11485], + [33088.69,277445.455,11490], + [33088.85,277352.611,11486], + [33088.98,277375.822,11487], + [33089.06,277399.033,11488], + [33089.17,277422.244,11489], + [33254.69,277491.944,11492], + [33254.94,277515.155,11493], + [33255.03,277538.366,11494], + [33255.21,277561.577,11495], + [33255.31,277584.788,11496], + [33255.41,277608,11497], + [33255.52,277468.733,11491], + [33397.55,277654.477,11499], + [33397.69,277677.688,11500], + [33397.79,277700.9,11501], + [33397.91,277724.111,11502], + [33397.99,277747.322,11503], + [33398.06,277770.533,11504], + [33398.13,277631.266,11498], + [33559.9,277933.077,11511], + [33560.06,277793.811,11505], + [33560.16,277817.022,11506], + [33560.24,277840.233,11507], + [33560.37,277863.444,11508], + [33560.43,277886.655,11509], + [33560.51,277909.866,11510], + [33723.06,277979.566,11513], + [33723.25,278002.777,11514], + [33723.35,278025.988,11515], + [33723.46,278049.2,11516], + [33723.66,278072.411,11517], + [33723.75,278095.622,11518], + [33723.88,277956.355,11512], + [33885.02,278142.1,11520], + [33885.16,278165.311,11521], + [33885.23,278188.522,11522], + [33885.31,278211.733,11523], + [33885.43,278234.944,11524], + [33885.5,278258.155,11525], + [33885.57,278118.888,11519], + [34050.42,278374.277,11530], + [34050.56,278397.488,11531], + [34050.65,278420.7,11532], + [34050.8,278304.644,11527], + [34050.88,278327.855,11528], + [34050.96,278281.433,11526], + [34051.04,278351.066,11529], + [34213.18,278536.811,11537], + [34213.43,278443.966,11533], + [34213.57,278467.177,11534], + [34213.72,278490.388,11535], + [34213.87,278560.022,11538], + [34214.02,278583.233,11539], + [34214.13,278513.6,11536], + [34350.88,278606.511,11540], + [34351.11,278629.722,11541], + [34351.24,278652.933,11542], + [34351.36,278676.144,11543], + [34351.48,278699.355,11544], + [34351.61,278722.566,11545], + [34520.88,278792.255,11548], + [34521.05,278815.466,11549], + [34521.13,278838.677,11550], + [34521.26,278861.888,11551], + [34521.34,278885.1,11552], + [34521.41,278745.833,11546], + [34521.53,278769.044,11547], + [34676.78,278931.577,11554], + [34676.94,278954.788,11555], + [34677.02,278978,11556], + [34677.13,279001.211,11557], + [34677.21,279024.422,11558], + [34677.27,279047.633,11559], + [34677.34,278908.366,11553], + [34845.22,279094.122,11561], + [34845.37,279117.333,11562], + [34845.45,279140.544,11563], + [34845.57,279163.755,11564], + [34845.65,279186.966,11565], + [34845.72,279210.177,11566], + [34845.8,279070.911,11560] + ] +} diff --git a/test/audio-quality/traces/relay-mic.json b/test/audio-quality/traces/relay-mic.json new file mode 100644 index 0000000000..3bb4d93735 --- /dev/null +++ b/test/audio-quality/traces/relay-mic.json @@ -0,0 +1,1763 @@ +{ + "version": 1, + "source": "https://cdn.moq.dev/anon, in headless Chromium capturing its fake audio device, recorded 2026-09-29", + "description": "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between.", + "transport": "webtransport", + "rtt": null, + "config": { + "codec": "opus", + "container": { + "kind": "legacy" + }, + "sampleRate": 48000, + "numberOfChannels": 2, + "bitrate": 64000 + }, + "arrivals": [ + [0,2324.745,0], + [16.53,2344.745,1], + [38.87,2364.745,2], + [58.56,2384.745,3], + [80.79,2404.745,4], + [90.63,2424.745,5], + [123.14,2444.745,6], + [133.15,2464.745,7], + [154.04,2484.745,8], + [175.89,2504.745,9], + [196.55,2524.745,10], + [217.73,2544.745,11], + [239.26,2564.745,12], + [249.41,2584.744,13], + [284.74,2604.745,14], + [293.14,2624.745,15], + [316.97,2644.745,16], + [341.57,2664.745,17], + [369.77,2684.745,18], + [383.67,2704.745,19], + [399.94,2724.745,20], + [412.31,2744.745,21], + [448.56,2764.745,22], + [462.16,2784.745,23], + [481.56,2804.745,24], + [500.93,2824.745,25], + [521.09,2844.744,26], + [549.96,2864.745,27], + [563.49,2884.745,28], + [571.93,2904.745,29], + [604.02,2924.745,30], + [614.12,2944.745,31], + [636.18,2964.745,32], + [655.33,2984.745,33], + [680.99,3004.745,34], + [699.9,3024.745,35], + [743.93,3044.745,36], + [744.29,3064.745,37], + [766.77,3084.745,38], + [773.76,3104.745,39], + [798.74,3124.745,40], + [821.71,3144.745,41], + [841.92,3164.745,42], + [862.21,3184.745,43], + [882.59,3204.745,44], + [900.28,3224.745,45], + [926.38,3244.745,46], + [937.7,3264.745,47], + [953.6,3284.745,48], + [974.8,3304.745,49], + [996.59,3324.745,50], + [1017.6,3344.745,51], + [1038.36,3364.744,52], + [1048.84,3384.745,53], + [1081.81,3404.745,54], + [1092.23,3424.745,55], + [1113.34,3444.745,56], + [1134.38,3464.745,57], + [1155.59,3484.745,58], + [1178.33,3504.745,59], + [1198.64,3524.745,60], + [1208.71,3544.745,61], + [1241.21,3564.745,62], + [1251.7,3584.745,63], + [1273.61,3604.745,64], + [1295.2,3624.745,65], + [1317.27,3644.745,66], + [1337.5,3664.745,67], + [1358.88,3684.745,68], + [1371.88,3704.745,69], + [1405.88,3724.745,70], + [1413.55,3744.745,71], + [1438.38,3764.745,72], + [1459.06,3784.745,73], + [1479,3804.745,74], + [1500.5,3824.745,75], + [1525.47,3844.745,76], + [1537.33,3864.745,77], + [1563.29,3884.745,78], + [1574.71,3904.745,79], + [1593.71,3924.745,80], + [1615.52,3944.745,81], + [1636.41,3964.745,82], + [1658.21,3984.745,83], + [1679.43,4004.745,84], + [1689.69,4024.745,85], + [1724.05,4044.745,86], + [1733.2,4064.745,87], + [1754.76,4084.745,88], + [1775.61,4104.745,89], + [1798.86,4124.745,90], + [1818.39,4144.745,91], + [1845.12,4164.745,92], + [1857.11,4184.745,93], + [1884.52,4204.745,94], + [1892.91,4224.745,95], + [1921.56,4244.745,96], + [1940.71,4264.745,97], + [1957.94,4284.745,98], + [1980.16,4304.745,99], + [2000.9,4324.745,100], + [2012,4344.745,101], + [2047.71,4364.745,102], + [2053.75,4384.744,103], + [2075.34,4404.744,104], + [2097.69,4424.745,105], + [2119.7,4444.745,106], + [2139.53,4464.745,107], + [2159.56,4484.745,108], + [2173.4,4504.745,109], + [2206.64,4524.745,110], + [2213.55,4544.745,111], + [2237.86,4564.745,112], + [2258.59,4584.745,113], + [2282.83,4604.745,114], + [2297.83,4624.745,115], + [2319.55,4644.745,116], + [2329.47,4664.745,117], + [2361.44,4684.745,118], + [2376.64,4704.745,119], + [2394.76,4724.745,120], + [2420.92,4744.745,121], + [2436.81,4764.745,122], + [2460.55,4784.745,123], + [2479.14,4804.745,124], + [2489.11,4824.745,125], + [2521.52,4844.745,126], + [2531.84,4864.745,127], + [2553.61,4884.745,128], + [2574.44,4904.745,129], + [2596.02,4924.745,130], + [2617.52,4944.745,131], + [2638.58,4964.745,132], + [2649.87,4984.745,133], + [2681.42,5004.745,134], + [2693.43,5024.745,135], + [2713.46,5044.745,136], + [2734.76,5064.745,137], + [2756.99,5084.745,138], + [2778.25,5104.745,139], + [2806,5124.745,140], + [2817.2,5144.745,141], + [2845.81,5164.745,142], + [2854.02,5184.745,143], + [2879.26,5204.745,144], + [2897.34,5224.745,145], + [2923.3,5244.745,146], + [2940.44,5264.745,147], + [2969.34,5284.745,148], + [2976.18,5304.745,149], + [3010.24,5324.745,150], + [3013.92,5344.745,151], + [3035.22,5364.745,152], + [3056.81,5384.745,153], + [3076.8,5404.745,154], + [3098.04,5424.745,155], + [3119.11,5444.745,156], + [3129.03,5464.745,157], + [3161.53,5484.745,158], + [3172.28,5504.745,159], + [3193.96,5524.745,160], + [3214.4,5544.745,161], + [3235.97,5564.745,162], + [3256.9,5584.745,163], + [3278.24,5604.745,164], + [3288.81,5624.745,165], + [3320.78,5644.745,166], + [3332.48,5664.745,167], + [3353.17,5684.745,168], + [3374.69,5704.745,169], + [3396.59,5724.745,170], + [3419.17,5744.745,171], + [3438.88,5764.745,172], + [3448.31,5784.745,173], + [3485.27,5804.745,174], + [3491.39,5824.745,175], + [3513.07,5844.745,176], + [3534.63,5864.745,177], + [3555.46,5884.745,178], + [3576.88,5904.745,179], + [3600.12,5924.745,180], + [3609.53,5944.745,181], + [3641.17,5964.745,182], + [3651.9,5984.745,183], + [3673.49,6004.745,184], + [3695.59,6024.745,185], + [3716.17,6044.745,186], + [3737.64,6064.745,187], + [3759.28,6084.745,188], + [3769.84,6104.745,189], + [3803.56,6124.745,190], + [3812.66,6144.745,191], + [3833.34,6164.745,192], + [3855.37,6184.745,193], + [3876.13,6204.745,194], + [3900.31,6224.745,195], + [3920.87,6244.745,196], + [3931.56,6264.745,197], + [3969,6284.745,198], + [3976.17,6304.745,199], + [3995.25,6324.745,200], + [4016.06,6344.745,201], + [4039.69,6364.745,202], + [4061.94,6384.745,203], + [4081.93,6404.745,204], + [4095.02,6424.744,205], + [4129.21,6444.744,206], + [4141.96,6464.744,207], + [4159.78,6484.744,208], + [4181.74,6504.744,209], + [4205.51,6524.745,210], + [4222.18,6544.745,211], + [4242.58,6564.745,212], + [4253.75,6584.745,213], + [4283.58,6604.745,214], + [4296.46,6624.745,215], + [4316.04,6644.745,216], + [4336.97,6664.745,217], + [4357.46,6684.745,218], + [4378.55,6704.745,219], + [4400.38,6724.745,220], + [4410.74,6744.745,221], + [4442.89,6764.745,222], + [4453.32,6784.745,223], + [4474.75,6804.745,224], + [4496.09,6824.745,225], + [4516.89,6844.745,226], + [4538.28,6864.745,227], + [4561.66,6884.745,228], + [4571.86,6904.745,229], + [4606.17,6924.745,230], + [4618.99,6944.745,231], + [4643.04,6964.745,232], + [4660.18,6984.745,233], + [4678.34,7004.745,234], + [4702.41,7024.745,235], + [4724.15,7044.745,236], + [4735.25,7064.745,237], + [4767.56,7084.745,238], + [4775.88,7104.745,239], + [4797.39,7124.745,240], + [4817.46,7144.745,241], + [4838.72,7164.745,242], + [4860.78,7184.745,243], + [4884.25,7204.745,244], + [4893.92,7224.745,245], + [4927.98,7244.745,246], + [4938.37,7264.745,247], + [4959.11,7284.745,248], + [4980.63,7304.745,249], + [5006.03,7324.745,250], + [5021.33,7344.745,251], + [5046.1,7364.745,252], + [5052.44,7384.745,253], + [5086.43,7404.745,254], + [5095.18,7424.745,255], + [5122.14,7444.745,256], + [5137.73,7464.745,257], + [5159.36,7484.745,258], + [5179.19,7504.745,259], + [5202.47,7524.745,260], + [5210.44,7544.745,261], + [5245.55,7564.745,262], + [5254.5,7584.745,263], + [5277.09,7604.745,264], + [5297.06,7624.745,265], + [5318.03,7644.745,266], + [5338.72,7664.745,267], + [5360.22,7684.745,268], + [5370.72,7704.745,269], + [5402.89,7724.745,270], + [5413.28,7744.745,271], + [5434.61,7764.745,272], + [5455.65,7784.745,273], + [5476.71,7804.745,274], + [5498.66,7824.745,275], + [5519.25,7844.745,276], + [5530.03,7864.745,277], + [5562.89,7884.745,278], + [5572.77,7904.745,279], + [5594.59,7924.745,280], + [5615.39,7944.745,281], + [5637.25,7964.745,282], + [5658.91,7984.745,283], + [5680.14,8004.745,284], + [5693.58,8024.745,285], + [5728.24,8044.745,286], + [5738.1,8064.745,287], + [5754.95,8084.745,288], + [5775.96,8104.745,289], + [5796.54,8124.745,290], + [5828.16,8144.745,291], + [5852.67,8164.745,292], + [5858.34,8184.745,293], + [5893.3,8204.745,294], + [5905.38,8224.745,295], + [5923.08,8244.745,296], + [5941.58,8264.745,297], + [5970.14,8284.745,298], + [6004.44,8304.745,299], + [6015.33,8324.745,300], + [6023.34,8344.745,301], + [6051.55,8364.745,302], + [6058.03,8384.745,303], + [6086.87,8404.745,304], + [6113.36,8424.745,305], + [6135.62,8444.745,306], + [6152.66,8464.745,307], + [6170.19,8484.745,308], + [6183.62,8504.745,309], + [6218.41,8524.745,310], + [6238.54,8544.745,311], + [6277.71,8564.745,312], + [6289.04,8584.745,313], + [6289.32,8604.745,314], + [6302.76,8624.745,315], + [6354.31,8644.745,316], + [6376.12,8664.745,317], + [6385.07,8684.745,318], + [6385.53,8704.745,319], + [6396.94,8724.745,320], + [6427.9,8744.745,321], + [6446.93,8764.745,322], + [6459.51,8784.745,323], + [6483.94,8804.745,324], + [6497.64,8824.745,325], + [6534.07,8844.745,326], + [6536.19,8864.745,327], + [6556.34,8884.745,328], + [6582.03,8904.745,329], + [6600.72,8924.745,330], + [6617.41,8944.745,331], + [6646.89,8964.745,332], + [6660.89,8984.745,333], + [6690.03,9004.745,334], + [6696.5,9024.745,335], + [6727.05,9044.745,336], + [6740.59,9064.745,337], + [6770.2,9084.745,338], + [6782.53,9104.745,339], + [6811.18,9124.745,340], + [6812.75,9144.745,341], + [6875.05,9164.745,342], + [6915.56,9184.745,343], + [6955.68,9204.745,344], + [6956.06,9224.745,345], + [6960.73,9244.745,346], + [6961.72,9264.745,347], + [6966.06,9284.745,348], + [6977.21,9304.745,349], + [7014.69,9324.745,350], + [7020.27,9344.745,351], + [7045.12,9364.745,352], + [7080.42,9384.745,353], + [7088.71,9404.745,354], + [7103.48,9424.745,355], + [7157.58,9444.745,356], + [7157.97,9464.745,357], + [7179.94,9484.745,358], + [7195.72,9504.745,359], + [7204.87,9524.745,360], + [7230.26,9544.745,361], + [7249.69,9564.745,362], + [7276.37,9584.745,363], + [7290.15,9604.745,364], + [7321.94,9624.745,365], + [7337.89,9644.745,366], + [7350.89,9664.745,367], + [7376.19,9684.745,368], + [7389.08,9704.745,369], + [7438.18,9724.745,370], + [7438.5,9744.745,371], + [7459.15,9764.745,372], + [7470.24,9784.745,373], + [7501.81,9804.745,374], + [7510.25,9824.745,375], + [7560.51,9844.745,376], + [7573.98,9864.745,377], + [7580.71,9884.745,378], + [7609.84,9904.745,379], + [7613.38,9924.745,380], + [7622.24,9944.745,381], + [7652.93,9964.745,382], + [7663.46,9984.745,383], + [7685.13,10004.745,384], + [7706.59,10024.745,385], + [7734.78,10044.745,386], + [7752.29,10064.745,387], + [7771.87,10084.745,388], + [7780.34,10104.745,389], + [7815.23,10124.745,390], + [7822.97,10144.745,391], + [7844.47,10164.745,392], + [7867.55,10184.745,393], + [7887.53,10204.745,394], + [7909.25,10224.745,395], + [7940.42,10244.745,396], + [7946.36,10264.745,397], + [7972.52,10284.745,398], + [7994.24,10304.745,399], + [8009.22,10324.745,400], + [8034.42,10344.745,401], + [8047.44,10364.745,402], + [8069.04,10384.745,403], + [8097.35,10404.745,404], + [8106.03,10424.745,405], + [8139.49,10444.745,406], + [8145.06,10464.745,407], + [8177.67,10484.745,408], + [8188.15,10504.745,409], + [8209.38,10524.744,410], + [8229.46,10544.744,411], + [8250.23,10564.744,412], + [8260.28,10584.744,413], + [8292.84,10604.744,414], + [8303.52,10624.744,415], + [8325.19,10644.744,416], + [8347.74,10664.744,417], + [8375.09,10684.744,418], + [8390.02,10704.744,419], + [8409.69,10724.745,420], + [8420.53,10744.745,421], + [8452.44,10764.745,422], + [8464.09,10784.745,423], + [8484.45,10804.745,424], + [8505.84,10824.745,425], + [8527.18,10844.745,426], + [8548.19,10864.745,427], + [8569.14,10884.745,428], + [8579.77,10904.745,429], + [8613.44,10924.745,430], + [8623.16,10944.745,431], + [8644.72,10964.745,432], + [8666.02,10984.745,433], + [8687.23,11004.745,434], + [8708.54,11024.745,435], + [8730.54,11044.745,436], + [8740.94,11064.745,437], + [8772.76,11084.745,438], + [8783.44,11104.745,439], + [8805.15,11124.745,440], + [8827.06,11144.745,441], + [8848.43,11164.745,442], + [8875.22,11184.745,443], + [8894.25,11204.745,444], + [8903.74,11224.745,445], + [8932.69,11244.745,446], + [8951.23,11264.745,447], + [8967.19,11284.745,448], + [8989.07,11304.745,449], + [9014.3,11324.745,450], + [9028.79,11344.745,451], + [9049.97,11364.745,452], + [9060.53,11384.745,453], + [9092.59,11404.745,454], + [9104.07,11424.745,455], + [9124.4,11444.745,456], + [9145.61,11464.745,457], + [9167.22,11484.745,458], + [9193.52,11504.745,459], + [9210.05,11524.745,460], + [9220.27,11544.745,461], + [9252.12,11564.745,462], + [9264.11,11584.745,463], + [9284.44,11604.745,464], + [9305.84,11624.745,465], + [9326.97,11644.745,466], + [9347.84,11664.745,467], + [9369.13,11684.745,468], + [9379.46,11704.745,469], + [9411.81,11724.745,470], + [9423.47,11744.745,471], + [9443.53,11764.745,472], + [9464.91,11784.745,473], + [9487.03,11804.745,474], + [9508.34,11824.745,475], + [9529.37,11844.745,476], + [9540.44,11864.745,477], + [9572.81,11884.745,478], + [9583.1,11904.745,479], + [9611.71,11924.745,480], + [9630.78,11944.745,481], + [9651.86,11964.745,482], + [9670.19,11984.745,483], + [9690.71,12004.745,484], + [9701.97,12024.745,485], + [9732.03,12044.745,486], + [9742.03,12064.745,487], + [9769.29,12084.745,488], + [9786.52,12104.745,489], + [9807.66,12124.745,490], + [9831.64,12144.745,491], + [9851.36,12164.745,492], + [9871.03,12184.745,493], + [9897.81,12204.745,494], + [9908.14,12224.745,495], + [9930.15,12244.745,496], + [9949.59,12264.745,497], + [9979,12284.745,498], + [9998.72,12304.745,499], + [10012.69,12324.745,500], + [10027.22,12344.745,501], + [10056.15,12364.745,502], + [10067.18,12384.745,503], + [10087.2,12404.745,504], + [10116.02,12424.745,505], + [10131.27,12444.745,506], + [10153.25,12464.745,507], + [10180.82,12484.745,508], + [10186.06,12504.745,509], + [10218.92,12524.745,510], + [10232.14,12544.745,511], + [10248.27,12564.745,512], + [10272.27,12584.745,513], + [10294.34,12604.745,514], + [10308.91,12624.745,515], + [10335.43,12644.745,516], + [10351.22,12664.745,517], + [10397.18,12684.745,518], + [10397.61,12704.745,519], + [10423.97,12724.745,520], + [10443.56,12744.745,521], + [10454.58,12764.745,522], + [10479.71,12784.745,523], + [10497.21,12804.745,524], + [10506.87,12824.745,525], + [10548.34,12844.745,526], + [10550.89,12864.745,527], + [10572.92,12884.745,528], + [10612.44,12904.745,529], + [10634.27,12924.745,530], + [10641.4,12944.745,531], + [10733.73,12964.745,532], + [10735.82,12984.745,533], + [10755.88,13004.745,534], + [10756.77,13024.745,535], + [10761.41,13044.745,536], + [10777.55,13064.745,537], + [10780.15,13084.745,538], + [10789.19,13104.745,539], + [10815,13124.745,540], + [10829.51,13144.745,541], + [10853.84,13164.745,542], + [10864.53,13184.745,543], + [10884.49,13204.745,544], + [10906.02,13224.745,545], + [10927.5,13244.745,546], + [10948.28,13264.745,547], + [10970.81,13284.745,548], + [10980.9,13304.745,549], + [11015.89,13324.745,550], + [11025.84,13344.745,551], + [11044.63,13364.745,552], + [11066.77,13384.745,553], + [11102.83,13404.745,554], + [11129.58,13424.745,555], + [11140.81,13444.745,556], + [11148.97,13464.745,557], + [11183.4,13484.745,558], + [11204.62,13504.745,559], + [11234.13,13524.745,560], + [11238.59,13544.745,561], + [11261.01,13564.745,562], + [11282.53,13584.745,563], + [11296.55,13604.745,564], + [11305.35,13624.745,565], + [11342.18,13644.745,566], + [11350.14,13664.745,567], + [11369.53,13684.745,568], + [11388.52,13704.745,569], + [11407.11,13724.745,570], + [11428.29,13744.745,571], + [11450.28,13764.745,572], + [11459.62,13784.745,573], + [11491.94,13804.745,574], + [11502.94,13824.745,575], + [11524.53,13844.745,576], + [11545.75,13864.745,577], + [11568.57,13884.745,578], + [11602.66,13904.745,579], + [11621.11,13924.745,580], + [11629.3,13944.745,581], + [11666.22,13964.745,582], + [11677.65,13984.745,583], + [11694.01,14004.745,584], + [11718.39,14024.745,585], + [11734.83,14044.745,586], + [11755.08,14064.745,587], + [11802.39,14084.745,588], + [11808.72,14104.745,589], + [11845.81,14124.745,590], + [11847.44,14144.745,591], + [11847.73,14164.745,592], + [11868.5,14184.745,593], + [11903.06,14204.745,594], + [11918.23,14224.745,595], + [11942.19,14244.745,596], + [11954.33,14264.745,597], + [11982.12,14284.745,598], + [11993.61,14304.745,599], + [12010.03,14324.745,600], + [12032.87,14344.745,601], + [12052.44,14364.745,602], + [12087.72,14384.745,603], + [12098.55,14404.745,604], + [12108.81,14424.745,605], + [12138.69,14444.745,606], + [12154.22,14464.745,607], + [12175.03,14484.745,608], + [12194.24,14504.745,609], + [12282.15,14524.745,610], + [12332.19,14544.745,611], + [12360.18,14564.745,612], + [12451.8,14584.745,613], + [12531.83,14604.745,614], + [12683.12,14624.745,615], + [12683.84,14644.745,616], + [12694.85,14664.745,617], + [12695.43,14684.745,618], + [12716.44,14724.745,620], + [12719.15,14704.745,619], + [12798.54,14744.745,621], + [12802.88,14764.745,622], + [12815.83,14784.745,623], + [12816.4,14804.745,624], + [12816.99,14824.745,625], + [12817.26,14844.745,626], + [12826.02,14864.745,627], + [12826.47,14884.745,628], + [12826.78,14904.745,629], + [12827.66,14924.745,630], + [12827.97,14944.745,631], + [12829.15,14964.745,632], + [12829.38,14984.745,633], + [12829.62,15004.745,634], + [12829.89,15024.745,635], + [12830.14,15044.745,636], + [12830.42,15064.745,637], + [12854.89,15084.745,638], + [12861.39,15104.745,639], + [12885.99,15124.745,640], + [12905.95,15144.745,641], + [12927.51,15164.745,642], + [12952.18,15184.745,643], + [12974.49,15204.745,644], + [12981.92,15224.745,645], + [13012.3,15244.745,646], + [13021.81,15264.745,647], + [13051.53,15284.745,648], + [13080.22,15304.745,649], + [13098.98,15324.745,650], + [13112.16,15344.745,651], + [13150.78,15364.745,652], + [13151.26,15384.745,653], + [13178.52,15404.745,654], + [13182.64,15424.745,655], + [13206.18,15444.745,656], + [13230.7,15464.745,657], + [13245.95,15484.745,658], + [13278.9,15504.745,659], + [13311.47,15524.745,660], + [13323.11,15544.745,661], + [13344.96,15564.745,662], + [13355.49,15584.745,663], + [13372.67,15604.745,664], + [13388.43,15624.745,665], + [13407.59,15644.745,666], + [13432.84,15664.745,667], + [13451.96,15684.745,668], + [13465.4,15704.745,669], + [13496.84,15724.745,670], + [13508.34,15744.745,671], + [13526.87,15764.745,672], + [13543.02,15784.745,673], + [13580.31,15804.745,674], + [13587.09,15824.745,675], + [13611.06,15844.745,676], + [13623.06,15864.745,677], + [13659.03,15884.745,678], + [13665.41,15904.745,679], + [13687.12,15924.745,680], + [13712.53,15944.745,681], + [13727.86,15964.745,682], + [13760.34,15984.745,683], + [13771.49,16004.745,684], + [13787.6,16024.745,685], + [13818.52,16044.745,686], + [13833.63,16064.745,687], + [13847.19,16084.745,688], + [13872.12,16104.745,689], + [13887.81,16124.745,690], + [13910.15,16144.745,691], + [13933.8,16164.745,692], + [13940.68,16184.745,693], + [13970.79,16204.745,694], + [13981.33,16224.745,695], + [14009.81,16244.745,696], + [14025.07,16264.745,697], + [14045.11,16284.745,698], + [14073.75,16304.745,699], + [14087.62,16324.745,700], + [14104.05,16344.745,701], + [14135.34,16364.745,702], + [14140.66,16384.745,703], + [14167.45,16404.745,704], + [14187.02,16424.745,705], + [14211.64,16444.745,706], + [14230.84,16464.745,707], + [14250.61,16484.745,708], + [14261.1,16504.745,709], + [14295.06,16524.745,710], + [14314.29,16544.745,711], + [14327.08,16564.745,712], + [14352.14,16584.745,713], + [14370.44,16604.745,714], + [14398.46,16624.745,715], + [14408.61,16644.745,716], + [14422.25,16664.745,717], + [14453.64,16684.745,718], + [14464.58,16704.745,719], + [14484.72,16724.745,720], + [14503.41,16744.745,721], + [14529.94,16764.745,722], + [14549.86,16784.745,723], + [14574.08,16804.745,724], + [14578.49,16824.745,725], + [14610.43,16844.745,726], + [14623.51,16864.745,727], + [14660.34,16884.745,728], + [14672.86,16904.745,729], + [14683.66,16924.745,730], + [14707.01,16944.745,731], + [14725.82,16964.745,732], + [14744.59,16984.745,733], + [14770.94,17004.745,734], + [14782.34,17024.745,735], + [14805.1,17044.745,736], + [14825.07,17064.745,737], + [14852.14,17084.745,738], + [14875.42,17104.745,739], + [14888.42,17124.745,740], + [14897.26,17144.745,741], + [14930.65,17164.745,742], + [14945.52,17184.745,743], + [14967.65,17204.745,744], + [14989.06,17224.745,745], + [15006.61,17244.745,746], + [15034.49,17264.745,747], + [15054.65,17284.745,748], + [15063.22,17304.745,749], + [15100.63,17324.745,750], + [15104.33,17344.745,751], + [15127.38,17364.745,752], + [15144.07,17384.745,753], + [15169.18,17404.745,754], + [15188.78,17424.745,755], + [15213.14,17444.745,756], + [15223.88,17464.745,757], + [15254.47,17484.745,758], + [15264.47,17504.745,759], + [15293.2,17524.745,760], + [15312.44,17544.745,761], + [15332.38,17564.745,762], + [15351.06,17584.745,763], + [15371.37,17604.745,764], + [15382.61,17624.745,765], + [15415.22,17644.745,766], + [15423.31,17664.745,767], + [15446.81,17684.745,768], + [15465.96,17704.745,769], + [15487.5,17724.745,770], + [15515.33,17744.745,771], + [15531.84,17764.745,772], + [15540.25,17784.745,773], + [15577.82,17804.745,774], + [15582.69,17824.745,775], + [15607.14,17844.745,776], + [15627.72,17864.745,777], + [15652.97,17884.745,778], + [15679.69,17904.745,779], + [15689.29,17924.745,780], + [15700.97,17944.745,781], + [15730.15,17964.745,782], + [15743.86,17984.745,783], + [15765.91,18004.745,784], + [15789.95,18024.745,785], + [15806.11,18044.745,786], + [15835.46,18064.745,787], + [15849.26,18084.745,788], + [15863.13,18104.745,789], + [15897.47,18124.745,790], + [15910.25,18144.745,791], + [15925.78,18164.745,792], + [15948.27,18184.745,793], + [15973.72,18204.745,794], + [15989.05,18224.745,795], + [16021,18244.745,796], + [16027.06,18264.745,797], + [16054.81,18284.745,798], + [16064.04,18304.745,799], + [16082.87,18324.745,800], + [16111.78,18344.745,801], + [16175.69,18364.745,802], + [16181.22,18384.745,803], + [16199.99,18404.745,804], + [16200.38,18424.745,805], + [16226.21,18444.745,806], + [16231.28,18464.745,807], + [16248.06,18484.745,808], + [16266.75,18504.745,809], + [16289.65,18524.745,810], + [16308.06,18544.745,811], + [16331.99,18564.745,812], + [16346.34,18584.745,813], + [16372.28,18604.745,814], + [16384.81,18624.745,815], + [16409.17,18644.745,816], + [16428.72,18664.745,817], + [16451.56,18684.745,818], + [16471.49,18704.745,819], + [16490.24,18724.744,820], + [16503.59,18744.744,821], + [16538.94,18764.744,822], + [16543.86,18784.744,823], + [16562.74,18804.744,824], + [16585.99,18824.744,825], + [16605.67,18844.744,826], + [16631.43,18864.744,827], + [16648.22,18884.744,828], + [16662.86,18904.744,829], + [16693.09,18924.744,830], + [16701.08,18944.744,831], + [16729.03,18964.744,832], + [16746.9,18984.744,833], + [16768.94,19004.744,834], + [16790.95,19024.744,835], + [16810.13,19044.744,836], + [16822.26,19064.744,837], + [16850.33,19084.744,838], + [16863.34,19104.745,839], + [16886.06,19124.745,840], + [16904.81,19144.745,841], + [16930.92,19164.745,842], + [16950.69,19184.745,843], + [16968.87,19204.745,844], + [16979.43,19224.745,845], + [17014.25,19244.745,846], + [17023.69,19264.745,847], + [17044.06,19284.745,848], + [17065.96,19304.745,849], + [17093.69,19324.745,850], + [17107.68,19344.745,851], + [17135.9,19364.745,852], + [17144.09,19384.745,853], + [17170.83,19404.745,854], + [17182.14,19424.745,855], + [17206.22,19444.745,856], + [17223.94,19464.745,857], + [17255.64,19484.745,858], + [17273.94,19504.745,859], + [17290.37,19524.745,860], + [17301.68,19544.745,861], + [17335.52,19564.745,862], + [17340.09,19584.745,863], + [17364.03,19604.745,864], + [17391.93,19624.745,865], + [17407.31,19644.745,866], + [17433.54,19664.745,867], + [17448.95,19684.745,868], + [17463.24,19704.745,869], + [17494.03,19724.745,870], + [17502.15,19744.745,871], + [17522,19764.745,872], + [17546.28,19784.745,873], + [17568.71,19804.745,874], + [17594.83,19824.745,875], + [17611.31,19844.745,876], + [17623.48,19864.745,877], + [17660.58,19884.745,878], + [17669.5,19904.745,879], + [17686.73,19924.745,880], + [17713.12,19944.745,881], + [17728.88,19964.745,882], + [17750.34,19984.745,883], + [17769.96,20004.745,884], + [17780.31,20024.745,885], + [17812.25,20044.745,886], + [17823.72,20064.745,887], + [17848.38,20084.745,888], + [17869.72,20104.745,889], + [17888.93,20124.745,890], + [17918.28,20144.745,891], + [17935.94,20164.745,892], + [17942.34,20184.745,893], + [17974.57,20204.745,894], + [17990.8,20224.745,895], + [18007.68,20244.745,896], + [18028.65,20264.745,897], + [18046.36,20284.745,898], + [18074.06,20304.745,899], + [18089.4,20324.745,900], + [18098.12,20344.745,901], + [18130.6,20364.745,902], + [18144.64,20384.745,903], + [18164.89,20404.745,904], + [18186.75,20424.745,905], + [18204.84,20444.745,906], + [18228.94,20464.745,907], + [18254.34,20484.745,908], + [18261.08,20504.745,909], + [18291.64,20524.745,910], + [18303.61,20544.745,911], + [18321.26,20564.745,912], + [18346.66,20584.745,913], + [18367.45,20604.745,914], + [18387.37,20624.745,915], + [18412.18,20644.745,916], + [18420.23,20664.745,917], + [18457.21,20684.745,918], + [18465.16,20704.745,919], + [18487.44,20724.745,920], + [18505.69,20744.745,921], + [18528.34,20764.745,922], + [18551.11,20784.745,923], + [18569.35,20804.745,924], + [18581.07,20824.745,925], + [18613.65,20844.745,926], + [18623.19,20864.745,927], + [18640.87,20884.745,928], + [18666.65,20904.745,929], + [18684.64,20924.745,930], + [18712,20944.745,931], + [18730.96,20964.745,932], + [18737.5,20984.745,933], + [18780.64,21004.745,934], + [18787.29,21024.745,935], + [18805.08,21044.745,936], + [18826.11,21064.745,937], + [18857.3,21084.745,938], + [18875.96,21104.745,939], + [18893.22,21124.745,940], + [18903.55,21144.745,941], + [18932.99,21164.745,942], + [18940.84,21184.745,943], + [18962.22,21204.745,944], + [18987.97,21224.745,945], + [19004.69,21244.745,946], + [19027.34,21264.745,947], + [19055.22,21284.745,948], + [19066.23,21304.745,949], + [19099.7,21324.745,950], + [19104.93,21344.745,951], + [19127.88,21364.745,952], + [19146.53,21384.745,953], + [19170.06,21404.745,954], + [19187.59,21424.745,955], + [19214.95,21444.745,956], + [19220.57,21464.745,957], + [19254.45,21484.745,958], + [19269.09,21504.745,959], + [19289.28,21524.745,960], + [19306.27,21544.745,961], + [19329.89,21564.745,962], + [19349.15,21584.745,963], + [19374.47,21604.745,964], + [19385.59,21624.745,965], + [19414.19,21644.745,966], + [19424.64,21664.745,967], + [19445.22,21684.745,968], + [19465.8,21704.745,969], + [19486.59,21724.745,970], + [19506.53,21744.745,971], + [19532.21,21764.745,972], + [19538.52,21784.745,973], + [19572.13,21804.745,974], + [19580.68,21824.745,975], + [19606.17,21844.745,976], + [19627.29,21864.745,977], + [19645.94,21884.745,978], + [19667.35,21904.745,979], + [19688.23,21924.745,980], + [19701.83,21944.745,981], + [19740.31,21964.745,982], + [19746.73,21984.745,983], + [19767.14,22004.745,984], + [19788.09,22024.745,985], + [19809.81,22044.745,986], + [19828.34,22064.745,987], + [19852.69,22084.745,988], + [19861.76,22104.745,989], + [19897.88,22124.745,990], + [19905.31,22144.745,991], + [19925.62,22164.745,992], + [19952.78,22184.745,993], + [19973.52,22204.745,994], + [19988.46,22224.745,995], + [20010.23,22244.745,996], + [20018.75,22264.745,997], + [20053.92,22284.745,998], + [20061.94,22304.745,999], + [20092.01,22324.745,1000], + [20108.16,22344.745,1001], + [20126.81,22364.745,1002], + [20150.27,22384.745,1003], + [20172.22,22404.745,1004], + [20186.41,22424.745,1005], + [20231.36,22444.745,1006], + [20233.46,22464.745,1007], + [20245.02,22484.745,1008], + [20276.72,22504.745,1009], + [20291.19,22524.745,1010], + [20307.55,22544.745,1011], + [20329.33,22564.745,1012], + [20338.57,22584.745,1013], + [20377.56,22604.745,1014], + [20398.18,22624.745,1015], + [20416.47,22644.745,1016], + [20424.66,22664.745,1017], + [20445.69,22684.745,1018], + [20473.22,22704.745,1019], + [20489.61,22724.745,1020], + [20499.79,22744.745,1021], + [20561.76,22764.745,1022], + [20566.46,22784.745,1023], + [20569.67,22804.745,1024], + [20595.55,22824.745,1025], + [20608.5,22844.745,1026], + [20629.6,22864.745,1027], + [20649.08,22884.745,1028], + [20663.03,22904.745,1029], + [20701.24,22924.745,1030], + [20708.01,22944.745,1031], + [20726.02,22964.745,1032], + [20747.29,22984.745,1033], + [20765.89,23004.745,1034], + [20787.19,23024.745,1035], + [20812.89,23044.745,1036], + [20821.8,23064.745,1037], + [20852.03,23084.745,1038], + [20862.96,23104.745,1039], + [20889.65,23124.745,1040], + [20905.12,23144.745,1041], + [20927.56,23164.745,1042], + [20946.2,23184.745,1043], + [20970.25,23204.745,1044], + [20978.92,23224.745,1045], + [21019.22,23244.745,1046], + [21027.17,23264.745,1047], + [21046.14,23284.745,1048], + [21064.84,23304.745,1049], + [21085.66,23324.745,1050], + [21109.81,23344.745,1051], + [21132.46,23364.745,1052], + [21153.24,23384.745,1053], + [21185.29,23404.745,1054], + [21193.65,23424.745,1055], + [21204.17,23444.745,1056], + [21230.16,23464.745,1057], + [21255.79,23484.745,1058], + [21271.3,23504.745,1059], + [21289.11,23524.745,1060], + [21297.08,23544.745,1061], + [21331.6,23564.745,1062], + [21339.07,23584.745,1063], + [21362.55,23604.745,1064], + [21387,23624.745,1065], + [21406.25,23644.745,1066], + [21425.71,23664.745,1067], + [21450.5,23684.745,1068], + [21460.08,23704.745,1069], + [21496.71,23724.745,1070], + [21508.12,23744.745,1071], + [21535.12,23764.745,1072], + [21553.94,23784.745,1073], + [21566.16,23804.745,1074], + [21592.14,23824.745,1075], + [21610.49,23844.745,1076], + [21623.01,23864.745,1077], + [21650.72,23884.745,1078], + [21665.94,23904.745,1079], + [21687.99,23924.745,1080], + [21705.19,23944.745,1081], + [21725.18,23964.745,1082], + [21749.41,23984.745,1083], + [21772.69,24004.745,1084], + [21777.97,24024.745,1085], + [21813.3,24044.745,1086], + [21821.05,24064.745,1087], + [21846.72,24084.745,1088], + [21864.73,24104.745,1089], + [21887.8,24124.745,1090], + [21909.22,24144.745,1091], + [21927.68,24164.745,1092], + [21944.41,24184.745,1093], + [21975.99,24204.745,1094], + [21981.84,24224.745,1095], + [22005.43,24244.745,1096], + [22050.17,24264.745,1097], + [22050.56,24284.745,1098], + [22067.53,24304.745,1099], + [22088.01,24324.745,1100], + [22099.11,24344.745,1101], + [22130.08,24364.745,1102], + [22140.17,24384.745,1103], + [22166.05,24404.745,1104], + [22186.64,24424.745,1105], + [22208.59,24444.745,1106], + [22236.03,24464.745,1107], + [22247.54,24484.745,1108], + [22262.98,24504.745,1109], + [22297.67,24524.745,1110], + [22302.87,24544.745,1111], + [22324.64,24564.745,1112], + [22343.21,24584.745,1113], + [22364.76,24604.745,1114], + [22388.21,24624.745,1115], + [22409.63,24644.745,1116], + [22420.59,24664.745,1117], + [22452.99,24684.745,1118], + [22463.58,24704.745,1119], + [22481.88,24724.745,1120], + [22509.56,24744.745,1121], + [22527.5,24764.745,1122], + [22545.19,24784.745,1123], + [22570.21,24804.745,1124], + [22582.88,24824.745,1125], + [22615.07,24844.745,1126], + [22622.19,24864.745,1127], + [22660.94,24884.745,1128], + [22674.41,24904.745,1129], + [22692.94,24924.745,1130], + [22715.66,24944.745,1131], + [22730.52,24964.745,1132], + [22748.88,24984.745,1133], + [22774.78,25004.745,1134], + [22780.86,25024.745,1135], + [22818.93,25044.745,1136], + [22832.89,25064.745,1137], + [22849.34,25084.745,1138], + [22874.8,25104.745,1139], + [22892.87,25124.745,1140], + [22901.09,25144.745,1141], + [22931.56,25164.745,1142], + [22940.98,25184.745,1143], + [22965.55,25204.745,1144], + [22982.46,25224.745,1145], + [23003.8,25244.745,1146], + [23039.55,25264.745,1147], + [23052.64,25284.745,1148], + [23058.4,25304.745,1149], + [23095.96,25324.745,1150], + [23106.78,25344.745,1151], + [23133.42,25364.745,1152], + [23144.36,25384.745,1153], + [23167.58,25404.745,1154], + [23187.89,25424.745,1155], + [23210.56,25444.745,1156], + [23224.16,25464.745,1157], + [23254.64,25484.745,1158], + [23265.24,25504.745,1159], + [23289.8,25524.745,1160], + [23305.65,25544.745,1161], + [23325.92,25564.745,1162], + [23351.41,25584.745,1163], + [23373.47,25604.745,1164], + [23387.58,25624.745,1165], + [23413.25,25644.745,1166], + [23419.11,25664.745,1167], + [23445.47,25684.745,1168], + [23465.53,25704.745,1169], + [23494.52,25724.745,1170], + [23509.51,25744.745,1171], + [23530.47,25764.745,1172], + [23541.52,25784.745,1173], + [23568.65,25804.745,1174], + [23582.87,25824.745,1175], + [23605.06,25844.745,1176], + [23627.37,25864.745,1177], + [23649.82,25884.745,1178], + [23677.29,25904.745,1179], + [23695.42,25924.745,1180], + [23705.53,25944.745,1181], + [23744.03,25964.745,1182], + [23750.62,25984.745,1183], + [23770.56,26004.745,1184], + [23791.64,26024.745,1185], + [23812.78,26044.745,1186], + [23833.56,26064.745,1187], + [23848.75,26084.745,1188], + [23859.92,26104.745,1189], + [23897.21,26124.745,1190], + [23905.83,26144.745,1191], + [23931,26164.745,1192], + [23953.71,26184.745,1193], + [23973.83,26204.745,1194], + [23990.6,26224.745,1195], + [24013.61,26244.745,1196], + [24030.32,26264.745,1197], + [24057.11,26284.745,1198], + [24074.47,26304.745,1199], + [24096.37,26324.745,1200], + [24112.47,26344.745,1201], + [24135.78,26364.745,1202], + [24147.33,26384.745,1203], + [24168.06,26404.745,1204], + [24193.34,26424.745,1205], + [24215.89,26444.745,1206], + [24225.75,26464.745,1207], + [24246.2,26484.745,1208], + [24276.68,26504.745,1209], + [24288.53,26524.745,1210], + [24311.25,26544.745,1211], + [24331.06,26564.745,1212], + [24342.69,26584.745,1213], + [24393.66,26604.745,1214], + [24394.7,26624.745,1215], + [24421.53,26644.745,1216], + [24431.37,26664.745,1217], + [24445.93,26684.745,1218], + [24472.03,26704.745,1219], + [24494.1,26724.745,1220], + [24504.47,26744.745,1221], + [24536.87,26764.745,1222], + [24545.84,26784.745,1223], + [24564.44,26804.745,1224], + [24589.21,26824.745,1225], + [24609.4,26844.745,1226], + [24632.79,26864.745,1227], + [24654.19,26884.745,1228], + [24662.9,26904.745,1229], + [24693.98,26924.745,1230], + [24701.06,26944.745,1231], + [24726.84,26964.745,1232], + [24748.64,26984.745,1233], + [24767.44,27004.745,1234], + [24797.2,27024.745,1235], + [24819.72,27044.745,1236], + [24829.07,27064.745,1237], + [24850.47,27084.745,1238], + [24867.72,27104.745,1239], + [24887.34,27124.745,1240], + [24906.53,27144.745,1241], + [24928.09,27164.745,1242], + [24947.66,27184.745,1243], + [24967.13,27204.745,1244], + [24979.17,27224.745,1245], + [25023.84,27244.745,1246], + [25028.97,27264.745,1247], + [25047.78,27284.745,1248], + [25071.99,27304.745,1249], + [25097.85,27324.745,1250], + [25113.73,27344.745,1251], + [25133.9,27364.745,1252], + [25142.75,27384.745,1253], + [25175.37,27404.745,1254], + [25185.5,27424.745,1255], + [25214.49,27444.745,1256], + [25229.84,27464.745,1257], + [25249.69,27484.745,1258], + [25271.05,27504.745,1259], + [25292.97,27524.745,1260], + [25305.13,27544.745,1261], + [25330.65,27564.745,1262], + [25342.11,27584.745,1263], + [25363.18,27604.745,1264], + [25386.65,27624.745,1265], + [25411.79,27644.745,1266], + [25436.82,27664.745,1267], + [25451.47,27684.745,1268], + [25462.09,27704.745,1269], + [25495.34,27724.745,1270], + [25504.92,27744.745,1271], + [25525.29,27764.745,1272], + [25554.46,27784.745,1273], + [25570.49,27804.745,1274], + [25589.7,27824.745,1275], + [25611.96,27844.745,1276], + [25628.56,27864.745,1277], + [25659.13,27884.745,1278], + [25672.93,27904.745,1279], + [25689.77,27924.745,1280], + [25706.4,27944.745,1281], + [25734.28,27964.745,1282], + [25753.91,27984.745,1283], + [25770.07,28004.745,1284], + [25786.1,28024.745,1285], + [25814.82,28044.745,1286], + [25824.55,28064.745,1287], + [25854.02,28084.745,1288], + [25865.66,28104.745,1289], + [25898.96,28124.745,1290], + [25911.78,28144.745,1291], + [25936.31,28164.745,1292], + [25939.53,28184.745,1293], + [25976.37,28204.745,1294], + [25985.36,28224.745,1295], + [26007.09,28244.745,1296], + [26035.46,28264.745,1297], + [26047.05,28284.745,1298], + [26068.22,28304.745,1299], + [26091.13,28324.745,1300], + [26108.54,28344.745,1301], + [26130.51,28364.745,1302], + [26147.74,28384.745,1303], + [26164.17,28404.745,1304], + [26184.77,28424.745,1305], + [26210.68,28444.745,1306], + [26226.63,28464.745,1307], + [26252.03,28484.745,1308], + [26257.62,28504.745,1309], + [26297.75,28524.745,1310], + [26306.53,28544.745,1311], + [26329.25,28564.745,1312], + [26347.22,28584.745,1313], + [26365.19,28604.745,1314], + [26387.06,28624.745,1315], + [26411.18,28644.745,1316], + [26420.76,28664.745,1317], + [26454.2,28684.745,1318], + [26463.83,28704.745,1319], + [26486.19,28724.745,1320], + [26502.83,28744.745,1321], + [26526.8,28764.745,1322], + [26548.26,28784.745,1323], + [26571.25,28804.745,1324], + [26585.75,28824.745,1325], + [26614.65,28844.745,1326], + [26631.75,28864.745,1327], + [26648.63,28884.745,1328], + [26667.74,28904.745,1329], + [26690.03,28924.745,1330], + [26713.28,28944.745,1331], + [26732.05,28964.745,1332], + [26738.81,28984.745,1333], + [26776.54,29004.745,1334], + [26789.47,29024.745,1335], + [26807.59,29044.745,1336], + [26838.52,29064.745,1337], + [26844.18,29084.745,1338], + [26872.27,29104.745,1339], + [26898.55,29124.745,1340], + [26907.36,29144.745,1341], + [26939.59,29164.745,1342], + [26942.64,29184.745,1343], + [26964.44,29204.745,1344], + [26989.94,29224.745,1345], + [27007.91,29244.745,1346], + [27033.69,29264.745,1347], + [27059.46,29284.745,1348], + [27066.43,29304.745,1349], + [27098,29324.745,1350], + [27110.91,29344.745,1351], + [27125.27,29364.745,1352], + [27145.16,29384.745,1353], + [27170.99,29404.745,1354], + [27213.57,29424.745,1355], + [27215.77,29444.745,1356], + [27218.53,29464.745,1357], + [27256.02,29484.745,1358], + [27262.27,29504.745,1359], + [27288.41,29524.745,1360], + [27311.98,29544.745,1361], + [27335.71,29564.745,1362], + [27353.25,29584.745,1363], + [27367.69,29604.745,1364], + [27378.59,29624.745,1365], + [27412,29644.745,1366], + [27421.5,29664.745,1367], + [27449.03,29684.745,1368], + [27472.77,29704.745,1369], + [27488.81,29724.745,1370], + [27514.43,29744.745,1371], + [27540.24,29764.745,1372], + [27546.29,29784.745,1373], + [27576.06,29804.745,1374], + [27587.97,29824.745,1375], + [27609,29844.745,1376], + [27629.24,29864.745,1377], + [27658.08,29884.745,1378], + [27671.68,29904.745,1379], + [27692.02,29924.745,1380], + [27697.62,29944.745,1381], + [27746.64,29964.745,1382], + [27747.31,29984.745,1383], + [27768.29,30004.745,1384], + [27793.66,30024.745,1385], + [27805.45,30044.745,1386], + [27834.9,30064.745,1387], + [27848.76,30084.745,1388], + [27868.08,30104.745,1389], + [27899.23,30124.745,1390], + [27905.76,30144.745,1391], + [27924.47,30164.745,1392], + [27950.41,30184.745,1393], + [27964.39,30204.745,1394], + [27989.23,30224.745,1395], + [28011.61,30244.745,1396], + [28029.47,30264.745,1397], + [28059.93,30284.745,1398], + [28069.34,30304.745,1399], + [28087.37,30324.745,1400], + [28104.95,30344.745,1401], + [28130.36,30364.745,1402], + [28148.51,30384.745,1403], + [28179.84,30404.745,1404], + [28232.78,30424.745,1405], + [28235.91,30444.745,1406], + [28236.37,30464.745,1407], + [28249.16,30484.745,1408], + [28268.32,30504.745,1409], + [28288.95,30524.745,1410], + [28311.84,30544.745,1411], + [28329.74,30564.745,1412], + [28336.01,30584.745,1413], + [28371.04,30604.745,1414], + [28389.23,30624.745,1415], + [28404.53,30644.745,1416], + [28436.23,30664.745,1417], + [28452.4,30684.745,1418], + [28472.75,30704.745,1419], + [28504.17,30724.745,1420], + [28504.5,30744.745,1421], + [28536.56,30764.745,1422], + [28542.55,30784.745,1423], + [28567.82,30804.745,1424], + [28593.77,30824.745,1425], + [28616.89,30844.745,1426], + [28630.41,30864.745,1427], + [28667.61,30884.745,1428], + [28668.33,30904.745,1429], + [28694.44,30924.745,1430], + [28702.15,30944.745,1431], + [28730.61,30964.745,1432], + [28747.68,30984.745,1433], + [28776.28,31004.745,1434], + [28788.25,31024.745,1435], + [28809.94,31044.745,1436], + [28826.78,31064.745,1437], + [28860.98,31084.745,1438], + [28877.76,31104.745,1439], + [28885.77,31124.745,1440], + [28906.71,31144.745,1441], + [28928.09,31164.745,1442], + [28954.15,31184.745,1443], + [28978.52,31204.745,1444], + [28982.46,31224.745,1445], + [29025.44,31244.745,1446], + [29034.44,31264.745,1447], + [29055.74,31284.745,1448], + [29069.68,31304.745,1449], + [29092.09,31324.745,1450], + [29108.25,31344.745,1451], + [29129.72,31364.745,1452], + [29140.09,31384.745,1453], + [29176.63,31404.745,1454], + [29194.42,31424.745,1455], + [29211.42,31444.745,1456], + [29231.97,31464.745,1457], + [29256.81,31484.745,1458], + [29269.43,31504.745,1459], + [29304.09,31524.745,1460], + [29307.56,31544.745,1461], + [29340.84,31564.745,1462], + [29361.66,31584.745,1463], + [29372.77,31604.745,1464], + [29404.91,31624.745,1465], + [29420.14,31644.745,1466], + [29432.52,31664.745,1467], + [29466.02,31684.745,1468], + [29478.46,31704.745,1469], + [29511.13,31724.745,1470], + [29523.84,31744.745,1471], + [29524.63,31764.745,1472], + [29544.81,31784.745,1473], + [29572.58,31804.745,1474], + [29592.4,31824.745,1475], + [29612.09,31844.745,1476], + [29623.78,31864.745,1477], + [29656.72,31884.745,1478], + [29670.59,31904.745,1479], + [29694.69,31924.745,1480], + [29707.62,31944.745,1481], + [29732.81,31964.745,1482], + [29751.53,31984.745,1483], + [29774.58,32004.745,1484], + [29779.78,32024.745,1485], + [29826.26,32044.745,1486], + [29826.69,32064.745,1487], + [29844.53,32084.745,1488], + [29868.37,32104.745,1489], + [29889.2,32124.745,1490], + [29910.3,32144.745,1491], + [29927.41,32164.745,1492], + [29943.92,32184.745,1493], + [29974.88,32204.745,1494], + [29986.15,32224.745,1495], + [30010.55,32244.745,1496], + [30029.15,32264.745,1497], + [30052.4,32284.745,1498], + [30070.1,32304.745,1499], + [30088.61,32324.745,1500], + [30102.3,32344.745,1501], + [30133.39,32364.745,1502], + [30143.52,32384.745,1503], + [30175.86,32404.745,1504], + [30188.72,32424.745,1505], + [30215.64,32444.745,1506], + [30232.71,32464.745,1507], + [30255.94,32484.745,1508], + [30264.34,32504.745,1509], + [30294.22,32524.745,1510], + [30307.52,32544.745,1511], + [30328.74,32564.745,1512], + [30346.08,32584.745,1513], + [30373.62,32604.745,1514], + [30392.43,32624.745,1515], + [30423.02,32644.745,1516], + [30428.04,32664.745,1517], + [30453.97,32684.745,1518], + [30464.41,32704.745,1519], + [30485.18,32724.745,1520], + [30508.08,32744.745,1521], + [30531.86,32764.745,1522], + [30547.62,32784.745,1523], + [30571.14,32804.745,1524], + [30582.96,32824.745,1525], + [30619.22,32844.745,1526], + [30624.19,32864.745,1527], + [30644.94,32884.745,1528], + [30672.69,32904.745,1529], + [30690,32924.745,1530], + [30718.78,32944.745,1531], + [30728.33,32964.745,1532], + [30741.84,32984.745,1533], + [30774.84,33004.745,1534], + [30780.37,33024.745,1535], + [30813.11,33044.745,1536], + [30829.81,33064.745,1537], + [30847.98,33084.745,1538], + [30878.75,33104.745,1539], + [30901.28,33124.745,1540], + [30902.58,33144.745,1541], + [30934.74,33164.745,1542], + [30964.14,33184.745,1543], + [30972.14,33204.745,1544], + [30987.01,33224.745,1545], + [31005.79,33244.745,1546], + [31030.16,33264.745,1547], + [31051.55,33284.745,1548], + [31057.52,33304.745,1549], + [31096.49,33324.745,1550], + [31107.38,33344.745,1551], + [31133.6,33364.745,1552], + [31167.45,33384.745,1553], + [31171.58,33404.745,1554], + [31193.78,33424.745,1555], + [31234.72,33444.745,1556], + [31247.71,33464.745,1557], + [31261.3,33484.745,1558], + [31273.39,33504.745,1559], + [31300.96,33524.745,1560], + [31315.84,33544.745,1561], + [31341.75,33564.745,1562], + [31358.54,33584.745,1563], + [31372.28,33604.745,1564], + [31383.66,33624.745,1565], + [31423.57,33644.745,1566], + [31429.9,33664.745,1567], + [31446.09,33684.745,1568], + [31467.75,33704.745,1569], + [31490.14,33724.745,1570], + [31514.66,33744.745,1571], + [31531.77,33764.745,1572], + [31546.41,33784.745,1573], + [31583.51,33804.745,1574], + [31590.83,33824.745,1575], + [31606.84,33844.745,1576], + [31629.36,33864.745,1577], + [31648.56,33884.745,1578], + [31683.84,33904.745,1579], + [31695.71,33924.745,1580], + [31708.11,33944.745,1581], + [31739.5,33964.745,1582], + [31749.13,33984.745,1583], + [31769.66,34004.745,1584], + [31786.4,34024.745,1585], + [31810.02,34044.745,1586], + [31829.59,34064.745,1587], + [31862.28,34084.745,1588], + [31869.93,34104.745,1589], + [31892.07,34124.745,1590], + [31910.09,34144.745,1591], + [31922.6,34164.745,1592], + [31955.69,34184.745,1593], + [31977.59,34204.745,1594], + [31990.16,34224.745,1595], + [32013.18,34244.745,1596], + [32022.53,34264.745,1597], + [32055.84,34284.745,1598], + [32061.38,34304.745,1599], + [32087.96,34324.745,1600], + [32111.78,34344.745,1601], + [32138.59,34364.745,1602], + [32145.59,34384.745,1603], + [32174.11,34404.745,1604], + [32182.26,34424.745,1605], + [32212.87,34444.745,1606], + [32225.2,34464.745,1607], + [32251.73,34484.745,1608], + [32270.94,34504.745,1609], + [32289.26,34524.745,1610], + [32309.45,34544.745,1611], + [32330.4,34564.745,1612], + [32342.56,34584.745,1613], + [32373.28,34604.745,1614], + [32382.06,34624.745,1615], + [32409.91,34644.745,1616], + [32428.18,34664.745,1617], + [32446.94,34684.745,1618], + [32468.71,34704.745,1619], + [32488.75,34724.745,1620], + [32502.42,34744.745,1621], + [32532.14,34764.745,1622], + [32542,34784.745,1623], + [32566.9,34804.745,1624], + [32588.19,34824.745,1625], + [32607.52,34844.745,1626], + [32628.47,34864.745,1627], + [32649.21,34884.745,1628], + [32665.42,34904.745,1629], + [32704.83,34924.745,1630], + [32714.33,34944.745,1631], + [32725.23,34964.745,1632], + [32756.48,34984.745,1633], + [32766.21,35004.745,1634], + [32786.23,35024.745,1635], + [32808.31,35044.745,1636], + [32843.97,35064.745,1637], + [32848.81,35084.745,1638], + [32871.66,35104.744,1639], + [32884.82,35124.744,1640], + [32909.82,35144.744,1641], + [32925.68,35164.744,1642], + [32953.94,35184.744,1643], + [32978.05,35204.744,1644], + [32984.49,35224.744,1645], + [33016.7,35244.744,1646], + [33023.36,35264.744,1647], + [33055.98,35284.744,1648], + [33069.43,35304.744,1649], + [33090.6,35324.744,1650], + [33108.59,35344.744,1651], + [33133.73,35364.744,1652], + [33138.11,35384.744,1653], + [33180.16,35404.744,1654], + [33198.44,35424.744,1655], + [33215.02,35444.744,1656], + [33244.39,35464.744,1657], + [33253.67,35484.744,1658], + [33277.44,35504.744,1659], + [33289.75,35524.744,1660], + [33308.04,35544.744,1661], + [33336.46,35564.744,1662], + [33348.34,35584.744,1663], + [33379.66,35604.744,1664], + [33394.03,35624.744,1665], + [33467.68,35644.744,1666], + [33487.94,35664.744,1667], + [33488.19,35684.744,1668], + [33488.42,35704.744,1669], + [33494.14,35724.744,1670], + [33504.19,35744.744,1671], + [33523.07,35764.744,1672], + [33551.59,35784.744,1673], + [33566.06,35804.744,1674], + [33596.17,35824.744,1675], + [33613.97,35844.744,1676], + [33625.5,35864.744,1677], + [33658.05,35884.745,1678], + [33663.83,35904.745,1679], + [33690.91,35924.745,1680], + [33723.63,35944.745,1681], + [33726.52,35964.745,1682], + [33754.03,35984.745,1683], + [33774.93,36004.745,1684], + [33788.19,36024.745,1685], + [33810.69,36044.745,1686], + [33820.76,36064.745,1687], + [33843.05,36084.745,1688], + [33870.92,36104.745,1689], + [33888.27,36124.745,1690], + [33923.34,36144.745,1691], + [33930.84,36164.745,1692], + [33938.01,36184.745,1693], + [33981.11,36204.745,1694], + [33984.38,36224.745,1695], + [34008.41,36244.745,1696], + [34029.34,36264.745,1697], + [34056.3,36284.745,1698], + [34074.65,36304.745,1699], + [34094.26,36324.745,1700], + [34107.72,36344.745,1701], + [34141.99,36364.745,1702], + [34148.11,36384.745,1703], + [34174.3,36404.745,1704], + [34187.58,36424.745,1705], + [34210.09,36444.745,1706], + [34238.46,36464.745,1707], + [34253.78,36484.745,1708], + [34263.96,36504.745,1709], + [34313.14,36524.745,1710], + [34314.44,36544.745,1711], + [34328.59,36564.745,1712], + [34353.72,36584.745,1713], + [34375.72,36604.745,1714], + [34395.25,36624.745,1715], + [34430.81,36644.745,1716], + [34474.11,36664.745,1717], + [34510.88,36684.745,1718], + [34511.25,36704.745,1719], + [34511.44,36724.745,1720], + [34521.42,36744.745,1721], + [34537.44,36764.745,1722], + [34552.49,36784.745,1723], + [34584.16,36804.745,1724], + [34593.16,36824.745,1725], + [34613.93,36844.745,1726], + [34625.03,36864.745,1727], + [34650.06,36884.745,1728], + [34667.72,36904.745,1729], + [34693.82,36924.745,1730], + [34721.68,36944.745,1731], + [34743.8,36964.745,1732], + [34748.39,36984.745,1733], + [34780.86,37004.745,1734], + [34797.81,37024.745,1735], + [34815.28,37044.745,1736], + [34832.37,37064.745,1737], + [34859.22,37084.745,1738], + [34880.69,37104.745,1739], + [34898.72,37124.745,1740], + [34909.97,37144.745,1741], + [34936.79,37164.745,1742], + [34951.06,37184.745,1743], + [34967.18,37204.745,1744] + ] +} diff --git a/test/justfile b/test/justfile index ae8c67339a..ed03d7cde9 100644 --- a/test/justfile +++ b/test/justfile @@ -85,6 +85,14 @@ drill *args: audio-quality *args: ./audio-quality/run.sh "$@" +# Records the arrival traces under audio-quality/traces/ that the replay rows grade: the +# public relay's bbb.hang, a local relay's AAC tone, and a browser microphone through the +# public relay. Needs the network. Narrow it with `--lanes relay-bbb --duration 35`. + +# Re-record the audio quality arrival traces. +audio-quality-record *args: + ./audio-quality/record.sh "$@" + # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. From caa7635a3d2e0cd49b2f57cfce8300fe933c96b4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:41:23 -0700 Subject: [PATCH 3/6] quest(audio-quality): plan replay CI and the real-mic re-record Replays run in seconds without a browser, so they gate PRs while the rest of the matrix stays nightly. The watch quest's manual run re-records relay-mic with a real microphone. Co-Authored-By: Claude Opus 5.5 --- quest/m0/audio-jitter-target/watch.md | 4 +++- quest/m0/audio-quality-harness/README.md | 5 ++++- quest/m0/audio-quality-harness/replay-ci.md | 22 +++++++++++++++++++++ 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 quest/m0/audio-quality-harness/replay-ci.md diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index 2b04c4c010..ac64a4fc0c 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -81,7 +81,9 @@ against it is likely cheaper than patching the branch's: `replay.test.ts`, move the replay's target onto the estimator, and tighten the harness's exact replay budgets with it. - Manual run against the public relay on Chrome and Safari, the two rows the - issue measured. Measure the publisher's audio encoder input-to-output lag in + issue measured. Re-record the `relay-mic` trace in the same run with a real + microphone (`just test audio-quality-record`); the checked-in one used + Chromium's fake capture device. Measure the publisher's audio encoder input-to-output lag in the same run using the reporter's instrumented harness; #3518 fixed the known cause, so the 7.35 s lag and the 88 to 275 ms/s drift the issue reported stand unconfirmed. If drift survives, the suspects are `writeFrame` opening a diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index 9656a4a84a..0330c28127 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -43,7 +43,9 @@ audio quality number makes a failure hard to attribute. Nightly, not per-PR: the matrix is jitter profiles by runtime by codec and sample rate, which is more than a merge gate should carry, and `nightly.yml` -already exists for exactly this trade. Budgets are keyed by the full row, since +already exists for exactly this trade. The recorded-trace replays are the +exception: they run in seconds without a browser, so [Replay +CI](/quest/m0/audio-quality-harness/replay-ci.md) gates PRs on them. Budgets are keyed by the full row, since each of those dimensions moves the expected floor. The budgets checked in with the browser lane (#4426) were recorded locally. @@ -54,6 +56,7 @@ quest](/quest/m0/audio-jitter-target/watch.md). ## Required +- [Replay CI](/quest/m0/audio-quality-harness/replay-ci.md) - the trace replays gate every PR that touches the player - [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/replay-ci.md b/quest/m0/audio-quality-harness/replay-ci.md new file mode 100644 index 0000000000..3eeef636c7 --- /dev/null +++ b/quest/m0/audio-quality-harness/replay-ci.md @@ -0,0 +1,22 @@ +# [XS] Trace replays gate every PR + +## Goal + +The recorded-trace replay rows run on any PR that touches `js/watch`, +`js/hang`, or `test/audio-quality`, graded against their exact budgets, so a +regression in the rings or `Sync` fails the PR that caused it instead of the +next nightly. The browser and shaper rows stay nightly. + +## Plan + +Replays need no browser, relay, or shaper: `run.sh` feeds each trace through +`test/audio-quality/clients/js/replay.ts` and they finish in seconds, so they +are cheap enough for a merge gate, unlike the rest of the matrix. Wire a +replays-only mode into the scoped JS path of `just check` (decided with the +maintainer over folding it into the watch quest's `replay.test.ts`), so CI +and local checks agree. + +Replay rows are deterministic, so a change that moves them updates +`budgets.json` in the same PR, with the new numbers in its description. + +Public API: none. Wire: none. From 69c14304d2a3241e5b729f761b24636c5b48638d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:14:08 -0700 Subject: [PATCH 4/6] test(audio-quality): replay traces through the consumer and gate PRs on them The replay now writes each recorded arrival into its group and lets the player's Container.Consumer order, hold, and skip groups at the delay's max age, resetting the ring on a discontinuity as the decoder does. local-aac's budgets move with it: its relay flushes groups newest-first, so the consumer holds a burst behind its oldest frame and the ring lands one quantum later. target() waits on the delay signal instead of a timer, run.sh skips the browser, relay, and publishers when only replays run (--replays), the recorder stops on the page clock, and replay.ts rejects unknown rings. .github/workflows/audio-quality.yml grades the replays on PRs touching js/watch, js/hang, or the harness. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/audio-quality.yml | 60 +++++ .github/workflows/nightly.yml | 4 +- js/watch/src/audio/replay.test.ts | 36 ++- js/watch/src/audio/replay.ts | 163 +++++++++++--- quest/m0/audio-jitter-target/watch.md | 9 +- quest/m0/audio-quality-harness/README.md | 13 +- quest/m0/audio-quality-harness/replay-ci.md | 22 -- test/audio-quality/README.md | 21 +- test/audio-quality/budgets.json | 12 +- test/audio-quality/clients/js/record.ts | 5 +- test/audio-quality/clients/js/replay.ts | 43 +++- test/audio-quality/run.sh | 230 +++++++++++--------- test/justfile | 1 + 13 files changed, 408 insertions(+), 211 deletions(-) create mode 100644 .github/workflows/audio-quality.yml delete mode 100644 quest/m0/audio-quality-harness/replay-ci.md diff --git a/.github/workflows/audio-quality.yml b/.github/workflows/audio-quality.yml new file mode 100644 index 0000000000..e04b976ad4 --- /dev/null +++ b/.github/workflows/audio-quality.yml @@ -0,0 +1,60 @@ +name: Audio quality + +# Grades the recorded-trace replays of test/audio-quality against their exact +# budgets on every PR that can move them. They replay the player's container +# consumer and rings on a simulated clock, so they are deterministic and take +# seconds, unlike the browser rows, which stay in nightly.yml. +on: + pull_request: + paths: + - "js/watch/**" + # The container consumer decides what a replay delivers, waits for, and skips. + - "js/hang/**" + - "test/audio-quality/**" + - "test/lib/**" + - "test/justfile" + - ".github/workflows/audio-quality.yml" + +concurrency: + group: audio-quality-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + replay: + name: Replay + runs-on: ubuntu-24.04-arm + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter; see check.yml. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + - name: Replay + run: nix develop --command just test audio-quality --replays --enforce + shell: bash -leo pipefail {0} + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + + # The harness keeps a failing run's directory: every row's samples and summary. + - name: Keep the failing run + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: audio-quality-replay + path: ${{ runner.temp }}/moq-test-runs/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 92807fe75e..f8b3941168 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,7 +23,9 @@ name: Nightly # number it grades is a timing one that moves with whatever else the runner is # doing. A per-PR arm would fail on unrelated load often enough to be ignored; # nightly it competes with nothing, and a regression surfaces a day late against -# a budget that still means something. +# a budget that still means something. The recorded-trace replays are the +# exception: deterministic and seconds long, so audio-quality.yml also gates +# PRs on them. # # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index 109019ffea..d4f23355cf 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -1,20 +1,20 @@ -import { describe, expect, it } from "bun:test"; +import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; import type * as Catalog from "@moq/hang/catalog"; import { Time } from "@moq/net"; import { type Arrival, type Options, QUANTUM, replay, target } from "./replay"; const RATE = 48000; -/** Twenty ms frames for `seconds`, each arriving `lateness(i)` ms after it was captured. */ +/** Twenty ms frames for `seconds`, one per group, each arriving `lateness(i)` ms after it was captured. */ function paced(seconds: number, lateness: (i: number) => number): Arrival[] { - return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20 })); + return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20, group: i })); } /** Quanta that came back short or empty while playing, once the ring had first played. */ -function underruns(trace: Arrival[], options: Options): number { +async function underruns(trace: Arrival[], options: Options): Promise { let started = false; let short = 0; - for (const { output, stalled } of replay(trace, options)) { + for await (const { output, stalled } of replay(trace, options)) { const filled = output.findLastIndex((v) => v !== 0) + 1; if (started && !stalled && filled < QUANTUM) short++; if (filled > 0) started = true; @@ -23,20 +23,36 @@ function underruns(trace: Arrival[], options: Options): number { } describe.each(["shared", "post"] as const)("%s ring", (ring) => { - it("plays an evenly paced sender without a gap", () => { + // The consumer warns on every group it skips. + let warn: Mock; + beforeEach(() => { + warn = spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => warn.mockRestore()); + + it("plays an evenly paced sender without a gap", async () => { expect( - underruns( + await underruns( paced(10, () => 30), { ring, rate: RATE, delay: 100 }, ), ).toBe(0); }); - it("underruns when a flush span outlasts the target, and not when it is covered", () => { + it("underruns when a flush span outlasts the target, and not when it is covered", async () => { // Five frames held and flushed at once: 80 ms of arrival spread. const trace = paced(10, (i) => 30 + (4 - (i % 5)) * 20); - expect(underruns(trace, { ring, rate: RATE, delay: 40 })).toBeGreaterThan(50); - expect(underruns(trace, { ring, rate: RATE, delay: 150 })).toBe(0); + expect(await underruns(trace, { ring, rate: RATE, delay: 40 })).toBeGreaterThan(50); + expect(await underruns(trace, { ring, rate: RATE, delay: 150 })).toBe(0); + }); + + it("holds newer groups behind a missing one until the max age gives up on it", async () => { + const trace = paced(10, () => 30).filter((arrival) => arrival.group !== 250); + // The consumer delivers in group order, so the ring runs dry for longer than the missing frame + // while the groups behind it wait. Writing arrivals straight into the ring plays straight through. + const frame = Math.ceil((20 / 1000) * (RATE / QUANTUM)); + expect(await underruns(trace, { ring, rate: RATE, delay: 100 })).toBeGreaterThan(frame); + expect(warn).toHaveBeenCalled(); }); }); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index c314a57a28..ba027dd643 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -1,11 +1,13 @@ /** - * Replay a recorded arrival trace through the player's real rings on a simulated clock. + * Replay a recorded arrival trace through the player's consumer and rings on a simulated clock. * - * Every frame is inserted at the instant the trace says it reached the container consumer, and one - * render quantum is read every quantum's worth of that same clock, which is what the AudioWorklet - * does. {@link target} resolves the delay with a real {@link Sync} fed the recorded catalog config - * and connection RTT, and the ring is sized from it the way the decoder sizes it, so a change to - * either moves what a replay hears. Decoding is taken as instant and sample exact. + * Every frame is written into its group at the instant the trace says it arrived, and the player's + * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max age the delay + * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder + * does. One render quantum is read every quantum's worth of that same clock, which is what the + * AudioWorklet does. {@link target} resolves the delay with a real {@link Sync} fed the recorded + * catalog config and connection RTT, and the ring is sized from it the way the decoder sizes it, so + * a change to any of them moves what a replay hears. Decoding is taken as instant and sample exact. * * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test * can assert exact counts. @@ -13,13 +15,15 @@ * @internal Test support, not part of the player. */ import type * as Catalog from "@moq/hang/catalog"; -import { Time } from "@moq/net"; +import * as Container from "@moq/hang/container"; +import { Group, Time, Track } from "@moq/net"; import { Signal } from "@moq/signals"; import { type Delay, Sync } from "../sync"; import { playbackJitter } from "./config"; import { ringSamples } from "./latency"; import { AudioRingBuffer } from "./ring-buffer"; import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; +import { Terminal } from "./terminal"; /** An AudioWorklet render quantum, in frames. */ export const QUANTUM = 128; @@ -27,18 +31,29 @@ export const QUANTUM = 128; /** A media gap longer than this is missing audio, not one long frame. */ const MAX_FRAME_MS = 100; +/** + * The replay's own subscription passes every recorded group through: the recorder's relay already + * applied a wire max age, so only the consumer's decisions are made again. + */ +const WIRE_MAX_AGE = Time.Milli(Number.MAX_SAFE_INTEGER); + +/** Any non-empty payload: an empty one is the legacy container's end marker. */ +const PAYLOAD = new Uint8Array(1); + /** One frame reaching the container consumer. */ export interface Arrival { /** When it arrived, on the viewer's monotonic clock, in ms. */ at: number; /** Its media timestamp, in ms. */ timestamp: number; + /** The group that carried it. */ + group: number; } /** What the player resolves its delay from. */ export interface Target { - /** The configured delay, as the element takes it. */ - delay: Delay; + /** The configured delay, as the element takes it. "instant" plays no audio, so it has no target. */ + delay: Exclude; /** The rendition as the catalog described it: its advertised jitter. */ config: Catalog.AudioConfig; /** The connection's round trip, in ms, which "auto" sizes from. Unset is no PROBE. */ @@ -51,10 +66,14 @@ export async function target(input: Target): Promise { delay: input.delay, probe: input.rtt === undefined ? undefined : { rtt: Time.Milli(input.rtt) }, }); - const unregister = sync.register(new Signal(playbackJitter(input.config))); + const jitter = playbackJitter(input.config); + const unregister = sync.register(new Signal(jitter)); try { - // Effects flush on a microtask, and a timer runs after all of them. - await new Promise((resolve) => setTimeout(resolve, 0)); + // Sync's effects run on microtasks. The delay has settled once it holds this rendition's jitter + // on top of the resolved jitter, the only rendition registered. + while (sync.out.delay.peek() !== Time.Milli.add(sync.out.jitter.peek(), jitter)) { + await sync.out.delay.changed(); + } return sync.out.delay.peek(); } finally { unregister(); @@ -68,7 +87,10 @@ export interface Options { ring: "shared" | "post"; /** The sample rate, in Hz. */ rate: number; - /** The resolved delay the ring is sized to, in ms. See {@link target}. */ + /** + * The resolved delay, in ms. See {@link target}. It sizes the ring and is the consumer's max age, + * which `Sync` sets to the delay plus a lookahead a live viewer does not configure. + */ delay: number; } @@ -88,6 +110,7 @@ export interface Quantum { interface Ring { insert(timestamp: Time.Micro, data: Float32Array[]): void; read(output: Float32Array[]): number; + reset(): void; readonly stalled: boolean; readonly timestamp: Time.Micro; } @@ -106,6 +129,7 @@ function post(rate: number, latency: number): Ring { return { insert: (timestamp, data) => ring.write(timestamp, data), read: (output) => ring.read(output), + reset: () => ring.reset(), get stalled() { return ring.stalled; }, @@ -115,31 +139,49 @@ function post(rate: number, latency: number): Ring { }; } +/** A trace's media timestamp as the container carries it. */ +const micros = (ms: number) => Math.round(ms * 1000) as Time.Micro; + /** - * Samples each frame carries: up to where the next one starts, so consecutive frames tile the - * timeline exactly. A frame followed by a hole, or by nothing, gets the trace's typical frame. + * Samples each frame carries, by its timestamp: up to where the next one starts, so consecutive + * frames tile the timeline exactly. A frame followed by a hole, or by nothing, gets the trace's + * typical frame. */ -function frameSamples(trace: Arrival[], rate: number): number[] { +function frameSamples(trace: Arrival[], rate: number): Map { const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b); const sample = (ms: number) => Math.round((ms * rate) / 1000); - const spans = new Map(); + const spans = new Map(); for (let i = 0; i < starts.length - 1; i++) { - if (starts[i + 1] - starts[i] < MAX_FRAME_MS) spans.set(starts[i], sample(starts[i + 1]) - sample(starts[i])); + if (starts[i + 1] - starts[i] < MAX_FRAME_MS) { + spans.set(micros(starts[i]), sample(starts[i + 1]) - sample(starts[i])); + } } const sorted = [...spans.values()].sort((a, b) => a - b); const typical = sorted[Math.floor(sorted.length / 2)]; if (typical === undefined) throw new Error("a trace needs two frames less than 100 ms apart"); - return trace.map((a) => spans.get(a.timestamp) ?? typical); + for (const start of starts) { + if (!spans.has(micros(start))) spans.set(micros(start), typical); + } + return spans; } +/** + * Run every microtask already queued, and every one those queue, before returning. + * + * A macrotask starts only once the microtask queue is empty, and nothing between a group write and + * the ring arms a timer, so this orders the replay behind the consumer rather than waiting on time. + */ +const settle = () => new Promise((resolve) => setImmediate(resolve)); + /** * Play `trace` in simulated real time, yielding every render quantum from the first arrival to the * last. * - * The audio is a constant, never zero, so what the ring did not supply is exactly the zeros at the - * end of a quantum, which is how the harness's output tap reads a real page. + * A group's stream finishes with its last recorded frame. The audio is a constant, never zero, so + * what the ring did not supply is exactly the zeros at the end of a quantum, which is how the + * harness's output tap reads a real page. */ -export function* replay(trace: Arrival[], options: Options): Generator { +export async function* replay(trace: Arrival[], options: Options): AsyncGenerator { const first = trace[0]; const last = trace.at(-1); if (!first || !last) return; @@ -148,21 +190,76 @@ export function* replay(trace: Arrival[], options: Options): Generator const samples = frameSamples(trace, rate); const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, Time.Milli(options.delay))); + // Where each group's stream finishes. + const ends = new Map(); + trace.forEach((arrival, i) => { + ends.set(arrival.group, i); + }); + + const track = new Track.Producer("audio"); + const consumer = new Container.Consumer(track.subscribe({ maxAge: WIRE_MAX_AGE }), { + format: new Container.Legacy.Format("audio"), + maxAge: Time.Milli(options.delay), + }); + + // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. + const terminal = new Terminal(); + let failure: { error: unknown } | undefined; + const decode = (async () => { + for (;;) { + const next = await consumer.next(); + if (!next) return; + if (terminal.update(next)) ring.reset(); + if (next.end !== undefined || !next.frame) continue; + + const { timestamp } = next.frame; + const length = samples.get(timestamp); + if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); + ring.insert(timestamp, [new Float32Array(length).fill(0.5)]); + } + })().catch((error: unknown) => { + failure = { error }; + }); + + const groups = new Map(); const step = (QUANTUM / rate) * 1000; const output = new Float32Array(QUANTUM); let next = 0; - for (let n = 1; first.at + (n - 1) * step <= last.at; n++) { - // The quantum ending at `now` renders once everything that arrived by then is in. - const now = first.at + n * step; - while (next < trace.length && trace[next].at <= now) { - const { timestamp } = trace[next]; - ring.insert(Time.Micro.fromMilli(timestamp as Time.Milli), [new Float32Array(samples[next]).fill(0.5)]); - next++; - } + try { + for (let n = 1; first.at + (n - 1) * step <= last.at; n++) { + // The quantum ending at `now` renders once everything that arrived by then is delivered. + const now = first.at + n * step; + const written = next; + while (next < trace.length && trace[next].at <= now) { + const arrival = trace[next]; + let group = groups.get(arrival.group); + if (!group) { + group = new Group.Producer(arrival.group); + groups.set(arrival.group, group); + track.writeGroup(group); + } + const timestamp = micros(arrival.timestamp); + group.writeFrame({ + payload: Container.Legacy.encodeFrame(PAYLOAD, timestamp), + timestamp: Time.Timestamp.fromMicros(timestamp), + }); + if (ends.get(arrival.group) === next) { + group.close(); + groups.delete(arrival.group); + } + next++; + } + if (next > written) await settle(); + if (failure) throw failure.error; - output.fill(0); - ring.read([output]); - yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp) }; + output.fill(0); + ring.read([output]); + yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp) }; + } + } finally { + consumer.close(); + track.close(); + await decode; } } diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index ac64a4fc0c..406dc95d98 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -76,10 +76,11 @@ against it is likely cheaper than patching the branch's: bug, and it also means auto gets far less real exposure than it looks like. - Replay recorded traces rather than synthetic ones of the same shape. `test/audio-quality/traces/` holds fresh ones (the #3477 traces are gone), - and `js/watch/src/audio/replay.ts` already plays a trace through both rings - on a simulated clock at the delay `Sync` resolves. Replay them in - `replay.test.ts`, move the replay's target onto the estimator, and tighten - the harness's exact replay budgets with it. + and `js/watch/src/audio/replay.ts` already plays a trace through + `Container.Consumer` and both rings on a simulated clock at the delay `Sync` + resolves. Replay them in `replay.test.ts`, move the replay's target onto the + estimator (its 100 ms fallback covers the public relay's missing PROBE RTT + until then), and tighten the harness's exact replay budgets with it. - Manual run against the public relay on Chrome and Safari, the two rows the issue measured. Re-record the `relay-mic` trace in the same run with a real microphone (`just test audio-quality-record`); the checked-in one used diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index 0330c28127..f3ad92d880 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -27,9 +27,10 @@ The browser lane has landed as `test/audio-quality/`, upstreamed from the reporter's fork on #3477 (`fperex/moq`, branch `debug-findings-solution`) on its own, without the fork's player, estimator, or shaper changes. Real arrival traces have landed beside it in `test/audio-quality/traces/`, graded as replay -rows and replayed by the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md). That leaves the bursty and step -profiles, which need shaper features the fork has and `main` does not. +rows through the player's container consumer and rings, and replayed by the +[jitter target's watch quest](/quest/m0/audio-jitter-target/watch.md). That +leaves the bursty and step profiles, which need shaper features the fork has +and `main` does not. Jitter comes from the seeded userspace UDP shaper the transport drills run under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake @@ -44,8 +45,9 @@ audio quality number makes a failure hard to attribute. Nightly, not per-PR: the matrix is jitter profiles by runtime by codec and sample rate, which is more than a merge gate should carry, and `nightly.yml` already exists for exactly this trade. The recorded-trace replays are the -exception: they run in seconds without a browser, so [Replay -CI](/quest/m0/audio-quality-harness/replay-ci.md) gates PRs on them. Budgets are keyed by the full row, since +exception: they run in seconds without a browser, so +`.github/workflows/audio-quality.yml` gates every PR touching `js/watch`, +`js/hang`, or the harness on them. Budgets are keyed by the full row, since each of those dimensions moves the expected floor. The budgets checked in with the browser lane (#4426) were recorded locally. @@ -56,7 +58,6 @@ quest](/quest/m0/audio-jitter-target/watch.md). ## Required -- [Replay CI](/quest/m0/audio-quality-harness/replay-ci.md) - the trace replays gate every PR that touches the player - [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/replay-ci.md b/quest/m0/audio-quality-harness/replay-ci.md deleted file mode 100644 index 3eeef636c7..0000000000 --- a/quest/m0/audio-quality-harness/replay-ci.md +++ /dev/null @@ -1,22 +0,0 @@ -# [XS] Trace replays gate every PR - -## Goal - -The recorded-trace replay rows run on any PR that touches `js/watch`, -`js/hang`, or `test/audio-quality`, graded against their exact budgets, so a -regression in the rings or `Sync` fails the PR that caused it instead of the -next nightly. The browser and shaper rows stay nightly. - -## Plan - -Replays need no browser, relay, or shaper: `run.sh` feeds each trace through -`test/audio-quality/clients/js/replay.ts` and they finish in seconds, so they -are cheap enough for a merge gate, unlike the rest of the matrix. Wire a -replays-only mode into the scoped JS path of `just check` (decided with the -maintainer over folding it into the watch quest's `replay.test.ts`), so CI -and local checks agree. - -Replay rows are deterministic, so a change that moves them updates -`budgets.json` in the same PR, with the new numbers in its description. - -Public API: none. Wire: none. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 3ebb3f483d..597ffb471c 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -12,7 +12,7 @@ just test audio-quality --profiles mild --codecs opus --duration 20 just test audio-quality --rings plain # the production path only just test audio-quality --out ~/runs/after # keep the run directory just test audio-quality --enforce # fail on the budgets -just test audio-quality --profiles relay-bbb,local-aac,relay-mic # the replays only, in seconds +just test audio-quality --replays # the replays only, in seconds ``` The matrix is codec x profile x ring: 16 browser rows, about 20 minutes, plus 6 [replay @@ -86,12 +86,14 @@ viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, t over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend on the delay, so the replay decides those again. -`replay.ts` plays a trace through the player's own rings on a simulated clock -([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a real -`Sync` resolves for the recorded catalog and round trip, and reads each quantum through the tap's -classifier. It writes the same samples a page does, so `analyze.ts` reduces both alike, and the -shaper, transport, and clock checks have nothing to void. Decoding is taken as instant. A replay is -deterministic, so its budgets are exactly what it measured. +`replay.ts` plays a trace through the player's own `Container.Consumer` and rings on a simulated +clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a +real `Sync` resolves for the recorded catalog and round trip, and reads each quantum through the +tap's classifier. The consumer makes the player's group ordering, max age skips, and discontinuity +resets again at that delay; a group's stream is taken to finish with its last recorded frame. It +writes the same samples a page does, so `analyze.ts` reduces both alike, and the shaper, transport, +and clock checks have nothing to void. Decoding is taken as instant. A replay is deterministic, so +its budgets are exactly what it measured. ```bash just test audio-quality-record # all three, 35 s each, needs the network @@ -187,7 +189,10 @@ unmeasured, a void row, or a row with no budget. Tightening a ceiling is a visib one needs a reason in review. The file's `note` says how the current ceilings were measured. The nightly `Audio quality` job runs the matrix under `--enforce` and keeps a failing run directory -for a week: each process's log, the shaper's counters, every row's raw samples and summary. +for a week: each process's log, the shaper's counters, every row's raw samples and summary. The +replays also run under `--enforce` on every PR that touches `js/watch`, `js/hang`, or the harness +(`.github/workflows/audio-quality.yml`), so a change that moves them updates `budgets.json` in the +same PR. ## Layout diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 9d8407e985..6d212be1f5 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -231,12 +231,12 @@ "rate": 44100, "profile": "local-aac", "ring": "isolated", - "underruns_per_min": 101.7, - "underrun_ms_per_min": 287.3, + "underruns_per_min": 103.7, + "underrun_ms_per_min": 293.2, "skip_aheads_per_min": 2, "discarded_ms_per_min": 94.4, "stalled_share": 0, - "silence_share": 0.004, + "silence_share": 0.005, "target_ms_p95": 186 }, { @@ -245,12 +245,12 @@ "rate": 44100, "profile": "local-aac", "ring": "plain", - "underruns_per_min": 101.7, - "underrun_ms_per_min": 287.3, + "underruns_per_min": 103.7, + "underrun_ms_per_min": 293.2, "skip_aheads_per_min": 2, "discarded_ms_per_min": 94.4, "stalled_share": 0, - "silence_share": 0.004, + "silence_share": 0.005, "target_ms_p95": 186 }, { diff --git a/test/audio-quality/clients/js/record.ts b/test/audio-quality/clients/js/record.ts index b374c905f9..ffd792496f 100644 --- a/test/audio-quality/clients/js/record.ts +++ b/test/audio-quality/clients/js/record.ts @@ -102,13 +102,16 @@ try { return drained.length > 0 ? drained : undefined; }).then((first) => arrivals.push(...first)); + // Ends on the page's clock, not on the next arrival, so a publisher that goes quiet is recorded + // as the silence it is instead of holding the recording open. const start = arrivals[0][0]; - while ((arrivals.at(-1)?.[0] ?? start) < start + durationMs) { + while ((await page.evaluate(() => performance.now())) < start + durationMs) { await sleep(1000); const error = await page.evaluate(() => globalThis.recorder.error()); if (error) throw new Error(`recorder: ${error}`); arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); } + arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); // The RTT is read last: the PROBE minimum only falls as the session runs. const rtt = await page.evaluate(() => globalThis.recorder.info()?.rtt ?? null); diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index 328143751c..0697928723 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -1,12 +1,12 @@ /** - * Replays the recorded traces through the player's rings, and writes each row the way `driver.ts` - * writes a browser row, so `analyze.ts` reduces both alike. + * Replays the recorded traces through the player's consumer and rings, and writes each row the way + * `driver.ts` writes a browser row, so `analyze.ts` reduces both alike. * - * The player half is `js/watch/src/audio/replay.ts`: the real rings on a simulated clock, sized to - * the delay a real `Sync` resolves for the recorded catalog and RTT. This half reads its quanta - * through the same classifier the page's output tap runs, and samples the ring every 250 ms of - * simulated time, as the probe samples a page. Every profile is the trace's name at the "auto" - * delay a viewer gets by default. + * The player half is `js/watch/src/audio/replay.ts`: the real container consumer and rings on a + * simulated clock, at the delay a real `Sync` resolves for the recorded catalog and RTT. This half + * reads its quanta through the same classifier the page's output tap runs, and samples the ring + * every 250 ms of simulated time, as the probe samples a page. Every profile is the trace's name at + * the "auto" delay a viewer gets by default. * * bun replay.ts --traces ../../traces --profiles relay-bbb --rings plain --codecs aac --list * bun replay.ts --traces ../../traces --out @@ -55,8 +55,18 @@ const traces = readdirSync(dir) .sort() .map((f) => ({ profile: basename(f, ".json"), trace: JSON.parse(readFileSync(join(dir, f), "utf8")) as Trace })); const profiles = values.profiles === undefined ? undefined : split(values.profiles); -const rings = split(values.rings) as Ring[]; +const rings = split(values.rings); const codecs = split(values.codecs); +for (const [name, list, known] of [ + ["ring", rings, ["isolated", "plain"]], + ["codec", codecs, ["opus", "aac"]], +] as const) { + const unknown = list.find((v) => !(known as readonly string[]).includes(v)); + if (unknown !== undefined) { + console.error(`unknown ${name} '${unknown}' (known: ${known.join(", ")})`); + process.exit(2); + } +} const rows: { row: Row; trace: Trace }[] = []; for (const { profile, trace } of traces) { @@ -64,7 +74,7 @@ for (const { profile, trace } of traces) { const codec = traceCodec(trace); if (profiles && !profiles.includes(profile)) continue; if (!codecs.includes(codec)) continue; - for (const ring of rings) { + for (const ring of rings as Ring[]) { rows.push({ row: { runtime: "replay", codec, rate: trace.config.sampleRate, profile, ring }, trace }); } } @@ -94,8 +104,16 @@ for (const { row, trace } of rows) { let frame = 0; let next = SAMPLE_INTERVAL_MS; - const arrivals = trace.arrivals.map(([at, timestamp]) => ({ at, timestamp })); - for (const quantum of replay(arrivals, { ring: row.ring === "isolated" ? "shared" : "post", rate, delay })) { + // The consumer's warnings, as the page's probe keeps a browser row's console. + const notes: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + if (notes.length < 200) notes.push(args.map(String).join(" ").slice(0, 200)); + }; + + const arrivals = trace.arrivals.map(([at, timestamp, group]) => ({ at, timestamp, group })); + const quanta = replay(arrivals, { ring: row.ring === "isolated" ? "shared" : "post", rate, delay }); + for await (const quantum of quanta) { // The render clock is the simulated one: the trace's `at`, which starts at zero. ledger.add(frame, quantum.output.length, classify([quantum.output])); frame += quantum.output.length; @@ -121,6 +139,7 @@ for (const { row, trace } of rows) { stalls = []; } } + console.warn = warn; const environment: Environment = { crossOriginIsolated: row.ring === "isolated", @@ -131,6 +150,6 @@ for (const { row, trace } of rows) { contextRate: rate, }; await Bun.write(join(out, `${tag}.ndjson`), `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); - await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes: [], voids: [] }, null, 1)); + await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids: [] }, null, 1)); console.log(`${tag}: ${samples.length} samples at a ${delay} ms target`); } diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index e8f47dfe03..1416a24dcd 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -56,6 +56,11 @@ while [[ $# -gt 0 ]]; do PROFILES=("${SPLIT[@]}") shift 2 ;; + --replays) + # Only the recorded traces: seconds, and no browser, relay, or shaper. + PROFILES=("${ALL_TRACES[@]}") + shift + ;; --rings) need "$1" $# "${2:-}" split "$2" @@ -183,7 +188,9 @@ if [[ $LIST -eq 1 ]]; then exit 0 fi -for tool in cargo bun ffmpeg; do +tools=(bun) +[[ ${#ROWS[@]} -eq 0 ]] || tools+=(cargo ffmpeg) +for tool in "${tools[@]}"; do command -v "$tool" >/dev/null 2>&1 || { echo "error: $tool not found; run inside 'nix develop'" >&2 exit 1 @@ -192,121 +199,128 @@ done harness_begin audio-quality "$RERUN" -# ── build ─────────────────────────────────────────────────────────────────── -flag=() -[[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") -echo "building moq-relay, moq, moq-shaper ($PROFILE)..." -(cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) -TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" -RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" -MOQ="$TARGET_BASE/$PROFILE/moq" -SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" - -# Into the run directory: vite empties its output first, so a shared dist/ would vanish under a -# concurrent run's page loads. -echo "building the page..." -( - cd "$CLIENT" - bun install --frozen-lockfile - bunx playwright install chromium - bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir -) - -# ── relay ─────────────────────────────────────────────────────────────────── -harness_port relay -RELAY_PORT="$HARNESS_PORT" -RELAY_URL="http://127.0.0.1:$RELAY_PORT" -sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" -harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" -if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then - echo "relay never became ready" >&2 - sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true - exit 1 -fi -harness_endpoint relay "$RELAY_URL" - -# ── publishers ────────────────────────────────────────────────────────────── -# A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. The -# publishers talk to the relay directly: impairing the ingest would grade the receiver on a stream -# already damaged before it was published. The AAC arm keeps ffmpeg's default PES packing, whose -# multi-frame bursts are the flush-span shape the reporter measured on the public relay (#3477). -# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' -publish_opus() { - ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ - -ac 2 -c:a libopus -b:a 128k \ - -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | - "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 -} +echo "installing the workspace..." +(cd "$CLIENT" && bun install --frozen-lockfile) -# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' -publish_aac() { - ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ - -ac 2 -c:a aac -b:a 128k -f mpegts - | - "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts -} +failed=0 -for codec in "${CODECS[@]}"; do - harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" -done +# The browser rows need the relay, the publishers, the shaper, and a page; replays need none of it. +if [[ ${#ROWS[@]} -gt 0 ]]; then + # ── build ─────────────────────────────────────────────────────────────── + flag=() + [[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") + echo "building moq-relay, moq, moq-shaper ($PROFILE)..." + (cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) + TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" + RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" + MOQ="$TARGET_BASE/$PROFILE/moq" + SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" + + # Into the run directory: vite empties its output first, so a shared dist/ would vanish under a + # concurrent run's page loads. + echo "building the page..." + ( + cd "$CLIENT" + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir + ) + + # ── relay ─────────────────────────────────────────────────────────────── + harness_port relay + RELAY_PORT="$HARNESS_PORT" + RELAY_URL="http://127.0.0.1:$RELAY_PORT" + sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" + harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" + if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then + echo "relay never became ready" >&2 + sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true + exit 1 + fi + harness_endpoint relay "$RELAY_URL" + + # ── publishers ────────────────────────────────────────────────────────── + # A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. + # The publishers talk to the relay directly: impairing the ingest would grade the receiver on a + # stream already damaged before it was published. The AAC arm keeps ffmpeg's default PES + # packing, whose multi-frame bursts are the flush-span shape the reporter measured on the public + # relay (#3477). + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish_opus() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -ac 2 -c:a libopus -b:a 128k \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 + } -# ── rows ──────────────────────────────────────────────────────────────────── -# One shaper port for the run: rows play one at a time. -harness_port shaper -SHAPER_PORT="$HARNESS_PORT" + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish_aac() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts + } -echo "" -echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" -failed=0 -for tag in "${ROWS[@]}"; do - IFS=- read -r _ codec _ rest <<<"$tag" - ring="${rest##*-}" - profile="${rest%-*}" - echo "" - echo "── $tag ──" - - # shellcheck disable=SC2046 # the profile's flags are meant to split - harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ - --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") - shaper_pid="$HARNESS_PID" - # It prints its seed and profile once bound; a page dialing before that would lose its handshake. - until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do - if harness_exited "$shaper_pid"; then - echo "shaper exited before binding for $tag" >&2 - sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true - exit 1 - fi - sleep 0.05 + for codec in "${CODECS[@]}"; do + harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" done - status=0 - harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ - --url "http://127.0.0.1:$SHAPER_PORT" \ - --fingerprint "$RELAY_URL/certificate.sha256" \ - --broadcast "tone-$codec.hang" \ - --page "$HARNESS_RUN/page" \ - --ring "$ring" \ - --delay "$(delay_of "$profile")" \ - --duration "$DURATION" \ - --tag "$tag" \ - --out "$HARNESS_RUN" - harness_wait "$HARNESS_PID" || status=$? - if [[ $status -ne 0 ]]; then - failed=1 - sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true - fi + # ── rows ──────────────────────────────────────────────────────────────── + # One shaper port for the run: rows play one at a time. + harness_port shaper + SHAPER_PORT="$HARNESS_PORT" - # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile - # acted on the way out, and exits nonzero if it did not. - kill -TERM -- -"$shaper_pid" 2>/dev/null || true - shaper_status=0 - harness_wait "$shaper_pid" || shaper_status=$? - echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + echo "" + echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" + for tag in "${ROWS[@]}"; do + IFS=- read -r _ codec _ rest <<<"$tag" + ring="${rest##*-}" + profile="${rest%-*}" + echo "" + echo "── $tag ──" + + # shellcheck disable=SC2046 # the profile's flags are meant to split + harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ + --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") + shaper_pid="$HARNESS_PID" + # It prints its seed and profile once bound; a page dialing before that would lose its handshake. + until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do + if harness_exited "$shaper_pid"; then + echo "shaper exited before binding for $tag" >&2 + sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true + exit 1 + fi + sleep 0.05 + done - bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { - echo "analyze failed for $tag" >&2 - failed=1 - } -done + status=0 + harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ + --url "http://127.0.0.1:$SHAPER_PORT" \ + --fingerprint "$RELAY_URL/certificate.sha256" \ + --broadcast "tone-$codec.hang" \ + --page "$HARNESS_RUN/page" \ + --ring "$ring" \ + --delay "$(delay_of "$profile")" \ + --duration "$DURATION" \ + --tag "$tag" \ + --out "$HARNESS_RUN" + harness_wait "$HARNESS_PID" || status=$? + if [[ $status -ne 0 ]]; then + failed=1 + sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true + fi + + # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile + # acted on the way out, and exits nonzero if it did not. + kill -TERM -- -"$shaper_pid" 2>/dev/null || true + shaper_status=0 + harness_wait "$shaper_pid" || shaper_status=$? + echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } + done +fi # ── replays ───────────────────────────────────────────────────────────────── # Simulated time, so the whole set takes seconds, and deterministic, so the budgets are exact. diff --git a/test/justfile b/test/justfile index ed03d7cde9..5262411960 100644 --- a/test/justfile +++ b/test/justfile @@ -79,6 +79,7 @@ drill *args: # heard (underruns, skip-aheads, silence, the resolved delay) against # audio-quality/budgets.json. The full matrix is 16 rows at 60 s; nightly runs it with # --enforce. Narrow it with `--profiles mild --codecs opus --rings plain --duration 20`. +# `--replays` grades only the recorded traces, in seconds, as PRs touching the player do. # See audio-quality/README.md. # Grade browser audio playout over an impaired path. From e7334dba2386ece9448474b0957cb11180554518 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:26:35 -0700 Subject: [PATCH 5/6] test(audio-quality): gate replays on net and signals, and validate the recording duration Co-Authored-By: Claude Opus 5.5 --- .github/workflows/audio-quality.yml | 5 ++++- quest/m0/audio-quality-harness/README.md | 4 ++-- test/audio-quality/README.md | 6 +++--- test/audio-quality/clients/js/record.ts | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/audio-quality.yml b/.github/workflows/audio-quality.yml index e04b976ad4..1ad4af43cc 100644 --- a/.github/workflows/audio-quality.yml +++ b/.github/workflows/audio-quality.yml @@ -8,8 +8,11 @@ on: pull_request: paths: - "js/watch/**" - # The container consumer decides what a replay delivers, waits for, and skips. + # The container consumer decides what a replay delivers, waits for, and skips, on the + # groups, tracks, and signals underneath it. - "js/hang/**" + - "js/net/**" + - "js/signals/**" - "test/audio-quality/**" - "test/lib/**" - "test/justfile" diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index f3ad92d880..a02c524c59 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -46,8 +46,8 @@ Nightly, not per-PR: the matrix is jitter profiles by runtime by codec and sample rate, which is more than a merge gate should carry, and `nightly.yml` already exists for exactly this trade. The recorded-trace replays are the exception: they run in seconds without a browser, so -`.github/workflows/audio-quality.yml` gates every PR touching `js/watch`, -`js/hang`, or the harness on them. Budgets are keyed by the full row, since +`.github/workflows/audio-quality.yml` gates every PR touching the player's +packages or the harness on them. Budgets are keyed by the full row, since each of those dimensions moves the expected floor. The budgets checked in with the browser lane (#4426) were recorded locally. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 597ffb471c..d351a84bba 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -190,9 +190,9 @@ one needs a reason in review. The file's `note` says how the current ceilings we The nightly `Audio quality` job runs the matrix under `--enforce` and keeps a failing run directory for a week: each process's log, the shaper's counters, every row's raw samples and summary. The -replays also run under `--enforce` on every PR that touches `js/watch`, `js/hang`, or the harness -(`.github/workflows/audio-quality.yml`), so a change that moves them updates `budgets.json` in the -same PR. +replays also run under `--enforce` on every PR that touches the player's packages (`js/watch`, +`js/hang`, `js/net`, `js/signals`) or the harness (`.github/workflows/audio-quality.yml`), so a +change that moves them updates `budgets.json` in the same PR. ## Layout diff --git a/test/audio-quality/clients/js/record.ts b/test/audio-quality/clients/js/record.ts index ffd792496f..ca4f6008f5 100644 --- a/test/audio-quality/clients/js/record.ts +++ b/test/audio-quality/clients/js/record.ts @@ -32,8 +32,8 @@ const { values } = parseArgs({ }); const { page: pageDir, url, broadcast, source, description, out } = values; -const durationMs = Number.parseFloat(values.duration) * 1000; -if (!pageDir || !url || !broadcast || !source || !description || !out || !(durationMs > 0)) { +const durationMs = Number(values.duration) * 1000; +if (!pageDir || !url || !broadcast || !source || !description || !out || !Number.isFinite(durationMs) || durationMs <= 0) { console.error( "usage: record.ts --page DIR --url U --broadcast B --source S --description D --out FILE [--duration S] [--mic]", ); From d360b87a82b5cf72d7f2cf70303b7540359d68d3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:43:11 -0700 Subject: [PATCH 6/6] test(audio-quality): refuse an unknown replay profile Co-Authored-By: Claude Opus 5.5 --- test/audio-quality/clients/js/replay.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index 0697928723..28e2d56a6d 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -57,11 +57,13 @@ const traces = readdirSync(dir) const profiles = values.profiles === undefined ? undefined : split(values.profiles); const rings = split(values.rings); const codecs = split(values.codecs); -for (const [name, list, known] of [ +const selectors: [string, string[], string[]][] = [ + ["profile", profiles ?? [], traces.map((t) => t.profile)], ["ring", rings, ["isolated", "plain"]], ["codec", codecs, ["opus", "aac"]], -] as const) { - const unknown = list.find((v) => !(known as readonly string[]).includes(v)); +]; +for (const [name, list, known] of selectors) { + const unknown = list.find((v) => !known.includes(v)); if (unknown !== undefined) { console.error(`unknown ${name} '${unknown}' (known: ${known.join(", ")})`); process.exit(2);