From 5a2994ac26a9e520c608ab0f1de42871def4630b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 13:26:56 -0700 Subject: [PATCH 01/27] quest(audio-quality-harness): open questline Co-Authored-By: Claude Opus 5.5 From 29b0a64bf7107851e9e45dde2369a8ab9000d32b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 17:38:15 -0700 Subject: [PATCH 02/27] test(audio-quality): grade browser audio playout against a budget, nightly (#4426) Co-authored-by: Claude Opus 5.5 Co-authored-by: fperex <270788582+fperex@users.noreply.github.com> --- .github/workflows/nightly.yml | 55 +++ bun.lock | 68 +++ js/justfile | 2 +- package.json | 1 + quest/m0/audio-jitter-target/watch.md | 6 +- quest/m0/audio-quality-harness/README.md | 18 +- quest/m0/audio-quality-harness/browser.md | 74 ---- .../audio-quality-harness/shaper-profiles.md | 34 ++ quest/m0/audio-quality-harness/traces.md | 33 ++ quest/m1/audio-quality-native.md | 7 +- test/README.md | 3 +- test/audio-quality/README.md | 183 ++++++++ test/audio-quality/budgets.json | 229 ++++++++++ test/audio-quality/clients/js/analyze.ts | 116 +++++ test/audio-quality/clients/js/driver.ts | 160 +++++++ test/audio-quality/clients/js/grade.ts | 102 +++++ test/audio-quality/clients/js/index.html | 10 + test/audio-quality/clients/js/package.json | 18 + .../clients/js/src/analyze.test.ts | 132 ++++++ test/audio-quality/clients/js/src/analyze.ts | 305 +++++++++++++ test/audio-quality/clients/js/src/page.ts | 43 ++ test/audio-quality/clients/js/src/probe.ts | 139 ++++++ .../clients/js/src/quantum.test.ts | 61 +++ test/audio-quality/clients/js/src/quantum.ts | 112 +++++ test/audio-quality/clients/js/src/schema.ts | 407 ++++++++++++++++++ .../clients/js/src/tap-worklet.ts | 39 ++ test/audio-quality/clients/js/src/tap.ts | 70 +++ test/audio-quality/clients/js/tsconfig.json | 7 + test/audio-quality/clients/js/vite.config.ts | 19 + test/audio-quality/relay.toml | 22 + test/audio-quality/run.sh | 300 +++++++++++++ test/justfile | 10 + 32 files changed, 2694 insertions(+), 91 deletions(-) delete mode 100644 quest/m0/audio-quality-harness/browser.md create mode 100644 quest/m0/audio-quality-harness/shaper-profiles.md create mode 100644 quest/m0/audio-quality-harness/traces.md create mode 100644 test/audio-quality/README.md create mode 100644 test/audio-quality/budgets.json create mode 100644 test/audio-quality/clients/js/analyze.ts create mode 100644 test/audio-quality/clients/js/driver.ts create mode 100644 test/audio-quality/clients/js/grade.ts create mode 100644 test/audio-quality/clients/js/index.html create mode 100644 test/audio-quality/clients/js/package.json create mode 100644 test/audio-quality/clients/js/src/analyze.test.ts create mode 100644 test/audio-quality/clients/js/src/analyze.ts create mode 100644 test/audio-quality/clients/js/src/page.ts create mode 100644 test/audio-quality/clients/js/src/probe.ts create mode 100644 test/audio-quality/clients/js/src/quantum.test.ts create mode 100644 test/audio-quality/clients/js/src/quantum.ts create mode 100644 test/audio-quality/clients/js/src/schema.ts create mode 100644 test/audio-quality/clients/js/src/tap-worklet.ts create mode 100644 test/audio-quality/clients/js/src/tap.ts create mode 100644 test/audio-quality/clients/js/tsconfig.json create mode 100644 test/audio-quality/clients/js/vite.config.ts create mode 100644 test/audio-quality/relay.toml create mode 100755 test/audio-quality/run.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 90002d9552..eed4b612bc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,6 +18,13 @@ name: Nightly # the wheel builds in a manylinux container, the Kotlin/Swift/Go/Dart packages # link the FFI on every platform) and nothing notices until the tag is pushed. # +# `audio-quality` is here because of what it measures. It plays sixteen rows of +# audio in real time, a minute each, which no merge gate should carry, and every +# number it grades is a timing one that moves with whatever else the runner is +# doing. A per-PR arm would fail on unrelated load often enough to be ignored; +# nightly it competes with nothing, and a regression surfaces a day late against +# a budget that still means something. +# # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. @@ -176,6 +183,54 @@ jobs: NEXTEST_PROFILE: ci MOQ_STRICT: 1 + # Plays the matrix in headless Chromium through moq-shaper and grades what the + # listener heard against test/audio-quality/budgets.json. ARM64 like wasm.yml, + # the only architecture cache.yml warms a Rust store for. + audio-quality: + name: Audio quality + runs-on: ubuntu-24.04-arm + # Sixteen rows at a minute each, plus the relay, CLI, and shaper built from source. + timeout-minutes: 60 + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Rust cache + uses: ./.github/actions/rust-cache + # Playwright's Chromium isn't in the nix devShell; `--with-deps` installs + # its apt runtime libs, and run.sh's own install then no-ops. + - name: Playwright Chromium + run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" + shell: bash -leo pipefail {0} + - name: Audio quality + run: nix develop --command just test audio-quality --enforce + shell: bash -leo pipefail {0} + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # The run directory is the diagnosis: each process's log, the shaper's + # counters, the raw samples, and every row's summary. The harness keeps it + # on a failure without being asked. + - name: Keep the failing run + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: audio-quality-run + path: ${{ runner.temp }}/moq-test-runs/ + retention-days: 7 + if-no-files-found: ignore + # Every tag-driven release workflow, run against main with publishing skipped: # the workflows gate their publish jobs on `github.ref_type == 'tag'`, and no # secrets are passed, so a dry-run cannot publish even by accident. diff --git a/bun.lock b/bun.lock index 661ebc2605..7ce6980b9c 100644 --- a/bun.lock +++ b/bun.lock @@ -343,6 +343,18 @@ "vite": "^8.3.0", }, }, + "test/audio-quality/clients/js": { + "name": "@moq/audio-quality", + "dependencies": { + "@moq/watch": "workspace:*", + }, + "devDependencies": { + "@types/audioworklet": "^0.0.100", + "esbuild": "^0.28.2", + "playwright": "^1.63.0", + "vite": "^8.3.0", + }, + }, "test/interop/clients/js": { "name": "@moq/interop-browser", "dependencies": { @@ -640,6 +652,8 @@ "@libav.js/variant-opus-af": ["@libav.js/variant-opus-af@6.10.9", "", {}, "sha512-yULJUCQIHDtwDPbT38t9pP/7EGgqwynVcOau46hZjtCe8TWMp67DTi2X0Emr2M4Ci3nzxNPf25IzO+IXSmHDBg=="], + "@moq/audio-quality": ["@moq/audio-quality@workspace:test/audio-quality/clients/js"], + "@moq/auth": ["@moq/auth@workspace:js/auth"], "@moq/binary": ["@moq/binary@workspace:js/binary"], @@ -1996,6 +2010,8 @@ "@types/concat-stream/@types/node": ["@types/node@26.0.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw=="], + "@vitejs/plugin-vue/vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], + "@vue/compiler-core/@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], "@vue/compiler-core/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], @@ -2172,6 +2188,10 @@ "@types/concat-stream/@types/node/undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + "@vitejs/plugin-vue/vite/esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], + + "@vitejs/plugin-vue/vite/postcss": ["postcss@8.5.16", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg=="], + "@vue/compiler-core/@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], "@vue/compiler-sfc/@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], @@ -2282,6 +2302,54 @@ "@npmcli/package-json/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "@vitejs/plugin-vue/vite/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="], + + "@vitejs/plugin-vue/vite/postcss/nanoid": ["nanoid@3.3.15", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA=="], + "@vue/compiler-core/@babel/parser/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], "@vue/compiler-core/@babel/parser/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], diff --git a/js/justfile b/js/justfile index 4be426446e..93e433102c 100644 --- a/js/justfile +++ b/js/justfile @@ -9,7 +9,7 @@ set working-directory := '.' # Paths a JS check depends on: every bun workspace (they are not all under js/, # see the root package.json) plus the config shared by all of them. `check`, # `fix`, and `ci` all skip themselves when a diff misses this. -scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' +scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/(audio-quality|interop)/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' # Run all checks by default. default: diff --git a/package.json b/package.json index 3a85e66642..0cae436c5b 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "demo/boy", "demo/web", "js/moq-boy", + "test/audio-quality/clients/js", "test/interop/clients/js", "test/interop/clients/js-native", "test/wasm" diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index a1a258f61e..75d43a4203 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -75,8 +75,8 @@ against it is likely cheaper than patching the branch's: that down: a stored preference silently winning over the default is its own bug, and it also means auto gets far less real exposure than it looks like. - Replay recorded traces rather than synthetic ones of the same shape. The - #3477 traces are gone, so record fresh ones with the [browser - harness](/quest/m0/audio-quality-harness/browser.md) (decided with the + #3477 traces are gone, so record fresh ones with the [audio quality + harness](/quest/m0/audio-quality-harness/traces.md) (decided with the maintainer instead of asking the reporter). Trim a copy into the repository and replay it through both rings in `replay.test.ts`. - Manual run against the public relay on Chrome and Safari, the two rows the @@ -97,7 +97,7 @@ that quest can adopt it without a second estimator change. ## Required -- [Browser harness](/quest/m0/audio-quality-harness/browser.md) - records the arrival traces this quest replays +- [Traces](/quest/m0/audio-quality-harness/traces.md) - records the arrival traces this quest replays ## Related diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index 457844c109..e3779f51ba 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -23,14 +23,13 @@ are. The native lane moved to m1 as a standalone quest (decided in the same budgets and metric schema so the two implementations can be compared rather than merely both passing. -The starting point is not a blank page. The reporter on #3477 already built a -working browser lane on their fork (`fperex/moq`, branch -`debug-findings-solution`, under `test/audio-quality/`): a Playwright-driven -matrix over `moq-shaper`, a budget file graded under `--enforce`, a nightly -job, and a replay runtime. Upstream that rather than reinventing it. The raw traces it shipped with are gone, so -this harness records fresh ones, and the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md) replays them (decided with the -maintainer during the merged-PR audit). +The browser lane has landed as `test/audio-quality/`, upstreamed from the +reporter's fork on #3477 (`fperex/moq`, branch `debug-findings-solution`) on +its own, without the fork's player, estimator, or shaper changes. That leaves +the two things that needed them: real arrival traces, which the [jitter +target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays, and the +bursty and step profiles, which need shaper features the fork has and `main` +does not. Jitter comes from the seeded userspace UDP shaper the transport drills run under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake @@ -49,7 +48,8 @@ each of those dimensions moves the expected floor. ## Required -- [Browser](/quest/m0/audio-quality-harness/browser.md) - upstream the fork's harness, grade it against a budget, run it nightly +- [Traces](/quest/m0/audio-quality-harness/traces.md) - real arrival traces recorded, checked in, and graded as replay rows +- [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/browser.md b/quest/m0/audio-quality-harness/browser.md deleted file mode 100644 index a47aed7969..0000000000 --- a/quest/m0/audio-quality-harness/browser.md +++ /dev/null @@ -1,74 +0,0 @@ -# [L] The browser audio lane is graded against a budget, nightly - -## Goal - -`just test audio-quality` plays a broadcast in a headless browser over a seeded, -impaired path and prints a graded table: underruns, short quanta, discarded -samples, skip-aheads, and where the end-to-end delay went. It fails when a -profile exceeds its budget. A nightly job runs the full matrix and keeps the -run directory of a failure. - -The same run works on both ring paths, isolated and postMessage, since the -production path is the one without cross-origin isolation. - -## Plan - -Upstream the reporter's lane from `fperex/moq` branch -`debug-findings-solution` rather than rebuilding it, keeping the attribution. -It already has a built `test/audio-quality/` lane: `just test audio-quality` -over `moq-shaper`, a Playwright-driven Chromium matrix, a `budgets.json` -graded by `grade.ts` under `--enforce`, a nightly job that keeps a failure's -run directory, and `chromium`, `safari`, and `replay` runtimes. It also -carries player, estimator, and shaper changes (checked 2026-09-28: 222 commits -ahead of and 148 behind `main`), so land the lane on its own and hold its -schema to the contract below rather than taking the branch wholesale. - -- Land the lane under `test/`, alongside the existing `interop` and `drill` - lanes, wired into the `justfile` the way they are. -- Keep the instrumentation ad-hoc for now. The probes stay a debug surface, not - public API; promoting them is [Latency - ledger](/quest/m2/latency-ledger.md), which nothing here waits on. -- The metric schema is the deliverable that outlives this quest, because the - native lane and the ledger both have to emit the same thing. It is a - contract, so write it as one: - - Every counter defined, not just the two that are obvious. An underrun is a - quantum the ring could only partly fill; a skip-ahead is a re-anchor that - discarded buffered audio; short quanta and discarded samples need the same - treatment rather than being left to the reader. - - Units and clock domain stated once. Durations in one unit, and every - timestamp on a named clock, so a browser value and a native value are the - same measurement. Naming the clock is not enough on its own: the stages span - the publisher, the relay and the viewer, which are separate processes and - often separate machines, so either reduce every timestamp to one common - monotonic reference or define the calibration and drift correction that maps - between them. The ledger's sum-to-end-to-end identity is arithmetic across - these values, and it is meaningless if they sit on unaligned clocks. - - Aggregation stated per metric: a count over the run, a max, or a - percentile. "Underruns: 3" and "underruns: 3/s" grade differently. - - Stages as exclusive, non-overlapping spans (capture, encode, publish flush, - network, jitter buffer, decode, render). The ledger's sum-to-end-to-end - identity is unimplementable if two stages can claim the same milliseconds, - and an unaccounted remainder is the finding, so give it a name too. -- Run the `moq-shaper` binary (`rs/moq-shaper`, see `test/drill/README.md`) - in front of the relay, so this harness and the drills share one impairment - implementation. It exits nonzero when the profile never acted, which the - harness must treat as a failed run: a profile that silently did nothing - turns an impaired run into an unimpaired pass. -- Profiles: near-zero, mild, bursty (the flush-span shape from #3477), and a - step change that forces the target to move mid-run. Fixed seeds, recorded - with the results. -- One profile runs a fixed delay preset rather than auto, as the control. Every - other profile exercises adaptation, so without this the whole matrix can pass - while a fixed preset regresses, and fixed presets are what a viewer lands on - today. -- Budgets in one checked-in file, keyed by the whole matrix row and not by - profile alone: runtime, codec, sample rate, profile, and the ring path, since - each of those moves the expected floor. A profile-only key silently grades - one row against another's threshold. Tightening a budget is then a visible - diff and loosening one needs a reason in review. -- Record real arrival traces (the #3477 release traces are gone), trim them - into a fixture, and replay them too, so a real recorded arrival pattern is - graded next to the synthetic profiles. The jitter target's watch quest - replays the same fixture. -- Add the lane to `nightly.yml`, and extend its header comment with why this - one is not a PR gate. diff --git a/quest/m0/audio-quality-harness/shaper-profiles.md b/quest/m0/audio-quality-harness/shaper-profiles.md new file mode 100644 index 0000000000..71536c518d --- /dev/null +++ b/quest/m0/audio-quality-harness/shaper-profiles.md @@ -0,0 +1,34 @@ +# [M] The audio quality matrix grades a bursty path and a mid-run step + +## Goal + +`just test audio-quality` gains the two profiles `moq-shaper` cannot express +today: `bursty`, which releases datagrams in batches (the flush-span shape from +#3477, seven datagrams per 160 ms window), and `step`, which changes the +profile part-way through a row and forces the adaptive target to move. Both get +budgets in `test/audio-quality/budgets.json` and run nightly with the rest. + +## Plan + +The browser lane landed on the shaper as it is: one uniform profile both ways, +fixed for the life of the process. Its `mild` and `wide` profiles spread +arrivals with uniform jitter, which also reorders datagrams whenever the +jitter exceeds their spacing, so `wide` measures reordering as much as spread. + +The reporter's fork (`fperex/moq`, branch `debug-findings-solution`, +`rs/moq-shaper`) already has what is missing, as separate commits: batches that +release datagrams together, steps that change a profile mid-run, an +order-preserving gaussian jitter model, named profiles loaded from TOML, and a +JSON report of the counters. Upstream the batch, step, and order-preserving +jitter changes with the fork author's credit, keeping each behind the options +the drills do not set so their lane is unchanged. Skip TCP passthrough: the +harness answers the page's certificate fetch itself. + +Then add `bursty` and `step` to `run.sh`, refuse a `--duration` that ends +before the step, and switch `mild` and `wide` to the order-preserving jitter so +their names mean spread alone. Re-record the budgets for every row whose profile +changed. + +## Related + +- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator the step profile exercises diff --git a/quest/m0/audio-quality-harness/traces.md b/quest/m0/audio-quality-harness/traces.md new file mode 100644 index 0000000000..ce0b749962 --- /dev/null +++ b/quest/m0/audio-quality-harness/traces.md @@ -0,0 +1,33 @@ +# [M] Real arrival traces are recorded, checked in, and graded + +## Goal + +A trimmed fixture of real audio arrival traces lives in the repository, and the +audio quality harness grades a replay of it next to its synthetic profiles. The +[jitter target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays +the same fixture through both rings in `replay.test.ts`. + +## Plan + +The #3477 traces are gone, so record fresh ones: a viewer on the public relay +playing `bbb.hang` (the AAC flush-span shape the issue measured), a local relay +as the shallow control, and a browser microphone published through the public +relay. Stamp each audio frame's arrival where the container consumer receives +it, with its media timestamp and group, on the viewer's monotonic clock (see +`test/audio-quality/clients/js/src/schema.ts`). + +The browser lane reads only public signals, and no public signal carries a +per-frame arrival, so decide where the recorder lives: a debug hook in +`@moq/watch` that the harness page enables, or a native subscriber (`moq` +CLI) that records the same path's arrivals. The reporter's fork (`fperex/moq`, +branch `debug-findings-solution`) recorded from a debug sampler in the player +and replayed through `js/watch/src/audio/replay.ts` on a simulated clock; +that replay module is the likely shape for the grading side, and landing it is +a player change this quest owns. + +Replay rows are deterministic, so their budgets are exactly what was measured, +with no headroom. + +## Related + +- [Latency ledger](/quest/m2/latency-ledger.md) - would promote an arrival stamp into public API diff --git a/quest/m1/audio-quality-native.md b/quest/m1/audio-quality-native.md index ff4d16ab81..11373ae103 100644 --- a/quest/m1/audio-quality-native.md +++ b/quest/m1/audio-quality-native.md @@ -25,7 +25,8 @@ from whichever platform a user happened to be on. - `moq play` needs to emit the run's counters and stage timings as JSON for the analyzer. Add that output, and keep it useful outside the test: a user debugging their own latency wants the same dump. -- Reuse the browser lane's analyzer and budget file, with the JSON field names +- Reuse the browser lane's analyzer and budget file (`test/audio-quality/`, the + schema is `clients/js/src/schema.ts`), with the JSON field names and meanings matching its schema exactly. Native rows get their own budget values in the same file, keyed the same way, since the device floor differs; they do not get their own schema. @@ -45,6 +46,6 @@ harness](/quest/m0/audio-quality-harness/README.md) line (decided in the 2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target it grades is done on the jitter target line. -## Required +## Related -- [Browser](/quest/m0/audio-quality-harness/browser.md) - defines the metric schema, the budget file, and the extracted shaper +- [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and step profiles both lanes should run diff --git a/test/README.md b/test/README.md index ae54517fa2..fcaf3e22bf 100644 --- a/test/README.md +++ b/test/README.md @@ -8,8 +8,9 @@ tests live in each language's own justfile. | [interop](interop/README.md) | `just test interop` | every client built from this checkout interoperates | | [wasm](wasm/README.md) | `just test wasm` | the `@moq/wasm` bindings work in a real browser | | [ts](ts/README.md) | `just test ts` | the subscriber's `export ts` output is IRD-compliant | +| [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser audio playout stays within budget over an impaired path | -All three stand up a `moq-relay` and clients, so two of them running at once, or +All four stand up a `moq-relay` and clients, so two of them running at once, or the same one running from two worktrees, would otherwise collide. `lib/harness.sh` is what keeps them apart. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md new file mode 100644 index 0000000000..242660731f --- /dev/null +++ b/test/audio-quality/README.md @@ -0,0 +1,183 @@ +# Audio quality + +Plays a tone in headless Chromium over a seeded, impaired UDP path and grades what the listener +would have heard: how often the ring ran dry, how much audio a re-anchor threw away, how much of the +run was silent, and where the playout delay settled. A regression in any of them fails a row +against [`budgets.json`](budgets.json) instead of arriving as a bug report. + +```bash +just test audio-quality # the whole matrix, 60s a row +just test audio-quality --list # what it would run +just test audio-quality --profiles mild --codecs opus --duration 20 +just test audio-quality --rings plain # the production path only +just test audio-quality --out ~/runs/after # keep the run directory +just test audio-quality --enforce # fail on the budgets +``` + +The matrix is codec x profile x ring: 16 rows, about 20 minutes. `--profiles`, `--rings`, and +`--codecs` take comma-separated lists; `--seed` replays a given impairment. The table prints either +way, and only `--enforce` (which the nightly job passes) turns a breach into a failed run. + +## Rows + +| Axis | Values | +| --- | --- | +| codec | `opus` at 48 kHz over fMP4, `aac` at 44.1 kHz over MPEG-TS | +| profile | see below | +| ring | `isolated` (SharedArrayBuffer, cross-origin isolated page) and `plain` (postMessage) | + +`plain` is the production path: most pages are not cross-origin isolated. One build is served under +`/isolated/` with COOP and COEP and under `/plain/` without, and the page's `crossOriginIsolated` +decides which ring ran. + +| Profile | Path | Delay | +| --- | --- | --- | +| `near-zero` | through the shaper, untreated | auto | +| `mild` | 5 ms one way, plus or minus 5 ms uniform | auto | +| `wide` | 40 ms one way, plus or minus 40 ms uniform | auto | +| `fixed-250` | through the shaper, untreated | a fixed 250 ms | + +`fixed-250` is the control: every other row adapts, so without it the whole matrix could pass while a +fixed preset regressed, and a fixed preset is what many viewers land on. Loss, reorder, and rate +limits stay off: the buffer's job is absorbing arrival spread, and congestion response would make a +failure hard to attribute. The shaper's uniform jitter does reorder datagrams whose spacing is +smaller than it, so `wide` measures some reordering along with the spread. The bursty and mid-run +step profiles need shaper features that are not here yet +([quest](../../quest/m0/audio-quality-harness/shaper-profiles.md)). + +The AAC arm keeps ffmpeg's default PES packing, whose multi-frame bursts are the flush-span shape +the reporter measured on the public relay (#3477). The source is a sine tone rather than a film, so a +quiet quantum at the output is always a gap or a stall and never the soundtrack. + +## The path + +```text + page ──UDP──▶ moq-shaper ──▶ moq-relay WebTransport, the impaired path + moq (opus, aac) ─────────────▶ moq-relay the publishers, on the clean path +``` + +Every port is reserved for the run ([the harness contract](../README.md)), and the shaper is the +same `moq-shaper` binary the [transport drills](../drill/README.md) run under, so the two share one +impairment implementation. It exits nonzero when a configured impairment never acted, and that +voids the row. + +The shaper carries UDP and nothing else. The page fetches `/certificate.sha256` from the address it +dials, so the driver answers that fetch with the relay's own hash (the certificate is pinned by +hash, so a different port needs nothing more). With nothing listening on TCP there, the WebSocket +fallback cannot connect: the session is WebTransport through the shaper, or the row is void. + +## The metric contract + +[`clients/js/src/schema.ts`](clients/js/src/schema.ts) is the contract. The page emits `Sample`s, +[`src/analyze.ts`](clients/js/src/analyze.ts) reduces a row to a `Summary`, and `grade.ts` reads that +against the budgets. A native lane or the latency ledger that emits the same `Summary` is comparable +by construction. + +**Units.** Every duration is milliseconds as a float, every count an integer, every share a fraction +of 1. Samples missing at the device rate are converted to ms before they reach a summary. + +**Clocks.** Every timestamp is taken on a named clock and reduced to `viewer`, the page's monotonic +`performance.now()`, before any two are subtracted: + +| Clock | What | Reduced to `viewer` by | +| --- | --- | --- | +| `viewer` | `performance.now()` | nothing: it is the reference | +| `render` | the `AudioContext` frame clock | offset and rate fitted from paired readings every 250 ms | +| `media` | the publisher's timestamps, through the stream | rate fitted; the offset is unknowable from the viewer | +| `publisher`, `relay` | other processes | an offset measured NTP style over the session, uncertain by half a round trip | + +Nothing in this lane carries a publisher or relay timestamp yet, so the stages that need one are +reported as unmeasured rather than guessed. A render clock more than 1% off the viewer's voids the +row: it is a throttled or stalled context, and everything counted on it would be wrong. + +**Aggregation.** A budget key is `_`: `total` over the graded window, +`per_min` (the total over the window's minutes), `p50`/`p95` (nearest rank over the window's +samples), `max`, `share` (of the window), or `last`. So "underruns: 3" and "underruns: 3/min" are +different keys and are never graded against each other. + +**Counters.** The player publishes no count of its own underruns, so the page fans a listening +AudioWorklet out from the player's output node. The player's render worklet copies what the ring +holds into the front of each 128-sample quantum and leaves the rest zero, so the tap reads every +quantum's fill level from the output itself: + +- `underruns`: quanta the ring filled only partly, or not at all, while playing. +- `short_quanta`: the underruns that were partly filled: the ring ran dry part-way through. +- `underrun_episodes`: maximal runs of consecutive underruns, one audible gap each. +- `underrun_ms`: the missing audio, summed, and the longest episode as `max`. +- `skip_aheads`: re-anchors that discarded buffered audio, seen as a forward step of more than 40 ms + in the lag between the viewer clock and the playhead (judged on the median of four samples either + side, after removing the media clock's drift). +- `discarded_ms`: the media those skip-aheads jumped over. Audio dropped before it reached the ring + moves no playhead, so it is not counted. +- `stalled`: the share of the window the ring spent stalled, refilling rather than playing. A gap + that starts within 60 ms of a stall is that stall, and is not an underrun. +- `silence`: the share of rendered quanta under -60 dBFS at the output, whatever the cause. +- `target_ms`: the resolved playout delay (`sync.out.delay`). +- `converge_ms`: from first audio to the last time the target moved more than one 20 ms bucket from + its final value. +- `render_load`: Chromium's render capacity, the share of each quantum's budget the graph used. + +Every row discards the first 5 s after first audio, which is the tune-in rather than the steady +state; `converge_ms` grades the tune-in instead. + +**Stages.** An end-to-end delay splits into exclusive spans, each defined by its two boundaries: +`capture`, `encode`, `publish_flush`, `network`, `jitter_buffer`, `decode`, `render`, `device`, and +the named remainder `unaccounted`. The identity is `sum == end-to-end` within the larger of 2 ms or +2%. This lane measures `jitter_buffer` (the resolved target) and `device` (`outputLatency` plus +`baseLatency`) on the viewer, and reports the publisher's declared `publish_flush` without summing it +with them. No clock spans the whole path here, so `end-to-end` is null rather than an identity that +holds by construction. + +## Void rules + +A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. + +| Void | Why | +| --- | --- | +| `shaper` | It exited nonzero (a configured impairment never acted), left no report, or forwarded nothing to the page. | +| `transport` | The session negotiated something other than WebTransport. | +| `ring` | `crossOriginIsolated` does not match the row's ring, so the other ring ran. | +| `clock` | The AudioContext clock was unreadable or ran more than 1% off the viewer's. | +| `window` | No audio after the warmup. | +| `driver` | The page never reached a session or first audio within 30 s, or the driver threw. | + +## Budgets + +`budgets.json` holds a ceiling per graded key per row, keyed by the whole row: runtime, codec, rate, +profile, and ring. Under `--enforce` a run fails on a breach, a budgeted key the run left +unmeasured, a void row, or a row with no budget. Tightening a ceiling is a visible diff; loosening +one needs a reason in review. The file's `note` says how the current ceilings were measured. + +The nightly `Audio quality` job runs the matrix under `--enforce` and keeps a failing run directory +for a week: each process's log, the shaper's counters, every row's raw samples and summary. + +## Layout + +```text +run.sh builds, starts the relay and publishers, runs each row behind the shaper +relay.toml anonymous relay, self-signed cert +budgets.json a ceiling per metric per row +clients/js/ + index.html src/page.ts the watch element, driven from the query string + src/probe.ts samples its public signals every 250 ms + src/tap.ts the listening worklet (tap-worklet.ts, quantum.ts) + src/schema.ts the metric contract + src/analyze.ts samples to a summary, unit tested + driver.ts one row in headless Chromium, and its void checks + analyze.ts grade.ts the summary per row, then the table and the verdict +``` + +## Attribution + +The harness is the reporter's, from #3477: fperex built it on their fork `fperex/moq` (branches +`debug/rt-audio` and `debug-findings-solution`), with the black-box probe, the analyzer's skip-ahead +rule, the Playwright matrix over `moq-shaper`, the budget file graded under `--enforce`, and the +nightly job. This lane upstreams it on its own, without the fork's player, estimator, or shaper +changes: the underrun count comes from a tap on the output rather than a counter in the player, and +the profiles are the ones the shaper on `main` can express. + +## Not covered + +Safari and Firefox, video, and perceptual scoring: the grade is glitches and latency, not an opinion +about how it sounds. Recorded arrival traces are +[their own quest](../../quest/m0/audio-quality-harness/traces.md). diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json new file mode 100644 index 0000000000..0cdde6b25c --- /dev/null +++ b/test/audio-quality/budgets.json @@ -0,0 +1,229 @@ +{ + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs.", + "rows": [ + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "fixed-250", + "ring": "isolated", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 440 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "fixed-250", + "ring": "plain", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 440 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "mild", + "ring": "isolated", + "underruns_per_min": 270, + "underrun_ms_per_min": 780, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 160, + "stalled_share": 0.01, + "silence_share": 0.012, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "mild", + "ring": "plain", + "underruns_per_min": 570, + "underrun_ms_per_min": 1700, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 120, + "stalled_share": 0.01, + "silence_share": 0.026, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "near-zero", + "ring": "isolated", + "underruns_per_min": 760, + "underrun_ms_per_min": 2200, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.028, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "near-zero", + "ring": "plain", + "underruns_per_min": 1500, + "underrun_ms_per_min": 4400, + "skip_aheads_per_min": 17, + "discarded_ms_per_min": 690, + "stalled_share": 0.01, + "silence_share": 0.066, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 430, + "underrun_ms_per_min": 1200, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 330, + "stalled_share": 0.01, + "silence_share": 0.018, + "target_ms_p95": 250 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "plain", + "underruns_per_min": 540, + "underrun_ms_per_min": 1500, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 260, + "stalled_share": 0.01, + "silence_share": 0.03, + "target_ms_p95": 250 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "fixed-250", + "ring": "isolated", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 300 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "fixed-250", + "ring": "plain", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 300 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "mild", + "ring": "isolated", + "underruns_per_min": 4600, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.19, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "mild", + "ring": "plain", + "underruns_per_min": 4800, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 130, + "stalled_share": 0.028, + "silence_share": 0.22, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "near-zero", + "ring": "isolated", + "underruns_per_min": 6100, + "underrun_ms_per_min": 16000, + "skip_aheads_per_min": 8.0, + "discarded_ms_per_min": 340, + "stalled_share": 0.034, + "silence_share": 0.25, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "near-zero", + "ring": "plain", + "underruns_per_min": 4600, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 290, + "stalled_share": 0.01, + "silence_share": 0.19, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 8100, + "underrun_ms_per_min": 22000, + "skip_aheads_per_min": 29, + "discarded_ms_per_min": 1700, + "stalled_share": 0.56, + "silence_share": 0.73, + "target_ms_p95": 82 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "wide", + "ring": "plain", + "underruns_per_min": 6700, + "underrun_ms_per_min": 17000, + "skip_aheads_per_min": 18, + "discarded_ms_per_min": 1300, + "stalled_share": 0.49, + "silence_share": 0.75, + "target_ms_p95": 82 + } + ] +} diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts new file mode 100644 index 0000000000..7a0eabaa1e --- /dev/null +++ b/test/audio-quality/clients/js/analyze.ts @@ -0,0 +1,116 @@ +/** + * Reduces one row of a run directory to `.summary.json`, and prints it. + * + * Reads what the row left: `.ndjson` and `.page.json` from the driver, and the shaper's log + * and exit status from run.sh. + * + * bun analyze.ts --run --row [--warmup 5] + * + * @module + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { analyze } from "./src/analyze.ts"; +import { + type Environment, + METRICS, + parseRow, + type Sample, + type Shaper, + type ShaperCounters, + STAGES, + type Void, +} from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + run: { type: "string" }, + row: { type: "string" }, + warmup: { type: "string", default: "5" }, + }, +}); +if (!values.run || !values.row) { + console.error("usage: analyze.ts --run --row [--warmup 5]"); + process.exit(2); +} +const run = values.run; +const tag = values.row; +const read = (file: string) => { + const path = join(run, file); + return existsSync(path) ? readFileSync(path, "utf8") : undefined; +}; + +const samples: Sample[] = (read(`${tag}.ndjson`) ?? "") + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as Sample); + +const page = JSON.parse(read(`${tag}.page.json`) ?? "null") as { + environment?: Environment; + notes: string[]; + voids: Void[]; +} | null; + +/** The shaper prints its seed at start and its counters at exit; run.sh records the exit status. */ +function shaperOf(): Shaper | null { + const log = read(`shaper-${tag}.log`); + const status = read(`shaper-${tag}.status`); + if (log === undefined || status === undefined) return null; + const counters = (direction: "up" | "down"): ShaperCounters | null => { + const match = log.match( + new RegExp( + `${direction}: (\\d+) packets, (\\d+) lost, (\\d+) overflowed, (\\d+) throttled, (\\d+) delayed, (\\d+) reordered`, + ), + ); + if (!match) return null; + const [packets, lost, overflowed, throttled, delayed, reordered] = match.slice(1).map(Number) as number[]; + return { packets, lost, overflowed, throttled, delayed, reordered } as ShaperCounters; + }; + const seed = log.match(/seed (\d+)/)?.[1]; + return { + seed: seed === undefined ? null : Number(seed), + status: Number.parseInt(status, 10), + up: counters("up"), + down: counters("down"), + }; +} + +const summary = analyze({ + row: parseRow(tag), + samples, + environment: page?.environment, + voids: page?.voids ?? [{ assertion: "driver", detail: "left no page report" }], + notes: page?.notes ?? [], + shaper: shaperOf(), + warmupMs: Number.parseFloat(values.warmup) * 1000, +}); +await Bun.write(join(run, `${tag}.summary.json`), JSON.stringify(summary, null, 1)); + +const lines = [`## ${tag}`, "", `window ${summary.windowMs} ms after a ${summary.warmupMs} ms warmup`, ""]; +if (summary.voids.length > 0) { + lines.push(`**void**: ${summary.voids.map((v) => `${v.assertion} (${v.detail})`).join("; ")}`, ""); +} +lines.push("| metric | unit | clock | aggregation | value |", "|---|---|---|---|---|"); +for (const [name, spec] of Object.entries(METRICS)) { + for (const aggregation of spec.aggregations) { + lines.push( + `| ${name} | ${spec.unit} | ${spec.clock} | ${aggregation} | ${summary.metrics[`${name}_${aggregation}`] ?? "n/a"} |`, + ); + } +} +lines.push( + "", + "| stage | ms | source |", + "|---|---|---|", + ...STAGES.map((s) => `| ${s} | ${summary.stages[s].ms ?? "n/a"} | ${summary.stages[s].source} |`), +); +const shaper = summary.shaper; +if (shaper) { + lines.push( + "", + `shaper seed ${shaper.seed} exit ${shaper.status}: up ${JSON.stringify(shaper.up)} down ${JSON.stringify(shaper.down)}`, + ); +} +if (summary.notes.length > 0) lines.push("", "notes:", ...summary.notes.slice(0, 20).map((n) => `- ${n}`)); +console.log(lines.join("\n")); diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts new file mode 100644 index 0000000000..3fec53e203 --- /dev/null +++ b/test/audio-quality/clients/js/driver.ts @@ -0,0 +1,160 @@ +/** + * Plays one matrix row in headless Chromium, and refuses a row whose numbers would mean something + * other than what its key says. + * + * The page dials the shaper, which forwards UDP and nothing else. So the page's fetch of the + * certificate hash is answered here with the relay's own (the certificate is pinned by hash, so + * dialing another port needs nothing more), and a WebSocket fallback finds nobody listening: the + * session is WebTransport through the shaper, or nothing. + * + * bun driver.ts --url http://127.0.0.1:4501 --fingerprint http://127.0.0.1:4500/certificate.sha256 \ + * --broadcast tone-opus.hang --page dist --ring plain --delay auto --duration 60 \ + * --tag chromium-opus-48000-mild-plain --out + * + * Writes `.ndjson` (the samples, appended as they are drained) and `.page.json` (the + * environment, console notes, and voids). Exits nonzero when the row could not be played at all. + * + * @module + */ +import { appendFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { launch, sleep } from "../../../interop/clients/js/harness.ts"; +import type { Environment, Ring, Void } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + url: { type: "string" }, + fingerprint: { type: "string" }, + broadcast: { type: "string" }, + page: { type: "string" }, + ring: { type: "string" }, + delay: { type: "string", default: "auto" }, + duration: { type: "string", default: "60" }, + tag: { type: "string" }, + out: { type: "string" }, + }, +}); + +const { url, fingerprint, broadcast, page: pageDir, tag, out } = values; +const durationMs = Number.parseFloat(values.duration) * 1000; +const ring = values.ring as Ring; +if ( + !url || + !fingerprint || + !broadcast || + !pageDir || + !tag || + !out || + !Number.isFinite(durationMs) || + durationMs <= 0 || + (ring !== "plain" && ring !== "isolated") +) { + console.error( + "usage: driver.ts --url U --fingerprint F --broadcast B --page DIR --ring plain|isolated --tag T --out DIR [--delay auto] [--duration S>0]", + ); + process.exit(2); +} + +/** How long the session and the first audio each get before the row is abandoned. */ +const STARTUP_MS = 30_000; +/** How often the probe is drained into the ndjson. */ +const DRAIN_MS = 1000; + +const hash = await fetch(fingerprint).then((r) => { + if (!r.ok) throw new Error(`${fingerprint}: ${r.status}`); + return r.text(); +}); + +// Cross-origin isolation is a property of the document, not the bundle, so one build served under +// two prefixes runs both rings. `/plain` is the production path: most viewers are not isolated, and +// get the postMessage ring. +const root = resolve(pageDir); +const isolation = { "cross-origin-opener-policy": "same-origin", "cross-origin-embedder-policy": "require-corp" }; +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const path = new URL(req.url).pathname; + const match = path.match(/^\/(isolated|plain)\/(.*)$/); + if (!match) return new Response("not found", { status: 404 }); + const file = Bun.file(join(root, match[2] || "index.html")); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { headers: match[1] === "isolated" ? isolation : undefined }); + }, +}); + +const query = new URLSearchParams({ url, broadcast, delay: values.delay }); +const pageUrl = `http://127.0.0.1:${server.port}/${ring}/?${query}`; +console.log(`page: ${pageUrl}`); + +const samplesFile = join(out, `${tag}.ndjson`); +const voids: Void[] = []; +const refuse = (assertion: string, detail: string) => { + console.error(`void: ${assertion}: ${detail}`); + voids.push({ assertion, detail }); +}; + +/** Poll the page until `ready` returns a value, or throw naming what never happened. */ +async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { + const deadline = Date.now() + STARTUP_MS; + while (Date.now() < deadline) { + const value = await page.evaluate(ready); + if (value !== undefined && value !== null) return value; + await sleep(200); + } + throw new Error(`${what} not within ${STARTUP_MS / 1000}s`); +} + +const drain = async (page: Page) => { + const samples = await page.evaluate(() => globalThis.audioQuality.drain()); + if (samples.length > 0) appendFileSync(samplesFile, `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); +}; + +// No fake devices: nothing here captures. The autoplay flag stands in for the click a viewer makes. +const browser = await launch(["--autoplay-policy=no-user-gesture-required"]); +let environment: Environment | undefined; +let notes: string[] = []; +let failed = false; +try { + const page = await browser.newPage(); + page.on("pageerror", (error) => console.error(`[page error] ${error.message}`)); + await page.route(new URL("/certificate.sha256", url).href, (route) => + route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), + ); + await page.goto(pageUrl, { waitUntil: "load" }); + + environment = await waitFor(page, "session", () => { + const env = globalThis.audioQuality?.environment(); + return env?.transport ? env : undefined; + }); + if (environment.transport !== "webtransport") { + refuse("transport", `negotiated ${environment.transport}, which never crosses the UDP shaper`); + } + if (environment.crossOriginIsolated !== (ring === "isolated")) { + refuse("ring", `asked for ${ring}, but crossOriginIsolated is ${environment.crossOriginIsolated}`); + } + + // The row's duration is audio played, not startup: the window starts once the ring does. + await waitFor(page, "first audio", () => globalThis.audioQuality.playing() || undefined); + + const end = Date.now() + durationMs; + while (Date.now() < end) { + await sleep(Math.min(DRAIN_MS, end - Date.now())); + await drain(page); + } + // The context rate is only known once the graph exists, which is after the catalog. + environment = (await page.evaluate(() => globalThis.audioQuality.environment())) ?? environment; + notes = await page.evaluate(() => globalThis.audioQuality.notes()); + await page.close(); +} catch (err) { + failed = true; + refuse("driver", err instanceof Error ? err.message : String(err)); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} + +await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids }, null, 1)); +process.exit(failed ? 1 : 0); diff --git a/test/audio-quality/clients/js/grade.ts b/test/audio-quality/clients/js/grade.ts new file mode 100644 index 0000000000..819b7be29f --- /dev/null +++ b/test/audio-quality/clients/js/grade.ts @@ -0,0 +1,102 @@ +/** + * Grades a run's summaries against `budgets.json`, prints the table, and writes `grade.md`. + * + * Every budget value is a ceiling on one `_` key. Under `--enforce` a run fails + * on any of: a value over its ceiling, a budgeted value the run never measured, a void row, or a row + * with no budget at all. Each of those would otherwise read as a pass while grading nothing. + * + * bun grade.ts --run --budgets [--enforce] + * + * @module + */ +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { type Budgets, METRIC_KEYS, type Row, rowKey, type Summary } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + run: { type: "string" }, + budgets: { type: "string" }, + enforce: { type: "boolean", default: false }, + }, +}); +if (!values.run || !values.budgets) { + console.error("usage: grade.ts --run --budgets [--enforce]"); + process.exit(2); +} +const run = values.run; + +const budgets = JSON.parse(readFileSync(values.budgets, "utf8")) as Budgets; +const summaries = readdirSync(run) + .filter((f) => f.endsWith(".summary.json")) + .sort() + .map((f) => JSON.parse(readFileSync(join(run, f), "utf8")) as Summary); +if (summaries.length === 0) { + console.error(`error: no summaries in ${run}`); + process.exit(1); +} + +/** A budget applies to exactly one matrix cell, matched on the whole row. */ +const budgetFor = (row: Row) => + budgets.rows.find( + (b) => + b.runtime === row.runtime && + b.codec === row.codec && + b.rate === row.rate && + b.profile === row.profile && + b.ring === row.ring, + ); + +const failures: string[] = []; +for (const summary of summaries) { + const key = rowKey(summary.row); + const budget = budgetFor(summary.row); + if (!budget) { + failures.push(`${key}: no budget`); + continue; + } + for (const v of summary.voids) failures.push(`${key}: void, ${v.assertion} (${v.detail})`); + if (summary.voids.length > 0) continue; + for (const metric of METRIC_KEYS) { + const ceiling = budget[metric]; + if (typeof ceiling !== "number") continue; + const value = summary.metrics[metric]; + if (value === null || value === undefined) failures.push(`${key}: ${metric} unmeasured, ceiling ${ceiling}`); + else if (value > ceiling) failures.push(`${key}: ${metric} ${value} > ${ceiling}`); + } +} + +/** The columns a reader scans first: what the listener would have noticed, and the delay paid for it. */ +const COLUMNS: [string, string][] = [ + ["underruns/min", "underruns_per_min"], + ["short/min", "short_quanta_per_min"], + ["episodes/min", "underrun_episodes_per_min"], + ["gap ms/min", "underrun_ms_per_min"], + ["skips/min", "skip_aheads_per_min"], + ["discarded ms/min", "discarded_ms_per_min"], + ["stalled", "stalled_share"], + ["silence", "silence_share"], + ["target p95", "target_ms_p95"], + ["converge ms", "converge_ms_last"], +]; + +const lines = [ + `| row | ${COLUMNS.map(([label]) => label).join(" | ")} | void |`, + `|---|${COLUMNS.map(() => "---|").join("")}---|`, + ...summaries.map( + (s) => + `| ${rowKey(s.row)} | ${COLUMNS.map(([, k]) => s.metrics[k] ?? "n/a").join(" | ")} | ${s.voids.map((v) => v.assertion).join(",") || "-"} |`, + ), + "", + ...(failures.length > 0 ? ["failures:", ...failures.map((f) => `- ${f}`)] : ["within budget"]), +]; +const report = lines.join("\n"); +console.log(report); +await Bun.write(join(run, "grade.md"), `${report}\n`); + +if (!values.enforce) { + if (failures.length > 0) console.log("\nreporting only: pass --enforce to fail on these"); + process.exit(0); +} +process.exit(failures.length > 0 ? 1 : 0); diff --git a/test/audio-quality/clients/js/index.html b/test/audio-quality/clients/js/index.html new file mode 100644 index 0000000000..8b151b202c --- /dev/null +++ b/test/audio-quality/clients/js/index.html @@ -0,0 +1,10 @@ + + + + + moq audio quality + + + + + diff --git a/test/audio-quality/clients/js/package.json b/test/audio-quality/clients/js/package.json new file mode 100644 index 0000000000..9534e140ed --- /dev/null +++ b/test/audio-quality/clients/js/package.json @@ -0,0 +1,18 @@ +{ + "name": "@moq/audio-quality", + "private": true, + "type": "module", + "scripts": { + "check": "tsc --noEmit", + "test": "bun test --only-failures" + }, + "dependencies": { + "@moq/watch": "workspace:*" + }, + "devDependencies": { + "@types/audioworklet": "^0.0.100", + "esbuild": "^0.28.2", + "playwright": "^1.63.0", + "vite": "^8.3.0" + } +} diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts new file mode 100644 index 0000000000..b6dba0ae50 --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test } from "bun:test"; +import { analyze, type Input, percentile, SKIP_MS, skipAheads } from "./analyze.ts"; +import type { Sample, Shaper } from "./schema.ts"; + +const row = { runtime: "chromium", codec: "opus", rate: 48000, profile: "mild", ring: "plain" } as const; +const counters = { packets: 100, lost: 0, overflowed: 0, throttled: 0, delayed: 100, reordered: 0 }; +const shaper: Shaper = { seed: 7, status: 0, up: counters, down: counters }; + +/** A clean run: 40 s of samples, the render clock 1000 ms behind the viewer's, the playhead 200 ms behind. */ +function clean(): Sample[] { + return Array.from({ length: 160 }, (_, i) => { + const at = 1000 + i * 250; + return { + at, + render: at - 1000, + timestamp: at - 200, + stalled: false, + delay: 120, + quanta: i * 94, + quiet: 0, + }; + }); +} + +const input = (samples: Sample[], extra: Partial = {}): Input => ({ + row, + samples, + environment: { crossOriginIsolated: false, transport: "webtransport" }, + voids: [], + notes: [], + shaper, + warmupMs: 5000, + ...extra, +}); + +describe("skipAheads", () => { + test("the quantization band is not a skip", () => { + const lags = Array.from({ length: 40 }, (_, i) => 200 + (i % 2 === 0 ? 25 : -25)); + expect(skipAheads(lags).count).toBe(0); + }); + + test("a step in the lag is one skip, of the step's size", () => { + const lags = Array.from({ length: 40 }, (_, i) => (i < 20 ? 300 : 200)); + expect(skipAheads(lags)).toEqual({ count: 1, ms: 100 }); + }); + + test("a step at the threshold is not one", () => { + const lags = Array.from({ length: 40 }, (_, i) => (i < 20 ? 200 + SKIP_MS : 200)); + expect(skipAheads(lags).count).toBe(0); + }); +}); + +test("percentile is nearest rank", () => { + expect(percentile([5, 1, 4, 2, 3], 50)).toBe(3); + expect(percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 95)).toBe(10); + expect(percentile([], 50)).toBeNull(); +}); + +describe("analyze", () => { + test("a clean run has nothing to report and nothing void", () => { + const summary = analyze(input(clean())); + expect(summary.voids).toEqual([]); + expect(summary.metrics).toMatchObject({ + underruns_total: 0, + skip_aheads_total: 0, + stalled_share: 0, + silence_share: 0, + target_ms_p95: 120, + converge_ms_last: 0, + }); + // Every key in the schema is present, measured or null. + expect(summary.metrics.render_load_max).toBeNull(); + expect(summary.drift.find((d) => d.clock === "render")).toEqual({ clock: "render", rate: 0, offset: 1000 }); + }); + + test("a gap on the render clock is counted where it lands on the viewer's", () => { + const samples = clean(); + // 20 s into the viewer clock is 19 s on the render clock. + samples[100] = { ...samples[100], gaps: [{ at: 19_000, ms: 8, quanta: 3, short: 1 }] } as Sample; + // Inside the warmup, so not graded. + samples[10] = { ...samples[10], gaps: [{ at: 1_000, ms: 4, quanta: 2, short: 1 }] } as Sample; + const summary = analyze(input(samples)); + expect(summary.metrics).toMatchObject({ + underruns_total: 3, + short_quanta_total: 1, + underrun_episodes_total: 1, + underrun_ms_total: 8, + underrun_ms_max: 8, + }); + }); + + test("a gap that is the ring stalling is not an underrun", () => { + const samples = clean(); + samples[100] = { + ...samples[100], + gaps: [{ at: 19_000, ms: 300, quanta: 110, short: 0 }], + stalls: [ + { at: 19_020, stalled: true }, + { at: 19_300, stalled: false }, + ], + } as Sample; + expect(analyze(input(samples)).metrics.underruns_total).toBe(0); + }); + + test("a re-anchor that discards audio is a skip-ahead", () => { + const samples = clean().map((s, i) => (i >= 80 ? { ...s, timestamp: s.at - 100 } : s)); + expect(analyze(input(samples)).metrics).toMatchObject({ skip_aheads_total: 1, discarded_ms_total: 100 }); + }); + + test("a shaper that failed or never carried the page voids the row", () => { + const failed = analyze(input(clean(), { shaper: { ...shaper, status: 1 } })); + expect(failed.voids.map((v) => v.assertion)).toEqual(["shaper"]); + const idle = analyze(input(clean(), { shaper: { ...shaper, down: { ...counters, packets: 0 } } })); + expect(idle.voids.map((v) => v.assertion)).toEqual(["shaper"]); + }); + + test("a render clock that is not running at wall rate voids the row", () => { + const samples = clean().map((s) => ({ ...s, render: (s.at - 1000) * 0.5 })); + expect(analyze(input(samples)).voids.map((v) => v.assertion)).toEqual(["clock"]); + }); + + test("no audio at all voids the window", () => { + const samples = clean().map((s) => ({ ...s, timestamp: undefined, stalled: true })); + expect(analyze(input(samples)).voids.map((v) => v.assertion)).toContain("window"); + }); + + test("convergence is the last move, not the first plateau", () => { + const samples = clean().map((s, i) => ({ ...s, delay: i < 40 ? 100 : i < 60 ? 300 : 100 })); + // First audio at sample 0, the target back on its final value from sample 60. + expect(analyze(input(samples)).metrics.converge_ms_last).toBe(60 * 250); + }); +}); diff --git a/test/audio-quality/clients/js/src/analyze.ts b/test/audio-quality/clients/js/src/analyze.ts new file mode 100644 index 0000000000..9ee7d6c62c --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.ts @@ -0,0 +1,305 @@ +/** + * Reduces one row's samples to the {@link Summary} the grader reads. + * + * Pure, so every derivation below is unit tested against synthetic samples rather than trusted. + * `../analyze.ts` is the command that feeds it a run directory. + * + * Adapted from `debug-findings/analysis/analyze.mjs` on the reporter's fork (`fperex/moq`, branch + * `debug/rt-audio`), whose skip-ahead rule and per-minute normalisation this keeps. The event stream + * it read came from probes patched into the player; this reads public signals and the output tap. + * + * @module + */ +import { + BUCKET_MS, + type Drift, + type Environment, + type Gap, + MAX_RENDER_DRIFT, + METRIC_KEYS, + type Ms, + type Row, + type Sample, + type Shaper, + STAGES, + type Stage, + type StageSpan, + type Summary, + type Void, +} from "./schema.ts"; + +/** Everything a row left behind. */ +export type Input = { + row: Row; + samples: Sample[]; + environment: Environment | undefined; + voids: Void[]; + notes: string[]; + shaper: Shaper | null; + /** Discarded after first audio: every profile spends its first seconds converging on purpose. */ + warmupMs: Ms; +}; + +/** + * A gap that starts this close to a stall is the stall, not an underrun. + * + * The page learns of a stall after the ring does: up to five quanta late on the postMessage ring, + * one poll late on the isolated one. + */ +export const STALL_SLACK_MS = 60; + +/** A step in the de-trended playhead lag larger than this is a skip-ahead. See {@link skipAheads}. */ +export const SKIP_MS = 40; + +/** Samples either side of a candidate skip reduced to a median before it is judged. */ +export const SKIP_WINDOW = 4; + +const median = (values: number[]): number | undefined => { + if (values.length === 0) return undefined; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +}; + +/** Nearest-rank percentile. Null for an empty series. */ +export function percentile(values: number[], p: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)] ?? null; +} + +/** Least-squares slope of `ys` over `xs`. Null with nothing to fit. */ +export function slope(xs: number[], ys: number[]): number | null { + const n = xs.length; + if (n < 2) return null; + const mx = xs.reduce((a, b) => a + b, 0) / n; + const my = ys.reduce((a, b) => a + b, 0) / n; + let num = 0; + let den = 0; + for (let i = 0; i < n; i++) { + num += ((xs[i] ?? 0) - mx) * ((ys[i] ?? 0) - my); + den += ((xs[i] ?? 0) - mx) ** 2; + } + return den === 0 ? null : num / den; +} + +/** + * Forward steps in the lag between the viewer clock and the playhead, and the media they skipped. + * + * `lags` is the de-trended `at - timestamp` per sample, undefined where there was no playhead or the + * ring was stalled. The playhead a page reads is quantized (it moves 240 to 296 ms per 250 ms sample + * on a clean path), so one sample's excess is noise: a skip is a step between the medians either side + * of a sample, larger than {@link SKIP_MS}, which is more than that band and more than one estimator + * bucket plus a render quantum. + */ +export function skipAheads(lags: (Ms | undefined)[]): { count: number; ms: Ms } { + let count = 0; + let ms = 0; + const defined = (slice: (Ms | undefined)[]) => slice.filter((l): l is Ms => l !== undefined); + for (let i = SKIP_WINDOW; i < lags.length - SKIP_WINDOW; i++) { + if (lags[i] === undefined) continue; + const before = median(defined(lags.slice(i - SKIP_WINDOW, i))); + const after = median(defined(lags.slice(i + 1, i + 1 + SKIP_WINDOW))); + if (before === undefined || after === undefined) continue; + const jumped = before - after; + if (jumped > SKIP_MS) { + count++; + ms += jumped; + // One step is one skip, and the trailing window straddles it for several samples. + i += SKIP_WINDOW; + } + } + return { count, ms }; +} + +/** + * Spans in which the ring was stalled, on the render clock. + * + * The ring starts stalled, before the page has a context to stamp that on, so a first change to + * unstalled closes a span that opened before anything was recorded. + */ +export function stallSpans(samples: Sample[]): { start: Ms; end: Ms }[] { + const changes = samples.flatMap((s) => s.stalls ?? []); + const spans: { start: Ms; end: Ms }[] = []; + let open: Ms | undefined = changes[0]?.stalled === false ? Number.NEGATIVE_INFINITY : undefined; + for (const change of changes) { + if (change.stalled && open === undefined) open = change.at; + else if (!change.stalled && open !== undefined) { + spans.push({ start: open, end: change.at }); + open = undefined; + } + } + if (open !== undefined) spans.push({ start: open, end: Number.POSITIVE_INFINITY }); + return spans; +} + +const round = (x: number | null | undefined, places = 1): number | null => + x === null || x === undefined || !Number.isFinite(x) ? null : Math.round(x * 10 ** places) / 10 ** places; + +/** Reduce one row. */ +export function analyze(input: Input): Summary { + const { row, samples, environment, warmupMs } = input; + const voids = [...input.voids]; + const notes = [...input.notes]; + + // ── the shaper's verdict ──────────────────────────────────────────────── + const shaper = input.shaper; + if (!shaper) { + voids.push({ assertion: "shaper", detail: "left no report, so nothing says the path was shaped" }); + } else if (shaper.status !== 0) { + voids.push({ assertion: "shaper", detail: `exited ${shaper.status}: the profile never acted as configured` }); + } else if ((shaper.down?.packets ?? 0) === 0) { + voids.push({ assertion: "shaper", detail: "forwarded nothing to the page, so the session never crossed it" }); + } + + // ── the window ────────────────────────────────────────────────────────── + const first = samples.find((s) => s.timestamp !== undefined && !s.stalled); + const t0 = (first?.at ?? 0) + warmupMs; + const t1 = samples.at(-1)?.at ?? t0; + const window = samples.filter((s) => s.at >= t0 && s.at <= t1); + const minutes = Math.max(1e-9, (t1 - t0) / 60_000); + if (!first || window.length < 2) { + voids.push({ assertion: "window", detail: `no audio after the warmup (${samples.length} samples in all)` }); + } + + // ── the render clock, reduced to the viewer's ─────────────────────────── + // Each sample reads both clocks back to back, so the pairs fit the render clock's rate and offset. + const paired = samples.filter((s) => s.render !== undefined); + const renderRate = slope( + paired.map((s) => s.at), + paired.map((s) => s.render as Ms), + ); + const renderOffset = median(paired.map((s) => s.at - (s.render as Ms))) ?? null; + if (renderRate === null || renderOffset === null) { + voids.push({ assertion: "clock", detail: "the AudioContext clock was never readable" }); + } else if (Math.abs(renderRate - 1) > MAX_RENDER_DRIFT) { + voids.push({ + assertion: "clock", + detail: `the AudioContext clock ran at ${round(renderRate, 3)}x the viewer's: throttled or stalled`, + }); + } + const toViewer = (render: Ms) => render + (renderOffset ?? 0); + + // ── underruns, from the tap ───────────────────────────────────────────── + const stalls = stallSpans(samples); + const chosen = (gap: Gap) => + stalls.some((s) => gap.at >= s.start - STALL_SLACK_MS && gap.at <= s.end + STALL_SLACK_MS); + const gaps = samples + .flatMap((s) => s.gaps ?? []) + .filter((g) => toViewer(g.at) >= t0 && toViewer(g.at) <= t1 && !chosen(g)); + const underruns = gaps.reduce((a, g) => a + g.quanta, 0); + const short = gaps.reduce((a, g) => a + g.short, 0); + const underrunMs = gaps.reduce((a, g) => a + g.ms, 0); + + const tapped = window.filter((s) => s.quanta !== undefined && s.quiet !== undefined); + const firstTap = tapped[0]; + const lastTap = tapped.at(-1); + const quanta = (lastTap?.quanta ?? 0) - (firstTap?.quanta ?? 0); + const silence = quanta > 0 ? ((lastTap?.quiet ?? 0) - (firstTap?.quiet ?? 0)) / quanta : null; + if (!lastTap) notes.push("tap: never reported, so underruns and silence are unmeasured"); + + // ── the playhead ──────────────────────────────────────────────────────── + // The lag drifts when the media timeline does not run at wall rate, a property of the source + // rather than the player. Fitted and removed first, or it reads as a skip every few seconds. + // A least-squares fit would read a skip itself as drift and hide part of it, so the rate is the + // median slope across spans of two skip windows, which a step contaminates only a few of. + const playing = window.filter((s) => s.timestamp !== undefined && !s.stalled); + const reach = 2 * SKIP_WINDOW; + const mediaRate = + median( + playing.slice(reach).map((s, i) => { + const from = playing[i] as Sample; + return (s.at - (s.timestamp as Ms) - (from.at - (from.timestamp as Ms))) / (s.at - from.at); + }), + ) ?? null; + const base = playing[0]?.at ?? 0; + const lags = window.map((s) => + s.timestamp === undefined || s.stalled ? undefined : s.at - s.timestamp - (mediaRate ?? 0) * (s.at - base), + ); + const skips = skipAheads(lags); + const stalledShare = window.length > 0 ? window.filter((s) => s.stalled).length / window.length : null; + + // ── the target ────────────────────────────────────────────────────────── + const targets = window.flatMap((s) => (s.delay === undefined ? [] : [s.delay])); + // Measured backwards from the end: a target that settles and moves again has not converged. + const series = samples.filter((s) => first && s.at >= first.at && s.delay !== undefined); + const final = series.at(-1)?.delay; + let settled = series[0]; + for (let i = series.length - 1; i >= 0; i--) { + if (Math.abs((series[i]?.delay ?? 0) - (final ?? 0)) > BUCKET_MS) { + settled = series[i + 1]; + break; + } + } + const converge = first && settled ? settled.at - first.at : null; + + const loads = window.flatMap((s) => (s.renderLoad === undefined ? [] : [s.renderLoad])); + + const metrics: Record = Object.fromEntries(METRIC_KEYS.map((k) => [k, null])); + const perMin = (total: number) => round(total / minutes); + Object.assign(metrics, { + underruns_total: lastTap ? underruns : null, + underruns_per_min: lastTap ? perMin(underruns) : null, + short_quanta_total: lastTap ? short : null, + short_quanta_per_min: lastTap ? perMin(short) : null, + underrun_episodes_total: lastTap ? gaps.length : null, + underrun_episodes_per_min: lastTap ? perMin(gaps.length) : null, + underrun_ms_total: lastTap ? round(underrunMs) : null, + underrun_ms_per_min: lastTap ? perMin(underrunMs) : null, + underrun_ms_max: lastTap ? round(Math.max(0, ...gaps.map((g) => g.ms))) : null, + skip_aheads_total: skips.count, + skip_aheads_per_min: perMin(skips.count), + discarded_ms_total: round(skips.ms), + discarded_ms_per_min: perMin(skips.ms), + stalled_share: round(stalledShare, 3), + silence_share: round(silence, 3), + target_ms_p50: round(percentile(targets, 50)), + target_ms_p95: round(percentile(targets, 95)), + target_ms_max: round(percentile(targets, 100)), + converge_ms_last: round(converge), + render_load_max: round(percentile(loads, 100), 3), + }); + + // ── stages ────────────────────────────────────────────────────────────── + // Only the viewer's clocks are readable here, so only the receiver's stages are measured, and no + // end-to-end exists to sum them to. `publish_flush` is what the publisher declares, reported but + // never summed with viewer-clock spans. + const last = window.at(-1); + const span = (ms: Ms | undefined, source: StageSpan["source"]): StageSpan => + ms === undefined ? { ms: null, source: "unmeasured" } : { ms: round(ms), source }; + const known: Partial> = { + publish_flush: span(environment?.jitter, "declared"), + jitter_buffer: span(last?.delay, "measured"), + device: + last?.outputLatency === undefined + ? span(undefined, "unmeasured") + : span(last.outputLatency + (last.baseLatency ?? 0), "measured"), + }; + const stages = Object.fromEntries(STAGES.map((s) => [s, known[s] ?? span(undefined, "unmeasured")])) as Record< + Stage, + StageSpan + >; + + const drift: Drift[] = [ + { clock: "viewer", rate: 0, offset: 0 }, + { clock: "render", rate: round(renderRate === null ? null : renderRate - 1, 6), offset: round(renderOffset) }, + // The lag's slope is the media clock's rate against the viewer's, negated. + { clock: "media", rate: round(mediaRate === null ? null : -mediaRate, 6), offset: null }, + { clock: "publisher", rate: null, offset: null }, + { clock: "relay", rate: null, offset: null }, + ]; + + return { + version: 1, + row, + windowMs: round(t1 - t0) ?? 0, + warmupMs, + environment: environment ?? null, + voids, + metrics, + stages, + endToEnd: null, + drift, + shaper, + notes: notes.slice(0, 50), + }; +} diff --git a/test/audio-quality/clients/js/src/page.ts b/test/audio-quality/clients/js/src/page.ts new file mode 100644 index 0000000000..ee742da8cc --- /dev/null +++ b/test/audio-quality/clients/js/src/page.ts @@ -0,0 +1,43 @@ +/** + * The page under measurement: the demo's player and nothing else. + * + * The element and its registration imports are the ones `demo/web` uses, so a regression in the + * element's attribute handling or the worklet reaches these numbers the way it reaches a viewer. + * Everything comes from the query string, so one build serves every row: + * + * ?url=http://127.0.0.1:4499&broadcast=tone-opus.hang&delay=auto + * + * The driver drains the probe through `globalThis.audioQuality`. + * + * @module + */ +import "@moq/watch/element"; +import type MoqWatch from "@moq/watch/element"; +import { type Probe, probe } from "./probe.ts"; + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const watch = document.createElement("moq-watch") as MoqWatch; +watch.setAttribute("url", required("url")); +watch.setAttribute("name", required("broadcast")); +// "auto" adapts, and is what every profile but the control runs. A fixed delay carries its unit: +// the element rejects `delay="250"`. +watch.setAttribute("delay", params.get("delay") ?? "auto"); +watch.appendChild(document.createElement("canvas")); +document.body.appendChild(watch); + +// Audio is the measurement: muted, nothing would reach the ring. Volume first, because un-muting +// restores the stashed volume. +watch.volume = 1; +watch.muted = false; +watch.paused = false; + +declare global { + var audioQuality: Probe; +} +globalThis.audioQuality = probe(watch); diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts new file mode 100644 index 0000000000..2245170d70 --- /dev/null +++ b/test/audio-quality/clients/js/src/probe.ts @@ -0,0 +1,139 @@ +/** + * Samples a `` every 250 ms, from its public signals and a tap on its output. + * + * Nothing here patches the player or reaches past its `out` surface, so a number it reports is one + * any consumer could read. What the player does not publish (how full each render quantum was) is + * read off its output by the tap in `tap.ts`, every quantum rather than a sample of them. + * + * Adapted from the black-box probe in `debug-findings/analysis/blackbox.js` on the reporter's fork + * (`fperex/moq`, branch `debug/rt-audio`). See ../../../README.md. + * + * @module + */ +import type MoqWatch from "@moq/watch/element"; +import { type Environment, SAMPLE_INTERVAL_MS, type Sample, SILENCE_RMS, type Stall } from "./schema.ts"; +import { type Tap, tap } from "./tap.ts"; + +/** Chromium's render capacity surface, which is not in lib.dom. */ +type RenderCapacity = { + start(options: { updateInterval: number }): void; + addEventListener(type: "update", listener: (event: { averageLoad: number }) => void): void; +}; + +/** What the probe has collected, until someone drains it. */ +export type Probe = { + /** Samples taken since the last drain. */ + drain(): Sample[]; + /** What the page is, once the catalog says the session is up. */ + environment(): Environment | undefined; + /** Console warnings and errors so far. */ + notes(): string[]; + /** Whether the ring has a playhead and is not stalled. */ + playing(): boolean; +}; + +/** Start sampling `watch`, from before it has produced anything, so startup is on the record too. */ +export function probe(watch: MoqWatch): Probe { + const samples: Sample[] = []; + const notes: string[] = []; + const note = (line: string) => { + if (notes.length < 200) notes.push(line.slice(0, 200)); + }; + + const { audio, sync, broadcast } = watch.player; + + // Stall changes are taken as they happen: a re-stall shorter than the sample grid is exactly the + // one a 250 ms read would miss, and it is what tells a stall from an underrun. + let stalls: Stall[] = []; + audio.out.stalled.subscribe((stalled) => { + const context = audio.out.context.peek(); + if (context) stalls.push({ at: context.currentTime * 1000, stalled }); + }); + + // The tap follows the output node, which the player rebuilds with its graph. A tap left on the old + // node would report a perfect run as one long silence. + let tapped: AudioNode | undefined; + let current: Tap | undefined; + let renderLoad: number | undefined; + const attach = () => { + const root = audio.out.root.peek(); + if (!root || root === tapped) return; + tapped = root; + current?.close(); + current = undefined; + + tap(root, { floor: SILENCE_RMS }).then( + (t) => { + if (tapped === root) current = t; + else t.close(); + }, + (err: unknown) => note(`tap: ${err instanceof Error ? err.message : String(err)}`), + ); + + const capacity = (root.context as unknown as { renderCapacity?: RenderCapacity }).renderCapacity; + if (!capacity) { + note("renderCapacity: unavailable, render_load will be null"); + return; + } + capacity.addEventListener("update", (event) => { + renderLoad = event.averageLoad; + }); + // A whole second: shorter intervals are refused by some builds, and load is smooth anyway. + capacity.start({ updateInterval: 1 }); + }; + + const sample = (): Sample => { + attach(); + const context = audio.out.context.peek(); + const counts = current?.counts(); + const gaps = current?.take() ?? []; + const taken = stalls; + stalls = []; + return { + // Read back to back, so the pair calibrates the render clock against the viewer's. + at: performance.now(), + render: context ? context.currentTime * 1000 : undefined, + timestamp: audio.out.timestamp.peek(), + stalled: audio.out.stalled.peek(), + delay: sync.out.delay.peek(), + outputLatency: context ? context.outputLatency * 1000 : undefined, + baseLatency: context ? context.baseLatency * 1000 : undefined, + renderLoad, + quanta: counts?.quanta, + quiet: counts?.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: taken.length > 0 ? taken : undefined, + }; + }; + + // Console output is evidence: a decoder reset shows up here before it shows up in a count. + for (const level of ["warn", "error"] as const) { + const original = console[level]; + console[level] = (...args: unknown[]) => { + note(`${level}: ${args.map(String).join(" ")}`); + original(...args); + }; + } + + setInterval(() => samples.push(sample()), SAMPLE_INTERVAL_MS); + + return { + drain: () => samples.splice(0, samples.length), + environment() { + const catalog = broadcast.out.catalog.peek(); + if (!catalog) return undefined; + // One rendition is published per broadcast here, so the first is the one playing. + const config = Object.values(catalog.audio?.renditions ?? {})[0]; + return { + crossOriginIsolated: globalThis.crossOriginIsolated === true, + transport: watch.connection.transport.peek(), + codec: config?.codec, + rate: config?.sampleRate, + jitter: config?.jitter, + contextRate: audio.out.context.peek()?.sampleRate, + }; + }, + notes: () => notes.slice(), + playing: () => audio.out.timestamp.peek() !== undefined && !audio.out.stalled.peek(), + }; +} diff --git a/test/audio-quality/clients/js/src/quantum.test.ts b/test/audio-quality/clients/js/src/quantum.test.ts new file mode 100644 index 0000000000..adf9d50d07 --- /dev/null +++ b/test/audio-quality/clients/js/src/quantum.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { classify, Ledger } from "./quantum.ts"; + +const quantum = (values: number[], channels = 2) => Array.from({ length: channels }, () => Float32Array.from(values)); +const tone = (length: number) => Array.from({ length }, (_, i) => 0.5 * Math.sin(i / 3 + 0.1)); + +describe("classify", () => { + test("a quantum the ring filled is full", () => { + expect(classify(quantum(tone(128))).fill).toBe("full"); + }); + + test("trailing zeros in every channel are the samples the ring did not have", () => { + const values = [...tone(100), ...new Array(28).fill(0)]; + expect(classify(quantum(values))).toMatchObject({ fill: "short", missing: 28 }); + }); + + test("one channel still playing means the ring supplied that sample", () => { + const left = Float32Array.from([...tone(100), ...new Array(28).fill(0)]); + const right = Float32Array.from(tone(128)); + expect(classify([left, right])).toMatchObject({ fill: "full", missing: 0 }); + }); + + test("all zeros is silent, however it came about", () => { + expect(classify(quantum(new Array(128).fill(0)))).toMatchObject({ fill: "silent", missing: 128 }); + expect(classify([])).toMatchObject({ fill: "silent", missing: 0 }); + }); +}); + +describe("Ledger", () => { + const RATE = 48000; + const full = classify(quantum(tone(128))); + const short = classify(quantum([...tone(64), ...new Array(64).fill(0)])); + const silent = classify(quantum(new Array(128).fill(0))); + + test("the tune-in before the first audible quantum is not a gap", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, silent); + ledger.add(128, 128, silent); + ledger.add(256, 128, full); + expect(ledger.counts).toEqual({ quanta: 1, quiet: 0 }); + expect(ledger.take()).toEqual([]); + }); + + test("consecutive unfilled quanta are one gap, closed by the next full one", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, full); + ledger.add(128, 128, short); + ledger.add(256, 128, silent); + ledger.add(384, 128, short); + expect(ledger.take()).toEqual([]); + + ledger.add(512, 128, full); + const [gap, ...rest] = ledger.take(); + expect(rest).toEqual([]); + // Starts at the first missing sample, half-way through the first short quantum. + expect(gap?.at).toBeCloseTo((192 / RATE) * 1000); + expect(gap?.ms).toBeCloseTo(((64 + 128 + 64) / RATE) * 1000); + expect(gap).toMatchObject({ quanta: 3, short: 2 }); + expect(ledger.counts).toEqual({ quanta: 5, quiet: 1 }); + }); +}); diff --git a/test/audio-quality/clients/js/src/quantum.ts b/test/audio-quality/clients/js/src/quantum.ts new file mode 100644 index 0000000000..e28ef166a3 --- /dev/null +++ b/test/audio-quality/clients/js/src/quantum.ts @@ -0,0 +1,112 @@ +/** + * What one render quantum at the player's output says about the ring behind it. + * + * The player's render worklet copies what its ring holds into the front of each output quantum and + * leaves the rest as the zeros Web Audio hands it. So a ring that ran dry shows up at the output as + * exact zeros at the end of a quantum across every channel, and an empty or stalled ring as a quantum + * of nothing but zeros. The published tone never lands on exact zero in every channel at once, which + * is what makes the output readable as the ring's fill level without a counter in the player. + * + * Free of worklet globals so the classification is unit tested rather than trusted. + * + * @module + */ +import type { Gap } from "./schema.ts"; + +/** How the ring filled one quantum. */ +export type Fill = "full" | "short" | "silent"; + +/** One quantum, classified. */ +export type Quantum = { + fill: Fill; + /** Samples per channel the ring did not supply: the trailing zeros, or the whole quantum. */ + missing: number; + /** Mean square over every sample of every channel. */ + power: number; +}; + +/** Classify one quantum of planar audio. No channels at all is a quantum that played nothing. */ +export function classify(channels: Float32Array[]): Quantum { + const length = channels[0]?.length ?? 0; + if (length === 0) return { fill: "silent", missing: 0, power: 0 }; + + let last = -1; + let sum = 0; + for (const channel of channels) { + for (let i = 0; i < channel.length; i++) { + const value = channel[i] ?? 0; + if (value !== 0 && i > last) last = i; + sum += value * value; + } + } + const power = sum / (length * channels.length); + if (last < 0) return { fill: "silent", missing: length, power }; + + const missing = length - 1 - last; + return { fill: missing > 0 ? "short" : "full", missing, power }; +} + +/** Cumulative counts since the first audible quantum. */ +export type Counts = { + /** Quanta rendered. */ + quanta: number; + /** Quanta under the silence floor, whatever the reason. */ + quiet: number; +}; + +/** + * Folds quanta into {@link Counts}, and closes a {@link Gap} each time the ring fills a quantum again. + * + * Nothing before the first non-silent quantum counts: the ring is filling for the first time, which + * is the tune-in, not a gap. + */ +export class Ledger { + readonly counts: Counts = { quanta: 0, quiet: 0 }; + readonly #rate: number; + readonly #floor: number; + #started = false; + #open: { frame: number; missing: number; quanta: number; short: number } | undefined; + #closed: Gap[] = []; + + /** `rate` is the context's sample rate; `floor` is the silence floor as an RMS level. */ + constructor(rate: number, floor: number) { + this.#rate = rate; + this.#floor = floor; + } + + /** Account for one quantum of `length` samples that started at `frame` on the render clock. */ + add(frame: number, length: number, quantum: Quantum): void { + if (!this.#started) { + if (quantum.fill === "silent") return; + this.#started = true; + } + + this.counts.quanta++; + if (Math.sqrt(quantum.power) < this.#floor) this.counts.quiet++; + if (quantum.fill === "full") { + this.#close(); + return; + } + + // A short quantum inside an open gap is a ring that got a little and ran out again: still one + // gap, since only a full quantum ends it. + this.#open ??= { frame: frame + length - quantum.missing, missing: 0, quanta: 0, short: 0 }; + const open = this.#open; + open.missing += quantum.missing; + open.quanta++; + if (quantum.fill === "short") open.short++; + } + + /** Gaps closed since the last call. A gap still open is reported once a full quantum ends it. */ + take(): Gap[] { + return this.#closed.splice(0, this.#closed.length); + } + + #close(): void { + if (!this.#open) return; + const ms = (samples: number) => (samples / this.#rate) * 1000; + const { frame, missing, quanta, short } = this.#open; + this.#closed.push({ at: ms(frame), ms: ms(missing), quanta, short }); + this.#open = undefined; + } +} diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts new file mode 100644 index 0000000000..3e36798d1b --- /dev/null +++ b/test/audio-quality/clients/js/src/schema.ts @@ -0,0 +1,407 @@ +/** + * The metric contract: what the page samples, what a row reduces to, and what a budget grades. + * + * This file is the schema, not a description of one. The page emits {@link Sample}s, `analyze.ts` + * reduces them to a {@link Summary}, and `grade.ts` reads that against `budgets.json`. {@link METRICS} + * says for every graded number what it counts, which clock it sits on, and how a series became one + * value. Another lane (native playout, the latency ledger) that emits the same {@link Summary} is + * comparable to this one by construction. + * + * Conventions, stated once for everything below: + * + * - Every duration is milliseconds as a float. A sample count is converted at the rate it was + * counted at before it reaches a {@link Summary}. + * - Every count is a non-negative integer. Every share is a fraction of 1. + * - Every timestamp sits on a named {@link Clock} and is reduced to `viewer` before any two are + * subtracted. {@link Drift} records what each reduction took. + * + * @module + */ + +/** Milliseconds, as a float. The unit of every duration in this schema. */ +export type Ms = number; + +/** + * The clocks a timestamp can be taken on. + * + * - `viewer`: the page's `performance.now()`, monotonic. The reference every other clock is reduced + * to, so a stage span is always a difference of two `viewer` values. + * - `render`: the page's `AudioContext` frame clock, the audio device's oscillator. Same host, so it + * is reduced by an offset and a drift fitted over the run from paired readings (see + * {@link MAX_RENDER_DRIFT}). + * - `media`: the publisher's presentation timestamps, seen through the stream. Not a clock the viewer + * can read directly: its rate against `viewer` is fitted, its offset is unknowable from here. + * - `publisher`, `relay`: other processes, often other machines. A timestamp taken there is reduced + * to `viewer` by an offset measured over the same session, NTP style: the offset at the midpoint of + * a round trip, with half the round trip as its uncertainty. A stage whose uncertainty exceeds + * {@link identityTolerance} is reported unmeasured rather than guessed. The browser lane carries no + * such timestamp yet, so every stage that needs one is null here. + */ +export type Clock = "viewer" | "render" | "media" | "publisher" | "relay"; + +/** + * How a series became the one number a budget grades. + * + * - `total`: summed over the graded window. + * - `per_min`: `total` divided by the graded window's length in minutes. + * - `p50`, `p95`: nearest-rank percentile of the window's samples. + * - `max`: the largest value in the window. + * - `share`: the fraction of the window for which the condition held. + * - `last`: the value at the end of the window. + */ +export type Aggregation = "total" | "per_min" | "p50" | "p95" | "max" | "share" | "last"; + +/** What one metric is: enough to read a number off a summary without guessing. */ +export type MetricSpec = { + /** What the raw measurement is counted in, before aggregation. */ + unit: "ms" | "count" | "share"; + /** The clock its timestamps were taken on, before reduction to `viewer`. */ + clock: Clock; + /** The aggregations reported for it, and therefore the budget keys that can grade it. */ + aggregations: Aggregation[]; + /** One plain line saying what it measures. */ + description: string; +}; + +/** + * Every graded metric, keyed by its base name. + * + * A budget key is `_`: `underruns` graded per minute is `underruns_per_min`. Naming + * the aggregation in the key is what keeps "underruns: 3" and "underruns: 3/min" from being graded + * against each other. + */ +export const METRICS = { + underruns: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: + "Render quanta the ring could only partly fill, or not fill at all, while it was playing. A quantum rendered while the ring is stalled is not one.", + }, + short_quanta: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: "Of the underruns, the quanta the ring filled the front of and then ran dry part-way through.", + }, + underrun_episodes: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: "Maximal runs of consecutive underruns: one audible gap each, however many quanta it spanned.", + }, + underrun_ms: { + unit: "ms", + clock: "render", + aggregations: ["total", "per_min", "max"], + description: + "Audio the ring failed to supply while playing: the missing samples at the device rate. `max` is the longest episode.", + }, + skip_aheads: { + unit: "count", + clock: "viewer", + aggregations: ["total", "per_min"], + description: + "Re-anchors that discarded buffered audio: a forward step in the lag between the viewer clock and the playhead.", + }, + discarded_ms: { + unit: "ms", + clock: "media", + aggregations: ["total", "per_min"], + description: + "Media time the skip-aheads jumped over without playing. Audio dropped before it reached the ring moves no playhead and is not counted.", + }, + stalled: { + unit: "share", + clock: "viewer", + aggregations: ["share"], + description: + "Share of the window the ring spent stalled, refilling rather than playing: silence the player chose.", + }, + silence: { + unit: "share", + clock: "render", + aggregations: ["share"], + description: + "Share of rendered quanta under the silence floor at the player's output, whatever the cause. The published tone is never quiet, so any of it is a gap or a stall.", + }, + target_ms: { + unit: "ms", + clock: "viewer", + aggregations: ["p50", "p95", "max"], + description: "The playout delay the receiver resolved (`sync.out.delay`), sampled through the window.", + }, + converge_ms: { + unit: "ms", + clock: "viewer", + aggregations: ["last"], + description: + "From first audio to the last time the resolved target moved more than one bucket from its final value.", + }, + render_load: { + unit: "share", + clock: "render", + aggregations: ["max"], + description: + "Chromium's render capacity: the share of each quantum's budget the graph used. High says the runner, not the player, was short of time.", + }, +} as const satisfies Record; + +/** One of the {@link METRICS} names. */ +export type Metric = keyof typeof METRICS; + +/** Every `_` key, in schema order, so tables read the same every run. */ +export const METRIC_KEYS: string[] = Object.entries(METRICS).flatMap(([name, spec]) => + spec.aggregations.map((aggregation) => `${name}_${aggregation}`), +); + +/** A quantum whose RMS at the player's output is under this is quiet. About -60 dBFS. */ +export const SILENCE_RMS = 0.001; + +/** How often the page samples. */ +export const SAMPLE_INTERVAL_MS = 250; + +/** A resolved target within this of its final value has settled. One estimator bucket. */ +export const BUCKET_MS = 20; + +/** + * The render clock may run this far from the viewer's, as a fraction, before the row is void. + * + * Both oscillators are on one host, so real drift is parts per million. Past 1% the device is not + * running at wall rate at all, which is a throttled or stalled context, and every render-clock count + * would be read off a clock that was not moving. + */ +export const MAX_RENDER_DRIFT = 0.01; + +/** + * The stages an end-to-end delay splits into, in order, as exclusive spans. + * + * Exclusive is what makes the sum mean anything: each stage is defined by the two boundaries that + * end the previous stage and start the next, so no two can claim the same millisecond. + * + * - `capture`: sound at the publisher's input to its samples being readable. + * - `encode`: samples readable to the encoded frame existing. + * - `publish_flush`: frame existing to it being written to the transport, where a publisher that + * batches frames holds them. + * - `network`: written by the publisher to delivered to the viewer's container consumer, relay + * included. + * - `jitter_buffer`: delivered to its decoded samples leaving the ring for a render quantum. The + * decoder runs inside this span in a live pipeline, so it is not also `decode`. + * - `decode`: decode time that did not overlap the jitter buffer, which is zero while the ring holds + * anything. + * - `render`: leaving the ring to leaving the audio graph. + * - `device`: leaving the graph to the speaker (`outputLatency` plus `baseLatency`). + * - `unaccounted`: end-to-end minus the sum of the rest. Named because a large one is the finding. + * + * The identity is `sum(stages) == end-to-end`, within {@link identityTolerance}. + */ +export const STAGES = [ + "capture", + "encode", + "publish_flush", + "network", + "jitter_buffer", + "decode", + "render", + "device", + "unaccounted", +] as const; + +/** One of {@link STAGES}. */ +export type Stage = (typeof STAGES)[number]; + +/** A stage span and where its number came from. */ +export type StageSpan = { + /** The span, or null when this lane cannot measure it. */ + ms: Ms | null; + /** `measured` on the viewer clock, `declared` by the publisher, or `unmeasured`. */ + source: "measured" | "declared" | "unmeasured"; +}; + +/** The tolerance on the sum-to-end-to-end identity: the larger of 2 ms or 2% of the total. */ +export const identityTolerance = (endToEnd: Ms): Ms => Math.max(2, endToEnd * 0.02); + +/** What reducing one clock to `viewer` took. */ +export type Drift = { + /** Which clock. */ + clock: Clock; + /** Its rate against `viewer`, minus one: 0.001 runs 1 ms fast per second. Null when unmeasured. */ + rate: number | null; + /** What was added to put it on the viewer epoch. Null when unmeasured or unknowable. */ + offset: Ms | null; +}; + +// ── the row ───────────────────────────────────────────────────────────────── + +/** Which ring the page ran, decided by whether the document is cross-origin isolated. */ +export type Ring = "isolated" | "plain"; + +/** The audio codecs the matrix publishes. */ +export type Codec = "opus" | "aac"; + +/** + * One matrix cell. + * + * A budget is keyed by the whole thing. The codec, its rate, and the ring each move the expected + * floor as much as the profile does, so a profile-only key would grade one cell against another's + * threshold. + */ +export type Row = { + runtime: "chromium"; + codec: Codec; + /** Sample rate in Hz. */ + rate: number; + profile: string; + ring: Ring; +}; + +/** The row as a flat string: a file name and a table label. */ +export const rowKey = (row: Row): string => `${row.runtime}-${row.codec}-${row.rate}-${row.profile}-${row.ring}`; + +/** Parse {@link rowKey}. The profile is the only field that may contain a dash. */ +export function parseRow(key: string): Row { + const parts = key.split("-"); + const [runtime, codec, rate] = parts; + const ring = parts.at(-1); + const profile = parts.slice(3, -1).join("-"); + const hz = Number.parseInt(rate ?? "", 10); + if ( + runtime !== "chromium" || + (codec !== "opus" && codec !== "aac") || + (ring !== "isolated" && ring !== "plain") || + !Number.isFinite(hz) || + profile === "" + ) { + throw new Error(`not a row key: ${key}`); + } + return { runtime, codec, rate: hz, profile, ring }; +} + +// ── what the page emits ───────────────────────────────────────────────────── + +/** A run of consecutive quanta the ring did not fill, as the output tap saw it. */ +export type Gap = { + /** Where the first missing sample would have played, on the `render` clock. */ + at: Ms; + /** The missing audio, at the device rate. */ + ms: Ms; + /** Quanta in it the ring did not fill, short and silent alike. */ + quanta: number; + /** Of those, the quanta it filled the front of. */ + short: number; +}; + +/** A change in whether the ring is stalled, stamped on the `render` clock when the page saw it. */ +export type Stall = { + at: Ms; + stalled: boolean; +}; + +/** One probe sample: what the page could read at that instant, plus what the tap reported since. */ +export type Sample = { + /** `performance.now()`: the `viewer` clock. */ + at: Ms; + /** `AudioContext.currentTime`, read at the same instant: the `render` clock. */ + render?: Ms; + + /** `audio.out.timestamp`: the playhead on the `media` clock. Undefined while there is none. */ + timestamp?: Ms; + /** `audio.out.stalled`: the ring is refilling rather than playing. */ + stalled?: boolean; + /** `sync.out.delay`: the resolved playout target. */ + delay?: Ms; + + /** `AudioContext.outputLatency`. */ + outputLatency?: Ms; + /** `AudioContext.baseLatency`. */ + baseLatency?: Ms; + /** `AudioContext.renderCapacity` average load, Chromium only. */ + renderLoad?: number; + + /** The tap's cumulative quanta rendered since the first audible one. */ + quanta?: number; + /** The tap's cumulative quanta under {@link SILENCE_RMS}. */ + quiet?: number; + /** Gaps the tap closed since the previous sample. */ + gaps?: Gap[]; + /** Stall changes since the previous sample. */ + stalls?: Stall[]; +}; + +/** What the page reports once the session is up. */ +export type Environment = { + /** Whether the document is cross-origin isolated, and therefore which ring runs. */ + crossOriginIsolated: boolean; + /** The transport the session negotiated. Anything but WebTransport never crossed the UDP shaper. */ + transport?: string; + /** The audio rendition's codec string. */ + codec?: string; + /** The audio rendition's sample rate, Hz. */ + rate?: number; + /** The publisher's declared flush span: the `publish_flush` stage. */ + jitter?: Ms; + /** The AudioContext's rate, Hz, which the render clock counts in. */ + contextRate?: number; +}; + +// ── what a row reduces to ─────────────────────────────────────────────────── + +/** The shaper's counters for one direction, as its exit line prints them. */ +export type ShaperCounters = { + packets: number; + lost: number; + overflowed: number; + throttled: number; + delayed: number; + reordered: number; +}; + +/** The shaper's run: its seed, how it exited, and what it did. */ +export type Shaper = { + seed: number | null; + /** Its exit status. Nonzero means the profile never acted, or forwarding stopped. */ + status: number | null; + up: ShaperCounters | null; + down: ShaperCounters | null; +}; + +/** Why a row cannot be graded. A void row is reported, and fails an enforced run. */ +export type Void = { + /** The assertion that failed. */ + assertion: string; + /** What was seen instead. */ + detail: string; +}; + +/** Everything one row produced: the graded numbers and what makes them trustworthy. */ +export type Summary = { + /** Bumped when a metric's meaning changes, not when one is added. */ + version: 1; + row: Row; + /** Graded window, after the warmup. */ + windowMs: Ms; + /** Discarded after first audio, before grading. */ + warmupMs: Ms; + environment: Environment | null; + /** Empty means the row is gradeable. */ + voids: Void[]; + /** Every {@link METRIC_KEYS} entry. Null is unmeasured, never zero. */ + metrics: Record; + stages: Record; + /** The delay the stages sum to, or null when no clock spans the whole path. */ + endToEnd: Ms | null; + drift: Drift[]; + shaper: Shaper | null; + /** Console warnings and errors, truncated. */ + notes: string[]; +}; + +/** One budget: the whole row it applies to, and a ceiling per graded key. */ +export type Budget = Row & { [key: string]: number | string }; + +/** The checked-in budget file. */ +export type Budgets = { + /** How the ceilings were arrived at. */ + note: string; + rows: Budget[]; +}; diff --git a/test/audio-quality/clients/js/src/tap-worklet.ts b/test/audio-quality/clients/js/src/tap-worklet.ts new file mode 100644 index 0000000000..e5a02245fb --- /dev/null +++ b/test/audio-quality/clients/js/src/tap-worklet.ts @@ -0,0 +1,39 @@ +/** + * The harness's own AudioWorklet, fanned out from the player's output: it listens and never plays. + * + * It sees every render quantum rather than a 250 ms sample of them, which is what lets a gap of a few + * milliseconds between two probe samples be counted at all. See `quantum.ts` for how a quantum is + * read. + * + * @module + */ +import { classify, Ledger } from "./quantum.ts"; +import type { TapInit, TapReport } from "./tap.ts"; + +/** Quanta between reports: about 170 ms at 48 kHz, inside the probe's 250 ms. */ +const REPORT_EVERY = 64; + +class Tap extends AudioWorkletProcessor { + #ledger: Ledger; + #since = 0; + + constructor(options: AudioWorkletNodeOptions) { + super(); + const init = options.processorOptions as TapInit; + this.#ledger = new Ledger(sampleRate, init.floor); + } + + process(inputs: Float32Array[][]): boolean { + const input = inputs[0] ?? []; + this.#ledger.add(currentFrame, input[0]?.length ?? 128, classify(input)); + + if (++this.#since >= REPORT_EVERY) { + this.#since = 0; + const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take() }; + this.port.postMessage(report); + } + return true; + } +} + +registerProcessor("audio-quality-tap", Tap); diff --git a/test/audio-quality/clients/js/src/tap.ts b/test/audio-quality/clients/js/src/tap.ts new file mode 100644 index 0000000000..7d0ac4dcc4 --- /dev/null +++ b/test/audio-quality/clients/js/src/tap.ts @@ -0,0 +1,70 @@ +/** + * Fan the tap worklet out from a player's output node, and collect what it reports. + * + * The tap reaches the destination through a zero gain, because Web Audio only renders what the + * destination pulls: left dangling it would never be asked for a quantum. The zero gain keeps it out + * of what the listener hears. + * + * @module + */ +import type { Counts } from "./quantum.ts"; +import type { Gap } from "./schema.ts"; +import TapWorklet from "./tap-worklet.ts?worklet"; + +/** What the worklet is constructed with. */ +export type TapInit = { + /** The silence floor, as an RMS level. */ + floor: number; +}; + +/** What the worklet posts every few dozen quanta. */ +export type TapReport = { + /** Cumulative since the first audible quantum. */ + counts: Counts; + /** Gaps closed since the last report. */ + gaps: Gap[]; +}; + +/** A running tap. */ +export type Tap = { + /** The latest cumulative counts, or undefined before the first report. */ + counts(): Counts | undefined; + /** Gaps closed since the last call. */ + take(): Gap[]; + close(): void; +}; + +/** Attach a tap to `root`. */ +export async function tap(root: AudioNode, init: TapInit): Promise { + const context = root.context as AudioContext; + await context.audioWorklet.addModule(TapWorklet); + + const node = new AudioWorkletNode(context, "audio-quality-tap", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: init, + }); + const mute = new GainNode(context, { gain: 0 }); + root.connect(node); + node.connect(mute); + mute.connect(context.destination); + + let counts: Counts | undefined; + const gaps: Gap[] = []; + node.port.onmessage = (event: MessageEvent) => { + counts = event.data.counts; + gaps.push(...event.data.gaps); + }; + + return { + counts: () => counts, + take: () => gaps.splice(0, gaps.length), + close() { + node.port.onmessage = null; + root.disconnect(node); + node.disconnect(); + mute.disconnect(); + }, + }; +} diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json new file mode 100644 index 0000000000..5a536d16f4 --- /dev/null +++ b/test/audio-quality/clients/js/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../../js/tsconfig.json", + "compilerOptions": { + "types": ["audioworklet", "bun", "vite/client"] + }, + "include": ["analyze.ts", "driver.ts", "grade.ts", "src", "vite.config.ts", "../../../../js/common/worklet.d.ts"] +} diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts new file mode 100644 index 0000000000..c971d84052 --- /dev/null +++ b/test/audio-quality/clients/js/vite.config.ts @@ -0,0 +1,19 @@ +/** + * Vite config for the audio quality page. `@moq/watch` is consumed as workspace source, so its render + * worklet is inlined as a blob URL by the same plugin its own build uses, along with the harness's tap + * worklet: a worklet fetched over the network would be one more thing that can stall inside the + * measurement. + * + * `base: "./"` because one build is served under both `/isolated/` and `/plain/`. + * + * @module + */ +import { defineConfig } from "vite"; +import { workletInline } from "../../../../js/common/vite-plugin-worklet"; + +/** esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. */ +export default defineConfig({ + base: "./", + plugins: [workletInline()], + build: { target: "esnext", outDir: "dist" }, +}); diff --git a/test/audio-quality/relay.toml b/test/audio-quality/relay.toml new file mode 100644 index 0000000000..5b8ed37b1d --- /dev/null +++ b/test/audio-quality/relay.toml @@ -0,0 +1,22 @@ +# Relay config for the audio quality harness. +# Anonymous access, self-signed localhost cert, QUIC + HTTP on 127.0.0.1:4443. +# +# run.sh rewrites the port into a copy in the run directory. The page reaches QUIC through the +# shaper on another port; the driver answers the page's /certificate.sha256 fetch with this relay's +# hash, so the shaper only carries UDP. + +[log] +level = "info" + +[listen] +# QUIC on UDP. 127.0.0.1 avoids IPv6 flakiness on CI runners. +bind = "127.0.0.1:4443" +tls.generate = ["localhost", "127.0.0.1"] + +[web.http] +# HTTP on TCP, serving /certificate.sha256 to the publishers and the driver. +listen = "127.0.0.1:4443" + +[auth] +# Allow anonymous access to everything. +public = "**" diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh new file mode 100755 index 0000000000..c10544597e --- /dev/null +++ b/test/audio-quality/run.sh @@ -0,0 +1,300 @@ +#!/usr/bin/env bash +# Plays a tone in headless Chromium over a seeded, impaired UDP path, and grades what it heard. +# +# One row is one page playing one codec through one shaper profile on one ring, for --duration +# seconds of audio. See README.md for the profiles, the metric contract, and the void rules. +set -euo pipefail + +AQ_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$AQ_DIR/../.." && pwd) +CLIENT="$AQ_DIR/clients/js" + +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$AQ_DIR/../lib/harness.sh" + +# Captured before the parse consumes it, so the rerun command carries every flag. +RERUN="$(harness_env AQ_PROFILE RELAY_BIN)just test audio-quality$(harness_argv "$@")" + +ALL_PROFILES=(near-zero mild wide fixed-250) +ALL_RINGS=(isolated plain) +ALL_CODECS=(opus aac) + +PROFILES=("${ALL_PROFILES[@]}") +RINGS=("${ALL_RINGS[@]}") +CODECS=("${ALL_CODECS[@]}") +DURATION=60 +SEED=7 +OUT="" +LIST=0 +ENFORCE=0 +PROFILE="${AQ_PROFILE:-debug}" + +split() { + local IFS=, + # shellcheck disable=SC2206 # splitting on commas is the point + SPLIT=($1) +} + +need() { + if [[ $2 -lt 2 || -z "${3:-}" || "${3:-}" == -* ]]; then + echo "error: $1 requires a value" >&2 + exit 2 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --profiles) + need "$1" $# "${2:-}" + split "$2" + PROFILES=("${SPLIT[@]}") + shift 2 + ;; + --rings) + need "$1" $# "${2:-}" + split "$2" + RINGS=("${SPLIT[@]}") + shift 2 + ;; + --codecs) + need "$1" $# "${2:-}" + split "$2" + CODECS=("${SPLIT[@]}") + shift 2 + ;; + --duration) + need "$1" $# "${2:-}" + DURATION="$2" + shift 2 + ;; + --seed) + need "$1" $# "${2:-}" + SEED="$2" + shift 2 + ;; + --out) + need "$1" $# "${2:-}" + OUT="$2" + shift 2 + ;; + --list) + LIST=1 + shift + ;; + --enforce) + ENFORCE=1 + shift + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +if [[ ! "$DURATION" =~ ^[0-9]+$ ]] || ((DURATION <= 5)); then + echo "error: --duration must be whole seconds, more than the 5s warmup (got '$DURATION')" >&2 + exit 2 +fi +if [[ ! "$SEED" =~ ^[0-9]+$ ]]; then + echo "error: --seed must be a non-negative integer (got '$SEED')" >&2 + exit 2 +fi + +valid() { + local want="$1" name="$2" + shift 2 + local known + for known in "$@"; do + [[ "$known" == "$want" ]] && return 0 + done + echo "error: unknown $name '$want' (known: $*)" >&2 + exit 2 +} +for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}"; done +for r in "${RINGS[@]}"; do valid "$r" ring "${ALL_RINGS[@]}"; done +for c in "${CODECS[@]}"; do valid "$c" codec "${ALL_CODECS[@]}"; done + +if [[ $LIST -eq 0 && -n "$OUT" && -d "$OUT" ]] && [[ -n "$(ls -A "$OUT" 2>/dev/null)" ]]; then + echo "error: --out $OUT is not empty; remove it or name a new directory" >&2 + exit 2 +fi + +# The rate is part of the row key: a 44.1 kHz stream's frames do not land on a 48 kHz quantum. +rate_of() { + case "$1" in + opus) echo 48000 ;; + aac) echo 44100 ;; + esac +} + +# Every profile but the control adapts. The element requires a unit on a fixed delay. +delay_of() { + case "$1" in + fixed-250) echo "250ms" ;; + *) echo "auto" ;; + esac +} + +# The path treatment, as moq-shaper flags. Loss, reorder, and rate limits stay off: the buffer's job +# is absorbing arrival spread, and congestion response would make a failure hard to attribute. +shaper_of() { + case "$1" in + near-zero | fixed-250) ;; + mild) echo "--delay 5ms --jitter 5ms" ;; + wide) echo "--delay 40ms --jitter 40ms" ;; + esac +} + +ROWS=() +for codec in "${CODECS[@]}"; do + for profile in "${PROFILES[@]}"; do + for ring in "${RINGS[@]}"; do + ROWS+=("chromium-$codec-$(rate_of "$codec")-$profile-$ring") + done + done +done + +if [[ $LIST -eq 1 ]]; then + printf '%s\n' "${ROWS[@]}" + echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED" >&2 + exit 0 +fi + +for tool in cargo bun ffmpeg; do + command -v "$tool" >/dev/null 2>&1 || { + echo "error: $tool not found; run inside 'nix develop'" >&2 + exit 1 + } +done + +harness_begin audio-quality "$RERUN" + +# ── build ─────────────────────────────────────────────────────────────────── +flag=() +[[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") +echo "building moq-relay, moq, moq-shaper ($PROFILE)..." +(cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) +TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" +RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" +MOQ="$TARGET_BASE/$PROFILE/moq" +SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" + +# Into the run directory: vite empties its output first, so a shared dist/ would vanish under a +# concurrent run's page loads. +echo "building the page..." +( + cd "$CLIENT" + bun install --frozen-lockfile + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir +) + +# ── relay ─────────────────────────────────────────────────────────────────── +harness_port relay +RELAY_PORT="$HARNESS_PORT" +RELAY_URL="http://127.0.0.1:$RELAY_PORT" +sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" +if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then + echo "relay never became ready" >&2 + sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true + exit 1 +fi +harness_endpoint relay "$RELAY_URL" + +# ── publishers ────────────────────────────────────────────────────────────── +# A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. The +# publishers talk to the relay directly: impairing the ingest would grade the receiver on a stream +# already damaged before it was published. The AAC arm keeps ffmpeg's default PES packing, whose +# multi-frame bursts are the flush-span shape the reporter measured on the public relay (#3477). +# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' +publish_opus() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -ac 2 -c:a libopus -b:a 128k \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 +} + +# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' +publish_aac() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts +} + +for codec in "${CODECS[@]}"; do + harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" +done + +# ── rows ──────────────────────────────────────────────────────────────────── +# One shaper port for the run: rows play one at a time. +harness_port shaper +SHAPER_PORT="$HARNESS_PORT" + +echo "" +echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" +failed=0 +for tag in "${ROWS[@]}"; do + IFS=- read -r _ codec _ rest <<<"$tag" + ring="${rest##*-}" + profile="${rest%-*}" + echo "" + echo "── $tag ──" + + # shellcheck disable=SC2046 # the profile's flags are meant to split + harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ + --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") + shaper_pid="$HARNESS_PID" + # It prints its seed and profile once bound; a page dialing before that would lose its handshake. + until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do + if harness_exited "$shaper_pid"; then + echo "shaper exited before binding for $tag" >&2 + sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true + exit 1 + fi + sleep 0.05 + done + + status=0 + harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ + --url "http://127.0.0.1:$SHAPER_PORT" \ + --fingerprint "$RELAY_URL/certificate.sha256" \ + --broadcast "tone-$codec.hang" \ + --page "$HARNESS_RUN/page" \ + --ring "$ring" \ + --delay "$(delay_of "$profile")" \ + --duration "$DURATION" \ + --tag "$tag" \ + --out "$HARNESS_RUN" + harness_wait "$HARNESS_PID" || status=$? + if [[ $status -ne 0 ]]; then + failed=1 + sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true + fi + + # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile + # acted on the way out, and exits nonzero if it did not. + kill -TERM -- -"$shaper_pid" 2>/dev/null || true + shaper_status=0 + harness_wait "$shaper_pid" || shaper_status=$? + echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } +done + +# ── grade ─────────────────────────────────────────────────────────────────── +echo "" +grade=(bun "$CLIENT/grade.ts" --run "$HARNESS_RUN" --budgets "$AQ_DIR/budgets.json") +[[ $ENFORCE -eq 0 ]] || grade+=(--enforce) +"${grade[@]}" || failed=1 + +if [[ -n "$OUT" ]]; then + mkdir -p "$OUT" + cp -R "$HARNESS_RUN/." "$OUT/" + echo "saved: $OUT" +fi +exit "$failed" diff --git a/test/justfile b/test/justfile index 1c88dcb677..d5ec917a5e 100644 --- a/test/justfile +++ b/test/justfile @@ -73,6 +73,16 @@ media *args: drill *args: just rs test -p moq-relay --test drills {{ args }} +# Plays a tone in headless Chromium through moq-shaper and grades what the listener +# heard (underruns, skip-aheads, silence, the resolved delay) against +# audio-quality/budgets.json. The full matrix is 16 rows at 60 s; nightly runs it with +# --enforce. Narrow it with `--profiles mild --codecs opus --rings plain --duration 20`. +# See audio-quality/README.md. + +# Grade browser audio playout over an impaired path. +audio-quality *args: + ./audio-quality/run.sh "$@" + # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. From 165377f8e8abebf8f6a1e7244180685ab8011723 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 22:05:17 -0700 Subject: [PATCH 03/27] chore: claim shaper profiles quest From 1c47bf8f11b27223f36678162e0ca779bf2f3d7d Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 06:57:25 -0400 Subject: [PATCH 04/27] feat(shaper): pipe TCP on the shaper's port to the target, as an option A relay serves HTTP on the port number it serves QUIC on, and a browser fetches /certificate.sha256 from it before it dials WebTransport. With only a UDP socket on the shaper's port that fetch fails, so a browser cannot be put behind the shaper at all. `Setup` is a `Config` plus the options it has no field for, and `Shaper::bind` takes anything that converts into one, so every existing caller passing a `Config` compiles and behaves as before. Its first option, `tcp_passthrough`, binds a TCP listener on the port the UDP socket got and copies each connection to the target untouched. TCP is not impaired: a reliable transport cannot shed load, so shaping it would measure TCP's retransmission instead of the profile. The CLI takes it as `--tcp-passthrough`. Off by default. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/Cargo.toml | 2 +- rs/moq-shaper/src/lib.rs | 128 ++++++++++++++++++++++++++++++++++++-- rs/moq-shaper/src/main.rs | 9 ++- 3 files changed, 133 insertions(+), 6 deletions(-) diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index a923d0f2f1..ea617a976d 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -21,4 +21,4 @@ anyhow = { workspace = true } clap = { workspace = true } humantime = { workspace = true } rand = { workspace = true } -tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 730d3a9814..c741f1cb6e 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -28,7 +28,12 @@ use std::{ use anyhow::Context; use rand::{RngExt, SeedableRng, rngs::Xoshiro256PlusPlus}; -use tokio::{net::UdpSocket, sync::mpsc, task::JoinSet, time::Instant}; +use tokio::{ + net::{TcpListener, TcpStream, UdpSocket}, + sync::mpsc, + task::JoinSet, + time::Instant, +}; /// How one direction of the path treats each datagram. /// @@ -146,6 +151,32 @@ pub struct Config { pub down: Profile, } +/// A [`Config`] plus the opt-in options it has no field for. +/// +/// Every option defaults to off, so a setup made from a config shapes exactly +/// as that config does. +#[derive(Clone, Debug)] +pub struct Setup { + /// Where to listen and forward, the seed, and each direction's profile. + pub config: Config, + /// Also accept TCP on the listening port and pipe it to the target untouched. + /// + /// A relay serves HTTP on the port number it serves QUIC on, and a browser + /// fetches the certificate hash from it before it dials WebTransport. TCP is + /// never impaired: a reliable transport cannot shed load, so shaping it + /// would measure how TCP retransmits. + pub tcp_passthrough: bool, +} + +impl From for Setup { + fn from(config: Config) -> Self { + Self { + config, + tcp_passthrough: false, + } + } +} + /// What one direction did, summed over every client. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Counters { @@ -200,7 +231,9 @@ pub struct Shaper { impl Shaper { /// Bind the listening socket and start forwarding. - pub async fn bind(config: Config) -> anyhow::Result { + pub async fn bind(setup: impl Into) -> anyhow::Result { + let setup = setup.into(); + let config = setup.config; config.up.validate().context("invalid up profile")?; config.down.validate().context("invalid down profile")?; @@ -209,6 +242,16 @@ impl Shaper { .with_context(|| format!("bind {}", config.bind))?; let addr = listen.local_addr()?; + // The port the UDP socket got, since a relay serves HTTP on its QUIC port. + let tcp = match setup.tcp_passthrough { + true => Some( + TcpListener::bind(addr) + .await + .with_context(|| format!("bind the TCP passthrough on {addr}"))?, + ), + false => None, + }; + let tally = Arc::new([Tally::default(), Tally::default()]); let failed = Arc::new(OnceLock::new()); let task = tokio::spawn({ @@ -216,7 +259,7 @@ impl Shaper { let tally = tally.clone(); let failed = failed.clone(); async move { - if let Err(err) = run(Arc::new(listen), config, tally).await { + if let Err(err) = run(Arc::new(listen), tcp, config, tally).await { let _ = failed.set(format!("{err:#}")); } } @@ -310,7 +353,12 @@ fn bump(counter: &AtomicU64) { /// /// A flow is a socket of its own toward the target, so the target sees one /// address per client just as it would without the shaper in the way. -async fn run(listen: Arc, config: Config, tally: Arc<[Tally; 2]>) -> anyhow::Result<()> { +async fn run( + listen: Arc, + tcp: Option, + config: Config, + tally: Arc<[Tally; 2]>, +) -> anyhow::Result<()> { let mut flows = HashMap::::new(); let mut tasks = JoinSet::new(); let mut buf = vec![0u8; u16::MAX as usize]; @@ -322,6 +370,11 @@ async fn run(listen: Arc, config: Config, tally: Arc<[Tally; 2]>) -> res.context("flow task panicked")??; continue; } + res = accept(tcp.as_ref()) => { + let (stream, _) = res.context("accept a TCP connection")?; + tasks.spawn(pipe(stream, config.target)); + continue; + } }; let now = Instant::now(); @@ -347,6 +400,23 @@ async fn run(listen: Arc, config: Config, tally: Arc<[Tally; 2]>) -> } } +/// The next TCP connection, or never without a passthrough. +async fn accept(tcp: Option<&TcpListener>) -> std::io::Result<(TcpStream, SocketAddr)> { + match tcp { + Some(listener) => listener.accept().await, + None => std::future::pending().await, + } +} + +/// Copy one TCP connection to the target and back, untouched. +async fn pipe(mut client: TcpStream, target: SocketAddr) -> anyhow::Result<()> { + // Either end refusing or hanging up ends that connection, not the shaper. + if let Ok(mut server) = TcpStream::connect(target).await { + let _ = tokio::io::copy_bidirectional(&mut client, &mut server).await; + } + Ok(()) +} + /// A socket that can reach `target`: loopback for a loopback target, so the /// shaper never opens a port beyond the host when it does not need to. async fn bind_toward(target: SocketAddr) -> anyhow::Result { @@ -724,4 +794,54 @@ mod tests { }; refused(undelayed, "needs a delay").await; } + + #[tokio::test] + async fn tcp_passes_through_untouched() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + // A relay answering HTTP on its QUIC port, reduced to an echo. + let server = TcpListener::bind(LOCALHOST).await.unwrap(); + let target = server.local_addr().unwrap(); + tokio::spawn(async move { + let (mut stream, _) = server.accept().await.unwrap(); + let mut buf = [0u8; 64]; + let size = stream.read(&mut buf).await.unwrap(); + stream.write_all(&buf[..size]).await.unwrap(); + }); + + // A profile that would lose every datagram, to show TCP skips it. + let blackhole = Profile { + loss: 1.0, + ..Default::default() + }; + let config = Config { + bind: LOCALHOST, + target, + seed: 13, + up: blackhole.clone(), + down: blackhole, + }; + let shaper = Shaper::bind(Setup { + tcp_passthrough: true, + ..config.into() + }) + .await + .unwrap(); + + let mut client = TcpStream::connect(shaper.addr()).await.unwrap(); + client.write_all(b"/certificate.sha256").await.unwrap(); + let mut buf = [0u8; 64]; + let size = tokio::time::timeout(Duration::from_secs(2), client.read(&mut buf)) + .await + .unwrap() + .unwrap(); + assert_eq!(&buf[..size], b"/certificate.sha256"); + assert_eq!(shaper.stats(), Stats::default(), "TCP reached the datagram path"); + } + + #[tokio::test] + async fn tcp_is_refused_without_the_passthrough() { + let (shaper, _client) = setup(1, Profile::default(), Profile::default()).await; + assert!(TcpStream::connect(shaper.addr()).await.is_err()); + } } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index 2a2841e101..cabb72f9ff 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -40,6 +40,9 @@ struct Args { /// The longest a datagram waits for the rate limit before it is dropped. #[arg(long, default_value = "100ms", value_parser = humantime::parse_duration, requires = "rate")] queue: Duration, + /// Also pipe TCP on the listening port to the target, untouched. + #[arg(long)] + tcp_passthrough: bool, } #[tokio::main] @@ -57,12 +60,16 @@ async fn main() -> anyhow::Result<()> { queue: args.queue, }), }; - let shaper = moq_shaper::Shaper::bind(moq_shaper::Config { + let config = moq_shaper::Config { bind: args.listen, target: args.target, seed: args.seed.unwrap_or_else(rand::random), up: profile.clone(), down: profile, + }; + let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { + tcp_passthrough: args.tcp_passthrough, + ..config.into() }) .await?; From a909d5faf108b401b9c86cafb3ff6574a0b1ff6f Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:01:19 -0400 Subject: [PATCH 05/27] feat(shaper): a gaussian jitter model that keeps the order, as an option The uniform jitter draws each datagram's delay on its own, so a later datagram can overtake an earlier one. A drill wants that, but QUIC reads the overtake as loss: it retransmits and backs off, and a run meant to measure a jittery path measures the congestion response instead. On a 5ms delay with 5ms of jitter, that alone dragged a receiver's audio buffer from 120ms to nearly two seconds. `Options::jitter_model` picks how `Profile::jitter` is drawn, per direction, through `Setup::up` and `Setup::down`. `Jitter::Uniform` stays the default and is unchanged, draws included. `Jitter::Gaussian` treats `jitter` as a sigma, clamps the draw at zero, and never lets a datagram leave before the one in front: queueing delay on a FIFO path, which varies the spacing and never the order. Only `reorder` still overtakes. Its sigma may exceed the delay, which the uniform model refuses. Box-Muller takes exactly two draws per datagram, so the other knobs' streams stay aligned. It counts `delayed` like the uniform model whenever a delay is configured, so `verify` holds it to the same chance. The CLI takes `--jitter-model gaussian`. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/src/lib.rs | 294 +++++++++++++++++++++++++++++++++++--- rs/moq-shaper/src/main.rs | 24 +++- 2 files changed, 299 insertions(+), 19 deletions(-) diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index c741f1cb6e..c9d54744ec 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -38,13 +38,14 @@ use tokio::{ /// How one direction of the path treats each datagram. /// /// A datagram is first subject to `loss`, then waits for the `rate` limit, then -/// takes `delay` plus or minus `jitter` to arrive, unless `reorder` sends it -/// ahead of everything still in flight. +/// takes `delay` plus or minus `jitter`, drawn as its [`Jitter`] model says, to +/// arrive, unless `reorder` sends it ahead of everything still in flight. #[derive(Clone, Debug, Default, PartialEq)] pub struct Profile { /// The base one-way delay. pub delay: Duration, - /// The most a datagram's delay varies from `delay`, uniformly either way. + /// How far a datagram's delay varies from `delay`: the most either way when + /// [`Jitter::Uniform`], the sigma when [`Jitter::Gaussian`]. pub jitter: Duration, /// The probability that a datagram is dropped. pub loss: f64, @@ -55,7 +56,7 @@ pub struct Profile { } impl Profile { - fn validate(&self) -> anyhow::Result<()> { + fn validate(&self, model: Jitter) -> anyhow::Result<()> { anyhow::ensure!( (0.0..=1.0).contains(&self.loss), "loss {} is not a probability", @@ -71,8 +72,9 @@ impl Profile { "reorder {} needs a delay to overtake", self.reorder ); + // A gaussian clamps its draw at zero instead, as a queue cannot run early. anyhow::ensure!( - self.jitter <= self.delay, + model == Jitter::Gaussian || self.jitter <= self.delay, "jitter {:?} exceeds delay {:?}, which would need a negative delay", self.jitter, self.delay @@ -84,6 +86,36 @@ impl Profile { } } +/// How a direction draws each datagram's jitter. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub enum Jitter { + /// Uniform within `jitter` either way of `delay`, drawn for each datagram on + /// its own, so a later datagram can overtake an earlier one. + #[default] + Uniform, + /// A gaussian with `jitter` as its sigma, clamped at zero, that never leaves + /// before the datagram in front: it varies the spacing, never the order. + /// + /// This is queueing delay on a FIFO path. A jitter that overtakes hands QUIC + /// a gap it can only read as loss, so the run measures its congestion + /// response rather than the jitter. + Gaussian, +} + +/// One direction's opt-in options beyond its [`Profile`]. The default adds nothing. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Options { + /// How the profile's `jitter` is drawn. + pub jitter_model: Jitter, +} + +impl Options { + /// Check the options, and `profile` as they draw it. + fn validate(&self, profile: &Profile) -> anyhow::Result<()> { + profile.validate(self.jitter_model) + } +} + /// Each impairment a profile draws for: its name, how likely it is to act on /// any one datagram (zero when unconfigured), and the count showing it did. /// @@ -166,6 +198,10 @@ pub struct Setup { /// never impaired: a reliable transport cannot shed load, so shaping it /// would measure how TCP retransmits. pub tcp_passthrough: bool, + /// The options from a client toward the target. + pub up: Options, + /// The options from the target back toward a client. + pub down: Options, } impl From for Setup { @@ -173,6 +209,8 @@ impl From for Setup { Self { config, tcp_passthrough: false, + up: Options::default(), + down: Options::default(), } } } @@ -222,7 +260,7 @@ impl fmt::Display for Stats { /// A running shaper. Dropping it stops forwarding. pub struct Shaper { addr: SocketAddr, - config: Config, + setup: Setup, tally: Arc<[Tally; 2]>, /// Why forwarding stopped, if it did. failed: Arc>, @@ -233,9 +271,9 @@ impl Shaper { /// Bind the listening socket and start forwarding. pub async fn bind(setup: impl Into) -> anyhow::Result { let setup = setup.into(); - let config = setup.config; - config.up.validate().context("invalid up profile")?; - config.down.validate().context("invalid down profile")?; + let config = &setup.config; + setup.up.validate(&config.up).context("invalid up profile")?; + setup.down.validate(&config.down).context("invalid down profile")?; let listen = UdpSocket::bind(config.bind) .await @@ -255,11 +293,11 @@ impl Shaper { let tally = Arc::new([Tally::default(), Tally::default()]); let failed = Arc::new(OnceLock::new()); let task = tokio::spawn({ - let config = config.clone(); + let setup = setup.clone(); let tally = tally.clone(); let failed = failed.clone(); async move { - if let Err(err) = run(Arc::new(listen), tcp, config, tally).await { + if let Err(err) = run(Arc::new(listen), tcp, setup, tally).await { let _ = failed.set(format!("{err:#}")); } } @@ -267,7 +305,7 @@ impl Shaper { Ok(Self { addr, - config, + setup, tally, failed, task, @@ -281,7 +319,7 @@ impl Shaper { /// The configuration this shaper runs, including its seed. pub fn config(&self) -> &Config { - &self.config + &self.setup.config } /// What the shaper has done so far. @@ -303,7 +341,7 @@ impl Shaper { anyhow::bail!("the shaper stopped forwarding: {err} ({stats})"); } - let missing = unapplied(&self.config, &stats); + let missing = unapplied(&self.setup.config, &stats); anyhow::ensure!( missing.is_empty(), "the profile never applied {} ({stats}), so this run was not impaired as configured", @@ -356,9 +394,10 @@ fn bump(counter: &AtomicU64) { async fn run( listen: Arc, tcp: Option, - config: Config, + setup: Setup, tally: Arc<[Tally; 2]>, ) -> anyhow::Result<()> { + let config = &setup.config; let mut flows = HashMap::::new(); let mut tasks = JoinSet::new(); let mut buf = vec![0u8; u16::MAX as usize]; @@ -390,10 +429,10 @@ async fn run( let (down_tx, down_rx) = mpsc::unbounded_channel(); tasks.spawn(deliver(down_rx, listen.clone(), from)); - let down = Link::new(&config.down, config.seed, stream + 1, now, down_tx); + let down = Link::new(&config.down, &setup.down, config.seed, stream + 1, now, down_tx); tasks.spawn(reply(upstream, config.target, down, tally.clone())); - entry.insert(Link::new(&config.up, config.seed, stream, now, up_tx)) + entry.insert(Link::new(&config.up, &setup.up, config.seed, stream, now, up_tx)) } }; link.push(now, buf[..size].to_vec(), &tally[UP]); @@ -491,9 +530,12 @@ async fn deliver( /// One direction of one flow: the decisions and the state they depend on. struct Link { profile: Profile, + jitter: Jitter, rng: Xoshiro256PlusPlus, /// When the token bucket would be full again, as in GCRA. full_at: Instant, + /// When the latest in-order datagram leaves, which the next one never beats. + floor: Instant, queue: mpsc::UnboundedSender<(Instant, Vec)>, } @@ -504,6 +546,7 @@ impl Link { /// client's ephemeral port is no identity across runs. fn new( profile: &Profile, + options: &Options, seed: u64, stream: u64, now: Instant, @@ -511,10 +554,12 @@ impl Link { ) -> Self { Self { profile: profile.clone(), + jitter: options.jitter_model, rng: Xoshiro256PlusPlus::seed_from_u64(seed ^ stream.wrapping_mul(0x9E37_79B9_7F4A_7C15)), // Not the moment the link is built, which is after `now`: that would // make the first datagram queue behind a bucket still refilling. full_at: now, + floor: now, queue, } } @@ -526,7 +571,10 @@ impl Link { // knob's outcome never shifts the stream another knob draws from. let lose = self.rng.random::() < self.profile.loss; let skip = self.rng.random::() < self.profile.reorder; - let spread = self.rng.random::() * 2.0 - 1.0; + let spread = match self.jitter { + Jitter::Uniform => self.rng.random::() * 2.0 - 1.0, + Jitter::Gaussian => gaussian(&mut self.rng), + }; if lose { bump(&tally.lost); @@ -554,6 +602,17 @@ impl Link { if skip { bump(&tally.reordered); + } else if self.jitter == Jitter::Gaussian { + // A queue: nothing leaves before the datagram in front of it. + let wait = self.profile.delay.as_secs_f64() + self.profile.jitter.as_secs_f64() * spread; + let leave = (depart + Duration::from_secs_f64(wait.max(0.0))).max(self.floor); + // Counted like the uniform model whenever a delay is configured, so + // the chance `verify` holds it to is the same. + if !self.profile.delay.is_zero() || leave > depart { + bump(&tally.delayed); + } + self.floor = leave; + depart = leave; } else if !self.profile.delay.is_zero() { let jitter = self.profile.jitter.as_secs_f64() * spread; depart += Duration::from_secs_f64(self.profile.delay.as_secs_f64() + jitter); @@ -565,6 +624,14 @@ impl Link { } } +/// A standard normal sample by Box-Muller, which always takes exactly two draws, +/// so the jitter never shifts the stream the next datagram's knobs draw from. +fn gaussian(rng: &mut Xoshiro256PlusPlus) -> f64 { + let u1 = rng.random::().max(f64::MIN_POSITIVE); + let u2 = rng.random::(); + (-2.0 * u1.ln()).sqrt() * (std::f64::consts::TAU * u2).cos() +} + #[cfg(test)] mod tests { use super::*; @@ -844,4 +911,195 @@ mod tests { let (shaper, _client) = setup(1, Profile::default(), Profile::default()).await; assert!(TcpStream::connect(shaper.addr()).await.is_err()); } + + /// An echo server, a shaper `setup` builds from a config aimed at it, and a + /// client dialing the shaper. + async fn shaped(setup: impl FnOnce(Config) -> Setup) -> (Shaper, UdpSocket) { + let echo = UdpSocket::bind(LOCALHOST).await.unwrap(); + let target = echo.local_addr().unwrap(); + tokio::spawn(async move { + let mut buf = vec![0u8; u16::MAX as usize]; + loop { + let (size, from) = echo.recv_from(&mut buf).await.unwrap(); + echo.send_to(&buf[..size], from).await.unwrap(); + } + }); + + let config = Config { + bind: LOCALHOST, + target, + seed: 3, + up: Profile::default(), + down: Profile::default(), + }; + let shaper = Shaper::bind(setup(config)).await.unwrap(); + + let client = UdpSocket::bind(LOCALHOST).await.unwrap(); + client.connect(shaper.addr()).await.unwrap(); + (shaper, client) + } + + /// Push `count` numbered datagrams through one link, `spacing` apart, and + /// return each id with its departure, in the order they would be sent. + fn departures( + profile: &Profile, + options: &Options, + count: u32, + spacing: Duration, + ) -> (Vec<(u32, Instant)>, Counters) { + let (queue, mut queued) = mpsc::unbounded_channel(); + let now = Instant::now(); + let mut link = Link::new(profile, options, 7, 0, now, queue); + let tally = Tally::default(); + for id in 0..count { + link.push(now + spacing * id, id.to_be_bytes().to_vec(), &tally); + } + + let mut sent = Vec::new(); + while let Ok((at, datagram)) = queued.try_recv() { + sent.push((u32::from_be_bytes(datagram[..4].try_into().unwrap()), at)); + } + // Ties leave in arrival order, which is id order here. + sent.sort_by_key(|&(id, at)| (at, id)); + (sent, tally.snapshot()) + } + + const GAUSSIAN: Options = Options { + jitter_model: Jitter::Gaussian, + }; + + #[tokio::test] + async fn only_the_gaussian_model_keeps_the_order() { + let jittery = Profile { + delay: Duration::from_millis(20), + jitter: Duration::from_millis(10), + ..Default::default() + }; + + let (_uniform, client) = shaped(|config| { + Config { + up: jittery.clone(), + ..config + } + .into() + }) + .await; + let got = round_trip(&client, 100).await; + let mut sorted = got.clone(); + sorted.sort(); + assert_eq!(sorted, (0..100).collect::>(), "jitter lost datagrams"); + assert_ne!(got, sorted, "uniform jitter never overtook, so this proves nothing"); + + let (gaussian, client) = shaped(|config| Setup { + up: GAUSSIAN, + ..Config { up: jittery, ..config }.into() + }) + .await; + let got = round_trip(&client, 100).await; + assert_eq!(got, (0..100).collect::>(), "gaussian jitter reordered"); + + let stats = gaussian.verify().unwrap(); + assert_eq!(stats.up.reordered, 0, "{stats}"); + assert_eq!(stats.up.delayed, 100, "{stats}"); + } + + #[test] + fn gaussian_jitter_never_leaves_before_it_arrived() { + // A sigma far past the delay, so most draws would be negative unclamped. + let wide = Profile { + jitter: Duration::from_millis(50), + ..Default::default() + }; + let now = Instant::now(); + let (sent, counters) = departures(&wide, &GAUSSIAN, 1000, Duration::ZERO); + assert_eq!(sent.len(), 1000); + assert!(sent.iter().all(|&(_, at)| at >= now)); + assert!(counters.delayed > 0, "{counters}"); + } + + #[test] + fn gaussian_jitter_alone_never_changes_the_order() { + // Ten times the arrival spacing as sigma, so an independent draw per + // datagram would shuffle nearly all of them. + let jittery = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(50), + ..Default::default() + }; + let (sent, counters) = departures(&jittery, &GAUSSIAN, 2000, Duration::from_millis(5)); + let ids: Vec = sent.iter().map(|&(id, _)| id).collect(); + assert_eq!(ids, (0..2000).collect::>(), "gaussian jitter reordered"); + assert_eq!(counters.reordered, 0); + } + + #[test] + fn gaussian_jitter_still_varies_the_spacing() { + let spacing = Duration::from_millis(20); + let jittery = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(5), + ..Default::default() + }; + let (sent, _) = departures(&jittery, &GAUSSIAN, 2000, spacing); + + // Keeping the order is not pacing the path: a datagram that drew more + // than the one in front falls further behind, and one that drew less + // closes up against it. + let gaps: Vec = sent.windows(2).map(|pair| pair[1].1 - pair[0].1).collect(); + assert!(gaps.iter().any(|&gap| gap < spacing), "nothing closed up"); + assert!(gaps.iter().any(|&gap| gap > spacing), "nothing fell behind"); + } + + #[test] + fn a_reorder_still_overtakes_gaussian_jitter() { + let shuffled = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(50), + reorder: 0.05, + ..Default::default() + }; + let (sent, counters) = departures(&shuffled, &GAUSSIAN, 2000, Duration::from_millis(5)); + let overtaken = sent.windows(2).filter(|pair| pair[0].0 > pair[1].0).count(); + assert!(overtaken > 0, "a reorder never overtook"); + assert!(counters.reordered > 0, "{counters}"); + } + + #[test] + fn the_seed_reproduces_the_gaussian_decisions() { + let everything = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(5), + loss: 0.05, + reorder: 0.05, + ..Default::default() + }; + let (first, first_counters) = departures(&everything, &GAUSSIAN, 1000, Duration::from_millis(1)); + let (second, second_counters) = departures(&everything, &GAUSSIAN, 1000, Duration::from_millis(1)); + + // The same ids survive and leave at the same offsets from their start. + let offsets = |sent: &[(u32, Instant)]| { + let start = sent.iter().map(|&(_, at)| at).min().unwrap(); + sent.iter().map(|&(id, at)| (id, at - start)).collect::>() + }; + assert_eq!(offsets(&first), offsets(&second)); + assert_eq!(first_counters, second_counters); + assert!(first_counters.lost > 0 && first_counters.reordered > 0); + } + + #[tokio::test] + async fn a_gaussian_sigma_may_exceed_the_delay() { + // The uniform model refuses this; a gaussian clamps its draw at zero. + let wide = Profile { + delay: Duration::from_millis(5), + jitter: Duration::from_millis(10), + ..Default::default() + }; + let (shaper, client) = shaped(|config| Setup { + up: GAUSSIAN, + ..Config { up: wide, ..config }.into() + }) + .await; + assert_eq!(round_trip(&client, 20).await, (0..20).collect::>()); + shaper.verify().unwrap(); + } } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index cabb72f9ff..eeef1c158f 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -22,9 +22,12 @@ struct Args { /// The base one-way delay, e.g. `20ms`. #[arg(long, default_value = "0s", value_parser = humantime::parse_duration)] delay: Duration, - /// The most the delay varies, uniformly either way. + /// How far the delay varies: the most either way, or the sigma of a gaussian. #[arg(long, default_value = "0s", value_parser = humantime::parse_duration)] jitter: Duration, + /// How the jitter is drawn. + #[arg(long, value_enum, default_value_t = JitterModel::Uniform)] + jitter_model: JitterModel, /// The probability that a datagram is dropped. #[arg(long, default_value_t = 0.0)] loss: f64, @@ -45,6 +48,14 @@ struct Args { tcp_passthrough: bool, } +#[derive(Clone, Copy, clap::ValueEnum)] +enum JitterModel { + /// Uniform either way of the delay, per datagram, so datagrams can overtake each other. + Uniform, + /// A gaussian clamped at zero that never lets a datagram overtake the one in front. + Gaussian, +} + #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); @@ -67,8 +78,16 @@ async fn main() -> anyhow::Result<()> { up: profile.clone(), down: profile, }; + let options = moq_shaper::Options { + jitter_model: match args.jitter_model { + JitterModel::Uniform => moq_shaper::Jitter::Uniform, + JitterModel::Gaussian => moq_shaper::Jitter::Gaussian, + }, + }; let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { tcp_passthrough: args.tcp_passthrough, + up: options.clone(), + down: options.clone(), ..config.into() }) .await?; @@ -81,6 +100,9 @@ async fn main() -> anyhow::Result<()> { config.seed, config.up ); + if options != moq_shaper::Options::default() { + println!("shaper: options {options:?}"); + } shutdown().await?; From 6b7964c1558803b9a8d948b853c0b1dca123ef03 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:05:27 -0400 Subject: [PATCH 06/27] feat(shaper): one link each way for every client, as an option Each client flow takes a link of its own each way, with its own seeded stream, rate limit and queue. That models a path per client. A browser page is several clients on one path: its own WebTransport session and its audio worker's, on one access link. There, the second flow's datagrams wait behind the first's, and a clump holds both. `Setup::shared` gives every flow one link each way, which is how the path the audio-quality lane was measured on behaved. A jitter that keeps the order then keeps it across clients, and one seeded stream and one rate limit serve them all. The shared links draw the streams the first flow's would, and a client's treatment now depends on how its datagrams interleave with the others'. The CLI takes it as `--shared`. Off by default, so each flow still gets its own. To route a shared link's output, a treated datagram carries the socket and address it leaves by, and `Link::treat` returns a departure without queueing it, so the link's decisions can be tested without sockets. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/src/lib.rs | 227 +++++++++++++++++++++++++++++--------- rs/moq-shaper/src/main.rs | 8 +- 2 files changed, 182 insertions(+), 53 deletions(-) diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index c9d54744ec..9a915854f5 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -20,7 +20,7 @@ use std::{ fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::{ - Arc, OnceLock, + Arc, Mutex, OnceLock, atomic::{AtomicU64, Ordering}, }, time::Duration, @@ -198,6 +198,13 @@ pub struct Setup { /// never impaired: a reliable transport cannot shed load, so shaping it /// would measure how TCP retransmits. pub tcp_passthrough: bool, + /// Every client shares one link each way, the way clients behind one access + /// link do, rather than each getting its own. + /// + /// Shared, a jitter that keeps the order keeps it across clients, and one + /// seeded stream and one rate limit serve them all, so a client's treatment + /// depends on how its datagrams interleave with the others'. + pub shared: bool, /// The options from a client toward the target. pub up: Options, /// The options from the target back toward a client. @@ -209,6 +216,7 @@ impl From for Setup { Self { config, tcp_passthrough: false, + shared: false, up: Options::default(), down: Options::default(), } @@ -390,7 +398,8 @@ fn bump(counter: &AtomicU64) { /// Accept datagrams from clients, giving each client its own flow. /// /// A flow is a socket of its own toward the target, so the target sees one -/// address per client just as it would without the shaper in the way. +/// address per client just as it would without the shaper in the way. Each +/// flow takes a link of its own each way, unless the path is shared. async fn run( listen: Arc, tcp: Option, @@ -398,10 +407,19 @@ async fn run( tally: Arc<[Tally; 2]>, ) -> anyhow::Result<()> { let config = &setup.config; - let mut flows = HashMap::::new(); + let mut flows = HashMap::::new(); let mut tasks = JoinSet::new(); let mut buf = vec![0u8; u16::MAX as usize]; + // A shared path is one link each way, drawing the streams the first flow would. + let shared = setup.shared.then(|| { + let now = Instant::now(); + [ + link(&mut tasks, &config.up, &setup.up, config.seed, 0, now), + link(&mut tasks, &config.down, &setup.down, config.seed, 1, now), + ] + }); + loop { let (size, from) = tokio::select! { res = listen.recv_from(&mut buf) => res.context("receive from a client")?, @@ -418,27 +436,56 @@ async fn run( let now = Instant::now(); let stream = 2 * flows.len() as u64; - let link = match flows.entry(from) { + let flow = match flows.entry(from) { hash_map::Entry::Occupied(entry) => entry.into_mut(), hash_map::Entry::Vacant(entry) => { let upstream = Arc::new(bind_toward(config.target).await?); - let (up_tx, up_rx) = mpsc::unbounded_channel(); - tasks.spawn(deliver(up_rx, upstream.clone(), config.target)); - - let (down_tx, down_rx) = mpsc::unbounded_channel(); - tasks.spawn(deliver(down_rx, listen.clone(), from)); - - let down = Link::new(&config.down, &setup.down, config.seed, stream + 1, now, down_tx); - tasks.spawn(reply(upstream, config.target, down, tally.clone())); - - entry.insert(Link::new(&config.up, &setup.up, config.seed, stream, now, up_tx)) + let [up, down] = match &shared { + Some(links) => links.clone(), + None => [ + link(&mut tasks, &config.up, &setup.up, config.seed, stream, now), + link(&mut tasks, &config.down, &setup.down, config.seed, stream + 1, now), + ], + }; + tasks.spawn(reply( + upstream.clone(), + config.target, + down, + listen.clone(), + from, + tally.clone(), + )); + + entry.insert(Flow { upstream, up }) } }; - link.push(now, buf[..size].to_vec(), &tally[UP]); + let datagram = buf[..size].to_vec(); + let mut up = flow.up.lock().expect("link poisoned"); + up.push(now, datagram, &flow.upstream, config.target, &tally[UP]); } } +/// One client: its socket toward the target, and the link it sends through. +struct Flow { + upstream: Arc, + up: Arc>, +} + +/// A link, and the task delivering what it treats. +fn link( + tasks: &mut JoinSet>, + profile: &Profile, + options: &Options, + seed: u64, + stream: u64, + now: Instant, +) -> Arc> { + let (queue, queued) = mpsc::unbounded_channel(); + tasks.spawn(deliver(queued)); + Arc::new(Mutex::new(Link::new(profile, options, seed, stream, now, queue))) +} + /// The next TCP connection, or never without a passthrough. async fn accept(tcp: Option<&TcpListener>) -> std::io::Result<(TcpStream, SocketAddr)> { match tcp { @@ -470,11 +517,13 @@ async fn bind_toward(target: SocketAddr) -> anyhow::Result { .with_context(|| format!("bind a socket toward {target}")) } -/// Feed what the target sends a flow into that flow's return link. +/// Feed what the target sends a flow into the link back toward its client. async fn reply( socket: Arc, target: SocketAddr, - mut link: Link, + link: Arc>, + listen: Arc, + client: SocketAddr, tally: Arc<[Tally; 2]>, ) -> anyhow::Result<()> { let mut buf = vec![0u8; u16::MAX as usize]; @@ -484,19 +533,17 @@ async fn reply( if from != target { continue; } - link.push(Instant::now(), buf[..size].to_vec(), &tally[DOWN]); + let datagram = buf[..size].to_vec(); + let mut down = link.lock().expect("link poisoned"); + down.push(Instant::now(), datagram, &listen, client, &tally[DOWN]); } } /// Send each treated datagram at its departure time. -async fn deliver( - mut queue: mpsc::UnboundedReceiver<(Instant, Vec)>, - socket: Arc, - dest: SocketAddr, -) -> anyhow::Result<()> { +async fn deliver(mut queue: mpsc::UnboundedReceiver) -> anyhow::Result<()> { // Ordered by departure, then by arrival, so ties keep their order. let mut pending = BinaryHeap::>::new(); - let mut datagrams = HashMap::>::new(); + let mut parcels = HashMap::::new(); let mut sequence = 0u64; loop { @@ -504,9 +551,9 @@ async fn deliver( tokio::select! { item = queue.recv() => { // The link is gone, so the flow is too. - let Some((at, datagram)) = item else { return Ok(()) }; - pending.push(Reverse((at, sequence))); - datagrams.insert(sequence, datagram); + let Some(parcel) = item else { return Ok(()) }; + pending.push(Reverse((parcel.at, sequence))); + parcels.insert(sequence, parcel); sequence += 1; } _ = tokio::time::sleep_until(next.unwrap_or_else(Instant::now)), if next.is_some() => { @@ -516,18 +563,27 @@ async fn deliver( break; } pending.pop(); - let datagram = datagrams.remove(&id).expect("queued datagram"); + let parcel = parcels.remove(&id).expect("queued datagram"); // A send error is the path losing the datagram, the way a // network does when the far end is gone (a killed relay, say), // so it is not the shaper failing. - let _ = socket.send_to(&datagram, dest).await; + let _ = parcel.socket.send_to(&parcel.datagram, parcel.dest).await; } } } } } -/// One direction of one flow: the decisions and the state they depend on. +/// A treated datagram: when it leaves, and the socket and address it leaves by. +struct Parcel { + at: Instant, + datagram: Vec, + socket: Arc, + dest: SocketAddr, +} + +/// One direction of one flow, or of every flow on a shared path: the decisions +/// and the state they depend on. struct Link { profile: Profile, jitter: Jitter, @@ -536,21 +592,21 @@ struct Link { full_at: Instant, /// When the latest in-order datagram leaves, which the next one never beats. floor: Instant, - queue: mpsc::UnboundedSender<(Instant, Vec)>, + queue: mpsc::UnboundedSender, } impl Link { - /// Each direction of each flow draws from its own stream of the one seed, - /// so a flow's decisions do not depend on how its datagrams interleave with - /// anyone else's. Flows are numbered in the order they first send, since a - /// client's ephemeral port is no identity across runs. + /// Each link draws from its own stream of the one seed, so a flow's + /// decisions do not depend on how its datagrams interleave with anyone + /// else's, unless the path is shared. Flows are numbered in the order they + /// first send, since a client's ephemeral port is no identity across runs. fn new( profile: &Profile, options: &Options, seed: u64, stream: u64, now: Instant, - queue: mpsc::UnboundedSender<(Instant, Vec)>, + queue: mpsc::UnboundedSender, ) -> Self { Self { profile: profile.clone(), @@ -564,7 +620,23 @@ impl Link { } } - fn push(&mut self, now: Instant, datagram: Vec, tally: &Tally) { + /// Treat a datagram and queue it to leave by `socket` for `dest`. + fn push(&mut self, now: Instant, datagram: Vec, socket: &Arc, dest: SocketAddr, tally: &Tally) { + let Some(at) = self.treat(now, datagram.len(), tally) else { + return; + }; + // The delivery task only ends once this link is dropped. + let _ = self.queue.send(Parcel { + at, + datagram, + socket: socket.clone(), + dest, + }); + } + + /// When a datagram of `size` bytes arriving `now` leaves, or `None` when it + /// is dropped. + fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option { bump(&tally.packets); // Every draw happens for every datagram, whatever the profile, so one @@ -578,20 +650,20 @@ impl Link { if lose { bump(&tally.lost); - return; + return None; } let mut depart = now; if let Some(rate) = &self.profile.rate { // GCRA: a datagram may leave once the backlog, itself included, is // within the burst allowance. - let full_at = self.full_at.max(now) + rate.cost(datagram.len() as u64); + let full_at = self.full_at.max(now) + rate.cost(size as u64); let start = full_at .checked_sub(rate.cost(rate.burst)) .map_or(now, |start| start.max(now)); if start - now > rate.queue { bump(&tally.overflowed); - return; + return None; } if start > now { bump(&tally.throttled); @@ -619,8 +691,7 @@ impl Link { bump(&tally.delayed); } - // The delivery task only ends once this link is dropped. - let _ = self.queue.send((depart, datagram)); + Some(depart) } } @@ -947,18 +1018,13 @@ mod tests { count: u32, spacing: Duration, ) -> (Vec<(u32, Instant)>, Counters) { - let (queue, mut queued) = mpsc::unbounded_channel(); + let (queue, _) = mpsc::unbounded_channel(); let now = Instant::now(); let mut link = Link::new(profile, options, 7, 0, now, queue); let tally = Tally::default(); - for id in 0..count { - link.push(now + spacing * id, id.to_be_bytes().to_vec(), &tally); - } - - let mut sent = Vec::new(); - while let Ok((at, datagram)) = queued.try_recv() { - sent.push((u32::from_be_bytes(datagram[..4].try_into().unwrap()), at)); - } + let mut sent: Vec<(u32, Instant)> = (0..count) + .filter_map(|id| Some((id, link.treat(now + spacing * id, 4, &tally)?))) + .collect(); // Ties leave in arrival order, which is id order here. sent.sort_by_key(|&(id, at)| (at, id)); (sent, tally.snapshot()) @@ -1102,4 +1168,63 @@ mod tests { assert_eq!(round_trip(&client, 20).await, (0..20).collect::>()); shaper.verify().unwrap(); } + + /// Two clients take turns sending numbered datagrams up one gaussian path, + /// and the target's arrival order comes back. + async fn two_clients(shared: bool) -> Vec { + let target = UdpSocket::bind(LOCALHOST).await.unwrap(); + let config = Config { + bind: LOCALHOST, + target: target.local_addr().unwrap(), + seed: 5, + up: Profile { + delay: Duration::from_millis(20), + jitter: Duration::from_millis(10), + ..Default::default() + }, + down: Profile::default(), + }; + let shaper = Shaper::bind(Setup { + shared, + up: GAUSSIAN, + ..config.into() + }) + .await + .unwrap(); + + let clients = [ + UdpSocket::bind(LOCALHOST).await.unwrap(), + UdpSocket::bind(LOCALHOST).await.unwrap(), + ]; + for id in 0..200u32 { + let client = &clients[id as usize % 2]; + client.send_to(&id.to_be_bytes(), shaper.addr()).await.unwrap(); + } + + let mut got = Vec::new(); + let mut buf = [0u8; 4]; + while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(200), target.recv_from(&mut buf)).await { + got.push(u32::from_be_bytes(buf)); + } + got + } + + #[tokio::test] + async fn a_shared_path_keeps_the_order_across_clients() { + // Separate links keep each client's order, but not the order between them. + let apart = two_clients(false).await; + let mut sorted = apart.clone(); + sorted.sort(); + assert_eq!(sorted, (0..200).collect::>(), "a client lost datagrams"); + assert_ne!( + apart, sorted, + "separate links kept the order anyway, so this proves nothing" + ); + for parity in 0..2 { + assert!(apart.iter().filter(|&&id| id % 2 == parity).is_sorted()); + } + + let together = two_clients(true).await; + assert_eq!(together, (0..200).collect::>(), "a shared path reordered"); + } } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index eeef1c158f..e8585c23b5 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -43,6 +43,9 @@ struct Args { /// The longest a datagram waits for the rate limit before it is dropped. #[arg(long, default_value = "100ms", value_parser = humantime::parse_duration, requires = "rate")] queue: Duration, + /// Every client shares one link each way, instead of each getting its own. + #[arg(long)] + shared: bool, /// Also pipe TCP on the listening port to the target, untouched. #[arg(long)] tcp_passthrough: bool, @@ -86,6 +89,7 @@ async fn main() -> anyhow::Result<()> { }; let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { tcp_passthrough: args.tcp_passthrough, + shared: args.shared, up: options.clone(), down: options.clone(), ..config.into() @@ -100,8 +104,8 @@ async fn main() -> anyhow::Result<()> { config.seed, config.up ); - if options != moq_shaper::Options::default() { - println!("shaper: options {options:?}"); + if options != moq_shaper::Options::default() || args.shared { + println!("shaper: options {options:?}, shared {}", args.shared); } shutdown().await?; From 7e7b7290500b007222db852f4dd8db2923e5deab Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:10:34 -0400 Subject: [PATCH 07/27] feat(shaper): batches that release datagrams together, as an option A paced hop, a Wi-Fi access point or a cellular scheduler bunches traffic: nothing moves for a while, then several datagrams arrive at once. That clump is what a receiver's jitter estimate has to see, and no delay or jitter produces it, since those make datagrams late rather than together. `Options::batch` holds each direction's datagrams until `count` are waiting, or until `window` after the first of them arrived, then sends the whole batch when its latest datagram would have left, and never before the batch closed. The delivery task holds it, so on a shared path one batch fills from every client. A hold counts as `delayed`, once, and `verify` fails a run whose batches implausibly never held anything, which a batch of `count` does to all but one datagram in `count`. The CLI takes `--batch --batch-window `. Off by default. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/src/lib.rs | 447 +++++++++++++++++++++++++++++++++----- rs/moq-shaper/src/main.rs | 10 + 2 files changed, 401 insertions(+), 56 deletions(-) diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 9a915854f5..2b31df9dae 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -102,17 +102,38 @@ pub enum Jitter { Gaussian, } +/// Hold datagrams, then release them together, the way a paced hop bunches them. +/// +/// A batch closes once `count` datagrams are waiting, or `window` after the +/// first of them arrived, and everything in it leaves when the latest would. +/// That clump is what a receiver's jitter estimate sees on such a path, and no +/// delay or jitter produces it: the datagrams arrive together rather than late. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Batch { + /// How many datagrams close a batch early. + pub count: usize, + /// How long a batch waits for that many. + pub window: Duration, +} + /// One direction's opt-in options beyond its [`Profile`]. The default adds nothing. #[derive(Clone, Debug, Default, PartialEq)] pub struct Options { /// How the profile's `jitter` is drawn. pub jitter_model: Jitter, + /// Hold datagrams and release them together, if at all. + pub batch: Option, } impl Options { /// Check the options, and `profile` as they draw it. fn validate(&self, profile: &Profile) -> anyhow::Result<()> { - profile.validate(self.jitter_model) + profile.validate(self.jitter_model)?; + if let Some(batch) = &self.batch { + anyhow::ensure!(batch.count > 0, "a batch of zero datagrams never closes"); + anyhow::ensure!(!batch.window.is_zero(), "a batch with no window never holds anything"); + } + Ok(()) } } @@ -150,6 +171,18 @@ fn unapplied(config: &Config, stats: &Stats) -> Vec<&'static str> { .collect() } +/// Whether the batches `setup` configures implausibly never held a datagram. +/// +/// A batch holds every datagram but the one that fills it, so it acts on at +/// least `1 - 1 / count` of them. What it holds counts as delayed once the +/// batch leaves. +fn unbatched(setup: &Setup, stats: &Stats) -> bool { + let chance = |options: &Options| options.batch.map_or(0.0, |batch| 1.0 - 1.0 / batch.count as f64); + let silence = (1.0 - chance(&setup.up)).powf(stats.up.packets as f64) + * (1.0 - chance(&setup.down)).powf(stats.down.packets as f64); + stats.up.delayed + stats.down.delayed == 0 && silence < IMPLAUSIBLE +} + /// A token-bucket rate limit with a bounded queue behind it. #[derive(Clone, Debug, PartialEq)] pub struct Rate { @@ -234,7 +267,7 @@ pub struct Counters { pub overflowed: u64, /// Datagrams that waited for the rate limit. pub throttled: u64, - /// Datagrams given a nonzero delay. + /// Datagrams given a nonzero delay, by the profile or by a batch holding them. pub delayed: u64, /// Datagrams sent ahead of the delay, overtaking any still in flight. pub reordered: u64, @@ -349,7 +382,10 @@ impl Shaper { anyhow::bail!("the shaper stopped forwarding: {err} ({stats})"); } - let missing = unapplied(&self.setup.config, &stats); + let mut missing = unapplied(&self.setup.config, &stats); + if unbatched(&self.setup, &stats) { + missing.push("batch"); + } anyhow::ensure!( missing.is_empty(), "the profile never applied {} ({stats}), so this run was not impaired as configured", @@ -414,10 +450,7 @@ async fn run( // A shared path is one link each way, drawing the streams the first flow would. let shared = setup.shared.then(|| { let now = Instant::now(); - [ - link(&mut tasks, &config.up, &setup.up, config.seed, 0, now), - link(&mut tasks, &config.down, &setup.down, config.seed, 1, now), - ] + [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, 0, now, &tally)) }); loop { @@ -435,7 +468,7 @@ async fn run( }; let now = Instant::now(); - let stream = 2 * flows.len() as u64; + let number = flows.len() as u64; let flow = match flows.entry(from) { hash_map::Entry::Occupied(entry) => entry.into_mut(), hash_map::Entry::Vacant(entry) => { @@ -443,10 +476,7 @@ async fn run( let [up, down] = match &shared { Some(links) => links.clone(), - None => [ - link(&mut tasks, &config.up, &setup.up, config.seed, stream, now), - link(&mut tasks, &config.down, &setup.down, config.seed, stream + 1, now), - ], + None => [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, number, now, &tally)), }; tasks.spawn(reply( upstream.clone(), @@ -472,18 +502,31 @@ struct Flow { up: Arc>, } -/// A link, and the task delivering what it treats. +/// One direction of the flow numbered `flow` as a link, and the task delivering +/// what it treats. fn link( tasks: &mut JoinSet>, - profile: &Profile, - options: &Options, - seed: u64, - stream: u64, + setup: &Setup, + direction: usize, + flow: u64, now: Instant, + tally: &Arc<[Tally; 2]>, ) -> Arc> { + let (profile, options) = match direction { + UP => (&setup.config.up, &setup.up), + _ => (&setup.config.down, &setup.down), + }; let (queue, queued) = mpsc::unbounded_channel(); - tasks.spawn(deliver(queued)); - Arc::new(Mutex::new(Link::new(profile, options, seed, stream, now, queue))) + tasks.spawn(deliver(queued, options.batch, tally.clone(), direction)); + let stream = 2 * flow + direction as u64; + Arc::new(Mutex::new(Link::new( + profile, + options, + setup.config.seed, + stream, + now, + queue, + ))) } /// The next TCP connection, or never without a passthrough. @@ -539,24 +582,34 @@ async fn reply( } } -/// Send each treated datagram at its departure time. -async fn deliver(mut queue: mpsc::UnboundedReceiver) -> anyhow::Result<()> { +/// Send each treated datagram at its departure time, after its batch if the +/// link has one. +async fn deliver( + mut queue: mpsc::UnboundedReceiver, + batch: Option, + tally: Arc<[Tally; 2]>, + direction: usize, +) -> anyhow::Result<()> { // Ordered by departure, then by arrival, so ties keep their order. let mut pending = BinaryHeap::>::new(); let mut parcels = HashMap::::new(); let mut sequence = 0u64; + let mut held = Held::default(); + let mut ready = Vec::new(); loop { let next = pending.peek().map(|Reverse((at, _))| *at); + let wake = next.into_iter().chain(held.closes).min(); tokio::select! { item = queue.recv() => { // The link is gone, so the flow is too. let Some(parcel) = item else { return Ok(()) }; - pending.push(Reverse((parcel.at, sequence))); - parcels.insert(sequence, parcel); - sequence += 1; + match batch { + Some(batch) => held.hold(batch, parcel, &mut ready, &tally[direction]), + None => ready.push(parcel), + } } - _ = tokio::time::sleep_until(next.unwrap_or_else(Instant::now)), if next.is_some() => { + _ = tokio::time::sleep_until(wake.unwrap_or_else(Instant::now)), if wake.is_some() => { let now = Instant::now(); while let Some(&Reverse((at, id))) = pending.peek() { if at > now { @@ -569,14 +622,64 @@ async fn deliver(mut queue: mpsc::UnboundedReceiver) -> anyhow::Result<( // so it is not the shaper failing. let _ = parcel.socket.send_to(&parcel.datagram, parcel.dest).await; } + // The window closes a batch that never filled. + if let Some(closes) = held.closes.filter(|&closes| closes <= now) { + held.release(closes, &mut ready, &tally[direction]); + } } } + for parcel in ready.drain(..) { + pending.push(Reverse((parcel.at, sequence))); + parcels.insert(sequence, parcel); + sequence += 1; + } } } -/// A treated datagram: when it leaves, and the socket and address it leaves by. +/// The datagrams a batch is holding, and when its window closes on them. +#[derive(Default)] +struct Held { + parcels: Vec, + closes: Option, +} + +impl Held { + /// Hold `parcel`, releasing the batch into `ready` once it is full. + fn hold(&mut self, batch: Batch, parcel: Parcel, ready: &mut Vec, tally: &Tally) { + let arrived = parcel.arrived; + self.closes.get_or_insert(arrived + batch.window); + self.parcels.push(parcel); + if self.parcels.len() >= batch.count { + self.release(arrived, ready, tally); + } + } + + /// Release the batch, closed at `closed`, into `ready`: every datagram leaves + /// when the latest would, and none before the batch closed. + fn release(&mut self, closed: Instant, ready: &mut Vec, tally: &Tally) { + self.closes = None; + let Some(latest) = self.parcels.iter().map(|parcel| parcel.at).max() else { + return; + }; + let at = latest.max(closed); + for mut parcel in self.parcels.drain(..) { + // A hold is a delay, counted once whichever stage gave it. + if at > parcel.at && !parcel.delayed { + bump(&tally.delayed); + } + parcel.at = at; + ready.push(parcel); + } + } +} + +/// A treated datagram: when it arrived and leaves, and the socket and address +/// it leaves by. struct Parcel { + arrived: Instant, at: Instant, + /// Whether the link already counted it as delayed. + delayed: bool, datagram: Vec, socket: Arc, dest: SocketAddr, @@ -622,21 +725,23 @@ impl Link { /// Treat a datagram and queue it to leave by `socket` for `dest`. fn push(&mut self, now: Instant, datagram: Vec, socket: &Arc, dest: SocketAddr, tally: &Tally) { - let Some(at) = self.treat(now, datagram.len(), tally) else { + let Some((at, delayed)) = self.treat(now, datagram.len(), tally) else { return; }; // The delivery task only ends once this link is dropped. let _ = self.queue.send(Parcel { + arrived: now, at, + delayed, datagram, socket: socket.clone(), dest, }); } - /// When a datagram of `size` bytes arriving `now` leaves, or `None` when it - /// is dropped. - fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option { + /// When a datagram of `size` bytes arriving `now` leaves, and whether it was + /// counted as delayed, or `None` when it is dropped. + fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option<(Instant, bool)> { bump(&tally.packets); // Every draw happens for every datagram, whatever the profile, so one @@ -672,6 +777,7 @@ impl Link { depart = start; } + let mut delayed = false; if skip { bump(&tally.reordered); } else if self.jitter == Jitter::Gaussian { @@ -680,18 +786,19 @@ impl Link { let leave = (depart + Duration::from_secs_f64(wait.max(0.0))).max(self.floor); // Counted like the uniform model whenever a delay is configured, so // the chance `verify` holds it to is the same. - if !self.profile.delay.is_zero() || leave > depart { - bump(&tally.delayed); - } + delayed = !self.profile.delay.is_zero() || leave > depart; self.floor = leave; depart = leave; } else if !self.profile.delay.is_zero() { let jitter = self.profile.jitter.as_secs_f64() * spread; depart += Duration::from_secs_f64(self.profile.delay.as_secs_f64() + jitter); + delayed = true; + } + if delayed { bump(&tally.delayed); } - Some(depart) + Some((depart, delayed)) } } @@ -1023,7 +1130,7 @@ mod tests { let mut link = Link::new(profile, options, 7, 0, now, queue); let tally = Tally::default(); let mut sent: Vec<(u32, Instant)> = (0..count) - .filter_map(|id| Some((id, link.treat(now + spacing * id, 4, &tally)?))) + .filter_map(|id| Some((id, link.treat(now + spacing * id, 4, &tally)?.0))) .collect(); // Ties leave in arrival order, which is id order here. sent.sort_by_key(|&(id, at)| (at, id)); @@ -1032,6 +1139,7 @@ mod tests { const GAUSSIAN: Options = Options { jitter_model: Jitter::Gaussian, + batch: None, }; #[tokio::test] @@ -1169,50 +1277,63 @@ mod tests { shaper.verify().unwrap(); } - /// Two clients take turns sending numbered datagrams up one gaussian path, - /// and the target's arrival order comes back. - async fn two_clients(shared: bool) -> Vec { + /// Two clients take turns sending `count` numbered datagrams up the path + /// `setup` builds, and what reaches the target comes back, in order, with + /// how long after the first send it did. + async fn two_clients(setup: impl FnOnce(Config) -> Setup, count: u32) -> Vec<(u32, Duration)> { let target = UdpSocket::bind(LOCALHOST).await.unwrap(); let config = Config { bind: LOCALHOST, target: target.local_addr().unwrap(), seed: 5, - up: Profile { - delay: Duration::from_millis(20), - jitter: Duration::from_millis(10), - ..Default::default() - }, + up: Profile::default(), down: Profile::default(), }; - let shaper = Shaper::bind(Setup { - shared, - up: GAUSSIAN, - ..config.into() - }) - .await - .unwrap(); + let shaper = Shaper::bind(setup(config)).await.unwrap(); let clients = [ UdpSocket::bind(LOCALHOST).await.unwrap(), UdpSocket::bind(LOCALHOST).await.unwrap(), ]; - for id in 0..200u32 { + let start = Instant::now(); + for id in 0..count { let client = &clients[id as usize % 2]; client.send_to(&id.to_be_bytes(), shaper.addr()).await.unwrap(); } let mut got = Vec::new(); let mut buf = [0u8; 4]; - while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(200), target.recv_from(&mut buf)).await { - got.push(u32::from_be_bytes(buf)); + while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(500), target.recv_from(&mut buf)).await { + got.push((u32::from_be_bytes(buf), start.elapsed())); } got } #[tokio::test] async fn a_shared_path_keeps_the_order_across_clients() { + let jittery = Profile { + delay: Duration::from_millis(20), + jitter: Duration::from_millis(10), + ..Default::default() + }; + let order = |shared: bool| { + let up = jittery.clone(); + async move { + let got = two_clients( + |config| Setup { + shared, + up: GAUSSIAN, + ..Config { up, ..config }.into() + }, + 200, + ) + .await; + got.into_iter().map(|(id, _)| id).collect::>() + } + }; + // Separate links keep each client's order, but not the order between them. - let apart = two_clients(false).await; + let apart = order(false).await; let mut sorted = apart.clone(); sorted.sort(); assert_eq!(sorted, (0..200).collect::>(), "a client lost datagrams"); @@ -1224,7 +1345,221 @@ mod tests { assert!(apart.iter().filter(|&&id| id % 2 == parity).is_sorted()); } - let together = two_clients(true).await; - assert_eq!(together, (0..200).collect::>(), "a shared path reordered"); + assert_eq!( + order(true).await, + (0..200).collect::>(), + "a shared path reordered" + ); + } + + /// Send `count` numbered datagrams `spacing` apart, and collect what echoes + /// back with when it did, alongside when each was sent. + async fn paced(client: &UdpSocket, count: u32, spacing: Duration) -> (Vec, Vec<(u32, Instant)>) { + let send = async { + let mut sent = Vec::new(); + for id in 0..count { + sent.push(Instant::now()); + client.send(&id.to_be_bytes()).await.unwrap(); + tokio::time::sleep(spacing).await; + } + sent + }; + let receive = async { + let mut got = Vec::new(); + let mut buf = [0u8; 4]; + while let Ok(Ok(_)) = tokio::time::timeout(Duration::from_millis(500), client.recv(&mut buf)).await { + got.push((u32::from_be_bytes(buf), Instant::now())); + } + got + }; + tokio::join!(send, receive) + } + + fn batched(count: usize, window: Duration) -> Options { + Options { + batch: Some(Batch { count, window }), + ..Default::default() + } + } + + #[tokio::test] + async fn a_batch_releases_on_count_and_on_the_window() { + let socket = Arc::new(UdpSocket::bind(LOCALHOST).await.unwrap()); + let ms = |ms| Duration::from_millis(ms); + let now = Instant::now(); + let parcel = |at: Instant| Parcel { + arrived: at, + at, + delayed: false, + datagram: Vec::new(), + socket: socket.clone(), + dest: LOCALHOST, + }; + let batch = Batch { + count: 3, + window: ms(160), + }; + let tally = Tally::default(); + let mut held = Held::default(); + let mut ready = Vec::new(); + + held.hold(batch, parcel(now), &mut ready, &tally); + held.hold(batch, parcel(now + ms(10)), &mut ready, &tally); + assert!(ready.is_empty(), "a partial batch leaked"); + assert_eq!( + held.closes, + Some(now + ms(160)), + "the window counts from the first arrival" + ); + + held.hold(batch, parcel(now + ms(20)), &mut ready, &tally); + let leaves: Vec = ready.drain(..).map(|parcel| parcel.at).collect(); + assert_eq!( + leaves, + [now + ms(20); 3], + "a full batch leaves together, as the latest would" + ); + assert_eq!( + tally.snapshot().delayed, + 2, + "the datagram that filled it waited for nothing" + ); + + // The window closes a batch that never fills, and nothing in it leaves sooner. + held.hold(batch, parcel(now + ms(30)), &mut ready, &tally); + let closes = held.closes.unwrap(); + assert_eq!(closes, now + ms(190)); + held.release(closes, &mut ready, &tally); + assert_eq!(ready.iter().map(|parcel| parcel.at).collect::>(), [closes]); + assert_eq!(tally.snapshot().delayed, 3); + } + + #[tokio::test] + async fn a_batch_releases_datagrams_together() { + let (shaper, client) = shaped(|config| Setup { + up: batched(7, Duration::from_millis(160)), + ..config.into() + }) + .await; + + // Three full batches, each filled well inside its window. + let (_, got) = paced(&client, 21, Duration::from_millis(10)).await; + assert_eq!( + got.iter().map(|&(id, _)| id).collect::>(), + (0..21).collect::>() + ); + + let batches: Vec<&[(u32, Instant)]> = got.chunks(7).collect(); + for batch in &batches { + let spread = batch[6].1 - batch[0].1; + assert!( + spread < Duration::from_millis(15), + "a batch arrived spread over {spread:?}" + ); + } + for pair in batches.windows(2) { + let gap = pair[1][0].1 - pair[0][0].1; + assert!(gap > Duration::from_millis(40), "two batches arrived {gap:?} apart"); + } + + let stats = shaper.verify().unwrap(); + assert_eq!( + stats.up.delayed, 18, + "every datagram but the last of each batch waits: {stats}" + ); + } + + #[tokio::test] + async fn the_window_releases_a_batch_that_never_fills() { + let (shaper, client) = shaped(|config| Setup { + up: batched(100, Duration::from_millis(80)), + ..config.into() + }) + .await; + + let (sent, got) = paced(&client, 3, Duration::ZERO).await; + assert_eq!(got.len(), 3); + let held = got[0].1 - sent[0]; + assert!(held >= Duration::from_millis(80), "the batch left after only {held:?}"); + assert_eq!(shaper.verify().unwrap().up.delayed, 3); + } + + #[tokio::test] + async fn a_shared_batch_fills_from_every_client() { + // Three datagrams from each of two clients, into batches of six: only a + // shared path fills one, and separate links wait out the window. + let window = Duration::from_millis(300); + for shared in [true, false] { + let got = two_clients( + |config| Setup { + shared, + up: batched(6, window), + ..config.into() + }, + 6, + ) + .await; + assert_eq!(got.len(), 6); + let last = got.iter().map(|&(_, after)| after).max().unwrap(); + assert_eq!( + last < window / 2, + shared, + "shared {shared}: the batch left after {last:?}" + ); + } + } + + #[tokio::test] + async fn a_batch_needs_a_count_and_a_window() { + let refused = |options: Options, why: &'static str| async move { + let config = Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: Profile::default(), + down: Profile::default(), + }; + let err = Shaper::bind(Setup { + up: options, + ..config.into() + }) + .await + .err() + .unwrap_or_else(|| panic!("accepted a batch where {why}")); + assert!(format!("{err:#}").contains(why), "{err:#}"); + }; + refused(batched(0, Duration::from_millis(10)), "never closes").await; + refused(batched(7, Duration::ZERO), "never holds").await; + } + + #[test] + fn a_batch_that_never_held_anything_is_unapplied() { + let config = Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: Profile::default(), + down: Profile::default(), + }; + let setup = Setup { + up: batched(7, Duration::from_millis(160)), + ..config.into() + }; + let quiet = |packets| Stats { + up: Counters { + packets, + ..Default::default() + }, + ..Default::default() + }; + + // (1/7)^2 is 2%: two datagrams through batches of seven prove nothing. + assert!(!unbatched(&setup, &quiet(2))); + // (1/7)^10 is 4e-9: ten datagrams a batch never held means no batch. + assert!(unbatched(&setup, &quiet(10))); + + let mut held = quiet(10); + held.up.delayed = 6; + assert!(!unbatched(&setup, &held)); } } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index e8585c23b5..6de67ba9e3 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -43,6 +43,12 @@ struct Args { /// The longest a datagram waits for the rate limit before it is dropped. #[arg(long, default_value = "100ms", value_parser = humantime::parse_duration, requires = "rate")] queue: Duration, + /// Hold datagrams until this many are waiting, then release them together. + #[arg(long, requires = "batch_window")] + batch: Option, + /// The longest a batch waits to fill before it leaves anyway. + #[arg(long, value_parser = humantime::parse_duration, requires = "batch")] + batch_window: Option, /// Every client shares one link each way, instead of each getting its own. #[arg(long)] shared: bool, @@ -86,6 +92,10 @@ async fn main() -> anyhow::Result<()> { JitterModel::Uniform => moq_shaper::Jitter::Uniform, JitterModel::Gaussian => moq_shaper::Jitter::Gaussian, }, + batch: args + .batch + .zip(args.batch_window) + .map(|(count, window)| moq_shaper::Batch { count, window }), }; let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { tcp_passthrough: args.tcp_passthrough, From b6230c5519b2b6b8778498de0ad6ded892945dd5 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:13:41 -0400 Subject: [PATCH 08/27] feat(shaper): steps that change a profile part-way through a run A path that stays one shape for a whole run cannot show how a receiver adapts when it changes: a delay that grows, a rate that narrows, loss that comes and goes. A steady profile measures convergence only. `Options::steps` is a list of `Step`s, each at a time into the run, naming the delay, jitter, loss, reorder or rate limit it changes. A step changes only what it names, so a later one puts a knob back without restating the rest. Steps count from when forwarding started, so every flow steps at the same moment. A rate step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there starts full. `bind` refuses steps out of order, a step that changes nothing, and any step that leaves the profile invalid. Steps have no CLI flags; a profile file names them. Off by default. Links now start their bucket and their order floor at the run's start rather than their first datagram. The bucket reads `full_at.max(now)` and the floor only ever rises, so an earlier start changes nothing. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/src/lib.rs | 351 ++++++++++++++++++++++++++++++++++++-- rs/moq-shaper/src/main.rs | 1 + 2 files changed, 339 insertions(+), 13 deletions(-) diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 2b31df9dae..90b01f777e 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -116,6 +116,48 @@ pub struct Batch { pub window: Duration, } +/// A change to one direction's profile once the run reaches `at`. +/// +/// A step changes only what it names, so a later step puts one knob back +/// without restating the rest. It can add or change a rate limit, never remove +/// one. +#[derive(Clone, Debug, Default, PartialEq)] +pub struct Step { + /// How far into the run the change happens. + pub at: Duration, + /// The delay from then on. + pub delay: Option, + /// The jitter from then on. + pub jitter: Option, + /// The loss from then on. + pub loss: Option, + /// The reorder from then on. + pub reorder: Option, + /// The rate limit from then on. + pub rate: Option, +} + +impl Step { + /// Change what this step names in `profile`. + fn apply(&self, profile: &mut Profile) { + if let Some(delay) = self.delay { + profile.delay = delay; + } + if let Some(jitter) = self.jitter { + profile.jitter = jitter; + } + if let Some(loss) = self.loss { + profile.loss = loss; + } + if let Some(reorder) = self.reorder { + profile.reorder = reorder; + } + if let Some(rate) = &self.rate { + profile.rate = Some(rate.clone()); + } + } +} + /// One direction's opt-in options beyond its [`Profile`]. The default adds nothing. #[derive(Clone, Debug, Default, PartialEq)] pub struct Options { @@ -123,16 +165,44 @@ pub struct Options { pub jitter_model: Jitter, /// Hold datagrams and release them together, if at all. pub batch: Option, + /// Changes to the profile part-way through the run, earliest first. + pub steps: Vec, } impl Options { - /// Check the options, and `profile` as they draw it. + /// Check the options, and `profile` as they draw it, after every step too. fn validate(&self, profile: &Profile) -> anyhow::Result<()> { profile.validate(self.jitter_model)?; if let Some(batch) = &self.batch { anyhow::ensure!(batch.count > 0, "a batch of zero datagrams never closes"); anyhow::ensure!(!batch.window.is_zero(), "a batch with no window never holds anything"); } + + let mut profile = profile.clone(); + let mut previous: Option = None; + for step in &self.steps { + // A link only ever looks at the next step due, so one out of order + // would be skipped without a word. + anyhow::ensure!( + previous.is_none_or(|previous| step.at > previous), + "the step at {:?} comes after the one at {previous:?}; steps go in order", + step.at + ); + // A step naming nothing is a profile that thinks it changes and does not. + anyhow::ensure!( + Step { + at: step.at, + ..Step::default() + } != *step, + "the step at {:?} changes nothing", + step.at + ); + step.apply(&mut profile); + profile + .validate(self.jitter_model) + .with_context(|| format!("after the step at {:?}", step.at))?; + previous = Some(step.at); + } Ok(()) } } @@ -447,11 +517,13 @@ async fn run( let mut tasks = JoinSet::new(); let mut buf = vec![0u8; u16::MAX as usize]; + // What a step's `at` counts from. + let start = Instant::now(); + // A shared path is one link each way, drawing the streams the first flow would. - let shared = setup.shared.then(|| { - let now = Instant::now(); - [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, 0, now, &tally)) - }); + let shared = setup + .shared + .then(|| [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, 0, start, &tally))); loop { let (size, from) = tokio::select! { @@ -476,7 +548,7 @@ async fn run( let [up, down] = match &shared { Some(links) => links.clone(), - None => [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, number, now, &tally)), + None => [UP, DOWN].map(|direction| link(&mut tasks, &setup, direction, number, start, &tally)), }; tasks.spawn(reply( upstream.clone(), @@ -509,7 +581,7 @@ fn link( setup: &Setup, direction: usize, flow: u64, - now: Instant, + start: Instant, tally: &Arc<[Tally; 2]>, ) -> Arc> { let (profile, options) = match direction { @@ -524,7 +596,7 @@ fn link( options, setup.config.seed, stream, - now, + start, queue, ))) } @@ -695,6 +767,10 @@ struct Link { full_at: Instant, /// When the latest in-order datagram leaves, which the next one never beats. floor: Instant, + /// When the run started, which a step's `at` counts from. + start: Instant, + /// The steps still to come, soonest last. + steps: Vec, queue: mpsc::UnboundedSender, } @@ -708,21 +784,48 @@ impl Link { options: &Options, seed: u64, stream: u64, - now: Instant, + start: Instant, queue: mpsc::UnboundedSender, ) -> Self { Self { profile: profile.clone(), jitter: options.jitter_model, rng: Xoshiro256PlusPlus::seed_from_u64(seed ^ stream.wrapping_mul(0x9E37_79B9_7F4A_7C15)), - // Not the moment the link is built, which is after `now`: that would - // make the first datagram queue behind a bucket still refilling. - full_at: now, - floor: now, + // Full since the run started, never since the link was built, which + // is after its first datagram arrived: that would make the datagram + // queue behind a bucket still refilling. + full_at: start, + floor: start, + start, + steps: options.steps.iter().rev().cloned().collect(), queue, } } + /// Apply every step the run has reached by `now`, in order. + fn step(&mut self, now: Instant) { + while self.steps.last().is_some_and(|step| self.start + step.at <= now) { + let step = self.steps.pop().expect("a step is due"); + if let Some(rate) = &step.rate { + self.full_at = self.refilled(rate, now); + } + step.apply(&mut self.profile); + } + } + + /// When `rate`'s bucket would be full again if it took over at `now`. + /// + /// A limit that was not there starts full, the way the run started. One + /// that was keeps its credit, or its debt, clipped to the new bucket. + fn refilled(&self, rate: &Rate, now: Instant) -> Instant { + let Some(old) = &self.profile.rate else { + return now; + }; + let owed = self.full_at.saturating_duration_since(now).as_secs_f64() * old.bits_per_second as f64 / 8.0; + let credit = (old.burst as f64 - owed).min(rate.burst as f64); + now + Duration::from_secs_f64((rate.burst as f64 - credit) * 8.0 / rate.bits_per_second as f64) + } + /// Treat a datagram and queue it to leave by `socket` for `dest`. fn push(&mut self, now: Instant, datagram: Vec, socket: &Arc, dest: SocketAddr, tally: &Tally) { let Some((at, delayed)) = self.treat(now, datagram.len(), tally) else { @@ -742,6 +845,7 @@ impl Link { /// When a datagram of `size` bytes arriving `now` leaves, and whether it was /// counted as delayed, or `None` when it is dropped. fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option<(Instant, bool)> { + self.step(now); bump(&tally.packets); // Every draw happens for every datagram, whatever the profile, so one @@ -1140,6 +1244,7 @@ mod tests { const GAUSSIAN: Options = Options { jitter_model: Jitter::Gaussian, batch: None, + steps: Vec::new(), }; #[tokio::test] @@ -1562,4 +1667,224 @@ mod tests { held.up.delayed = 6; assert!(!unbatched(&setup, &held)); } + + /// How long after `at` each of `sizes` leaves one link, fed at the same instant. + fn owed(link: &mut Link, at: Instant, sizes: &[usize]) -> Vec { + let tally = Tally::default(); + sizes + .iter() + .map(|&size| link.treat(at, size, &tally).expect("dropped").0 - at) + .collect() + } + + fn stepped(steps: Vec) -> Options { + Options { + steps, + ..Default::default() + } + } + + #[test] + fn each_step_applies_at_its_own_time() { + let ms = Duration::from_millis; + let profile = Profile { + delay: ms(5), + ..Default::default() + }; + let options = stepped(vec![ + Step { + at: ms(100), + delay: Some(ms(60)), + ..Default::default() + }, + Step { + at: ms(200), + delay: Some(ms(5)), + ..Default::default() + }, + ]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&profile, &options, 7, 0, start, queue); + + assert_eq!(owed(&mut link, start, &[16]), [ms(5)]); + assert_eq!(owed(&mut link, start + ms(100), &[16]), [ms(60)]); + // The step back restores what the run opened with. + assert_eq!(owed(&mut link, start + ms(200), &[16]), [ms(5)]); + } + + #[test] + fn a_loss_step_turns_the_loss_on_and_off_again() { + let secs = Duration::from_secs; + let options = stepped(vec![ + Step { + at: secs(30), + loss: Some(1.0), + ..Default::default() + }, + Step { + at: secs(60), + loss: Some(0.0), + ..Default::default() + }, + ]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&Profile::default(), &options, 7, 0, start, queue); + let tally = Tally::default(); + let mut lost = |at: Instant| (0..100).filter(|_| link.treat(at, 16, &tally).is_none()).count(); + + assert_eq!(lost(start), 0, "the profile opens clean"); + assert_eq!(lost(start + secs(30)), 100, "the step lost nothing"); + assert_eq!(lost(start + secs(60)), 0, "the step back never cleared"); + } + + #[test] + fn a_rate_step_narrows_the_bucket_and_widens_it_again() { + let secs = Duration::from_secs; + let rate = |bytes_per_second: u64| Rate { + bits_per_second: bytes_per_second * 8, + burst: 100, + queue: secs(10), + }; + let profile = Profile { + rate: Some(rate(4000)), + ..Default::default() + }; + let options = stepped(vec![ + Step { + at: secs(30), + rate: Some(rate(1000)), + ..Default::default() + }, + Step { + at: secs(60), + rate: Some(rate(4000)), + ..Default::default() + }, + ]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&profile, &options, 7, 0, start, queue); + + // The bucket is full at each stretch, so the first 100 bytes leave at + // once and the next 100 owe one bucket's refill at the rate in force. + let ms = Duration::from_millis; + assert_eq!(owed(&mut link, start, &[100, 100]), [ms(0), ms(25)]); + assert_eq!(owed(&mut link, start + secs(30), &[100, 100]), [ms(0), ms(100)]); + assert_eq!(owed(&mut link, start + secs(60), &[100, 100]), [ms(0), ms(25)]); + } + + #[test] + fn a_rate_step_keeps_the_debt_it_takes_over() { + let ms = Duration::from_millis; + let rate = |bytes_per_second: u64| Rate { + bits_per_second: bytes_per_second * 8, + burst: 100, + queue: Duration::from_secs(10), + }; + let profile = Profile { + rate: Some(rate(1000)), + ..Default::default() + }; + // Halfway through a 300 byte backlog at 1000 bytes a second, the rate doubles. + let options = stepped(vec![Step { + at: ms(100), + rate: Some(rate(2000)), + ..Default::default() + }]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&profile, &options, 7, 0, start, queue); + assert_eq!(owed(&mut link, start, &[100, 100, 100]), [ms(0), ms(100), ms(200)]); + + // 100 bytes of the backlog are still owed, and now drain in 50ms, so + // a fresh 100 bytes leave once those and themselves are paid. + assert_eq!(owed(&mut link, start + ms(100), &[100]), [ms(100)]); + } + + #[tokio::test] + async fn a_step_gets_worse_part_way_through() { + let ms = Duration::from_millis; + let (_shaper, client) = shaped(|config| Setup { + up: stepped(vec![Step { + at: ms(150), + delay: Some(ms(60)), + ..Default::default() + }]), + ..Config { + up: Profile { + delay: ms(5), + ..Default::default() + }, + ..config + } + .into() + }) + .await; + + let (sent, got) = paced(&client, 40, ms(10)).await; + assert_eq!(got.len(), 40); + let median = |range: std::ops::Range| { + let mut latency: Vec = got[range].iter().map(|&(id, at)| at - sent[id as usize]).collect(); + latency.sort(); + latency[latency.len() / 2] + }; + + // The first ten leave well inside the first 150ms, the last fifteen well after. + let (before, after) = (median(0..10), median(25..40)); + assert!( + after > before + ms(30), + "median latency went from {before:?} to {after:?}" + ); + } + + #[tokio::test] + async fn steps_that_would_be_skipped_or_do_nothing_are_refused() { + let refused = |steps: Vec, why: &'static str| async move { + let config = Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: Profile { + delay: Duration::from_millis(20), + ..Default::default() + }, + down: Profile::default(), + }; + let err = Shaper::bind(Setup { + down: stepped(steps), + ..config.into() + }) + .await + .err() + .unwrap_or_else(|| panic!("accepted steps where {why}")); + assert!(format!("{err:#}").contains(why), "{err:#}"); + }; + let at = |secs| Step { + at: Duration::from_secs(secs), + loss: Some(0.1), + ..Default::default() + }; + + refused(vec![at(60), at(30)], "steps go in order").await; + refused( + vec![Step { + at: Duration::from_secs(30), + ..Default::default() + }], + "changes nothing", + ) + .await; + // Uniform jitter past the delay, reached by a step rather than at the start. + refused( + vec![Step { + at: Duration::from_secs(30), + jitter: Some(Duration::from_millis(10)), + ..Default::default() + }], + "exceeds delay", + ) + .await; + } } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index 6de67ba9e3..6b467067b0 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -96,6 +96,7 @@ async fn main() -> anyhow::Result<()> { .batch .zip(args.batch_window) .map(|(count, window)| moq_shaper::Batch { count, window }), + ..Default::default() }; let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { tcp_passthrough: args.tcp_passthrough, From f56f47780f00c1e93f352fb545bcce16816f939e Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:18:03 -0400 Subject: [PATCH 09/27] feat(shaper): named profiles for both directions, loaded from TOML A run that has to be repeated, compared, or graded against a budget needs its path written down under a name, not rebuilt from flags each time, and the flags cannot say everything anyway: they apply one profile both ways and have no steps. `Preset::load` reads a built-in profile by name, or a TOML file by path, and `Preset::setup` turns it into a `Setup` for an address pair. A file names itself and its seed, says whether the path is shared, and gives each direction one table holding its profile and options: delay, jitter, jitter_model, loss, reorder, rate, batch and steps. Unknown fields are refused, and so is a direction whose delay varies without naming its jitter_model, since whether datagrams can overtake each other then depends on it. The CLI takes `--profile ` in place of the flags that shape the path, with `--seed` still overriding the file's. The built-ins are embedded from profiles/: near-zero, mild, bursty, step and high-rtt, the audio-quality lane's paths, which keep their numbers, their in-order gaussian jitter and their shared path; and lossy, 2% loss with 1% of datagrams skipping a 20ms delay, since a reorder here overtakes by skipping the delay rather than by adding one. `Rate`, `Jitter`, `Batch` and `Step` derive `Deserialize`, and the crate now depends on serde, toml and humantime-serde. Cargo.lock gains only those edges and humantime-serde 1.1.1, the version it had before. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 13 ++ rs/moq-shaper/Cargo.toml | 3 + rs/moq-shaper/profiles/bursty.toml | 16 ++ rs/moq-shaper/profiles/high-rtt.toml | 15 ++ rs/moq-shaper/profiles/lossy.toml | 16 ++ rs/moq-shaper/profiles/mild.toml | 15 ++ rs/moq-shaper/profiles/near-zero.toml | 7 + rs/moq-shaper/profiles/step.toml | 24 +++ rs/moq-shaper/src/lib.rs | 25 ++- rs/moq-shaper/src/main.rs | 82 ++++++--- rs/moq-shaper/src/preset.rs | 251 ++++++++++++++++++++++++++ 11 files changed, 438 insertions(+), 29 deletions(-) create mode 100644 rs/moq-shaper/profiles/bursty.toml create mode 100644 rs/moq-shaper/profiles/high-rtt.toml create mode 100644 rs/moq-shaper/profiles/lossy.toml create mode 100644 rs/moq-shaper/profiles/mild.toml create mode 100644 rs/moq-shaper/profiles/near-zero.toml create mode 100644 rs/moq-shaper/profiles/step.toml create mode 100644 rs/moq-shaper/src/preset.rs diff --git a/Cargo.lock b/Cargo.lock index db0d69c052..2ffc9d3146 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3036,6 +3036,16 @@ version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15" +[[package]] +name = "humantime-serde" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c" +dependencies = [ + "humantime", + "serde", +] + [[package]] name = "hybrid-array" version = "0.4.15" @@ -4713,8 +4723,11 @@ dependencies = [ "anyhow", "clap", "humantime", + "humantime-serde", "rand 0.10.3", + "serde", "tokio", + "toml 1.1.6+spec-1.1.0", ] [[package]] diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index ea617a976d..36add548dd 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -20,5 +20,8 @@ doc = false anyhow = { workspace = true } clap = { workspace = true } humantime = { workspace = true } +humantime-serde = { workspace = true } rand = { workspace = true } +serde = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +toml = { workspace = true } diff --git a/rs/moq-shaper/profiles/bursty.toml b/rs/moq-shaper/profiles/bursty.toml new file mode 100644 index 0000000000..307c8f2f4e --- /dev/null +++ b/rs/moq-shaper/profiles/bursty.toml @@ -0,0 +1,16 @@ +# A paced hop that bunches traffic: nothing moves until seven datagrams are +# waiting or the window closes, and then they all leave together. +# +# This is the shape a receiver's jitter estimate has to see, and no delay-and- +# spread profile produces it: the datagrams arrive together rather than late. +name = "bursty" +seed = 3 +shared = true + +[up.batch] +count = 7 +window = "160ms" + +[down.batch] +count = 7 +window = "160ms" diff --git a/rs/moq-shaper/profiles/high-rtt.toml b/rs/moq-shaper/profiles/high-rtt.toml new file mode 100644 index 0000000000..1f600fbbf3 --- /dev/null +++ b/rs/moq-shaper/profiles/high-rtt.toml @@ -0,0 +1,15 @@ +# An intercontinental path: 75ms each way with a wide spread on top, queued in +# order. +name = "high-rtt" +seed = 5 +shared = true + +[up] +delay = "75ms" +jitter = "30ms" +jitter_model = "gaussian" + +[down] +delay = "75ms" +jitter = "30ms" +jitter_model = "gaussian" diff --git a/rs/moq-shaper/profiles/lossy.toml b/rs/moq-shaper/profiles/lossy.toml new file mode 100644 index 0000000000..816063a752 --- /dev/null +++ b/rs/moq-shaper/profiles/lossy.toml @@ -0,0 +1,16 @@ +# A path that loses 2%, and on which 1% of what survives skips the 20ms delay +# the rest take, arriving ahead of what is still in flight: the loss and the +# reordering a recovery path has to cope with. +name = "lossy" +seed = 6 +shared = true + +[up] +delay = "20ms" +loss = 0.02 +reorder = 0.01 + +[down] +delay = "20ms" +loss = 0.02 +reorder = 0.01 diff --git a/rs/moq-shaper/profiles/mild.toml b/rs/moq-shaper/profiles/mild.toml new file mode 100644 index 0000000000..de101bb5bf --- /dev/null +++ b/rs/moq-shaper/profiles/mild.toml @@ -0,0 +1,15 @@ +# A healthy wired LAN: a few milliseconds of delay with the same spread again, +# queued in order. +name = "mild" +seed = 2 +shared = true + +[up] +delay = "5ms" +jitter = "5ms" +jitter_model = "gaussian" + +[down] +delay = "5ms" +jitter = "5ms" +jitter_model = "gaussian" diff --git a/rs/moq-shaper/profiles/near-zero.toml b/rs/moq-shaper/profiles/near-zero.toml new file mode 100644 index 0000000000..627f619487 --- /dev/null +++ b/rs/moq-shaper/profiles/near-zero.toml @@ -0,0 +1,7 @@ +# The control: the shaper sits in the path but treats nothing. +# +# Every other profile is graded against this one, and a harness asserting that an +# impairment applied needs a profile where `delayed == 0` is the right answer. +name = "near-zero" +seed = 1 +shared = true diff --git a/rs/moq-shaper/profiles/step.toml b/rs/moq-shaper/profiles/step.toml new file mode 100644 index 0000000000..bdd0170d2f --- /dev/null +++ b/rs/moq-shaper/profiles/step.toml @@ -0,0 +1,24 @@ +# A path that gets worse mid-run: 5ms for the first thirty seconds, then 60ms. +# +# An estimator that only ever converges downward passes every steady profile and +# fails this one. It never recovers, which is what makes it the baseline a +# recovery profile is read against. +name = "step" +seed = 4 +shared = true + +[up] +delay = "5ms" +jitter_model = "gaussian" + +[[up.steps]] +at = "30s" +delay = "60ms" + +[down] +delay = "5ms" +jitter_model = "gaussian" + +[[down.steps]] +at = "30s" +delay = "60ms" diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 90b01f777e..c3c471abdd 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -26,8 +26,13 @@ use std::{ time::Duration, }; +mod preset; + +pub use preset::Preset; + use anyhow::Context; use rand::{RngExt, SeedableRng, rngs::Xoshiro256PlusPlus}; +use serde::Deserialize; use tokio::{ net::{TcpListener, TcpStream, UdpSocket}, sync::mpsc, @@ -87,7 +92,8 @@ impl Profile { } /// How a direction draws each datagram's jitter. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum Jitter { /// Uniform within `jitter` either way of `delay`, drawn for each datagram on /// its own, so a later datagram can overtake an earlier one. @@ -108,11 +114,13 @@ pub enum Jitter { /// first of them arrived, and everything in it leaves when the latest would. /// That clump is what a receiver's jitter estimate sees on such a path, and no /// delay or jitter produces it: the datagrams arrive together rather than late. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Batch { /// How many datagrams close a batch early. pub count: usize, /// How long a batch waits for that many. + #[serde(with = "humantime_serde")] pub window: Duration, } @@ -121,19 +129,26 @@ pub struct Batch { /// A step changes only what it names, so a later step puts one knob back /// without restating the rest. It can add or change a rate limit, never remove /// one. -#[derive(Clone, Debug, Default, PartialEq)] +#[derive(Clone, Debug, Default, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Step { /// How far into the run the change happens. + #[serde(with = "humantime_serde")] pub at: Duration, /// The delay from then on. + #[serde(default, with = "humantime_serde")] pub delay: Option, /// The jitter from then on. + #[serde(default, with = "humantime_serde")] pub jitter: Option, /// The loss from then on. + #[serde(default)] pub loss: Option, /// The reorder from then on. + #[serde(default)] pub reorder: Option, /// The rate limit from then on. + #[serde(default)] pub rate: Option, } @@ -254,13 +269,15 @@ fn unbatched(setup: &Setup, stats: &Stats) -> bool { } /// A token-bucket rate limit with a bounded queue behind it. -#[derive(Clone, Debug, PartialEq)] +#[derive(Clone, Debug, PartialEq, Deserialize)] +#[serde(deny_unknown_fields)] pub struct Rate { /// The sustained rate. pub bits_per_second: u64, /// How many bytes may go out ahead of the sustained rate after an idle spell. pub burst: u64, /// The longest a datagram waits for the bucket; one that would wait longer is dropped. + #[serde(with = "humantime_serde")] pub queue: Duration, } diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index 6b467067b0..d05c07c323 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -1,7 +1,9 @@ //! Put a seeded, impaired UDP path in front of a server. //! -//! Both directions get the same profile. The seed and profile print at start, -//! and the counters at exit, which fails if the profile never acted. +//! Both directions get the same profile from the flags, unless `--profile` +//! names a profile file, which sets each direction on its own. The seed and +//! profile print at start, and the counters at exit, which fails if the profile +//! never acted. use std::{net::SocketAddr, time::Duration}; @@ -16,9 +18,19 @@ struct Args { /// The address every datagram is forwarded to. #[arg(long)] target: SocketAddr, - /// Seeds every treatment decision; random when omitted. + /// Seeds every treatment decision; the profile's own, or random, when omitted. #[arg(long)] seed: Option, + /// A built-in profile's name, or a profile TOML file, in place of the flags + /// that shape the path. + #[arg( + long, + conflicts_with_all = [ + "delay", "jitter", "jitter_model", "loss", "reorder", "rate", "burst", "queue", "batch", "batch_window", + "shared", + ], + )] + profile: Option, /// The base one-way delay, e.g. `20ms`. #[arg(long, default_value = "0s", value_parser = humantime::parse_duration)] delay: Duration, @@ -69,6 +81,46 @@ enum JitterModel { async fn main() -> anyhow::Result<()> { let args = Args::parse(); + let (name, mut setup) = match &args.profile { + Some(profile) => { + let preset = moq_shaper::Preset::load(profile)?; + let mut setup = preset.setup(args.listen, args.target); + if let Some(seed) = args.seed { + setup.config.seed = seed; + } + (Some(preset.name), setup) + } + None => (None, flags(&args)), + }; + setup.tcp_passthrough = args.tcp_passthrough; + + let treatment = match &name { + Some(name) => format!("profile {name}"), + None if setup.shared || setup.up != moq_shaper::Options::default() => format!( + "profile {:?}, options {:?}, shared {}", + setup.config.up, setup.up, setup.shared + ), + None => format!("profile {:?}", setup.config.up), + }; + let shaper = moq_shaper::Shaper::bind(setup).await?; + + let config = shaper.config(); + println!( + "shaper: {} -> {}, seed {}, {treatment}", + shaper.addr(), + config.target, + config.seed + ); + + shutdown().await?; + + let stats = shaper.verify()?; + println!("shaper: {stats}"); + Ok(()) +} + +/// The setup the flags describe: one profile and one set of options, both ways. +fn flags(args: &Args) -> moq_shaper::Setup { let profile = moq_shaper::Profile { delay: args.delay, jitter: args.jitter, @@ -98,32 +150,12 @@ async fn main() -> anyhow::Result<()> { .map(|(count, window)| moq_shaper::Batch { count, window }), ..Default::default() }; - let shaper = moq_shaper::Shaper::bind(moq_shaper::Setup { - tcp_passthrough: args.tcp_passthrough, + moq_shaper::Setup { shared: args.shared, up: options.clone(), - down: options.clone(), + down: options, ..config.into() - }) - .await?; - - let config = shaper.config(); - println!( - "shaper: {} -> {}, seed {}, profile {:?}", - shaper.addr(), - config.target, - config.seed, - config.up - ); - if options != moq_shaper::Options::default() || args.shared { - println!("shaper: options {options:?}, shared {}", args.shared); } - - shutdown().await?; - - let stats = shaper.verify()?; - println!("shaper: {stats}"); - Ok(()) } /// Wait for Ctrl-C, or SIGTERM where there is one, which is how a harness stops a child. diff --git a/rs/moq-shaper/src/preset.rs b/rs/moq-shaper/src/preset.rs new file mode 100644 index 0000000000..56c01a1778 --- /dev/null +++ b/rs/moq-shaper/src/preset.rs @@ -0,0 +1,251 @@ +use std::{net::SocketAddr, time::Duration}; + +use anyhow::Context; +use serde::Deserialize; + +use crate::{Batch, Config, Jitter, Options, Profile, Rate, Setup, Step}; + +/// The profiles shipped under `profiles/`, embedded so a binary needs no data files. +const BUILTIN: &[(&str, &str)] = &[ + ("near-zero", include_str!("../profiles/near-zero.toml")), + ("mild", include_str!("../profiles/mild.toml")), + ("bursty", include_str!("../profiles/bursty.toml")), + ("step", include_str!("../profiles/step.toml")), + ("high-rtt", include_str!("../profiles/high-rtt.toml")), + ("lossy", include_str!("../profiles/lossy.toml")), +]; + +/// A named, seeded treatment of both directions, loaded from a TOML profile. +#[derive(Clone, Debug)] +pub struct Preset { + /// The name a run records its counters under. + pub name: String, + seed: u64, + shared: bool, + up: (Profile, Options), + down: (Profile, Options), +} + +impl Preset { + /// Load a built-in profile by name, or a profile file by path. + pub fn load(name_or_path: &str) -> anyhow::Result { + let text = match BUILTIN.iter().find(|(name, _)| *name == name_or_path) { + Some((_, text)) => (*text).to_string(), + // An unknown name is almost always a typo, so say what the choices were. + None => std::fs::read_to_string(name_or_path).with_context(|| { + format!( + "no built-in profile or readable file named {name_or_path:?}; the built-ins are {}", + Self::names().collect::>().join(", ") + ) + })?, + }; + + let file: File = toml::from_str(&text).with_context(|| format!("parse profile {name_or_path:?}"))?; + file.preset() + .with_context(|| format!("invalid profile {name_or_path:?}")) + } + + /// The built-in profiles' names. + pub fn names() -> impl Iterator { + BUILTIN.iter().map(|(name, _)| *name) + } + + /// This preset listening on `bind` and forwarding to `target`, with its own seed. + pub fn setup(&self, bind: SocketAddr, target: SocketAddr) -> Setup { + Setup { + config: Config { + bind, + target, + seed: self.seed, + up: self.up.0.clone(), + down: self.down.0.clone(), + }, + tcp_passthrough: false, + shared: self.shared, + up: self.up.1.clone(), + down: self.down.1.clone(), + } + } +} + +/// A profile file. +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct File { + name: String, + seed: u64, + #[serde(default)] + shared: bool, + #[serde(default)] + up: Side, + #[serde(default)] + down: Side, +} + +impl File { + fn preset(self) -> anyhow::Result { + anyhow::ensure!(!self.name.is_empty(), "the profile has no name"); + Ok(Preset { + up: self.up.split().context("up")?, + down: self.down.split().context("down")?, + name: self.name, + seed: self.seed, + shared: self.shared, + }) + } +} + +/// One direction of a profile file: its [`Profile`] and [`Options`] in one table. +#[derive(Default, Deserialize)] +#[serde(default, deny_unknown_fields)] +struct Side { + #[serde(with = "humantime_serde")] + delay: Duration, + #[serde(with = "humantime_serde")] + jitter: Duration, + jitter_model: Option, + loss: f64, + reorder: f64, + rate: Option, + batch: Option, + steps: Vec, +} + +impl Side { + fn split(self) -> anyhow::Result<(Profile, Options)> { + // Whether datagrams can overtake each other depends on the model as soon + // as their delays differ, so a file has to say which it means rather + // than inherit whichever is the default. + let varies = !self.jitter.is_zero() + || self + .steps + .iter() + .any(|step| step.delay.is_some() || step.jitter.is_some_and(|jitter| !jitter.is_zero())); + anyhow::ensure!( + self.jitter_model.is_some() || !varies, + "the delay varies, so name a jitter_model: \"uniform\" lets a datagram overtake the one in front, \"gaussian\" never does" + ); + + let profile = Profile { + delay: self.delay, + jitter: self.jitter, + loss: self.loss, + reorder: self.reorder, + rate: self.rate, + }; + let options = Options { + jitter_model: self.jitter_model.unwrap_or_default(), + batch: self.batch, + steps: self.steps, + }; + options.validate(&profile)?; + Ok((profile, options)) + } +} + +#[cfg(test)] +mod tests { + use std::net::{IpAddr, Ipv4Addr}; + + use super::*; + + const LOCALHOST: SocketAddr = SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), 0); + + #[test] + fn builtins_parse_and_are_named_after_their_file() { + for name in Preset::names() { + let preset = Preset::load(name).unwrap_or_else(|err| panic!("{name}: {err:#}")); + assert_eq!(preset.name, name); + } + } + + #[test] + fn builtins_treat_the_path_except_the_control() { + for name in Preset::names() { + let preset = Preset::load(name).unwrap(); + let untreated = (Profile::default(), Options::default()); + let idle = preset.up == untreated && preset.down == untreated; + assert_eq!(idle, name == "near-zero", "{name} treats the wrong amount of traffic"); + } + } + + #[test] + fn the_step_builtin_steps_both_directions_at_thirty_seconds() { + let setup = Preset::load("step").unwrap().setup(LOCALHOST, LOCALHOST); + for (which, profile, options) in [ + ("up", &setup.config.up, &setup.up), + ("down", &setup.config.down, &setup.down), + ] { + assert_eq!(profile.delay, Duration::from_millis(5), "{which}"); + assert_eq!(options.jitter_model, Jitter::Gaussian, "{which}"); + assert_eq!(options.steps.len(), 1, "{which}"); + assert_eq!(options.steps[0].at, Duration::from_secs(30), "{which}"); + assert_eq!(options.steps[0].delay, Some(Duration::from_millis(60)), "{which}"); + } + } + + #[test] + fn an_unknown_name_lists_the_builtins() { + let err = format!("{:#}", Preset::load("nope").unwrap_err()); + assert!(err.contains("near-zero"), "{err}"); + } + + #[test] + fn an_unknown_field_is_refused() { + // A field the schema does not know, like a rate in bytes, is refused rather than ignored. + let Err(err) = toml::from_str::("name = \"x\"\nseed = 1\n[up.rate]\nbytes_per_second = 1000\n") else { + panic!("a rate in bytes parsed"); + }; + assert!(err.to_string().contains("bytes_per_second"), "{err}"); + } + + #[test] + fn a_varying_delay_has_to_name_its_model() { + let parse = |text: &str| toml::from_str::(text).unwrap().preset(); + + let err = parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\njitter = \"5ms\"\n").unwrap_err(); + assert!(format!("{err:#}").contains("jitter_model"), "{err:#}"); + + let err = parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\n[[up.steps]]\nat = \"1s\"\ndelay = \"9ms\"\n") + .unwrap_err(); + assert!(format!("{err:#}").contains("jitter_model"), "{err:#}"); + + // A fixed delay cannot reorder anything, so it needs no model. + parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\n").unwrap(); + parse("name = \"x\"\nseed = 1\n[up]\ndelay = \"5ms\"\njitter = \"5ms\"\njitter_model = \"uniform\"\n").unwrap(); + } + + #[test] + fn a_profile_file_loads_by_path() { + let path = std::env::temp_dir().join(format!("moq-shaper-preset-{}.toml", std::process::id())); + std::fs::write( + &path, + "name = \"file\"\nseed = 9\n[down.batch]\ncount = 3\nwindow = \"50ms\"\n", + ) + .unwrap(); + let preset = Preset::load(path.to_str().unwrap()); + std::fs::remove_file(&path).unwrap(); + + let setup = preset.unwrap().setup(LOCALHOST, LOCALHOST); + assert_eq!(setup.config.seed, 9); + assert_eq!( + setup.down.batch, + Some(Batch { + count: 3, + window: Duration::from_millis(50) + }) + ); + } + + #[tokio::test] + async fn the_builtins_all_bind() { + for name in Preset::names() { + // Binding proves the profile is usable, not just parseable. + let preset = Preset::load(name).unwrap(); + let shaper = crate::Shaper::bind(preset.setup(LOCALHOST, LOCALHOST)) + .await + .unwrap_or_else(|err| panic!("{name}: {err:#}")); + shaper.verify().unwrap(); + } + } +} From c630613c9a59d04e00ae39f0ebc7bd9b2e03df44 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:19:31 -0400 Subject: [PATCH 10/27] feat(shaper): a JSON report of the counters, as an option A harness grading a run needs the shaper's counters in a form it can read, and a run that proved nothing (a profile that never acted) has to be told apart from one that passed. The exit line is prose, and it is printed only after the verdict, so a failed run leaves nothing to read. `Counters` and `Stats` derive `Serialize`. The CLI's `--report ` writes `{"profile", "seed", "up", "down"}` at exit, before the verdict, so a run that fails it still leaves its counters behind; `profile` is the profile's name, or null when the flags built it. `--report-interval` prints the same object as one line on stdout each period, which also survives a shaper killed before it could write the file. Off by default. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 1 + rs/moq-shaper/Cargo.toml | 1 + rs/moq-shaper/src/lib.rs | 6 +-- rs/moq-shaper/src/main.rs | 56 ++++++++++++++++++++++++- rs/moq-shaper/tests/cli.rs | 83 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 142 insertions(+), 5 deletions(-) create mode 100644 rs/moq-shaper/tests/cli.rs diff --git a/Cargo.lock b/Cargo.lock index 2ffc9d3146..7273dc240f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4726,6 +4726,7 @@ dependencies = [ "humantime-serde", "rand 0.10.3", "serde", + "serde_json", "tokio", "toml 1.1.6+spec-1.1.0", ] diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index 36add548dd..f81759d5e1 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -23,5 +23,6 @@ humantime = { workspace = true } humantime-serde = { workspace = true } rand = { workspace = true } serde = { workspace = true } +serde_json = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } toml = { workspace = true } diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index c3c471abdd..073dcb5c79 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -32,7 +32,7 @@ pub use preset::Preset; use anyhow::Context; use rand::{RngExt, SeedableRng, rngs::Xoshiro256PlusPlus}; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tokio::{ net::{TcpListener, TcpStream, UdpSocket}, sync::mpsc, @@ -344,7 +344,7 @@ impl From for Setup { } /// What one direction did, summed over every client. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] pub struct Counters { /// Datagrams received. pub packets: u64, @@ -371,7 +371,7 @@ impl fmt::Display for Counters { } /// What both directions did. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)] pub struct Stats { /// From clients toward the target. pub up: Counters, diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index d05c07c323..0fac2dacd7 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -5,8 +5,9 @@ //! profile print at start, and the counters at exit, which fails if the profile //! never acted. -use std::{net::SocketAddr, time::Duration}; +use std::{net::SocketAddr, path::PathBuf, time::Duration}; +use anyhow::Context; use clap::Parser; #[derive(Parser)] @@ -67,6 +68,12 @@ struct Args { /// Also pipe TCP on the listening port to the target, untouched. #[arg(long)] tcp_passthrough: bool, + /// Write the profile, seed and counters to this file as JSON at exit. + #[arg(long)] + report: Option, + /// Print the same JSON as one line on stdout this often. + #[arg(long, value_parser = humantime::parse_duration)] + report_interval: Option, } #[derive(Clone, Copy, clap::ValueEnum)] @@ -112,7 +119,32 @@ async fn main() -> anyhow::Result<()> { config.seed ); - shutdown().await?; + let report = || Report { + profile: name.as_deref(), + seed: config.seed, + stats: shaper.stats(), + }; + + // The first tick is a full period in, so a line reports traffic, not the start. + let mut interval = args + .report_interval + .map(|period| tokio::time::interval_at(tokio::time::Instant::now() + period, period)); + let shutdown = shutdown(); + tokio::pin!(shutdown); + loop { + tokio::select! { + res = &mut shutdown => { + res?; + break; + } + _ = tick(&mut interval) => println!("{}", serde_json::to_string(&report())?), + } + } + + // Before the verdict, so a run that fails it still leaves its counters behind. + if let Some(path) = &args.report { + std::fs::write(path, serde_json::to_vec(&report())?).with_context(|| format!("write {}", path.display()))?; + } let stats = shaper.verify()?; println!("shaper: {stats}"); @@ -158,6 +190,26 @@ fn flags(args: &Args) -> moq_shaper::Setup { } } +/// What a run did, as the JSON a harness reads back. +#[derive(serde::Serialize)] +struct Report<'a> { + /// The profile's name, or null when the flags built it. + profile: Option<&'a str>, + seed: u64, + #[serde(flatten)] + stats: moq_shaper::Stats, +} + +/// The next report tick, or never without an interval. +async fn tick(interval: &mut Option) { + match interval { + Some(interval) => { + interval.tick().await; + } + None => std::future::pending().await, + } +} + /// Wait for Ctrl-C, or SIGTERM where there is one, which is how a harness stops a child. async fn shutdown() -> anyhow::Result<()> { #[cfg(unix)] diff --git a/rs/moq-shaper/tests/cli.rs b/rs/moq-shaper/tests/cli.rs new file mode 100644 index 0000000000..fb37ba9d1f --- /dev/null +++ b/rs/moq-shaper/tests/cli.rs @@ -0,0 +1,83 @@ +//! The binary end to end: a profile by name, the TCP passthrough, and the +//! counters it reports while it runs and when it stops. +#![cfg(unix)] + +use std::{ + io::{BufRead, BufReader, Read, Write}, + net::{SocketAddr, TcpListener, TcpStream, UdpSocket}, + process::{Command, Stdio}, + time::Duration, +}; + +#[test] +fn a_profile_run_reports_its_counters() { + // The target answers TCP on the port number it takes datagrams on, as a relay does. + let target = UdpSocket::bind("127.0.0.1:0").unwrap(); + let target_addr = target.local_addr().unwrap(); + let http = TcpListener::bind(target_addr).unwrap(); + std::thread::spawn(move || { + let (mut stream, _) = http.accept().unwrap(); + let mut buf = [0u8; 64]; + let size = stream.read(&mut buf).unwrap(); + stream.write_all(&buf[..size]).unwrap(); + }); + + let report = std::env::temp_dir().join(format!("moq-shaper-report-{}.json", std::process::id())); + let mut shaper = Command::new(env!("CARGO_BIN_EXE_moq-shaper")) + .args(["--listen", "127.0.0.1:0", "--target", &target_addr.to_string()]) + .args(["--profile", "mild", "--seed", "7", "--tcp-passthrough"]) + .args(["--report-interval", "100ms", "--report", report.to_str().unwrap()]) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + let mut stdout = BufReader::new(shaper.stdout.take().unwrap()); + + // The first line says where it listens, and under what. + let mut line = String::new(); + stdout.read_line(&mut line).unwrap(); + assert!(line.ends_with("seed 7, profile mild\n"), "{line}"); + let addr: SocketAddr = line.split_whitespace().nth(1).unwrap().parse().unwrap(); + + let client = UdpSocket::bind("127.0.0.1:0").unwrap(); + for id in 0..50u32 { + client.send_to(&id.to_be_bytes(), addr).unwrap(); + } + target.set_read_timeout(Some(Duration::from_secs(5))).unwrap(); + let mut buf = [0u8; 4]; + for _ in 0..50 { + target.recv_from(&mut buf).expect("a datagram never arrived"); + } + + let mut tcp = TcpStream::connect(addr).unwrap(); + tcp.write_all(b"/certificate.sha256").unwrap(); + let mut buf = [0u8; 64]; + let size = tcp.read(&mut buf).unwrap(); + assert_eq!(&buf[..size], b"/certificate.sha256"); + + // A line each interval, once the traffic has been counted. + let interim = loop { + line.clear(); + stdout.read_line(&mut line).unwrap(); + let json: serde_json::Value = serde_json::from_str(&line).unwrap(); + if json["up"]["packets"] == 50 { + break json; + } + }; + assert_eq!(interim["profile"], "mild"); + + let status = Command::new("kill") + .args(["-TERM", &shaper.id().to_string()]) + .status() + .unwrap(); + assert!(status.success()); + assert!(shaper.wait().unwrap().success(), "the profile never acted"); + + let written = std::fs::read(&report).expect("no report at exit"); + std::fs::remove_file(&report).unwrap(); + let json: serde_json::Value = serde_json::from_slice(&written).unwrap(); + assert_eq!(json["profile"], "mild"); + assert_eq!(json["seed"], 7); + assert_eq!(json["up"]["packets"], 50); + assert_eq!(json["up"]["delayed"], 50); + assert_eq!(json["up"]["lost"], 0); +} From f564161e2ed49a252b190aa382383b737b8b61b9 Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:25:42 -0400 Subject: [PATCH 11/27] fix(shaper): send a datagram that is already due at once, not on a timer The delivery task sent datagrams only when its timer fired, even ones whose departure had already come. tokio rounds a deadline up to the end of its millisecond, so every datagram waited for the next tick: on loopback with an empty profile, the shaper added 1.5ms each way where a forward takes 0.3ms. Every profile carried that on top of its own delay, so the control profile was not a control, and a batch that filled left a tick late. After each datagram arrives, and each time the timer fires, whatever is due now leaves now. Nothing leaves before its departure, and the timer still wakes for the ones that are not due yet. A paused-clock test pins it: an undelayed round trip takes no time, where before the fix it took 2ms, a tick each way. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/Cargo.toml | 3 +++ rs/moq-shaper/src/lib.rs | 45 ++++++++++++++++++++++++++++------------ 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index f81759d5e1..a18c3cc8fc 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -26,3 +26,6 @@ serde = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } toml = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["test-util"] } diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 073dcb5c79..7d0f17d99a 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -699,20 +699,8 @@ async fn deliver( } } _ = tokio::time::sleep_until(wake.unwrap_or_else(Instant::now)), if wake.is_some() => { - let now = Instant::now(); - while let Some(&Reverse((at, id))) = pending.peek() { - if at > now { - break; - } - pending.pop(); - let parcel = parcels.remove(&id).expect("queued datagram"); - // A send error is the path losing the datagram, the way a - // network does when the far end is gone (a killed relay, say), - // so it is not the shaper failing. - let _ = parcel.socket.send_to(&parcel.datagram, parcel.dest).await; - } // The window closes a batch that never filled. - if let Some(closes) = held.closes.filter(|&closes| closes <= now) { + if let Some(closes) = held.closes.filter(|&closes| closes <= Instant::now()) { held.release(closes, &mut ready, &tally[direction]); } } @@ -722,6 +710,21 @@ async fn deliver( parcels.insert(sequence, parcel); sequence += 1; } + + // Whatever is due leaves now. The timer rounds up to the next + // millisecond, which would delay even a datagram given no delay. + let now = Instant::now(); + while let Some(&Reverse((at, id))) = pending.peek() { + if at > now { + break; + } + pending.pop(); + let parcel = parcels.remove(&id).expect("queued datagram"); + // A send error is the path losing the datagram, the way a network + // does when the far end is gone (a killed relay, say), so it is not + // the shaper failing. + let _ = parcel.socket.send_to(&parcel.datagram, parcel.dest).await; + } } } @@ -988,6 +991,22 @@ mod tests { assert_eq!(stats.down.packets, 50); } + #[tokio::test(start_paused = true)] + async fn an_undelayed_datagram_waits_for_no_timer() { + let (_shaper, client) = setup(1, Profile::default(), Profile::default()).await; + + // Off a millisecond boundary, where a timer would round up to the next one. + tokio::time::advance(Duration::from_micros(500)).await; + let start = Instant::now(); + client.send(b"ping").await.unwrap(); + let mut buf = [0u8; 4]; + client.recv(&mut buf).await.unwrap(); + + // A paused clock only moves when every task waits on a timer, so a round + // trip that took any time slept on one. + assert_eq!(start.elapsed(), Duration::ZERO); + } + #[tokio::test] async fn the_seed_reproduces_the_losses() { let lossy = Profile { From 78046628c78a4a07095c0f1c4e4bbec7e2cd661a Mon Sep 17 00:00:00 2001 From: fperex <270788582+fperex@users.noreply.github.com> Date: Fri, 25 Sep 2026 07:29:18 -0400 Subject: [PATCH 12/27] docs(shaper): a README for the shaper's usage and its options The crate had no README, and its options are only worth taking with the reason behind each: why jitter should keep the order, why a page's sessions share one path, why a batch is not a delay, why TCP passes through untouched. The README covers upstream's usage and flags, what a seed fixes and what it does not, each option, the profile file format and the built-ins, the counters, and the report. The crate docs point at the options. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 179 +++++++++++++++++++++++++++++++++++++++ rs/moq-shaper/src/lib.rs | 4 + 2 files changed, 183 insertions(+) create mode 100644 rs/moq-shaper/README.md diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md new file mode 100644 index 0000000000..8eb817054a --- /dev/null +++ b/rs/moq-shaper/README.md @@ -0,0 +1,179 @@ +# moq-shaper + +A seeded userspace UDP impairment relay, so a test can put an impaired path in front of a server. + +The shaper binds one socket, forwards every datagram to a target after a profile's treatment, and +forwards the replies back. A client dials the shaper instead of the server and reaches it through a +path with loss, a rate limit, delay, jitter, and reordering. QUIC is indifferent to the extra hop, so +this impairs a real transport with no capabilities and nothing touching the host's network, on macOS +and Linux alike. Kernel impairment (`netem` in a network namespace) is Linux only, needs +`CAP_NET_ADMIN`, and changes the host's network for everything running on it. + +It is a datagram relay, not an HTTP interceptor or a TCP proxy, neither of which can impair QUIC. + +## Usage + +```bash +moq-relay --listen 127.0.0.1:4443 & + +moq-shaper --listen 127.0.0.1:4444 --target 127.0.0.1:4443 \ + --delay 20ms --jitter 5ms --loss 0.05 --reorder 0.02 --rate 10000000 --seed 1 +``` + +Point the client at the shaper's port. The flags build one profile, applied both ways. The seed and +profile print at start, and the counters at exit (Ctrl-C or SIGTERM), which exits nonzero if the +profile never acted. + +| Flag | What it does | +| --- | --- | +| `--listen` | The address clients send to. | +| `--target` | The address every datagram is forwarded to. | +| `--seed` | Seeds every decision; the profile's own, or random, when omitted. | +| `--delay` | The base one-way delay. | +| `--jitter` | How far the delay varies: the most either way, or the sigma of a gaussian. | +| `--jitter-model` | `uniform` (the default) or `gaussian`. See [Jitter](#jitter). | +| `--loss` | The probability that a datagram is dropped. | +| `--reorder` | The probability that a datagram skips the delay, overtaking those in flight. | +| `--rate`, `--burst`, `--queue` | A token bucket: bits per second, the bytes it lets out ahead after an idle spell (1500), and the longest a datagram waits before it is dropped (100ms). | +| `--batch`, `--batch-window` | Hold datagrams until this many wait, or the window closes, then release them together. See [Batches](#batches). | +| `--shared` | Every client shares one link each way. See [Shared path](#shared-path). | +| `--profile` | A built-in profile's name, or a profile TOML file, in place of the flags that shape the path. See [Profiles](#profiles). | +| `--tcp-passthrough` | Also pipe TCP on the listening port to the target, untouched. See [TCP](#tcp). | +| `--report` | Write the profile, seed and counters to this file as JSON at exit. See [Report](#report). | +| `--report-interval` | Print the same JSON as one line on stdout this often. | + +As a library, `Shaper::bind` takes a `Config`: where to listen and forward, the seed, and a `Profile` +per direction. It also takes a `Setup`, which is a `Config` plus the opt-in options below, each off by +default. `Shaper::verify` fails when the shaper stopped forwarding, or when an impairment the profile +configures never acted and the traffic makes that silence implausible. The relay's drills +(`rs/moq-relay/tests/drills.rs`, described in `test/drill/README.md`) run every scenario through it. + +## What a seed does and does not fix + +Every decision comes from the seed: each link draws loss, reorder and jitter from its own stream, in the +same order for every datagram whether or not the knob is on. Replay a run with the same seed and the +same datagrams are dropped, the same ones skip the delay, and each draws the same jitter. + +The seed does not make the kernel's delivery clock reproducible: when a held datagram actually lands is +the host's scheduling. A test asserting on a replay should assert on the decisions, not on arrival times. + +## Jitter + +The default, `uniform`, draws each datagram's delay on its own within `jitter` either way of `delay`, so a +later datagram can overtake an earlier one. A drill wants that, and `jitter` may not exceed `delay`. + +`gaussian` treats `jitter` as a sigma, clamps the draw at zero, and never lets a datagram leave before the +one in front of it: queueing delay on a FIFO path, which stretches and compresses the spacing and never +changes the order. Only `reorder` overtakes. This matters beyond realism: QUIC reads an overtake as loss, +retransmits and backs off, so a run meant to measure a jittery path ends up measuring the congestion +response. On a 5ms delay with 5ms of jitter, reordering alone dragged a receiver's audio buffer from +120ms to nearly two seconds. + +## Shared path + +Each client flow gets a link of its own each way by default: its own seeded stream, rate limit and queue, +so its decisions do not depend on anyone else's traffic. + +With `shared`, every flow takes one link each way, the way clients behind one access link do. A browser +page with a WebTransport session of its own and another in its audio worker is two flows on one path: +shared, a gaussian path keeps the order across both, a batch fills from both, and one rate limit serves +both. A flow's treatment then depends on how its datagrams interleave with the others'. + +## Batches + +A paced hop, a Wi-Fi access point or a cellular scheduler bunches traffic: nothing moves for a while, then +several datagrams arrive at once. A batch holds datagrams until `count` are waiting, or until `window` +after the first arrived, then sends them all when the latest would have left. No delay or jitter produces +that clump, and it is what a receiver's jitter estimate has to cope with. + +## Steps + +A step changes a direction's delay, jitter, loss, reorder or rate limit once the run reaches `at`, counted +from when forwarding started. It changes only what it names, so a later step puts one knob back. A rate +step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there +starts full. A step can add or change a rate limit, never remove one. Steps come from a profile file. + +## Profiles + +A profile file names both directions' treatment, the seed, and whether the path is shared. `up` is from +the client to the target, `down` the way back. Every field of a direction is optional and off when left +out, and unknown fields are refused. + +```toml +name = "example" +seed = 7 +shared = true + +[up] +delay = "5ms" +jitter = "5ms" +jitter_model = "gaussian" # required once the delay varies, through jitter or steps +loss = 0.02 +reorder = 0.01 # skips the delay, so it needs one + +[up.rate] +bits_per_second = 2000000 +burst = 12500 +queue = "250ms" + +[up.batch] +count = 7 +window = "160ms" + +[[up.steps]] # at 30s the delay grows, +at = "30s" +delay = "60ms" + +[[up.steps]] # and at 60s it recovers +at = "60s" +delay = "5ms" + +[down] +delay = "5ms" +``` + +`--profile` takes a file's path, or one of the built-ins in `profiles/`, embedded in the binary. The +audio-quality lane (`test/audio-quality/`) runs the first five. + +| Name | What it models | +| --- | --- | +| `near-zero` | The control. The shaper is in the path but treats nothing. | +| `mild` | A healthy wired LAN: 5ms delay, 5ms sigma, in order. | +| `bursty` | A paced hop: seven datagrams per 160ms window, released together. | +| `step` | 5ms for thirty seconds, then 60ms, with no recovery. | +| `high-rtt` | An intercontinental path: 75ms one way, 30ms sigma, in order. | +| `lossy` | 2% loss, with 1% of datagrams skipping a 20ms delay to overtake. | + +Each built-in is a shared path, as a browser page's sessions on one host would be. + +## TCP + +A relay serves `/certificate.sha256` over HTTP on the port number it serves QUIC on, and a browser fetches +it before it dials WebTransport. With `tcp_passthrough`, the shaper listens for TCP on its own port and +pipes each connection to the target untouched, so the page loads through it. TCP is never impaired: a +reliable transport cannot shed load, so shaping it would only measure how TCP retransmits. + +## Counters + +`Shaper::stats` counts what each direction did, summed over every client. + +| Counter | Meaning | +| --- | --- | +| `packets` | Datagrams received. | +| `lost` | Datagrams dropped by `loss`. | +| `overflowed` | Datagrams dropped because the rate limit's queue was full. | +| `throttled` | Datagrams that waited for the rate limit. | +| `delayed` | Datagrams given a nonzero delay, by the profile or by a batch holding them. | +| `reordered` | Datagrams sent ahead of the delay, overtaking any still in flight. | + +## Report + +`--report ` writes `{"profile", "seed", "up", "down"}` at exit, before the verdict, so a run that +fails it still leaves its counters behind. `profile` is the profile's name, or null when the flags built +it, and `up` and `down` hold the counters above. `--report-interval` prints the same object as one line on +stdout each period, which also survives a shaper killed before it could write the file. + +A profile that silently did nothing turns an impaired run into an unimpaired pass. `verify` catches that +for the shaper's own exit; a harness grading a run should also check the report, where `near-zero` is the +only profile for which a `delayed` of zero is the right answer. Keep the name, the seed and the counters +with the run's artifacts: the seed is what turns a failing run into one that can be looked at again. diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 7d0f17d99a..fa84248661 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -13,6 +13,10 @@ //! A profile that silently did nothing would turn an impaired run into an //! unimpaired pass, so [`Shaper::verify`] fails when an impairment the profile //! configures never acted and the traffic makes that silence implausible. +//! +//! A [`Setup`] adds opt-in options to a [`Config`]: a TCP passthrough, a jitter +//! model that keeps the order, one link shared by every client, batches, steps, +//! and named profiles loaded as a [`Preset`]. The README says why each exists. use std::{ cmp::Reverse, From 53137d666a46dcea107d0bbd5c719af7fbe916f9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 22:49:39 -0700 Subject: [PATCH 13/27] test(audio-quality): grade bursty and changing paths Co-Authored-By: GPT-6 --- .github/workflows/nightly.yml | 4 +- Cargo.toml | 1 + quest/m0/audio-quality-harness/README.md | 14 +- .../audio-quality-harness/shaper-profiles.md | 34 --- quest/m1/audio-quality-native.md | 4 - rs/moq-shaper/Cargo.toml | 2 +- rs/moq-shaper/README.md | 12 +- rs/moq-shaper/profiles/high-rtt.toml | 15 -- rs/moq-shaper/profiles/lossy.toml | 16 -- rs/moq-shaper/src/lib.rs | 119 ++-------- rs/moq-shaper/src/main.rs | 10 +- rs/moq-shaper/src/preset.rs | 3 - rs/moq-shaper/tests/cli.rs | 40 ++-- test/audio-quality/README.md | 21 +- test/audio-quality/budgets.json | 212 +++++++++++++----- test/audio-quality/clients/js/src/run.test.ts | 30 +++ test/audio-quality/run.sh | 15 +- test/justfile | 2 +- 18 files changed, 267 insertions(+), 287 deletions(-) delete mode 100644 quest/m0/audio-quality-harness/shaper-profiles.md delete mode 100644 rs/moq-shaper/profiles/high-rtt.toml delete mode 100644 rs/moq-shaper/profiles/lossy.toml create mode 100644 test/audio-quality/clients/js/src/run.test.ts diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index eed4b612bc..a34677533c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,7 +18,7 @@ name: Nightly # the wheel builds in a manylinux container, the Kotlin/Swift/Go/Dart packages # link the FFI on every platform) and nothing notices until the tag is pushed. # -# `audio-quality` is here because of what it measures. It plays sixteen rows of +# `audio-quality` is here because of what it measures. It plays twenty-four rows of # audio in real time, a minute each, which no merge gate should carry, and every # number it grades is a timing one that moves with whatever else the runner is # doing. A per-PR arm would fail on unrelated load often enough to be ignored; @@ -189,7 +189,7 @@ jobs: audio-quality: name: Audio quality runs-on: ubuntu-24.04-arm - # Sixteen rows at a minute each, plus the relay, CLI, and shaper built from source. + # Twenty-four rows at a minute each, plus the relay, CLI, and shaper built from source. timeout-minutes: 60 steps: - name: Free disk space diff --git a/Cargo.toml b/Cargo.toml index 789ecbbfd0..8665f822e1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -124,6 +124,7 @@ hex = "0.4" # HMAC-SHA256 for the mDNS membership proofs (moq-tokio's `mdns` feature). hmac = "0.13" humantime = "2.3" +humantime-serde = "1.1.1" io-uring = "0.7.14" jsonwebtoken = "11" kio = { version = "0.6.1", path = "rs/kio" } diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index e3779f51ba..35388e5b76 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -23,13 +23,12 @@ are. The native lane moved to m1 as a standalone quest (decided in the same budgets and metric schema so the two implementations can be compared rather than merely both passing. -The browser lane has landed as `test/audio-quality/`, upstreamed from the -reporter's fork on #3477 (`fperex/moq`, branch `debug-findings-solution`) on -its own, without the fork's player, estimator, or shaper changes. That leaves -the two things that needed them: real arrival traces, which the [jitter -target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays, and the -bursty and step profiles, which need shaper features the fork has and `main` -does not. +The browser lane and its opt-in shaper profiles have landed in `test/audio-quality/`, +upstreamed from the reporter's fork on #3477 (`fperex/moq`, branch +`debug-findings-solution`), without the fork's player or estimator changes. +The matrix includes bursty and mid-run step paths, and mild/wide jitter preserves +packet order. Real arrival traces remain for the [jitter target's watch +quest](/quest/m0/audio-jitter-target/watch.md) to replay. Jitter comes from the seeded userspace UDP shaper the transport drills run under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake @@ -49,7 +48,6 @@ each of those dimensions moves the expected floor. ## Required - [Traces](/quest/m0/audio-quality-harness/traces.md) - real arrival traces recorded, checked in, and graded as replay rows -- [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/shaper-profiles.md b/quest/m0/audio-quality-harness/shaper-profiles.md deleted file mode 100644 index 71536c518d..0000000000 --- a/quest/m0/audio-quality-harness/shaper-profiles.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] The audio quality matrix grades a bursty path and a mid-run step - -## Goal - -`just test audio-quality` gains the two profiles `moq-shaper` cannot express -today: `bursty`, which releases datagrams in batches (the flush-span shape from -#3477, seven datagrams per 160 ms window), and `step`, which changes the -profile part-way through a row and forces the adaptive target to move. Both get -budgets in `test/audio-quality/budgets.json` and run nightly with the rest. - -## Plan - -The browser lane landed on the shaper as it is: one uniform profile both ways, -fixed for the life of the process. Its `mild` and `wide` profiles spread -arrivals with uniform jitter, which also reorders datagrams whenever the -jitter exceeds their spacing, so `wide` measures reordering as much as spread. - -The reporter's fork (`fperex/moq`, branch `debug-findings-solution`, -`rs/moq-shaper`) already has what is missing, as separate commits: batches that -release datagrams together, steps that change a profile mid-run, an -order-preserving gaussian jitter model, named profiles loaded from TOML, and a -JSON report of the counters. Upstream the batch, step, and order-preserving -jitter changes with the fork author's credit, keeping each behind the options -the drills do not set so their lane is unchanged. Skip TCP passthrough: the -harness answers the page's certificate fetch itself. - -Then add `bursty` and `step` to `run.sh`, refuse a `--duration` that ends -before the step, and switch `mild` and `wide` to the order-preserving jitter so -their names mean spread alone. Re-record the budgets for every row whose profile -changed. - -## Related - -- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator the step profile exercises diff --git a/quest/m1/audio-quality-native.md b/quest/m1/audio-quality-native.md index 11373ae103..232cc61deb 100644 --- a/quest/m1/audio-quality-native.md +++ b/quest/m1/audio-quality-native.md @@ -45,7 +45,3 @@ Standalone in m1 rather than a child of the m0 [Audio quality harness](/quest/m0/audio-quality-harness/README.md) line (decided in the 2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target it grades is done on the jitter target line. - -## Related - -- [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and step profiles both lanes should run diff --git a/rs/moq-shaper/Cargo.toml b/rs/moq-shaper/Cargo.toml index a18c3cc8fc..ebeb0190d7 100644 --- a/rs/moq-shaper/Cargo.toml +++ b/rs/moq-shaper/Cargo.toml @@ -24,7 +24,7 @@ humantime-serde = { workspace = true } rand = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "signal", "sync", "time"] } +tokio = { workspace = true, features = ["macros", "net", "rt-multi-thread", "signal", "sync", "time"] } toml = { workspace = true } [dev-dependencies] diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index 8eb817054a..89ad49d53b 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -38,7 +38,6 @@ profile never acted. | `--batch`, `--batch-window` | Hold datagrams until this many wait, or the window closes, then release them together. See [Batches](#batches). | | `--shared` | Every client shares one link each way. See [Shared path](#shared-path). | | `--profile` | A built-in profile's name, or a profile TOML file, in place of the flags that shape the path. See [Profiles](#profiles). | -| `--tcp-passthrough` | Also pipe TCP on the listening port to the target, untouched. See [TCP](#tcp). | | `--report` | Write the profile, seed and counters to this file as JSON at exit. See [Report](#report). | | `--report-interval` | Print the same JSON as one line on stdout this often. | @@ -133,7 +132,7 @@ delay = "5ms" ``` `--profile` takes a file's path, or one of the built-ins in `profiles/`, embedded in the binary. The -audio-quality lane (`test/audio-quality/`) runs the first five. +audio-quality lane (`test/audio-quality/`) uses `bursty` and `step`, plus its own flag profiles. | Name | What it models | | --- | --- | @@ -141,18 +140,9 @@ audio-quality lane (`test/audio-quality/`) runs the first five. | `mild` | A healthy wired LAN: 5ms delay, 5ms sigma, in order. | | `bursty` | A paced hop: seven datagrams per 160ms window, released together. | | `step` | 5ms for thirty seconds, then 60ms, with no recovery. | -| `high-rtt` | An intercontinental path: 75ms one way, 30ms sigma, in order. | -| `lossy` | 2% loss, with 1% of datagrams skipping a 20ms delay to overtake. | Each built-in is a shared path, as a browser page's sessions on one host would be. -## TCP - -A relay serves `/certificate.sha256` over HTTP on the port number it serves QUIC on, and a browser fetches -it before it dials WebTransport. With `tcp_passthrough`, the shaper listens for TCP on its own port and -pipes each connection to the target untouched, so the page loads through it. TCP is never impaired: a -reliable transport cannot shed load, so shaping it would only measure how TCP retransmits. - ## Counters `Shaper::stats` counts what each direction did, summed over every client. diff --git a/rs/moq-shaper/profiles/high-rtt.toml b/rs/moq-shaper/profiles/high-rtt.toml deleted file mode 100644 index 1f600fbbf3..0000000000 --- a/rs/moq-shaper/profiles/high-rtt.toml +++ /dev/null @@ -1,15 +0,0 @@ -# An intercontinental path: 75ms each way with a wide spread on top, queued in -# order. -name = "high-rtt" -seed = 5 -shared = true - -[up] -delay = "75ms" -jitter = "30ms" -jitter_model = "gaussian" - -[down] -delay = "75ms" -jitter = "30ms" -jitter_model = "gaussian" diff --git a/rs/moq-shaper/profiles/lossy.toml b/rs/moq-shaper/profiles/lossy.toml deleted file mode 100644 index 816063a752..0000000000 --- a/rs/moq-shaper/profiles/lossy.toml +++ /dev/null @@ -1,16 +0,0 @@ -# A path that loses 2%, and on which 1% of what survives skips the 20ms delay -# the rest take, arriving ahead of what is still in flight: the loss and the -# reordering a recovery path has to cope with. -name = "lossy" -seed = 6 -shared = true - -[up] -delay = "20ms" -loss = 0.02 -reorder = 0.01 - -[down] -delay = "20ms" -loss = 0.02 -reorder = 0.01 diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index fa84248661..5b0e0f24ca 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -14,7 +14,7 @@ //! unimpaired pass, so [`Shaper::verify`] fails when an impairment the profile //! configures never acted and the traffic makes that silence implausible. //! -//! A [`Setup`] adds opt-in options to a [`Config`]: a TCP passthrough, a jitter +//! A [`Setup`] adds opt-in options to a [`Config`]: a jitter //! model that keeps the order, one link shared by every client, batches, steps, //! and named profiles loaded as a [`Preset`]. The README says why each exists. @@ -37,12 +37,7 @@ pub use preset::Preset; use anyhow::Context; use rand::{RngExt, SeedableRng, rngs::Xoshiro256PlusPlus}; use serde::{Deserialize, Serialize}; -use tokio::{ - net::{TcpListener, TcpStream, UdpSocket}, - sync::mpsc, - task::JoinSet, - time::Instant, -}; +use tokio::{net::UdpSocket, sync::mpsc, task::JoinSet, time::Instant}; /// How one direction of the path treats each datagram. /// @@ -315,13 +310,6 @@ pub struct Config { pub struct Setup { /// Where to listen and forward, the seed, and each direction's profile. pub config: Config, - /// Also accept TCP on the listening port and pipe it to the target untouched. - /// - /// A relay serves HTTP on the port number it serves QUIC on, and a browser - /// fetches the certificate hash from it before it dials WebTransport. TCP is - /// never impaired: a reliable transport cannot shed load, so shaping it - /// would measure how TCP retransmits. - pub tcp_passthrough: bool, /// Every client shares one link each way, the way clients behind one access /// link do, rather than each getting its own. /// @@ -339,7 +327,6 @@ impl From for Setup { fn from(config: Config) -> Self { Self { config, - tcp_passthrough: false, shared: false, up: Options::default(), down: Options::default(), @@ -412,16 +399,6 @@ impl Shaper { .with_context(|| format!("bind {}", config.bind))?; let addr = listen.local_addr()?; - // The port the UDP socket got, since a relay serves HTTP on its QUIC port. - let tcp = match setup.tcp_passthrough { - true => Some( - TcpListener::bind(addr) - .await - .with_context(|| format!("bind the TCP passthrough on {addr}"))?, - ), - false => None, - }; - let tally = Arc::new([Tally::default(), Tally::default()]); let failed = Arc::new(OnceLock::new()); let task = tokio::spawn({ @@ -429,7 +406,7 @@ impl Shaper { let tally = tally.clone(); let failed = failed.clone(); async move { - if let Err(err) = run(Arc::new(listen), tcp, setup, tally).await { + if let Err(err) = run(Arc::new(listen), setup, tally).await { let _ = failed.set(format!("{err:#}")); } } @@ -527,12 +504,7 @@ fn bump(counter: &AtomicU64) { /// A flow is a socket of its own toward the target, so the target sees one /// address per client just as it would without the shaper in the way. Each /// flow takes a link of its own each way, unless the path is shared. -async fn run( - listen: Arc, - tcp: Option, - setup: Setup, - tally: Arc<[Tally; 2]>, -) -> anyhow::Result<()> { +async fn run(listen: Arc, setup: Setup, tally: Arc<[Tally; 2]>) -> anyhow::Result<()> { let config = &setup.config; let mut flows = HashMap::::new(); let mut tasks = JoinSet::new(); @@ -553,11 +525,7 @@ async fn run( res.context("flow task panicked")??; continue; } - res = accept(tcp.as_ref()) => { - let (stream, _) = res.context("accept a TCP connection")?; - tasks.spawn(pipe(stream, config.target)); - continue; - } + }; let now = Instant::now(); @@ -622,23 +590,6 @@ fn link( ))) } -/// The next TCP connection, or never without a passthrough. -async fn accept(tcp: Option<&TcpListener>) -> std::io::Result<(TcpStream, SocketAddr)> { - match tcp { - Some(listener) => listener.accept().await, - None => std::future::pending().await, - } -} - -/// Copy one TCP connection to the target and back, untouched. -async fn pipe(mut client: TcpStream, target: SocketAddr) -> anyhow::Result<()> { - // Either end refusing or hanging up ends that connection, not the shaper. - if let Ok(mut server) = TcpStream::connect(target).await { - let _ = tokio::io::copy_bidirectional(&mut client, &mut server).await; - } - Ok(()) -} - /// A socket that can reach `target`: loopback for a loopback target, so the /// shaper never opens a port beyond the host when it does not need to. async fn bind_toward(target: SocketAddr) -> anyhow::Result { @@ -831,7 +782,9 @@ impl Link { while self.steps.last().is_some_and(|step| self.start + step.at <= now) { let step = self.steps.pop().expect("a step is due"); if let Some(rate) = &step.rate { - self.full_at = self.refilled(rate, now); + // Credit accrues at the new rate from the scheduled transition, + // even when no datagram arrived at that instant. + self.full_at = self.refilled(rate, self.start + step.at); } step.apply(&mut self.profile); } @@ -1184,56 +1137,6 @@ mod tests { refused(undelayed, "needs a delay").await; } - #[tokio::test] - async fn tcp_passes_through_untouched() { - use tokio::io::{AsyncReadExt, AsyncWriteExt}; - - // A relay answering HTTP on its QUIC port, reduced to an echo. - let server = TcpListener::bind(LOCALHOST).await.unwrap(); - let target = server.local_addr().unwrap(); - tokio::spawn(async move { - let (mut stream, _) = server.accept().await.unwrap(); - let mut buf = [0u8; 64]; - let size = stream.read(&mut buf).await.unwrap(); - stream.write_all(&buf[..size]).await.unwrap(); - }); - - // A profile that would lose every datagram, to show TCP skips it. - let blackhole = Profile { - loss: 1.0, - ..Default::default() - }; - let config = Config { - bind: LOCALHOST, - target, - seed: 13, - up: blackhole.clone(), - down: blackhole, - }; - let shaper = Shaper::bind(Setup { - tcp_passthrough: true, - ..config.into() - }) - .await - .unwrap(); - - let mut client = TcpStream::connect(shaper.addr()).await.unwrap(); - client.write_all(b"/certificate.sha256").await.unwrap(); - let mut buf = [0u8; 64]; - let size = tokio::time::timeout(Duration::from_secs(2), client.read(&mut buf)) - .await - .unwrap() - .unwrap(); - assert_eq!(&buf[..size], b"/certificate.sha256"); - assert_eq!(shaper.stats(), Stats::default(), "TCP reached the datagram path"); - } - - #[tokio::test] - async fn tcp_is_refused_without_the_passthrough() { - let (shaper, _client) = setup(1, Profile::default(), Profile::default()).await; - assert!(TcpStream::connect(shaper.addr()).await.is_err()); - } - /// An echo server, a shaper `setup` builds from a config aimed at it, and a /// client dialing the shaper. async fn shaped(setup: impl FnOnce(Config) -> Setup) -> (Shaper, UdpSocket) { @@ -1841,6 +1744,12 @@ mod tests { // 100 bytes of the backlog are still owed, and now drain in 50ms, so // a fresh 100 bytes leave once those and themselves are paid. assert_eq!(owed(&mut link, start + ms(100), &[100]), [ms(100)]); + + // A quiet link still changes rates at the step, not at its next arrival. + let (queue, _) = mpsc::unbounded_channel(); + let mut link = Link::new(&profile, &options, 7, 0, start, queue); + owed(&mut link, start, &[100, 100, 100]); + assert_eq!(owed(&mut link, start + ms(200), &[100]), [ms(0)]); } #[tokio::test] diff --git a/rs/moq-shaper/src/main.rs b/rs/moq-shaper/src/main.rs index 0fac2dacd7..c16eb30578 100644 --- a/rs/moq-shaper/src/main.rs +++ b/rs/moq-shaper/src/main.rs @@ -65,9 +65,6 @@ struct Args { /// Every client shares one link each way, instead of each getting its own. #[arg(long)] shared: bool, - /// Also pipe TCP on the listening port to the target, untouched. - #[arg(long)] - tcp_passthrough: bool, /// Write the profile, seed and counters to this file as JSON at exit. #[arg(long)] report: Option, @@ -87,8 +84,12 @@ enum JitterModel { #[tokio::main] async fn main() -> anyhow::Result<()> { let args = Args::parse(); + anyhow::ensure!( + args.report_interval.is_none_or(|period| !period.is_zero()), + "the report interval must be positive" + ); - let (name, mut setup) = match &args.profile { + let (name, setup) = match &args.profile { Some(profile) => { let preset = moq_shaper::Preset::load(profile)?; let mut setup = preset.setup(args.listen, args.target); @@ -99,7 +100,6 @@ async fn main() -> anyhow::Result<()> { } None => (None, flags(&args)), }; - setup.tcp_passthrough = args.tcp_passthrough; let treatment = match &name { Some(name) => format!("profile {name}"), diff --git a/rs/moq-shaper/src/preset.rs b/rs/moq-shaper/src/preset.rs index 56c01a1778..e8cd8095a9 100644 --- a/rs/moq-shaper/src/preset.rs +++ b/rs/moq-shaper/src/preset.rs @@ -11,8 +11,6 @@ const BUILTIN: &[(&str, &str)] = &[ ("mild", include_str!("../profiles/mild.toml")), ("bursty", include_str!("../profiles/bursty.toml")), ("step", include_str!("../profiles/step.toml")), - ("high-rtt", include_str!("../profiles/high-rtt.toml")), - ("lossy", include_str!("../profiles/lossy.toml")), ]; /// A named, seeded treatment of both directions, loaded from a TOML profile. @@ -60,7 +58,6 @@ impl Preset { up: self.up.0.clone(), down: self.down.0.clone(), }, - tcp_passthrough: false, shared: self.shared, up: self.up.1.clone(), down: self.down.1.clone(), diff --git a/rs/moq-shaper/tests/cli.rs b/rs/moq-shaper/tests/cli.rs index fb37ba9d1f..8c15351647 100644 --- a/rs/moq-shaper/tests/cli.rs +++ b/rs/moq-shaper/tests/cli.rs @@ -1,31 +1,23 @@ -//! The binary end to end: a profile by name, the TCP passthrough, and the +//! The binary end to end: a profile by name and the //! counters it reports while it runs and when it stops. #![cfg(unix)] use std::{ - io::{BufRead, BufReader, Read, Write}, - net::{SocketAddr, TcpListener, TcpStream, UdpSocket}, + io::{BufRead, BufReader}, + net::{SocketAddr, UdpSocket}, process::{Command, Stdio}, time::Duration, }; #[test] fn a_profile_run_reports_its_counters() { - // The target answers TCP on the port number it takes datagrams on, as a relay does. let target = UdpSocket::bind("127.0.0.1:0").unwrap(); let target_addr = target.local_addr().unwrap(); - let http = TcpListener::bind(target_addr).unwrap(); - std::thread::spawn(move || { - let (mut stream, _) = http.accept().unwrap(); - let mut buf = [0u8; 64]; - let size = stream.read(&mut buf).unwrap(); - stream.write_all(&buf[..size]).unwrap(); - }); let report = std::env::temp_dir().join(format!("moq-shaper-report-{}.json", std::process::id())); let mut shaper = Command::new(env!("CARGO_BIN_EXE_moq-shaper")) .args(["--listen", "127.0.0.1:0", "--target", &target_addr.to_string()]) - .args(["--profile", "mild", "--seed", "7", "--tcp-passthrough"]) + .args(["--profile", "mild", "--seed", "7"]) .args(["--report-interval", "100ms", "--report", report.to_str().unwrap()]) .stdout(Stdio::piped()) .spawn() @@ -48,12 +40,6 @@ fn a_profile_run_reports_its_counters() { target.recv_from(&mut buf).expect("a datagram never arrived"); } - let mut tcp = TcpStream::connect(addr).unwrap(); - tcp.write_all(b"/certificate.sha256").unwrap(); - let mut buf = [0u8; 64]; - let size = tcp.read(&mut buf).unwrap(); - assert_eq!(&buf[..size], b"/certificate.sha256"); - // A line each interval, once the traffic has been counted. let interim = loop { line.clear(); @@ -81,3 +67,21 @@ fn a_profile_run_reports_its_counters() { assert_eq!(json["up"]["delayed"], 50); assert_eq!(json["up"]["lost"], 0); } + +#[test] +fn a_zero_report_interval_is_refused_before_binding() { + let result = Command::new(env!("CARGO_BIN_EXE_moq-shaper")) + .args([ + "--listen", + "127.0.0.1:0", + "--target", + "127.0.0.1:9", + "--report-interval", + "0s", + ]) + .output() + .unwrap(); + assert!(!result.status.success()); + assert!(result.stdout.is_empty(), "bound before refusing the interval"); + assert!(String::from_utf8_lossy(&result.stderr).contains("report interval must be positive")); +} diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 242660731f..c9640eb993 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -14,7 +14,7 @@ just test audio-quality --out ~/runs/after # keep the run di just test audio-quality --enforce # fail on the budgets ``` -The matrix is codec x profile x ring: 16 rows, about 20 minutes. `--profiles`, `--rings`, and +The matrix is codec x profile x ring: 24 rows, about 30 minutes. `--profiles`, `--rings`, and `--codecs` take comma-separated lists; `--seed` replays a given impairment. The table prints either way, and only `--enforce` (which the nightly job passes) turns a breach into a failed run. @@ -33,17 +33,18 @@ decides which ring ran. | Profile | Path | Delay | | --- | --- | --- | | `near-zero` | through the shaper, untreated | auto | -| `mild` | 5 ms one way, plus or minus 5 ms uniform | auto | -| `wide` | 40 ms one way, plus or minus 40 ms uniform | auto | +| `mild` | 5 ms one way, 5 ms Gaussian sigma, in order | auto | +| `wide` | 40 ms one way, 40 ms Gaussian sigma, in order | auto | +| `bursty` | batches of seven datagrams, released within 160 ms | auto | +| `step` | 5 ms one way, stepping to 60 ms at 30 s | auto | | `fixed-250` | through the shaper, untreated | a fixed 250 ms | `fixed-250` is the control: every other row adapts, so without it the whole matrix could pass while a fixed preset regressed, and a fixed preset is what many viewers land on. Loss, reorder, and rate limits stay off: the buffer's job is absorbing arrival spread, and congestion response would make a -failure hard to attribute. The shaper's uniform jitter does reorder datagrams whose spacing is -smaller than it, so `wide` measures some reordering along with the spread. The bursty and mid-run -step profiles need shaper features that are not here yet -([quest](../../quest/m0/audio-quality-harness/shaper-profiles.md)). +failure hard to attribute. The `mild` and `wide` profiles preserve datagram order and share one path across the page's sessions. +`bursty` and `step` use the shaper's named profiles on that same shared path. The step is timed from +shaper startup, before the page connects; `--duration` must exceed 30 s so playback spans it. The AAC arm keeps ffmpeg's default PES packing, whose multi-frame bursts are the flush-span shape the reporter measured on the public relay (#3477). The source is a sine tone rather than a film, so a @@ -172,9 +173,9 @@ clients/js/ The harness is the reporter's, from #3477: fperex built it on their fork `fperex/moq` (branches `debug/rt-audio` and `debug-findings-solution`), with the black-box probe, the analyzer's skip-ahead rule, the Playwright matrix over `moq-shaper`, the budget file graded under `--enforce`, and the -nightly job. This lane upstreams it on its own, without the fork's player, estimator, or shaper -changes: the underrun count comes from a tap on the output rather than a counter in the player, and -the profiles are the ones the shaper on `main` can express. +nightly job. The harness and opt-in shaper features preserve that author's credit. The player and estimator +changes remain separate: the underrun count comes from a tap on the output rather than a counter in the player, and +the profiles now include the fork's batch, step, and order-preserving jitter treatments. ## Not covered diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 0cdde6b25c..16971ef4d0 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,6 +1,34 @@ { - "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs.", + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. Updated 2026-09-29: mild and wide now use shared order-preserving Gaussian jitter; bursty and step are new. Those 16 rows were re-recorded for 60 s each, seed 7, on the same shared Linux workstation with concurrent bounded builds. Their ceilings use twice the measurement rounded upward to two significant figures and the same floors above; target_ms_p95 adds one 20 ms bucket. Other rows retain their previous measurements. These remain current-player regression ceilings, not target audio quality; re-record on the nightly runner before tightening them.", "rows": [ + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "bursty", + "ring": "isolated", + "underruns_per_min": 1100.0, + "underrun_ms_per_min": 3000.0, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 500.0, + "stalled_share": 0.01, + "silence_share": 0.05, + "target_ms_p95": 404.8 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "bursty", + "ring": "plain", + "underruns_per_min": 740.0, + "underrun_ms_per_min": 2200.0, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 350.0, + "stalled_share": 0.01, + "silence_share": 0.034, + "target_ms_p95": 368.5 + }, { "runtime": "chromium", "codec": "aac", @@ -35,13 +63,13 @@ "rate": 44100, "profile": "mild", "ring": "isolated", - "underruns_per_min": 270, - "underrun_ms_per_min": 780, - "skip_aheads_per_min": 2.2, - "discarded_ms_per_min": 160, + "underruns_per_min": 320.0, + "underrun_ms_per_min": 890.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, "stalled_share": 0.01, - "silence_share": 0.012, - "target_ms_p95": 210 + "silence_share": 0.014, + "target_ms_p95": 211 }, { "runtime": "chromium", @@ -49,13 +77,13 @@ "rate": 44100, "profile": "mild", "ring": "plain", - "underruns_per_min": 570, - "underrun_ms_per_min": 1700, - "skip_aheads_per_min": 2.2, - "discarded_ms_per_min": 120, + "underruns_per_min": 340.0, + "underrun_ms_per_min": 970.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, "stalled_share": 0.01, - "silence_share": 0.026, - "target_ms_p95": 210 + "silence_share": 0.014, + "target_ms_p95": 206 }, { "runtime": "chromium", @@ -89,15 +117,43 @@ "runtime": "chromium", "codec": "aac", "rate": 44100, - "profile": "wide", + "profile": "step", "ring": "isolated", - "underruns_per_min": 430, - "underrun_ms_per_min": 1200, + "underruns_per_min": 970.0, + "underrun_ms_per_min": 2800.0, "skip_aheads_per_min": 6.6, - "discarded_ms_per_min": 330, + "discarded_ms_per_min": 380.0, "stalled_share": 0.01, - "silence_share": 0.018, - "target_ms_p95": 250 + "silence_share": 0.042, + "target_ms_p95": 207.3 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "step", + "ring": "plain", + "underruns_per_min": 450.0, + "underrun_ms_per_min": 1200.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 120.0, + "stalled_share": 0.01, + "silence_share": 0.02, + "target_ms_p95": 212.3 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 140.0, + "underrun_ms_per_min": 360.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 140.0, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 319.8 }, { "runtime": "chromium", @@ -105,13 +161,41 @@ "rate": 44100, "profile": "wide", "ring": "plain", - "underruns_per_min": 540, - "underrun_ms_per_min": 1500, - "skip_aheads_per_min": 4.4, - "discarded_ms_per_min": 260, + "underruns_per_min": 330.0, + "underrun_ms_per_min": 940.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, "stalled_share": 0.01, - "silence_share": 0.03, - "target_ms_p95": 250 + "silence_share": 0.018, + "target_ms_p95": 308.5 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "bursty", + "ring": "isolated", + "underruns_per_min": 4800.0, + "underrun_ms_per_min": 13000.0, + "skip_aheads_per_min": 8.8, + "discarded_ms_per_min": 490.0, + "stalled_share": 0.018, + "silence_share": 0.2, + "target_ms_p95": 128 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "bursty", + "ring": "plain", + "underruns_per_min": 2900.0, + "underrun_ms_per_min": 7100.0, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 470.0, + "stalled_share": 0.028, + "silence_share": 0.13, + "target_ms_p95": 139.3 }, { "runtime": "chromium", @@ -147,12 +231,12 @@ "rate": 48000, "profile": "mild", "ring": "isolated", - "underruns_per_min": 4600, - "underrun_ms_per_min": 12000, - "skip_aheads_per_min": 1, - "discarded_ms_per_min": 100, + "underruns_per_min": 1300.0, + "underrun_ms_per_min": 2600.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, "stalled_share": 0.01, - "silence_share": 0.19, + "silence_share": 0.026, "target_ms_p95": 63 }, { @@ -161,12 +245,12 @@ "rate": 48000, "profile": "mild", "ring": "plain", - "underruns_per_min": 4800, - "underrun_ms_per_min": 12000, - "skip_aheads_per_min": 2.2, - "discarded_ms_per_min": 130, - "stalled_share": 0.028, - "silence_share": 0.22, + "underruns_per_min": 1400.0, + "underrun_ms_per_min": 2900.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.032, "target_ms_p95": 63 }, { @@ -197,19 +281,47 @@ "silence_share": 0.19, "target_ms_p95": 63 }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "step", + "ring": "isolated", + "underruns_per_min": 1300.0, + "underrun_ms_per_min": 2700.0, + "skip_aheads_per_min": 1.0, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.034, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "step", + "ring": "plain", + "underruns_per_min": 2500.0, + "underrun_ms_per_min": 5700.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.084, + "target_ms_p95": 63 + }, { "runtime": "chromium", "codec": "opus", "rate": 48000, "profile": "wide", "ring": "isolated", - "underruns_per_min": 8100, - "underrun_ms_per_min": 22000, - "skip_aheads_per_min": 29, - "discarded_ms_per_min": 1700, - "stalled_share": 0.56, - "silence_share": 0.73, - "target_ms_p95": 82 + "underruns_per_min": 2200.0, + "underrun_ms_per_min": 5600.0, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 100.0, + "stalled_share": 0.01, + "silence_share": 0.092, + "target_ms_p95": 150.5 }, { "runtime": "chromium", @@ -217,13 +329,13 @@ "rate": 48000, "profile": "wide", "ring": "plain", - "underruns_per_min": 6700, - "underrun_ms_per_min": 17000, - "skip_aheads_per_min": 18, - "discarded_ms_per_min": 1300, - "stalled_share": 0.49, - "silence_share": 0.75, - "target_ms_p95": 82 + "underruns_per_min": 2300.0, + "underrun_ms_per_min": 5700.0, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 200.0, + "stalled_share": 0.01, + "silence_share": 0.09, + "target_ms_p95": 154.3 } ] } diff --git a/test/audio-quality/clients/js/src/run.test.ts b/test/audio-quality/clients/js/src/run.test.ts new file mode 100644 index 0000000000..7cd6a67d10 --- /dev/null +++ b/test/audio-quality/clients/js/src/run.test.ts @@ -0,0 +1,30 @@ +import { expect, test } from "bun:test"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("../../../run.sh", import.meta.url)); + +test("a step row must play beyond the profile change", () => { + for (const duration of ["20", "30"]) { + const result = Bun.spawnSync(["bash", script, "--profiles", "step", "--duration", duration, "--list"]); + expect(result.exitCode).toBe(2); + expect(result.stderr.toString()).toContain("must exceed the step at 30s"); + } + const result = Bun.spawnSync(["bash", script, "--profiles", "step", "--duration", "31", "--list"]); + expect(result.exitCode).toBe(0); + expect(result.stdout.toString()).toContain("chromium-opus-48000-step-plain"); +}); + +test("short steady rows remain supported", () => { + const result = Bun.spawnSync(["bash", script, "--profiles", "mild", "--duration", "20", "--list"]); + expect(result.exitCode).toBe(0); +}); + +test("the nightly matrix includes both new profiles", () => { + const result = Bun.spawnSync(["bash", script, "--list"]); + expect(result.exitCode).toBe(0); + const rows = result.stdout.toString().trim().split("\n"); + expect(rows).toHaveLength(24); + for (const profile of ["bursty", "step"]) { + expect(rows.filter((row) => row.includes(`-${profile}-`))).toHaveLength(4); + } +}); diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index c10544597e..062576f313 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -15,7 +15,7 @@ source "$AQ_DIR/../lib/harness.sh" # Captured before the parse consumes it, so the rerun command carries every flag. RERUN="$(harness_env AQ_PROFILE RELAY_BIN)just test audio-quality$(harness_argv "$@")" -ALL_PROFILES=(near-zero mild wide fixed-250) +ALL_PROFILES=(near-zero mild wide bursty step fixed-250) ALL_RINGS=(isolated plain) ALL_CODECS=(opus aac) @@ -111,7 +111,13 @@ valid() { echo "error: unknown $name '$want' (known: $*)" >&2 exit 2 } -for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}"; done +for p in "${PROFILES[@]}"; do + valid "$p" profile "${ALL_PROFILES[@]}" + if [[ "$p" == step ]] && ((DURATION <= 30)); then + echo "error: --duration must exceed the step at 30s" >&2 + exit 2 + fi +done for r in "${RINGS[@]}"; do valid "$r" ring "${ALL_RINGS[@]}"; done for c in "${CODECS[@]}"; do valid "$c" codec "${ALL_CODECS[@]}"; done @@ -141,8 +147,9 @@ delay_of() { shaper_of() { case "$1" in near-zero | fixed-250) ;; - mild) echo "--delay 5ms --jitter 5ms" ;; - wide) echo "--delay 40ms --jitter 40ms" ;; + mild) echo "--delay 5ms --jitter 5ms --jitter-model gaussian --shared" ;; + wide) echo "--delay 40ms --jitter 40ms --jitter-model gaussian --shared" ;; + bursty | step) echo "--profile $1" ;; esac } diff --git a/test/justfile b/test/justfile index d5ec917a5e..6461b9ad98 100644 --- a/test/justfile +++ b/test/justfile @@ -75,7 +75,7 @@ drill *args: # Plays a tone in headless Chromium through moq-shaper and grades what the listener # heard (underruns, skip-aheads, silence, the resolved delay) against -# audio-quality/budgets.json. The full matrix is 16 rows at 60 s; nightly runs it with +# audio-quality/budgets.json. The full matrix is 24 rows at 60 s; nightly runs it with # --enforce. Narrow it with `--profiles mild --codecs opus --rings plain --duration 20`. # See audio-quality/README.md. From 756200eb8585b654621f9ad563a7e7174a1adb0a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 10:12:12 -0700 Subject: [PATCH 14/27] fix(shaper): verify fails a step no traffic reached, and a late datagram starts a new batch `verify` now fails when a direction carried traffic but none of it reached a configured step, so a run that ended early no longer passes as stepped. A datagram that arrives after a batch's window closed, but before its timer fired, now releases the closed batch first and starts the next one. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 6 ++-- rs/moq-shaper/src/lib.rs | 78 +++++++++++++++++++++++++++++++++------- 2 files changed, 70 insertions(+), 14 deletions(-) diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index 89ad49d53b..7adfee3d35 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -43,8 +43,9 @@ profile never acted. As a library, `Shaper::bind` takes a `Config`: where to listen and forward, the seed, and a `Profile` per direction. It also takes a `Setup`, which is a `Config` plus the opt-in options below, each off by -default. `Shaper::verify` fails when the shaper stopped forwarding, or when an impairment the profile -configures never acted and the traffic makes that silence implausible. The relay's drills +default. `Shaper::verify` fails when the shaper stopped forwarding, when an impairment the profile +configures never acted and the traffic makes that silence implausible, or when a direction carried +traffic but none of it reached a step. The relay's drills (`rs/moq-relay/tests/drills.rs`, described in `test/drill/README.md`) run every scenario through it. ## What a seed does and does not fix @@ -91,6 +92,7 @@ A step changes a direction's delay, jitter, loss, reorder or rate limit once the from when forwarding started. It changes only what it names, so a later step puts one knob back. A rate step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there starts full. A step can add or change a rate limit, never remove one. Steps come from a profile file. +A run that ends before its last step fails `verify`, since that traffic never saw the path change. ## Profiles diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 5b0e0f24ca..797d8568e6 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -439,20 +439,35 @@ impl Shaper { } } - /// Fail unless the shaper is still forwarding and every impairment the - /// profile configures acted on some datagram, in either direction. + /// Fail unless the shaper is still forwarding, every impairment the profile + /// configures acted on some datagram, and some datagram saw every step, in + /// either direction. /// /// An impairment is only held to that once the traffic makes its silence - /// implausible: a short run can see no loss, but never no delay. + /// implausible: a short run can see no loss, but never no delay. A step is + /// held to it once its direction carried anything: traffic that all came + /// before the step never saw the path change. pub fn verify(&self) -> anyhow::Result { let stats = self.stats(); if let Some(err) = self.failed.get() { anyhow::bail!("the shaper stopped forwarding: {err} ({stats})"); } - let mut missing = unapplied(&self.setup.config, &stats); + let mut missing: Vec = unapplied(&self.setup.config, &stats) + .into_iter() + .map(String::from) + .collect(); if unbatched(&self.setup, &stats) { - missing.push("batch"); + missing.push("batch".to_string()); + } + for (name, options, tally, counters) in [ + ("up", &self.setup.up, &self.tally[UP], &stats.up), + ("down", &self.setup.down, &self.tally[DOWN], &stats.down), + ] { + let reached = tally.stepped.load(Ordering::Relaxed) as usize; + if let Some(step) = options.steps.get(reached).filter(|_| counters.packets > 0) { + missing.push(format!("the {name} step at {:?}", step.at)); + } } anyhow::ensure!( missing.is_empty(), @@ -480,6 +495,8 @@ struct Tally { throttled: AtomicU64, delayed: AtomicU64, reordered: AtomicU64, + /// The most steps any one link has applied. + stepped: AtomicU64, } impl Tally { @@ -694,6 +711,11 @@ impl Held { /// Hold `parcel`, releasing the batch into `ready` once it is full. fn hold(&mut self, batch: Batch, parcel: Parcel, ready: &mut Vec, tally: &Tally) { let arrived = parcel.arrived; + // The window may have closed before its timer fired; the batch it closed + // leaves without this datagram, which starts the next one. + if let Some(closes) = self.closes.filter(|&closes| closes <= arrived) { + self.release(closes, ready, tally); + } self.closes.get_or_insert(arrived + batch.window); self.parcels.push(parcel); if self.parcels.len() >= batch.count { @@ -744,8 +766,10 @@ struct Link { floor: Instant, /// When the run started, which a step's `at` counts from. start: Instant, - /// The steps still to come, soonest last. + /// Every step, earliest first. steps: Vec, + /// How many of `steps` have applied. + stepped: usize, queue: mpsc::UnboundedSender, } @@ -772,21 +796,23 @@ impl Link { full_at: start, floor: start, start, - steps: options.steps.iter().rev().cloned().collect(), + steps: options.steps.clone(), + stepped: 0, queue, } } /// Apply every step the run has reached by `now`, in order. - fn step(&mut self, now: Instant) { - while self.steps.last().is_some_and(|step| self.start + step.at <= now) { - let step = self.steps.pop().expect("a step is due"); + fn step(&mut self, now: Instant, tally: &Tally) { + while let Some(step) = self.steps.get(self.stepped).filter(|step| self.start + step.at <= now) { if let Some(rate) = &step.rate { // Credit accrues at the new rate from the scheduled transition, // even when no datagram arrived at that instant. self.full_at = self.refilled(rate, self.start + step.at); } step.apply(&mut self.profile); + self.stepped += 1; + tally.stepped.fetch_max(self.stepped as u64, Ordering::Relaxed); } } @@ -822,7 +848,7 @@ impl Link { /// When a datagram of `size` bytes arriving `now` leaves, and whether it was /// counted as delayed, or `None` when it is dropped. fn treat(&mut self, now: Instant, size: usize, tally: &Tally) -> Option<(Instant, bool)> { - self.step(now); + self.step(now, tally); bump(&tally.packets); // Every draw happens for every datagram, whatever the profile, so one @@ -1480,6 +1506,16 @@ mod tests { held.release(closes, &mut ready, &tally); assert_eq!(ready.iter().map(|parcel| parcel.at).collect::>(), [closes]); assert_eq!(tally.snapshot().delayed, 3); + ready.clear(); + + // A datagram that arrives after the window closed, before its timer fired, + // starts the next batch rather than joining the closed one. + held.hold(batch, parcel(now + ms(200)), &mut ready, &tally); + let closes = held.closes.unwrap(); + held.hold(batch, parcel(closes + ms(1)), &mut ready, &tally); + assert_eq!(ready.iter().map(|parcel| parcel.at).collect::>(), [closes]); + assert_eq!(held.parcels.len(), 1, "the late datagram joined the closed batch"); + assert_eq!(held.closes, Some(closes + ms(1) + ms(160))); } #[tokio::test] @@ -1755,7 +1791,7 @@ mod tests { #[tokio::test] async fn a_step_gets_worse_part_way_through() { let ms = Duration::from_millis; - let (_shaper, client) = shaped(|config| Setup { + let (shaper, client) = shaped(|config| Setup { up: stepped(vec![Step { at: ms(150), delay: Some(ms(60)), @@ -1786,6 +1822,24 @@ mod tests { after > before + ms(30), "median latency went from {before:?} to {after:?}" ); + shaper.verify().unwrap(); + } + + #[tokio::test] + async fn a_step_the_run_never_reached_is_unapplied() { + let (shaper, client) = shaped(|config| Setup { + down: stepped(vec![Step { + at: Duration::from_secs(60), + delay: Some(Duration::from_millis(60)), + ..Default::default() + }]), + ..config.into() + }) + .await; + assert_eq!(round_trip(&client, 10).await, (0..10).collect::>()); + + let err = shaper.verify().expect_err("a run that ended before its step passed"); + assert!(format!("{err:#}").contains("the down step at 60s"), "{err:#}"); } #[tokio::test] From d9a9de7550504baffb4ff0384acb0790a3d784e9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 11:42:40 -0700 Subject: [PATCH 15/27] fix(shaper): refuse a step that changes nothing and a batch of one A step restating the value already in force left the profile unchanged yet counted as reached. A batch of one released every datagram on arrival. Both now fail before binding. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 6 ++++-- rs/moq-shaper/src/lib.rs | 33 +++++++++++++++++++++------------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index 7adfee3d35..9d2145cb9c 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -84,14 +84,16 @@ both. A flow's treatment then depends on how its datagrams interleave with the o A paced hop, a Wi-Fi access point or a cellular scheduler bunches traffic: nothing moves for a while, then several datagrams arrive at once. A batch holds datagrams until `count` are waiting, or until `window` after the first arrived, then sends them all when the latest would have left. No delay or jitter produces -that clump, and it is what a receiver's jitter estimate has to cope with. +that clump, and it is what a receiver's jitter estimate has to cope with. A `count` below two is refused, +since the datagram that fills a batch never waits. ## Steps A step changes a direction's delay, jitter, loss, reorder or rate limit once the run reaches `at`, counted from when forwarding started. It changes only what it names, so a later step puts one knob back. A rate step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there -starts full. A step can add or change a rate limit, never remove one. Steps come from a profile file. +starts full. A step can add or change a rate limit, never remove one. A step that leaves the profile as +it was is refused. Steps come from a profile file. A run that ends before its last step fails `verify`, since that traffic never saw the path change. ## Profiles diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 797d8568e6..dd7d27f63c 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -188,7 +188,8 @@ impl Options { fn validate(&self, profile: &Profile) -> anyhow::Result<()> { profile.validate(self.jitter_model)?; if let Some(batch) = &self.batch { - anyhow::ensure!(batch.count > 0, "a batch of zero datagrams never closes"); + // The datagram that fills a batch leaves at once, so one of one holds nothing. + anyhow::ensure!(batch.count > 1, "a batch of {} never holds anything", batch.count); anyhow::ensure!(!batch.window.is_zero(), "a batch with no window never holds anything"); } @@ -202,16 +203,10 @@ impl Options { "the step at {:?} comes after the one at {previous:?}; steps go in order", step.at ); - // A step naming nothing is a profile that thinks it changes and does not. - anyhow::ensure!( - Step { - at: step.at, - ..Step::default() - } != *step, - "the step at {:?} changes nothing", - step.at - ); + // A step that leaves the profile as it was thinks it changes and does not. + let before = profile.clone(); step.apply(&mut profile); + anyhow::ensure!(profile != before, "the step at {:?} changes nothing", step.at); profile .validate(self.jitter_model) .with_context(|| format!("after the step at {:?}", step.at))?; @@ -1612,8 +1607,9 @@ mod tests { .unwrap_or_else(|| panic!("accepted a batch where {why}")); assert!(format!("{err:#}").contains(why), "{err:#}"); }; - refused(batched(0, Duration::from_millis(10)), "never closes").await; - refused(batched(7, Duration::ZERO), "never holds").await; + refused(batched(0, Duration::from_millis(10)), "a batch of 0 never holds").await; + refused(batched(1, Duration::from_millis(10)), "a batch of 1 never holds").await; + refused(batched(7, Duration::ZERO), "no window never holds").await; } #[test] @@ -1879,6 +1875,19 @@ mod tests { "changes nothing", ) .await; + // Restating the value already in force changes nothing either. + refused( + vec![ + at(30), + Step { + at: Duration::from_secs(60), + loss: Some(0.1), + ..Default::default() + }, + ], + "the step at 60s changes nothing", + ) + .await; // Uniform jitter past the delay, reached by a step rather than at the start. refused( vec![Step { From 748447b53b97e764c030010739f699595f7be279 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:00:56 -0700 Subject: [PATCH 16/27] fix(audio-quality): measure a step row from before its step, and count only batch holds The driver now gives the session and first audio one 20s deadline, short of the step profile's change at 30s, so a step row never grades only the post-step path. `verify` now proves a batch acted from a counter only batches bump, so a profile delay no longer passes a batch that never changed a departure. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/src/lib.rs | 56 +++++++++++++++++++------ test/audio-quality/README.md | 5 ++- test/audio-quality/clients/js/driver.ts | 11 +++-- 3 files changed, 53 insertions(+), 19 deletions(-) diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index dd7d27f63c..1bcfd2fa6f 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -253,13 +253,14 @@ fn unapplied(config: &Config, stats: &Stats) -> Vec<&'static str> { /// Whether the batches `setup` configures implausibly never held a datagram. /// /// A batch holds every datagram but the one that fills it, so it acts on at -/// least `1 - 1 / count` of them. What it holds counts as delayed once the -/// batch leaves. -fn unbatched(setup: &Setup, stats: &Stats) -> bool { +/// least `1 - 1 / count` of them. `batched` counts only what a batch held, not +/// what the profile delayed, so a batch that never changed a departure is caught +/// even behind a delay. +fn unbatched(setup: &Setup, stats: &Stats, batched: u64) -> bool { let chance = |options: &Options| options.batch.map_or(0.0, |batch| 1.0 - 1.0 / batch.count as f64); let silence = (1.0 - chance(&setup.up)).powf(stats.up.packets as f64) * (1.0 - chance(&setup.down)).powf(stats.down.packets as f64); - stats.up.delayed + stats.down.delayed == 0 && silence < IMPLAUSIBLE + batched == 0 && silence < IMPLAUSIBLE } /// A token-bucket rate limit with a bounded queue behind it. @@ -452,7 +453,12 @@ impl Shaper { .into_iter() .map(String::from) .collect(); - if unbatched(&self.setup, &stats) { + let batched = self + .tally + .iter() + .map(|tally| tally.batched.load(Ordering::Relaxed)) + .sum(); + if unbatched(&self.setup, &stats, batched) { missing.push("batch".to_string()); } for (name, options, tally, counters) in [ @@ -492,6 +498,8 @@ struct Tally { reordered: AtomicU64, /// The most steps any one link has applied. stepped: AtomicU64, + /// Datagrams a batch held past when they would otherwise have left. + batched: AtomicU64, } impl Tally { @@ -727,9 +735,12 @@ impl Held { }; let at = latest.max(closed); for mut parcel in self.parcels.drain(..) { - // A hold is a delay, counted once whichever stage gave it. - if at > parcel.at && !parcel.delayed { - bump(&tally.delayed); + if at > parcel.at { + bump(&tally.batched); + // A hold is a delay, counted once whichever stage gave it. + if !parcel.delayed { + bump(&tally.delayed); + } } parcel.at = at; ready.push(parcel); @@ -1511,6 +1522,23 @@ mod tests { assert_eq!(ready.iter().map(|parcel| parcel.at).collect::>(), [closes]); assert_eq!(held.parcels.len(), 1, "the late datagram joined the closed batch"); assert_eq!(held.closes, Some(closes + ms(1) + ms(160))); + assert_eq!(tally.snapshot().delayed, 4); + assert_eq!(tally.batched.load(Ordering::Relaxed), 4); + + // Behind a delay longer than the window, a lone datagram leaves when the + // delay says: the batch held nothing, whatever `delayed` counts. + ready.clear(); + let mut held = Held::default(); + let tally = Tally::default(); + let late = Parcel { + at: now + ms(1000), + delayed: true, + ..parcel(now) + }; + held.hold(batch, late, &mut ready, &tally); + held.release(held.closes.unwrap(), &mut ready, &tally); + assert_eq!(ready[0].at, now + ms(1000)); + assert_eq!(tally.batched.load(Ordering::Relaxed), 0); } #[tokio::test] @@ -1634,13 +1662,15 @@ mod tests { }; // (1/7)^2 is 2%: two datagrams through batches of seven prove nothing. - assert!(!unbatched(&setup, &quiet(2))); + assert!(!unbatched(&setup, &quiet(2), 0)); // (1/7)^10 is 4e-9: ten datagrams a batch never held means no batch. - assert!(unbatched(&setup, &quiet(10))); + assert!(unbatched(&setup, &quiet(10), 0)); + assert!(!unbatched(&setup, &quiet(10), 6)); - let mut held = quiet(10); - held.up.delayed = 6; - assert!(!unbatched(&setup, &held)); + // A delay the profile gave is not a batch holding anything. + let mut delayed = quiet(10); + delayed.up.delayed = 10; + assert!(unbatched(&setup, &delayed, 0)); } /// How long after `at` each of `sizes` leaves one link, fed at the same instant. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index c9640eb993..c64b180aeb 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -44,7 +44,8 @@ fixed preset regressed, and a fixed preset is what many viewers land on. Loss, r limits stay off: the buffer's job is absorbing arrival spread, and congestion response would make a failure hard to attribute. The `mild` and `wide` profiles preserve datagram order and share one path across the page's sessions. `bursty` and `step` use the shaper's named profiles on that same shared path. The step is timed from -shaper startup, before the page connects; `--duration` must exceed 30 s so playback spans it. +shaper startup, before the page connects; the driver gives up on a row with no audio 20 s in, so the +measured window starts before the step, and `--duration` must exceed 30 s so playback spans it. The AAC arm keeps ffmpeg's default PES packing, whose multi-frame bursts are the flush-span shape the reporter measured on the public relay (#3477). The source is a sine tone rather than a film, so a @@ -140,7 +141,7 @@ A row that cannot be trusted is void, reported rather than graded, and fails an | `ring` | `crossOriginIsolated` does not match the row's ring, so the other ring ran. | | `clock` | The AudioContext clock was unreadable or ran more than 1% off the viewer's. | | `window` | No audio after the warmup. | -| `driver` | The page never reached a session or first audio within 30 s, or the driver threw. | +| `driver` | The page never reached a session and first audio within 20 s, or the driver threw. | ## Budgets diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts index 3fec53e203..e640a584f9 100644 --- a/test/audio-quality/clients/js/driver.ts +++ b/test/audio-quality/clients/js/driver.ts @@ -57,8 +57,12 @@ if ( process.exit(2); } -/** How long the session and the first audio each get before the row is abandoned. */ -const STARTUP_MS = 30_000; +/** + * How long the session and the first audio get, together, before the row is abandoned. It stays short + * of the step profile's change at 30s, so a step row always measures audio from before the change. + */ +const STARTUP_MS = 20_000; +const startupDeadline = Date.now() + STARTUP_MS; /** How often the probe is drained into the ndjson. */ const DRAIN_MS = 1000; @@ -98,8 +102,7 @@ const refuse = (assertion: string, detail: string) => { /** Poll the page until `ready` returns a value, or throw naming what never happened. */ async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { - const deadline = Date.now() + STARTUP_MS; - while (Date.now() < deadline) { + while (Date.now() < startupDeadline) { const value = await page.evaluate(ready); if (value !== undefined && value !== null) return value; await sleep(200); From 2ecb0abecc611eafbdbdc5007cdcb728a61db310 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 12:43:31 -0700 Subject: [PATCH 17/27] fix(shaper): refuse jitter with no delay to vary A gaussian profile with jitter but no delay passed `verify` even when every draw clamped to zero, since a zero delay has no chance to act. The uniform model already refused it as jitter exceeding delay. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 2 +- rs/moq-shaper/src/lib.rs | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index 9d2145cb9c..b7d8001184 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -64,7 +64,7 @@ later datagram can overtake an earlier one. A drill wants that, and `jitter` may `gaussian` treats `jitter` as a sigma, clamps the draw at zero, and never lets a datagram leave before the one in front of it: queueing delay on a FIFO path, which stretches and compresses the spacing and never -changes the order. Only `reorder` overtakes. This matters beyond realism: QUIC reads an overtake as loss, +changes the order. The sigma may exceed `delay`, but a nonzero `jitter` still needs a nonzero `delay`. Only `reorder` overtakes. This matters beyond realism: QUIC reads an overtake as loss, retransmits and backs off, so a run meant to measure a jittery path ends up measuring the congestion response. On a 5ms delay with 5ms of jitter, reordering alone dragged a receiver's audio buffer from 120ms to nearly two seconds. diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 1bcfd2fa6f..8e1bdf9cfa 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -83,6 +83,12 @@ impl Profile { self.jitter, self.delay ); + // With no delay to vary, `verify` has nothing to hold the jitter to. + anyhow::ensure!( + self.jitter.is_zero() || !self.delay.is_zero(), + "jitter {:?} needs a delay to vary", + self.jitter + ); if let Some(rate) = &self.rate { anyhow::ensure!(rate.bits_per_second > 0, "a rate limit of zero passes nothing"); } @@ -1261,6 +1267,7 @@ mod tests { fn gaussian_jitter_never_leaves_before_it_arrived() { // A sigma far past the delay, so most draws would be negative unclamped. let wide = Profile { + delay: Duration::from_millis(1), jitter: Duration::from_millis(50), ..Default::default() }; @@ -1355,6 +1362,27 @@ mod tests { .await; assert_eq!(round_trip(&client, 20).await, (0..20).collect::>()); shaper.verify().unwrap(); + + // With no delay, `verify` could never tell the jitter acted. + let undelayed = Profile { + jitter: Duration::from_millis(5), + ..Default::default() + }; + let err = Shaper::bind(Setup { + up: GAUSSIAN, + ..Config { + bind: LOCALHOST, + target: LOCALHOST, + seed: 0, + up: undelayed, + down: Profile::default(), + } + .into() + }) + .await + .err() + .expect("accepted gaussian jitter with no delay"); + assert!(format!("{err:#}").contains("needs a delay to vary"), "{err:#}"); } /// Two clients take turns sending `count` numbered datagrams up the path From 0f024e3e01e490135eb59590d0699e202f2aaa93 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:41:59 -0700 Subject: [PATCH 18/27] test(audio-quality): record real arrival traces and grade their replay per PR (#4525) Co-authored-by: Claude Opus 5.5 --- .github/workflows/audio-quality.yml | 63 + .github/workflows/nightly.yml | 4 +- bun.lock | 4 + js/watch/src/audio/replay.test.ts | 70 + js/watch/src/audio/replay.ts | 265 +++ quest/m0/audio-jitter-target/watch.md | 20 +- quest/m0/audio-quality-harness/README.md | 17 +- quest/m0/audio-quality-harness/traces.md | 33 - test/audio-quality/README.md | 62 +- test/audio-quality/budgets.json | 86 +- test/audio-quality/clients/js/mic.html | 10 + test/audio-quality/clients/js/package.json | 4 + test/audio-quality/clients/js/record.ts | 143 ++ test/audio-quality/clients/js/recorder.html | 10 + test/audio-quality/clients/js/replay.ts | 157 ++ test/audio-quality/clients/js/src/analyze.ts | 5 +- test/audio-quality/clients/js/src/mic.ts | 27 + test/audio-quality/clients/js/src/recorder.ts | 130 ++ test/audio-quality/clients/js/src/schema.ts | 52 +- test/audio-quality/clients/js/tsconfig.json | 11 +- test/audio-quality/clients/js/vite.config.ts | 11 +- test/audio-quality/record.sh | 107 + test/audio-quality/run.sh | 278 ++- test/audio-quality/traces/local-aac.json | 1566 +++++++++++++ test/audio-quality/traces/relay-bbb.json | 1954 +++++++++++++++++ test/audio-quality/traces/relay-mic.json | 1763 +++++++++++++++ test/justfile | 9 + 27 files changed, 6682 insertions(+), 179 deletions(-) create mode 100644 .github/workflows/audio-quality.yml create mode 100644 js/watch/src/audio/replay.test.ts create mode 100644 js/watch/src/audio/replay.ts delete mode 100644 quest/m0/audio-quality-harness/traces.md create mode 100644 test/audio-quality/clients/js/mic.html create mode 100644 test/audio-quality/clients/js/record.ts create mode 100644 test/audio-quality/clients/js/recorder.html create mode 100644 test/audio-quality/clients/js/replay.ts create mode 100644 test/audio-quality/clients/js/src/mic.ts create mode 100644 test/audio-quality/clients/js/src/recorder.ts create mode 100755 test/audio-quality/record.sh create mode 100644 test/audio-quality/traces/local-aac.json create mode 100644 test/audio-quality/traces/relay-bbb.json create mode 100644 test/audio-quality/traces/relay-mic.json diff --git a/.github/workflows/audio-quality.yml b/.github/workflows/audio-quality.yml new file mode 100644 index 0000000000..1ad4af43cc --- /dev/null +++ b/.github/workflows/audio-quality.yml @@ -0,0 +1,63 @@ +name: Audio quality + +# Grades the recorded-trace replays of test/audio-quality against their exact +# budgets on every PR that can move them. They replay the player's container +# consumer and rings on a simulated clock, so they are deterministic and take +# seconds, unlike the browser rows, which stay in nightly.yml. +on: + pull_request: + paths: + - "js/watch/**" + # The container consumer decides what a replay delivers, waits for, and skips, on the + # groups, tracks, and signals underneath it. + - "js/hang/**" + - "js/net/**" + - "js/signals/**" + - "test/audio-quality/**" + - "test/lib/**" + - "test/justfile" + - ".github/workflows/audio-quality.yml" + +concurrency: + group: audio-quality-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + replay: + name: Replay + runs-on: ubuntu-24.04-arm + timeout-minutes: 20 + + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter; see check.yml. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + - name: Replay + run: nix develop --command just test audio-quality --replays --enforce + shell: bash -leo pipefail {0} + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + + # The harness keeps a failing run's directory: every row's samples and summary. + - name: Keep the failing run + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: audio-quality-replay + path: ${{ runner.temp }}/moq-test-runs/ + retention-days: 7 + if-no-files-found: ignore diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 92807fe75e..f8b3941168 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -23,7 +23,9 @@ name: Nightly # number it grades is a timing one that moves with whatever else the runner is # doing. A per-PR arm would fail on unrelated load often enough to be ignored; # nightly it competes with nothing, and a regression surfaces a day late against -# a budget that still means something. +# a budget that still means something. The recorded-trace replays are the +# exception: deterministic and seconds long, so audio-quality.yml also gates +# PRs on them. # # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. diff --git a/bun.lock b/bun.lock index 7ce6980b9c..1968391f8d 100644 --- a/bun.lock +++ b/bun.lock @@ -346,6 +346,10 @@ "test/audio-quality/clients/js": { "name": "@moq/audio-quality", "dependencies": { + "@moq/hang": "workspace:*", + "@moq/net": "workspace:*", + "@moq/publish": "workspace:*", + "@moq/signals": "workspace:*", "@moq/watch": "workspace:*", }, "devDependencies": { diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts new file mode 100644 index 0000000000..d4f23355cf --- /dev/null +++ b/js/watch/src/audio/replay.test.ts @@ -0,0 +1,70 @@ +import { afterEach, beforeEach, describe, expect, it, type Mock, spyOn } from "bun:test"; +import type * as Catalog from "@moq/hang/catalog"; +import { Time } from "@moq/net"; +import { type Arrival, type Options, QUANTUM, replay, target } from "./replay"; + +const RATE = 48000; + +/** Twenty ms frames for `seconds`, one per group, each arriving `lateness(i)` ms after it was captured. */ +function paced(seconds: number, lateness: (i: number) => number): Arrival[] { + return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20, group: i })); +} + +/** Quanta that came back short or empty while playing, once the ring had first played. */ +async function underruns(trace: Arrival[], options: Options): Promise { + let started = false; + let short = 0; + for await (const { output, stalled } of replay(trace, options)) { + const filled = output.findLastIndex((v) => v !== 0) + 1; + if (started && !stalled && filled < QUANTUM) short++; + if (filled > 0) started = true; + } + return short; +} + +describe.each(["shared", "post"] as const)("%s ring", (ring) => { + // The consumer warns on every group it skips. + let warn: Mock; + beforeEach(() => { + warn = spyOn(console, "warn").mockImplementation(() => {}); + }); + afterEach(() => warn.mockRestore()); + + it("plays an evenly paced sender without a gap", async () => { + expect( + await underruns( + paced(10, () => 30), + { ring, rate: RATE, delay: 100 }, + ), + ).toBe(0); + }); + + it("underruns when a flush span outlasts the target, and not when it is covered", async () => { + // Five frames held and flushed at once: 80 ms of arrival spread. + const trace = paced(10, (i) => 30 + (4 - (i % 5)) * 20); + expect(await underruns(trace, { ring, rate: RATE, delay: 40 })).toBeGreaterThan(50); + expect(await underruns(trace, { ring, rate: RATE, delay: 150 })).toBe(0); + }); + + it("holds newer groups behind a missing one until the max age gives up on it", async () => { + const trace = paced(10, () => 30).filter((arrival) => arrival.group !== 250); + // The consumer delivers in group order, so the ring runs dry for longer than the missing frame + // while the groups behind it wait. Writing arrivals straight into the ring plays straight through. + const frame = Math.ceil((20 / 1000) * (RATE / QUANTUM)); + expect(await underruns(trace, { ring, rate: RATE, delay: 100 })).toBeGreaterThan(frame); + expect(warn).toHaveBeenCalled(); + }); +}); + +describe("target", () => { + const config = { codec: "opus", sampleRate: RATE, numberOfChannels: 1 } as Catalog.AudioConfig; + + it("sizes auto from the RTT and the rendition's jitter", async () => { + // 1.25 x 40 ms of RTT, plus the Opus frame and a render quantum. + expect(await target({ delay: "auto", config, rtt: 40 })).toBe(Time.Milli(50 + 20 + 3)); + }); + + it("adds the rendition's jitter to a fixed delay", async () => { + expect(await target({ delay: Time.Milli(250), config })).toBe(Time.Milli(250 + 20 + 3)); + }); +}); diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts new file mode 100644 index 0000000000..ba027dd643 --- /dev/null +++ b/js/watch/src/audio/replay.ts @@ -0,0 +1,265 @@ +/** + * Replay a recorded arrival trace through the player's consumer and rings on a simulated clock. + * + * Every frame is written into its group at the instant the trace says it arrived, and the player's + * real `Container.Consumer` decides what to deliver, wait for, and skip, at the max age the delay + * sets. Delivered frames go into the real ring, and a discontinuity resets it, the way the decoder + * does. One render quantum is read every quantum's worth of that same clock, which is what the + * AudioWorklet does. {@link target} resolves the delay with a real {@link Sync} fed the recorded + * catalog config and connection RTT, and the ring is sized from it the way the decoder sizes it, so + * a change to any of them moves what a replay hears. Decoding is taken as instant and sample exact. + * + * Deterministic, so the audio quality harness grades its output with no headroom, and a unit test + * can assert exact counts. + * + * @internal Test support, not part of the player. + */ +import type * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import { Group, Time, Track } from "@moq/net"; +import { Signal } from "@moq/signals"; +import { type Delay, Sync } from "../sync"; +import { playbackJitter } from "./config"; +import { ringSamples } from "./latency"; +import { AudioRingBuffer } from "./ring-buffer"; +import { allocSharedRingBuffer, SharedRingBuffer } from "./shared-ring-buffer"; +import { Terminal } from "./terminal"; + +/** An AudioWorklet render quantum, in frames. */ +export const QUANTUM = 128; + +/** A media gap longer than this is missing audio, not one long frame. */ +const MAX_FRAME_MS = 100; + +/** + * The replay's own subscription passes every recorded group through: the recorder's relay already + * applied a wire max age, so only the consumer's decisions are made again. + */ +const WIRE_MAX_AGE = Time.Milli(Number.MAX_SAFE_INTEGER); + +/** Any non-empty payload: an empty one is the legacy container's end marker. */ +const PAYLOAD = new Uint8Array(1); + +/** One frame reaching the container consumer. */ +export interface Arrival { + /** When it arrived, on the viewer's monotonic clock, in ms. */ + at: number; + /** Its media timestamp, in ms. */ + timestamp: number; + /** The group that carried it. */ + group: number; +} + +/** What the player resolves its delay from. */ +export interface Target { + /** The configured delay, as the element takes it. "instant" plays no audio, so it has no target. */ + delay: Exclude; + /** The rendition as the catalog described it: its advertised jitter. */ + config: Catalog.AudioConfig; + /** The connection's round trip, in ms, which "auto" sizes from. Unset is no PROBE. */ + rtt?: number; +} + +/** The delay a player resolves for `input`, in ms, from a real {@link Sync}. */ +export async function target(input: Target): Promise { + const sync = new Sync({ + delay: input.delay, + probe: input.rtt === undefined ? undefined : { rtt: Time.Milli(input.rtt) }, + }); + const jitter = playbackJitter(input.config); + const unregister = sync.register(new Signal(jitter)); + try { + // Sync's effects run on microtasks. The delay has settled once it holds this rendition's jitter + // on top of the resolved jitter, the only rendition registered. + while (sync.out.delay.peek() !== Time.Milli.add(sync.out.jitter.peek(), jitter)) { + await sync.out.delay.changed(); + } + return sync.out.delay.peek(); + } finally { + unregister(); + sync.close(); + } +} + +/** What a replay plays through. */ +export interface Options { + /** `shared` is the SharedArrayBuffer ring a cross-origin isolated page runs; `post` is the rest. */ + ring: "shared" | "post"; + /** The sample rate, in Hz. */ + rate: number; + /** + * The resolved delay, in ms. See {@link target}. It sizes the ring and is the consumer's max age, + * which `Sync` sets to the delay plus a lookahead a live viewer does not configure. + */ + delay: number; +} + +/** One render quantum, as the worklet produced it. */ +export interface Quantum { + /** When it finished rendering, on the trace's clock, in ms. */ + at: number; + /** The quantum: what the ring supplied at the front, zeros after it. Reused between yields. */ + output: Float32Array; + /** Whether the ring was stalled after this quantum. */ + stalled: boolean; + /** The playhead after this quantum, on the media clock, in ms. */ + timestamp: number; +} + +/** The two rings behind the calls the decoder and the worklet make on them. */ +interface Ring { + insert(timestamp: Time.Micro, data: Float32Array[]): void; + read(output: Float32Array[]): number; + reset(): void; + readonly stalled: boolean; + readonly timestamp: Time.Micro; +} + +/** The shared ring, sized the way `SharedAudioBuffer` sizes it. */ +function shared(rate: number, latency: number): Ring { + const ring = new SharedRingBuffer(allocSharedRingBuffer(1, Math.max(rate, latency * 2), rate)); + ring.setLatency(latency); + return ring; +} + +/** The postMessage ring, sized the way `PostAudioBuffer` asks the worklet to size it. */ +function post(rate: number, latency: number): Ring { + const ms = Time.Milli.fromSecond((latency / rate) as Time.Second); + const ring = new AudioRingBuffer({ rate, channels: 1, latency: ms }); + return { + insert: (timestamp, data) => ring.write(timestamp, data), + read: (output) => ring.read(output), + reset: () => ring.reset(), + get stalled() { + return ring.stalled; + }, + get timestamp() { + return ring.timestamp; + }, + }; +} + +/** A trace's media timestamp as the container carries it. */ +const micros = (ms: number) => Math.round(ms * 1000) as Time.Micro; + +/** + * Samples each frame carries, by its timestamp: up to where the next one starts, so consecutive + * frames tile the timeline exactly. A frame followed by a hole, or by nothing, gets the trace's + * typical frame. + */ +function frameSamples(trace: Arrival[], rate: number): Map { + const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b); + const sample = (ms: number) => Math.round((ms * rate) / 1000); + const spans = new Map(); + for (let i = 0; i < starts.length - 1; i++) { + if (starts[i + 1] - starts[i] < MAX_FRAME_MS) { + spans.set(micros(starts[i]), sample(starts[i + 1]) - sample(starts[i])); + } + } + const sorted = [...spans.values()].sort((a, b) => a - b); + const typical = sorted[Math.floor(sorted.length / 2)]; + if (typical === undefined) throw new Error("a trace needs two frames less than 100 ms apart"); + for (const start of starts) { + if (!spans.has(micros(start))) spans.set(micros(start), typical); + } + return spans; +} + +/** + * Run every microtask already queued, and every one those queue, before returning. + * + * A macrotask starts only once the microtask queue is empty, and nothing between a group write and + * the ring arms a timer, so this orders the replay behind the consumer rather than waiting on time. + */ +const settle = () => new Promise((resolve) => setImmediate(resolve)); + +/** + * Play `trace` in simulated real time, yielding every render quantum from the first arrival to the + * last. + * + * A group's stream finishes with its last recorded frame. The audio is a constant, never zero, so + * what the ring did not supply is exactly the zeros at the end of a quantum, which is how the + * harness's output tap reads a real page. + */ +export async function* replay(trace: Arrival[], options: Options): AsyncGenerator { + const first = trace[0]; + const last = trace.at(-1); + if (!first || !last) return; + + const { rate } = options; + const samples = frameSamples(trace, rate); + const ring = (options.ring === "shared" ? shared : post)(rate, ringSamples(rate, Time.Milli(options.delay))); + + // Where each group's stream finishes. + const ends = new Map(); + trace.forEach((arrival, i) => { + ends.set(arrival.group, i); + }); + + const track = new Track.Producer("audio"); + const consumer = new Container.Consumer(track.subscribe({ maxAge: WIRE_MAX_AGE }), { + format: new Container.Legacy.Format("audio"), + maxAge: Time.Milli(options.delay), + }); + + // The decoder's read loop, with the codec taken out. A failure is rethrown at the next quantum. + const terminal = new Terminal(); + let failure: { error: unknown } | undefined; + const decode = (async () => { + for (;;) { + const next = await consumer.next(); + if (!next) return; + if (terminal.update(next)) ring.reset(); + if (next.end !== undefined || !next.frame) continue; + + const { timestamp } = next.frame; + const length = samples.get(timestamp); + if (length === undefined) throw new Error(`the consumer delivered an unrecorded frame at ${timestamp} us`); + ring.insert(timestamp, [new Float32Array(length).fill(0.5)]); + } + })().catch((error: unknown) => { + failure = { error }; + }); + + const groups = new Map(); + const step = (QUANTUM / rate) * 1000; + const output = new Float32Array(QUANTUM); + let next = 0; + + try { + for (let n = 1; first.at + (n - 1) * step <= last.at; n++) { + // The quantum ending at `now` renders once everything that arrived by then is delivered. + const now = first.at + n * step; + const written = next; + while (next < trace.length && trace[next].at <= now) { + const arrival = trace[next]; + let group = groups.get(arrival.group); + if (!group) { + group = new Group.Producer(arrival.group); + groups.set(arrival.group, group); + track.writeGroup(group); + } + const timestamp = micros(arrival.timestamp); + group.writeFrame({ + payload: Container.Legacy.encodeFrame(PAYLOAD, timestamp), + timestamp: Time.Timestamp.fromMicros(timestamp), + }); + if (ends.get(arrival.group) === next) { + group.close(); + groups.delete(arrival.group); + } + next++; + } + if (next > written) await settle(); + if (failure) throw failure.error; + + output.fill(0); + ring.read([output]); + yield { at: now, output, stalled: ring.stalled, timestamp: Time.Milli.fromMicro(ring.timestamp) }; + } + } finally { + consumer.close(); + track.close(); + await decode; + } +} diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index 75d43a4203..406dc95d98 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -74,13 +74,17 @@ against it is likely cheaper than patching the branch's: came up on the 100 ms chip, so something is restoring or overriding it. Pin that down: a stored preference silently winning over the default is its own bug, and it also means auto gets far less real exposure than it looks like. -- Replay recorded traces rather than synthetic ones of the same shape. The - #3477 traces are gone, so record fresh ones with the [audio quality - harness](/quest/m0/audio-quality-harness/traces.md) (decided with the - maintainer instead of asking the reporter). Trim a copy into the repository - and replay it through both rings in `replay.test.ts`. +- Replay recorded traces rather than synthetic ones of the same shape. + `test/audio-quality/traces/` holds fresh ones (the #3477 traces are gone), + and `js/watch/src/audio/replay.ts` already plays a trace through + `Container.Consumer` and both rings on a simulated clock at the delay `Sync` + resolves. Replay them in `replay.test.ts`, move the replay's target onto the + estimator (its 100 ms fallback covers the public relay's missing PROBE RTT + until then), and tighten the harness's exact replay budgets with it. - Manual run against the public relay on Chrome and Safari, the two rows the - issue measured. Measure the publisher's audio encoder input-to-output lag in + issue measured. Re-record the `relay-mic` trace in the same run with a real + microphone (`just test audio-quality-record`); the checked-in one used + Chromium's fake capture device. Measure the publisher's audio encoder input-to-output lag in the same run using the reporter's instrumented harness; #3518 fixed the known cause, so the 7.35 s lag and the 88 to 275 ms/s drift the issue reported stand unconfirmed. If drift survives, the suspects are `writeFrame` opening a @@ -95,10 +99,6 @@ reading `probe`; removing it is part of the `SyncInput` reshape in [Plan: A/V clock](/quest/m0/plan-av-clock.md). Land the estimator so that quest can adopt it without a second estimator change. -## Required - -- [Traces](/quest/m0/audio-quality-harness/traces.md) - records the arrival traces this quest replays - ## Related - [Plan: A/V clock](/quest/m0/plan-av-clock.md) - reshapes `SyncInput` around the per-track spread this quest produces diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index dfc3b6150c..a02c524c59 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -25,11 +25,12 @@ rather than merely both passing. The browser lane has landed as `test/audio-quality/`, upstreamed from the reporter's fork on #3477 (`fperex/moq`, branch `debug-findings-solution`) on -its own, without the fork's player, estimator, or shaper changes. That leaves -the two things that needed them: real arrival traces, which the [jitter -target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays, and the -bursty and step profiles, which need shaper features the fork has and `main` -does not. +its own, without the fork's player, estimator, or shaper changes. Real arrival +traces have landed beside it in `test/audio-quality/traces/`, graded as replay +rows through the player's container consumer and rings, and replayed by the +[jitter target's watch quest](/quest/m0/audio-jitter-target/watch.md). That +leaves the bursty and step profiles, which need shaper features the fork has +and `main` does not. Jitter comes from the seeded userspace UDP shaper the transport drills run under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake @@ -43,7 +44,10 @@ audio quality number makes a failure hard to attribute. Nightly, not per-PR: the matrix is jitter profiles by runtime by codec and sample rate, which is more than a merge gate should carry, and `nightly.yml` -already exists for exactly this trade. Budgets are keyed by the full row, since +already exists for exactly this trade. The recorded-trace replays are the +exception: they run in seconds without a browser, so +`.github/workflows/audio-quality.yml` gates every PR touching the player's +packages or the harness on them. Budgets are keyed by the full row, since each of those dimensions moves the expected floor. The budgets checked in with the browser lane (#4426) were recorded locally. @@ -54,7 +58,6 @@ quest](/quest/m0/audio-jitter-target/watch.md). ## Required -- [Traces](/quest/m0/audio-quality-harness/traces.md) - real arrival traces recorded, checked in, and graded as replay rows - [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/traces.md b/quest/m0/audio-quality-harness/traces.md deleted file mode 100644 index ce0b749962..0000000000 --- a/quest/m0/audio-quality-harness/traces.md +++ /dev/null @@ -1,33 +0,0 @@ -# [M] Real arrival traces are recorded, checked in, and graded - -## Goal - -A trimmed fixture of real audio arrival traces lives in the repository, and the -audio quality harness grades a replay of it next to its synthetic profiles. The -[jitter target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays -the same fixture through both rings in `replay.test.ts`. - -## Plan - -The #3477 traces are gone, so record fresh ones: a viewer on the public relay -playing `bbb.hang` (the AAC flush-span shape the issue measured), a local relay -as the shallow control, and a browser microphone published through the public -relay. Stamp each audio frame's arrival where the container consumer receives -it, with its media timestamp and group, on the viewer's monotonic clock (see -`test/audio-quality/clients/js/src/schema.ts`). - -The browser lane reads only public signals, and no public signal carries a -per-frame arrival, so decide where the recorder lives: a debug hook in -`@moq/watch` that the harness page enables, or a native subscriber (`moq` -CLI) that records the same path's arrivals. The reporter's fork (`fperex/moq`, -branch `debug-findings-solution`) recorded from a debug sampler in the player -and replayed through `js/watch/src/audio/replay.ts` on a simulated clock; -that replay module is the likely shape for the grading side, and landing it is -a player change this quest owns. - -Replay rows are deterministic, so their budgets are exactly what was measured, -with no headroom. - -## Related - -- [Latency ledger](/quest/m2/latency-ledger.md) - would promote an arrival stamp into public API diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 242660731f..d351a84bba 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -12,11 +12,13 @@ just test audio-quality --profiles mild --codecs opus --duration 20 just test audio-quality --rings plain # the production path only just test audio-quality --out ~/runs/after # keep the run directory just test audio-quality --enforce # fail on the budgets +just test audio-quality --replays # the replays only, in seconds ``` -The matrix is codec x profile x ring: 16 rows, about 20 minutes. `--profiles`, `--rings`, and -`--codecs` take comma-separated lists; `--seed` replays a given impairment. The table prints either -way, and only `--enforce` (which the nightly job passes) turns a breach into a failed run. +The matrix is codec x profile x ring: 16 browser rows, about 20 minutes, plus 6 [replay +rows](#traces) of recorded arrivals. `--profiles`, `--rings`, and `--codecs` take comma-separated +lists; `--seed` replays a given impairment. The table prints either way, and only `--enforce` (which +the nightly job passes) turns a breach into a failed run. ## Rows @@ -66,6 +68,43 @@ dials, so the driver answers that fetch with the relay's own hash (the certifica hash, so a different port needs nothing more). With nothing listening on TCP there, the WebSocket fallback cannot connect: the session is WebTransport through the shaper, or the row is void. +## Traces + +The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, each graded as a +`replay` row whose profile is the file's name: + +| Trace | Recorded from | +| --- | --- | +| `relay-bbb` | the public relay's `bbb.hang`: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured | +| `local-aac` | this harness's AAC publisher through a local relay: the shallow control | +| `relay-mic` | `` in headless Chromium, publishing its fake capture device through the public relay | + +A trace is a [`Trace`](clients/js/src/schema.ts): the catalog's audio rendition, the smallest +PROBE round trip the session saw, and every frame's `[at, timestamp, group]`, with `at` on the +viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, the way +`Container.Consumer` receives it, using only public `@moq/net` and `@moq/hang` in the same Chromium +over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend +on the delay, so the replay decides those again. + +`replay.ts` plays a trace through the player's own `Container.Consumer` and rings on a simulated +clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a +real `Sync` resolves for the recorded catalog and round trip, and reads each quantum through the +tap's classifier. The consumer makes the player's group ordering, max age skips, and discontinuity +resets again at that delay; a group's stream is taken to finish with its last recorded frame. It +writes the same samples a page does, so `analyze.ts` reduces both alike, and the shaper, transport, +and clock checks have nothing to void. Decoding is taken as instant. A replay is deterministic, so +its budgets are exactly what it measured. + +```bash +just test audio-quality-record # all three, 35 s each, needs the network +just test audio-quality-record --lanes relay-bbb +``` + +Recording is by hand: the public relay and a browser publisher are not something a nightly run +should depend on. Re-measure the replay budgets after re-recording. The recorder asks the relay for +ten seconds of backlog so a late group is kept, which makes a trace's tune-in heavier than a +player's; the warmup excludes it from the grade. + ## The metric contract [`clients/js/src/schema.ts`](clients/js/src/schema.ts) is the contract. The page emits `Sample`s, @@ -130,7 +169,8 @@ holds by construction. ## Void rules -A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. +A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. A +replay row crosses no shaper, so the `shaper` void does not apply to it. | Void | Why | | --- | --- | @@ -149,14 +189,19 @@ unmeasured, a void row, or a row with no budget. Tightening a ceiling is a visib one needs a reason in review. The file's `note` says how the current ceilings were measured. The nightly `Audio quality` job runs the matrix under `--enforce` and keeps a failing run directory -for a week: each process's log, the shaper's counters, every row's raw samples and summary. +for a week: each process's log, the shaper's counters, every row's raw samples and summary. The +replays also run under `--enforce` on every PR that touches the player's packages (`js/watch`, +`js/hang`, `js/net`, `js/signals`) or the harness (`.github/workflows/audio-quality.yml`), so a +change that moves them updates `budgets.json` in the same PR. ## Layout ```text run.sh builds, starts the relay and publishers, runs each row behind the shaper +record.sh records traces/, by hand relay.toml anonymous relay, self-signed cert budgets.json a ceiling per metric per row +traces/ recorded arrivals, one replay profile each clients/js/ index.html src/page.ts the watch element, driven from the query string src/probe.ts samples its public signals every 250 ms @@ -164,6 +209,9 @@ clients/js/ src/schema.ts the metric contract src/analyze.ts samples to a summary, unit tested driver.ts one row in headless Chromium, and its void checks + recorder.html mic.html the recorder and the microphone publisher (src/recorder.ts, src/mic.ts) + record.ts one trace in headless Chromium + replay.ts the replay rows, written as a page's samples analyze.ts grade.ts the summary per row, then the table and the verdict ``` @@ -179,5 +227,5 @@ the profiles are the ones the shaper on `main` can express. ## Not covered Safari and Firefox, video, and perceptual scoring: the grade is glitches and latency, not an opinion -about how it sounds. Recorded arrival traces are -[their own quest](../../quest/m0/audio-quality-harness/traces.md). +about how it sounds. The `relay-mic` trace captures Chromium's fake device rather than a real +microphone, so it carries a browser publisher's encode and send cadence but not a sound card's. diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 0cdde6b25c..6d212be1f5 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs.", + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-09-29, with no headroom: any regression fails them, and an improvement should be tightened in.", "rows": [ { "runtime": "chromium", @@ -224,6 +224,90 @@ "stalled_share": 0.49, "silence_share": 0.75, "target_ms_p95": 82 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "local-aac", + "ring": "isolated", + "underruns_per_min": 103.7, + "underrun_ms_per_min": 293.2, + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 94.4, + "stalled_share": 0, + "silence_share": 0.005, + "target_ms_p95": 186 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "local-aac", + "ring": "plain", + "underruns_per_min": 103.7, + "underrun_ms_per_min": 293.2, + "skip_aheads_per_min": 2, + "discarded_ms_per_min": 94.4, + "stalled_share": 0, + "silence_share": 0.005, + "target_ms_p95": 186 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "relay-bbb", + "ring": "isolated", + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0, + "target_ms_p95": 289 + }, + { + "runtime": "replay", + "codec": "aac", + "rate": 44100, + "profile": "relay-bbb", + "ring": "plain", + "underruns_per_min": 0, + "underrun_ms_per_min": 0, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0, + "target_ms_p95": 289 + }, + { + "runtime": "replay", + "codec": "opus", + "rate": 48000, + "profile": "relay-mic", + "ring": "isolated", + "underruns_per_min": 227.8, + "underrun_ms_per_min": 593.2, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0.01, + "target_ms_p95": 123 + }, + { + "runtime": "replay", + "codec": "opus", + "rate": 48000, + "profile": "relay-mic", + "ring": "plain", + "underruns_per_min": 227.8, + "underrun_ms_per_min": 593.2, + "skip_aheads_per_min": 0, + "discarded_ms_per_min": 0, + "stalled_share": 0, + "silence_share": 0.01, + "target_ms_p95": 123 } ] } diff --git a/test/audio-quality/clients/js/mic.html b/test/audio-quality/clients/js/mic.html new file mode 100644 index 0000000000..43c0cb2d81 --- /dev/null +++ b/test/audio-quality/clients/js/mic.html @@ -0,0 +1,10 @@ + + + + + moq audio quality microphone + + + + + diff --git a/test/audio-quality/clients/js/package.json b/test/audio-quality/clients/js/package.json index 9534e140ed..dd2918b166 100644 --- a/test/audio-quality/clients/js/package.json +++ b/test/audio-quality/clients/js/package.json @@ -7,6 +7,10 @@ "test": "bun test --only-failures" }, "dependencies": { + "@moq/hang": "workspace:*", + "@moq/net": "workspace:*", + "@moq/publish": "workspace:*", + "@moq/signals": "workspace:*", "@moq/watch": "workspace:*" }, "devDependencies": { diff --git a/test/audio-quality/clients/js/record.ts b/test/audio-quality/clients/js/record.ts new file mode 100644 index 0000000000..ca4f6008f5 --- /dev/null +++ b/test/audio-quality/clients/js/record.ts @@ -0,0 +1,143 @@ +/** + * Records one arrival trace in headless Chromium, and writes it as a checked-in `Trace`. + * + * The recorder page subscribes to the broadcast's audio and stamps every frame the container consumer + * hands over. With `--mic`, a second page first publishes Chromium's fake capture device through + * `` to the same relay and broadcast, so the trace is a browser publisher's cadence. + * + * bun record.ts --page dist --url https://cdn.moq.dev/demo --broadcast bbb.hang --duration 35 \ + * --source "..." --description "..." --out ../../traces/relay-bbb.json + * + * The recording starts at the first arrival, tune-in burst included, since a player tunes in too. + * + * @module + */ +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { launch, sleep } from "../../../interop/clients/js/harness.ts"; +import type { Arrival, Trace } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + page: { type: "string" }, + url: { type: "string" }, + broadcast: { type: "string" }, + duration: { type: "string", default: "35" }, + source: { type: "string" }, + description: { type: "string" }, + mic: { type: "boolean", default: false }, + out: { type: "string" }, + }, +}); + +const { page: pageDir, url, broadcast, source, description, out } = values; +const durationMs = Number(values.duration) * 1000; +if (!pageDir || !url || !broadcast || !source || !description || !out || !Number.isFinite(durationMs) || durationMs <= 0) { + console.error( + "usage: record.ts --page DIR --url U --broadcast B --source S --description D --out FILE [--duration S] [--mic]", + ); + process.exit(2); +} + +/** How long the session and the first arrival get before the recording is abandoned. */ +const STARTUP_MS = 30_000; + +// Cross-origin isolated, for the 5 us `performance.now()` a non-isolated page coarsens to 100 us. +const root = resolve(pageDir); +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const file = Bun.file(join(root, new URL(req.url).pathname)); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { + headers: { "cross-origin-opener-policy": "same-origin", "cross-origin-embedder-policy": "require-corp" }, + }); + }, +}); +const pageUrl = (file: string) => `http://127.0.0.1:${server.port}/${file}?${new URLSearchParams({ url, broadcast })}`; + +/** Poll the page until `ready` returns a value, or throw naming what never happened. */ +async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { + const deadline = Date.now() + STARTUP_MS; + while (Date.now() < deadline) { + const error = await page.evaluate(() => globalThis.recorder?.error()); + if (error) throw new Error(`recorder: ${error}`); + const value = await page.evaluate(ready); + if (value !== undefined && value !== null) return value; + await sleep(200); + } + throw new Error(`${what} not within ${STARTUP_MS / 1000}s`); +} + +const browser = await launch(["--use-fake-device-for-media-stream", "--use-fake-ui-for-media-stream"]); +let failed = false; +try { + const context = await browser.newContext({ permissions: ["microphone", "camera"] }); + if (values.mic) { + const publisher = await context.newPage(); + publisher.on("pageerror", (error) => console.error(`[mic error] ${error.message}`)); + await publisher.goto(pageUrl("mic.html"), { waitUntil: "load" }); + // Subscribing before the element has an audio rendition finds no catalog to read. + const deadline = Date.now() + STARTUP_MS; + while (!(await publisher.evaluate(() => document.querySelector("moq-publish")?.audio.out.catalog.peek()))) { + if (Date.now() > deadline) throw new Error(`the microphone never published within ${STARTUP_MS / 1000}s`); + await sleep(200); + } + await sleep(2000); + } + + const page = await context.newPage(); + page.on("pageerror", (error) => console.error(`[recorder error] ${error.message}`)); + page.on("console", (msg) => { + if (msg.type() === "error" || msg.type() === "warning") console.error(`[recorder ${msg.type()}] ${msg.text()}`); + }); + await page.goto(pageUrl("recorder.html"), { waitUntil: "load" }); + + const info = await waitFor(page, "an audio subscription", () => globalThis.recorder.info()); + const arrivals: Arrival[] = []; + await waitFor(page, "a first arrival", () => { + const drained = globalThis.recorder.drain(); + return drained.length > 0 ? drained : undefined; + }).then((first) => arrivals.push(...first)); + + // Ends on the page's clock, not on the next arrival, so a publisher that goes quiet is recorded + // as the silence it is instead of holding the recording open. + const start = arrivals[0][0]; + while ((await page.evaluate(() => performance.now())) < start + durationMs) { + await sleep(1000); + const error = await page.evaluate(() => globalThis.recorder.error()); + if (error) throw new Error(`recorder: ${error}`); + arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); + } + arrivals.push(...(await page.evaluate(() => globalThis.recorder.drain()))); + // The RTT is read last: the PROBE minimum only falls as the session runs. + const rtt = await page.evaluate(() => globalThis.recorder.info()?.rtt ?? null); + + const trimmed = arrivals.filter(([at]) => at - start <= durationMs); + const round = (x: number, places: number) => Math.round(x * 10 ** places) / 10 ** places; + const trace: Trace = { + version: 1, + source, + description, + transport: info.transport, + rtt: rtt === null ? null : round(rtt, 1), + config: info.config as Trace["config"], + arrivals: trimmed.map(([at, timestamp, group]) => [round(at - start, 2), round(timestamp, 3), group]), + }; + // One arrival per line, so the file reads as a table. + const { arrivals: rows, ...header } = trace; + const head = JSON.stringify(header, null, "\t").slice(0, -2); + await Bun.write(out, `${head},\n\t"arrivals": [\n${rows.map((r) => `\t\t${JSON.stringify(r)}`).join(",\n")}\n\t]\n}\n`); + console.log( + `${out}: ${trace.arrivals.length} arrivals over ${round(durationMs / 1000, 1)} s, ${trace.config.codec} at ${trace.config.sampleRate} Hz, ${trace.transport}, rtt ${trace.rtt} ms`, + ); +} catch (err) { + failed = true; + console.error(err instanceof Error ? err.message : String(err)); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} +process.exit(failed ? 1 : 0); diff --git a/test/audio-quality/clients/js/recorder.html b/test/audio-quality/clients/js/recorder.html new file mode 100644 index 0000000000..f3185f8b36 --- /dev/null +++ b/test/audio-quality/clients/js/recorder.html @@ -0,0 +1,10 @@ + + + + + moq audio quality recorder + + + + + diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts new file mode 100644 index 0000000000..28e2d56a6d --- /dev/null +++ b/test/audio-quality/clients/js/replay.ts @@ -0,0 +1,157 @@ +/** + * Replays the recorded traces through the player's consumer and rings, and writes each row the way + * `driver.ts` writes a browser row, so `analyze.ts` reduces both alike. + * + * The player half is `js/watch/src/audio/replay.ts`: the real container consumer and rings on a + * simulated clock, at the delay a real `Sync` resolves for the recorded catalog and RTT. This half + * reads its quanta through the same classifier the page's output tap runs, and samples the ring + * every 250 ms of simulated time, as the probe samples a page. Every profile is the trace's name at + * the "auto" delay a viewer gets by default. + * + * bun replay.ts --traces ../../traces --profiles relay-bbb --rings plain --codecs aac --list + * bun replay.ts --traces ../../traces --out + * + * `--list` prints the row tags; otherwise it writes `.ndjson` and `.page.json` per row. + * + * @module + */ +import { readdirSync, readFileSync } from "node:fs"; +import { basename, join } from "node:path"; +import { parseArgs } from "node:util"; +import type * as Catalog from "@moq/hang/catalog"; +import { classify, Ledger } from "./src/quantum.ts"; +import { + type Environment, + type Ring, + type Row, + rowKey, + SAMPLE_INTERVAL_MS, + type Sample, + SILENCE_RMS, + type Stall, + type Trace, + traceCodec, +} from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + traces: { type: "string" }, + profiles: { type: "string" }, + rings: { type: "string", default: "isolated,plain" }, + codecs: { type: "string", default: "opus,aac" }, + out: { type: "string" }, + list: { type: "boolean", default: false }, + }, +}); +if (!values.traces || (!values.list && !values.out)) { + console.error("usage: replay.ts --traces DIR [--profiles a,b] [--rings r] [--codecs c] (--list | --out DIR)"); + process.exit(2); +} +const dir = values.traces; +const split = (list: string) => list.split(",").filter((s) => s !== ""); + +const traces = readdirSync(dir) + .filter((f) => f.endsWith(".json")) + .sort() + .map((f) => ({ profile: basename(f, ".json"), trace: JSON.parse(readFileSync(join(dir, f), "utf8")) as Trace })); +const profiles = values.profiles === undefined ? undefined : split(values.profiles); +const rings = split(values.rings); +const codecs = split(values.codecs); +const selectors: [string, string[], string[]][] = [ + ["profile", profiles ?? [], traces.map((t) => t.profile)], + ["ring", rings, ["isolated", "plain"]], + ["codec", codecs, ["opus", "aac"]], +]; +for (const [name, list, known] of selectors) { + const unknown = list.find((v) => !known.includes(v)); + if (unknown !== undefined) { + console.error(`unknown ${name} '${unknown}' (known: ${known.join(", ")})`); + process.exit(2); + } +} + +const rows: { row: Row; trace: Trace }[] = []; +for (const { profile, trace } of traces) { + if (trace.version !== 1) throw new Error(`${profile}: trace version ${trace.version}, expected 1`); + const codec = traceCodec(trace); + if (profiles && !profiles.includes(profile)) continue; + if (!codecs.includes(codec)) continue; + for (const ring of rings as Ring[]) { + rows.push({ row: { runtime: "replay", codec, rate: trace.config.sampleRate, profile, ring }, trace }); + } +} + +if (values.list) { + for (const { row } of rows) console.log(rowKey(row)); + process.exit(0); +} +const out = values.out as string; + +// Loaded only to play: listing runs before `run.sh` has installed the workspace the player needs. +const { replay, target } = await import("../../../../js/watch/src/audio/replay.ts"); + +for (const { row, trace } of rows) { + const tag = rowKey(row); + const rate = row.rate; + const delay = await target({ + delay: "auto", + config: trace.config as unknown as Catalog.AudioConfig, + rtt: trace.rtt ?? undefined, + }); + + const ledger = new Ledger(rate, SILENCE_RMS); + const samples: Sample[] = []; + let stalls: Stall[] = []; + let stalled = true; + let frame = 0; + let next = SAMPLE_INTERVAL_MS; + + // The consumer's warnings, as the page's probe keeps a browser row's console. + const notes: string[] = []; + const warn = console.warn; + console.warn = (...args: unknown[]) => { + if (notes.length < 200) notes.push(args.map(String).join(" ").slice(0, 200)); + }; + + const arrivals = trace.arrivals.map(([at, timestamp, group]) => ({ at, timestamp, group })); + const quanta = replay(arrivals, { ring: row.ring === "isolated" ? "shared" : "post", rate, delay }); + for await (const quantum of quanta) { + // The render clock is the simulated one: the trace's `at`, which starts at zero. + ledger.add(frame, quantum.output.length, classify([quantum.output])); + frame += quantum.output.length; + if (quantum.stalled !== stalled) { + stalled = quantum.stalled; + stalls.push({ at: quantum.at, stalled }); + } + + if (quantum.at >= next) { + next += SAMPLE_INTERVAL_MS; + const gaps = ledger.take(); + samples.push({ + at: quantum.at, + render: quantum.at, + timestamp: quantum.timestamp, + stalled: quantum.stalled, + delay, + quanta: ledger.counts.quanta, + quiet: ledger.counts.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: stalls.length > 0 ? stalls : undefined, + }); + stalls = []; + } + } + console.warn = warn; + + const environment: Environment = { + crossOriginIsolated: row.ring === "isolated", + transport: trace.transport, + codec: trace.config.codec, + rate, + jitter: trace.config.jitter, + contextRate: rate, + }; + await Bun.write(join(out, `${tag}.ndjson`), `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); + await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids: [] }, null, 1)); + console.log(`${tag}: ${samples.length} samples at a ${delay} ms target`); +} diff --git a/test/audio-quality/clients/js/src/analyze.ts b/test/audio-quality/clients/js/src/analyze.ts index 9ee7d6c62c..84d398b6bd 100644 --- a/test/audio-quality/clients/js/src/analyze.ts +++ b/test/audio-quality/clients/js/src/analyze.ts @@ -142,8 +142,11 @@ export function analyze(input: Input): Summary { const notes = [...input.notes]; // ── the shaper's verdict ──────────────────────────────────────────────── + // A replay row never crosses one: its path is the recording. const shaper = input.shaper; - if (!shaper) { + if (row.runtime === "replay") { + // Nothing to check. + } else if (!shaper) { voids.push({ assertion: "shaper", detail: "left no report, so nothing says the path was shaped" }); } else if (shaper.status !== 0) { voids.push({ assertion: "shaper", detail: `exited ${shaper.status}: the profile never acted as configured` }); diff --git a/test/audio-quality/clients/js/src/mic.ts b/test/audio-quality/clients/js/src/mic.ts new file mode 100644 index 0000000000..ae536caa9c --- /dev/null +++ b/test/audio-quality/clients/js/src/mic.ts @@ -0,0 +1,27 @@ +/** + * The publisher page for the microphone trace: the demo's `` capturing audio only. + * + * Whatever device Chromium offers is captured through the element's real path (getUserMedia, the + * capture worklet, WebCodecs Opus), so the trace carries a browser publisher's own cadence. Under + * `record.ts` that device is Chromium's fake one, a tone. + * + * ?url=https://cdn.moq.dev/anon&broadcast=aq-mic-1234.hang + * + * @module + */ +import "@moq/publish/element"; + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const publish = document.createElement("moq-publish"); +publish.setAttribute("url", required("url")); +publish.setAttribute("name", required("broadcast")); +publish.setAttribute("source", "camera"); +publish.setAttribute("invisible", ""); +publish.setAttribute("preview", "none"); +document.body.appendChild(publish); diff --git a/test/audio-quality/clients/js/src/recorder.ts b/test/audio-quality/clients/js/src/recorder.ts new file mode 100644 index 0000000000..c0494c5cc1 --- /dev/null +++ b/test/audio-quality/clients/js/src/recorder.ts @@ -0,0 +1,130 @@ +/** + * The recorder page: subscribes to a broadcast's audio and stamps each frame as it comes off its + * group's stream, which is where the player's container consumer receives it. + * + * Built only on the public `@moq/net` and `@moq/hang` surface, so it records the path a viewer's + * player takes (a WebTransport session in Chromium, every group on its own stream) without a debug + * hook in the player. It reads groups the way `Container.Consumer` does, concurrently and decoded + * with the same format, but stamps them before the consumer's in-order delivery: a group the + * consumer holds back waiting for a missing one, or skips for age, is a decision the replay makes + * again at its own delay, and baking one in would censor the trace. + * + * ?url=https://cdn.moq.dev/demo&broadcast=bbb.hang + * + * `record.ts` drains it through `globalThis.recorder`. + * + * @module + */ +import * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import * as Moq from "@moq/net"; +import type { Arrival } from "./schema.ts"; + +/** How far back the relay may serve a group. Long, so a late group is recorded rather than dropped. */ +const MAX_AGE = Moq.Time.Milli(10_000); + +/** What the recorder has, until the driver drains it. */ +export type Recorder = { + /** Arrivals since the last drain, `at` on this page's `performance.now()`. */ + drain(): Arrival[]; + /** The session and rendition, once subscribed. */ + info(): { transport: string; rtt: number | null; config: Catalog.AudioConfig } | undefined; + /** The failure that stopped recording, if one did. */ + error(): string | undefined; +}; + +declare global { + var recorder: Recorder; +} + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const arrivals: Arrival[] = []; +let info: ReturnType; +let failure: string | undefined; +const fail = (err: unknown) => { + failure ??= err instanceof Error ? err.message : String(err); + console.error("recorder:", failure); +}; + +globalThis.recorder = { + drain: () => arrivals.splice(0, arrivals.length), + info: () => info, + error: () => failure, +}; + +/** The container format the player's decoder would pick for `config`. */ +function format(config: Catalog.AudioConfig): Container.Format { + const container = config.container; + if (container.kind === "legacy") return new Container.Legacy.Format(config); + if (container.kind === "loc") return new Container.Loc.Format("audio"); + if (container.kind === "cmaf") { + const init = Uint8Array.from(atob(container.init), (c) => c.charCodeAt(0)); + return new Container.Cmaf.Format(Container.Cmaf.decodeInitSegment(init)); + } + throw new Error(`unsupported container: ${JSON.stringify(container)}`); +} + +/** Stamp every frame of one group as it is read. Markers carry no media, so they are not arrivals. */ +async function readGroup(group: Moq.Group.Consumer, decoder: Container.Format): Promise { + for (;;) { + const next = await group.readFrame(); + const at = performance.now(); + if (!next) return; + for (const frame of decoder.decode(next.payload)) { + if (decoder.end?.(frame) !== undefined) continue; + arrivals.push([at, frame.timestamp / 1000, group.sequence]); + } + } +} + +async function record(): Promise { + const origin = new Moq.Origin.Producer(); + const connection = await Moq.Connection.connect({ url: new URL(required("url")), consume: origin }); + + let rtt: number | null = null; + connection.probe.subscribe((probe) => { + if (probe.rtt !== undefined) rtt = rtt === null ? probe.rtt : Math.min(rtt, probe.rtt); + }); + + const request = origin.request(Moq.Path.from(required("broadcast"))); + let broadcast = request.active.peek(); + while (!broadcast) { + await request.active.changed(); + broadcast = request.active.peek(); + } + + let root: Catalog.Root | undefined; + for await (const update of Catalog.watch(broadcast)) { + if (Object.keys(update.audio?.renditions ?? {}).length > 0) { + root = update; + break; + } + } + // Every recorded broadcast carries one audio rendition, so the first is the one a player picks. + const [name, config] = Object.entries(root?.audio?.renditions ?? {})[0] ?? []; + if (!name || !config) throw new Error("the catalog ended without an audio rendition"); + + const decoder = format(config); + const track = broadcast.track(name).subscribe({ priority: Catalog.PRIORITY.audio, maxAge: MAX_AGE }); + info = { + transport: connection.transport, + get rtt() { + return rtt; + }, + config, + }; + + for (;;) { + const group = await track.recvGroup(); + if (!group) throw new Error("the track ended"); + readGroup(group, decoder).catch(fail); + } +} + +record().catch(fail); diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index 3e36798d1b..ca17583f6c 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -239,6 +239,12 @@ export type Ring = "isolated" | "plain"; /** The audio codecs the matrix publishes. */ export type Codec = "opus" | "aac"; +/** + * Where a row played: a headless browser over the shaper, or a {@link Trace} replayed through the + * player's rings on a simulated clock, whose profile is the trace's name. + */ +export type Runtime = "chromium" | "replay"; + /** * One matrix cell. * @@ -247,7 +253,7 @@ export type Codec = "opus" | "aac"; * threshold. */ export type Row = { - runtime: "chromium"; + runtime: Runtime; codec: Codec; /** Sample rate in Hz. */ rate: number; @@ -266,7 +272,7 @@ export function parseRow(key: string): Row { const profile = parts.slice(3, -1).join("-"); const hz = Number.parseInt(rate ?? "", 10); if ( - runtime !== "chromium" || + (runtime !== "chromium" && runtime !== "replay") || (codec !== "opus" && codec !== "aac") || (ring !== "isolated" && ring !== "plain") || !Number.isFinite(hz) || @@ -277,6 +283,48 @@ export function parseRow(key: string): Row { return { runtime, codec, rate: hz, profile, ring }; } +// ── what the recorder keeps ───────────────────────────────────────────────── + +/** + * One audio frame reaching the viewer's container consumer: when, on the `viewer` clock; its + * timestamp, on the `media` clock; and the group that carried it. + */ +export type Arrival = [at: Ms, timestamp: Ms, group: number]; + +/** A recorded arrival trace, as checked in under `traces/`. The file name is its profile. */ +export type Trace = { + /** Bumped when a field's meaning changes. */ + version: 1; + /** Where it was recorded: the relay, the broadcast, its publisher, the date. */ + source: string; + /** The shape it carries, and why it is kept. */ + description: string; + /** The transport the session negotiated. */ + transport: string; + /** The smallest round trip the connection's PROBE reported, which "auto" sizes from. */ + rtt: Ms | null; + /** The audio rendition, as the catalog advertised it. */ + config: { + codec: string; + sampleRate: number; + numberOfChannels: number; + container: { kind: string }; + jitter?: number; + delay?: number; + [key: string]: unknown; + }; + /** Every arrival in order, `at` counted from the first. */ + arrivals: Arrival[]; +}; + +/** The codec a trace's rendition is, as a row names it. */ +export function traceCodec(trace: Trace): Codec { + const codec = trace.config.codec; + if (codec === "opus") return "opus"; + if (codec.startsWith("mp4a")) return "aac"; + throw new Error(`no row codec for ${codec}`); +} + // ── what the page emits ───────────────────────────────────────────────────── /** A run of consecutive quanta the ring did not fill, as the output tap saw it. */ diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json index 5a536d16f4..b61d3dce46 100644 --- a/test/audio-quality/clients/js/tsconfig.json +++ b/test/audio-quality/clients/js/tsconfig.json @@ -3,5 +3,14 @@ "compilerOptions": { "types": ["audioworklet", "bun", "vite/client"] }, - "include": ["analyze.ts", "driver.ts", "grade.ts", "src", "vite.config.ts", "../../../../js/common/worklet.d.ts"] + "include": [ + "analyze.ts", + "driver.ts", + "grade.ts", + "record.ts", + "replay.ts", + "src", + "vite.config.ts", + "../../../../js/common/worklet.d.ts" + ] } diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts index c971d84052..68040577ab 100644 --- a/test/audio-quality/clients/js/vite.config.ts +++ b/test/audio-quality/clients/js/vite.config.ts @@ -11,9 +11,16 @@ import { defineConfig } from "vite"; import { workletInline } from "../../../../js/common/vite-plugin-worklet"; -/** esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. */ +/** + * esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. Three pages: the player + * under measurement, and the recorder and microphone publisher that `record.ts` drives. + */ export default defineConfig({ base: "./", plugins: [workletInline()], - build: { target: "esnext", outDir: "dist" }, + build: { + target: "esnext", + outDir: "dist", + rollupOptions: { input: ["index.html", "recorder.html", "mic.html"] }, + }, }); diff --git a/test/audio-quality/record.sh b/test/audio-quality/record.sh new file mode 100755 index 0000000000..2211436969 --- /dev/null +++ b/test/audio-quality/record.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# Records the arrival traces under traces/, which the replay rows grade. See README.md. +# +# Three lanes, each --duration seconds from the first arrival: +# relay-bbb the public relay's Big Buck Bunny, the AAC flush-span shape #3477 measured +# local-aac an AAC tone through a local relay, the shallow control +# relay-mic a browser publishing Chromium's fake capture device through the public relay +# +# Not part of a run: re-record by hand when the path or the publishers change, and re-measure the +# replay budgets after. +set -euo pipefail + +AQ_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$AQ_DIR/../.." && pwd) +CLIENT="$AQ_DIR/clients/js" + +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$AQ_DIR/../lib/harness.sh" + +RERUN="just test audio-quality-record$(harness_argv "$@")" + +ALL_LANES=(relay-bbb local-aac relay-mic) +LANES=("${ALL_LANES[@]}") +DURATION=35 +PUBLIC="https://cdn.moq.dev" + +while [[ $# -gt 0 ]]; do + case "$1" in + --lanes) + IFS=, read -r -a LANES <<<"${2:?--lanes requires a value}" + shift 2 + ;; + --duration) + DURATION="${2:?--duration requires a value}" + shift 2 + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done +for lane in "${LANES[@]}"; do + [[ " ${ALL_LANES[*]} " == *" $lane "* ]] || { + echo "error: unknown lane '$lane' (known: ${ALL_LANES[*]})" >&2 + exit 2 + } +done + +harness_begin audio-quality-record "$RERUN" + +echo "building the pages..." +( + cd "$CLIENT" + bun install --frozen-lockfile + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir +) + +DATE=$(date -u +%Y-%m-%d) +record() { + local lane="$1" + shift + echo "" + echo "── $lane ──" + bun "$CLIENT/record.ts" --page "$HARNESS_RUN/page" --duration "$DURATION" \ + --out "$AQ_DIR/traces/$lane.json" "$@" +} + +for lane in "${LANES[@]}"; do + case "$lane" in + relay-bbb) + record "$lane" --url "$PUBLIC/demo" --broadcast bbb.hang \ + --source "$PUBLIC/demo bbb.hang, the public demo's MPEG-TS import, recorded $DATE" \ + --description "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured." + ;; + local-aac) + (cd "$WORKSPACE" && cargo build --locked -p moq-relay -p moq-cli) + target="${CARGO_TARGET_DIR:-$WORKSPACE/target}/debug" + harness_port relay + url="http://127.0.0.1:$HARNESS_PORT" + sed "s/4443/$HARNESS_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" + harness_spawn relay "$HARNESS_RUN/relay.log" "$target/moq-relay" "$HARNESS_RUN/relay.toml" + harness_ready "$url/certificate.sha256" 30 "$HARNESS_PID" || { + echo "relay never became ready" >&2 + exit 1 + } + # The harness's own AAC publisher, so the control is the shape its rows play. + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$target/moq" --connect "$url" --broadcast tone-aac.hang import ts + } + harness_spawn pub-aac "$HARNESS_RUN/pub-aac.log" publish + record "$lane" --url "$url" --broadcast tone-aac.hang \ + --source "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded $DATE" \ + --description "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer." + ;; + relay-mic) + broadcast="aq-mic-$RANDOM$RANDOM.hang" + record "$lane" --mic --url "$PUBLIC/anon" --broadcast "$broadcast" \ + --source "$PUBLIC/anon, in headless Chromium capturing its fake audio device, recorded $DATE" \ + --description "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between." + ;; + esac +done diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index c10544597e..1416a24dcd 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -19,7 +19,13 @@ ALL_PROFILES=(near-zero mild wide fixed-250) ALL_RINGS=(isolated plain) ALL_CODECS=(opus aac) -PROFILES=("${ALL_PROFILES[@]}") +# Every recorded trace is a profile too, replayed through the rings instead of played in a browser. +ALL_TRACES=() +for trace in "$AQ_DIR"/traces/*.json; do + ALL_TRACES+=("$(basename "$trace" .json)") +done + +PROFILES=("${ALL_PROFILES[@]}" "${ALL_TRACES[@]}") RINGS=("${ALL_RINGS[@]}") CODECS=("${ALL_CODECS[@]}") DURATION=60 @@ -50,6 +56,11 @@ while [[ $# -gt 0 ]]; do PROFILES=("${SPLIT[@]}") shift 2 ;; + --replays) + # Only the recorded traces: seconds, and no browser, relay, or shaper. + PROFILES=("${ALL_TRACES[@]}") + shift + ;; --rings) need "$1" $# "${2:-}" split "$2" @@ -111,7 +122,7 @@ valid() { echo "error: unknown $name '$want' (known: $*)" >&2 exit 2 } -for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}"; done +for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}" "${ALL_TRACES[@]}"; done for r in "${RINGS[@]}"; do valid "$r" ring "${ALL_RINGS[@]}"; done for c in "${CODECS[@]}"; do valid "$c" codec "${ALL_CODECS[@]}"; done @@ -147,21 +158,39 @@ shaper_of() { } ROWS=() -for codec in "${CODECS[@]}"; do - for profile in "${PROFILES[@]}"; do +TRACES=() +for profile in "${PROFILES[@]}"; do + if [[ " ${ALL_TRACES[*]} " == *" $profile "* ]]; then + TRACES+=("$profile") + continue + fi + for codec in "${CODECS[@]}"; do for ring in "${RINGS[@]}"; do ROWS+=("chromium-$codec-$(rate_of "$codec")-$profile-$ring") done done done +# A trace's codec and rate are its recording's, so replay.ts names those rows. +joined() { + local IFS=, + echo "$*" +} +replay=(bun "$CLIENT/replay.ts" --traces "$AQ_DIR/traces" --profiles "$(joined "${TRACES[@]}")") +replay+=(--rings "$(joined "${RINGS[@]}")" --codecs "$(joined "${CODECS[@]}")") +listed=$("${replay[@]}" --list) +mapfile -t REPLAYS <<<"$listed" +[[ -n "$listed" ]] || REPLAYS=() + if [[ $LIST -eq 1 ]]; then - printf '%s\n' "${ROWS[@]}" - echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED" >&2 + printf '%s\n' "${ROWS[@]}" "${REPLAYS[@]}" + echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED; ${#REPLAYS[@]} replays of their recorded trace" >&2 exit 0 fi -for tool in cargo bun ffmpeg; do +tools=(bun) +[[ ${#ROWS[@]} -eq 0 ]] || tools+=(cargo ffmpeg) +for tool in "${tools[@]}"; do command -v "$tool" >/dev/null 2>&1 || { echo "error: $tool not found; run inside 'nix develop'" >&2 exit 1 @@ -170,121 +199,142 @@ done harness_begin audio-quality "$RERUN" -# ── build ─────────────────────────────────────────────────────────────────── -flag=() -[[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") -echo "building moq-relay, moq, moq-shaper ($PROFILE)..." -(cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) -TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" -RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" -MOQ="$TARGET_BASE/$PROFILE/moq" -SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" - -# Into the run directory: vite empties its output first, so a shared dist/ would vanish under a -# concurrent run's page loads. -echo "building the page..." -( - cd "$CLIENT" - bun install --frozen-lockfile - bunx playwright install chromium - bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir -) - -# ── relay ─────────────────────────────────────────────────────────────────── -harness_port relay -RELAY_PORT="$HARNESS_PORT" -RELAY_URL="http://127.0.0.1:$RELAY_PORT" -sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" -harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" -if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then - echo "relay never became ready" >&2 - sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true - exit 1 -fi -harness_endpoint relay "$RELAY_URL" - -# ── publishers ────────────────────────────────────────────────────────────── -# A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. The -# publishers talk to the relay directly: impairing the ingest would grade the receiver on a stream -# already damaged before it was published. The AAC arm keeps ffmpeg's default PES packing, whose -# multi-frame bursts are the flush-span shape the reporter measured on the public relay (#3477). -# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' -publish_opus() { - ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ - -ac 2 -c:a libopus -b:a 128k \ - -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | - "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 -} +echo "installing the workspace..." +(cd "$CLIENT" && bun install --frozen-lockfile) -# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' -publish_aac() { - ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ - -ac 2 -c:a aac -b:a 128k -f mpegts - | - "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts -} +failed=0 -for codec in "${CODECS[@]}"; do - harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" -done +# The browser rows need the relay, the publishers, the shaper, and a page; replays need none of it. +if [[ ${#ROWS[@]} -gt 0 ]]; then + # ── build ─────────────────────────────────────────────────────────────── + flag=() + [[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") + echo "building moq-relay, moq, moq-shaper ($PROFILE)..." + (cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) + TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" + RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" + MOQ="$TARGET_BASE/$PROFILE/moq" + SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" + + # Into the run directory: vite empties its output first, so a shared dist/ would vanish under a + # concurrent run's page loads. + echo "building the page..." + ( + cd "$CLIENT" + bunx playwright install chromium + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir + ) + + # ── relay ─────────────────────────────────────────────────────────────── + harness_port relay + RELAY_PORT="$HARNESS_PORT" + RELAY_URL="http://127.0.0.1:$RELAY_PORT" + sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" + harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" + if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then + echo "relay never became ready" >&2 + sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true + exit 1 + fi + harness_endpoint relay "$RELAY_URL" + + # ── publishers ────────────────────────────────────────────────────────── + # A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. + # The publishers talk to the relay directly: impairing the ingest would grade the receiver on a + # stream already damaged before it was published. The AAC arm keeps ffmpeg's default PES + # packing, whose multi-frame bursts are the flush-span shape the reporter measured on the public + # relay (#3477). + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish_opus() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -ac 2 -c:a libopus -b:a 128k \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 + } + + # shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' + publish_aac() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts + } -# ── rows ──────────────────────────────────────────────────────────────────── -# One shaper port for the run: rows play one at a time. -harness_port shaper -SHAPER_PORT="$HARNESS_PORT" + for codec in "${CODECS[@]}"; do + harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" + done + + # ── rows ──────────────────────────────────────────────────────────────── + # One shaper port for the run: rows play one at a time. + harness_port shaper + SHAPER_PORT="$HARNESS_PORT" -echo "" -echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" -failed=0 -for tag in "${ROWS[@]}"; do - IFS=- read -r _ codec _ rest <<<"$tag" - ring="${rest##*-}" - profile="${rest%-*}" echo "" - echo "── $tag ──" - - # shellcheck disable=SC2046 # the profile's flags are meant to split - harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ - --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") - shaper_pid="$HARNESS_PID" - # It prints its seed and profile once bound; a page dialing before that would lose its handshake. - until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do - if harness_exited "$shaper_pid"; then - echo "shaper exited before binding for $tag" >&2 - sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true - exit 1 - fi - sleep 0.05 - done + echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" + for tag in "${ROWS[@]}"; do + IFS=- read -r _ codec _ rest <<<"$tag" + ring="${rest##*-}" + profile="${rest%-*}" + echo "" + echo "── $tag ──" + + # shellcheck disable=SC2046 # the profile's flags are meant to split + harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ + --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") + shaper_pid="$HARNESS_PID" + # It prints its seed and profile once bound; a page dialing before that would lose its handshake. + until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do + if harness_exited "$shaper_pid"; then + echo "shaper exited before binding for $tag" >&2 + sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true + exit 1 + fi + sleep 0.05 + done - status=0 - harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ - --url "http://127.0.0.1:$SHAPER_PORT" \ - --fingerprint "$RELAY_URL/certificate.sha256" \ - --broadcast "tone-$codec.hang" \ - --page "$HARNESS_RUN/page" \ - --ring "$ring" \ - --delay "$(delay_of "$profile")" \ - --duration "$DURATION" \ - --tag "$tag" \ - --out "$HARNESS_RUN" - harness_wait "$HARNESS_PID" || status=$? - if [[ $status -ne 0 ]]; then - failed=1 - sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true - fi + status=0 + harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ + --url "http://127.0.0.1:$SHAPER_PORT" \ + --fingerprint "$RELAY_URL/certificate.sha256" \ + --broadcast "tone-$codec.hang" \ + --page "$HARNESS_RUN/page" \ + --ring "$ring" \ + --delay "$(delay_of "$profile")" \ + --duration "$DURATION" \ + --tag "$tag" \ + --out "$HARNESS_RUN" + harness_wait "$HARNESS_PID" || status=$? + if [[ $status -ne 0 ]]; then + failed=1 + sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true + fi - # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile - # acted on the way out, and exits nonzero if it did not. - kill -TERM -- -"$shaper_pid" 2>/dev/null || true - shaper_status=0 - harness_wait "$shaper_pid" || shaper_status=$? - echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile + # acted on the way out, and exits nonzero if it did not. + kill -TERM -- -"$shaper_pid" 2>/dev/null || true + shaper_status=0 + harness_wait "$shaper_pid" || shaper_status=$? + echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } + done +fi - bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { - echo "analyze failed for $tag" >&2 - failed=1 - } -done +# ── replays ───────────────────────────────────────────────────────────────── +# Simulated time, so the whole set takes seconds, and deterministic, so the budgets are exact. +if [[ ${#REPLAYS[@]} -gt 0 ]]; then + echo "" + echo "replaying ${#REPLAYS[@]} rows" + "${replay[@]}" --out "$HARNESS_RUN" || failed=1 + for tag in "${REPLAYS[@]}"; do + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } + done +fi # ── grade ─────────────────────────────────────────────────────────────────── echo "" diff --git a/test/audio-quality/traces/local-aac.json b/test/audio-quality/traces/local-aac.json new file mode 100644 index 0000000000..807d609bd7 --- /dev/null +++ b/test/audio-quality/traces/local-aac.json @@ -0,0 +1,1566 @@ +{ + "version": 1, + "source": "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded 2026-09-29", + "description": "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer.", + "transport": "webtransport", + "rtt": 1, + "config": { + "codec": "mp4a.40.2", + "container": { + "kind": "legacy" + }, + "description": "1210", + "sampleRate": 44100, + "numberOfChannels": 2, + "jitter": 163 + }, + "arrivals": [ + [0,10016.133,0], + [2.55,10039.344,1], + [5.71,10062.555,2], + [5.98,10085.766,3], + [6.4,10108.977,4], + [6.61,10132.188,5], + [6.84,10155.4,6], + [7.09,10178.677,7], + [7.59,10201.888,8], + [7.73,10225.1,9], + [7.87,10248.311,10], + [8.02,10271.522,11], + [8.17,10294.733,12], + [8.34,10317.944,13], + [9.43,10341.211,14], + [9.59,10364.422,15], + [9.75,10387.633,16], + [10.38,10410.844,17], + [10.53,10434.055,18], + [10.69,10457.266,19], + [10.84,10480.477,20], + [10.96,10503.755,21], + [11.11,10526.966,22], + [11.26,10550.177,23], + [11.4,10573.388,24], + [11.55,10596.6,25], + [11.72,10619.811,26], + [11.9,10643.022,27], + [12.08,10666.288,28], + [12.26,10689.5,29], + [12.45,10712.711,30], + [12.66,10735.922,31], + [12.83,10759.133,32], + [13.02,10782.344,33], + [13.2,10805.555,34], + [14.59,10828.833,35], + [14.84,10852.044,36], + [15.16,10875.255,37], + [15.72,10968.1,41], + [15.88,10944.888,40], + [16.06,10921.677,39], + [16.2,10898.466,38], + [123.89,11014.588,43], + [124.13,10991.377,42], + [124.91,11130.644,48], + [125.08,11107.433,47], + [125.19,11084.222,46], + [125.34,11061.011,45], + [125.43,11037.8,44], + [287.9,11200.333,51], + [288.18,11177.122,50], + [288.3,11153.911,49], + [288.42,11293.177,55], + [288.64,11269.966,54], + [288.75,11246.755,53], + [288.89,11223.544,52], + [450.27,11432.511,61], + [450.48,11409.3,60], + [450.59,11386.088,59], + [450.71,11362.877,58], + [450.93,11339.666,57], + [451.07,11316.455,56], + [451.23,11455.722,62], + [613.15,11595.044,68], + [613.37,11571.833,67], + [613.47,11548.622,66], + [613.56,11525.411,65], + [613.67,11502.2,64], + [613.87,11478.988,63], + [613.97,11618.255,69], + [774.29,11757.588,75], + [774.5,11734.377,74], + [774.59,11711.166,73], + [774.69,11687.955,72], + [774.88,11664.744,71], + [774.97,11641.533,70], + [775.08,11780.8,76], + [937.57,11920.122,82], + [937.76,11896.911,81], + [937.87,11873.7,80], + [937.98,11850.488,79], + [938.17,11827.277,78], + [938.29,11804.066,77], + [938.4,11943.333,83], + [1100.28,12036.244,87], + [1100.49,12013.033,86], + [1100.59,11989.822,85], + [1100.7,11966.611,84], + [1100.87,12105.877,90], + [1100.98,12082.666,89], + [1101.08,12059.455,88], + [1262.9,12152.366,92], + [1263.13,12129.155,91], + [1263.3,12268.422,97], + [1263.43,12245.211,96], + [1263.64,12222,95], + [1263.74,12198.788,94], + [1263.88,12175.577,93], + [1426.26,12407.744,103], + [1426.47,12384.533,102], + [1426.59,12361.322,101], + [1426.73,12338.111,100], + [1426.95,12314.9,99], + [1427.06,12291.688,98], + [1427.2,12430.955,104], + [1587.95,12570.288,110], + [1588.2,12547.077,109], + [1588.31,12523.866,108], + [1588.43,12500.655,107], + [1588.64,12477.444,106], + [1588.73,12454.233,105], + [1588.84,12593.5,111], + [1751.58,12732.822,117], + [1751.77,12709.611,116], + [1751.88,12686.4,115], + [1751.99,12663.188,114], + [1752.17,12639.977,113], + [1752.3,12616.766,112], + [1752.43,12756.033,118], + [1916.67,12895.366,124], + [1917,12872.155,123], + [1917.16,12848.944,122], + [1917.41,12825.733,121], + [1917.55,12802.522,120], + [1917.67,12779.311,119], + [1917.81,12918.577,125], + [2077.07,13057.9,131], + [2077.3,13034.688,130], + [2077.46,13011.477,129], + [2077.61,12988.266,128], + [2077.91,12965.055,127], + [2078.04,12941.844,126], + [2078.14,13081.111,132], + [2238.87,13220.444,138], + [2239.24,13197.233,137], + [2239.38,13174.022,136], + [2239.53,13150.811,135], + [2239.66,13127.6,134], + [2239.93,13104.388,133], + [2240.06,13243.655,139], + [2400.81,13382.988,145], + [2401.13,13359.777,144], + [2401.28,13336.566,143], + [2401.42,13313.355,142], + [2401.56,13290.144,141], + [2401.9,13266.933,140], + [2402.04,13406.2,146], + [2564.49,13545.522,152], + [2564.8,13522.311,151], + [2564.92,13499.1,150], + [2565.06,13475.888,149], + [2565.16,13452.677,148], + [2565.24,13429.466,147], + [2565.32,13568.733,153], + [2726.35,13708.066,159], + [2726.66,13684.855,158], + [2726.82,13661.644,157], + [2727.09,13638.433,156], + [2727.2,13615.222,155], + [2727.32,13592.011,154], + [2727.44,13731.277,160], + [2888.53,13870.6,166], + [2888.76,13847.388,165], + [2888.87,13824.177,164], + [2889.11,13800.966,163], + [2889.22,13777.755,162], + [2889.43,13754.544,161], + [2889.61,13893.811,167], + [3051.61,14033.144,173], + [3051.87,14009.933,172], + [3052.02,13986.722,171], + [3052.18,13963.511,170], + [3052.33,13940.3,169], + [3052.68,13917.088,168], + [3052.82,14056.355,174], + [3213.91,14079.622,175], + [3214.18,14218.888,181], + [3215.8,14172.466,179], + [3216.03,14149.255,178], + [3216.22,14126.044,177], + [3216.36,14102.833,176], + [3216.53,14195.677,180], + [3375.85,14335.011,186], + [3376.06,14311.8,185], + [3376.16,14288.588,184], + [3376.38,14265.377,183], + [3376.49,14242.166,182], + [3376.59,14381.433,188], + [3376.72,14358.222,187], + [3538.87,14520.766,194], + [3539.21,14497.555,193], + [3539.38,14474.344,192], + [3539.52,14451.133,191], + [3539.74,14427.922,190], + [3539.85,14404.711,189], + [3539.98,14543.977,195], + [3711.34,14706.511,202], + [3711.56,14683.3,201], + [3711.88,14660.088,200], + [3712.04,14636.877,199], + [3712.79,14613.666,198], + [3713,14590.455,197], + [3713.21,14567.244,196], + [3868.88,14845.844,208], + [3869.2,14822.633,207], + [3869.32,14799.422,206], + [3869.43,14776.211,205], + [3869.61,14753,204], + [3869.75,14729.788,203], + [3869.95,14869.055,209], + [4026.26,15008.377,215], + [4026.49,14985.166,214], + [4026.61,14961.955,213], + [4026.74,14938.744,212], + [4026.94,14915.533,211], + [4027.06,14892.322,210], + [4027.19,15031.588,216], + [4188.73,15170.922,222], + [4188.97,15147.711,221], + [4189.08,15124.5,220], + [4189.25,15101.288,219], + [4189.35,15078.077,218], + [4189.48,15054.866,217], + [4189.62,15194.133,223], + [4351.39,15333.455,229], + [4351.65,15310.244,228], + [4351.78,15287.033,227], + [4351.9,15263.822,226], + [4352.15,15240.611,225], + [4352.27,15217.4,224], + [4352.42,15356.666,230], + [4514.23,15496,236], + [4514.45,15472.788,235], + [4514.59,15449.577,234], + [4514.71,15426.366,233], + [4514.9,15403.155,232], + [4515,15379.944,231], + [4515.14,15519.211,237], + [4686.54,15658.544,243], + [4686.87,15635.333,242], + [4688.84,15588.911,240], + [4689.05,15565.7,239], + [4689.37,15542.488,238], + [4689.54,15681.755,244], + [4689.68,15612.122,241], + [4839.79,15821.077,250], + [4840.04,15797.866,249], + [4840.2,15774.655,248], + [4840.39,15751.444,247], + [4840.51,15728.233,246], + [4840.62,15705.022,245], + [4840.78,15844.288,251], + [5001.26,15983.622,257], + [5001.71,15960.411,256], + [5002.02,15937.2,255], + [5002.21,15913.988,254], + [5002.41,15890.777,253], + [5002.74,15867.566,252], + [5002.88,16006.833,258], + [5165.08,16146.155,264], + [5165.3,16122.944,263], + [5165.42,16099.733,262], + [5165.52,16076.522,261], + [5165.75,16053.311,260], + [5165.85,16030.1,259], + [5166.02,16169.366,265], + [5333.68,16192.644,266], + [5333.97,16308.7,271], + [5334.26,16285.488,270], + [5334.39,16262.277,269], + [5334.52,16239.066,268], + [5334.66,16215.855,267], + [5334.84,16331.911,272], + [5493.92,16471.233,278], + [5494.18,16448.022,277], + [5494.28,16424.811,276], + [5494.38,16401.6,275], + [5494.57,16378.388,274], + [5494.67,16355.177,273], + [5494.78,16494.444,279], + [5657.09,16633.777,285], + [5657.29,16610.566,284], + [5657.39,16587.355,283], + [5657.47,16564.144,282], + [5657.56,16540.933,281], + [5657.77,16517.722,280], + [5657.87,16656.988,286], + [5813.98,16796.322,292], + [5814.21,16773.111,291], + [5814.34,16749.9,290], + [5814.44,16726.688,289], + [5814.59,16703.477,288], + [5814.69,16680.266,287], + [5814.92,16819.533,293], + [5977.89,16958.855,299], + [5978.19,16935.644,298], + [5978.3,16912.433,297], + [5978.47,16889.222,296], + [5978.57,16866.011,295], + [5978.66,16842.8,294], + [5978.87,16982.066,300], + [6139.22,17121.4,306], + [6139.41,17098.188,305], + [6139.52,17074.977,304], + [6139.68,17051.766,303], + [6139.77,17028.555,302], + [6139.87,17005.344,301], + [6139.99,17144.611,307], + [6311.7,17283.933,313], + [6312.03,17260.722,312], + [6312.19,17237.511,311], + [6312.34,17214.3,310], + [6312.49,17191.088,309], + [6312.84,17167.877,308], + [6312.99,17307.144,314], + [6464.69,17446.477,320], + [6464.96,17423.266,319], + [6465.11,17400.055,318], + [6465.21,17376.844,317], + [6465.29,17353.633,316], + [6465.38,17330.422,315], + [6465.6,17469.688,321], + [6627.69,17609.011,327], + [6628.22,17585.8,326], + [6628.41,17562.588,325], + [6628.55,17539.377,324], + [6628.95,17516.166,323], + [6629.06,17492.955,322], + [6629.37,17632.222,328], + [6788.63,17771.555,334], + [6788.84,17748.344,333], + [6788.94,17725.133,332], + [6789.06,17701.922,331], + [6789.26,17678.711,330], + [6789.34,17655.5,329], + [6789.45,17794.766,335], + [6952.27,17934.1,341], + [6952.49,17910.888,340], + [6952.62,17887.677,339], + [6952.79,17864.466,338], + [6952.89,17841.255,337], + [6953,17818.044,336], + [6953.21,17957.311,342], + [7114.68,18096.633,348], + [7114.92,18073.422,347], + [7115.09,18050.211,346], + [7115.21,18027,345], + [7115.46,18003.788,344], + [7115.57,17980.577,343], + [7115.69,18119.844,349], + [7277.22,18259.177,355], + [7277.54,18235.966,354], + [7277.66,18212.755,353], + [7277.77,18189.544,352], + [7277.89,18166.333,351], + [7278.14,18143.122,350], + [7278.24,18282.388,356], + [7438.99,18421.711,362], + [7439.21,18398.5,361], + [7439.32,18375.288,360], + [7439.45,18352.077,359], + [7439.68,18328.866,358], + [7439.79,18305.655,357], + [7439.89,18444.922,363], + [7601.84,18584.255,369], + [7602.06,18561.044,368], + [7602.18,18537.833,367], + [7602.29,18514.622,366], + [7602.56,18491.411,365], + [7602.71,18468.2,364], + [7602.88,18607.466,370], + [7764.61,18746.788,376], + [7764.84,18723.577,375], + [7764.95,18700.366,374], + [7765.13,18677.155,373], + [7765.24,18653.944,372], + [7765.34,18630.733,371], + [7765.49,18770,377], + [7926.66,18909.333,383], + [7926.85,18886.122,382], + [7926.96,18862.911,381], + [7927.13,18839.7,380], + [7927.21,18816.488,379], + [7927.31,18793.277,378], + [7927.41,18932.544,384], + [8089.77,19071.877,390], + [8090.01,19048.666,389], + [8090.12,19025.455,388], + [8090.34,19002.244,387], + [8090.45,18979.033,386], + [8090.59,18955.822,385], + [8090.78,19095.088,391], + [8251.55,19234.411,397], + [8251.75,19211.2,396], + [8251.92,19187.988,395], + [8252.09,19164.777,394], + [8252.38,19141.566,393], + [8252.48,19118.355,392], + [8252.59,19257.622,398], + [8414.65,19396.955,404], + [8414.87,19373.744,403], + [8414.99,19350.533,402], + [8415.09,19327.322,401], + [8415.28,19304.111,400], + [8415.36,19280.9,399], + [8415.47,19420.166,405], + [8577.91,19559.488,411], + [8578.23,19536.277,410], + [8578.38,19513.066,409], + [8578.75,19489.855,408], + [8578.89,19466.644,407], + [8579.06,19443.433,406], + [8579.18,19582.7,412], + [8739.86,19722.033,418], + [8740.09,19698.822,417], + [8740.21,19675.611,416], + [8740.31,19652.4,415], + [8740.57,19629.188,414], + [8740.67,19605.977,413], + [8740.8,19745.244,419], + [8904.17,19884.566,425], + [8904.5,19861.355,424], + [8904.68,19838.144,423], + [8905,19814.933,422], + [8905.15,19791.722,421], + [8905.31,19768.511,420], + [8905.49,19907.777,426], + [9064.8,20047.111,432], + [9065.11,20023.9,431], + [9065.23,20000.688,430], + [9065.39,19977.477,429], + [9065.62,19954.266,428], + [9065.72,19931.055,427], + [9065.83,20070.322,433], + [9228,20209.655,439], + [9228.27,20186.444,438], + [9228.38,20163.233,437], + [9228.48,20140.022,436], + [9228.59,20116.811,435], + [9228.87,20093.6,434], + [9229.3,20232.866,440], + [9390.18,20372.188,446], + [9390.56,20348.977,445], + [9390.71,20325.766,444], + [9390.84,20302.555,443], + [9390.99,20279.344,442], + [9391.12,20256.133,441], + [9391.31,20395.4,447], + [9570.51,20511.522,452], + [9570.64,20534.733,453], + [9570.9,20488.311,451], + [9571.07,20465.1,450], + [9571.23,20441.888,449], + [9571.41,20418.677,448], + [9572.07,20557.944,454], + [9728.41,20697.266,460], + [9733.17,20674.055,459], + [9733.53,20650.844,458], + [9733.84,20627.633,457], + [9736.38,20604.422,456], + [9736.55,20581.211,455], + [9736.67,20720.477,461], + [9877.29,20859.811,467], + [9877.49,20836.6,466], + [9877.58,20813.388,465], + [9877.76,20790.177,464], + [9877.84,20766.966,463], + [9877.94,20743.755,462], + [9878.05,20883.022,468], + [10039.92,21022.344,474], + [10040.41,20999.133,473], + [10040.62,20975.922,472], + [10040.83,20952.711,471], + [10040.94,20929.5,470], + [10041.05,20906.288,469], + [10041.14,21045.555,475], + [10202.67,21184.888,481], + [10202.87,21161.677,480], + [10203.01,21138.466,479], + [10203.15,21115.255,478], + [10203.37,21092.044,477], + [10203.48,21068.833,476], + [10203.59,21208.1,482], + [10364.37,21347.433,488], + [10364.59,21324.222,487], + [10364.69,21301.011,486], + [10364.84,21277.8,485], + [10364.92,21254.588,484], + [10365.06,21231.377,483], + [10365.15,21370.644,489], + [10536.68,21509.966,495], + [10536.93,21486.755,494], + [10537.08,21463.544,493], + [10537.21,21440.333,492], + [10537.47,21393.911,490], + [10537.66,21533.177,496], + [10537.77,21417.122,491], + [10718.08,21579.666,498], + [10718.25,21672.511,502], + [10720.29,21649.3,501], + [10720.46,21626.088,500], + [10720.62,21602.877,499], + [10720.88,21556.455,497], + [10721.45,21695.722,503], + [10852.5,21835.044,509], + [10852.69,21811.833,508], + [10852.81,21788.622,507], + [10852.96,21765.411,506], + [10853.17,21742.2,505], + [10853.52,21718.988,504], + [10853.64,21858.255,510], + [11014.78,21997.588,516], + [11015.06,21974.377,515], + [11015.16,21951.166,514], + [11015.26,21927.955,513], + [11015.44,21904.744,512], + [11015.53,21881.533,511], + [11015.63,22020.8,517], + [11180.89,22136.911,522], + [11181.07,22160.122,523], + [11181.74,22113.7,521], + [11182.13,22090.488,520], + [11182.3,22067.277,519], + [11182.46,22044.066,518], + [11182.58,22183.333,524], + [11341.58,22322.666,530], + [11341.82,22299.455,529], + [11341.94,22276.244,528], + [11342.07,22253.033,527], + [11342.34,22229.822,526], + [11342.45,22206.611,525], + [11342.57,22345.877,531], + [11504.42,22485.211,537], + [11504.62,22462,536], + [11504.7,22438.788,535], + [11504.85,22415.577,534], + [11504.94,22392.366,533], + [11505.06,22369.155,532], + [11505.24,22508.422,538], + [11665.44,22647.744,544], + [11665.6,22624.533,543], + [11665.68,22601.322,542], + [11665.83,22578.111,541], + [11665.91,22554.9,540], + [11666.02,22531.688,539], + [11666.23,22670.955,545], + [11828.72,22810.288,551], + [11829.11,22787.077,550], + [11829.23,22763.866,549], + [11829.3,22740.655,548], + [11829.71,22717.444,547], + [11829.81,22694.233,546], + [11829.91,22833.5,552], + [11990.61,22972.822,558], + [11990.8,22949.611,557], + [11990.89,22926.4,556], + [11991.04,22903.188,555], + [11991.11,22879.977,554], + [11991.2,22856.766,553], + [11991.28,22996.033,559], + [12155.63,23135.366,565], + [12155.83,23112.155,564], + [12155.94,23088.944,563], + [12156.07,23065.733,562], + [12156.3,23042.522,561], + [12156.39,23019.311,560], + [12156.55,23158.577,566], + [12317.01,23297.9,572], + [12317.17,23274.688,571], + [12317.28,23251.477,570], + [12317.41,23228.266,569], + [12317.46,23205.055,568], + [12317.53,23181.844,567], + [12317.64,23321.111,573], + [12477.48,23460.444,579], + [12477.66,23437.233,578], + [12477.78,23414.022,577], + [12477.87,23390.811,576], + [12477.99,23367.6,575], + [12478.07,23344.388,574], + [12478.2,23483.655,580], + [12641.84,23622.988,586], + [12642.02,23599.777,585], + [12642.11,23576.566,584], + [12642.24,23553.355,583], + [12642.31,23530.144,582], + [12642.39,23506.933,581], + [12642.57,23646.2,587], + [12803.44,23785.522,593], + [12803.62,23762.311,592], + [12803.71,23739.1,591], + [12803.8,23715.888,590], + [12803.95,23692.677,589], + [12804.04,23669.466,588], + [12804.16,23808.733,594], + [12968.53,23878.433,597], + [12968.66,23948.066,600], + [12968.89,23924.855,599], + [12969.03,23901.644,598], + [12969.2,23855.222,596], + [12969.39,23832.011,595], + [12969.71,23971.277,601], + [13128.76,24110.6,607], + [13128.92,24087.388,606], + [13129.01,24064.177,605], + [13129.19,24040.966,604], + [13129.28,24017.755,603], + [13129.36,23994.544,602], + [13129.53,24133.811,608], + [13291.69,24273.144,614], + [13291.9,24249.933,613], + [13292.02,24226.722,612], + [13292.14,24203.511,611], + [13292.34,24180.3,610], + [13292.46,24157.088,609], + [13292.57,24296.355,615], + [13456.24,24435.677,621], + [13456.44,24412.466,620], + [13456.56,24389.255,619], + [13456.67,24366.044,618], + [13456.78,24342.833,617], + [13456.87,24319.622,616], + [13456.99,24458.888,622], + [13622.09,24505.377,624], + [13622.38,24482.166,623], + [13623.46,24598.222,628], + [13624.14,24575.011,627], + [13625.59,24551.8,626], + [13625.77,24528.588,625], + [13627.41,24621.433,629], + [13870.43,24760.766,635], + [13873.91,24737.555,634], + [13876.49,24714.344,633], + [13883.19,24691.133,632], + [13889,24667.922,631], + [13891.52,24644.711,630], + [13892.02,24783.977,636], + [13992.79,24923.3,642], + [13993.03,24900.088,641], + [13993.18,24876.877,640], + [13993.26,24853.666,639], + [13993.39,24830.455,638], + [13993.73,24807.244,637], + [13994.1,24946.511,643], + [14174.97,25016.211,646], + [14175.28,25085.844,649], + [14175.79,25062.633,648], + [14175.91,25039.422,647], + [14176.04,24993,645], + [14176.17,24969.788,644], + [14176.27,25109.055,650], + [14286.82,25248.377,656], + [14292.13,25225.166,655], + [14294,25178.744,653], + [14294.68,25155.533,652], + [14295.73,25132.322,651], + [14296.09,25271.588,657], + [14296.46,25201.955,654], + [14472.09,25318.077,659], + [14472.22,25294.866,658], + [14472.45,25434.133,664], + [14472.66,25410.922,663], + [14472.85,25387.711,662], + [14472.94,25364.5,661], + [14473.1,25341.288,660], + [14674.91,25573.455,670], + [14675.71,25550.244,669], + [14676.58,25503.822,667], + [14680.92,25480.611,666], + [14681.54,25457.4,665], + [14687.88,25596.666,671], + [14688.29,25527.033,668], + [14776.93,25736,677], + [14777.53,25712.788,676], + [14777.66,25689.577,675], + [14777.8,25666.366,674], + [14777.91,25643.155,673], + [14778.21,25619.944,672], + [14778.88,25759.211,678], + [14942.41,25898.544,684], + [14942.61,25875.333,683], + [14942.82,25852.122,682], + [14943,25828.911,681], + [14943.36,25805.7,680], + [14943.5,25782.488,679], + [14944.51,25921.755,685], + [15084.12,26061.077,691], + [15084.39,26037.866,690], + [15085.33,26014.655,689], + [15086.9,25991.444,688], + [15087.67,25968.233,687], + [15088.27,25945.022,686], + [15089.31,26084.288,692], + [15247.24,26223.622,698], + [15247.94,26200.411,697], + [15248.07,26177.2,696], + [15248.27,26153.988,695], + [15248.36,26130.777,694], + [15248.45,26107.566,693], + [15248.55,26246.833,699], + [15411.58,26386.155,705], + [15412.06,26362.944,704], + [15412.23,26339.733,703], + [15412.52,26316.522,702], + [15412.7,26293.311,701], + [15412.89,26270.1,700], + [15413.07,26409.366,706], + [15567.84,26548.7,712], + [15568.09,26525.488,711], + [15568.2,26502.277,710], + [15568.37,26479.066,709], + [15568.47,26455.855,708], + [15568.61,26432.644,707], + [15568.72,26571.911,713], + [15735.29,26711.233,719], + [15735.6,26688.022,718], + [15735.74,26664.811,717], + [15735.88,26641.6,716], + [15736.18,26618.388,715], + [15736.31,26595.177,714], + [15736.46,26734.444,720], + [15895.29,26873.777,726], + [15895.63,26850.566,725], + [15895.79,26827.355,724], + [15895.94,26804.144,723], + [15896.26,26780.933,722], + [15896.38,26757.722,721], + [15896.51,26896.988,727], + [16059.43,27036.322,733], + [16059.7,27013.111,732], + [16059.83,26989.9,731], + [16059.97,26966.688,730], + [16060.11,26943.477,729], + [16060.26,26920.266,728], + [16060.4,27059.533,734], + [16219.19,27198.855,740], + [16219.51,27175.644,739], + [16219.69,27152.433,738], + [16219.97,27129.222,737], + [16220.06,27106.011,736], + [16220.15,27082.8,735], + [16220.24,27222.066,741], + [16382.51,27361.4,747], + [16382.77,27338.188,746], + [16382.9,27314.977,745], + [16383.09,27291.766,744], + [16383.17,27268.555,743], + [16383.25,27245.344,742], + [16383.33,27384.611,748], + [16545.33,27523.933,754], + [16545.69,27500.722,753], + [16545.83,27477.511,752], + [16546.09,27454.3,751], + [16546.22,27431.088,750], + [16546.35,27407.877,749], + [16546.46,27547.144,755], + [16707.81,27686.477,761], + [16708.14,27663.266,760], + [16708.28,27640.055,759], + [16708.45,27616.844,758], + [16708.53,27593.633,757], + [16708.59,27570.422,756], + [16708.71,27709.688,762], + [16868,27849.011,768], + [16868.21,27825.8,767], + [16868.3,27802.588,766], + [16868.5,27779.377,765], + [16868.61,27756.166,764], + [16868.73,27732.955,763], + [16868.87,27872.222,769], + [17029.84,28011.555,775], + [17030.07,27988.344,774], + [17030.18,27965.133,773], + [17030.28,27941.922,772], + [17030.5,27918.711,771], + [17030.61,27895.5,770], + [17030.73,28034.766,776], + [17200.69,28174.1,782], + [17200.89,28150.888,781], + [17200.97,28127.677,780], + [17201.05,28104.466,779], + [17201.2,28081.255,778], + [17201.28,28058.044,777], + [17201.36,28197.311,783], + [17356.81,28336.633,789], + [17357.26,28313.422,788], + [17357.44,28290.211,787], + [17357.6,28267,786], + [17357.78,28243.788,785], + [17358.17,28220.577,784], + [17358.29,28359.844,790], + [17519.52,28499.177,796], + [17519.78,28475.966,795], + [17519.89,28452.755,794], + [17520.02,28429.544,793], + [17520.41,28406.333,792], + [17520.69,28383.122,791], + [17520.87,28522.388,797], + [17684,28661.711,803], + [17684.66,28638.5,802], + [17685.2,28615.288,801], + [17685.51,28592.077,800], + [17685.77,28568.866,799], + [17686,28545.655,798], + [17686.4,28684.922,804], + [17842.95,28824.255,810], + [17843.22,28801.044,809], + [17843.34,28777.833,808], + [17843.52,28754.622,807], + [17843.62,28731.411,806], + [17843.7,28708.2,805], + [17843.86,28847.466,811], + [18012.8,28986.788,817], + [18012.99,28963.577,816], + [18013.08,28940.366,815], + [18013.21,28917.155,814], + [18013.28,28893.944,813], + [18013.35,28870.733,812], + [18013.58,29010,818], + [18167.89,29149.333,824], + [18168.08,29126.122,823], + [18168.16,29102.911,822], + [18168.27,29079.7,821], + [18168.34,29056.488,820], + [18168.41,29033.277,819], + [18168.55,29172.544,825], + [18331.13,29311.877,831], + [18331.34,29288.666,830], + [18331.46,29265.455,829], + [18331.57,29242.244,828], + [18331.64,29219.033,827], + [18331.8,29195.822,826], + [18331.88,29335.088,832], + [18497.56,29474.411,838], + [18497.9,29451.2,837], + [18498.08,29427.988,836], + [18498.38,29404.777,835], + [18498.5,29381.566,834], + [18498.63,29358.355,833], + [18498.85,29497.622,839], + [18655.23,29636.955,845], + [18655.43,29613.744,844], + [18655.53,29590.533,843], + [18655.62,29567.322,842], + [18655.79,29544.111,841], + [18655.88,29520.9,840], + [18655.99,29660.166,846], + [18822.2,29799.488,852], + [18822.63,29776.277,851], + [18822.8,29753.066,850], + [18823.05,29729.855,849], + [18823.24,29706.644,848], + [18823.83,29683.433,847], + [18824,29822.7,853], + [18988.77,29962.033,859], + [18989.25,29938.822,858], + [18989.48,29915.611,857], + [18989.64,29892.4,856], + [18989.97,29869.188,855], + [18990.09,29845.977,854], + [18990.24,29985.244,860], + [19152.68,30124.566,866], + [19152.82,30101.355,865], + [19152.89,30078.144,864], + [19152.96,30054.933,863], + [19153.05,30031.722,862], + [19153.33,30008.511,861], + [19153.46,30147.777,867], + [19334.36,30171.055,868], + [19337.72,30310.322,874], + [19352.35,30287.111,873], + [19352.91,30263.9,872], + [19353.25,30240.688,871], + [19357.88,30217.477,870], + [19364.11,30194.266,869], + [19469.63,30426.444,879], + [19469.95,30403.233,878], + [19470.11,30380.022,877], + [19470.37,30356.811,876], + [19470.47,30333.6,875], + [19470.69,30472.866,881], + [19470.87,30449.655,880], + [19631.34,30612.188,887], + [19631.51,30588.977,886], + [19631.59,30565.766,885], + [19631.71,30542.555,884], + [19631.79,30519.344,883], + [19631.84,30496.133,882], + [19631.93,30635.4,888], + [19792.62,30774.733,894], + [19792.8,30751.522,893], + [19792.9,30728.311,892], + [19792.99,30705.1,891], + [19793.06,30681.888,890], + [19793.22,30658.677,889], + [19793.3,30797.944,895], + [19954.65,30937.266,901], + [19954.84,30914.055,900], + [19954.93,30890.844,899], + [19955.09,30867.633,898], + [19955.17,30844.422,897], + [19955.26,30821.211,896], + [19955.44,30960.477,902], + [20118.29,31099.811,908], + [20118.48,31076.6,907], + [20118.57,31053.388,906], + [20118.74,31030.177,905], + [20118.81,31006.966,904], + [20118.89,30983.755,903], + [20119,31123.022,909], + [20280.22,31262.344,915], + [20280.41,31239.133,914], + [20280.5,31215.922,913], + [20280.66,31192.711,912], + [20280.74,31169.5,911], + [20280.81,31146.288,910], + [20280.91,31285.555,916], + [20442.9,31424.888,922], + [20443.1,31401.677,921], + [20443.2,31378.466,920], + [20443.33,31355.255,919], + [20443.41,31332.044,918], + [20443.48,31308.833,917], + [20443.57,31448.1,923], + [20605.07,31587.433,929], + [20605.24,31564.222,928], + [20605.32,31541.011,927], + [20605.45,31517.8,926], + [20605.52,31494.588,925], + [20605.59,31471.377,924], + [20605.69,31610.644,930], + [20767.74,31749.966,936], + [20767.92,31726.755,935], + [20768.01,31703.544,934], + [20768.16,31680.333,933], + [20768.24,31657.122,932], + [20768.32,31633.911,931], + [20768.41,31773.177,937], + [20930.12,31912.511,943], + [20930.3,31889.3,942], + [20930.38,31866.088,941], + [20930.53,31842.877,940], + [20930.6,31819.666,939], + [20930.68,31796.455,938], + [20930.77,31935.722,944], + [21093.23,32075.044,950], + [21093.4,32051.833,949], + [21093.49,32028.622,948], + [21093.64,32005.411,947], + [21093.72,31982.2,946], + [21093.8,31958.988,945], + [21093.95,32098.255,951], + [21254.99,32237.588,957], + [21255.18,32214.377,956], + [21255.27,32191.166,955], + [21255.41,32167.955,954], + [21255.5,32144.744,953], + [21255.58,32121.533,952], + [21255.67,32260.8,958], + [21417.68,32400.122,964], + [21417.87,32376.911,963], + [21418,32353.7,962], + [21418.09,32330.488,961], + [21418.23,32307.277,960], + [21418.31,32284.066,959], + [21418.41,32423.333,965], + [21580.61,32562.666,971], + [21580.8,32539.455,970], + [21580.94,32516.244,969], + [21581.04,32493.033,968], + [21581.13,32469.822,967], + [21581.31,32446.611,966], + [21581.39,32585.877,972], + [21743.16,32725.211,978], + [21743.34,32702,977], + [21743.43,32678.788,976], + [21743.59,32655.577,975], + [21743.67,32632.366,974], + [21743.74,32609.155,973], + [21743.84,32748.422,979], + [21905.34,32887.744,985], + [21905.53,32864.533,984], + [21905.63,32841.322,983], + [21905.77,32818.111,982], + [21905.87,32794.9,981], + [21905.98,32771.688,980], + [21906.08,32910.955,986], + [22067.64,33050.288,992], + [22067.83,33027.077,991], + [22067.91,33003.866,990], + [22068.01,32980.655,989], + [22068.18,32957.444,988], + [22068.3,32934.233,987], + [22068.39,33073.5,993], + [22231.3,33212.822,999], + [22231.52,33189.611,998], + [22231.76,33166.4,997], + [22231.98,33143.188,996], + [22232.07,33119.977,995], + [22232.17,33096.766,994], + [22232.28,33236.033,1000], + [22395.19,33375.366,1006], + [22395.45,33352.155,1005], + [22395.63,33328.944,1004], + [22395.81,33305.733,1003], + [22395.98,33282.522,1002], + [22396.12,33259.311,1001], + [22396.27,33398.577,1007], + [22556.88,33537.9,1013], + [22557.18,33514.688,1012], + [22557.32,33491.477,1011], + [22557.45,33468.266,1010], + [22557.73,33445.055,1009], + [22557.9,33421.844,1008], + [22558.04,33561.111,1014], + [22720.22,33700.444,1020], + [22720.47,33677.233,1019], + [22720.67,33654.022,1018], + [22720.77,33630.811,1017], + [22720.98,33607.6,1016], + [22721.08,33584.388,1015], + [22721.19,33723.655,1021], + [22882.47,33862.988,1027], + [22882.74,33839.777,1026], + [22882.95,33816.566,1025], + [22883.05,33793.355,1024], + [22883.15,33770.144,1023], + [22883.34,33746.933,1022], + [22883.45,33886.2,1028], + [23044.69,34025.522,1034], + [23044.92,34002.311,1033], + [23045.09,33979.1,1032], + [23045.19,33955.888,1031], + [23045.34,33932.677,1030], + [23045.43,33909.466,1029], + [23045.53,34048.733,1035], + [23207.07,34188.066,1041], + [23207.29,34164.855,1040], + [23207.4,34141.644,1039], + [23207.55,34118.433,1038], + [23207.79,34095.222,1037], + [23207.9,34072.011,1036], + [23208.06,34211.277,1042], + [23370,34350.6,1048], + [23370.25,34327.388,1047], + [23370.36,34304.177,1046], + [23370.46,34280.966,1045], + [23370.7,34257.755,1044], + [23370.79,34234.544,1043], + [23370.91,34373.811,1049], + [23532.84,34513.144,1055], + [23533.06,34489.933,1054], + [23533.17,34466.722,1053], + [23533.29,34443.511,1052], + [23533.51,34420.3,1051], + [23533.6,34397.088,1050], + [23533.71,34536.355,1056], + [23696.22,34675.677,1062], + [23696.48,34652.466,1061], + [23696.58,34629.255,1060], + [23696.73,34606.044,1059], + [23696.83,34582.833,1058], + [23696.91,34559.622,1057], + [23697.07,34698.888,1063], + [23858.15,34838.222,1069], + [23858.39,34815.011,1068], + [23858.64,34791.8,1067], + [23858.84,34768.588,1066], + [23858.97,34745.377,1065], + [23859.07,34722.166,1064], + [23859.18,34861.433,1070], + [24019.44,35000.766,1076], + [24019.67,34977.555,1075], + [24019.8,34954.344,1074], + [24019.87,34931.133,1073], + [24019.95,34907.922,1072], + [24020.12,34884.711,1071], + [24020.19,35023.977,1077], + [24181.11,35163.3,1083], + [24181.28,35140.088,1082], + [24181.35,35116.877,1081], + [24181.49,35093.666,1080], + [24181.56,35070.455,1079], + [24181.63,35047.244,1078], + [24181.71,35186.511,1084], + [24345.36,35325.844,1090], + [24345.66,35302.633,1089], + [24345.77,35279.422,1088], + [24345.95,35256.211,1087], + [24346.05,35233,1086], + [24346.14,35209.788,1085], + [24346.26,35349.055,1091], + [24509.21,35488.377,1097], + [24509.52,35465.166,1096], + [24509.64,35441.955,1095], + [24509.77,35418.744,1094], + [24510.03,35395.533,1093], + [24510.14,35372.322,1092], + [24510.27,35511.588,1098], + [24673.12,35650.922,1104], + [24673.44,35627.711,1103], + [24673.58,35604.5,1102], + [24673.71,35581.288,1101], + [24673.96,35558.077,1100], + [24674.08,35534.866,1099], + [24674.22,35674.133,1105], + [24833.74,35813.455,1111], + [24833.99,35790.244,1110], + [24834.12,35767.033,1109], + [24834.21,35743.822,1108], + [24834.37,35720.611,1107], + [24834.46,35697.4,1106], + [24834.61,35836.666,1112], + [24997.09,35976,1118], + [24997.33,35952.788,1117], + [24997.47,35929.577,1116], + [24997.56,35906.366,1115], + [24997.68,35883.155,1114], + [24997.86,35859.944,1113], + [24997.99,35999.211,1119], + [25159.71,36138.544,1125], + [25159.88,36115.333,1124], + [25160,36092.122,1123], + [25160.08,36068.911,1122], + [25160.2,36045.7,1121], + [25160.27,36022.488,1120], + [25160.4,36161.755,1126], + [25322.74,36301.077,1132], + [25322.97,36277.866,1131], + [25323.07,36254.655,1130], + [25323.18,36231.444,1129], + [25323.3,36208.233,1128], + [25323.42,36185.022,1127], + [25323.55,36324.288,1133], + [25486.23,36463.622,1139], + [25486.51,36440.411,1138], + [25486.61,36417.2,1137], + [25486.78,36393.988,1136], + [25486.88,36370.777,1135], + [25487,36347.566,1134], + [25487.21,36486.833,1140], + [25645.47,36626.155,1146], + [25645.64,36602.944,1145], + [25645.75,36579.733,1144], + [25645.88,36556.522,1143], + [25646.01,36533.311,1142], + [25646.13,36510.1,1141], + [25646.51,36649.366,1147], + [25817.49,36695.855,1149], + [25817.7,36672.644,1148], + [25818.43,36742.277,1151], + [25818.57,36811.911,1154], + [25818.83,36788.7,1153], + [25819.08,36765.488,1152], + [25819.39,36719.066,1150], + [25981.76,36974.444,1161], + [25982.05,36951.233,1160], + [25982.15,36904.811,1158], + [25982.3,36881.6,1157], + [25982.38,36858.388,1156], + [25982.46,36835.177,1155], + [25982.62,36928.022,1159], + [26132.89,37113.777,1167], + [26133.16,37090.566,1166], + [26133.27,37067.355,1165], + [26133.47,37044.144,1164], + [26133.58,37020.933,1163], + [26133.69,36997.722,1162], + [26133.81,37136.988,1168], + [26296.13,37276.322,1174], + [26296.4,37253.111,1173], + [26296.51,37229.9,1172], + [26296.62,37206.688,1171], + [26296.87,37183.477,1170], + [26296.99,37160.266,1169], + [26297.11,37299.533,1175], + [26457.47,37438.855,1181], + [26457.68,37415.644,1180], + [26457.78,37392.433,1179], + [26457.98,37369.222,1178], + [26458.08,37346.011,1177], + [26458.21,37322.8,1176], + [26458.33,37462.066,1182], + [26620.12,37601.4,1188], + [26620.36,37578.188,1187], + [26620.46,37554.977,1186], + [26620.62,37531.766,1185], + [26620.72,37508.555,1184], + [26620.82,37485.344,1183], + [26620.94,37624.611,1189], + [26783.36,37763.933,1195], + [26783.56,37740.722,1194], + [26783.66,37717.511,1193], + [26783.84,37694.3,1192], + [26783.94,37671.088,1191], + [26784.03,37647.877,1190], + [26784.21,37787.144,1196], + [26945.3,37926.477,1202], + [26945.52,37903.266,1201], + [26945.62,37880.055,1200], + [26945.81,37856.844,1199], + [26945.91,37833.633,1198], + [26946.07,37810.422,1197], + [26946.2,37949.688,1203], + [27109.08,38089.011,1209], + [27109.31,38065.8,1208], + [27109.42,38042.588,1207], + [27109.61,38019.377,1206], + [27109.72,37996.166,1205], + [27109.84,37972.955,1204], + [27109.98,38112.222,1210], + [27271.98,38251.555,1216], + [27272.19,38228.344,1215], + [27272.29,38205.133,1214], + [27272.42,38181.922,1213], + [27272.63,38158.711,1212], + [27272.74,38135.5,1211], + [27272.86,38274.766,1217], + [27435.38,38414.1,1223], + [27435.63,38390.888,1222], + [27435.74,38367.677,1221], + [27435.93,38344.466,1220], + [27436.03,38321.255,1219], + [27436.12,38298.044,1218], + [27436.39,38437.311,1224], + [27596.35,38576.633,1230], + [27596.62,38553.422,1229], + [27596.73,38530.211,1228], + [27596.93,38507,1227], + [27597.08,38483.788,1226], + [27597.31,38460.577,1225], + [27597.45,38599.844,1231], + [27757.81,38739.177,1237], + [27758.02,38715.966,1236], + [27758.12,38692.755,1235], + [27758.21,38669.544,1234], + [27758.39,38646.333,1233], + [27758.48,38623.122,1232], + [27758.59,38762.388,1238], + [27922.07,38901.711,1244], + [27922.25,38878.5,1243], + [27922.34,38855.288,1242], + [27922.42,38832.077,1241], + [27922.54,38808.866,1240], + [27922.63,38785.655,1239], + [27922.72,38924.922,1245], + [28084.02,39064.255,1251], + [28084.23,39041.044,1250], + [28084.3,39017.833,1249], + [28084.42,38994.622,1248], + [28084.49,38971.411,1247], + [28084.59,38948.2,1246], + [28084.84,39087.466,1252], + [28247.27,39226.788,1258], + [28247.5,39203.577,1257], + [28247.61,39180.366,1256], + [28247.8,39157.155,1255], + [28247.91,39133.944,1254], + [28248.03,39110.733,1253], + [28248.24,39250,1259], + [28409.79,39389.333,1265], + [28410.01,39366.122,1264], + [28410.16,39342.911,1263], + [28410.26,39319.7,1262], + [28410.4,39296.488,1261], + [28410.73,39273.277,1260], + [28410.86,39412.544,1266], + [28573.39,39551.877,1272], + [28573.67,39528.666,1271], + [28573.79,39505.455,1270], + [28573.94,39482.244,1269], + [28574.06,39459.033,1268], + [28574.15,39435.822,1267], + [28574.31,39575.088,1273], + [28734.46,39714.411,1279], + [28734.79,39691.2,1278], + [28734.94,39667.988,1277], + [28735.17,39644.777,1276], + [28735.27,39621.566,1275], + [28735.38,39598.355,1274], + [28735.52,39737.622,1280], + [28897.75,39876.955,1286], + [28898.07,39853.744,1285], + [28898.43,39830.533,1284], + [28898.65,39807.322,1283], + [28898.75,39784.111,1282], + [28898.86,39760.9,1281], + [28898.99,39900.166,1287], + [29060.62,40039.488,1293], + [29060.87,40016.277,1292], + [29061.02,39993.066,1291], + [29061.29,39969.855,1290], + [29061.41,39946.644,1289], + [29061.57,39923.433,1288], + [29061.84,40062.7,1294], + [29221.44,40202.033,1300], + [29221.7,40178.822,1299], + [29221.82,40155.611,1298], + [29221.94,40132.4,1297], + [29222.23,40109.188,1296], + [29222.37,40085.977,1295], + [29222.51,40225.244,1301], + [29383.83,40364.566,1307], + [29385.58,40318.144,1305], + [29385.79,40294.933,1304], + [29385.92,40271.722,1303], + [29386.06,40248.511,1302], + [29386.19,40387.777,1308], + [29386.32,40341.355,1306], + [29546.27,40527.111,1314], + [29546.56,40503.9,1313], + [29547.11,40480.688,1312], + [29547.25,40457.477,1311], + [29547.36,40434.266,1310], + [29547.47,40411.055,1309], + [29547.58,40550.322,1315], + [29712.46,40689.655,1321], + [29712.74,40666.444,1320], + [29712.88,40643.233,1319], + [29713.11,40620.022,1318], + [29713.22,40596.811,1317], + [29713.32,40573.6,1316], + [29713.46,40712.866,1322], + [29872.6,40852.188,1328], + [29872.84,40828.977,1327], + [29872.94,40805.766,1326], + [29873.14,40782.555,1325], + [29873.22,40759.344,1324], + [29873.3,40736.133,1323], + [29873.45,40875.4,1329], + [30037.37,41014.733,1335], + [30037.59,40991.522,1334], + [30037.72,40968.311,1333], + [30037.91,40945.1,1332], + [30038.05,40921.888,1331], + [30038.17,40898.677,1330], + [30038.29,41037.944,1336], + [30209.54,41177.266,1342], + [30211.82,41154.055,1341], + [30212.55,41130.844,1340], + [30212.65,41107.633,1339], + [30217.43,41061.211,1337], + [30217.86,41200.477,1343], + [30218.12,41084.422,1338], + [30367.11,41339.811,1349], + [30367.3,41316.6,1348], + [30367.41,41293.388,1347], + [30367.54,41270.177,1346], + [30367.74,41246.966,1345], + [30367.82,41223.755,1344], + [30367.91,41363.022,1350], + [30524.24,41502.344,1356], + [30524.54,41479.133,1355], + [30524.66,41455.922,1354], + [30524.8,41432.711,1353], + [30525.17,41409.5,1352], + [30525.33,41386.288,1351], + [30525.48,41525.555,1357], + [30695.69,41664.888,1363], + [30696.02,41641.677,1362], + [30696.24,41618.466,1361], + [30696.45,41595.255,1360], + [30696.62,41572.044,1359], + [30696.88,41548.833,1358], + [30700.84,41688.1,1364], + [30850.04,41827.433,1370], + [30851.28,41804.222,1369], + [30851.44,41781.011,1368], + [30851.76,41757.8,1367], + [30851.93,41734.588,1366], + [30852.12,41711.377,1365], + [30852.3,41850.644,1371], + [31012.69,41989.966,1377], + [31013.13,42013.177,1378], + [31013.35,41966.755,1376], + [31013.53,41943.544,1375], + [31013.7,41920.333,1374], + [31013.87,41897.122,1373], + [31014.05,41873.911,1372], + [31176.13,42152.511,1384], + [31176.5,42129.3,1383], + [31176.63,42106.088,1382], + [31176.81,42082.877,1381], + [31176.94,42059.666,1380], + [31177.1,42036.455,1379], + [31177.27,42175.722,1385], + [31338.66,42315.044,1391], + [31339.11,42291.833,1390], + [31339.29,42268.622,1389], + [31339.54,42245.411,1388], + [31339.72,42222.2,1387], + [31339.91,42198.988,1386], + [31340.18,42338.255,1392], + [31497.75,42477.588,1398], + [31498,42454.377,1397], + [31498.13,42431.166,1396], + [31498.34,42407.955,1395], + [31498.45,42384.744,1394], + [31498.58,42361.533,1393], + [31498.7,42500.8,1399], + [31665.88,42640.122,1405], + [31666.22,42616.911,1404], + [31666.42,42593.7,1403], + [31666.61,42570.488,1402], + [31666.78,42547.277,1401], + [31666.99,42524.066,1400], + [31667.24,42663.333,1406], + [31826.52,42733.033,1409], + [31826.67,42802.666,1412], + [31826.91,42779.455,1411], + [31827.05,42756.244,1410], + [31827.19,42709.822,1408], + [31827.29,42686.611,1407], + [31827.42,42825.877,1413], + [31989.84,42965.211,1419], + [31991.38,42942,1418], + [31991.5,42918.788,1417], + [31991.62,42895.577,1416], + [31991.71,42872.366,1415], + [31992.28,42849.155,1414], + [31992.43,42988.422,1420], + [32146.34,43127.744,1426], + [32146.57,43104.533,1425], + [32146.67,43081.322,1424], + [32146.78,43058.111,1423], + [32146.91,43034.9,1422], + [32147.16,43011.688,1421], + [32147.26,43150.955,1427], + [32309.4,43290.288,1433], + [32309.62,43174.233,1428], + [32309.74,43313.5,1434], + [32309.84,43267.077,1432], + [32309.95,43243.866,1431], + [32310.09,43220.655,1430], + [32310.22,43197.444,1429], + [32472.45,43452.822,1440], + [32472.7,43429.611,1439], + [32472.81,43406.4,1438], + [32472.99,43383.188,1437], + [32473.14,43359.977,1436], + [32473.24,43336.766,1435], + [32473.36,43476.033,1441], + [32635.04,43615.366,1447], + [32635.3,43592.155,1446], + [32635.5,43568.944,1445], + [32635.63,43545.733,1444], + [32635.9,43522.522,1443], + [32636.04,43499.311,1442], + [32636.19,43638.577,1448], + [32798.25,43777.9,1454], + [32798.51,43754.688,1453], + [32798.62,43731.477,1452], + [32798.74,43708.266,1451], + [32798.98,43685.055,1450], + [32799.09,43661.844,1449], + [32799.22,43801.111,1455], + [32962.33,43940.444,1461], + [32963.18,43917.233,1460], + [32963.36,43894.022,1459], + [32963.49,43870.811,1458], + [32963.61,43847.6,1457], + [32963.74,43824.388,1456], + [32963.87,43963.655,1462], + [33125.55,43986.933,1463], + [33125.63,44102.988,1468], + [33125.81,44079.777,1467], + [33125.88,44056.566,1466], + [33125.97,44033.355,1465], + [33126.06,44010.144,1464], + [33126.15,44126.2,1469], + [33287.84,44265.522,1475], + [33288.12,44242.311,1474], + [33288.22,44219.1,1473], + [33288.32,44195.888,1472], + [33288.51,44172.677,1471], + [33288.6,44149.466,1470], + [33288.75,44288.733,1476], + [33453.51,44428.066,1482], + [33454.39,44404.855,1481], + [33454.56,44381.644,1480], + [33454.68,44358.433,1479], + [33454.88,44335.222,1478], + [33454.98,44312.011,1477], + [33455.11,44451.277,1483], + [33613.18,44590.6,1489], + [33614.42,44567.388,1488], + [33614.71,44544.177,1487], + [33614.88,44520.966,1486], + [33615.02,44497.755,1485], + [33615.18,44474.544,1484], + [33615.34,44613.811,1490], + [33775.06,44776.355,1497], + [33775.2,44753.144,1496], + [33775.44,44729.933,1495], + [33775.57,44706.722,1494], + [33775.69,44683.511,1493], + [33775.8,44660.3,1492], + [33775.94,44637.088,1491], + [33938.08,44915.677,1503], + [33938.3,44892.466,1502], + [33938.39,44869.255,1501], + [33938.55,44846.044,1500], + [33938.64,44822.833,1499], + [33938.79,44799.622,1498], + [33938.94,44938.888,1504], + [34098.71,45078.222,1510], + [34098.93,45055.011,1509], + [34099.04,45031.8,1508], + [34099.24,45008.588,1507], + [34099.34,44985.377,1506], + [34099.45,44962.166,1505], + [34099.57,45101.433,1511], + [34260.46,45240.766,1517], + [34260.69,45217.555,1516], + [34260.81,45194.344,1515], + [34260.93,45171.133,1514], + [34261.16,45147.922,1513], + [34261.26,45124.711,1512], + [34261.39,45263.977,1518], + [34425.51,45287.244,1519], + [34425.7,45403.3,1524], + [34426.04,45380.088,1523], + [34426.2,45356.877,1522], + [34426.35,45333.666,1521], + [34426.5,45310.455,1520], + [34426.71,45426.511,1525], + [34589.27,45565.844,1531], + [34589.57,45542.633,1530], + [34591.72,45519.422,1529], + [34591.97,45496.211,1528], + [34592.09,45473,1527], + [34592.22,45449.788,1526], + [34592.38,45589.055,1532], + [34747.99,45728.377,1538], + [34748.24,45705.166,1537], + [34748.34,45681.955,1536], + [34748.51,45658.744,1535], + [34748.6,45635.533,1534], + [34748.7,45612.322,1533], + [34748.89,45751.588,1539], + [34910.41,45890.922,1545], + [34910.63,45867.711,1544], + [34910.74,45844.5,1543], + [34910.93,45821.288,1542], + [34911.05,45798.077,1541], + [34911.16,45774.866,1540], + [34911.31,45914.133,1546] + ] +} diff --git a/test/audio-quality/traces/relay-bbb.json b/test/audio-quality/traces/relay-bbb.json new file mode 100644 index 0000000000..b18b143ab6 --- /dev/null +++ b/test/audio-quality/traces/relay-bbb.json @@ -0,0 +1,1954 @@ +{ + "version": 1, + "source": "https://cdn.moq.dev/demo bbb.hang, the public demo's MPEG-TS import, recorded 2026-09-29", + "description": "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured.", + "transport": "webtransport", + "rtt": null, + "config": { + "codec": "mp4a.40.2", + "container": { + "kind": "legacy" + }, + "description": "1210", + "sampleRate": 44100, + "numberOfChannels": 2, + "bitrate": 139613, + "jitter": 186 + }, + "arrivals": [ + [0,234535,9642], + [2.98,240014.922,9878], + [9.78,240038.133,9879], + [42.28,240061.344,9880], + [42.65,234558.211,9643], + [42.87,240084.555,9881], + [43.17,240107.766,9882], + [43.41,240131.044,9883], + [43.69,240154.255,9884], + [43.97,240177.466,9885], + [44.19,240200.677,9886], + [44.4,234581.422,9644], + [44.65,240223.888,9887], + [44.85,240247.1,9888], + [45.08,240270.311,9889], + [46.07,240293.588,9890], + [46.25,240316.8,9891], + [46.46,240340.011,9892], + [46.65,240363.222,9893], + [47.12,240386.433,9894], + [47.32,234604.633,9645], + [47.78,240409.644,9895], + [47.99,240432.855,9896], + [48.33,234627.911,9646], + [48.59,240456.122,9897], + [48.81,240479.333,9898], + [49,240502.544,9899], + [49.2,234651.122,9647], + [49.41,240525.755,9900], + [49.6,240548.966,9901], + [49.81,234674.333,9648], + [50.03,240572.177,9902], + [50.27,240595.388,9903], + [50.55,234697.544,9649], + [51.48,240618.666,9904], + [51.71,235394.133,9679], + [51.88,234720.755,9650], + [52.03,234883.3,9657], + [52.18,234743.966,9651], + [52.38,234767.177,9652], + [52.59,234326.033,9633], + [52.85,234790.455,9653], + [53.07,234349.244,9634], + [53.31,234372.455,9635], + [53.54,234813.666,9654], + [53.88,234395.666,9636], + [54.16,234418.877,9637], + [57.75,234442.155,9638], + [58.49,234465.366,9639], + [58.78,238900.355,9830], + [58.93,234860.088,9656], + [59.06,234488.577,9640], + [59.42,234511.788,9641], + [59.59,234836.877,9655], + [59.74,238969.988,9833], + [59.91,238993.266,9834], + [60.05,239039.688,9836], + [60.19,239062.9,9837], + [60.34,239086.111,9838], + [60.47,239109.322,9839], + [60.63,239132.533,9840], + [60.83,239155.811,9841], + [60.97,239179.022,9842], + [61.11,239202.233,9843], + [61.25,239225.444,9844], + [61.38,239248.655,9845], + [61.52,239271.866,9846], + [61.66,239504.1,9856], + [61.81,239527.311,9857], + [61.97,239550.522,9858], + [62.11,239573.733,9859], + [62.34,239596.944,9860], + [62.52,239620.155,9861], + [62.71,239295.077,9847], + [63.2,239318.344,9848], + [63.37,239341.555,9849], + [63.53,239643.422,9862], + [63.68,239666.633,9863], + [63.9,239689.844,9864], + [64.05,239364.766,9850], + [64.75,239387.977,9851], + [64.98,239411.188,9852], + [65.18,239713.055,9865], + [65.37,239736.266,9866], + [65.57,239759.477,9867], + [65.76,239782.688,9868], + [66.32,239805.966,9869], + [66.56,239829.177,9870], + [66.77,239434.4,9853], + [67,239457.611,9854], + [67.21,239480.888,9855], + [67.43,239852.388,9871], + [67.65,239875.6,9872], + [67.88,239898.811,9873], + [68.09,239922.022,9874], + [68.34,239945.233,9875], + [68.76,239968.5,9876], + [68.99,239991.711,9877], + [69.18,238761.033,9824], + [69.35,238714.611,9822], + [69.53,238737.822,9823], + [69.72,238784.244,9825], + [69.92,238807.455,9826], + [70.13,238830.722,9827], + [70.5,238853.933,9828], + [70.71,238877.144,9829], + [77.44,238923.566,9831], + [77.63,238946.777,9832], + [77.8,239016.477,9835], + [77.98,238644.911,9819], + [78.14,238668.188,9820], + [78.3,238691.4,9821], + [78.45,238598.488,9817], + [78.6,238621.7,9818], + [78.76,237971.544,9790], + [78.94,237994.755,9791], + [79.12,238018.033,9792], + [79.31,238041.244,9793], + [79.52,238064.455,9794], + [79.71,238087.666,9795], + [79.88,238110.877,9796], + [80.04,238134.088,9797], + [80.19,238157.3,9798], + [80.34,238180.566,9799], + [80.52,238203.777,9800], + [80.69,238226.988,9801], + [80.88,238250.2,9802], + [81.05,238273.411,9803], + [81.25,238296.622,9804], + [81.41,238319.833,9805], + [81.63,238343.111,9806], + [81.81,238366.322,9807], + [81.98,238389.533,9808], + [82.22,238412.744,9809], + [82.38,238435.955,9810], + [82.54,238459.166,9811], + [82.77,238482.377,9812], + [82.99,238505.644,9813], + [83.16,238528.855,9814], + [83.37,238552.066,9815], + [83.56,238575.277,9816], + [83.72,237483.933,9769], + [83.96,237182.066,9756], + [84.13,237205.277,9757], + [84.63,237228.555,9758], + [84.8,237251.766,9759], + [85.95,237274.977,9760], + [86.24,237298.188,9761], + [86.44,237321.4,9762], + [86.63,237344.611,9763], + [86.9,237367.822,9764], + [87.11,237391.088,9765], + [87.3,237414.3,9766], + [87.47,236044.288,9707], + [87.69,236067.5,9708], + [87.88,237437.511,9767], + [88.07,237507.144,9770], + [88.27,237530.355,9771], + [88.46,237553.633,9772], + [88.63,237576.844,9773], + [88.85,236090.777,9709], + [89.02,236113.988,9710], + [89.19,236137.2,9711], + [89.37,236160.411,9712], + [89.55,237600.055,9774], + [89.72,236183.622,9713], + [89.9,236206.833,9714], + [90.1,236230.044,9715], + [90.5,236276.522,9717], + [90.72,236299.733,9718], + [91.03,236346.155,9720], + [91.19,237623.266,9775], + [91.42,236392.577,9722], + [91.57,236415.855,9723], + [91.8,236439.066,9724], + [91.95,236462.277,9725], + [92.09,236485.488,9726], + [92.32,237692.944,9778], + [92.5,236508.7,9727], + [92.82,235231.588,9672], + [93.02,235812.111,9697], + [93.29,236555.122,9729], + [93.46,236578.388,9730], + [93.78,236624.811,9732], + [94,236648.022,9733], + [94.2,236671.233,9734], + [94.49,237739.366,9780], + [94.66,237762.577,9781], + [95.03,236694.444,9735], + [95.28,237809,9783], + [95.5,236717.655,9736], + [95.75,237832.211,9784], + [95.97,236740.933,9737], + [96.32,236787.355,9739], + [96.57,236810.566,9740], + [96.97,236856.988,9742], + [97.19,237855.488,9785], + [97.4,236880.2,9743], + [97.57,236903.466,9744], + [97.82,236926.677,9745], + [98.03,236949.888,9746], + [98.27,236973.1,9747], + [98.5,236996.311,9748], + [98.82,237019.522,9749], + [99.15,237042.733,9750], + [99.31,237066.011,9751], + [99.5,237089.222,9752], + [99.68,237112.433,9753], + [99.86,237135.644,9754], + [100.03,237158.855,9755], + [100.2,237878.7,9786], + [100.39,237901.911,9787], + [102.17,237925.122,9788], + [102.47,237948.333,9789], + [102.69,237460.722,9768], + [102.9,235254.8,9673], + [103.09,235278.077,9674], + [103.92,235301.288,9675], + [104.1,235324.5,9676], + [104.32,235347.711,9677], + [104.5,235370.922,9678], + [104.65,235161.955,9669], + [104.8,235185.166,9670], + [104.97,235208.377,9671], + [105.12,235858.533,9699], + [105.29,235881.744,9700], + [105.46,235904.955,9701], + [105.61,235928.233,9702], + [105.79,235951.444,9703], + [105.97,235974.655,9704], + [106.14,235997.866,9705], + [106.34,236021.077,9706], + [106.56,234906.511,9658], + [106.76,234929.722,9659], + [106.95,234953,9660], + [107.12,234976.211,9661], + [107.34,234999.422,9662], + [107.52,235022.633,9663], + [107.71,235045.844,9664], + [107.93,235069.055,9665], + [108.14,235092.266,9666], + [108.32,235115.533,9667], + [108.54,235138.744,9668], + [109.17,235417.344,9680], + [109.4,235440.611,9681], + [109.59,235463.822,9682], + [109.8,240897.255,9916], + [110.02,240920.466,9917], + [110.24,240943.744,9918], + [110.45,240966.955,9919], + [110.68,240990.166,9920], + [110.9,241013.377,9921], + [111.12,241036.588,9922], + [111.34,241059.8,9923], + [111.58,241083.011,9924], + [111.85,241106.277,9925], + [112.1,241129.488,9926], + [112.36,241152.7,9927], + [115.86,235487.033,9683], + [116.5,235510.244,9684], + [116.87,235533.455,9685], + [117.19,235556.666,9686], + [117.55,235579.877,9687], + [117.84,235603.155,9688], + [118.08,235626.366,9689], + [118.32,235649.577,9690], + [118.58,235672.788,9691], + [118.82,235696,9692], + [119.11,235719.211,9693], + [119.44,235742.422,9694], + [119.7,235765.688,9695], + [120.07,235788.9,9696], + [120.42,241175.911,9928], + [120.69,243474.677,10027], + [121.3,242824.522,9999], + [121.55,243242.511,10017], + [121.81,242847.733,10000], + [122.07,242801.311,9998], + [122.36,242870.944,10001], + [122.63,243265.722,10018], + [122.96,243219.3,10016], + [123.26,243288.933,10019], + [123.53,242894.222,10002], + [123.88,243312.144,10020], + [124.18,243335.355,10021], + [124.49,242917.433,10003], + [125.22,243358.566,10022], + [125.51,242522.655,9986], + [125.78,242940.644,10004], + [126.09,242545.866,9987], + [126.36,242569.144,9988], + [126.65,242592.355,9989], + [127.34,242615.566,9990], + [128.12,242638.777,9991], + [128.82,242661.988,9992], + [129.38,243381.833,10023], + [129.97,242685.2,9993], + [131.38,242708.411,9994], + [132.19,242731.677,9995], + [132.6,242754.888,9996], + [132.97,242778.1,9997], + [133.36,242499.444,9985], + [133.72,242476.233,9984], + [134.14,242453.022,9983], + [134.47,242429.811,9982], + [134.77,242406.6,9981], + [135.22,242963.855,10005], + [135.52,242987.066,10006], + [136.13,243405.044,10024], + [136.43,243428.255,10025], + [136.72,243010.277,10007], + [137.05,243033.488,10008], + [137.41,243451.466,10026], + [137.75,243079.966,10010], + [138.07,243103.177,10011], + [138.36,243497.888,10028], + [138.66,243521.1,10029], + [139,243126.388,10012], + [139.59,243149.6,10013], + [139.93,243544.377,10030], + [140.28,243172.811,10014], + [140.62,240641.877,9905], + [140.94,240665.088,9906], + [141.25,240688.3,9907], + [141.59,240711.511,9908], + [141.92,240734.722,9909], + [142.25,240757.933,9910], + [142.38,237785.788,9782], + [143.09,237716.155,9779], + [143.4,237669.688,9777], + [143.68,237646.477,9776], + [143.97,236833.777,9741], + [144.26,236764.144,9738], + [144.55,236601.6,9731], + [144.83,236531.911,9728], + [145.11,236369.366,9721], + [145.84,236322.944,9719], + [146.35,236253.311,9716], + [146.74,235835.322,9698], + [147.43,240781.2,9911], + [147.84,240804.411,9912], + [148.72,240827.622,9913], + [149.34,240850.833,9914], + [150.47,243056.755,10009], + [151.16,242127.944,9969], + [151.94,242151.155,9970], + [152.75,242174.366,9971], + [153.65,242197.577,9972], + [154.65,242220.788,9973], + [155.68,242244.055,9974], + [156.55,242267.266,9975], + [157.66,242290.477,9976], + [158.09,242313.688,9977], + [158.54,242336.9,9978], + [159.02,242360.111,9979], + [159.44,242383.322,9980], + [159.85,241918.977,9960], + [160.5,241942.188,9961], + [161.22,241965.4,9962], + [161.93,241988.611,9963], + [162.73,242011.822,9964], + [163.45,242035.033,9965], + [164.22,242058.244,9966], + [164.95,242081.522,9967], + [165.68,242104.733,9968], + [166.67,241779.655,9954], + [167.23,241802.866,9955], + [167.68,241826.077,9956], + [168.12,241849.288,9957], + [168.65,241872.5,9958], + [169.09,241895.711,9959], + [169.49,244171.266,10057], + [169.9,241570.633,9945], + [170.39,241593.9,9946], + [171.01,241617.111,9947], + [171.49,241640.322,9948], + [171.91,241663.533,9949], + [172.32,241408.088,9938], + [172.88,243776.555,10040], + [173.72,241431.366,9939], + [174.18,241454.577,9940], + [174.6,243799.766,10041], + [175.03,243822.977,10042], + [175.45,241477.788,9941], + [175.89,243846.188,10043], + [176.34,243869.455,10044], + [176.76,241501,9942], + [178.37,243892.666,10045], + [178.8,243915.877,10046], + [179.25,241524.211,9943], + [179.69,243939.088,10047], + [180.12,243962.3,10048], + [181.03,243985.511,10049], + [181.56,244008.722,10050], + [182,244032,10051], + [182.44,244055.211,10052], + [182.93,244078.422,10053], + [183.38,244101.633,10054], + [183.84,241547.422,9944], + [184.41,244124.844,10055], + [184.87,243753.344,10039], + [185.34,241384.877,9937], + [185.82,243730.133,10038], + [186.28,243706.922,10037], + [187.33,241361.666,9936], + [228.22,241338.455,9935], + [228.43,243683.644,10036], + [228.61,243660.433,10035], + [228.84,241315.244,9934], + [229.06,241292.033,9933], + [229.27,243637.222,10034], + [229.5,243614.011,10033], + [229.69,241268.822,9932], + [229.88,244148.055,10056], + [230.08,241686.744,9950], + [230.25,241709.955,9951], + [230.42,241733.166,9952], + [230.59,241756.444,9953], + [230.77,244287.377,10062], + [230.94,244310.588,10063], + [231.12,244333.8,10064], + [231.28,243196.022,10015], + [231.78,243567.588,10031], + [231.98,241199.122,9929], + [232.15,241222.333,9930], + [232.32,243590.8,10032], + [232.5,241245.544,9931], + [232.69,244194.533,10058], + [232.9,244217.744,10059], + [233.09,244240.955,10060], + [233.3,244264.166,10061], + [233.5,244449.922,10069], + [233.69,244473.133,10070], + [233.88,244496.344,10071], + [234.07,244357.077,10065], + [234.27,244380.288,10066], + [234.47,244403.5,10067], + [234.7,244426.711,10068], + [235.54,240874.044,9915], + [309.69,244542.822,10073], + [309.93,244566.033,10074], + [310.03,244589.244,10075], + [310.13,244612.455,10076], + [310.31,244635.666,10077], + [310.41,244658.877,10078], + [310.59,244519.611,10072], + [502.34,244705.366,10080], + [502.54,244728.577,10081], + [502.63,244751.788,10082], + [502.79,244775,10083], + [502.88,244798.211,10084], + [502.97,244821.422,10085], + [503.07,244682.155,10079], + [638.06,244867.911,10087], + [638.23,244891.122,10088], + [638.32,244960.755,10091], + [638.47,244914.333,10089], + [638.58,244937.544,10090], + [638.66,244844.7,10086], + [638.77,244983.966,10092], + [818.46,245030.444,10094], + [818.66,245053.655,10095], + [818.75,245076.866,10096], + [818.87,245100.077,10097], + [819.04,245123.288,10098], + [819.12,245146.5,10099], + [819.22,245007.233,10093], + [957.28,245192.988,10101], + [957.49,245216.2,10102], + [957.57,245239.411,10103], + [957.75,245262.622,10104], + [957.86,245285.833,10105], + [958.05,245309.044,10106], + [958.16,245169.777,10100], + [1107.13,245355.522,10108], + [1107.32,245378.733,10109], + [1107.44,245401.944,10110], + [1107.6,245425.155,10111], + [1107.71,245448.366,10112], + [1107.84,245471.577,10113], + [1108.03,245332.311,10107], + [1261.86,245518.066,10115], + [1262.11,245541.277,10116], + [1262.25,245564.488,10117], + [1262.41,245587.7,10118], + [1262.5,245610.911,10119], + [1262.6,245634.122,10120], + [1262.76,245494.855,10114], + [1424.35,245750.233,10125], + [1424.53,245773.444,10126], + [1424.63,245796.655,10127], + [1424.72,245657.388,10121], + [1424.91,245680.6,10122], + [1424.99,245703.811,10123], + [1425.09,245727.022,10124], + [1588.68,245843.144,10129], + [1588.91,245866.355,10130], + [1589.03,245889.566,10131], + [1589.19,245912.777,10132], + [1589.28,245935.988,10133], + [1589.37,245959.2,10134], + [1589.47,245819.933,10128], + [1753.91,246005.688,10136], + [1754.11,246028.9,10137], + [1754.21,246052.111,10138], + [1754.37,246075.322,10139], + [1754.46,246098.533,10140], + [1754.57,246121.744,10141], + [1754.71,245982.477,10135], + [1921.03,246168.222,10143], + [1921.34,246191.433,10144], + [1921.44,246214.644,10145], + [1921.66,246237.855,10146], + [1921.78,246261.066,10147], + [1921.9,246284.277,10148], + [1922.1,246145.011,10142], + [2087.59,246330.766,10150], + [2087.85,246353.977,10151], + [2087.97,246377.188,10152], + [2088.12,246400.4,10153], + [2088.21,246423.611,10154], + [2088.3,246446.822,10155], + [2088.38,246307.555,10149], + [2255.17,246493.3,10157], + [2255.32,246516.511,10158], + [2255.41,246586.144,10161], + [2255.54,246539.722,10159], + [2255.63,246562.933,10160], + [2255.71,246470.088,10156], + [2255.9,246609.355,10162], + [2424.94,246655.844,10164], + [2425.25,246679.055,10165], + [2425.35,246702.266,10166], + [2425.51,246725.477,10167], + [2425.59,246748.688,10168], + [2425.66,246771.9,10169], + [2425.73,246632.633,10163], + [2575.38,246818.377,10171], + [2575.52,246841.588,10172], + [2575.59,246864.8,10173], + [2575.72,246888.011,10174], + [2575.78,246911.222,10175], + [2575.88,246934.433,10176], + [2575.95,246795.166,10170], + [2749.89,246957.711,10177], + [2750.14,246980.922,10178], + [2750.25,247004.133,10179], + [2750.44,247027.344,10180], + [2750.56,247050.555,10181], + [2750.66,247073.766,10182], + [2750.78,247096.977,10183], + [2900.56,247120.255,10184], + [2900.73,247143.466,10185], + [2900.84,247166.677,10186], + [2900.99,247189.888,10187], + [2901.07,247213.1,10188], + [2901.15,247236.311,10189], + [2901.31,247259.522,10190], + [3071.44,247282.788,10191], + [3071.62,247306,10192], + [3071.7,247329.211,10193], + [3071.87,247352.422,10194], + [3071.95,247375.633,10195], + [3072.03,247398.844,10196], + [3072.17,247422.055,10197], + [3212.59,247468.544,10199], + [3212.79,247445.333,10198], + [3212.9,247514.966,10201], + [3213.05,247491.755,10200], + [3213.15,247538.177,10202], + [3213.24,247561.388,10203], + [3213.38,247584.6,10204], + [3378.91,247631.077,10206], + [3379.09,247654.288,10207], + [3379.18,247607.866,10205], + [3379.3,247677.5,10208], + [3379.4,247723.922,10210], + [3379.48,247747.133,10211], + [3379.6,247700.711,10209], + [3543.59,247770.411,10212], + [3544.84,247886.466,10217], + [3544.94,247909.677,10218], + [3545.05,247793.622,10213], + [3545.15,247816.833,10214], + [3545.24,247840.044,10215], + [3545.31,247863.255,10216], + [3716.75,248072.211,10225], + [3717.96,248049,10224], + [3718.09,247932.944,10219], + [3718.38,248025.788,10223], + [3718.49,248002.577,10222], + [3718.6,247979.366,10221], + [3718.68,247956.155,10220], + [3878.51,248118.7,10227], + [3878.66,248141.911,10228], + [3878.72,248165.122,10229], + [3878.87,248188.333,10230], + [3878.94,248211.544,10231], + [3879,248234.755,10232], + [3879.09,248095.488,10226], + [4024.78,248258.033,10233], + [4024.93,248397.3,10239], + [4025.03,248281.244,10234], + [4025.12,248304.455,10235], + [4025.22,248374.088,10238], + [4025.29,248327.666,10236], + [4025.37,248350.877,10237], + [4186.69,248420.566,10240], + [4186.87,248443.777,10241], + [4186.95,248466.988,10242], + [4187.18,248490.2,10243], + [4187.28,248513.411,10244], + [4187.41,248536.622,10245], + [4187.53,248559.833,10246], + [4347.91,248583.111,10247], + [4348.73,248606.322,10248], + [4348.86,248629.533,10249], + [4348.99,248652.744,10250], + [4349.12,248675.955,10251], + [4349.21,248699.166,10252], + [4349.28,248722.377,10253], + [4532.82,248768.855,10255], + [4533,248792.066,10256], + [4533.07,248815.277,10257], + [4533.21,248838.488,10258], + [4533.27,248861.7,10259], + [4533.4,248884.911,10260], + [4533.52,248745.644,10254], + [4692.91,248908.188,10261], + [4693.07,249047.455,10267], + [4693.18,248931.4,10262], + [4693.27,248954.611,10263], + [4693.34,249024.244,10266], + [4693.5,248977.822,10264], + [4693.58,249001.033,10265], + [4866.71,249070.722,10268], + [4866.88,249093.933,10269], + [4866.96,249117.144,10270], + [4867.09,249140.355,10271], + [4867.16,249163.566,10272], + [4867.24,249186.777,10273], + [4867.31,249209.988,10274], + [5018.38,249326.111,10279], + [5018.5,249349.322,10280], + [5018.59,249372.533,10281], + [5018.65,249233.266,10275], + [5018.75,249302.9,10278], + [5018.82,249256.477,10276], + [5018.9,249279.688,10277], + [5184.16,249395.811,10282], + [5184.35,249419.022,10283], + [5184.44,249442.233,10284], + [5184.53,249465.444,10285], + [5184.59,249488.655,10286], + [5184.72,249511.866,10287], + [5184.79,249535.077,10288], + [5338.05,249558.344,10289], + [5338.18,249581.555,10290], + [5338.29,249604.766,10291], + [5338.36,249627.977,10292], + [5338.5,249651.188,10293], + [5338.57,249674.4,10294], + [5338.65,249697.611,10295], + [5503.97,249720.888,10296], + [5504.18,249744.1,10297], + [5504.28,249767.311,10298], + [5504.46,249790.522,10299], + [5504.56,249813.733,10300], + [5504.65,249836.944,10301], + [5504.84,249860.155,10302], + [5650.21,249883.422,10303], + [5650.35,249906.633,10304], + [5650.45,249929.844,10305], + [5650.53,249953.055,10306], + [5650.65,249976.266,10307], + [5650.72,249999.477,10308], + [5650.8,250022.688,10309], + [5812.98,250045.966,10310], + [5813.17,250069.177,10311], + [5813.27,250092.388,10312], + [5813.4,250115.6,10313], + [5813.47,250138.811,10314], + [5813.55,250162.022,10315], + [5813.69,250185.233,10316], + [5974.33,250208.5,10317], + [5974.48,250347.766,10323], + [5974.59,250324.555,10322], + [5974.67,250301.344,10321], + [5974.8,250254.922,10319], + [5974.93,250231.711,10318], + [5975.03,250278.133,10320], + [6137.69,250371.044,10324], + [6137.88,250394.255,10325], + [6137.97,250417.466,10326], + [6138.11,250440.677,10327], + [6138.19,250510.311,10330], + [6138.28,250487.1,10329], + [6138.43,250463.888,10328], + [6304.4,250533.588,10331], + [6304.56,250556.8,10332], + [6304.64,250580.011,10333], + [6304.77,250603.222,10334], + [6304.9,250626.433,10335], + [6304.99,250649.644,10336], + [6305.13,250672.855,10337], + [6462.38,250696.122,10338], + [6463.72,250719.333,10339], + [6463.87,250742.544,10340], + [6463.96,250765.755,10341], + [6464.09,250788.966,10342], + [6464.17,250812.177,10343], + [6464.27,250835.388,10344], + [6629,250997.933,10351], + [6629.24,250881.877,10346], + [6629.38,250905.088,10347], + [6629.53,250974.722,10350], + [6629.8,250928.3,10348], + [6629.94,250951.511,10349], + [6630.08,250858.666,10345], + [6795.28,251021.2,10352], + [6795.44,251044.411,10353], + [6795.52,251067.622,10354], + [6795.66,251090.833,10355], + [6795.75,251114.044,10356], + [6795.85,251137.255,10357], + [6795.94,251160.466,10358], + [6962.14,251253.377,10362], + [6962.27,251276.588,10363], + [6962.33,251299.8,10364], + [6962.44,251323.011,10365], + [6962.5,251206.955,10360], + [6962.56,251230.166,10361], + [6962.68,251183.744,10359], + [7128.44,251346.277,10366], + [7128.59,251485.544,10372], + [7128.66,251462.333,10371], + [7128.79,251439.122,10370], + [7128.87,251369.488,10367], + [7128.95,251392.7,10368], + [7129.03,251415.911,10369], + [7294.6,251508.822,10373], + [7294.77,251532.033,10374], + [7294.87,251601.666,10377], + [7295,251555.244,10375], + [7295.12,251578.455,10376], + [7295.2,251624.877,10378], + [7295.28,251648.088,10379], + [7437.72,251694.577,10381], + [7437.94,251717.788,10382], + [7438.03,251741,10383], + [7438.17,251764.211,10384], + [7438.24,251787.422,10385], + [7438.32,251810.633,10386], + [7438.39,251671.366,10380], + [7602.92,251833.9,10387], + [7603.62,251973.166,10393], + [7604.08,251857.111,10388], + [7604.31,251880.322,10389], + [7604.6,251949.955,10392], + [7604.86,251903.533,10390], + [7605.18,251926.744,10391], + [7771.25,251996.444,10394], + [7771.68,252019.655,10395], + [7771.87,252042.866,10396], + [7772.03,252066.077,10397], + [7772.31,252089.288,10398], + [7772.43,252112.5,10399], + [7772.57,252135.711,10400], + [7933.8,252158.977,10401], + [7933.97,252182.188,10402], + [7934.03,252205.4,10403], + [7934.16,252228.611,10404], + [7934.22,252251.822,10405], + [7934.28,252275.033,10406], + [7934.36,252298.244,10407], + [8092.4,252344.733,10409], + [8092.54,252367.944,10410], + [8092.61,252391.155,10411], + [8092.75,252414.366,10412], + [8092.82,252437.577,10413], + [8093.09,252460.788,10414], + [8093.21,252321.522,10408], + [8254.12,252600.111,10420], + [8254.28,252623.322,10421], + [8254.37,252484.055,10415], + [8254.54,252530.477,10417], + [8254.62,252553.688,10418], + [8267.91,252576.9,10419], + [8268.05,252507.266,10416], + [8466.57,252785.866,10428], + [8466.72,252646.6,10422], + [8466.8,252669.811,10423], + [8466.92,252739.444,10426], + [8466.99,252762.655,10427], + [8467.06,252693.022,10424], + [8467.19,252716.233,10425], + [8586.91,252832.355,10430], + [8587.04,252855.566,10431], + [8587.1,252878.777,10432], + [8587.17,252901.988,10433], + [8587.31,252925.2,10434], + [8587.37,252948.411,10435], + [8587.46,252809.144,10429], + [8760.22,253064.522,10440], + [8760.37,253110.944,10442], + [8760.44,252994.888,10437], + [8760.55,253018.1,10438], + [8760.63,253087.733,10441], + [8760.69,253041.311,10439], + [8760.77,252971.677,10436], + [8967.66,253203.855,10446], + [8967.84,253227.066,10447], + [8968,253250.277,10448], + [8968.13,253134.222,10443], + [8968.23,253296.7,10450], + [8968.4,253157.433,10444], + [8968.49,253180.644,10445], + [8968.58,253273.488,10449], + [9092.01,253459.244,10457], + [9111.41,253412.822,10455], + [9111.57,253319.977,10451], + [9111.65,253343.188,10452], + [9111.78,253366.4,10453], + [9111.85,253389.611,10454], + [9111.93,253436.033,10456], + [9255.93,253482.522,10458], + [9256.09,253528.944,10460], + [9256.16,253552.155,10461], + [9256.34,253575.366,10462], + [9256.41,253598.577,10463], + [9256.49,253621.788,10464], + [9256.62,253505.733,10459], + [9421.78,253668.266,10466], + [9421.96,253691.477,10467], + [9422.04,253714.688,10468], + [9422.12,253737.9,10469], + [9422.31,253761.111,10470], + [9422.4,253784.322,10471], + [9422.5,253645.055,10465], + [9588.17,253830.811,10473], + [9588.44,253854.022,10474], + [9588.53,253877.233,10475], + [9588.81,253900.444,10476], + [9589.1,253923.655,10477], + [9589.18,253946.866,10478], + [9589.26,253807.6,10472], + [9754.88,254062.977,10483], + [9755.11,254086.188,10484], + [9755.25,254109.4,10485], + [9755.38,253993.344,10480], + [9755.45,254016.555,10481], + [9755.53,253970.133,10479], + [9755.6,254039.766,10482], + [9937.53,254225.522,10490], + [9937.72,254132.677,10486], + [9937.82,254155.888,10487], + [9937.94,254179.1,10488], + [9938.07,254248.733,10491], + [9938.15,254271.944,10492], + [9938.22,254202.311,10489], + [10093.97,254434.488,10499], + [10094.12,254295.222,10493], + [10094.19,254318.433,10494], + [10094.3,254341.644,10495], + [10094.36,254364.855,10496], + [10094.43,254388.066,10497], + [10094.5,254411.277,10498], + [10226.07,254480.966,10501], + [10226.2,254504.177,10502], + [10226.3,254527.388,10503], + [10226.35,254550.6,10504], + [10226.47,254573.811,10505], + [10226.53,254597.022,10506], + [10226.6,254457.755,10500], + [10410.06,254643.511,10508], + [10410.21,254666.722,10509], + [10410.28,254736.355,10512], + [10410.41,254689.933,10510], + [10410.47,254713.144,10511], + [10410.55,254620.3,10507], + [10410.62,254759.566,10513], + [10563.9,254806.044,10515], + [10564.03,254829.255,10516], + [10564.1,254852.466,10517], + [10564.22,254875.677,10518], + [10564.29,254898.888,10519], + [10564.35,254922.1,10520], + [10564.49,254782.833,10514], + [10729.24,254968.588,10522], + [10729.44,254991.8,10523], + [10729.63,255015.011,10524], + [10729.77,255038.222,10525], + [10729.97,255084.644,10527], + [10730.2,254945.377,10521], + [10730.31,255061.433,10526], + [10888.53,255131.122,10529], + [10888.82,255154.333,10530], + [10888.93,255177.544,10531], + [10889,255200.755,10532], + [10889.13,255223.966,10533], + [10889.19,255247.177,10534], + [10889.43,255107.911,10528], + [11084.73,255293.666,10536], + [11084.97,255316.877,10537], + [11085.09,255340.088,10538], + [11085.3,255363.3,10539], + [11085.43,255386.511,10540], + [11085.55,255409.722,10541], + [11085.69,255270.455,10535], + [11220.5,255456.211,10543], + [11220.65,255479.422,10544], + [11220.75,255502.633,10545], + [11220.84,255525.844,10546], + [11220.97,255549.055,10547], + [11221.06,255572.266,10548], + [11221.16,255433,10542], + [11379.03,255618.744,10550], + [11379.19,255641.955,10551], + [11379.27,255665.166,10552], + [11379.39,255688.377,10553], + [11379.47,255711.588,10554], + [11379.55,255734.8,10555], + [11379.63,255595.533,10549], + [11524.88,255781.288,10557], + [11525.08,255804.5,10558], + [11525.2,255827.711,10559], + [11525.31,255850.922,10560], + [11525.52,255874.133,10561], + [11525.61,255897.344,10562], + [11525.7,255758.077,10556], + [11688,255943.822,10564], + [11688.25,255967.033,10565], + [11688.36,255990.244,10566], + [11688.55,256013.455,10567], + [11688.65,256036.666,10568], + [11688.76,256059.877,10569], + [11688.88,255920.611,10563], + [11849.95,256106.366,10571], + [11850.16,256129.577,10572], + [11850.28,256152.788,10573], + [11850.42,256176,10574], + [11850.5,256199.211,10575], + [11850.56,256222.422,10576], + [11850.65,256083.155,10570], + [12012.25,256268.9,10578], + [12012.43,256292.111,10579], + [12012.53,256315.322,10580], + [12012.69,256338.533,10581], + [12012.77,256361.744,10582], + [12012.89,256384.955,10583], + [12012.99,256245.688,10577], + [12204.19,256431.444,10585], + [12204.41,256454.655,10586], + [12204.49,256477.866,10587], + [12204.68,256501.077,10588], + [12204.78,256524.288,10589], + [12204.88,256547.5,10590], + [12204.98,256408.233,10584], + [12340.82,256593.988,10592], + [12341.63,256617.2,10593], + [12341.77,256686.833,10596], + [12341.88,256640.411,10594], + [12342,256663.622,10595], + [12342.08,256570.777,10591], + [12342.17,256710.044,10597], + [12512.62,256849.366,10603], + [12512.78,256872.577,10604], + [12516.18,256733.311,10598], + [12516.31,256756.522,10599], + [12516.43,256779.733,10600], + [12516.5,256802.944,10601], + [12516.63,256826.155,10602], + [12671.85,256965.488,10608], + [12672.04,256988.7,10609], + [12672.41,257011.911,10610], + [12672.53,257035.122,10611], + [12672.63,256895.855,10605], + [12672.72,256919.066,10606], + [12672.96,256942.277,10607], + [12840.08,257081.6,10613], + [12840.27,257104.811,10614], + [12840.38,257128.022,10615], + [12840.52,257151.233,10616], + [12840.59,257174.444,10617], + [12840.66,257197.655,10618], + [12840.75,257058.388,10612], + [13003.86,257360.2,10625], + [13004.03,257313.777,10623], + [13004.16,257336.988,10624], + [13004.24,257290.566,10622], + [13004.41,257267.355,10621], + [13004.5,257244.144,10620], + [13004.57,257220.933,10619], + [13150.1,257522.733,10632], + [13150.26,257383.466,10626], + [13150.37,257406.677,10627], + [13150.46,257429.888,10628], + [13150.56,257453.1,10629], + [13150.64,257476.311,10630], + [13150.72,257499.522,10631], + [13313.7,257685.277,10639], + [13314,257569.222,10634], + [13314.09,257592.433,10635], + [13314.16,257615.644,10636], + [13314.27,257638.855,10637], + [13314.37,257662.066,10638], + [13314.47,257546.011,10633], + [13476.18,257731.766,10641], + [13476.34,257754.977,10642], + [13476.44,257778.188,10643], + [13476.52,257801.4,10644], + [13476.63,257824.611,10645], + [13476.71,257847.822,10646], + [13476.78,257708.555,10640], + [13638.15,257894.3,10648], + [13638.41,257917.511,10649], + [13638.58,257987.144,10652], + [13638.87,257940.722,10650], + [13639.47,257963.933,10651], + [13639.6,257871.088,10647], + [13640.33,258010.355,10653], + [13801.11,258056.844,10655], + [13801.31,258080.055,10656], + [13801.58,258103.266,10657], + [13801.71,258126.477,10658], + [13801.85,258149.688,10659], + [13802.07,258172.9,10660], + [13802.21,258033.633,10654], + [13964.66,258312.222,10666], + [13964.8,258335.433,10667], + [13964.89,258289.011,10665], + [13965.01,258196.166,10661], + [13965.08,258219.377,10662], + [13965.15,258242.588,10663], + [13965.28,258265.8,10664], + [14133.5,258358.711,10668], + [14133.65,258405.133,10670], + [14133.72,258428.344,10671], + [14133.85,258451.555,10672], + [14133.93,258474.766,10673], + [14134,258497.977,10674], + [14134.07,258381.922,10669], + [14318.44,258544.455,10676], + [14318.69,258567.666,10677], + [14318.85,258590.877,10678], + [14318.93,258614.088,10679], + [14319,258637.3,10680], + [14319.19,258660.511,10681], + [14319.31,258521.244,10675], + [14491.59,258707,10683], + [14491.75,258730.211,10684], + [14491.84,258753.422,10685], + [14491.97,258776.633,10686], + [14492.04,258799.844,10687], + [14492.11,258846.266,10689], + [14492.25,258823.055,10688], + [14492.31,258683.788,10682], + [14677.09,258869.544,10690], + [14677.25,258892.755,10691], + [14677.33,258915.966,10692], + [14677.46,258939.177,10693], + [14677.53,258962.388,10694], + [14677.6,258985.6,10695], + [14820.59,259055.288,10698], + [14820.78,259078.5,10699], + [14820.94,259101.711,10700], + [14821.09,259124.922,10701], + [14821.16,259148.133,10702], + [14821.25,259008.866,10696], + [14821.33,259032.077,10697], + [14964.22,259287.466,10708], + [14964.37,259310.677,10709], + [14964.44,259171.411,10703], + [14964.56,259194.622,10704], + [14964.63,259217.833,10705], + [14964.7,259241.044,10706], + [14964.96,259264.255,10707], + [15100.84,259333.944,10710], + [15101.09,259357.155,10711], + [15101.2,259380.366,10712], + [15101.35,259403.577,10713], + [15101.45,259426.788,10714], + [15101.59,259450,10715], + [15101.71,259473.211,10716], + [15269.97,259519.7,10718], + [15270.12,259542.911,10719], + [15270.22,259566.122,10720], + [15270.29,259589.333,10721], + [15270.35,259612.544,10722], + [15270.49,259635.755,10723], + [15270.56,259496.488,10717], + [15425.1,259682.233,10725], + [15425.25,259705.444,10726], + [15425.33,259728.655,10727], + [15425.46,259751.866,10728], + [15425.52,259775.077,10729], + [15425.61,259798.288,10730], + [15425.69,259659.022,10724], + [15590.53,259844.777,10732], + [15590.66,259867.988,10733], + [15590.77,259891.2,10734], + [15590.88,259914.411,10735], + [15590.94,259937.622,10736], + [15591.06,259960.833,10737], + [15591.19,259821.566,10731], + [15755.92,260007.322,10739], + [15756.06,260030.533,10740], + [15756.16,260053.744,10741], + [15756.25,260076.955,10742], + [15756.34,260100.166,10743], + [15756.44,260123.377,10744], + [15756.56,259984.111,10738], + [15920.88,260146.644,10745], + [15921,260239.488,10749], + [15921.11,260262.7,10750], + [15921.22,260285.911,10751], + [15921.38,260216.277,10748], + [15921.54,260169.855,10746], + [15921.63,260193.066,10747], + [16087.75,260378.822,10755], + [16087.94,260425.244,10757], + [16088.03,260448.455,10758], + [16088.15,260309.188,10752], + [16088.22,260332.4,10753], + [16088.29,260355.611,10754], + [16088.37,260402.033,10756], + [16254.79,260494.933,10760], + [16254.97,260518.144,10761], + [16255.08,260541.355,10762], + [16255.18,260564.566,10763], + [16255.28,260587.777,10764], + [16255.41,260610.988,10765], + [16255.52,260471.722,10759], + [16421.4,260657.477,10767], + [16421.72,260680.688,10768], + [16421.94,260703.9,10769], + [16422.03,260727.111,10770], + [16422.11,260750.322,10771], + [16422.19,260773.533,10772], + [16422.37,260634.266,10766], + [16563.5,260820.011,10774], + [16563.69,260843.222,10775], + [16563.78,260866.433,10776], + [16563.89,260889.644,10777], + [16564.07,260912.855,10778], + [16564.16,260936.066,10779], + [16564.25,260796.8,10773], + [16725.87,260982.555,10781], + [16726.04,261005.766,10782], + [16726.13,261028.977,10783], + [16726.21,261052.188,10784], + [16726.35,261075.4,10785], + [16726.43,261098.611,10786], + [16726.53,260959.344,10780], + [16889.66,261145.1,10788], + [16889.81,261168.311,10789], + [16889.88,261191.522,10790], + [16889.96,261214.733,10791], + [16890.03,261237.944,10792], + [16890.1,261261.155,10793], + [16890.19,261121.888,10787], + [17051.54,261307.633,10795], + [17051.7,261330.844,10796], + [17051.78,261354.055,10797], + [17051.93,261377.266,10798], + [17052.01,261400.477,10799], + [17052.08,261423.688,10800], + [17052.25,261284.422,10794], + [17221.22,261493.388,10803], + [17221.41,261516.6,10804], + [17221.48,261539.811,10805], + [17221.62,261563.022,10806], + [17221.69,261586.233,10807], + [17221.76,261446.966,10801], + [17221.92,261470.177,10802], + [17379.66,261655.922,10810], + [17379.85,261679.133,10811], + [17379.97,261725.555,10813], + [17380.05,261748.766,10814], + [17380.19,261609.5,10808], + [17380.25,261632.711,10809], + [17380.33,261702.344,10812], + [17546.72,261795.255,10816], + [17546.99,261818.466,10817], + [17547.14,261841.677,10818], + [17547.27,261864.888,10819], + [17547.5,261888.1,10820], + [17547.6,261911.311,10821], + [17547.74,261772.044,10815], + [17711.03,261934.577,10822], + [17712.21,262027.422,10826], + [17712.38,262050.633,10827], + [17712.55,262073.844,10828], + [17712.66,261957.788,10823], + [17712.81,261981,10824], + [17712.91,262004.211,10825], + [17890.29,262213.177,10834], + [17890.45,262097.122,10829], + [17890.56,262120.333,10830], + [17890.63,262143.544,10831], + [17890.76,262166.755,10832], + [17890.84,262236.388,10835], + [17890.91,262189.966,10833], + [18045.28,262352.511,10840], + [18045.44,262375.722,10841], + [18045.55,262398.933,10842], + [18045.63,262282.877,10837], + [18045.75,262306.088,10838], + [18045.84,262259.666,10836], + [18045.97,262329.3,10839], + [18240.02,262515.044,10847], + [18240.16,262422.2,10843], + [18240.22,262445.411,10844], + [18240.35,262468.622,10845], + [18240.42,262538.255,10848], + [18240.5,262561.466,10849], + [18240.63,262491.833,10846], + [18352.12,262607.955,10851], + [18352.27,262631.166,10852], + [18352.35,262654.377,10853], + [18352.47,262677.588,10854], + [18352.55,262700.8,10855], + [18352.61,262724.011,10856], + [18352.69,262584.744,10850], + [18515.84,262793.7,10859], + [18516,262816.911,10860], + [18516.09,262840.122,10861], + [18516.22,262863.333,10862], + [18516.31,262886.544,10863], + [18516.38,262747.277,10857], + [18516.52,262770.488,10858], + [18676.4,263049.088,10870], + [18676.53,262909.822,10864], + [18676.63,262933.033,10865], + [18676.71,262956.244,10866], + [18676.85,262979.455,10867], + [18676.93,263002.666,10868], + [18677.04,263025.877,10869], + [18848.16,263165.2,10875], + [18848.31,263188.411,10876], + [18848.4,263211.622,10877], + [18848.52,263095.566,10872], + [18848.59,263118.777,10873], + [18848.66,263072.355,10871], + [18848.77,263141.988,10874], + [19026.36,263304.533,10881], + [19026.53,263234.9,10878], + [19026.62,263258.111,10879], + [19026.74,263327.744,10882], + [19026.82,263350.955,10883], + [19026.9,263281.322,10880], + [19027.04,263374.166,10884], + [19178.5,263467.077,10888], + [19178.71,263513.5,10890], + [19178.81,263536.711,10891], + [19179.01,263397.444,10885], + [19179.13,263420.655,10886], + [19179.25,263443.866,10887], + [19179.59,263490.288,10889], + [19355.32,263583.188,10893], + [19355.51,263606.4,10894], + [19355.6,263559.977,10892], + [19355.74,263629.611,10895], + [19355.82,263652.822,10896], + [19355.9,263676.033,10897], + [19355.97,263699.244,10898], + [19504.23,263722.522,10899], + [19505.66,263745.733,10900], + [19505.82,263768.944,10901], + [19505.95,263792.155,10902], + [19506.03,263815.366,10903], + [19506.17,263838.577,10904], + [19506.25,263861.788,10905], + [19672.07,263954.688,10909], + [19672.25,263977.9,10910], + [19672.4,264001.111,10911], + [19672.5,263885.055,10906], + [19672.6,263908.266,10907], + [19672.76,263931.477,10908], + [19672.84,264024.322,10912], + [19815.57,264070.811,10914], + [19815.74,264094.022,10915], + [19815.85,264117.233,10916], + [19815.94,264140.444,10917], + [19816.06,264163.655,10918], + [19816.16,264186.866,10919], + [19816.24,264047.6,10913], + [20050.35,264233.344,10921], + [20050.51,264256.555,10922], + [20050.59,264326.188,10925], + [20050.72,264279.766,10923], + [20050.8,264302.977,10924], + [20050.91,264210.133,10920], + [20051.05,264349.4,10926], + [20151.41,264465.522,10931], + [20151.57,264488.733,10932], + [20151.66,264511.944,10933], + [20151.78,264372.677,10927], + [20151.87,264395.888,10928], + [20151.95,264419.1,10929], + [20152.09,264442.311,10930], + [20307.34,264535.222,10934], + [20307.53,264558.433,10935], + [20307.68,264581.644,10936], + [20307.78,264604.855,10937], + [20307.94,264628.066,10938], + [20308.02,264651.277,10939], + [20308.1,264674.488,10940], + [20466.32,264720.966,10942], + [20466.5,264744.177,10943], + [20466.58,264813.811,10946], + [20466.72,264767.388,10944], + [20466.8,264790.6,10945], + [20466.89,264697.755,10941], + [20467.06,264837.022,10947], + [20641.56,264883.511,10949], + [20641.76,264906.722,10950], + [20641.88,264929.933,10951], + [20642.07,264953.144,10952], + [20642.19,264976.355,10953], + [20642.31,264999.566,10954], + [20642.42,264860.3,10948], + [20796,265138.888,10960], + [20796.17,265162.1,10961], + [20796.27,265115.677,10959], + [20796.44,265022.833,10955], + [20796.53,265046.044,10956], + [20796.63,265069.255,10957], + [20796.74,265092.466,10958], + [20980.12,265231.8,10964], + [20980.27,265208.588,10963], + [20980.37,265255.011,10965], + [20980.44,265278.222,10966], + [20980.6,265301.433,10967], + [20980.68,265324.644,10968], + [20980.73,265185.377,10962], + [21133.86,265347.911,10969], + [21134,265394.333,10971], + [21134.09,265417.544,10972], + [21134.25,265440.755,10973], + [21134.34,265463.966,10974], + [21134.44,265487.177,10975], + [21134.66,265371.122,10970], + [21301.55,265533.666,10977], + [21301.71,265556.877,10978], + [21301.82,265580.088,10979], + [21301.9,265603.3,10980], + [21302.02,265626.511,10981], + [21302.09,265649.722,10982], + [21302.22,265510.455,10976], + [21440.19,265696.211,10984], + [21440.37,265719.422,10985], + [21440.49,265742.633,10986], + [21440.57,265765.844,10987], + [21440.71,265789.055,10988], + [21440.78,265812.266,10989], + [21440.92,265673,10983], + [21602.06,265974.8,10996], + [21602.28,265951.588,10995], + [21602.4,265928.377,10994], + [21602.49,265881.955,10992], + [21602.69,265905.166,10993], + [21602.77,265858.744,10991], + [21602.87,265835.533,10990], + [21764.78,266067.711,11000], + [21764.96,266090.922,11001], + [21765.05,266114.133,11002], + [21765.21,266137.344,11003], + [21765.28,265998.077,10997], + [21765.37,266044.5,10999], + [21765.46,266021.288,10998], + [21927.49,266253.455,11008], + [21927.66,266230.244,11007], + [21927.77,266276.666,11009], + [21927.85,266299.877,11010], + [21927.96,266183.822,11005], + [21928.03,266207.033,11006], + [21928.1,266160.611,11004], + [22090.88,266416,11015], + [22091.03,266439.211,11016], + [22091.11,266462.422,11017], + [22091.24,266346.366,11012], + [22091.3,266369.577,11013], + [22091.38,266323.155,11011], + [22091.53,266392.788,11014], + [22321.65,266508.9,11019], + [22321.82,266532.111,11020], + [22321.94,266578.533,11022], + [22322.11,266601.744,11023], + [22322.19,266624.955,11024], + [22322.25,266485.688,11018], + [22322.4,266555.322,11021], + [22433.75,266741.077,11029], + [22433.99,266764.288,11030], + [22434.12,266787.5,11031], + [22437.23,266671.444,11026], + [22437.45,266694.655,11027], + [22437.55,266648.233,11025], + [22437.66,266717.866,11028], + [22615.91,266903.622,11036], + [22616.09,266810.777,11032], + [22616.18,266833.988,11033], + [22616.31,266857.2,11034], + [22616.39,266926.833,11037], + [22616.47,266950.044,11038], + [22616.62,266880.411,11035], + [22763.72,267019.733,11041], + [22763.91,266973.311,11039], + [22764.05,267042.944,11042], + [22764.13,267066.155,11043], + [22764.27,267089.366,11044], + [22764.34,267112.577,11045], + [22764.43,267135.788,11046], + [22764.53,266996.522,11040], + [22925.46,267159.077,11047], + [22925.62,267182.288,11048], + [22925.69,267205.5,11049], + [22925.87,267228.711,11050], + [22925.94,267251.922,11051], + [22926.02,267275.133,11052], + [22926.1,267298.344,11053], + [23088.63,267344.822,11055], + [23088.78,267368.033,11056], + [23088.87,267391.244,11057], + [23089.03,267414.455,11058], + [23089.11,267437.666,11059], + [23089.19,267460.877,11060], + [23089.25,267321.611,11054], + [23253.25,267623.422,11067], + [23253.4,267553.788,11064], + [23253.51,267577,11065], + [23253.59,267600.211,11066], + [23253.67,267530.577,11063], + [23253.82,267507.366,11062], + [23253.91,267484.155,11061], + [23421.5,267669.9,11069], + [23421.66,267693.111,11070], + [23421.74,267716.322,11071], + [23421.88,267739.533,11072], + [23422.03,267785.955,11074], + [23422.13,267762.744,11073], + [23422.25,267646.688,11068], + [23587.5,267809.233,11075], + [23587.66,267832.444,11076], + [23587.77,267855.655,11077], + [23587.97,267878.866,11078], + [23588.06,267902.077,11079], + [23588.16,267925.288,11080], + [23588.29,267948.5,11081], + [23754.12,267994.977,11083], + [23754.28,268018.188,11084], + [23754.35,268041.4,11085], + [23754.44,268064.611,11086], + [23754.59,268087.822,11087], + [23754.68,268111.033,11088], + [23754.88,267971.766,11082], + [23922.3,268157.522,11090], + [23922.44,268180.733,11091], + [23922.55,268203.944,11092], + [23922.63,268227.155,11093], + [23922.77,268250.366,11094], + [23922.84,268273.577,11095], + [23923,268134.311,11089], + [24063.78,268320.066,11097], + [24063.96,268343.277,11098], + [24064.09,268366.488,11099], + [24064.19,268389.7,11100], + [24064.34,268412.911,11101], + [24064.41,268436.122,11102], + [24064.5,268296.855,11096], + [24225.27,268529.022,11106], + [24225.45,268552.233,11107], + [24225.55,268575.444,11108], + [24225.68,268598.655,11109], + [24225.75,268459.388,11103], + [24225.84,268482.6,11104], + [24225.93,268505.811,11105], + [24390.13,268737.988,11115], + [24390.29,268621.933,11110], + [24390.37,268645.144,11111], + [24390.53,268668.355,11112], + [24390.6,268761.2,11116], + [24390.68,268691.566,11113], + [24390.75,268714.777,11114], + [24566.88,268807.677,11118], + [24567.05,268830.888,11119], + [24567.14,268854.1,11120], + [24567.27,268877.311,11121], + [24567.37,268900.522,11122], + [24567.44,268923.733,11123], + [24567.6,268784.466,11117], + [24738.47,268970.222,11125], + [24738.63,268993.433,11126], + [24738.71,269016.644,11127], + [24738.85,269039.855,11128], + [24738.92,269063.066,11129], + [24739,269086.277,11130], + [24739.08,268947.011,11124], + [24879.15,269202.388,11135], + [24879.31,269225.6,11136], + [24879.41,269109.544,11131], + [24879.55,269132.755,11132], + [24879.63,269155.966,11133], + [24879.7,269179.177,11134], + [24879.79,269248.811,11137], + [25054.02,269272.088,11138], + [25055.04,269295.3,11139], + [25055.16,269318.511,11140], + [25055.26,269341.722,11141], + [25055.33,269364.933,11142], + [25055.6,269388.144,11143], + [25055.67,269411.355,11144], + [25212.62,269573.9,11151], + [25212.79,269434.633,11145], + [25212.9,269457.844,11146], + [25213.06,269481.055,11147], + [25213.14,269504.266,11148], + [25213.23,269527.477,11149], + [25213.39,269550.688,11150], + [25380.99,269597.166,11152], + [25381.31,269620.377,11153], + [25381.4,269643.588,11154], + [25381.55,269666.8,11155], + [25381.62,269690.011,11156], + [25381.71,269713.222,11157], + [25381.8,269736.433,11158], + [25548.94,269875.766,11164], + [25549.1,269898.977,11165], + [25549.19,269852.555,11163], + [25549.37,269759.711,11159], + [25549.44,269782.922,11160], + [25549.62,269806.133,11161], + [25549.71,269829.344,11162], + [25713.68,269922.244,11166], + [25714.61,269991.877,11169], + [25714.74,269945.455,11167], + [25714.86,269968.666,11168], + [25714.94,270015.088,11170], + [25715.03,270038.3,11171], + [25715.19,270061.511,11172], + [25715.26,270084.722,11173], + [25888.14,270177.644,11177], + [25888.28,270200.855,11178], + [25888.37,270224.066,11179], + [25888.51,270108.011,11174], + [25888.61,270131.222,11175], + [25888.7,270154.433,11176], + [25888.79,270247.277,11180], + [26047.55,270409.811,11187], + [26047.72,270363.388,11185], + [26047.82,270386.6,11186], + [26047.99,270316.966,11183], + [26048.07,270293.755,11182], + [26048.17,270340.177,11184], + [26048.3,270270.544,11181], + [26214.11,270479.511,11190], + [26214.21,270525.933,11192], + [26214.34,270433.088,11188], + [26214.43,270456.3,11189], + [26214.55,270502.722,11191], + [26214.65,270549.144,11193], + [26214.76,270572.355,11194], + [26380.43,270618.833,11196], + [26380.67,270642.044,11197], + [26380.78,270665.255,11198], + [26380.91,270688.466,11199], + [26381.18,270711.677,11200], + [26381.27,270734.888,11201], + [26381.35,270595.622,11195], + [26524.57,270781.377,11203], + [26524.77,270804.588,11204], + [26524.87,270827.8,11205], + [26524.97,270851.011,11206], + [26525.16,270874.222,11207], + [26525.23,270897.433,11208], + [26525.32,270758.166,11202], + [26687.77,270943.911,11210], + [26687.95,270967.122,11211], + [26688.07,270990.333,11212], + [26688.15,271013.544,11213], + [26688.28,271036.755,11214], + [26688.35,271059.966,11215], + [26688.47,270920.7,11209], + [26850.96,271106.455,11217], + [26851.13,271129.666,11218], + [26851.22,271152.877,11219], + [26851.34,271176.088,11220], + [26851.42,271199.3,11221], + [26851.49,271222.511,11222], + [26851.66,271083.244,11216], + [27012.06,271269,11224], + [27012.22,271292.211,11225], + [27012.3,271315.422,11226], + [27012.43,271338.633,11227], + [27012.5,271361.844,11228], + [27012.58,271385.055,11229], + [27012.72,271245.788,11223], + [27174.88,271454.744,11232], + [27175.05,271477.955,11233], + [27175.15,271501.166,11234], + [27175.28,271524.377,11235], + [27175.36,271408.322,11230], + [27175.44,271431.533,11231], + [27175.52,271547.588,11236], + [27340.27,271594.077,11238], + [27340.44,271617.288,11239], + [27340.62,271640.5,11240], + [27340.73,271663.711,11241], + [27340.9,271686.922,11242], + [27340.97,271710.133,11243], + [27341.07,271570.866,11237], + [27513.14,271756.611,11245], + [27513.37,271779.822,11246], + [27513.49,271803.033,11247], + [27513.67,271826.244,11248], + [27513.75,271849.455,11249], + [27513.84,271872.666,11250], + [27513.94,271733.4,11244], + [27701.13,271942.366,11253], + [27701.32,271965.577,11254], + [27701.43,271988.788,11255], + [27701.53,272012,11256], + [27701.75,272035.211,11257], + [27701.86,271895.944,11251], + [27701.99,271919.155,11252], + [27849.3,272128.111,11261], + [27849.49,272151.322,11262], + [27849.59,272174.533,11263], + [27849.8,272058.477,11258], + [27849.91,272081.688,11259], + [27849.98,272197.744,11264], + [27850.06,272104.9,11260], + [28004.41,272313.866,11269], + [28004.6,272221.022,11265], + [28004.69,272244.233,11266], + [28004.83,272267.444,11267], + [28004.91,272337.077,11270], + [28004.97,272360.288,11271], + [28005.05,272290.655,11268], + [28150.41,272383.566,11272], + [28150.57,272406.777,11273], + [28150.66,272429.988,11274], + [28150.8,272453.2,11275], + [28150.88,272476.411,11276], + [28150.96,272522.833,11278], + [28151.05,272499.622,11277], + [28339.67,272592.522,11281], + [28339.83,272615.733,11282], + [28339.91,272638.944,11283], + [28340.04,272662.155,11284], + [28340.12,272685.366,11285], + [28340.19,272546.1,11279], + [28340.28,272708.577,11286], + [28340.44,272569.311,11280], + [28502.85,272731.866,11287], + [28504.1,272755.077,11288], + [28504.23,272778.288,11289], + [28504.31,272801.5,11290], + [28504.44,272824.711,11291], + [28504.53,272847.922,11292], + [28504.63,272871.133,11293], + [28671.16,272940.822,11296], + [28671.43,272964.033,11297], + [28671.54,272987.244,11298], + [28671.69,273010.455,11299], + [28671.76,273033.666,11300], + [28671.87,272894.4,11294], + [28671.97,272917.611,11295], + [28840.35,273080.155,11302], + [28840.53,273103.366,11303], + [28840.63,273126.577,11304], + [28840.77,273149.788,11305], + [28840.85,273173,11306], + [28840.94,273196.211,11307], + [28841.01,273056.944,11301], + [29004.41,273312.322,11312], + [29004.55,273335.533,11313], + [29004.62,273358.744,11314], + [29004.74,273242.688,11309], + [29004.82,273265.9,11310], + [29004.91,273219.477,11308], + [29005.05,273289.111,11311], + [29165.59,273474.866,11319], + [29165.75,273382.022,11315], + [29165.85,273405.233,11316], + [29166,273428.444,11317], + [29166.08,273498.077,11320], + [29166.16,273521.288,11321], + [29166.25,273451.655,11318], + [29311.41,273660.611,11327], + [29311.63,273683.822,11328], + [29311.76,273544.555,11322], + [29311.88,273614.188,11325], + [29312.07,273637.4,11326], + [29312.14,273567.766,11323], + [29312.22,273590.977,11324], + [29474.61,273776.733,11332], + [29474.77,273846.366,11335], + [29474.9,273707.1,11329], + [29475.06,273730.311,11330], + [29475.13,273799.944,11333], + [29475.21,273823.155,11334], + [29475.3,273753.522,11331], + [29636.09,273892.855,11337], + [29636.25,273916.066,11338], + [29636.34,273939.277,11339], + [29636.47,273962.488,11340], + [29636.55,273985.7,11341], + [29636.63,274008.911,11342], + [29636.72,273869.644,11336], + [29799.95,274055.388,11344], + [29800.1,274078.6,11345], + [29800.19,274101.811,11346], + [29800.31,274125.022,11347], + [29800.38,274148.233,11348], + [29800.46,274171.444,11349], + [29800.6,274032.177,11343], + [29964.7,274217.933,11351], + [29964.85,274241.144,11352], + [29964.94,274264.355,11353], + [29965.06,274287.566,11354], + [29965.15,274310.777,11355], + [29965.22,274333.988,11356], + [29965.36,274194.722,11350], + [30130.13,274380.466,11358], + [30130.31,274403.677,11359], + [30130.39,274426.888,11360], + [30130.47,274450.1,11361], + [30130.65,274473.311,11362], + [30130.73,274496.522,11363], + [30130.87,274357.255,11357], + [30295.18,274612.644,11368], + [30295.38,274659.066,11370], + [30295.5,274635.855,11369], + [30295.69,274543.011,11365], + [30295.84,274566.222,11366], + [30295.97,274519.8,11364], + [30296.09,274589.433,11367], + [30462.77,274682.333,11371], + [30463.07,274705.544,11372], + [30463.24,274775.177,11375], + [30463.35,274728.755,11373], + [30463.71,274798.388,11376], + [30463.84,274821.6,11377], + [30463.98,274751.966,11374], + [30679.69,274868.088,11379], + [30679.93,274891.3,11380], + [30680.08,274914.511,11381], + [30680.27,274937.722,11382], + [30680.4,274960.933,11383], + [30680.5,274984.144,11384], + [30680.62,274844.877,11378], + [30774.94,275030.633,11386], + [30775.11,275053.844,11387], + [30775.21,275077.055,11388], + [30775.45,275100.266,11389], + [30775.63,275123.477,11390], + [30775.75,275146.688,11391], + [30775.87,275007.422,11385], + [30950.08,275286.011,11397], + [30950.25,275309.222,11398], + [30950.35,275169.955,11392], + [30950.5,275193.166,11393], + [30950.59,275216.377,11394], + [30950.69,275239.588,11395], + [30950.84,275262.8,11396], + [31111.47,275332.5,11399], + [31111.65,275355.711,11400], + [31111.74,275378.922,11401], + [31111.83,275402.133,11402], + [31112,275425.344,11403], + [31112.07,275448.555,11404], + [31112.16,275471.766,11405], + [31261.41,275518.244,11407], + [31261.55,275541.455,11408], + [31261.63,275564.666,11409], + [31261.75,275587.877,11410], + [31261.84,275611.088,11411], + [31261.92,275634.3,11412], + [31262.01,275495.033,11406], + [31424.69,275680.788,11414], + [31424.87,275704,11415], + [31424.95,275727.211,11416], + [31425.07,275750.422,11417], + [31425.15,275773.633,11418], + [31425.24,275796.844,11419], + [31425.31,275657.577,11413], + [31589.23,275843.322,11421], + [31589.38,275866.533,11422], + [31589.46,275936.166,11425], + [31589.6,275889.744,11423], + [31589.69,275912.955,11424], + [31589.77,275820.111,11420], + [31589.93,275959.377,11426], + [31756.16,276005.866,11428], + [31756.33,276029.077,11429], + [31756.42,276052.288,11430], + [31756.59,276075.5,11431], + [31756.66,276098.711,11432], + [31756.75,276121.922,11433], + [31756.85,275982.655,11427], + [31921.65,276168.411,11435], + [31921.8,276191.622,11436], + [31921.88,276214.833,11437], + [31922,276238.044,11438], + [31922.07,276261.255,11439], + [31922.16,276284.466,11440], + [31922.29,276145.2,11434], + [32092.05,276330.944,11442], + [32092.16,276354.155,11443], + [32092.24,276377.366,11444], + [32092.36,276400.577,11445], + [32092.43,276423.788,11446], + [32092.49,276447,11447], + [32092.61,276307.733,11441], + [32265.74,276493.488,11449], + [32265.89,276516.7,11450], + [32265.97,276539.911,11451], + [32266.09,276563.122,11452], + [32266.16,276586.333,11453], + [32266.22,276609.544,11454], + [32266.35,276470.277,11448], + [32399.35,276725.655,11459], + [32399.48,276748.866,11460], + [32399.55,276772.077,11461], + [32399.66,276656.022,11456], + [32399.74,276679.233,11457], + [32399.84,276632.811,11455], + [32400,276702.444,11458], + [32587.97,276795.355,11462], + [32589.35,276888.2,11466], + [32589.47,276911.411,11467], + [32589.57,276818.566,11463], + [32589.69,276841.777,11464], + [32589.77,276864.988,11465], + [32589.91,276934.622,11468], + [32589.99,276957.833,11469], + [32753,277004.322,11471], + [32753.18,277027.533,11472], + [32753.28,277050.744,11473], + [32753.43,277073.955,11474], + [32753.52,277097.166,11475], + [32753.6,277120.377,11476], + [32753.7,276981.111,11470], + [32920.49,277190.077,11479], + [32920.74,277213.288,11480], + [32920.86,277236.5,11481], + [32921.03,277259.711,11482], + [32921.12,277282.922,11483], + [32921.22,277143.655,11477], + [32921.34,277166.866,11478], + [33088.34,277306.188,11484], + [33088.57,277329.4,11485], + [33088.69,277445.455,11490], + [33088.85,277352.611,11486], + [33088.98,277375.822,11487], + [33089.06,277399.033,11488], + [33089.17,277422.244,11489], + [33254.69,277491.944,11492], + [33254.94,277515.155,11493], + [33255.03,277538.366,11494], + [33255.21,277561.577,11495], + [33255.31,277584.788,11496], + [33255.41,277608,11497], + [33255.52,277468.733,11491], + [33397.55,277654.477,11499], + [33397.69,277677.688,11500], + [33397.79,277700.9,11501], + [33397.91,277724.111,11502], + [33397.99,277747.322,11503], + [33398.06,277770.533,11504], + [33398.13,277631.266,11498], + [33559.9,277933.077,11511], + [33560.06,277793.811,11505], + [33560.16,277817.022,11506], + [33560.24,277840.233,11507], + [33560.37,277863.444,11508], + [33560.43,277886.655,11509], + [33560.51,277909.866,11510], + [33723.06,277979.566,11513], + [33723.25,278002.777,11514], + [33723.35,278025.988,11515], + [33723.46,278049.2,11516], + [33723.66,278072.411,11517], + [33723.75,278095.622,11518], + [33723.88,277956.355,11512], + [33885.02,278142.1,11520], + [33885.16,278165.311,11521], + [33885.23,278188.522,11522], + [33885.31,278211.733,11523], + [33885.43,278234.944,11524], + [33885.5,278258.155,11525], + [33885.57,278118.888,11519], + [34050.42,278374.277,11530], + [34050.56,278397.488,11531], + [34050.65,278420.7,11532], + [34050.8,278304.644,11527], + [34050.88,278327.855,11528], + [34050.96,278281.433,11526], + [34051.04,278351.066,11529], + [34213.18,278536.811,11537], + [34213.43,278443.966,11533], + [34213.57,278467.177,11534], + [34213.72,278490.388,11535], + [34213.87,278560.022,11538], + [34214.02,278583.233,11539], + [34214.13,278513.6,11536], + [34350.88,278606.511,11540], + [34351.11,278629.722,11541], + [34351.24,278652.933,11542], + [34351.36,278676.144,11543], + [34351.48,278699.355,11544], + [34351.61,278722.566,11545], + [34520.88,278792.255,11548], + [34521.05,278815.466,11549], + [34521.13,278838.677,11550], + [34521.26,278861.888,11551], + [34521.34,278885.1,11552], + [34521.41,278745.833,11546], + [34521.53,278769.044,11547], + [34676.78,278931.577,11554], + [34676.94,278954.788,11555], + [34677.02,278978,11556], + [34677.13,279001.211,11557], + [34677.21,279024.422,11558], + [34677.27,279047.633,11559], + [34677.34,278908.366,11553], + [34845.22,279094.122,11561], + [34845.37,279117.333,11562], + [34845.45,279140.544,11563], + [34845.57,279163.755,11564], + [34845.65,279186.966,11565], + [34845.72,279210.177,11566], + [34845.8,279070.911,11560] + ] +} diff --git a/test/audio-quality/traces/relay-mic.json b/test/audio-quality/traces/relay-mic.json new file mode 100644 index 0000000000..3bb4d93735 --- /dev/null +++ b/test/audio-quality/traces/relay-mic.json @@ -0,0 +1,1763 @@ +{ + "version": 1, + "source": "https://cdn.moq.dev/anon, in headless Chromium capturing its fake audio device, recorded 2026-09-29", + "description": "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between.", + "transport": "webtransport", + "rtt": null, + "config": { + "codec": "opus", + "container": { + "kind": "legacy" + }, + "sampleRate": 48000, + "numberOfChannels": 2, + "bitrate": 64000 + }, + "arrivals": [ + [0,2324.745,0], + [16.53,2344.745,1], + [38.87,2364.745,2], + [58.56,2384.745,3], + [80.79,2404.745,4], + [90.63,2424.745,5], + [123.14,2444.745,6], + [133.15,2464.745,7], + [154.04,2484.745,8], + [175.89,2504.745,9], + [196.55,2524.745,10], + [217.73,2544.745,11], + [239.26,2564.745,12], + [249.41,2584.744,13], + [284.74,2604.745,14], + [293.14,2624.745,15], + [316.97,2644.745,16], + [341.57,2664.745,17], + [369.77,2684.745,18], + [383.67,2704.745,19], + [399.94,2724.745,20], + [412.31,2744.745,21], + [448.56,2764.745,22], + [462.16,2784.745,23], + [481.56,2804.745,24], + [500.93,2824.745,25], + [521.09,2844.744,26], + [549.96,2864.745,27], + [563.49,2884.745,28], + [571.93,2904.745,29], + [604.02,2924.745,30], + [614.12,2944.745,31], + [636.18,2964.745,32], + [655.33,2984.745,33], + [680.99,3004.745,34], + [699.9,3024.745,35], + [743.93,3044.745,36], + [744.29,3064.745,37], + [766.77,3084.745,38], + [773.76,3104.745,39], + [798.74,3124.745,40], + [821.71,3144.745,41], + [841.92,3164.745,42], + [862.21,3184.745,43], + [882.59,3204.745,44], + [900.28,3224.745,45], + [926.38,3244.745,46], + [937.7,3264.745,47], + [953.6,3284.745,48], + [974.8,3304.745,49], + [996.59,3324.745,50], + [1017.6,3344.745,51], + [1038.36,3364.744,52], + [1048.84,3384.745,53], + [1081.81,3404.745,54], + [1092.23,3424.745,55], + [1113.34,3444.745,56], + [1134.38,3464.745,57], + [1155.59,3484.745,58], + [1178.33,3504.745,59], + [1198.64,3524.745,60], + [1208.71,3544.745,61], + [1241.21,3564.745,62], + [1251.7,3584.745,63], + [1273.61,3604.745,64], + [1295.2,3624.745,65], + [1317.27,3644.745,66], + [1337.5,3664.745,67], + [1358.88,3684.745,68], + [1371.88,3704.745,69], + [1405.88,3724.745,70], + [1413.55,3744.745,71], + [1438.38,3764.745,72], + [1459.06,3784.745,73], + [1479,3804.745,74], + [1500.5,3824.745,75], + [1525.47,3844.745,76], + [1537.33,3864.745,77], + [1563.29,3884.745,78], + [1574.71,3904.745,79], + [1593.71,3924.745,80], + [1615.52,3944.745,81], + [1636.41,3964.745,82], + [1658.21,3984.745,83], + [1679.43,4004.745,84], + [1689.69,4024.745,85], + [1724.05,4044.745,86], + [1733.2,4064.745,87], + [1754.76,4084.745,88], + [1775.61,4104.745,89], + [1798.86,4124.745,90], + [1818.39,4144.745,91], + [1845.12,4164.745,92], + [1857.11,4184.745,93], + [1884.52,4204.745,94], + [1892.91,4224.745,95], + [1921.56,4244.745,96], + [1940.71,4264.745,97], + [1957.94,4284.745,98], + [1980.16,4304.745,99], + [2000.9,4324.745,100], + [2012,4344.745,101], + [2047.71,4364.745,102], + [2053.75,4384.744,103], + [2075.34,4404.744,104], + [2097.69,4424.745,105], + [2119.7,4444.745,106], + [2139.53,4464.745,107], + [2159.56,4484.745,108], + [2173.4,4504.745,109], + [2206.64,4524.745,110], + [2213.55,4544.745,111], + [2237.86,4564.745,112], + [2258.59,4584.745,113], + [2282.83,4604.745,114], + [2297.83,4624.745,115], + [2319.55,4644.745,116], + [2329.47,4664.745,117], + [2361.44,4684.745,118], + [2376.64,4704.745,119], + [2394.76,4724.745,120], + [2420.92,4744.745,121], + [2436.81,4764.745,122], + [2460.55,4784.745,123], + [2479.14,4804.745,124], + [2489.11,4824.745,125], + [2521.52,4844.745,126], + [2531.84,4864.745,127], + [2553.61,4884.745,128], + [2574.44,4904.745,129], + [2596.02,4924.745,130], + [2617.52,4944.745,131], + [2638.58,4964.745,132], + [2649.87,4984.745,133], + [2681.42,5004.745,134], + [2693.43,5024.745,135], + [2713.46,5044.745,136], + [2734.76,5064.745,137], + [2756.99,5084.745,138], + [2778.25,5104.745,139], + [2806,5124.745,140], + [2817.2,5144.745,141], + [2845.81,5164.745,142], + [2854.02,5184.745,143], + [2879.26,5204.745,144], + [2897.34,5224.745,145], + [2923.3,5244.745,146], + [2940.44,5264.745,147], + [2969.34,5284.745,148], + [2976.18,5304.745,149], + [3010.24,5324.745,150], + [3013.92,5344.745,151], + [3035.22,5364.745,152], + [3056.81,5384.745,153], + [3076.8,5404.745,154], + [3098.04,5424.745,155], + [3119.11,5444.745,156], + [3129.03,5464.745,157], + [3161.53,5484.745,158], + [3172.28,5504.745,159], + [3193.96,5524.745,160], + [3214.4,5544.745,161], + [3235.97,5564.745,162], + [3256.9,5584.745,163], + [3278.24,5604.745,164], + [3288.81,5624.745,165], + [3320.78,5644.745,166], + [3332.48,5664.745,167], + [3353.17,5684.745,168], + [3374.69,5704.745,169], + [3396.59,5724.745,170], + [3419.17,5744.745,171], + [3438.88,5764.745,172], + [3448.31,5784.745,173], + [3485.27,5804.745,174], + [3491.39,5824.745,175], + [3513.07,5844.745,176], + [3534.63,5864.745,177], + [3555.46,5884.745,178], + [3576.88,5904.745,179], + [3600.12,5924.745,180], + [3609.53,5944.745,181], + [3641.17,5964.745,182], + [3651.9,5984.745,183], + [3673.49,6004.745,184], + [3695.59,6024.745,185], + [3716.17,6044.745,186], + [3737.64,6064.745,187], + [3759.28,6084.745,188], + [3769.84,6104.745,189], + [3803.56,6124.745,190], + [3812.66,6144.745,191], + [3833.34,6164.745,192], + [3855.37,6184.745,193], + [3876.13,6204.745,194], + [3900.31,6224.745,195], + [3920.87,6244.745,196], + [3931.56,6264.745,197], + [3969,6284.745,198], + [3976.17,6304.745,199], + [3995.25,6324.745,200], + [4016.06,6344.745,201], + [4039.69,6364.745,202], + [4061.94,6384.745,203], + [4081.93,6404.745,204], + [4095.02,6424.744,205], + [4129.21,6444.744,206], + [4141.96,6464.744,207], + [4159.78,6484.744,208], + [4181.74,6504.744,209], + [4205.51,6524.745,210], + [4222.18,6544.745,211], + [4242.58,6564.745,212], + [4253.75,6584.745,213], + [4283.58,6604.745,214], + [4296.46,6624.745,215], + [4316.04,6644.745,216], + [4336.97,6664.745,217], + [4357.46,6684.745,218], + [4378.55,6704.745,219], + [4400.38,6724.745,220], + [4410.74,6744.745,221], + [4442.89,6764.745,222], + [4453.32,6784.745,223], + [4474.75,6804.745,224], + [4496.09,6824.745,225], + [4516.89,6844.745,226], + [4538.28,6864.745,227], + [4561.66,6884.745,228], + [4571.86,6904.745,229], + [4606.17,6924.745,230], + [4618.99,6944.745,231], + [4643.04,6964.745,232], + [4660.18,6984.745,233], + [4678.34,7004.745,234], + [4702.41,7024.745,235], + [4724.15,7044.745,236], + [4735.25,7064.745,237], + [4767.56,7084.745,238], + [4775.88,7104.745,239], + [4797.39,7124.745,240], + [4817.46,7144.745,241], + [4838.72,7164.745,242], + [4860.78,7184.745,243], + [4884.25,7204.745,244], + [4893.92,7224.745,245], + [4927.98,7244.745,246], + [4938.37,7264.745,247], + [4959.11,7284.745,248], + [4980.63,7304.745,249], + [5006.03,7324.745,250], + [5021.33,7344.745,251], + [5046.1,7364.745,252], + [5052.44,7384.745,253], + [5086.43,7404.745,254], + [5095.18,7424.745,255], + [5122.14,7444.745,256], + [5137.73,7464.745,257], + [5159.36,7484.745,258], + [5179.19,7504.745,259], + [5202.47,7524.745,260], + [5210.44,7544.745,261], + [5245.55,7564.745,262], + [5254.5,7584.745,263], + [5277.09,7604.745,264], + [5297.06,7624.745,265], + [5318.03,7644.745,266], + [5338.72,7664.745,267], + [5360.22,7684.745,268], + [5370.72,7704.745,269], + [5402.89,7724.745,270], + [5413.28,7744.745,271], + [5434.61,7764.745,272], + [5455.65,7784.745,273], + [5476.71,7804.745,274], + [5498.66,7824.745,275], + [5519.25,7844.745,276], + [5530.03,7864.745,277], + [5562.89,7884.745,278], + [5572.77,7904.745,279], + [5594.59,7924.745,280], + [5615.39,7944.745,281], + [5637.25,7964.745,282], + [5658.91,7984.745,283], + [5680.14,8004.745,284], + [5693.58,8024.745,285], + [5728.24,8044.745,286], + [5738.1,8064.745,287], + [5754.95,8084.745,288], + [5775.96,8104.745,289], + [5796.54,8124.745,290], + [5828.16,8144.745,291], + [5852.67,8164.745,292], + [5858.34,8184.745,293], + [5893.3,8204.745,294], + [5905.38,8224.745,295], + [5923.08,8244.745,296], + [5941.58,8264.745,297], + [5970.14,8284.745,298], + [6004.44,8304.745,299], + [6015.33,8324.745,300], + [6023.34,8344.745,301], + [6051.55,8364.745,302], + [6058.03,8384.745,303], + [6086.87,8404.745,304], + [6113.36,8424.745,305], + [6135.62,8444.745,306], + [6152.66,8464.745,307], + [6170.19,8484.745,308], + [6183.62,8504.745,309], + [6218.41,8524.745,310], + [6238.54,8544.745,311], + [6277.71,8564.745,312], + [6289.04,8584.745,313], + [6289.32,8604.745,314], + [6302.76,8624.745,315], + [6354.31,8644.745,316], + [6376.12,8664.745,317], + [6385.07,8684.745,318], + [6385.53,8704.745,319], + [6396.94,8724.745,320], + [6427.9,8744.745,321], + [6446.93,8764.745,322], + [6459.51,8784.745,323], + [6483.94,8804.745,324], + [6497.64,8824.745,325], + [6534.07,8844.745,326], + [6536.19,8864.745,327], + [6556.34,8884.745,328], + [6582.03,8904.745,329], + [6600.72,8924.745,330], + [6617.41,8944.745,331], + [6646.89,8964.745,332], + [6660.89,8984.745,333], + [6690.03,9004.745,334], + [6696.5,9024.745,335], + [6727.05,9044.745,336], + [6740.59,9064.745,337], + [6770.2,9084.745,338], + [6782.53,9104.745,339], + [6811.18,9124.745,340], + [6812.75,9144.745,341], + [6875.05,9164.745,342], + [6915.56,9184.745,343], + [6955.68,9204.745,344], + [6956.06,9224.745,345], + [6960.73,9244.745,346], + [6961.72,9264.745,347], + [6966.06,9284.745,348], + [6977.21,9304.745,349], + [7014.69,9324.745,350], + [7020.27,9344.745,351], + [7045.12,9364.745,352], + [7080.42,9384.745,353], + [7088.71,9404.745,354], + [7103.48,9424.745,355], + [7157.58,9444.745,356], + [7157.97,9464.745,357], + [7179.94,9484.745,358], + [7195.72,9504.745,359], + [7204.87,9524.745,360], + [7230.26,9544.745,361], + [7249.69,9564.745,362], + [7276.37,9584.745,363], + [7290.15,9604.745,364], + [7321.94,9624.745,365], + [7337.89,9644.745,366], + [7350.89,9664.745,367], + [7376.19,9684.745,368], + [7389.08,9704.745,369], + [7438.18,9724.745,370], + [7438.5,9744.745,371], + [7459.15,9764.745,372], + [7470.24,9784.745,373], + [7501.81,9804.745,374], + [7510.25,9824.745,375], + [7560.51,9844.745,376], + [7573.98,9864.745,377], + [7580.71,9884.745,378], + [7609.84,9904.745,379], + [7613.38,9924.745,380], + [7622.24,9944.745,381], + [7652.93,9964.745,382], + [7663.46,9984.745,383], + [7685.13,10004.745,384], + [7706.59,10024.745,385], + [7734.78,10044.745,386], + [7752.29,10064.745,387], + [7771.87,10084.745,388], + [7780.34,10104.745,389], + [7815.23,10124.745,390], + [7822.97,10144.745,391], + [7844.47,10164.745,392], + [7867.55,10184.745,393], + [7887.53,10204.745,394], + [7909.25,10224.745,395], + [7940.42,10244.745,396], + [7946.36,10264.745,397], + [7972.52,10284.745,398], + [7994.24,10304.745,399], + [8009.22,10324.745,400], + [8034.42,10344.745,401], + [8047.44,10364.745,402], + [8069.04,10384.745,403], + [8097.35,10404.745,404], + [8106.03,10424.745,405], + [8139.49,10444.745,406], + [8145.06,10464.745,407], + [8177.67,10484.745,408], + [8188.15,10504.745,409], + [8209.38,10524.744,410], + [8229.46,10544.744,411], + [8250.23,10564.744,412], + [8260.28,10584.744,413], + [8292.84,10604.744,414], + [8303.52,10624.744,415], + [8325.19,10644.744,416], + [8347.74,10664.744,417], + [8375.09,10684.744,418], + [8390.02,10704.744,419], + [8409.69,10724.745,420], + [8420.53,10744.745,421], + [8452.44,10764.745,422], + [8464.09,10784.745,423], + [8484.45,10804.745,424], + [8505.84,10824.745,425], + [8527.18,10844.745,426], + [8548.19,10864.745,427], + [8569.14,10884.745,428], + [8579.77,10904.745,429], + [8613.44,10924.745,430], + [8623.16,10944.745,431], + [8644.72,10964.745,432], + [8666.02,10984.745,433], + [8687.23,11004.745,434], + [8708.54,11024.745,435], + [8730.54,11044.745,436], + [8740.94,11064.745,437], + [8772.76,11084.745,438], + [8783.44,11104.745,439], + [8805.15,11124.745,440], + [8827.06,11144.745,441], + [8848.43,11164.745,442], + [8875.22,11184.745,443], + [8894.25,11204.745,444], + [8903.74,11224.745,445], + [8932.69,11244.745,446], + [8951.23,11264.745,447], + [8967.19,11284.745,448], + [8989.07,11304.745,449], + [9014.3,11324.745,450], + [9028.79,11344.745,451], + [9049.97,11364.745,452], + [9060.53,11384.745,453], + [9092.59,11404.745,454], + [9104.07,11424.745,455], + [9124.4,11444.745,456], + [9145.61,11464.745,457], + [9167.22,11484.745,458], + [9193.52,11504.745,459], + [9210.05,11524.745,460], + [9220.27,11544.745,461], + [9252.12,11564.745,462], + [9264.11,11584.745,463], + [9284.44,11604.745,464], + [9305.84,11624.745,465], + [9326.97,11644.745,466], + [9347.84,11664.745,467], + [9369.13,11684.745,468], + [9379.46,11704.745,469], + [9411.81,11724.745,470], + [9423.47,11744.745,471], + [9443.53,11764.745,472], + [9464.91,11784.745,473], + [9487.03,11804.745,474], + [9508.34,11824.745,475], + [9529.37,11844.745,476], + [9540.44,11864.745,477], + [9572.81,11884.745,478], + [9583.1,11904.745,479], + [9611.71,11924.745,480], + [9630.78,11944.745,481], + [9651.86,11964.745,482], + [9670.19,11984.745,483], + [9690.71,12004.745,484], + [9701.97,12024.745,485], + [9732.03,12044.745,486], + [9742.03,12064.745,487], + [9769.29,12084.745,488], + [9786.52,12104.745,489], + [9807.66,12124.745,490], + [9831.64,12144.745,491], + [9851.36,12164.745,492], + [9871.03,12184.745,493], + [9897.81,12204.745,494], + [9908.14,12224.745,495], + [9930.15,12244.745,496], + [9949.59,12264.745,497], + [9979,12284.745,498], + [9998.72,12304.745,499], + [10012.69,12324.745,500], + [10027.22,12344.745,501], + [10056.15,12364.745,502], + [10067.18,12384.745,503], + [10087.2,12404.745,504], + [10116.02,12424.745,505], + [10131.27,12444.745,506], + [10153.25,12464.745,507], + [10180.82,12484.745,508], + [10186.06,12504.745,509], + [10218.92,12524.745,510], + [10232.14,12544.745,511], + [10248.27,12564.745,512], + [10272.27,12584.745,513], + [10294.34,12604.745,514], + [10308.91,12624.745,515], + [10335.43,12644.745,516], + [10351.22,12664.745,517], + [10397.18,12684.745,518], + [10397.61,12704.745,519], + [10423.97,12724.745,520], + [10443.56,12744.745,521], + [10454.58,12764.745,522], + [10479.71,12784.745,523], + [10497.21,12804.745,524], + [10506.87,12824.745,525], + [10548.34,12844.745,526], + [10550.89,12864.745,527], + [10572.92,12884.745,528], + [10612.44,12904.745,529], + [10634.27,12924.745,530], + [10641.4,12944.745,531], + [10733.73,12964.745,532], + [10735.82,12984.745,533], + [10755.88,13004.745,534], + [10756.77,13024.745,535], + [10761.41,13044.745,536], + [10777.55,13064.745,537], + [10780.15,13084.745,538], + [10789.19,13104.745,539], + [10815,13124.745,540], + [10829.51,13144.745,541], + [10853.84,13164.745,542], + [10864.53,13184.745,543], + [10884.49,13204.745,544], + [10906.02,13224.745,545], + [10927.5,13244.745,546], + [10948.28,13264.745,547], + [10970.81,13284.745,548], + [10980.9,13304.745,549], + [11015.89,13324.745,550], + [11025.84,13344.745,551], + [11044.63,13364.745,552], + [11066.77,13384.745,553], + [11102.83,13404.745,554], + [11129.58,13424.745,555], + [11140.81,13444.745,556], + [11148.97,13464.745,557], + [11183.4,13484.745,558], + [11204.62,13504.745,559], + [11234.13,13524.745,560], + [11238.59,13544.745,561], + [11261.01,13564.745,562], + [11282.53,13584.745,563], + [11296.55,13604.745,564], + [11305.35,13624.745,565], + [11342.18,13644.745,566], + [11350.14,13664.745,567], + [11369.53,13684.745,568], + [11388.52,13704.745,569], + [11407.11,13724.745,570], + [11428.29,13744.745,571], + [11450.28,13764.745,572], + [11459.62,13784.745,573], + [11491.94,13804.745,574], + [11502.94,13824.745,575], + [11524.53,13844.745,576], + [11545.75,13864.745,577], + [11568.57,13884.745,578], + [11602.66,13904.745,579], + [11621.11,13924.745,580], + [11629.3,13944.745,581], + [11666.22,13964.745,582], + [11677.65,13984.745,583], + [11694.01,14004.745,584], + [11718.39,14024.745,585], + [11734.83,14044.745,586], + [11755.08,14064.745,587], + [11802.39,14084.745,588], + [11808.72,14104.745,589], + [11845.81,14124.745,590], + [11847.44,14144.745,591], + [11847.73,14164.745,592], + [11868.5,14184.745,593], + [11903.06,14204.745,594], + [11918.23,14224.745,595], + [11942.19,14244.745,596], + [11954.33,14264.745,597], + [11982.12,14284.745,598], + [11993.61,14304.745,599], + [12010.03,14324.745,600], + [12032.87,14344.745,601], + [12052.44,14364.745,602], + [12087.72,14384.745,603], + [12098.55,14404.745,604], + [12108.81,14424.745,605], + [12138.69,14444.745,606], + [12154.22,14464.745,607], + [12175.03,14484.745,608], + [12194.24,14504.745,609], + [12282.15,14524.745,610], + [12332.19,14544.745,611], + [12360.18,14564.745,612], + [12451.8,14584.745,613], + [12531.83,14604.745,614], + [12683.12,14624.745,615], + [12683.84,14644.745,616], + [12694.85,14664.745,617], + [12695.43,14684.745,618], + [12716.44,14724.745,620], + [12719.15,14704.745,619], + [12798.54,14744.745,621], + [12802.88,14764.745,622], + [12815.83,14784.745,623], + [12816.4,14804.745,624], + [12816.99,14824.745,625], + [12817.26,14844.745,626], + [12826.02,14864.745,627], + [12826.47,14884.745,628], + [12826.78,14904.745,629], + [12827.66,14924.745,630], + [12827.97,14944.745,631], + [12829.15,14964.745,632], + [12829.38,14984.745,633], + [12829.62,15004.745,634], + [12829.89,15024.745,635], + [12830.14,15044.745,636], + [12830.42,15064.745,637], + [12854.89,15084.745,638], + [12861.39,15104.745,639], + [12885.99,15124.745,640], + [12905.95,15144.745,641], + [12927.51,15164.745,642], + [12952.18,15184.745,643], + [12974.49,15204.745,644], + [12981.92,15224.745,645], + [13012.3,15244.745,646], + [13021.81,15264.745,647], + [13051.53,15284.745,648], + [13080.22,15304.745,649], + [13098.98,15324.745,650], + [13112.16,15344.745,651], + [13150.78,15364.745,652], + [13151.26,15384.745,653], + [13178.52,15404.745,654], + [13182.64,15424.745,655], + [13206.18,15444.745,656], + [13230.7,15464.745,657], + [13245.95,15484.745,658], + [13278.9,15504.745,659], + [13311.47,15524.745,660], + [13323.11,15544.745,661], + [13344.96,15564.745,662], + [13355.49,15584.745,663], + [13372.67,15604.745,664], + [13388.43,15624.745,665], + [13407.59,15644.745,666], + [13432.84,15664.745,667], + [13451.96,15684.745,668], + [13465.4,15704.745,669], + [13496.84,15724.745,670], + [13508.34,15744.745,671], + [13526.87,15764.745,672], + [13543.02,15784.745,673], + [13580.31,15804.745,674], + [13587.09,15824.745,675], + [13611.06,15844.745,676], + [13623.06,15864.745,677], + [13659.03,15884.745,678], + [13665.41,15904.745,679], + [13687.12,15924.745,680], + [13712.53,15944.745,681], + [13727.86,15964.745,682], + [13760.34,15984.745,683], + [13771.49,16004.745,684], + [13787.6,16024.745,685], + [13818.52,16044.745,686], + [13833.63,16064.745,687], + [13847.19,16084.745,688], + [13872.12,16104.745,689], + [13887.81,16124.745,690], + [13910.15,16144.745,691], + [13933.8,16164.745,692], + [13940.68,16184.745,693], + [13970.79,16204.745,694], + [13981.33,16224.745,695], + [14009.81,16244.745,696], + [14025.07,16264.745,697], + [14045.11,16284.745,698], + [14073.75,16304.745,699], + [14087.62,16324.745,700], + [14104.05,16344.745,701], + [14135.34,16364.745,702], + [14140.66,16384.745,703], + [14167.45,16404.745,704], + [14187.02,16424.745,705], + [14211.64,16444.745,706], + [14230.84,16464.745,707], + [14250.61,16484.745,708], + [14261.1,16504.745,709], + [14295.06,16524.745,710], + [14314.29,16544.745,711], + [14327.08,16564.745,712], + [14352.14,16584.745,713], + [14370.44,16604.745,714], + [14398.46,16624.745,715], + [14408.61,16644.745,716], + [14422.25,16664.745,717], + [14453.64,16684.745,718], + [14464.58,16704.745,719], + [14484.72,16724.745,720], + [14503.41,16744.745,721], + [14529.94,16764.745,722], + [14549.86,16784.745,723], + [14574.08,16804.745,724], + [14578.49,16824.745,725], + [14610.43,16844.745,726], + [14623.51,16864.745,727], + [14660.34,16884.745,728], + [14672.86,16904.745,729], + [14683.66,16924.745,730], + [14707.01,16944.745,731], + [14725.82,16964.745,732], + [14744.59,16984.745,733], + [14770.94,17004.745,734], + [14782.34,17024.745,735], + [14805.1,17044.745,736], + [14825.07,17064.745,737], + [14852.14,17084.745,738], + [14875.42,17104.745,739], + [14888.42,17124.745,740], + [14897.26,17144.745,741], + [14930.65,17164.745,742], + [14945.52,17184.745,743], + [14967.65,17204.745,744], + [14989.06,17224.745,745], + [15006.61,17244.745,746], + [15034.49,17264.745,747], + [15054.65,17284.745,748], + [15063.22,17304.745,749], + [15100.63,17324.745,750], + [15104.33,17344.745,751], + [15127.38,17364.745,752], + [15144.07,17384.745,753], + [15169.18,17404.745,754], + [15188.78,17424.745,755], + [15213.14,17444.745,756], + [15223.88,17464.745,757], + [15254.47,17484.745,758], + [15264.47,17504.745,759], + [15293.2,17524.745,760], + [15312.44,17544.745,761], + [15332.38,17564.745,762], + [15351.06,17584.745,763], + [15371.37,17604.745,764], + [15382.61,17624.745,765], + [15415.22,17644.745,766], + [15423.31,17664.745,767], + [15446.81,17684.745,768], + [15465.96,17704.745,769], + [15487.5,17724.745,770], + [15515.33,17744.745,771], + [15531.84,17764.745,772], + [15540.25,17784.745,773], + [15577.82,17804.745,774], + [15582.69,17824.745,775], + [15607.14,17844.745,776], + [15627.72,17864.745,777], + [15652.97,17884.745,778], + [15679.69,17904.745,779], + [15689.29,17924.745,780], + [15700.97,17944.745,781], + [15730.15,17964.745,782], + [15743.86,17984.745,783], + [15765.91,18004.745,784], + [15789.95,18024.745,785], + [15806.11,18044.745,786], + [15835.46,18064.745,787], + [15849.26,18084.745,788], + [15863.13,18104.745,789], + [15897.47,18124.745,790], + [15910.25,18144.745,791], + [15925.78,18164.745,792], + [15948.27,18184.745,793], + [15973.72,18204.745,794], + [15989.05,18224.745,795], + [16021,18244.745,796], + [16027.06,18264.745,797], + [16054.81,18284.745,798], + [16064.04,18304.745,799], + [16082.87,18324.745,800], + [16111.78,18344.745,801], + [16175.69,18364.745,802], + [16181.22,18384.745,803], + [16199.99,18404.745,804], + [16200.38,18424.745,805], + [16226.21,18444.745,806], + [16231.28,18464.745,807], + [16248.06,18484.745,808], + [16266.75,18504.745,809], + [16289.65,18524.745,810], + [16308.06,18544.745,811], + [16331.99,18564.745,812], + [16346.34,18584.745,813], + [16372.28,18604.745,814], + [16384.81,18624.745,815], + [16409.17,18644.745,816], + [16428.72,18664.745,817], + [16451.56,18684.745,818], + [16471.49,18704.745,819], + [16490.24,18724.744,820], + [16503.59,18744.744,821], + [16538.94,18764.744,822], + [16543.86,18784.744,823], + [16562.74,18804.744,824], + [16585.99,18824.744,825], + [16605.67,18844.744,826], + [16631.43,18864.744,827], + [16648.22,18884.744,828], + [16662.86,18904.744,829], + [16693.09,18924.744,830], + [16701.08,18944.744,831], + [16729.03,18964.744,832], + [16746.9,18984.744,833], + [16768.94,19004.744,834], + [16790.95,19024.744,835], + [16810.13,19044.744,836], + [16822.26,19064.744,837], + [16850.33,19084.744,838], + [16863.34,19104.745,839], + [16886.06,19124.745,840], + [16904.81,19144.745,841], + [16930.92,19164.745,842], + [16950.69,19184.745,843], + [16968.87,19204.745,844], + [16979.43,19224.745,845], + [17014.25,19244.745,846], + [17023.69,19264.745,847], + [17044.06,19284.745,848], + [17065.96,19304.745,849], + [17093.69,19324.745,850], + [17107.68,19344.745,851], + [17135.9,19364.745,852], + [17144.09,19384.745,853], + [17170.83,19404.745,854], + [17182.14,19424.745,855], + [17206.22,19444.745,856], + [17223.94,19464.745,857], + [17255.64,19484.745,858], + [17273.94,19504.745,859], + [17290.37,19524.745,860], + [17301.68,19544.745,861], + [17335.52,19564.745,862], + [17340.09,19584.745,863], + [17364.03,19604.745,864], + [17391.93,19624.745,865], + [17407.31,19644.745,866], + [17433.54,19664.745,867], + [17448.95,19684.745,868], + [17463.24,19704.745,869], + [17494.03,19724.745,870], + [17502.15,19744.745,871], + [17522,19764.745,872], + [17546.28,19784.745,873], + [17568.71,19804.745,874], + [17594.83,19824.745,875], + [17611.31,19844.745,876], + [17623.48,19864.745,877], + [17660.58,19884.745,878], + [17669.5,19904.745,879], + [17686.73,19924.745,880], + [17713.12,19944.745,881], + [17728.88,19964.745,882], + [17750.34,19984.745,883], + [17769.96,20004.745,884], + [17780.31,20024.745,885], + [17812.25,20044.745,886], + [17823.72,20064.745,887], + [17848.38,20084.745,888], + [17869.72,20104.745,889], + [17888.93,20124.745,890], + [17918.28,20144.745,891], + [17935.94,20164.745,892], + [17942.34,20184.745,893], + [17974.57,20204.745,894], + [17990.8,20224.745,895], + [18007.68,20244.745,896], + [18028.65,20264.745,897], + [18046.36,20284.745,898], + [18074.06,20304.745,899], + [18089.4,20324.745,900], + [18098.12,20344.745,901], + [18130.6,20364.745,902], + [18144.64,20384.745,903], + [18164.89,20404.745,904], + [18186.75,20424.745,905], + [18204.84,20444.745,906], + [18228.94,20464.745,907], + [18254.34,20484.745,908], + [18261.08,20504.745,909], + [18291.64,20524.745,910], + [18303.61,20544.745,911], + [18321.26,20564.745,912], + [18346.66,20584.745,913], + [18367.45,20604.745,914], + [18387.37,20624.745,915], + [18412.18,20644.745,916], + [18420.23,20664.745,917], + [18457.21,20684.745,918], + [18465.16,20704.745,919], + [18487.44,20724.745,920], + [18505.69,20744.745,921], + [18528.34,20764.745,922], + [18551.11,20784.745,923], + [18569.35,20804.745,924], + [18581.07,20824.745,925], + [18613.65,20844.745,926], + [18623.19,20864.745,927], + [18640.87,20884.745,928], + [18666.65,20904.745,929], + [18684.64,20924.745,930], + [18712,20944.745,931], + [18730.96,20964.745,932], + [18737.5,20984.745,933], + [18780.64,21004.745,934], + [18787.29,21024.745,935], + [18805.08,21044.745,936], + [18826.11,21064.745,937], + [18857.3,21084.745,938], + [18875.96,21104.745,939], + [18893.22,21124.745,940], + [18903.55,21144.745,941], + [18932.99,21164.745,942], + [18940.84,21184.745,943], + [18962.22,21204.745,944], + [18987.97,21224.745,945], + [19004.69,21244.745,946], + [19027.34,21264.745,947], + [19055.22,21284.745,948], + [19066.23,21304.745,949], + [19099.7,21324.745,950], + [19104.93,21344.745,951], + [19127.88,21364.745,952], + [19146.53,21384.745,953], + [19170.06,21404.745,954], + [19187.59,21424.745,955], + [19214.95,21444.745,956], + [19220.57,21464.745,957], + [19254.45,21484.745,958], + [19269.09,21504.745,959], + [19289.28,21524.745,960], + [19306.27,21544.745,961], + [19329.89,21564.745,962], + [19349.15,21584.745,963], + [19374.47,21604.745,964], + [19385.59,21624.745,965], + [19414.19,21644.745,966], + [19424.64,21664.745,967], + [19445.22,21684.745,968], + [19465.8,21704.745,969], + [19486.59,21724.745,970], + [19506.53,21744.745,971], + [19532.21,21764.745,972], + [19538.52,21784.745,973], + [19572.13,21804.745,974], + [19580.68,21824.745,975], + [19606.17,21844.745,976], + [19627.29,21864.745,977], + [19645.94,21884.745,978], + [19667.35,21904.745,979], + [19688.23,21924.745,980], + [19701.83,21944.745,981], + [19740.31,21964.745,982], + [19746.73,21984.745,983], + [19767.14,22004.745,984], + [19788.09,22024.745,985], + [19809.81,22044.745,986], + [19828.34,22064.745,987], + [19852.69,22084.745,988], + [19861.76,22104.745,989], + [19897.88,22124.745,990], + [19905.31,22144.745,991], + [19925.62,22164.745,992], + [19952.78,22184.745,993], + [19973.52,22204.745,994], + [19988.46,22224.745,995], + [20010.23,22244.745,996], + [20018.75,22264.745,997], + [20053.92,22284.745,998], + [20061.94,22304.745,999], + [20092.01,22324.745,1000], + [20108.16,22344.745,1001], + [20126.81,22364.745,1002], + [20150.27,22384.745,1003], + [20172.22,22404.745,1004], + [20186.41,22424.745,1005], + [20231.36,22444.745,1006], + [20233.46,22464.745,1007], + [20245.02,22484.745,1008], + [20276.72,22504.745,1009], + [20291.19,22524.745,1010], + [20307.55,22544.745,1011], + [20329.33,22564.745,1012], + [20338.57,22584.745,1013], + [20377.56,22604.745,1014], + [20398.18,22624.745,1015], + [20416.47,22644.745,1016], + [20424.66,22664.745,1017], + [20445.69,22684.745,1018], + [20473.22,22704.745,1019], + [20489.61,22724.745,1020], + [20499.79,22744.745,1021], + [20561.76,22764.745,1022], + [20566.46,22784.745,1023], + [20569.67,22804.745,1024], + [20595.55,22824.745,1025], + [20608.5,22844.745,1026], + [20629.6,22864.745,1027], + [20649.08,22884.745,1028], + [20663.03,22904.745,1029], + [20701.24,22924.745,1030], + [20708.01,22944.745,1031], + [20726.02,22964.745,1032], + [20747.29,22984.745,1033], + [20765.89,23004.745,1034], + [20787.19,23024.745,1035], + [20812.89,23044.745,1036], + [20821.8,23064.745,1037], + [20852.03,23084.745,1038], + [20862.96,23104.745,1039], + [20889.65,23124.745,1040], + [20905.12,23144.745,1041], + [20927.56,23164.745,1042], + [20946.2,23184.745,1043], + [20970.25,23204.745,1044], + [20978.92,23224.745,1045], + [21019.22,23244.745,1046], + [21027.17,23264.745,1047], + [21046.14,23284.745,1048], + [21064.84,23304.745,1049], + [21085.66,23324.745,1050], + [21109.81,23344.745,1051], + [21132.46,23364.745,1052], + [21153.24,23384.745,1053], + [21185.29,23404.745,1054], + [21193.65,23424.745,1055], + [21204.17,23444.745,1056], + [21230.16,23464.745,1057], + [21255.79,23484.745,1058], + [21271.3,23504.745,1059], + [21289.11,23524.745,1060], + [21297.08,23544.745,1061], + [21331.6,23564.745,1062], + [21339.07,23584.745,1063], + [21362.55,23604.745,1064], + [21387,23624.745,1065], + [21406.25,23644.745,1066], + [21425.71,23664.745,1067], + [21450.5,23684.745,1068], + [21460.08,23704.745,1069], + [21496.71,23724.745,1070], + [21508.12,23744.745,1071], + [21535.12,23764.745,1072], + [21553.94,23784.745,1073], + [21566.16,23804.745,1074], + [21592.14,23824.745,1075], + [21610.49,23844.745,1076], + [21623.01,23864.745,1077], + [21650.72,23884.745,1078], + [21665.94,23904.745,1079], + [21687.99,23924.745,1080], + [21705.19,23944.745,1081], + [21725.18,23964.745,1082], + [21749.41,23984.745,1083], + [21772.69,24004.745,1084], + [21777.97,24024.745,1085], + [21813.3,24044.745,1086], + [21821.05,24064.745,1087], + [21846.72,24084.745,1088], + [21864.73,24104.745,1089], + [21887.8,24124.745,1090], + [21909.22,24144.745,1091], + [21927.68,24164.745,1092], + [21944.41,24184.745,1093], + [21975.99,24204.745,1094], + [21981.84,24224.745,1095], + [22005.43,24244.745,1096], + [22050.17,24264.745,1097], + [22050.56,24284.745,1098], + [22067.53,24304.745,1099], + [22088.01,24324.745,1100], + [22099.11,24344.745,1101], + [22130.08,24364.745,1102], + [22140.17,24384.745,1103], + [22166.05,24404.745,1104], + [22186.64,24424.745,1105], + [22208.59,24444.745,1106], + [22236.03,24464.745,1107], + [22247.54,24484.745,1108], + [22262.98,24504.745,1109], + [22297.67,24524.745,1110], + [22302.87,24544.745,1111], + [22324.64,24564.745,1112], + [22343.21,24584.745,1113], + [22364.76,24604.745,1114], + [22388.21,24624.745,1115], + [22409.63,24644.745,1116], + [22420.59,24664.745,1117], + [22452.99,24684.745,1118], + [22463.58,24704.745,1119], + [22481.88,24724.745,1120], + [22509.56,24744.745,1121], + [22527.5,24764.745,1122], + [22545.19,24784.745,1123], + [22570.21,24804.745,1124], + [22582.88,24824.745,1125], + [22615.07,24844.745,1126], + [22622.19,24864.745,1127], + [22660.94,24884.745,1128], + [22674.41,24904.745,1129], + [22692.94,24924.745,1130], + [22715.66,24944.745,1131], + [22730.52,24964.745,1132], + [22748.88,24984.745,1133], + [22774.78,25004.745,1134], + [22780.86,25024.745,1135], + [22818.93,25044.745,1136], + [22832.89,25064.745,1137], + [22849.34,25084.745,1138], + [22874.8,25104.745,1139], + [22892.87,25124.745,1140], + [22901.09,25144.745,1141], + [22931.56,25164.745,1142], + [22940.98,25184.745,1143], + [22965.55,25204.745,1144], + [22982.46,25224.745,1145], + [23003.8,25244.745,1146], + [23039.55,25264.745,1147], + [23052.64,25284.745,1148], + [23058.4,25304.745,1149], + [23095.96,25324.745,1150], + [23106.78,25344.745,1151], + [23133.42,25364.745,1152], + [23144.36,25384.745,1153], + [23167.58,25404.745,1154], + [23187.89,25424.745,1155], + [23210.56,25444.745,1156], + [23224.16,25464.745,1157], + [23254.64,25484.745,1158], + [23265.24,25504.745,1159], + [23289.8,25524.745,1160], + [23305.65,25544.745,1161], + [23325.92,25564.745,1162], + [23351.41,25584.745,1163], + [23373.47,25604.745,1164], + [23387.58,25624.745,1165], + [23413.25,25644.745,1166], + [23419.11,25664.745,1167], + [23445.47,25684.745,1168], + [23465.53,25704.745,1169], + [23494.52,25724.745,1170], + [23509.51,25744.745,1171], + [23530.47,25764.745,1172], + [23541.52,25784.745,1173], + [23568.65,25804.745,1174], + [23582.87,25824.745,1175], + [23605.06,25844.745,1176], + [23627.37,25864.745,1177], + [23649.82,25884.745,1178], + [23677.29,25904.745,1179], + [23695.42,25924.745,1180], + [23705.53,25944.745,1181], + [23744.03,25964.745,1182], + [23750.62,25984.745,1183], + [23770.56,26004.745,1184], + [23791.64,26024.745,1185], + [23812.78,26044.745,1186], + [23833.56,26064.745,1187], + [23848.75,26084.745,1188], + [23859.92,26104.745,1189], + [23897.21,26124.745,1190], + [23905.83,26144.745,1191], + [23931,26164.745,1192], + [23953.71,26184.745,1193], + [23973.83,26204.745,1194], + [23990.6,26224.745,1195], + [24013.61,26244.745,1196], + [24030.32,26264.745,1197], + [24057.11,26284.745,1198], + [24074.47,26304.745,1199], + [24096.37,26324.745,1200], + [24112.47,26344.745,1201], + [24135.78,26364.745,1202], + [24147.33,26384.745,1203], + [24168.06,26404.745,1204], + [24193.34,26424.745,1205], + [24215.89,26444.745,1206], + [24225.75,26464.745,1207], + [24246.2,26484.745,1208], + [24276.68,26504.745,1209], + [24288.53,26524.745,1210], + [24311.25,26544.745,1211], + [24331.06,26564.745,1212], + [24342.69,26584.745,1213], + [24393.66,26604.745,1214], + [24394.7,26624.745,1215], + [24421.53,26644.745,1216], + [24431.37,26664.745,1217], + [24445.93,26684.745,1218], + [24472.03,26704.745,1219], + [24494.1,26724.745,1220], + [24504.47,26744.745,1221], + [24536.87,26764.745,1222], + [24545.84,26784.745,1223], + [24564.44,26804.745,1224], + [24589.21,26824.745,1225], + [24609.4,26844.745,1226], + [24632.79,26864.745,1227], + [24654.19,26884.745,1228], + [24662.9,26904.745,1229], + [24693.98,26924.745,1230], + [24701.06,26944.745,1231], + [24726.84,26964.745,1232], + [24748.64,26984.745,1233], + [24767.44,27004.745,1234], + [24797.2,27024.745,1235], + [24819.72,27044.745,1236], + [24829.07,27064.745,1237], + [24850.47,27084.745,1238], + [24867.72,27104.745,1239], + [24887.34,27124.745,1240], + [24906.53,27144.745,1241], + [24928.09,27164.745,1242], + [24947.66,27184.745,1243], + [24967.13,27204.745,1244], + [24979.17,27224.745,1245], + [25023.84,27244.745,1246], + [25028.97,27264.745,1247], + [25047.78,27284.745,1248], + [25071.99,27304.745,1249], + [25097.85,27324.745,1250], + [25113.73,27344.745,1251], + [25133.9,27364.745,1252], + [25142.75,27384.745,1253], + [25175.37,27404.745,1254], + [25185.5,27424.745,1255], + [25214.49,27444.745,1256], + [25229.84,27464.745,1257], + [25249.69,27484.745,1258], + [25271.05,27504.745,1259], + [25292.97,27524.745,1260], + [25305.13,27544.745,1261], + [25330.65,27564.745,1262], + [25342.11,27584.745,1263], + [25363.18,27604.745,1264], + [25386.65,27624.745,1265], + [25411.79,27644.745,1266], + [25436.82,27664.745,1267], + [25451.47,27684.745,1268], + [25462.09,27704.745,1269], + [25495.34,27724.745,1270], + [25504.92,27744.745,1271], + [25525.29,27764.745,1272], + [25554.46,27784.745,1273], + [25570.49,27804.745,1274], + [25589.7,27824.745,1275], + [25611.96,27844.745,1276], + [25628.56,27864.745,1277], + [25659.13,27884.745,1278], + [25672.93,27904.745,1279], + [25689.77,27924.745,1280], + [25706.4,27944.745,1281], + [25734.28,27964.745,1282], + [25753.91,27984.745,1283], + [25770.07,28004.745,1284], + [25786.1,28024.745,1285], + [25814.82,28044.745,1286], + [25824.55,28064.745,1287], + [25854.02,28084.745,1288], + [25865.66,28104.745,1289], + [25898.96,28124.745,1290], + [25911.78,28144.745,1291], + [25936.31,28164.745,1292], + [25939.53,28184.745,1293], + [25976.37,28204.745,1294], + [25985.36,28224.745,1295], + [26007.09,28244.745,1296], + [26035.46,28264.745,1297], + [26047.05,28284.745,1298], + [26068.22,28304.745,1299], + [26091.13,28324.745,1300], + [26108.54,28344.745,1301], + [26130.51,28364.745,1302], + [26147.74,28384.745,1303], + [26164.17,28404.745,1304], + [26184.77,28424.745,1305], + [26210.68,28444.745,1306], + [26226.63,28464.745,1307], + [26252.03,28484.745,1308], + [26257.62,28504.745,1309], + [26297.75,28524.745,1310], + [26306.53,28544.745,1311], + [26329.25,28564.745,1312], + [26347.22,28584.745,1313], + [26365.19,28604.745,1314], + [26387.06,28624.745,1315], + [26411.18,28644.745,1316], + [26420.76,28664.745,1317], + [26454.2,28684.745,1318], + [26463.83,28704.745,1319], + [26486.19,28724.745,1320], + [26502.83,28744.745,1321], + [26526.8,28764.745,1322], + [26548.26,28784.745,1323], + [26571.25,28804.745,1324], + [26585.75,28824.745,1325], + [26614.65,28844.745,1326], + [26631.75,28864.745,1327], + [26648.63,28884.745,1328], + [26667.74,28904.745,1329], + [26690.03,28924.745,1330], + [26713.28,28944.745,1331], + [26732.05,28964.745,1332], + [26738.81,28984.745,1333], + [26776.54,29004.745,1334], + [26789.47,29024.745,1335], + [26807.59,29044.745,1336], + [26838.52,29064.745,1337], + [26844.18,29084.745,1338], + [26872.27,29104.745,1339], + [26898.55,29124.745,1340], + [26907.36,29144.745,1341], + [26939.59,29164.745,1342], + [26942.64,29184.745,1343], + [26964.44,29204.745,1344], + [26989.94,29224.745,1345], + [27007.91,29244.745,1346], + [27033.69,29264.745,1347], + [27059.46,29284.745,1348], + [27066.43,29304.745,1349], + [27098,29324.745,1350], + [27110.91,29344.745,1351], + [27125.27,29364.745,1352], + [27145.16,29384.745,1353], + [27170.99,29404.745,1354], + [27213.57,29424.745,1355], + [27215.77,29444.745,1356], + [27218.53,29464.745,1357], + [27256.02,29484.745,1358], + [27262.27,29504.745,1359], + [27288.41,29524.745,1360], + [27311.98,29544.745,1361], + [27335.71,29564.745,1362], + [27353.25,29584.745,1363], + [27367.69,29604.745,1364], + [27378.59,29624.745,1365], + [27412,29644.745,1366], + [27421.5,29664.745,1367], + [27449.03,29684.745,1368], + [27472.77,29704.745,1369], + [27488.81,29724.745,1370], + [27514.43,29744.745,1371], + [27540.24,29764.745,1372], + [27546.29,29784.745,1373], + [27576.06,29804.745,1374], + [27587.97,29824.745,1375], + [27609,29844.745,1376], + [27629.24,29864.745,1377], + [27658.08,29884.745,1378], + [27671.68,29904.745,1379], + [27692.02,29924.745,1380], + [27697.62,29944.745,1381], + [27746.64,29964.745,1382], + [27747.31,29984.745,1383], + [27768.29,30004.745,1384], + [27793.66,30024.745,1385], + [27805.45,30044.745,1386], + [27834.9,30064.745,1387], + [27848.76,30084.745,1388], + [27868.08,30104.745,1389], + [27899.23,30124.745,1390], + [27905.76,30144.745,1391], + [27924.47,30164.745,1392], + [27950.41,30184.745,1393], + [27964.39,30204.745,1394], + [27989.23,30224.745,1395], + [28011.61,30244.745,1396], + [28029.47,30264.745,1397], + [28059.93,30284.745,1398], + [28069.34,30304.745,1399], + [28087.37,30324.745,1400], + [28104.95,30344.745,1401], + [28130.36,30364.745,1402], + [28148.51,30384.745,1403], + [28179.84,30404.745,1404], + [28232.78,30424.745,1405], + [28235.91,30444.745,1406], + [28236.37,30464.745,1407], + [28249.16,30484.745,1408], + [28268.32,30504.745,1409], + [28288.95,30524.745,1410], + [28311.84,30544.745,1411], + [28329.74,30564.745,1412], + [28336.01,30584.745,1413], + [28371.04,30604.745,1414], + [28389.23,30624.745,1415], + [28404.53,30644.745,1416], + [28436.23,30664.745,1417], + [28452.4,30684.745,1418], + [28472.75,30704.745,1419], + [28504.17,30724.745,1420], + [28504.5,30744.745,1421], + [28536.56,30764.745,1422], + [28542.55,30784.745,1423], + [28567.82,30804.745,1424], + [28593.77,30824.745,1425], + [28616.89,30844.745,1426], + [28630.41,30864.745,1427], + [28667.61,30884.745,1428], + [28668.33,30904.745,1429], + [28694.44,30924.745,1430], + [28702.15,30944.745,1431], + [28730.61,30964.745,1432], + [28747.68,30984.745,1433], + [28776.28,31004.745,1434], + [28788.25,31024.745,1435], + [28809.94,31044.745,1436], + [28826.78,31064.745,1437], + [28860.98,31084.745,1438], + [28877.76,31104.745,1439], + [28885.77,31124.745,1440], + [28906.71,31144.745,1441], + [28928.09,31164.745,1442], + [28954.15,31184.745,1443], + [28978.52,31204.745,1444], + [28982.46,31224.745,1445], + [29025.44,31244.745,1446], + [29034.44,31264.745,1447], + [29055.74,31284.745,1448], + [29069.68,31304.745,1449], + [29092.09,31324.745,1450], + [29108.25,31344.745,1451], + [29129.72,31364.745,1452], + [29140.09,31384.745,1453], + [29176.63,31404.745,1454], + [29194.42,31424.745,1455], + [29211.42,31444.745,1456], + [29231.97,31464.745,1457], + [29256.81,31484.745,1458], + [29269.43,31504.745,1459], + [29304.09,31524.745,1460], + [29307.56,31544.745,1461], + [29340.84,31564.745,1462], + [29361.66,31584.745,1463], + [29372.77,31604.745,1464], + [29404.91,31624.745,1465], + [29420.14,31644.745,1466], + [29432.52,31664.745,1467], + [29466.02,31684.745,1468], + [29478.46,31704.745,1469], + [29511.13,31724.745,1470], + [29523.84,31744.745,1471], + [29524.63,31764.745,1472], + [29544.81,31784.745,1473], + [29572.58,31804.745,1474], + [29592.4,31824.745,1475], + [29612.09,31844.745,1476], + [29623.78,31864.745,1477], + [29656.72,31884.745,1478], + [29670.59,31904.745,1479], + [29694.69,31924.745,1480], + [29707.62,31944.745,1481], + [29732.81,31964.745,1482], + [29751.53,31984.745,1483], + [29774.58,32004.745,1484], + [29779.78,32024.745,1485], + [29826.26,32044.745,1486], + [29826.69,32064.745,1487], + [29844.53,32084.745,1488], + [29868.37,32104.745,1489], + [29889.2,32124.745,1490], + [29910.3,32144.745,1491], + [29927.41,32164.745,1492], + [29943.92,32184.745,1493], + [29974.88,32204.745,1494], + [29986.15,32224.745,1495], + [30010.55,32244.745,1496], + [30029.15,32264.745,1497], + [30052.4,32284.745,1498], + [30070.1,32304.745,1499], + [30088.61,32324.745,1500], + [30102.3,32344.745,1501], + [30133.39,32364.745,1502], + [30143.52,32384.745,1503], + [30175.86,32404.745,1504], + [30188.72,32424.745,1505], + [30215.64,32444.745,1506], + [30232.71,32464.745,1507], + [30255.94,32484.745,1508], + [30264.34,32504.745,1509], + [30294.22,32524.745,1510], + [30307.52,32544.745,1511], + [30328.74,32564.745,1512], + [30346.08,32584.745,1513], + [30373.62,32604.745,1514], + [30392.43,32624.745,1515], + [30423.02,32644.745,1516], + [30428.04,32664.745,1517], + [30453.97,32684.745,1518], + [30464.41,32704.745,1519], + [30485.18,32724.745,1520], + [30508.08,32744.745,1521], + [30531.86,32764.745,1522], + [30547.62,32784.745,1523], + [30571.14,32804.745,1524], + [30582.96,32824.745,1525], + [30619.22,32844.745,1526], + [30624.19,32864.745,1527], + [30644.94,32884.745,1528], + [30672.69,32904.745,1529], + [30690,32924.745,1530], + [30718.78,32944.745,1531], + [30728.33,32964.745,1532], + [30741.84,32984.745,1533], + [30774.84,33004.745,1534], + [30780.37,33024.745,1535], + [30813.11,33044.745,1536], + [30829.81,33064.745,1537], + [30847.98,33084.745,1538], + [30878.75,33104.745,1539], + [30901.28,33124.745,1540], + [30902.58,33144.745,1541], + [30934.74,33164.745,1542], + [30964.14,33184.745,1543], + [30972.14,33204.745,1544], + [30987.01,33224.745,1545], + [31005.79,33244.745,1546], + [31030.16,33264.745,1547], + [31051.55,33284.745,1548], + [31057.52,33304.745,1549], + [31096.49,33324.745,1550], + [31107.38,33344.745,1551], + [31133.6,33364.745,1552], + [31167.45,33384.745,1553], + [31171.58,33404.745,1554], + [31193.78,33424.745,1555], + [31234.72,33444.745,1556], + [31247.71,33464.745,1557], + [31261.3,33484.745,1558], + [31273.39,33504.745,1559], + [31300.96,33524.745,1560], + [31315.84,33544.745,1561], + [31341.75,33564.745,1562], + [31358.54,33584.745,1563], + [31372.28,33604.745,1564], + [31383.66,33624.745,1565], + [31423.57,33644.745,1566], + [31429.9,33664.745,1567], + [31446.09,33684.745,1568], + [31467.75,33704.745,1569], + [31490.14,33724.745,1570], + [31514.66,33744.745,1571], + [31531.77,33764.745,1572], + [31546.41,33784.745,1573], + [31583.51,33804.745,1574], + [31590.83,33824.745,1575], + [31606.84,33844.745,1576], + [31629.36,33864.745,1577], + [31648.56,33884.745,1578], + [31683.84,33904.745,1579], + [31695.71,33924.745,1580], + [31708.11,33944.745,1581], + [31739.5,33964.745,1582], + [31749.13,33984.745,1583], + [31769.66,34004.745,1584], + [31786.4,34024.745,1585], + [31810.02,34044.745,1586], + [31829.59,34064.745,1587], + [31862.28,34084.745,1588], + [31869.93,34104.745,1589], + [31892.07,34124.745,1590], + [31910.09,34144.745,1591], + [31922.6,34164.745,1592], + [31955.69,34184.745,1593], + [31977.59,34204.745,1594], + [31990.16,34224.745,1595], + [32013.18,34244.745,1596], + [32022.53,34264.745,1597], + [32055.84,34284.745,1598], + [32061.38,34304.745,1599], + [32087.96,34324.745,1600], + [32111.78,34344.745,1601], + [32138.59,34364.745,1602], + [32145.59,34384.745,1603], + [32174.11,34404.745,1604], + [32182.26,34424.745,1605], + [32212.87,34444.745,1606], + [32225.2,34464.745,1607], + [32251.73,34484.745,1608], + [32270.94,34504.745,1609], + [32289.26,34524.745,1610], + [32309.45,34544.745,1611], + [32330.4,34564.745,1612], + [32342.56,34584.745,1613], + [32373.28,34604.745,1614], + [32382.06,34624.745,1615], + [32409.91,34644.745,1616], + [32428.18,34664.745,1617], + [32446.94,34684.745,1618], + [32468.71,34704.745,1619], + [32488.75,34724.745,1620], + [32502.42,34744.745,1621], + [32532.14,34764.745,1622], + [32542,34784.745,1623], + [32566.9,34804.745,1624], + [32588.19,34824.745,1625], + [32607.52,34844.745,1626], + [32628.47,34864.745,1627], + [32649.21,34884.745,1628], + [32665.42,34904.745,1629], + [32704.83,34924.745,1630], + [32714.33,34944.745,1631], + [32725.23,34964.745,1632], + [32756.48,34984.745,1633], + [32766.21,35004.745,1634], + [32786.23,35024.745,1635], + [32808.31,35044.745,1636], + [32843.97,35064.745,1637], + [32848.81,35084.745,1638], + [32871.66,35104.744,1639], + [32884.82,35124.744,1640], + [32909.82,35144.744,1641], + [32925.68,35164.744,1642], + [32953.94,35184.744,1643], + [32978.05,35204.744,1644], + [32984.49,35224.744,1645], + [33016.7,35244.744,1646], + [33023.36,35264.744,1647], + [33055.98,35284.744,1648], + [33069.43,35304.744,1649], + [33090.6,35324.744,1650], + [33108.59,35344.744,1651], + [33133.73,35364.744,1652], + [33138.11,35384.744,1653], + [33180.16,35404.744,1654], + [33198.44,35424.744,1655], + [33215.02,35444.744,1656], + [33244.39,35464.744,1657], + [33253.67,35484.744,1658], + [33277.44,35504.744,1659], + [33289.75,35524.744,1660], + [33308.04,35544.744,1661], + [33336.46,35564.744,1662], + [33348.34,35584.744,1663], + [33379.66,35604.744,1664], + [33394.03,35624.744,1665], + [33467.68,35644.744,1666], + [33487.94,35664.744,1667], + [33488.19,35684.744,1668], + [33488.42,35704.744,1669], + [33494.14,35724.744,1670], + [33504.19,35744.744,1671], + [33523.07,35764.744,1672], + [33551.59,35784.744,1673], + [33566.06,35804.744,1674], + [33596.17,35824.744,1675], + [33613.97,35844.744,1676], + [33625.5,35864.744,1677], + [33658.05,35884.745,1678], + [33663.83,35904.745,1679], + [33690.91,35924.745,1680], + [33723.63,35944.745,1681], + [33726.52,35964.745,1682], + [33754.03,35984.745,1683], + [33774.93,36004.745,1684], + [33788.19,36024.745,1685], + [33810.69,36044.745,1686], + [33820.76,36064.745,1687], + [33843.05,36084.745,1688], + [33870.92,36104.745,1689], + [33888.27,36124.745,1690], + [33923.34,36144.745,1691], + [33930.84,36164.745,1692], + [33938.01,36184.745,1693], + [33981.11,36204.745,1694], + [33984.38,36224.745,1695], + [34008.41,36244.745,1696], + [34029.34,36264.745,1697], + [34056.3,36284.745,1698], + [34074.65,36304.745,1699], + [34094.26,36324.745,1700], + [34107.72,36344.745,1701], + [34141.99,36364.745,1702], + [34148.11,36384.745,1703], + [34174.3,36404.745,1704], + [34187.58,36424.745,1705], + [34210.09,36444.745,1706], + [34238.46,36464.745,1707], + [34253.78,36484.745,1708], + [34263.96,36504.745,1709], + [34313.14,36524.745,1710], + [34314.44,36544.745,1711], + [34328.59,36564.745,1712], + [34353.72,36584.745,1713], + [34375.72,36604.745,1714], + [34395.25,36624.745,1715], + [34430.81,36644.745,1716], + [34474.11,36664.745,1717], + [34510.88,36684.745,1718], + [34511.25,36704.745,1719], + [34511.44,36724.745,1720], + [34521.42,36744.745,1721], + [34537.44,36764.745,1722], + [34552.49,36784.745,1723], + [34584.16,36804.745,1724], + [34593.16,36824.745,1725], + [34613.93,36844.745,1726], + [34625.03,36864.745,1727], + [34650.06,36884.745,1728], + [34667.72,36904.745,1729], + [34693.82,36924.745,1730], + [34721.68,36944.745,1731], + [34743.8,36964.745,1732], + [34748.39,36984.745,1733], + [34780.86,37004.745,1734], + [34797.81,37024.745,1735], + [34815.28,37044.745,1736], + [34832.37,37064.745,1737], + [34859.22,37084.745,1738], + [34880.69,37104.745,1739], + [34898.72,37124.745,1740], + [34909.97,37144.745,1741], + [34936.79,37164.745,1742], + [34951.06,37184.745,1743], + [34967.18,37204.745,1744] + ] +} diff --git a/test/justfile b/test/justfile index 0730d38bde..7a8c76c30d 100644 --- a/test/justfile +++ b/test/justfile @@ -79,12 +79,21 @@ drill *args: # heard (underruns, skip-aheads, silence, the resolved delay) against # audio-quality/budgets.json. The full matrix is 16 rows at 60 s; nightly runs it with # --enforce. Narrow it with `--profiles mild --codecs opus --rings plain --duration 20`. +# `--replays` grades only the recorded traces, in seconds, as PRs touching the player do. # See audio-quality/README.md. # Grade browser audio playout over an impaired path. audio-quality *args: ./audio-quality/run.sh "$@" +# Records the arrival traces under audio-quality/traces/ that the replay rows grade: the +# public relay's bbb.hang, a local relay's AAC tone, and a browser microphone through the +# public relay. Needs the network. Narrow it with `--lanes relay-bbb --duration 35`. + +# Re-record the audio quality arrival traces. +audio-quality-record *args: + ./audio-quality/record.sh "$@" + # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. From 94c34831db1d993ceb72fd8844723c91f5cdb9c2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 16:13:30 -0700 Subject: [PATCH 19/27] test(audio-quality): count browser rows apart from replays; size the questline Co-Authored-By: Claude Opus 5.5 --- quest/m0/audio-quality-harness/README.md | 2 +- test/audio-quality/clients/js/src/run.test.ts | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index 2fff2083b9..42ac07ff3d 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -1,4 +1,4 @@ -# Audio quality harness +# [S] Audio quality harness ## Goal diff --git a/test/audio-quality/clients/js/src/run.test.ts b/test/audio-quality/clients/js/src/run.test.ts index 7cd6a67d10..13bda94a13 100644 --- a/test/audio-quality/clients/js/src/run.test.ts +++ b/test/audio-quality/clients/js/src/run.test.ts @@ -22,7 +22,11 @@ test("short steady rows remain supported", () => { test("the nightly matrix includes both new profiles", () => { const result = Bun.spawnSync(["bash", script, "--list"]); expect(result.exitCode).toBe(0); - const rows = result.stdout.toString().trim().split("\n"); + const rows = result.stdout + .toString() + .trim() + .split("\n") + .filter((row) => row.startsWith("chromium-")); expect(rows).toHaveLength(24); for (const profile of ["bursty", "step"]) { expect(rows.filter((row) => row.includes(`-${profile}-`))).toHaveLength(4); From b7a48e6f2fcd8fb898e4a40ba8b895a93e9d6082 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 21:40:20 -0700 Subject: [PATCH 20/27] fix(moq-shaper): verify every step's phase saw traffic A step counted as reached when it applied, so a quiet stretch could skip an intermediate phase and still pass verify. Record the phase each link treats a datagram in and require each step's. Refuse a step at zero, and treat a step past what the clock can hold as never reached instead of panicking. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 7 +-- rs/moq-shaper/src/lib.rs | 97 +++++++++++++++++++++++++++++++++++----- 2 files changed, 91 insertions(+), 13 deletions(-) diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index b7d8001184..0a034b4e63 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -45,7 +45,7 @@ As a library, `Shaper::bind` takes a `Config`: where to listen and forward, the per direction. It also takes a `Setup`, which is a `Config` plus the opt-in options below, each off by default. `Shaper::verify` fails when the shaper stopped forwarding, when an impairment the profile configures never acted and the traffic makes that silence implausible, or when a direction carried -traffic but none of it reached a step. The relay's drills +traffic but none of it saw some step's profile. The relay's drills (`rs/moq-relay/tests/drills.rs`, described in `test/drill/README.md`) run every scenario through it. ## What a seed does and does not fix @@ -93,8 +93,9 @@ A step changes a direction's delay, jitter, loss, reorder or rate limit once the from when forwarding started. It changes only what it names, so a later step puts one knob back. A rate step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there starts full. A step can add or change a rate limit, never remove one. A step that leaves the profile as -it was is refused. Steps come from a profile file. -A run that ends before its last step fails `verify`, since that traffic never saw the path change. +it was, or comes at zero, is refused. Steps come from a profile file. +A run where no datagram saw some step's profile fails `verify`, whether it ended before the step or +went quiet until a later step replaced it. ## Profiles diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 8e1bdf9cfa..d46a0785ea 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -20,7 +20,7 @@ use std::{ cmp::Reverse, - collections::{BinaryHeap, HashMap, hash_map}, + collections::{BTreeSet, BinaryHeap, HashMap, hash_map}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::{ @@ -202,6 +202,9 @@ impl Options { let mut profile = profile.clone(); let mut previous: Option = None; for step in &self.steps { + // The profile is what the run opens with, so a step at zero would + // replace it before any datagram saw it. + anyhow::ensure!(!step.at.is_zero(), "a step at zero replaces the profile; change the profile instead"); // A link only ever looks at the next step due, so one out of order // would be skipped without a word. anyhow::ensure!( @@ -471,9 +474,14 @@ impl Shaper { ("up", &self.setup.up, &self.tally[UP], &stats.up), ("down", &self.setup.down, &self.tally[DOWN], &stats.down), ] { - let reached = tally.stepped.load(Ordering::Relaxed) as usize; - if let Some(step) = options.steps.get(reached).filter(|_| counters.packets > 0) { - missing.push(format!("the {name} step at {:?}", step.at)); + if counters.packets == 0 { + continue; + } + let phases = tally.phases.lock().unwrap(); + for (index, step) in options.steps.iter().enumerate() { + if !phases.contains(&(index + 1)) { + missing.push(format!("the {name} step at {:?}", step.at)); + } } } anyhow::ensure!( @@ -502,8 +510,8 @@ struct Tally { throttled: AtomicU64, delayed: AtomicU64, reordered: AtomicU64, - /// The most steps any one link has applied. - stepped: AtomicU64, + /// The phases some link treated a datagram in: 0 before the first step, n after the nth. + phases: Mutex>, /// Datagrams a batch held past when they would otherwise have left. batched: AtomicU64, } @@ -782,6 +790,8 @@ struct Link { steps: Vec, /// How many of `steps` have applied. stepped: usize, + /// Whether a datagram was treated since the latest step applied. + treated: bool, queue: mpsc::UnboundedSender, } @@ -810,21 +820,32 @@ impl Link { start, steps: options.steps.clone(), stepped: 0, + treated: false, queue, } } - /// Apply every step the run has reached by `now`, in order. + /// Apply every step the run has reached by `now`, in order, and record the + /// phase a datagram arriving then sees. fn step(&mut self, now: Instant, tally: &Tally) { - while let Some(step) = self.steps.get(self.stepped).filter(|step| self.start + step.at <= now) { + while let Some(step) = self.steps.get(self.stepped) { + // A step past what the clock can hold is one the run never reaches. + let Some(at) = self.start.checked_add(step.at).filter(|at| *at <= now) else { + break; + }; if let Some(rate) = &step.rate { // Credit accrues at the new rate from the scheduled transition, // even when no datagram arrived at that instant. - self.full_at = self.refilled(rate, self.start + step.at); + self.full_at = self.refilled(rate, at); } step.apply(&mut self.profile); self.stepped += 1; - tally.stepped.fetch_max(self.stepped as u64, Ordering::Relaxed); + self.treated = false; + } + // Applying a step is no evidence it acted, so record the phase a datagram sees. + if !self.treated { + self.treated = true; + tally.phases.lock().unwrap().insert(self.stepped); } } @@ -1896,6 +1917,61 @@ mod tests { assert!(format!("{err:#}").contains("the down step at 60s"), "{err:#}"); } + #[tokio::test(start_paused = true)] + async fn a_step_no_datagram_saw_is_unapplied() { + let ms = Duration::from_millis; + let (shaper, client) = shaped(|config| Setup { + up: stepped(vec![ + Step { + at: ms(100), + delay: Some(ms(60)), + ..Default::default() + }, + Step { + at: ms(200), + delay: Some(ms(5)), + ..Default::default() + }, + ]), + ..Config { + up: Profile { + delay: ms(5), + ..Default::default() + }, + ..config + } + .into() + }) + .await; + + // One datagram before the first step and one after the second, so the + // 60ms phase passes with nothing in it. + let mut buf = [0u8; 4]; + client.send(&0u32.to_be_bytes()).await.unwrap(); + client.recv(&mut buf).await.unwrap(); + tokio::time::sleep(ms(250)).await; + client.send(&1u32.to_be_bytes()).await.unwrap(); + client.recv(&mut buf).await.unwrap(); + + let err = shaper.verify().expect_err("a run that skipped a step passed"); + let err = format!("{err:#}"); + assert!(err.contains("the up step at 100ms"), "{err}"); + assert!(!err.contains("the up step at 200ms"), "{err}"); + } + + #[test] + fn a_step_past_the_clock_never_applies() { + let options = stepped(vec![Step { + at: Duration::MAX, + delay: Some(Duration::from_millis(60)), + ..Default::default() + }]); + let (queue, _) = mpsc::unbounded_channel(); + let start = Instant::now(); + let mut link = Link::new(&Profile::default(), &options, 7, 0, start, queue); + assert_eq!(owed(&mut link, start + Duration::from_secs(1), &[16]), [Duration::ZERO]); + } + #[tokio::test] async fn steps_that_would_be_skipped_or_do_nothing_are_refused() { let refused = |steps: Vec, why: &'static str| async move { @@ -1925,6 +2001,7 @@ mod tests { }; refused(vec![at(60), at(30)], "steps go in order").await; + refused(vec![at(0)], "a step at zero").await; refused( vec![Step { at: Duration::from_secs(30), From 01ab236d95868dd85f7e9977975c58f152d6f8c4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 21:47:24 -0700 Subject: [PATCH 21/27] style(moq-shaper): format Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/src/lib.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index d46a0785ea..8755d387e6 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -204,7 +204,10 @@ impl Options { for step in &self.steps { // The profile is what the run opens with, so a step at zero would // replace it before any datagram saw it. - anyhow::ensure!(!step.at.is_zero(), "a step at zero replaces the profile; change the profile instead"); + anyhow::ensure!( + !step.at.is_zero(), + "a step at zero replaces the profile; change the profile instead" + ); // A link only ever looks at the next step due, so one out of order // would be skipped without a word. anyhow::ensure!( From beeb93d18cc82fd1d9c0667da29208e8973a1734 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 23:18:40 -0700 Subject: [PATCH 22/27] fix(moq-shaper): require traffic before the first step too A direction whose traffic started after its first step never saw the transition, yet verify passed. Require the phase before the first step alongside each step's. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 7 +++--- rs/moq-shaper/src/lib.rs | 49 ++++++++++++++++++++++++++++++++++------ 2 files changed, 46 insertions(+), 10 deletions(-) diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index 0a034b4e63..af9e215f0a 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -45,7 +45,7 @@ As a library, `Shaper::bind` takes a `Config`: where to listen and forward, the per direction. It also takes a `Setup`, which is a `Config` plus the opt-in options below, each off by default. `Shaper::verify` fails when the shaper stopped forwarding, when an impairment the profile configures never acted and the traffic makes that silence implausible, or when a direction carried -traffic but none of it saw some step's profile. The relay's drills +traffic but no datagram saw one of its phases, before the first step or after one. The relay's drills (`rs/moq-relay/tests/drills.rs`, described in `test/drill/README.md`) run every scenario through it. ## What a seed does and does not fix @@ -94,8 +94,9 @@ from when forwarding started. It changes only what it names, so a later step put step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there starts full. A step can add or change a rate limit, never remove one. A step that leaves the profile as it was, or comes at zero, is refused. Steps come from a profile file. -A run where no datagram saw some step's profile fails `verify`, whether it ended before the step or -went quiet until a later step replaced it. +A direction that carried traffic fails `verify` unless some datagram saw each phase: the profile before +the first step and after every step. Traffic that starts late, ends early, or goes quiet across a step +never saw the path change. ## Profiles diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 8755d387e6..5da562780e 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -448,13 +448,13 @@ impl Shaper { } /// Fail unless the shaper is still forwarding, every impairment the profile - /// configures acted on some datagram, and some datagram saw every step, in - /// either direction. + /// configures acted on some datagram, and some datagram saw every phase of + /// a direction with steps, before the first and after each. /// /// An impairment is only held to that once the traffic makes its silence - /// implausible: a short run can see no loss, but never no delay. A step is - /// held to it once its direction carried anything: traffic that all came - /// before the step never saw the path change. + /// implausible: a short run can see no loss, but never no delay. The phases + /// are held to it once their direction carried anything: traffic that all + /// came on one side of a step never saw the path change. pub fn verify(&self) -> anyhow::Result { let stats = self.stats(); if let Some(err) = self.failed.get() { @@ -477,10 +477,13 @@ impl Shaper { ("up", &self.setup.up, &self.tally[UP], &stats.up), ("down", &self.setup.down, &self.tally[DOWN], &stats.down), ] { - if counters.packets == 0 { + let Some(first) = options.steps.first().filter(|_| counters.packets > 0) else { continue; - } + }; let phases = tally.phases.lock().unwrap(); + if !phases.contains(&0) { + missing.push(format!("the {name} profile before its step at {:?}", first.at)); + } for (index, step) in options.steps.iter().enumerate() { if !phases.contains(&(index + 1)) { missing.push(format!("the {name} step at {:?}", step.at)); @@ -1962,6 +1965,38 @@ mod tests { assert!(!err.contains("the up step at 200ms"), "{err}"); } + #[tokio::test(start_paused = true)] + async fn a_profile_no_datagram_saw_before_its_step_is_unapplied() { + let ms = Duration::from_millis; + let (shaper, client) = shaped(|config| Setup { + up: stepped(vec![Step { + at: ms(100), + delay: Some(ms(60)), + ..Default::default() + }]), + ..Config { + up: Profile { + delay: ms(5), + ..Default::default() + }, + ..config + } + .into() + }) + .await; + + // The first datagram comes after the step, so the 5ms phase passes with nothing in it. + tokio::time::sleep(ms(250)).await; + let mut buf = [0u8; 4]; + client.send(&0u32.to_be_bytes()).await.unwrap(); + client.recv(&mut buf).await.unwrap(); + + let err = shaper.verify().expect_err("a run that started after its step passed"); + let err = format!("{err:#}"); + assert!(err.contains("the up profile before its step at 100ms"), "{err}"); + assert!(!err.contains("the up step at"), "{err}"); + } + #[test] fn a_step_past_the_clock_never_applies() { let options = stepped(vec![Step { From 9fbc43e8ab4134181889f763545972ca680294b0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 10:30:33 -0700 Subject: [PATCH 23/27] fix(test): pass audio-quality recipe args through after the merge main replaced test/justfile's global positional-arguments with a per-recipe attribute, so the line's audio-quality recipes dropped every flag and --replays ran the full browser matrix until CI timed out. Co-Authored-By: Claude Opus 5.5 --- test/justfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/justfile b/test/justfile index 5547a1809a..a013fb897a 100644 --- a/test/justfile +++ b/test/justfile @@ -75,6 +75,7 @@ drill *args: # See audio-quality/README.md. # Grade browser audio playout over an impaired path. +[positional-arguments] audio-quality *args: ./audio-quality/run.sh "$@" @@ -83,6 +84,7 @@ audio-quality *args: # public relay. Needs the network. Narrow it with `--lanes relay-bbb --duration 35`. # Re-record the audio quality arrival traces. +[positional-arguments] audio-quality-record *args: ./audio-quality/record.sh "$@" From fea8fe4210beaed214f206bce8d2f5f41734ddbd Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 13:39:48 -0700 Subject: [PATCH 24/27] quest: finish the audio quality harness line Delete the line README and its references. The post-landing budget re-record from nightly moves into the jitter target's watch quest, which already tightens those budgets. Co-Authored-By: Claude Opus 5.5 --- quest/m0/README.md | 10 ++-- quest/m0/audio-jitter-target/README.md | 6 +-- quest/m0/audio-jitter-target/watch.md | 5 +- quest/m0/audio-quality-harness/README.md | 64 ------------------------ quest/m1/audio-quality-native.md | 5 +- quest/m2/audio-opus-backend.md | 4 -- quest/m2/latency-ledger.md | 6 +-- 7 files changed, 14 insertions(+), 86 deletions(-) delete mode 100644 quest/m0/audio-quality-harness/README.md diff --git a/quest/m0/README.md b/quest/m0/README.md index 31469deaa0..79d4d9bedb 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -28,11 +28,10 @@ work is done on the line branch and waits to land. Serving the relay's ingested-only view (`origin::Consumer::local()`) to localhost workers belongs to moq.pro's edge, which embeds moq-relay; it moved there on 2026-09-28. -Audio playout: the jitter target replaces the round-trip guess. The harness's -browser lane grades it nightly and records the traces it replays; the native -lane is a standalone m1 quest, since nothing here waits on it. The harness -line lands before the jitter line, since both add -`js/watch/src/audio/replay.test.ts`. The [A/V clock](/quest/m1/av-clock.md) +Audio playout: the jitter target replaces the round-trip guess. The browser +audio quality harness in `test/audio-quality/` has landed; it grades the target +nightly and records the traces it replays. Its native lane is a standalone m1 +quest, since nothing here waits on it. The [A/V clock](/quest/m1/av-clock.md) moved to m1 in the 2026-09-30 audit: it waits on the whole jitter line and is a published `@moq/watch` break on dev. @@ -49,7 +48,6 @@ Published API or wire breaks still land on dev; each quest's Plan says so. - [qmux credit](/quest/m0/qmux-credit.md) - qmux returns connection credit for dropped and stopped streams and delivers its close frame, on both lines - [Shared fronts](/quest/m0/shared-fronts.md) - viewer sessions share a front, so fronts scale with peers, not viewers - [Wildcard](/quest/m0/wildcard/README.md) - a relay resolves subscriptions against advertised prefixes, a service claims the prefix it could serve and refuses the rest instead of enumerating broadcasts, and the browser player treats a covering claim as availability -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - a browser playout latency regression fails a nightly run instead of arriving as a bug report, and its recorder supplies the jitter target's replay traces - [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the audio playout target is a measured estimate of arrival timing in both languages, not a round-trip guess ## Related diff --git a/quest/m0/audio-jitter-target/README.md b/quest/m0/audio-jitter-target/README.md index 9b5c88ad7d..1056392c71 100644 --- a/quest/m0/audio-jitter-target/README.md +++ b/quest/m0/audio-jitter-target/README.md @@ -35,9 +35,8 @@ was wrong, and both compile unchanged for existing callers. The line branch is about 200 commits behind `main` with conflicts in `js/watch/src/sync.ts` and `rs/moq-cli`; merge `main` in (never rebase the shared branch) before finishing the watch quest. The raw #3477 traces are gone, so record fresh -traces with the [audio quality -harness](/quest/m0/audio-quality-harness/README.md) instead of asking the -reporter; they replace the #3477 traces wherever the quests name them. +traces with the audio quality harness in `test/audio-quality/` instead of +asking the reporter; they replace the #3477 traces wherever the quests name them. The algorithm is written down at `doc/concept/audio-jitter.md`, with a conformance corpus beside it that both implementations will read. @@ -76,7 +75,6 @@ buffer against uneven arrivals. ## Required -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - lands first, since both lines add `js/watch/src/audio/replay.test.ts`; it also records the traces the watch quest replays - [Watch](/quest/m0/audio-jitter-target/watch.md) - js/watch and js/hang bring the #3954 estimator into conformance - [Native](/quest/m0/audio-jitter-target/native.md) - rs/moq-audio grows a measured jitter buffer from the same algorithm diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index dd07660988..dbb390c750 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -80,7 +80,10 @@ against it is likely cheaper than patching the branch's: `Container.Consumer` and both rings on a simulated clock at the delay `Sync` resolves. Replay them in `replay.test.ts`, move the replay's target onto the estimator (its 100 ms fallback covers the public relay's missing PROBE RTT - until then), and tighten the harness's exact replay budgets with it. + until then), and tighten the harness's exact replay budgets with it. The + checked-in `test/audio-quality/budgets.json` was recorded locally (#4426); + re-record it from the nightly runner's first runs on `main` before + tightening, since nightly only runs `main`'s code. - Manual run against the public relay on Chrome and Safari, the two rows the issue measured. Re-record the `relay-mic` trace in the same run with a real microphone (`just test audio-quality-record`); the checked-in one used diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md deleted file mode 100644 index 42ac07ff3d..0000000000 --- a/quest/m0/audio-quality-harness/README.md +++ /dev/null @@ -1,64 +0,0 @@ -# [S] Audio quality harness - -## Goal - -A regression in audio playout latency fails a run instead of arriving as a bug -report. The harness plays a broadcast over an impaired path, counts what the -listener would actually have heard (underruns, short quanta, discarded samples, -skip-aheads) and what each stage of the pipeline contributed to the delay, and -grades the result against a checked-in budget. This line is the browser; -the native lane is [Native audio -quality](/quest/m1/audio-quality-native.md), on the same jitter profiles and -reporting the same numbers. - -Boundaries: audio only. The stage breakdown is defined generically so video can -adopt it later, but no video assertion ships here. No perceptual scoring: the -grade is glitches and latency, not an opinion about how it sounds. - -## Plan - -Browser only, because that is where the traces and the reported bug both -are. The native lane moved to m1 as a standalone quest (decided in the -2026-09-28 quest audit): nothing in m0 waits on it, and it follows against the -same budgets and metric schema so the two implementations can be compared -rather than merely both passing. - -The browser lane and its opt-in shaper profiles have landed in `test/audio-quality/`, -upstreamed from the reporter's fork on #3477 (`fperex/moq`, branch -`debug-findings-solution`), without the fork's player or estimator changes. -The matrix includes bursty and mid-run step paths, and mild/wide jitter preserves -packet order. Real arrival traces have landed beside it in -`test/audio-quality/traces/`, graded as replay rows through the player's -container consumer and rings, and replayed by the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md). - -Jitter comes from the seeded userspace UDP shaper the transport drills run -under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake -arrival clock, so the transport's own contribution is measured rather than -assumed. Its `moq-shaper` binary is what a JS harness process puts in front of -a relay. - -Loss, reorder and rate limiting stay switched off in the profiles here. The -buffer's job is absorbing arrival spread, and mixing congestion response into an -audio quality number makes a failure hard to attribute. - -Nightly, not per-PR: the matrix is jitter profiles by runtime by codec and -sample rate, which is more than a merge gate should carry, and `nightly.yml` -already exists for exactly this trade. The recorded-trace replays are the -exception: they run in seconds without a browser, so -`.github/workflows/audio-quality.yml` gates every PR touching the player's -packages or the harness on them. Budgets are keyed by the full row, since -each of those dimensions moves the expected floor. - -The budgets checked in with the browser lane (#4426) were recorded locally. -Once the line lands, re-record `test/audio-quality/budgets.json` from the -nightly runner's first runs, since nightly only runs `main`'s code. Tightening -the auto rows belongs to the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md). - -## Related - -- [Native audio quality](/quest/m1/audio-quality-native.md) - the same profiles and budgets through `moq play` on a dummy device -- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator this exists to keep honest -- [Latency ledger](/quest/m2/latency-ledger.md) - promotes this harness's probes into a public API -- [Time stretch](/quest/m1/watch-audio-time-stretch.md) - graded by this harness once it lands diff --git a/quest/m1/audio-quality-native.md b/quest/m1/audio-quality-native.md index 232cc61deb..76b845c16e 100644 --- a/quest/m1/audio-quality-native.md +++ b/quest/m1/audio-quality-native.md @@ -41,7 +41,6 @@ budget. Only then compare end-to-end totals, with the backend-dependent stages isolated: this lane deliberately accepts real device callback noise, so a difference in totals alone proves nothing about the estimator. -Standalone in m1 rather than a child of the m0 [Audio quality -harness](/quest/m0/audio-quality-harness/README.md) line (decided in the -2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target +Standalone in m1 rather than a child of the m0 audio quality harness line +(decided in the 2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target it grades is done on the jitter target line. diff --git a/quest/m2/audio-opus-backend.md b/quest/m2/audio-opus-backend.md index 0a5ccf0e6f..ee9a225680 100644 --- a/quest/m2/audio-opus-backend.md +++ b/quest/m2/audio-opus-backend.md @@ -40,7 +40,3 @@ tested only by reading the codec's control value. Retain fixtures and repeatable measurements in the existing CI/nightly audio harness. Public API and wire: no change for the study; validate compatibility before a separately scoped backend implementation. - -## Related - -- [Audio quality](/quest/m0/audio-quality-harness/README.md) - shared measurement infrastructure diff --git a/quest/m2/latency-ledger.md b/quest/m2/latency-ledger.md index ad3cee94e8..0ccc99977a 100644 --- a/quest/m2/latency-ledger.md +++ b/quest/m2/latency-ledger.md @@ -9,8 +9,8 @@ come from the same place. ## Plan -The audio quality harness lands on ad-hoc debug probes, which is the right -trade to get it running. This quest promotes them. +The audio quality harness (`test/audio-quality/`) landed on ad-hoc debug +probes, which was the right trade to get it running. This quest promotes them. - Take the stage schema the harness already defines (capture, encode, publish flush, network, jitter buffer, decode, render) and expose it as fields of @@ -33,10 +33,8 @@ trade to get it running. This quest promotes them. ## Required -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - defines the stage schema and lands the probes this promotes - [Media stats schema](/quest/m1/stats/schema.md) - adds the stats and feedback snapshots this extends ## Related -- [Audio quality harness](/quest/m0/audio-quality-harness/README.md) - defines the stages and is the first consumer - [QoS](/quest/m1/qos/README.md) - relay-side health, the same idea from the other end From a5ecd606b220e831850790766d4d23f5fe4d0226 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 14:25:25 -0700 Subject: [PATCH 25/27] fix(moq-shaper): charge each step phase only its own traffic; drop rate steps verify held the opening profile's impairments to the whole run's packets, so a loss stepped off after one surviving datagram failed a valid run. Each phase now counts its own datagrams and is judged on them. A rate step left already queued departures at the old rate, so a speed-up reordered datagrams. Nothing steps the rate, so steps no longer take one. The two socket-based step tests run on the wall clock, since a paused clock can jump a step boundary while a datagram is in the kernel. Co-Authored-By: Claude Opus 5.5 --- rs/moq-shaper/README.md | 10 +- rs/moq-shaper/src/lib.rs | 232 +++++++++++++++------------------------ 2 files changed, 93 insertions(+), 149 deletions(-) diff --git a/rs/moq-shaper/README.md b/rs/moq-shaper/README.md index af9e215f0a..1ccad1cea2 100644 --- a/rs/moq-shaper/README.md +++ b/rs/moq-shaper/README.md @@ -89,11 +89,11 @@ since the datagram that fills a batch never waits. ## Steps -A step changes a direction's delay, jitter, loss, reorder or rate limit once the run reaches `at`, counted -from when forwarding started. It changes only what it names, so a later step puts one knob back. A rate -step keeps the old bucket's credit, or its debt, clipped to the new one, and a limit that was not there -starts full. A step can add or change a rate limit, never remove one. A step that leaves the profile as -it was, or comes at zero, is refused. Steps come from a profile file. +A step changes a direction's delay, jitter, loss or reorder once the run reaches `at`, counted from when +forwarding started. It changes only what it names, so a later step puts one knob back. The rate limit +never steps, since datagrams already queued behind it would keep the old rate's departures and a new +rate would reorder them. A step that leaves the profile as it was, or comes at zero, is refused. Steps +come from a profile file. `verify` holds each phase's impairments only to the traffic that phase treated. A direction that carried traffic fails `verify` unless some datagram saw each phase: the profile before the first step and after every step. Traffic that starts late, ends early, or goes quiet across a step never saw the path change. diff --git a/rs/moq-shaper/src/lib.rs b/rs/moq-shaper/src/lib.rs index 5da562780e..4e9ca01e5a 100644 --- a/rs/moq-shaper/src/lib.rs +++ b/rs/moq-shaper/src/lib.rs @@ -20,7 +20,7 @@ use std::{ cmp::Reverse, - collections::{BTreeSet, BinaryHeap, HashMap, hash_map}, + collections::{BTreeMap, BinaryHeap, HashMap, hash_map}, fmt, net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}, sync::{ @@ -132,8 +132,9 @@ pub struct Batch { /// A change to one direction's profile once the run reaches `at`. /// /// A step changes only what it names, so a later step puts one knob back -/// without restating the rest. It can add or change a rate limit, never remove -/// one. +/// without restating the rest. The rate limit never steps: datagrams already +/// queued behind it keep the old rate's departures, so a new rate would +/// reorder them. #[derive(Clone, Debug, Default, PartialEq, Deserialize)] #[serde(deny_unknown_fields)] pub struct Step { @@ -152,9 +153,6 @@ pub struct Step { /// The reorder from then on. #[serde(default)] pub reorder: Option, - /// The rate limit from then on. - #[serde(default)] - pub rate: Option, } impl Step { @@ -172,9 +170,6 @@ impl Step { if let Some(reorder) = self.reorder { profile.reorder = reorder; } - if let Some(rate) = &self.rate { - profile.rate = Some(rate.clone()); - } } } @@ -249,19 +244,38 @@ const IMPAIRMENTS: [Impairment; 3] = [ /// cannot, and that is when the silence means the profile is not in the path. const IMPLAUSIBLE: f64 = 1e-4; -/// The impairments `config` configures that `stats` shows implausibly never acted. -fn unapplied(config: &Config, stats: &Stats) -> Vec<&'static str> { +/// The impairments that `stats` shows implausibly never acted, given every +/// phase of both directions as the profile in force and the datagrams it treated. +/// +/// Each phase is charged only its own traffic, so the clean traffic after a step +/// that turns an impairment off never counts against the phase before it. +fn unapplied(phases: &[(Profile, u64)], stats: &Stats) -> Vec<&'static str> { IMPAIRMENTS .iter() .filter(|(_, chance, acted)| { - let silence = (1.0 - chance(&config.up)).powf(stats.up.packets as f64) - * (1.0 - chance(&config.down)).powf(stats.down.packets as f64); + let silence: f64 = phases + .iter() + .map(|(profile, packets)| (1.0 - chance(profile)).powf(*packets as f64)) + .product(); acted(&stats.up) + acted(&stats.down) == 0 && silence < IMPLAUSIBLE }) .map(|(name, ..)| *name) .collect() } +/// `profile` as the run opens and as each of `steps` leaves it, paired with the +/// datagrams `counts` says that phase treated. +fn phased(profile: &Profile, steps: &[Step], counts: &BTreeMap) -> Vec<(Profile, u64)> { + let count = |phase: usize| counts.get(&phase).copied().unwrap_or(0); + let mut profile = profile.clone(); + let mut phases = vec![(profile.clone(), count(0))]; + for (index, step) in steps.iter().enumerate() { + step.apply(&mut profile); + phases.push((profile.clone(), count(index + 1))); + } + phases +} + /// Whether the batches `setup` configures implausibly never held a datagram. /// /// A batch holds every datagram but the one that fills it, so it acts on at @@ -461,10 +475,15 @@ impl Shaper { anyhow::bail!("the shaper stopped forwarding: {err} ({stats})"); } - let mut missing: Vec = unapplied(&self.setup.config, &stats) - .into_iter() - .map(String::from) - .collect(); + let counts = [UP, DOWN].map(|dir| self.tally[dir].phases.lock().unwrap().clone()); + let phases: Vec<(Profile, u64)> = [ + (&self.setup.config.up, &self.setup.up, &counts[UP]), + (&self.setup.config.down, &self.setup.down, &counts[DOWN]), + ] + .into_iter() + .flat_map(|(profile, options, counts)| phased(profile, &options.steps, counts)) + .collect(); + let mut missing: Vec = unapplied(&phases, &stats).into_iter().map(String::from).collect(); let batched = self .tally .iter() @@ -473,19 +492,18 @@ impl Shaper { if unbatched(&self.setup, &stats, batched) { missing.push("batch".to_string()); } - for (name, options, tally, counters) in [ - ("up", &self.setup.up, &self.tally[UP], &stats.up), - ("down", &self.setup.down, &self.tally[DOWN], &stats.down), + for (name, options, phases, counters) in [ + ("up", &self.setup.up, &counts[UP], &stats.up), + ("down", &self.setup.down, &counts[DOWN], &stats.down), ] { let Some(first) = options.steps.first().filter(|_| counters.packets > 0) else { continue; }; - let phases = tally.phases.lock().unwrap(); - if !phases.contains(&0) { + if !phases.contains_key(&0) { missing.push(format!("the {name} profile before its step at {:?}", first.at)); } for (index, step) in options.steps.iter().enumerate() { - if !phases.contains(&(index + 1)) { + if !phases.contains_key(&(index + 1)) { missing.push(format!("the {name} step at {:?}", step.at)); } } @@ -516,8 +534,8 @@ struct Tally { throttled: AtomicU64, delayed: AtomicU64, reordered: AtomicU64, - /// The phases some link treated a datagram in: 0 before the first step, n after the nth. - phases: Mutex>, + /// Datagrams treated in each phase: 0 before the first step, n after the nth. + phases: Mutex>, /// Datagrams a batch held past when they would otherwise have left. batched: AtomicU64, } @@ -796,8 +814,6 @@ struct Link { steps: Vec, /// How many of `steps` have applied. stepped: usize, - /// Whether a datagram was treated since the latest step applied. - treated: bool, queue: mpsc::UnboundedSender, } @@ -826,7 +842,6 @@ impl Link { start, steps: options.steps.clone(), stepped: 0, - treated: false, queue, } } @@ -836,36 +851,14 @@ impl Link { fn step(&mut self, now: Instant, tally: &Tally) { while let Some(step) = self.steps.get(self.stepped) { // A step past what the clock can hold is one the run never reaches. - let Some(at) = self.start.checked_add(step.at).filter(|at| *at <= now) else { + if self.start.checked_add(step.at).is_none_or(|at| at > now) { break; - }; - if let Some(rate) = &step.rate { - // Credit accrues at the new rate from the scheduled transition, - // even when no datagram arrived at that instant. - self.full_at = self.refilled(rate, at); } step.apply(&mut self.profile); self.stepped += 1; - self.treated = false; - } - // Applying a step is no evidence it acted, so record the phase a datagram sees. - if !self.treated { - self.treated = true; - tally.phases.lock().unwrap().insert(self.stepped); } - } - - /// When `rate`'s bucket would be full again if it took over at `now`. - /// - /// A limit that was not there starts full, the way the run started. One - /// that was keeps its credit, or its debt, clipped to the new bucket. - fn refilled(&self, rate: &Rate, now: Instant) -> Instant { - let Some(old) = &self.profile.rate else { - return now; - }; - let owed = self.full_at.saturating_duration_since(now).as_secs_f64() * old.bits_per_second as f64 / 8.0; - let credit = (old.burst as f64 - owed).min(rate.burst as f64); - now + Duration::from_secs_f64((rate.burst as f64 - credit) * 8.0 / rate.bits_per_second as f64) + // Applying a step is no evidence it acted, so count the datagrams each phase treated. + *tally.phases.lock().unwrap().entry(self.stepped).or_default() += 1; } /// Treat a datagram and queue it to leave by `socket` for `dest`. @@ -1129,16 +1122,11 @@ mod tests { #[test] fn silence_is_only_a_failure_once_it_is_implausible() { - let config = Config { - bind: LOCALHOST, - target: LOCALHOST, - seed: 0, - up: Profile { - loss: 0.05, - ..Default::default() - }, - down: Profile::default(), + let lossy = Profile { + loss: 0.05, + ..Default::default() }; + let both = |packets| [(lossy.clone(), packets), (Profile::default(), packets)]; let quiet = |packets| Stats { up: Counters { packets, @@ -1151,24 +1139,49 @@ mod tests { }; // 0.95^20 is about a third: a short run seeing no loss proves nothing. - assert!(unapplied(&config, &quiet(20)).is_empty()); + assert!(unapplied(&both(20), &quiet(20)).is_empty()); // 0.95^1000 is about 5e-23: the loss is not in the path. - assert_eq!(unapplied(&config, &quiet(1000)), ["loss"]); + assert_eq!(unapplied(&both(1000), &quiet(1000)), ["loss"]); - let mut lossy = quiet(1000); - lossy.up.lost = 1; - assert!(unapplied(&config, &lossy).is_empty()); + let mut lost = quiet(1000); + lost.up.lost = 1; + assert!(unapplied(&both(1000), &lost).is_empty()); // A delay acts on every datagram, so even one undelayed datagram is a // shaper that is not in the path. - let config = Config { - down: Profile { - delay: Duration::from_millis(10), + let delayed = Profile { + delay: Duration::from_millis(10), + ..Default::default() + }; + assert_eq!(unapplied(&[(lossy, 1), (delayed, 1)], &quiet(1)), ["delay"]); + } + + #[test] + fn a_phase_is_charged_only_its_own_traffic() { + let ms = Duration::from_millis; + let lossy = Profile { + loss: 0.02, + ..Default::default() + }; + let steps = [Step { + at: ms(100), + loss: Some(0.0), + ..Default::default() + }]; + // One datagram survived the 2% loss, then the loss stepped off for a thousand. + let phases = phased(&lossy, &steps, &BTreeMap::from([(0, 1), (1, 1000)])); + let stats = Stats { + up: Counters { + packets: 1001, ..Default::default() }, - ..config + down: Counters::default(), }; - assert_eq!(unapplied(&config, &quiet(1)), ["delay"]); + assert!(unapplied(&phases, &stats).is_empty()); + + // The same thousand under the loss still fail. + let phases = phased(&lossy, &steps, &BTreeMap::from([(0, 1000), (1, 1)])); + assert_eq!(unapplied(&phases, &stats), ["loss"]); } #[tokio::test] @@ -1799,76 +1812,6 @@ mod tests { assert_eq!(lost(start + secs(60)), 0, "the step back never cleared"); } - #[test] - fn a_rate_step_narrows_the_bucket_and_widens_it_again() { - let secs = Duration::from_secs; - let rate = |bytes_per_second: u64| Rate { - bits_per_second: bytes_per_second * 8, - burst: 100, - queue: secs(10), - }; - let profile = Profile { - rate: Some(rate(4000)), - ..Default::default() - }; - let options = stepped(vec![ - Step { - at: secs(30), - rate: Some(rate(1000)), - ..Default::default() - }, - Step { - at: secs(60), - rate: Some(rate(4000)), - ..Default::default() - }, - ]); - let (queue, _) = mpsc::unbounded_channel(); - let start = Instant::now(); - let mut link = Link::new(&profile, &options, 7, 0, start, queue); - - // The bucket is full at each stretch, so the first 100 bytes leave at - // once and the next 100 owe one bucket's refill at the rate in force. - let ms = Duration::from_millis; - assert_eq!(owed(&mut link, start, &[100, 100]), [ms(0), ms(25)]); - assert_eq!(owed(&mut link, start + secs(30), &[100, 100]), [ms(0), ms(100)]); - assert_eq!(owed(&mut link, start + secs(60), &[100, 100]), [ms(0), ms(25)]); - } - - #[test] - fn a_rate_step_keeps_the_debt_it_takes_over() { - let ms = Duration::from_millis; - let rate = |bytes_per_second: u64| Rate { - bits_per_second: bytes_per_second * 8, - burst: 100, - queue: Duration::from_secs(10), - }; - let profile = Profile { - rate: Some(rate(1000)), - ..Default::default() - }; - // Halfway through a 300 byte backlog at 1000 bytes a second, the rate doubles. - let options = stepped(vec![Step { - at: ms(100), - rate: Some(rate(2000)), - ..Default::default() - }]); - let (queue, _) = mpsc::unbounded_channel(); - let start = Instant::now(); - let mut link = Link::new(&profile, &options, 7, 0, start, queue); - assert_eq!(owed(&mut link, start, &[100, 100, 100]), [ms(0), ms(100), ms(200)]); - - // 100 bytes of the backlog are still owed, and now drain in 50ms, so - // a fresh 100 bytes leave once those and themselves are paid. - assert_eq!(owed(&mut link, start + ms(100), &[100]), [ms(100)]); - - // A quiet link still changes rates at the step, not at its next arrival. - let (queue, _) = mpsc::unbounded_channel(); - let mut link = Link::new(&profile, &options, 7, 0, start, queue); - owed(&mut link, start, &[100, 100, 100]); - assert_eq!(owed(&mut link, start + ms(200), &[100]), [ms(0)]); - } - #[tokio::test] async fn a_step_gets_worse_part_way_through() { let ms = Duration::from_millis; @@ -1923,7 +1866,8 @@ mod tests { assert!(format!("{err:#}").contains("the down step at 60s"), "{err:#}"); } - #[tokio::test(start_paused = true)] + // Real sockets, so the wall clock: a paused one could jump past a step while a datagram is still in the kernel. + #[tokio::test] async fn a_step_no_datagram_saw_is_unapplied() { let ms = Duration::from_millis; let (shaper, client) = shaped(|config| Setup { @@ -1965,7 +1909,7 @@ mod tests { assert!(!err.contains("the up step at 200ms"), "{err}"); } - #[tokio::test(start_paused = true)] + #[tokio::test] async fn a_profile_no_datagram_saw_before_its_step_is_unapplied() { let ms = Duration::from_millis; let (shaper, client) = shaped(|config| Setup { From 2278508bb4541459f642e7ef7bffa05f852c1a60 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 14:25:25 -0700 Subject: [PATCH 26/27] fix(audio-quality): count a gap the run ends in; replay to the recording's end A gap only closed on the next full quantum, so a run ending mid-underrun never reported it. The tap and the replay now close it at the end. Replay stretched a frame over any gap under 100 ms, filling a frame that never arrived, and stopped at the last arrival, cutting a trailing outage. Frames are now the trace's median spacing, and traces carry the duration they observed (version 2) so replay renders through it. The three checked-in traces were all recorded at record.sh's default 35 s. Replay budgets tighten to the longer window; the totals are unchanged. Also detach the previous context's renderCapacity listener on a new root. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/audio/replay.test.ts | 38 ++++++++++++++-- js/watch/src/audio/replay.ts | 42 ++++++++++-------- quest/m0/README.md | 2 +- test/audio-quality/README.md | 8 ++-- test/audio-quality/budgets.json | 22 +++++----- test/audio-quality/clients/js/driver.ts | 3 ++ test/audio-quality/clients/js/record.ts | 3 +- test/audio-quality/clients/js/replay.ts | 44 ++++++++++++------- test/audio-quality/clients/js/src/probe.ts | 18 +++++++- .../clients/js/src/quantum.test.ts | 14 ++++++ test/audio-quality/clients/js/src/quantum.ts | 5 +++ test/audio-quality/clients/js/src/schema.ts | 4 +- .../clients/js/src/tap-worklet.ts | 17 ++++--- test/audio-quality/clients/js/src/tap.ts | 12 +++++ test/audio-quality/traces/local-aac.json | 3 +- test/audio-quality/traces/relay-bbb.json | 3 +- test/audio-quality/traces/relay-mic.json | 3 +- 17 files changed, 176 insertions(+), 65 deletions(-) diff --git a/js/watch/src/audio/replay.test.ts b/js/watch/src/audio/replay.test.ts index d4f23355cf..3abcc74208 100644 --- a/js/watch/src/audio/replay.test.ts +++ b/js/watch/src/audio/replay.test.ts @@ -10,6 +10,9 @@ function paced(seconds: number, lateness: (i: number) => number): Arrival[] { return Array.from({ length: seconds * 50 }, (_, i) => ({ at: i * 20 + lateness(i), timestamp: i * 20, group: i })); } +/** How long {@link paced} observes `seconds` of frames: the frames, plus one more. */ +const observed = (seconds: number) => seconds * 1000 + 20; + /** Quanta that came back short or empty while playing, once the ring had first played. */ async function underruns(trace: Arrival[], options: Options): Promise { let started = false; @@ -34,7 +37,7 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { expect( await underruns( paced(10, () => 30), - { ring, rate: RATE, delay: 100 }, + { ring, rate: RATE, delay: 100, duration: observed(10) }, ), ).toBe(0); }); @@ -42,8 +45,8 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { it("underruns when a flush span outlasts the target, and not when it is covered", async () => { // Five frames held and flushed at once: 80 ms of arrival spread. const trace = paced(10, (i) => 30 + (4 - (i % 5)) * 20); - expect(await underruns(trace, { ring, rate: RATE, delay: 40 })).toBeGreaterThan(50); - expect(await underruns(trace, { ring, rate: RATE, delay: 150 })).toBe(0); + expect(await underruns(trace, { ring, rate: RATE, delay: 40, duration: observed(10) })).toBeGreaterThan(50); + expect(await underruns(trace, { ring, rate: RATE, delay: 150, duration: observed(10) })).toBe(0); }); it("holds newer groups behind a missing one until the max age gives up on it", async () => { @@ -51,9 +54,36 @@ describe.each(["shared", "post"] as const)("%s ring", (ring) => { // The consumer delivers in group order, so the ring runs dry for longer than the missing frame // while the groups behind it wait. Writing arrivals straight into the ring plays straight through. const frame = Math.ceil((20 / 1000) * (RATE / QUANTUM)); - expect(await underruns(trace, { ring, rate: RATE, delay: 100 })).toBeGreaterThan(frame); + expect(await underruns(trace, { ring, rate: RATE, delay: 100, duration: observed(10) })).toBeGreaterThan(frame); expect(warn).toHaveBeenCalled(); }); + + it("leaves a frame missing inside a group as missing audio", async () => { + // Five frames a group, delivered on time, with one from the middle of a group never sent. + const trace = paced(10, () => 30) + .map((arrival, i) => ({ ...arrival, group: Math.floor(i / 5) })) + .filter((_, i) => i !== 252); + expect(await underruns(trace, { ring, rate: RATE, delay: 100, duration: observed(10) })).toBeGreaterThan(0); + }); + + it("renders through the end of the observation, past the last arrival", async () => { + let last = 0; + let heard = 0; + for await (const { at, output } of replay( + paced(10, () => 30), + { + ring, + rate: RATE, + delay: 100, + duration: 15_000, + }, + )) { + last = at; + if (at > 11_000 && output.some((v) => v !== 0)) heard++; + } + expect(last).toBeGreaterThanOrEqual(15_000); + expect(heard).toBe(0); + }); }); describe("target", () => { diff --git a/js/watch/src/audio/replay.ts b/js/watch/src/audio/replay.ts index ba027dd643..6102d61c10 100644 --- a/js/watch/src/audio/replay.ts +++ b/js/watch/src/audio/replay.ts @@ -28,9 +28,6 @@ import { Terminal } from "./terminal"; /** An AudioWorklet render quantum, in frames. */ export const QUANTUM = 128; -/** A media gap longer than this is missing audio, not one long frame. */ -const MAX_FRAME_MS = 100; - /** * The replay's own subscription passes every recorded group through: the recorder's relay already * applied a wire max age, so only the consumer's decisions are made again. @@ -92,6 +89,11 @@ export interface Options { * which `Sync` sets to the delay plus a lookahead a live viewer does not configure. */ delay: number; + /** + * How long the trace observed, from its first arrival, in ms. Rendering runs to here, so an + * outage after the last arrival is heard rather than cut off. + */ + duration: number; } /** One render quantum, as the worklet produced it. */ @@ -143,25 +145,25 @@ function post(rate: number, latency: number): Ring { const micros = (ms: number) => Math.round(ms * 1000) as Time.Micro; /** - * Samples each frame carries, by its timestamp: up to where the next one starts, so consecutive - * frames tile the timeline exactly. A frame followed by a hole, or by nothing, gets the trace's - * typical frame. + * Samples each frame carries, by its timestamp: the trace's typical frame, the median spacing. + * + * A frame within half a frame of that runs up to where the next one starts instead, so consecutive + * frames tile the timeline exactly despite rounding. Any wider spacing is a frame that never + * arrived, which stays missing audio rather than stretching the frame before it. */ function frameSamples(trace: Arrival[], rate: number): Map { const starts = [...new Set(trace.map((a) => a.timestamp))].sort((a, b) => a - b); const sample = (ms: number) => Math.round((ms * rate) / 1000); + const gaps = starts.slice(1).map((next, i) => sample(next) - sample(starts[i])); + const typical = [...gaps].sort((a, b) => a - b)[Math.floor(gaps.length / 2)]; + if (typical === undefined) throw new Error("a trace needs two frames"); + const spans = new Map(); - for (let i = 0; i < starts.length - 1; i++) { - if (starts[i + 1] - starts[i] < MAX_FRAME_MS) { - spans.set(micros(starts[i]), sample(starts[i + 1]) - sample(starts[i])); - } - } - const sorted = [...spans.values()].sort((a, b) => a - b); - const typical = sorted[Math.floor(sorted.length / 2)]; - if (typical === undefined) throw new Error("a trace needs two frames less than 100 ms apart"); - for (const start of starts) { - if (!spans.has(micros(start))) spans.set(micros(start), typical); - } + starts.forEach((start, i) => { + const gap = gaps[i]; + const tiles = gap !== undefined && Math.abs(gap - typical) * 2 <= typical; + spans.set(micros(start), tiles ? gap : typical); + }); return spans; } @@ -175,7 +177,7 @@ const settle = () => new Promise((resolve) => setImmediate(resolve)); /** * Play `trace` in simulated real time, yielding every render quantum from the first arrival to the - * last. + * end of the observation. * * A group's stream finishes with its last recorded frame. The audio is a constant, never zero, so * what the ring did not supply is exactly the zeros at the end of a quantum, which is how the @@ -185,6 +187,8 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato const first = trace[0]; const last = trace.at(-1); if (!first || !last) return; + const end = first.at + options.duration; + if (end < last.at) throw new Error(`an arrival at ${last.at} ms is past the ${options.duration} ms observed`); const { rate } = options; const samples = frameSamples(trace, rate); @@ -227,7 +231,7 @@ export async function* replay(trace: Arrival[], options: Options): AsyncGenerato let next = 0; try { - for (let n = 1; first.at + (n - 1) * step <= last.at; n++) { + for (let n = 1; first.at + (n - 1) * step <= end; n++) { // The quantum ending at `now` renders once everything that arrived by then is delivered. const now = first.at + n * step; const written = next; diff --git a/quest/m0/README.md b/quest/m0/README.md index 79d4d9bedb..2781ce6732 100644 --- a/quest/m0/README.md +++ b/quest/m0/README.md @@ -29,7 +29,7 @@ ingested-only view (`origin::Consumer::local()`) to localhost workers belongs to moq.pro's edge, which embeds moq-relay; it moved there on 2026-09-28. Audio playout: the jitter target replaces the round-trip guess. The browser -audio quality harness in `test/audio-quality/` has landed; it grades the target +audio quality harness in `test/audio-quality/` has landed; it grades playout nightly and records the traces it replays. Its native lane is a standalone m1 quest, since nothing here waits on it. The [A/V clock](/quest/m1/av-clock.md) moved to m1 in the 2026-09-30 audit: it waits on the whole jitter line and is diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md index 60bdb96da6..b79baa83b1 100644 --- a/test/audio-quality/README.md +++ b/test/audio-quality/README.md @@ -82,8 +82,8 @@ The shaper's profiles are synthetic. [`traces/`](traces) holds real arrivals, ea | `relay-mic` | `` in headless Chromium, publishing its fake capture device through the public relay | A trace is a [`Trace`](clients/js/src/schema.ts): the catalog's audio rendition, the smallest -PROBE round trip the session saw, and every frame's `[at, timestamp, group]`, with `at` on the -viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, the way +PROBE round trip the session saw, how long it observed, and every frame's `[at, timestamp, group]`, +with `at` on the viewer's clock. `record.ts` stamps a frame as it comes off its group's stream, the way `Container.Consumer` receives it, using only public `@moq/net` and `@moq/hang` in the same Chromium over WebTransport. It stamps before the consumer's in-order delivery, whose waits and skips depend on the delay, so the replay decides those again. @@ -92,7 +92,9 @@ on the delay, so the replay decides those again. clock ([`js/watch/src/audio/replay.ts`](../../js/watch/src/audio/replay.ts)), at the "auto" delay a real `Sync` resolves for the recorded catalog and round trip, and reads each quantum through the tap's classifier. The consumer makes the player's group ordering, max age skips, and discontinuity -resets again at that delay; a group's stream is taken to finish with its last recorded frame. It +resets again at that delay; a group's stream is taken to finish with its last recorded frame. Every +frame is the trace's median spacing long, so a frame that never arrived stays missing audio, and +rendering runs to the end of the observation, so an outage after the last arrival is heard. It writes the same samples a page does, so `analyze.ts` reduces both alike, and the shaper, transport, and clock checks have nothing to void. Decoding is taken as instant. A replay is deterministic, so its budgets are exactly what it measured. diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 7a9bb8f4c3..3682697b25 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,5 +1,5 @@ { - "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. Updated 2026-09-29: mild and wide now use shared order-preserving Gaussian jitter; bursty and step are new. Those 16 rows were re-recorded for 60 s each, seed 7, on the same shared Linux workstation with concurrent bounded builds. Their ceilings use twice the measurement rounded upward to two significant figures and the same floors above; target_ms_p95 adds one 20 ms bucket. Other rows retain their previous measurements. These remain current-player regression ceilings, not target audio quality; re-record on the nightly runner before tightening them. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-09-29, with no headroom: any regression fails them, and an improvement should be tightened in.", + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs. Updated 2026-09-29: mild and wide now use shared order-preserving Gaussian jitter; bursty and step are new. Those 16 rows were re-recorded for 60 s each, seed 7, on the same shared Linux workstation with concurrent bounded builds. Their ceilings use twice the measurement rounded upward to two significant figures and the same floors above; target_ms_p95 adds one 20 ms bucket. Other rows retain their previous measurements. These remain current-player regression ceilings, not target audio quality; re-record on the nightly runner before tightening them. The replay rows are deterministic, so their ceilings are exactly what they measured on 2026-10-01, through the end of each recording, with no headroom: any regression fails them, and an improvement should be tightened in.", "rows": [ { "runtime": "chromium", @@ -343,10 +343,10 @@ "rate": 44100, "profile": "local-aac", "ring": "isolated", - "underruns_per_min": 103.7, - "underrun_ms_per_min": 293.2, + "underruns_per_min": 102.9, + "underrun_ms_per_min": 290.8, "skip_aheads_per_min": 2, - "discarded_ms_per_min": 94.4, + "discarded_ms_per_min": 93.6, "stalled_share": 0, "silence_share": 0.005, "target_ms_p95": 186 @@ -357,10 +357,10 @@ "rate": 44100, "profile": "local-aac", "ring": "plain", - "underruns_per_min": 103.7, - "underrun_ms_per_min": 293.2, + "underruns_per_min": 102.9, + "underrun_ms_per_min": 290.8, "skip_aheads_per_min": 2, - "discarded_ms_per_min": 94.4, + "discarded_ms_per_min": 93.6, "stalled_share": 0, "silence_share": 0.005, "target_ms_p95": 186 @@ -399,8 +399,8 @@ "rate": 48000, "profile": "relay-mic", "ring": "isolated", - "underruns_per_min": 227.8, - "underrun_ms_per_min": 593.2, + "underruns_per_min": 225.9, + "underrun_ms_per_min": 588.2, "skip_aheads_per_min": 0, "discarded_ms_per_min": 0, "stalled_share": 0, @@ -413,8 +413,8 @@ "rate": 48000, "profile": "relay-mic", "ring": "plain", - "underruns_per_min": 227.8, - "underrun_ms_per_min": 593.2, + "underruns_per_min": 225.9, + "underrun_ms_per_min": 588.2, "skip_aheads_per_min": 0, "discarded_ms_per_min": 0, "stalled_share": 0, diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts index e640a584f9..4d18ea295c 100644 --- a/test/audio-quality/clients/js/driver.ts +++ b/test/audio-quality/clients/js/driver.ts @@ -147,6 +147,9 @@ try { await sleep(Math.min(DRAIN_MS, end - Date.now())); await drain(page); } + // A run that ends inside a gap still counts it. + await page.evaluate(() => globalThis.audioQuality.finish()); + await drain(page); // The context rate is only known once the graph exists, which is after the catalog. environment = (await page.evaluate(() => globalThis.audioQuality.environment())) ?? environment; notes = await page.evaluate(() => globalThis.audioQuality.notes()); diff --git a/test/audio-quality/clients/js/record.ts b/test/audio-quality/clients/js/record.ts index ca4f6008f5..7c710de081 100644 --- a/test/audio-quality/clients/js/record.ts +++ b/test/audio-quality/clients/js/record.ts @@ -118,12 +118,13 @@ try { const trimmed = arrivals.filter(([at]) => at - start <= durationMs); const round = (x: number, places: number) => Math.round(x * 10 ** places) / 10 ** places; const trace: Trace = { - version: 1, + version: 2, source, description, transport: info.transport, rtt: rtt === null ? null : round(rtt, 1), config: info.config as Trace["config"], + duration: durationMs, arrivals: trimmed.map(([at, timestamp, group]) => [round(at - start, 2), round(timestamp, 3), group]), }; // One arrival per line, so the file reads as a table. diff --git a/test/audio-quality/clients/js/replay.ts b/test/audio-quality/clients/js/replay.ts index 28e2d56a6d..9b07265862 100644 --- a/test/audio-quality/clients/js/replay.ts +++ b/test/audio-quality/clients/js/replay.ts @@ -72,7 +72,7 @@ for (const [name, list, known] of selectors) { const rows: { row: Row; trace: Trace }[] = []; for (const { profile, trace } of traces) { - if (trace.version !== 1) throw new Error(`${profile}: trace version ${trace.version}, expected 1`); + if (trace.version !== 2) throw new Error(`${profile}: trace version ${trace.version}, expected 2`); const codec = traceCodec(trace); if (profiles && !profiles.includes(profile)) continue; if (!codecs.includes(codec)) continue; @@ -114,7 +114,29 @@ for (const { row, trace } of rows) { }; const arrivals = trace.arrivals.map(([at, timestamp, group]) => ({ at, timestamp, group })); - const quanta = replay(arrivals, { ring: row.ring === "isolated" ? "shared" : "post", rate, delay }); + const sample = (quantum: { at: number; timestamp: number; stalled: boolean }) => { + const gaps = ledger.take(); + samples.push({ + at: quantum.at, + render: quantum.at, + timestamp: quantum.timestamp, + stalled: quantum.stalled, + delay, + quanta: ledger.counts.quanta, + quiet: ledger.counts.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: stalls.length > 0 ? stalls : undefined, + }); + stalls = []; + }; + + const quanta = replay(arrivals, { + ring: row.ring === "isolated" ? "shared" : "post", + rate, + delay, + duration: trace.duration, + }); + let last: { at: number; timestamp: number; stalled: boolean } | undefined; for await (const quantum of quanta) { // The render clock is the simulated one: the trace's `at`, which starts at zero. ledger.add(frame, quantum.output.length, classify([quantum.output])); @@ -123,24 +145,16 @@ for (const { row, trace } of rows) { stalled = quantum.stalled; stalls.push({ at: quantum.at, stalled }); } + last = { at: quantum.at, timestamp: quantum.timestamp, stalled: quantum.stalled }; if (quantum.at >= next) { next += SAMPLE_INTERVAL_MS; - const gaps = ledger.take(); - samples.push({ - at: quantum.at, - render: quantum.at, - timestamp: quantum.timestamp, - stalled: quantum.stalled, - delay, - quanta: ledger.counts.quanta, - quiet: ledger.counts.quiet, - gaps: gaps.length > 0 ? gaps : undefined, - stalls: stalls.length > 0 ? stalls : undefined, - }); - stalls = []; + sample(last); } } + // A trace that ends inside a gap still counts it. + ledger.finish(); + if (last) sample(last); console.warn = warn; const environment: Environment = { diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts index 2245170d70..195f3b852a 100644 --- a/test/audio-quality/clients/js/src/probe.ts +++ b/test/audio-quality/clients/js/src/probe.ts @@ -18,12 +18,15 @@ import { type Tap, tap } from "./tap.ts"; type RenderCapacity = { start(options: { updateInterval: number }): void; addEventListener(type: "update", listener: (event: { averageLoad: number }) => void): void; + removeEventListener(type: "update", listener: (event: { averageLoad: number }) => void): void; }; /** What the probe has collected, until someone drains it. */ export type Probe = { /** Samples taken since the last drain. */ drain(): Sample[]; + /** End the measurement: take one last sample, with any gap still open closed into it. */ + finish(): Promise; /** What the page is, once the catalog says the session is up. */ environment(): Environment | undefined; /** Console warnings and errors so far. */ @@ -55,6 +58,7 @@ export function probe(watch: MoqWatch): Probe { let tapped: AudioNode | undefined; let current: Tap | undefined; let renderLoad: number | undefined; + let unlisten: (() => void) | undefined; const attach = () => { const root = audio.out.root.peek(); if (!root || root === tapped) return; @@ -70,14 +74,19 @@ export function probe(watch: MoqWatch): Probe { (err: unknown) => note(`tap: ${err instanceof Error ? err.message : String(err)}`), ); + // The previous root's context stops reporting into this one's reading. + unlisten?.(); + unlisten = undefined; const capacity = (root.context as unknown as { renderCapacity?: RenderCapacity }).renderCapacity; if (!capacity) { note("renderCapacity: unavailable, render_load will be null"); return; } - capacity.addEventListener("update", (event) => { + const update = (event: { averageLoad: number }) => { renderLoad = event.averageLoad; - }); + }; + capacity.addEventListener("update", update); + unlisten = () => capacity.removeEventListener("update", update); // A whole second: shorter intervals are refused by some builds, and load is smooth anyway. capacity.start({ updateInterval: 1 }); }; @@ -119,6 +128,11 @@ export function probe(watch: MoqWatch): Probe { return { drain: () => samples.splice(0, samples.length), + async finish() { + attach(); + await current?.finish(); + samples.push(sample()); + }, environment() { const catalog = broadcast.out.catalog.peek(); if (!catalog) return undefined; diff --git a/test/audio-quality/clients/js/src/quantum.test.ts b/test/audio-quality/clients/js/src/quantum.test.ts index adf9d50d07..7fb5444889 100644 --- a/test/audio-quality/clients/js/src/quantum.test.ts +++ b/test/audio-quality/clients/js/src/quantum.test.ts @@ -58,4 +58,18 @@ describe("Ledger", () => { expect(gap).toMatchObject({ quanta: 3, short: 2 }); expect(ledger.counts).toEqual({ quanta: 5, quiet: 1 }); }); + + test("a run that ends inside a gap still reports it once finished", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, full); + ledger.add(128, 128, silent); + ledger.add(256, 128, silent); + expect(ledger.take()).toEqual([]); + + ledger.finish(); + const [gap, ...rest] = ledger.take(); + expect(rest).toEqual([]); + expect(gap?.ms).toBeCloseTo((256 / RATE) * 1000); + expect(gap).toMatchObject({ quanta: 2, short: 0 }); + }); }); diff --git a/test/audio-quality/clients/js/src/quantum.ts b/test/audio-quality/clients/js/src/quantum.ts index e28ef166a3..a3cb38d3b8 100644 --- a/test/audio-quality/clients/js/src/quantum.ts +++ b/test/audio-quality/clients/js/src/quantum.ts @@ -102,6 +102,11 @@ export class Ledger { return this.#closed.splice(0, this.#closed.length); } + /** Close a gap still open, for the next {@link take}: the measurement ended inside it. */ + finish(): void { + this.#close(); + } + #close(): void { if (!this.#open) return; const ms = (samples: number) => (samples / this.#rate) * 1000; diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts index ca17583f6c..6f65d35b60 100644 --- a/test/audio-quality/clients/js/src/schema.ts +++ b/test/audio-quality/clients/js/src/schema.ts @@ -294,7 +294,7 @@ export type Arrival = [at: Ms, timestamp: Ms, group: number]; /** A recorded arrival trace, as checked in under `traces/`. The file name is its profile. */ export type Trace = { /** Bumped when a field's meaning changes. */ - version: 1; + version: 2; /** Where it was recorded: the relay, the broadcast, its publisher, the date. */ source: string; /** The shape it carries, and why it is kept. */ @@ -313,6 +313,8 @@ export type Trace = { delay?: number; [key: string]: unknown; }; + /** How long the recording observed, from the first arrival: silence after the last one is an outage. */ + duration: Ms; /** Every arrival in order, `at` counted from the first. */ arrivals: Arrival[]; }; diff --git a/test/audio-quality/clients/js/src/tap-worklet.ts b/test/audio-quality/clients/js/src/tap-worklet.ts index e5a02245fb..fd770c457a 100644 --- a/test/audio-quality/clients/js/src/tap-worklet.ts +++ b/test/audio-quality/clients/js/src/tap-worklet.ts @@ -21,17 +21,24 @@ class Tap extends AudioWorkletProcessor { super(); const init = options.processorOptions as TapInit; this.#ledger = new Ledger(sampleRate, init.floor); + // The run is over: close a gap still open and report at once, rather than lose it. + this.port.onmessage = () => { + this.#ledger.finish(); + this.#report(); + }; + } + + #report(): void { + this.#since = 0; + const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take() }; + this.port.postMessage(report); } process(inputs: Float32Array[][]): boolean { const input = inputs[0] ?? []; this.#ledger.add(currentFrame, input[0]?.length ?? 128, classify(input)); - if (++this.#since >= REPORT_EVERY) { - this.#since = 0; - const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take() }; - this.port.postMessage(report); - } + if (++this.#since >= REPORT_EVERY) this.#report(); return true; } } diff --git a/test/audio-quality/clients/js/src/tap.ts b/test/audio-quality/clients/js/src/tap.ts index 7d0ac4dcc4..f2909fe7a9 100644 --- a/test/audio-quality/clients/js/src/tap.ts +++ b/test/audio-quality/clients/js/src/tap.ts @@ -31,6 +31,8 @@ export type Tap = { counts(): Counts | undefined; /** Gaps closed since the last call. */ take(): Gap[]; + /** Close a gap still open, and resolve once its report is in for {@link take}. */ + finish(): Promise; close(): void; }; @@ -52,14 +54,24 @@ export async function tap(root: AudioNode, init: TapInit): Promise { let counts: Counts | undefined; const gaps: Gap[] = []; + let reported: (() => void) | undefined; node.port.onmessage = (event: MessageEvent) => { counts = event.data.counts; gaps.push(...event.data.gaps); + reported?.(); }; return { counts: () => counts, take: () => gaps.splice(0, gaps.length), + finish() { + // The port is ordered, so the report answering this one is the last to resolve it. + const done = new Promise((resolve) => { + reported = resolve; + }); + node.port.postMessage("finish"); + return done; + }, close() { node.port.onmessage = null; root.disconnect(node); diff --git a/test/audio-quality/traces/local-aac.json b/test/audio-quality/traces/local-aac.json index 807d609bd7..13e0221589 100644 --- a/test/audio-quality/traces/local-aac.json +++ b/test/audio-quality/traces/local-aac.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "source": "a local moq-relay on 127.0.0.1, ffmpeg's AAC sine over MPEG-TS through moq import ts, recorded 2026-09-29", "description": "The shallow control: the audio quality harness's AAC publisher with nothing but loopback between it and the viewer.", "transport": "webtransport", @@ -14,6 +14,7 @@ "numberOfChannels": 2, "jitter": 163 }, + "duration": 35000, "arrivals": [ [0,10016.133,0], [2.55,10039.344,1], diff --git a/test/audio-quality/traces/relay-bbb.json b/test/audio-quality/traces/relay-bbb.json index b18b143ab6..fd605e36af 100644 --- a/test/audio-quality/traces/relay-bbb.json +++ b/test/audio-quality/traces/relay-bbb.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "source": "https://cdn.moq.dev/demo bbb.hang, the public demo's MPEG-TS import, recorded 2026-09-29", "description": "The public relay's Big Buck Bunny: one AAC frame per group, flushed in bursts of about seven, the flush-span shape #3477 measured.", "transport": "webtransport", @@ -15,6 +15,7 @@ "bitrate": 139613, "jitter": 186 }, + "duration": 35000, "arrivals": [ [0,234535,9642], [2.98,240014.922,9878], diff --git a/test/audio-quality/traces/relay-mic.json b/test/audio-quality/traces/relay-mic.json index 3bb4d93735..05a9987c25 100644 --- a/test/audio-quality/traces/relay-mic.json +++ b/test/audio-quality/traces/relay-mic.json @@ -1,5 +1,5 @@ { - "version": 1, + "version": 2, "source": "https://cdn.moq.dev/anon, in headless Chromium capturing its fake audio device, recorded 2026-09-29", "description": "A browser publisher through the public relay: getUserMedia, the capture worklet, and WebCodecs Opus, with the relay's round trip in between.", "transport": "webtransport", @@ -13,6 +13,7 @@ "numberOfChannels": 2, "bitrate": 64000 }, + "duration": 35000, "arrivals": [ [0,2324.745,0], [16.53,2344.745,1], From 7c230c6b8656ecbfe3b967adc98428f4f78b79b8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 1 Oct 2026 14:51:10 -0700 Subject: [PATCH 27/27] fix(audio-quality): resolve the tap's finish only on its own answer A periodic report already in flight resolved finish() before the worklet closed the open gap, so the final sample could still miss it. The finish answer is now marked and only it resolves. Also hand exact replay frame lengths to the jitter target's watch quest, which brings the codec frame duration to the container. Co-Authored-By: Claude Opus 5.5 --- quest/m0/audio-jitter-target/watch.md | 5 ++++- test/audio-quality/clients/js/src/tap-worklet.ts | 6 +++--- test/audio-quality/clients/js/src/tap.ts | 6 ++++-- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index dbb390c750..ea2f8c97c5 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -83,7 +83,10 @@ against it is likely cheaper than patching the branch's: until then), and tighten the harness's exact replay budgets with it. The checked-in `test/audio-quality/budgets.json` was recorded locally (#4426); re-record it from the nightly runner's first runs on `main` before - tightening, since nightly only runs `main`'s code. + tightening, since nightly only runs `main`'s code. The replay sizes every + frame as the trace's median spacing, which a trace missing most of its + frames defeats; once the container knows the codec's frame duration, record + it in the trace and replay that instead. - Manual run against the public relay on Chrome and Safari, the two rows the issue measured. Re-record the `relay-mic` trace in the same run with a real microphone (`just test audio-quality-record`); the checked-in one used diff --git a/test/audio-quality/clients/js/src/tap-worklet.ts b/test/audio-quality/clients/js/src/tap-worklet.ts index fd770c457a..81e1a9d18b 100644 --- a/test/audio-quality/clients/js/src/tap-worklet.ts +++ b/test/audio-quality/clients/js/src/tap-worklet.ts @@ -24,13 +24,13 @@ class Tap extends AudioWorkletProcessor { // The run is over: close a gap still open and report at once, rather than lose it. this.port.onmessage = () => { this.#ledger.finish(); - this.#report(); + this.#report(true); }; } - #report(): void { + #report(finished = false): void { this.#since = 0; - const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take() }; + const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take(), finished }; this.port.postMessage(report); } diff --git a/test/audio-quality/clients/js/src/tap.ts b/test/audio-quality/clients/js/src/tap.ts index f2909fe7a9..ef0d3e5d31 100644 --- a/test/audio-quality/clients/js/src/tap.ts +++ b/test/audio-quality/clients/js/src/tap.ts @@ -23,6 +23,8 @@ export type TapReport = { counts: Counts; /** Gaps closed since the last report. */ gaps: Gap[]; + /** Whether this answers a finish, so no gap is still open. */ + finished: boolean; }; /** A running tap. */ @@ -58,14 +60,14 @@ export async function tap(root: AudioNode, init: TapInit): Promise { node.port.onmessage = (event: MessageEvent) => { counts = event.data.counts; gaps.push(...event.data.gaps); - reported?.(); + // A periodic report already in flight is not the answer, and may still miss the open gap. + if (event.data.finished) reported?.(); }; return { counts: () => counts, take: () => gaps.splice(0, gaps.length), finish() { - // The port is ordered, so the report answering this one is the last to resolve it. const done = new Promise((resolve) => { reported = resolve; });