From 823a42e3d5bcca97ac0ebbfa01ea49014f1aba54 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 13:27:15 -0700 Subject: [PATCH 1/4] quest(audio-quality-harness/browser): claim Co-Authored-By: Claude Opus 5.5 From ae5d52a8960ca3ed8ddcd3c6875f74d2b857ed24 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 16:15:24 -0700 Subject: [PATCH 2/4] test(audio-quality): grade browser audio playout over an impaired path Upstreams the browser audio quality lane from fperex's fork (fperex/moq, debug-findings-solution) as test/audio-quality/, without the fork's player, estimator, or shaper changes. Plays a tone in headless Chromium through moq-shaper and grades underruns, skip-aheads, silence, and the resolved delay against budgets.json. Runs nightly under --enforce. Co-Authored-By: fperex <270788582+fperex@users.noreply.github.com> Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 57 +++ bun.lock | 68 +++ package.json | 1 + quest/m0/audio-jitter-target/watch.md | 6 +- quest/m0/audio-quality-harness/README.md | 18 +- quest/m0/audio-quality-harness/browser.md | 74 ---- .../audio-quality-harness/shaper-profiles.md | 34 ++ quest/m0/audio-quality-harness/traces.md | 33 ++ quest/m1/audio-quality-native.md | 7 +- test/README.md | 3 +- test/audio-quality/README.md | 183 ++++++++ test/audio-quality/budgets.json | 4 + test/audio-quality/clients/js/analyze.ts | 116 +++++ test/audio-quality/clients/js/driver.ts | 160 +++++++ test/audio-quality/clients/js/grade.ts | 102 +++++ test/audio-quality/clients/js/index.html | 10 + test/audio-quality/clients/js/package.json | 18 + .../clients/js/src/analyze.test.ts | 132 ++++++ test/audio-quality/clients/js/src/analyze.ts | 305 +++++++++++++ test/audio-quality/clients/js/src/page.ts | 43 ++ test/audio-quality/clients/js/src/probe.ts | 139 ++++++ .../clients/js/src/quantum.test.ts | 61 +++ test/audio-quality/clients/js/src/quantum.ts | 112 +++++ test/audio-quality/clients/js/src/schema.ts | 407 ++++++++++++++++++ .../clients/js/src/tap-worklet.ts | 39 ++ test/audio-quality/clients/js/src/tap.ts | 70 +++ test/audio-quality/clients/js/tsconfig.json | 7 + test/audio-quality/clients/js/vite.config.ts | 19 + test/audio-quality/relay.toml | 22 + test/audio-quality/run.sh | 298 +++++++++++++ test/justfile | 10 + 31 files changed, 2468 insertions(+), 90 deletions(-) delete mode 100644 quest/m0/audio-quality-harness/browser.md create mode 100644 quest/m0/audio-quality-harness/shaper-profiles.md create mode 100644 quest/m0/audio-quality-harness/traces.md create mode 100644 test/audio-quality/README.md create mode 100644 test/audio-quality/budgets.json create mode 100644 test/audio-quality/clients/js/analyze.ts create mode 100644 test/audio-quality/clients/js/driver.ts create mode 100644 test/audio-quality/clients/js/grade.ts create mode 100644 test/audio-quality/clients/js/index.html create mode 100644 test/audio-quality/clients/js/package.json create mode 100644 test/audio-quality/clients/js/src/analyze.test.ts create mode 100644 test/audio-quality/clients/js/src/analyze.ts create mode 100644 test/audio-quality/clients/js/src/page.ts create mode 100644 test/audio-quality/clients/js/src/probe.ts create mode 100644 test/audio-quality/clients/js/src/quantum.test.ts create mode 100644 test/audio-quality/clients/js/src/quantum.ts create mode 100644 test/audio-quality/clients/js/src/schema.ts create mode 100644 test/audio-quality/clients/js/src/tap-worklet.ts create mode 100644 test/audio-quality/clients/js/src/tap.ts create mode 100644 test/audio-quality/clients/js/tsconfig.json create mode 100644 test/audio-quality/clients/js/vite.config.ts create mode 100644 test/audio-quality/relay.toml create mode 100755 test/audio-quality/run.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 231fa33aec..2760eb02a5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,6 +18,13 @@ name: Nightly # the wheel builds in a manylinux container, the Kotlin/Swift/Go/Dart packages # link the FFI on every platform) and nothing notices until the tag is pushed. # +# `audio-quality` is here because of what it measures. It plays sixteen rows of +# audio in real time, a minute each, which no merge gate should carry, and every +# number it grades is a timing one that moves with whatever else the runner is +# doing. A per-PR arm would fail on unrelated load often enough to be ignored; +# nightly it competes with nothing, and a regression surfaces a day late against +# a budget that still means something. +# # A failure here lands on main rather than being caught in review, which is the # accepted trade. Anything that must block a merge belongs in check.yml. @@ -176,6 +183,56 @@ jobs: NEXTEST_PROFILE: ci MOQ_STRICT: 1 + # Plays the matrix in headless Chromium through moq-shaper and grades what the + # listener heard against test/audio-quality/budgets.json. x86 rather than arm, + # like interop.yml and wasm.yml, because that is where Playwright's Chromium runs. + audio-quality: + name: Audio quality + runs-on: ubuntu-latest + # Sixteen rows at a minute each, plus the relay, CLI, and shaper built from source. + timeout-minutes: 60 + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + steps: + - name: Free disk space + uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main + with: + tool-cache: false + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + - name: Rust cache + uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + cache-on-failure: true + # Playwright's Chromium isn't in the nix devShell; `--with-deps` installs + # its apt runtime libs, and run.sh's own install then no-ops. + - name: Playwright Chromium + run: nix develop --command bash -c "bun install --frozen-lockfile && bunx playwright install --with-deps chromium" + shell: bash -leo pipefail {0} + - name: Audio quality + run: nix develop --command just test audio-quality --enforce + shell: bash -leo pipefail {0} + # The run directory is the diagnosis: each process's log, the shaper's + # counters, the raw samples, and every row's summary. The harness keeps it + # on a failure without being asked. + - name: Keep the failing run + if: ${{ failure() }} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: audio-quality-run + path: ${{ runner.temp }}/moq-test-runs/ + retention-days: 7 + if-no-files-found: ignore + # Every tag-driven release workflow, run against main with publishing skipped: # the workflows gate their publish jobs on `github.ref_type == 'tag'`, and no # secrets are passed, so a dry-run cannot publish even by accident. diff --git a/bun.lock b/bun.lock index 661ebc2605..7ce6980b9c 100644 --- a/bun.lock +++ b/bun.lock @@ -343,6 +343,18 @@ "vite": "^8.3.0", }, }, + "test/audio-quality/clients/js": { + "name": "@moq/audio-quality", + "dependencies": { + "@moq/watch": "workspace:*", + }, + "devDependencies": { + "@types/audioworklet": "^0.0.100", + "esbuild": "^0.28.2", + "playwright": "^1.63.0", + "vite": "^8.3.0", + }, + }, "test/interop/clients/js": { "name": "@moq/interop-browser", "dependencies": { @@ -640,6 +652,8 @@ "@libav.js/variant-opus-af": ["@libav.js/variant-opus-af@6.10.9", "", {}, "sha512-yULJUCQIHDtwDPbT38t9pP/7EGgqwynVcOau46hZjtCe8TWMp67DTi2X0Emr2M4Ci3nzxNPf25IzO+IXSmHDBg=="], + "@moq/audio-quality": ["@moq/audio-quality@workspace:test/audio-quality/clients/js"], + "@moq/auth": ["@moq/auth@workspace:js/auth"], "@moq/binary": ["@moq/binary@workspace:js/binary"], @@ -1996,6 +2010,8 @@ "@types/concat-stream/@types/node": ["@types/node@26.0.1", "", { "dependencies": { "undici-types": "~8.3.0" } }, "sha512-fc3KiUoBt6kie0N9bIW3E47vZsuaMf0PM2AaUpLCLT0s/LvX1nxAim6Fc049cNxODPpGm6qRAuUOB86SkRuPQw=="], + "@vitejs/plugin-vue/vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], + "@vue/compiler-core/@babel/parser": ["@babel/parser@7.29.7", "", { "dependencies": { "@babel/types": "^7.29.7" }, "bin": "./bin/babel-parser.js" }, "sha512-hnORnjP/1P/zFEndoeX+n+t1RwWRJiJpM/jO7FW32Kn9r5+sJB2JWOdYo4L6k78j15eCwY3Gm/7364B1EMwtNg=="], "@vue/compiler-core/entities": ["entities@7.0.1", "", {}, "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA=="], @@ -2172,6 +2188,10 @@ "@types/concat-stream/@types/node/undici-types": ["undici-types@8.3.0", "", {}, "sha512-j375ScV60dom+YkPFIfTLcOiPxkN/buHz5GobjLhixFuANaNs3C9l4GmrWqejgXWJ7BbJcFYpTEUkS1Ge8bpZQ=="], + "@vitejs/plugin-vue/vite/esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], + + "@vitejs/plugin-vue/vite/postcss": ["postcss@8.5.16", "", { "dependencies": { "nanoid": "^3.3.12", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-vuwillviilfKZsg0VGj5R/YwwcHx4SLsIOI/7K6mQkWx+l5cUHTjj5g0AasTBcyXsbfTgrwsUNmVUb5xVwyPwg=="], + "@vue/compiler-core/@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], "@vue/compiler-sfc/@babel/parser/@babel/types": ["@babel/types@7.29.7", "", { "dependencies": { "@babel/helper-string-parser": "^7.29.7", "@babel/helper-validator-identifier": "^7.29.7" } }, "sha512-4zBIxpPzowiZpusoFkyGVwakdRJUyuH5PxQ/PrqghfdFWWasvnCdPfQXHrenDai+gyLARulZjZowCOj6fjT4pA=="], @@ -2282,6 +2302,54 @@ "@npmcli/package-json/glob/path-scurry/lru-cache": ["lru-cache@10.4.3", "", {}, "sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ=="], + "@vitejs/plugin-vue/vite/esbuild/@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="], + + "@vitejs/plugin-vue/vite/esbuild/@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="], + + "@vitejs/plugin-vue/vite/postcss/nanoid": ["nanoid@3.3.15", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-y7Wygv/7mEOvxTuEQDB8StXdMRBWf1kR/tlhAzBRUFkB2jfcLOAxO/SHmOO2zgz1pVgK29/kyupn059/bCHdjA=="], + "@vue/compiler-core/@babel/parser/@babel/types/@babel/helper-string-parser": ["@babel/helper-string-parser@7.29.7", "", {}, "sha512-Pb5ijPrZ89GDH8223L4UP8i6QApWxs04RbPQJTeWDV0/keR2E36MeKnyr6LYmUUvqRRI+Iv87SuF1W6ErINzYw=="], "@vue/compiler-core/@babel/parser/@babel/types/@babel/helper-validator-identifier": ["@babel/helper-validator-identifier@7.29.7", "", {}, "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg=="], diff --git a/package.json b/package.json index 3a85e66642..0cae436c5b 100644 --- a/package.json +++ b/package.json @@ -46,6 +46,7 @@ "demo/boy", "demo/web", "js/moq-boy", + "test/audio-quality/clients/js", "test/interop/clients/js", "test/interop/clients/js-native", "test/wasm" diff --git a/quest/m0/audio-jitter-target/watch.md b/quest/m0/audio-jitter-target/watch.md index a1a258f61e..75d43a4203 100644 --- a/quest/m0/audio-jitter-target/watch.md +++ b/quest/m0/audio-jitter-target/watch.md @@ -75,8 +75,8 @@ against it is likely cheaper than patching the branch's: that down: a stored preference silently winning over the default is its own bug, and it also means auto gets far less real exposure than it looks like. - Replay recorded traces rather than synthetic ones of the same shape. The - #3477 traces are gone, so record fresh ones with the [browser - harness](/quest/m0/audio-quality-harness/browser.md) (decided with the + #3477 traces are gone, so record fresh ones with the [audio quality + harness](/quest/m0/audio-quality-harness/traces.md) (decided with the maintainer instead of asking the reporter). Trim a copy into the repository and replay it through both rings in `replay.test.ts`. - Manual run against the public relay on Chrome and Safari, the two rows the @@ -97,7 +97,7 @@ that quest can adopt it without a second estimator change. ## Required -- [Browser harness](/quest/m0/audio-quality-harness/browser.md) - records the arrival traces this quest replays +- [Traces](/quest/m0/audio-quality-harness/traces.md) - records the arrival traces this quest replays ## Related diff --git a/quest/m0/audio-quality-harness/README.md b/quest/m0/audio-quality-harness/README.md index 457844c109..e3779f51ba 100644 --- a/quest/m0/audio-quality-harness/README.md +++ b/quest/m0/audio-quality-harness/README.md @@ -23,14 +23,13 @@ are. The native lane moved to m1 as a standalone quest (decided in the same budgets and metric schema so the two implementations can be compared rather than merely both passing. -The starting point is not a blank page. The reporter on #3477 already built a -working browser lane on their fork (`fperex/moq`, branch -`debug-findings-solution`, under `test/audio-quality/`): a Playwright-driven -matrix over `moq-shaper`, a budget file graded under `--enforce`, a nightly -job, and a replay runtime. Upstream that rather than reinventing it. The raw traces it shipped with are gone, so -this harness records fresh ones, and the [jitter target's watch -quest](/quest/m0/audio-jitter-target/watch.md) replays them (decided with the -maintainer during the merged-PR audit). +The browser lane has landed as `test/audio-quality/`, upstreamed from the +reporter's fork on #3477 (`fperex/moq`, branch `debug-findings-solution`) on +its own, without the fork's player, estimator, or shaper changes. That leaves +the two things that needed them: real arrival traces, which the [jitter +target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays, and the +bursty and step profiles, which need shaper features the fork has and `main` +does not. Jitter comes from the seeded userspace UDP shaper the transport drills run under (`rs/moq-shaper`, documented in `test/drill/README.md`), not from a fake @@ -49,7 +48,8 @@ each of those dimensions moves the expected floor. ## Required -- [Browser](/quest/m0/audio-quality-harness/browser.md) - upstream the fork's harness, grade it against a budget, run it nightly +- [Traces](/quest/m0/audio-quality-harness/traces.md) - real arrival traces recorded, checked in, and graded as replay rows +- [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and mid-run step profiles, once moq-shaper can express them ## Related diff --git a/quest/m0/audio-quality-harness/browser.md b/quest/m0/audio-quality-harness/browser.md deleted file mode 100644 index a47aed7969..0000000000 --- a/quest/m0/audio-quality-harness/browser.md +++ /dev/null @@ -1,74 +0,0 @@ -# [L] The browser audio lane is graded against a budget, nightly - -## Goal - -`just test audio-quality` plays a broadcast in a headless browser over a seeded, -impaired path and prints a graded table: underruns, short quanta, discarded -samples, skip-aheads, and where the end-to-end delay went. It fails when a -profile exceeds its budget. A nightly job runs the full matrix and keeps the -run directory of a failure. - -The same run works on both ring paths, isolated and postMessage, since the -production path is the one without cross-origin isolation. - -## Plan - -Upstream the reporter's lane from `fperex/moq` branch -`debug-findings-solution` rather than rebuilding it, keeping the attribution. -It already has a built `test/audio-quality/` lane: `just test audio-quality` -over `moq-shaper`, a Playwright-driven Chromium matrix, a `budgets.json` -graded by `grade.ts` under `--enforce`, a nightly job that keeps a failure's -run directory, and `chromium`, `safari`, and `replay` runtimes. It also -carries player, estimator, and shaper changes (checked 2026-09-28: 222 commits -ahead of and 148 behind `main`), so land the lane on its own and hold its -schema to the contract below rather than taking the branch wholesale. - -- Land the lane under `test/`, alongside the existing `interop` and `drill` - lanes, wired into the `justfile` the way they are. -- Keep the instrumentation ad-hoc for now. The probes stay a debug surface, not - public API; promoting them is [Latency - ledger](/quest/m2/latency-ledger.md), which nothing here waits on. -- The metric schema is the deliverable that outlives this quest, because the - native lane and the ledger both have to emit the same thing. It is a - contract, so write it as one: - - Every counter defined, not just the two that are obvious. An underrun is a - quantum the ring could only partly fill; a skip-ahead is a re-anchor that - discarded buffered audio; short quanta and discarded samples need the same - treatment rather than being left to the reader. - - Units and clock domain stated once. Durations in one unit, and every - timestamp on a named clock, so a browser value and a native value are the - same measurement. Naming the clock is not enough on its own: the stages span - the publisher, the relay and the viewer, which are separate processes and - often separate machines, so either reduce every timestamp to one common - monotonic reference or define the calibration and drift correction that maps - between them. The ledger's sum-to-end-to-end identity is arithmetic across - these values, and it is meaningless if they sit on unaligned clocks. - - Aggregation stated per metric: a count over the run, a max, or a - percentile. "Underruns: 3" and "underruns: 3/s" grade differently. - - Stages as exclusive, non-overlapping spans (capture, encode, publish flush, - network, jitter buffer, decode, render). The ledger's sum-to-end-to-end - identity is unimplementable if two stages can claim the same milliseconds, - and an unaccounted remainder is the finding, so give it a name too. -- Run the `moq-shaper` binary (`rs/moq-shaper`, see `test/drill/README.md`) - in front of the relay, so this harness and the drills share one impairment - implementation. It exits nonzero when the profile never acted, which the - harness must treat as a failed run: a profile that silently did nothing - turns an impaired run into an unimpaired pass. -- Profiles: near-zero, mild, bursty (the flush-span shape from #3477), and a - step change that forces the target to move mid-run. Fixed seeds, recorded - with the results. -- One profile runs a fixed delay preset rather than auto, as the control. Every - other profile exercises adaptation, so without this the whole matrix can pass - while a fixed preset regresses, and fixed presets are what a viewer lands on - today. -- Budgets in one checked-in file, keyed by the whole matrix row and not by - profile alone: runtime, codec, sample rate, profile, and the ring path, since - each of those moves the expected floor. A profile-only key silently grades - one row against another's threshold. Tightening a budget is then a visible - diff and loosening one needs a reason in review. -- Record real arrival traces (the #3477 release traces are gone), trim them - into a fixture, and replay them too, so a real recorded arrival pattern is - graded next to the synthetic profiles. The jitter target's watch quest - replays the same fixture. -- Add the lane to `nightly.yml`, and extend its header comment with why this - one is not a PR gate. diff --git a/quest/m0/audio-quality-harness/shaper-profiles.md b/quest/m0/audio-quality-harness/shaper-profiles.md new file mode 100644 index 0000000000..71536c518d --- /dev/null +++ b/quest/m0/audio-quality-harness/shaper-profiles.md @@ -0,0 +1,34 @@ +# [M] The audio quality matrix grades a bursty path and a mid-run step + +## Goal + +`just test audio-quality` gains the two profiles `moq-shaper` cannot express +today: `bursty`, which releases datagrams in batches (the flush-span shape from +#3477, seven datagrams per 160 ms window), and `step`, which changes the +profile part-way through a row and forces the adaptive target to move. Both get +budgets in `test/audio-quality/budgets.json` and run nightly with the rest. + +## Plan + +The browser lane landed on the shaper as it is: one uniform profile both ways, +fixed for the life of the process. Its `mild` and `wide` profiles spread +arrivals with uniform jitter, which also reorders datagrams whenever the +jitter exceeds their spacing, so `wide` measures reordering as much as spread. + +The reporter's fork (`fperex/moq`, branch `debug-findings-solution`, +`rs/moq-shaper`) already has what is missing, as separate commits: batches that +release datagrams together, steps that change a profile mid-run, an +order-preserving gaussian jitter model, named profiles loaded from TOML, and a +JSON report of the counters. Upstream the batch, step, and order-preserving +jitter changes with the fork author's credit, keeping each behind the options +the drills do not set so their lane is unchanged. Skip TCP passthrough: the +harness answers the page's certificate fetch itself. + +Then add `bursty` and `step` to `run.sh`, refuse a `--duration` that ends +before the step, and switch `mild` and `wide` to the order-preserving jitter so +their names mean spread alone. Re-record the budgets for every row whose profile +changed. + +## Related + +- [Audio jitter target](/quest/m0/audio-jitter-target/README.md) - the estimator the step profile exercises diff --git a/quest/m0/audio-quality-harness/traces.md b/quest/m0/audio-quality-harness/traces.md new file mode 100644 index 0000000000..ce0b749962 --- /dev/null +++ b/quest/m0/audio-quality-harness/traces.md @@ -0,0 +1,33 @@ +# [M] Real arrival traces are recorded, checked in, and graded + +## Goal + +A trimmed fixture of real audio arrival traces lives in the repository, and the +audio quality harness grades a replay of it next to its synthetic profiles. The +[jitter target's watch quest](/quest/m0/audio-jitter-target/watch.md) replays +the same fixture through both rings in `replay.test.ts`. + +## Plan + +The #3477 traces are gone, so record fresh ones: a viewer on the public relay +playing `bbb.hang` (the AAC flush-span shape the issue measured), a local relay +as the shallow control, and a browser microphone published through the public +relay. Stamp each audio frame's arrival where the container consumer receives +it, with its media timestamp and group, on the viewer's monotonic clock (see +`test/audio-quality/clients/js/src/schema.ts`). + +The browser lane reads only public signals, and no public signal carries a +per-frame arrival, so decide where the recorder lives: a debug hook in +`@moq/watch` that the harness page enables, or a native subscriber (`moq` +CLI) that records the same path's arrivals. The reporter's fork (`fperex/moq`, +branch `debug-findings-solution`) recorded from a debug sampler in the player +and replayed through `js/watch/src/audio/replay.ts` on a simulated clock; +that replay module is the likely shape for the grading side, and landing it is +a player change this quest owns. + +Replay rows are deterministic, so their budgets are exactly what was measured, +with no headroom. + +## Related + +- [Latency ledger](/quest/m2/latency-ledger.md) - would promote an arrival stamp into public API diff --git a/quest/m1/audio-quality-native.md b/quest/m1/audio-quality-native.md index ff4d16ab81..11373ae103 100644 --- a/quest/m1/audio-quality-native.md +++ b/quest/m1/audio-quality-native.md @@ -25,7 +25,8 @@ from whichever platform a user happened to be on. - `moq play` needs to emit the run's counters and stage timings as JSON for the analyzer. Add that output, and keep it useful outside the test: a user debugging their own latency wants the same dump. -- Reuse the browser lane's analyzer and budget file, with the JSON field names +- Reuse the browser lane's analyzer and budget file (`test/audio-quality/`, the + schema is `clients/js/src/schema.ts`), with the JSON field names and meanings matching its schema exactly. Native rows get their own budget values in the same file, keyed the same way, since the device floor differs; they do not get their own schema. @@ -45,6 +46,6 @@ harness](/quest/m0/audio-quality-harness/README.md) line (decided in the 2026-09-28 quest audit): nothing in m0 waits on it. The native jitter target it grades is done on the jitter target line. -## Required +## Related -- [Browser](/quest/m0/audio-quality-harness/browser.md) - defines the metric schema, the budget file, and the extracted shaper +- [Shaper profiles](/quest/m0/audio-quality-harness/shaper-profiles.md) - the bursty and step profiles both lanes should run diff --git a/test/README.md b/test/README.md index 0e9ad969f7..82e9870f8b 100644 --- a/test/README.md +++ b/test/README.md @@ -8,8 +8,9 @@ tests live in each language's own justfile. | [interop](interop/README.md) | `just test interop` | every client built from this checkout interoperates | | [wasm](wasm/README.md) | `just test wasm` | the `@moq/wasm` bindings work in a real browser | | [ts](ts/README.md) | `just test ts` | the subscriber's `export ts` output is IRD-compliant | +| [audio-quality](audio-quality/README.md) | `just test audio-quality` | browser audio playout stays within budget over an impaired path | -All three stand up a `moq-relay` and clients, so two of them running at once, or +All four stand up a `moq-relay` and clients, so two of them running at once, or the same one running from two worktrees, would otherwise collide. `lib/harness.sh` is what keeps them apart. diff --git a/test/audio-quality/README.md b/test/audio-quality/README.md new file mode 100644 index 0000000000..242660731f --- /dev/null +++ b/test/audio-quality/README.md @@ -0,0 +1,183 @@ +# Audio quality + +Plays a tone in headless Chromium over a seeded, impaired UDP path and grades what the listener +would have heard: how often the ring ran dry, how much audio a re-anchor threw away, how much of the +run was silent, and where the playout delay settled. A regression in any of them fails a row +against [`budgets.json`](budgets.json) instead of arriving as a bug report. + +```bash +just test audio-quality # the whole matrix, 60s a row +just test audio-quality --list # what it would run +just test audio-quality --profiles mild --codecs opus --duration 20 +just test audio-quality --rings plain # the production path only +just test audio-quality --out ~/runs/after # keep the run directory +just test audio-quality --enforce # fail on the budgets +``` + +The matrix is codec x profile x ring: 16 rows, about 20 minutes. `--profiles`, `--rings`, and +`--codecs` take comma-separated lists; `--seed` replays a given impairment. The table prints either +way, and only `--enforce` (which the nightly job passes) turns a breach into a failed run. + +## Rows + +| Axis | Values | +| --- | --- | +| codec | `opus` at 48 kHz over fMP4, `aac` at 44.1 kHz over MPEG-TS | +| profile | see below | +| ring | `isolated` (SharedArrayBuffer, cross-origin isolated page) and `plain` (postMessage) | + +`plain` is the production path: most pages are not cross-origin isolated. One build is served under +`/isolated/` with COOP and COEP and under `/plain/` without, and the page's `crossOriginIsolated` +decides which ring ran. + +| Profile | Path | Delay | +| --- | --- | --- | +| `near-zero` | through the shaper, untreated | auto | +| `mild` | 5 ms one way, plus or minus 5 ms uniform | auto | +| `wide` | 40 ms one way, plus or minus 40 ms uniform | auto | +| `fixed-250` | through the shaper, untreated | a fixed 250 ms | + +`fixed-250` is the control: every other row adapts, so without it the whole matrix could pass while a +fixed preset regressed, and a fixed preset is what many viewers land on. Loss, reorder, and rate +limits stay off: the buffer's job is absorbing arrival spread, and congestion response would make a +failure hard to attribute. The shaper's uniform jitter does reorder datagrams whose spacing is +smaller than it, so `wide` measures some reordering along with the spread. The bursty and mid-run +step profiles need shaper features that are not here yet +([quest](../../quest/m0/audio-quality-harness/shaper-profiles.md)). + +The AAC arm keeps ffmpeg's default PES packing, whose multi-frame bursts are the flush-span shape +the reporter measured on the public relay (#3477). The source is a sine tone rather than a film, so a +quiet quantum at the output is always a gap or a stall and never the soundtrack. + +## The path + +```text + page ──UDP──▶ moq-shaper ──▶ moq-relay WebTransport, the impaired path + moq (opus, aac) ─────────────▶ moq-relay the publishers, on the clean path +``` + +Every port is reserved for the run ([the harness contract](../README.md)), and the shaper is the +same `moq-shaper` binary the [transport drills](../drill/README.md) run under, so the two share one +impairment implementation. It exits nonzero when a configured impairment never acted, and that +voids the row. + +The shaper carries UDP and nothing else. The page fetches `/certificate.sha256` from the address it +dials, so the driver answers that fetch with the relay's own hash (the certificate is pinned by +hash, so a different port needs nothing more). With nothing listening on TCP there, the WebSocket +fallback cannot connect: the session is WebTransport through the shaper, or the row is void. + +## The metric contract + +[`clients/js/src/schema.ts`](clients/js/src/schema.ts) is the contract. The page emits `Sample`s, +[`src/analyze.ts`](clients/js/src/analyze.ts) reduces a row to a `Summary`, and `grade.ts` reads that +against the budgets. A native lane or the latency ledger that emits the same `Summary` is comparable +by construction. + +**Units.** Every duration is milliseconds as a float, every count an integer, every share a fraction +of 1. Samples missing at the device rate are converted to ms before they reach a summary. + +**Clocks.** Every timestamp is taken on a named clock and reduced to `viewer`, the page's monotonic +`performance.now()`, before any two are subtracted: + +| Clock | What | Reduced to `viewer` by | +| --- | --- | --- | +| `viewer` | `performance.now()` | nothing: it is the reference | +| `render` | the `AudioContext` frame clock | offset and rate fitted from paired readings every 250 ms | +| `media` | the publisher's timestamps, through the stream | rate fitted; the offset is unknowable from the viewer | +| `publisher`, `relay` | other processes | an offset measured NTP style over the session, uncertain by half a round trip | + +Nothing in this lane carries a publisher or relay timestamp yet, so the stages that need one are +reported as unmeasured rather than guessed. A render clock more than 1% off the viewer's voids the +row: it is a throttled or stalled context, and everything counted on it would be wrong. + +**Aggregation.** A budget key is `_`: `total` over the graded window, +`per_min` (the total over the window's minutes), `p50`/`p95` (nearest rank over the window's +samples), `max`, `share` (of the window), or `last`. So "underruns: 3" and "underruns: 3/min" are +different keys and are never graded against each other. + +**Counters.** The player publishes no count of its own underruns, so the page fans a listening +AudioWorklet out from the player's output node. The player's render worklet copies what the ring +holds into the front of each 128-sample quantum and leaves the rest zero, so the tap reads every +quantum's fill level from the output itself: + +- `underruns`: quanta the ring filled only partly, or not at all, while playing. +- `short_quanta`: the underruns that were partly filled: the ring ran dry part-way through. +- `underrun_episodes`: maximal runs of consecutive underruns, one audible gap each. +- `underrun_ms`: the missing audio, summed, and the longest episode as `max`. +- `skip_aheads`: re-anchors that discarded buffered audio, seen as a forward step of more than 40 ms + in the lag between the viewer clock and the playhead (judged on the median of four samples either + side, after removing the media clock's drift). +- `discarded_ms`: the media those skip-aheads jumped over. Audio dropped before it reached the ring + moves no playhead, so it is not counted. +- `stalled`: the share of the window the ring spent stalled, refilling rather than playing. A gap + that starts within 60 ms of a stall is that stall, and is not an underrun. +- `silence`: the share of rendered quanta under -60 dBFS at the output, whatever the cause. +- `target_ms`: the resolved playout delay (`sync.out.delay`). +- `converge_ms`: from first audio to the last time the target moved more than one 20 ms bucket from + its final value. +- `render_load`: Chromium's render capacity, the share of each quantum's budget the graph used. + +Every row discards the first 5 s after first audio, which is the tune-in rather than the steady +state; `converge_ms` grades the tune-in instead. + +**Stages.** An end-to-end delay splits into exclusive spans, each defined by its two boundaries: +`capture`, `encode`, `publish_flush`, `network`, `jitter_buffer`, `decode`, `render`, `device`, and +the named remainder `unaccounted`. The identity is `sum == end-to-end` within the larger of 2 ms or +2%. This lane measures `jitter_buffer` (the resolved target) and `device` (`outputLatency` plus +`baseLatency`) on the viewer, and reports the publisher's declared `publish_flush` without summing it +with them. No clock spans the whole path here, so `end-to-end` is null rather than an identity that +holds by construction. + +## Void rules + +A row that cannot be trusted is void, reported rather than graded, and fails an enforced run. + +| Void | Why | +| --- | --- | +| `shaper` | It exited nonzero (a configured impairment never acted), left no report, or forwarded nothing to the page. | +| `transport` | The session negotiated something other than WebTransport. | +| `ring` | `crossOriginIsolated` does not match the row's ring, so the other ring ran. | +| `clock` | The AudioContext clock was unreadable or ran more than 1% off the viewer's. | +| `window` | No audio after the warmup. | +| `driver` | The page never reached a session or first audio within 30 s, or the driver threw. | + +## Budgets + +`budgets.json` holds a ceiling per graded key per row, keyed by the whole row: runtime, codec, rate, +profile, and ring. Under `--enforce` a run fails on a breach, a budgeted key the run left +unmeasured, a void row, or a row with no budget. Tightening a ceiling is a visible diff; loosening +one needs a reason in review. The file's `note` says how the current ceilings were measured. + +The nightly `Audio quality` job runs the matrix under `--enforce` and keeps a failing run directory +for a week: each process's log, the shaper's counters, every row's raw samples and summary. + +## Layout + +```text +run.sh builds, starts the relay and publishers, runs each row behind the shaper +relay.toml anonymous relay, self-signed cert +budgets.json a ceiling per metric per row +clients/js/ + index.html src/page.ts the watch element, driven from the query string + src/probe.ts samples its public signals every 250 ms + src/tap.ts the listening worklet (tap-worklet.ts, quantum.ts) + src/schema.ts the metric contract + src/analyze.ts samples to a summary, unit tested + driver.ts one row in headless Chromium, and its void checks + analyze.ts grade.ts the summary per row, then the table and the verdict +``` + +## Attribution + +The harness is the reporter's, from #3477: fperex built it on their fork `fperex/moq` (branches +`debug/rt-audio` and `debug-findings-solution`), with the black-box probe, the analyzer's skip-ahead +rule, the Playwright matrix over `moq-shaper`, the budget file graded under `--enforce`, and the +nightly job. This lane upstreams it on its own, without the fork's player, estimator, or shaper +changes: the underrun count comes from a tap on the output rather than a counter in the player, and +the profiles are the ones the shaper on `main` can express. + +## Not covered + +Safari and Firefox, video, and perceptual scoring: the grade is glitches and latency, not an opinion +about how it sounds. Recorded arrival traces are +[their own quest](../../quest/m0/audio-quality-harness/traces.md). diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json new file mode 100644 index 0000000000..327fd2b15e --- /dev/null +++ b/test/audio-quality/budgets.json @@ -0,0 +1,4 @@ +{ + "note": "placeholder", + "rows": [] +} diff --git a/test/audio-quality/clients/js/analyze.ts b/test/audio-quality/clients/js/analyze.ts new file mode 100644 index 0000000000..7a0eabaa1e --- /dev/null +++ b/test/audio-quality/clients/js/analyze.ts @@ -0,0 +1,116 @@ +/** + * Reduces one row of a run directory to `.summary.json`, and prints it. + * + * Reads what the row left: `.ndjson` and `.page.json` from the driver, and the shaper's log + * and exit status from run.sh. + * + * bun analyze.ts --run --row [--warmup 5] + * + * @module + */ +import { existsSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { analyze } from "./src/analyze.ts"; +import { + type Environment, + METRICS, + parseRow, + type Sample, + type Shaper, + type ShaperCounters, + STAGES, + type Void, +} from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + run: { type: "string" }, + row: { type: "string" }, + warmup: { type: "string", default: "5" }, + }, +}); +if (!values.run || !values.row) { + console.error("usage: analyze.ts --run --row [--warmup 5]"); + process.exit(2); +} +const run = values.run; +const tag = values.row; +const read = (file: string) => { + const path = join(run, file); + return existsSync(path) ? readFileSync(path, "utf8") : undefined; +}; + +const samples: Sample[] = (read(`${tag}.ndjson`) ?? "") + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as Sample); + +const page = JSON.parse(read(`${tag}.page.json`) ?? "null") as { + environment?: Environment; + notes: string[]; + voids: Void[]; +} | null; + +/** The shaper prints its seed at start and its counters at exit; run.sh records the exit status. */ +function shaperOf(): Shaper | null { + const log = read(`shaper-${tag}.log`); + const status = read(`shaper-${tag}.status`); + if (log === undefined || status === undefined) return null; + const counters = (direction: "up" | "down"): ShaperCounters | null => { + const match = log.match( + new RegExp( + `${direction}: (\\d+) packets, (\\d+) lost, (\\d+) overflowed, (\\d+) throttled, (\\d+) delayed, (\\d+) reordered`, + ), + ); + if (!match) return null; + const [packets, lost, overflowed, throttled, delayed, reordered] = match.slice(1).map(Number) as number[]; + return { packets, lost, overflowed, throttled, delayed, reordered } as ShaperCounters; + }; + const seed = log.match(/seed (\d+)/)?.[1]; + return { + seed: seed === undefined ? null : Number(seed), + status: Number.parseInt(status, 10), + up: counters("up"), + down: counters("down"), + }; +} + +const summary = analyze({ + row: parseRow(tag), + samples, + environment: page?.environment, + voids: page?.voids ?? [{ assertion: "driver", detail: "left no page report" }], + notes: page?.notes ?? [], + shaper: shaperOf(), + warmupMs: Number.parseFloat(values.warmup) * 1000, +}); +await Bun.write(join(run, `${tag}.summary.json`), JSON.stringify(summary, null, 1)); + +const lines = [`## ${tag}`, "", `window ${summary.windowMs} ms after a ${summary.warmupMs} ms warmup`, ""]; +if (summary.voids.length > 0) { + lines.push(`**void**: ${summary.voids.map((v) => `${v.assertion} (${v.detail})`).join("; ")}`, ""); +} +lines.push("| metric | unit | clock | aggregation | value |", "|---|---|---|---|---|"); +for (const [name, spec] of Object.entries(METRICS)) { + for (const aggregation of spec.aggregations) { + lines.push( + `| ${name} | ${spec.unit} | ${spec.clock} | ${aggregation} | ${summary.metrics[`${name}_${aggregation}`] ?? "n/a"} |`, + ); + } +} +lines.push( + "", + "| stage | ms | source |", + "|---|---|---|", + ...STAGES.map((s) => `| ${s} | ${summary.stages[s].ms ?? "n/a"} | ${summary.stages[s].source} |`), +); +const shaper = summary.shaper; +if (shaper) { + lines.push( + "", + `shaper seed ${shaper.seed} exit ${shaper.status}: up ${JSON.stringify(shaper.up)} down ${JSON.stringify(shaper.down)}`, + ); +} +if (summary.notes.length > 0) lines.push("", "notes:", ...summary.notes.slice(0, 20).map((n) => `- ${n}`)); +console.log(lines.join("\n")); diff --git a/test/audio-quality/clients/js/driver.ts b/test/audio-quality/clients/js/driver.ts new file mode 100644 index 0000000000..3fec53e203 --- /dev/null +++ b/test/audio-quality/clients/js/driver.ts @@ -0,0 +1,160 @@ +/** + * Plays one matrix row in headless Chromium, and refuses a row whose numbers would mean something + * other than what its key says. + * + * The page dials the shaper, which forwards UDP and nothing else. So the page's fetch of the + * certificate hash is answered here with the relay's own (the certificate is pinned by hash, so + * dialing another port needs nothing more), and a WebSocket fallback finds nobody listening: the + * session is WebTransport through the shaper, or nothing. + * + * bun driver.ts --url http://127.0.0.1:4501 --fingerprint http://127.0.0.1:4500/certificate.sha256 \ + * --broadcast tone-opus.hang --page dist --ring plain --delay auto --duration 60 \ + * --tag chromium-opus-48000-mild-plain --out + * + * Writes `.ndjson` (the samples, appended as they are drained) and `.page.json` (the + * environment, console notes, and voids). Exits nonzero when the row could not be played at all. + * + * @module + */ +import { appendFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import type { Page } from "playwright"; +import { launch, sleep } from "../../../interop/clients/js/harness.ts"; +import type { Environment, Ring, Void } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + url: { type: "string" }, + fingerprint: { type: "string" }, + broadcast: { type: "string" }, + page: { type: "string" }, + ring: { type: "string" }, + delay: { type: "string", default: "auto" }, + duration: { type: "string", default: "60" }, + tag: { type: "string" }, + out: { type: "string" }, + }, +}); + +const { url, fingerprint, broadcast, page: pageDir, tag, out } = values; +const durationMs = Number.parseFloat(values.duration) * 1000; +const ring = values.ring as Ring; +if ( + !url || + !fingerprint || + !broadcast || + !pageDir || + !tag || + !out || + !Number.isFinite(durationMs) || + durationMs <= 0 || + (ring !== "plain" && ring !== "isolated") +) { + console.error( + "usage: driver.ts --url U --fingerprint F --broadcast B --page DIR --ring plain|isolated --tag T --out DIR [--delay auto] [--duration S>0]", + ); + process.exit(2); +} + +/** How long the session and the first audio each get before the row is abandoned. */ +const STARTUP_MS = 30_000; +/** How often the probe is drained into the ndjson. */ +const DRAIN_MS = 1000; + +const hash = await fetch(fingerprint).then((r) => { + if (!r.ok) throw new Error(`${fingerprint}: ${r.status}`); + return r.text(); +}); + +// Cross-origin isolation is a property of the document, not the bundle, so one build served under +// two prefixes runs both rings. `/plain` is the production path: most viewers are not isolated, and +// get the postMessage ring. +const root = resolve(pageDir); +const isolation = { "cross-origin-opener-policy": "same-origin", "cross-origin-embedder-policy": "require-corp" }; +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + async fetch(req) { + const path = new URL(req.url).pathname; + const match = path.match(/^\/(isolated|plain)\/(.*)$/); + if (!match) return new Response("not found", { status: 404 }); + const file = Bun.file(join(root, match[2] || "index.html")); + if (!(await file.exists())) return new Response("not found", { status: 404 }); + return new Response(file, { headers: match[1] === "isolated" ? isolation : undefined }); + }, +}); + +const query = new URLSearchParams({ url, broadcast, delay: values.delay }); +const pageUrl = `http://127.0.0.1:${server.port}/${ring}/?${query}`; +console.log(`page: ${pageUrl}`); + +const samplesFile = join(out, `${tag}.ndjson`); +const voids: Void[] = []; +const refuse = (assertion: string, detail: string) => { + console.error(`void: ${assertion}: ${detail}`); + voids.push({ assertion, detail }); +}; + +/** Poll the page until `ready` returns a value, or throw naming what never happened. */ +async function waitFor(page: Page, what: string, ready: () => T | undefined): Promise { + const deadline = Date.now() + STARTUP_MS; + while (Date.now() < deadline) { + const value = await page.evaluate(ready); + if (value !== undefined && value !== null) return value; + await sleep(200); + } + throw new Error(`${what} not within ${STARTUP_MS / 1000}s`); +} + +const drain = async (page: Page) => { + const samples = await page.evaluate(() => globalThis.audioQuality.drain()); + if (samples.length > 0) appendFileSync(samplesFile, `${samples.map((s) => JSON.stringify(s)).join("\n")}\n`); +}; + +// No fake devices: nothing here captures. The autoplay flag stands in for the click a viewer makes. +const browser = await launch(["--autoplay-policy=no-user-gesture-required"]); +let environment: Environment | undefined; +let notes: string[] = []; +let failed = false; +try { + const page = await browser.newPage(); + page.on("pageerror", (error) => console.error(`[page error] ${error.message}`)); + await page.route(new URL("/certificate.sha256", url).href, (route) => + route.fulfill({ body: hash, headers: { "access-control-allow-origin": "*" } }), + ); + await page.goto(pageUrl, { waitUntil: "load" }); + + environment = await waitFor(page, "session", () => { + const env = globalThis.audioQuality?.environment(); + return env?.transport ? env : undefined; + }); + if (environment.transport !== "webtransport") { + refuse("transport", `negotiated ${environment.transport}, which never crosses the UDP shaper`); + } + if (environment.crossOriginIsolated !== (ring === "isolated")) { + refuse("ring", `asked for ${ring}, but crossOriginIsolated is ${environment.crossOriginIsolated}`); + } + + // The row's duration is audio played, not startup: the window starts once the ring does. + await waitFor(page, "first audio", () => globalThis.audioQuality.playing() || undefined); + + const end = Date.now() + durationMs; + while (Date.now() < end) { + await sleep(Math.min(DRAIN_MS, end - Date.now())); + await drain(page); + } + // The context rate is only known once the graph exists, which is after the catalog. + environment = (await page.evaluate(() => globalThis.audioQuality.environment())) ?? environment; + notes = await page.evaluate(() => globalThis.audioQuality.notes()); + await page.close(); +} catch (err) { + failed = true; + refuse("driver", err instanceof Error ? err.message : String(err)); +} finally { + await browser.close().catch(() => {}); + server.stop(true); +} + +await Bun.write(join(out, `${tag}.page.json`), JSON.stringify({ environment, notes, voids }, null, 1)); +process.exit(failed ? 1 : 0); diff --git a/test/audio-quality/clients/js/grade.ts b/test/audio-quality/clients/js/grade.ts new file mode 100644 index 0000000000..819b7be29f --- /dev/null +++ b/test/audio-quality/clients/js/grade.ts @@ -0,0 +1,102 @@ +/** + * Grades a run's summaries against `budgets.json`, prints the table, and writes `grade.md`. + * + * Every budget value is a ceiling on one `_` key. Under `--enforce` a run fails + * on any of: a value over its ceiling, a budgeted value the run never measured, a void row, or a row + * with no budget at all. Each of those would otherwise read as a pass while grading nothing. + * + * bun grade.ts --run --budgets [--enforce] + * + * @module + */ +import { readdirSync, readFileSync } from "node:fs"; +import { join } from "node:path"; +import { parseArgs } from "node:util"; +import { type Budgets, METRIC_KEYS, type Row, rowKey, type Summary } from "./src/schema.ts"; + +const { values } = parseArgs({ + options: { + run: { type: "string" }, + budgets: { type: "string" }, + enforce: { type: "boolean", default: false }, + }, +}); +if (!values.run || !values.budgets) { + console.error("usage: grade.ts --run --budgets [--enforce]"); + process.exit(2); +} +const run = values.run; + +const budgets = JSON.parse(readFileSync(values.budgets, "utf8")) as Budgets; +const summaries = readdirSync(run) + .filter((f) => f.endsWith(".summary.json")) + .sort() + .map((f) => JSON.parse(readFileSync(join(run, f), "utf8")) as Summary); +if (summaries.length === 0) { + console.error(`error: no summaries in ${run}`); + process.exit(1); +} + +/** A budget applies to exactly one matrix cell, matched on the whole row. */ +const budgetFor = (row: Row) => + budgets.rows.find( + (b) => + b.runtime === row.runtime && + b.codec === row.codec && + b.rate === row.rate && + b.profile === row.profile && + b.ring === row.ring, + ); + +const failures: string[] = []; +for (const summary of summaries) { + const key = rowKey(summary.row); + const budget = budgetFor(summary.row); + if (!budget) { + failures.push(`${key}: no budget`); + continue; + } + for (const v of summary.voids) failures.push(`${key}: void, ${v.assertion} (${v.detail})`); + if (summary.voids.length > 0) continue; + for (const metric of METRIC_KEYS) { + const ceiling = budget[metric]; + if (typeof ceiling !== "number") continue; + const value = summary.metrics[metric]; + if (value === null || value === undefined) failures.push(`${key}: ${metric} unmeasured, ceiling ${ceiling}`); + else if (value > ceiling) failures.push(`${key}: ${metric} ${value} > ${ceiling}`); + } +} + +/** The columns a reader scans first: what the listener would have noticed, and the delay paid for it. */ +const COLUMNS: [string, string][] = [ + ["underruns/min", "underruns_per_min"], + ["short/min", "short_quanta_per_min"], + ["episodes/min", "underrun_episodes_per_min"], + ["gap ms/min", "underrun_ms_per_min"], + ["skips/min", "skip_aheads_per_min"], + ["discarded ms/min", "discarded_ms_per_min"], + ["stalled", "stalled_share"], + ["silence", "silence_share"], + ["target p95", "target_ms_p95"], + ["converge ms", "converge_ms_last"], +]; + +const lines = [ + `| row | ${COLUMNS.map(([label]) => label).join(" | ")} | void |`, + `|---|${COLUMNS.map(() => "---|").join("")}---|`, + ...summaries.map( + (s) => + `| ${rowKey(s.row)} | ${COLUMNS.map(([, k]) => s.metrics[k] ?? "n/a").join(" | ")} | ${s.voids.map((v) => v.assertion).join(",") || "-"} |`, + ), + "", + ...(failures.length > 0 ? ["failures:", ...failures.map((f) => `- ${f}`)] : ["within budget"]), +]; +const report = lines.join("\n"); +console.log(report); +await Bun.write(join(run, "grade.md"), `${report}\n`); + +if (!values.enforce) { + if (failures.length > 0) console.log("\nreporting only: pass --enforce to fail on these"); + process.exit(0); +} +process.exit(failures.length > 0 ? 1 : 0); diff --git a/test/audio-quality/clients/js/index.html b/test/audio-quality/clients/js/index.html new file mode 100644 index 0000000000..8b151b202c --- /dev/null +++ b/test/audio-quality/clients/js/index.html @@ -0,0 +1,10 @@ + + + + + moq audio quality + + + + + diff --git a/test/audio-quality/clients/js/package.json b/test/audio-quality/clients/js/package.json new file mode 100644 index 0000000000..9534e140ed --- /dev/null +++ b/test/audio-quality/clients/js/package.json @@ -0,0 +1,18 @@ +{ + "name": "@moq/audio-quality", + "private": true, + "type": "module", + "scripts": { + "check": "tsc --noEmit", + "test": "bun test --only-failures" + }, + "dependencies": { + "@moq/watch": "workspace:*" + }, + "devDependencies": { + "@types/audioworklet": "^0.0.100", + "esbuild": "^0.28.2", + "playwright": "^1.63.0", + "vite": "^8.3.0" + } +} diff --git a/test/audio-quality/clients/js/src/analyze.test.ts b/test/audio-quality/clients/js/src/analyze.test.ts new file mode 100644 index 0000000000..b6dba0ae50 --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, test } from "bun:test"; +import { analyze, type Input, percentile, SKIP_MS, skipAheads } from "./analyze.ts"; +import type { Sample, Shaper } from "./schema.ts"; + +const row = { runtime: "chromium", codec: "opus", rate: 48000, profile: "mild", ring: "plain" } as const; +const counters = { packets: 100, lost: 0, overflowed: 0, throttled: 0, delayed: 100, reordered: 0 }; +const shaper: Shaper = { seed: 7, status: 0, up: counters, down: counters }; + +/** A clean run: 40 s of samples, the render clock 1000 ms behind the viewer's, the playhead 200 ms behind. */ +function clean(): Sample[] { + return Array.from({ length: 160 }, (_, i) => { + const at = 1000 + i * 250; + return { + at, + render: at - 1000, + timestamp: at - 200, + stalled: false, + delay: 120, + quanta: i * 94, + quiet: 0, + }; + }); +} + +const input = (samples: Sample[], extra: Partial = {}): Input => ({ + row, + samples, + environment: { crossOriginIsolated: false, transport: "webtransport" }, + voids: [], + notes: [], + shaper, + warmupMs: 5000, + ...extra, +}); + +describe("skipAheads", () => { + test("the quantization band is not a skip", () => { + const lags = Array.from({ length: 40 }, (_, i) => 200 + (i % 2 === 0 ? 25 : -25)); + expect(skipAheads(lags).count).toBe(0); + }); + + test("a step in the lag is one skip, of the step's size", () => { + const lags = Array.from({ length: 40 }, (_, i) => (i < 20 ? 300 : 200)); + expect(skipAheads(lags)).toEqual({ count: 1, ms: 100 }); + }); + + test("a step at the threshold is not one", () => { + const lags = Array.from({ length: 40 }, (_, i) => (i < 20 ? 200 + SKIP_MS : 200)); + expect(skipAheads(lags).count).toBe(0); + }); +}); + +test("percentile is nearest rank", () => { + expect(percentile([5, 1, 4, 2, 3], 50)).toBe(3); + expect(percentile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 95)).toBe(10); + expect(percentile([], 50)).toBeNull(); +}); + +describe("analyze", () => { + test("a clean run has nothing to report and nothing void", () => { + const summary = analyze(input(clean())); + expect(summary.voids).toEqual([]); + expect(summary.metrics).toMatchObject({ + underruns_total: 0, + skip_aheads_total: 0, + stalled_share: 0, + silence_share: 0, + target_ms_p95: 120, + converge_ms_last: 0, + }); + // Every key in the schema is present, measured or null. + expect(summary.metrics.render_load_max).toBeNull(); + expect(summary.drift.find((d) => d.clock === "render")).toEqual({ clock: "render", rate: 0, offset: 1000 }); + }); + + test("a gap on the render clock is counted where it lands on the viewer's", () => { + const samples = clean(); + // 20 s into the viewer clock is 19 s on the render clock. + samples[100] = { ...samples[100], gaps: [{ at: 19_000, ms: 8, quanta: 3, short: 1 }] } as Sample; + // Inside the warmup, so not graded. + samples[10] = { ...samples[10], gaps: [{ at: 1_000, ms: 4, quanta: 2, short: 1 }] } as Sample; + const summary = analyze(input(samples)); + expect(summary.metrics).toMatchObject({ + underruns_total: 3, + short_quanta_total: 1, + underrun_episodes_total: 1, + underrun_ms_total: 8, + underrun_ms_max: 8, + }); + }); + + test("a gap that is the ring stalling is not an underrun", () => { + const samples = clean(); + samples[100] = { + ...samples[100], + gaps: [{ at: 19_000, ms: 300, quanta: 110, short: 0 }], + stalls: [ + { at: 19_020, stalled: true }, + { at: 19_300, stalled: false }, + ], + } as Sample; + expect(analyze(input(samples)).metrics.underruns_total).toBe(0); + }); + + test("a re-anchor that discards audio is a skip-ahead", () => { + const samples = clean().map((s, i) => (i >= 80 ? { ...s, timestamp: s.at - 100 } : s)); + expect(analyze(input(samples)).metrics).toMatchObject({ skip_aheads_total: 1, discarded_ms_total: 100 }); + }); + + test("a shaper that failed or never carried the page voids the row", () => { + const failed = analyze(input(clean(), { shaper: { ...shaper, status: 1 } })); + expect(failed.voids.map((v) => v.assertion)).toEqual(["shaper"]); + const idle = analyze(input(clean(), { shaper: { ...shaper, down: { ...counters, packets: 0 } } })); + expect(idle.voids.map((v) => v.assertion)).toEqual(["shaper"]); + }); + + test("a render clock that is not running at wall rate voids the row", () => { + const samples = clean().map((s) => ({ ...s, render: (s.at - 1000) * 0.5 })); + expect(analyze(input(samples)).voids.map((v) => v.assertion)).toEqual(["clock"]); + }); + + test("no audio at all voids the window", () => { + const samples = clean().map((s) => ({ ...s, timestamp: undefined, stalled: true })); + expect(analyze(input(samples)).voids.map((v) => v.assertion)).toContain("window"); + }); + + test("convergence is the last move, not the first plateau", () => { + const samples = clean().map((s, i) => ({ ...s, delay: i < 40 ? 100 : i < 60 ? 300 : 100 })); + // First audio at sample 0, the target back on its final value from sample 60. + expect(analyze(input(samples)).metrics.converge_ms_last).toBe(60 * 250); + }); +}); diff --git a/test/audio-quality/clients/js/src/analyze.ts b/test/audio-quality/clients/js/src/analyze.ts new file mode 100644 index 0000000000..9ee7d6c62c --- /dev/null +++ b/test/audio-quality/clients/js/src/analyze.ts @@ -0,0 +1,305 @@ +/** + * Reduces one row's samples to the {@link Summary} the grader reads. + * + * Pure, so every derivation below is unit tested against synthetic samples rather than trusted. + * `../analyze.ts` is the command that feeds it a run directory. + * + * Adapted from `debug-findings/analysis/analyze.mjs` on the reporter's fork (`fperex/moq`, branch + * `debug/rt-audio`), whose skip-ahead rule and per-minute normalisation this keeps. The event stream + * it read came from probes patched into the player; this reads public signals and the output tap. + * + * @module + */ +import { + BUCKET_MS, + type Drift, + type Environment, + type Gap, + MAX_RENDER_DRIFT, + METRIC_KEYS, + type Ms, + type Row, + type Sample, + type Shaper, + STAGES, + type Stage, + type StageSpan, + type Summary, + type Void, +} from "./schema.ts"; + +/** Everything a row left behind. */ +export type Input = { + row: Row; + samples: Sample[]; + environment: Environment | undefined; + voids: Void[]; + notes: string[]; + shaper: Shaper | null; + /** Discarded after first audio: every profile spends its first seconds converging on purpose. */ + warmupMs: Ms; +}; + +/** + * A gap that starts this close to a stall is the stall, not an underrun. + * + * The page learns of a stall after the ring does: up to five quanta late on the postMessage ring, + * one poll late on the isolated one. + */ +export const STALL_SLACK_MS = 60; + +/** A step in the de-trended playhead lag larger than this is a skip-ahead. See {@link skipAheads}. */ +export const SKIP_MS = 40; + +/** Samples either side of a candidate skip reduced to a median before it is judged. */ +export const SKIP_WINDOW = 4; + +const median = (values: number[]): number | undefined => { + if (values.length === 0) return undefined; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.floor(sorted.length / 2)]; +}; + +/** Nearest-rank percentile. Null for an empty series. */ +export function percentile(values: number[], p: number): number | null { + if (values.length === 0) return null; + const sorted = [...values].sort((a, b) => a - b); + return sorted[Math.max(0, Math.ceil((p / 100) * sorted.length) - 1)] ?? null; +} + +/** Least-squares slope of `ys` over `xs`. Null with nothing to fit. */ +export function slope(xs: number[], ys: number[]): number | null { + const n = xs.length; + if (n < 2) return null; + const mx = xs.reduce((a, b) => a + b, 0) / n; + const my = ys.reduce((a, b) => a + b, 0) / n; + let num = 0; + let den = 0; + for (let i = 0; i < n; i++) { + num += ((xs[i] ?? 0) - mx) * ((ys[i] ?? 0) - my); + den += ((xs[i] ?? 0) - mx) ** 2; + } + return den === 0 ? null : num / den; +} + +/** + * Forward steps in the lag between the viewer clock and the playhead, and the media they skipped. + * + * `lags` is the de-trended `at - timestamp` per sample, undefined where there was no playhead or the + * ring was stalled. The playhead a page reads is quantized (it moves 240 to 296 ms per 250 ms sample + * on a clean path), so one sample's excess is noise: a skip is a step between the medians either side + * of a sample, larger than {@link SKIP_MS}, which is more than that band and more than one estimator + * bucket plus a render quantum. + */ +export function skipAheads(lags: (Ms | undefined)[]): { count: number; ms: Ms } { + let count = 0; + let ms = 0; + const defined = (slice: (Ms | undefined)[]) => slice.filter((l): l is Ms => l !== undefined); + for (let i = SKIP_WINDOW; i < lags.length - SKIP_WINDOW; i++) { + if (lags[i] === undefined) continue; + const before = median(defined(lags.slice(i - SKIP_WINDOW, i))); + const after = median(defined(lags.slice(i + 1, i + 1 + SKIP_WINDOW))); + if (before === undefined || after === undefined) continue; + const jumped = before - after; + if (jumped > SKIP_MS) { + count++; + ms += jumped; + // One step is one skip, and the trailing window straddles it for several samples. + i += SKIP_WINDOW; + } + } + return { count, ms }; +} + +/** + * Spans in which the ring was stalled, on the render clock. + * + * The ring starts stalled, before the page has a context to stamp that on, so a first change to + * unstalled closes a span that opened before anything was recorded. + */ +export function stallSpans(samples: Sample[]): { start: Ms; end: Ms }[] { + const changes = samples.flatMap((s) => s.stalls ?? []); + const spans: { start: Ms; end: Ms }[] = []; + let open: Ms | undefined = changes[0]?.stalled === false ? Number.NEGATIVE_INFINITY : undefined; + for (const change of changes) { + if (change.stalled && open === undefined) open = change.at; + else if (!change.stalled && open !== undefined) { + spans.push({ start: open, end: change.at }); + open = undefined; + } + } + if (open !== undefined) spans.push({ start: open, end: Number.POSITIVE_INFINITY }); + return spans; +} + +const round = (x: number | null | undefined, places = 1): number | null => + x === null || x === undefined || !Number.isFinite(x) ? null : Math.round(x * 10 ** places) / 10 ** places; + +/** Reduce one row. */ +export function analyze(input: Input): Summary { + const { row, samples, environment, warmupMs } = input; + const voids = [...input.voids]; + const notes = [...input.notes]; + + // ── the shaper's verdict ──────────────────────────────────────────────── + const shaper = input.shaper; + if (!shaper) { + voids.push({ assertion: "shaper", detail: "left no report, so nothing says the path was shaped" }); + } else if (shaper.status !== 0) { + voids.push({ assertion: "shaper", detail: `exited ${shaper.status}: the profile never acted as configured` }); + } else if ((shaper.down?.packets ?? 0) === 0) { + voids.push({ assertion: "shaper", detail: "forwarded nothing to the page, so the session never crossed it" }); + } + + // ── the window ────────────────────────────────────────────────────────── + const first = samples.find((s) => s.timestamp !== undefined && !s.stalled); + const t0 = (first?.at ?? 0) + warmupMs; + const t1 = samples.at(-1)?.at ?? t0; + const window = samples.filter((s) => s.at >= t0 && s.at <= t1); + const minutes = Math.max(1e-9, (t1 - t0) / 60_000); + if (!first || window.length < 2) { + voids.push({ assertion: "window", detail: `no audio after the warmup (${samples.length} samples in all)` }); + } + + // ── the render clock, reduced to the viewer's ─────────────────────────── + // Each sample reads both clocks back to back, so the pairs fit the render clock's rate and offset. + const paired = samples.filter((s) => s.render !== undefined); + const renderRate = slope( + paired.map((s) => s.at), + paired.map((s) => s.render as Ms), + ); + const renderOffset = median(paired.map((s) => s.at - (s.render as Ms))) ?? null; + if (renderRate === null || renderOffset === null) { + voids.push({ assertion: "clock", detail: "the AudioContext clock was never readable" }); + } else if (Math.abs(renderRate - 1) > MAX_RENDER_DRIFT) { + voids.push({ + assertion: "clock", + detail: `the AudioContext clock ran at ${round(renderRate, 3)}x the viewer's: throttled or stalled`, + }); + } + const toViewer = (render: Ms) => render + (renderOffset ?? 0); + + // ── underruns, from the tap ───────────────────────────────────────────── + const stalls = stallSpans(samples); + const chosen = (gap: Gap) => + stalls.some((s) => gap.at >= s.start - STALL_SLACK_MS && gap.at <= s.end + STALL_SLACK_MS); + const gaps = samples + .flatMap((s) => s.gaps ?? []) + .filter((g) => toViewer(g.at) >= t0 && toViewer(g.at) <= t1 && !chosen(g)); + const underruns = gaps.reduce((a, g) => a + g.quanta, 0); + const short = gaps.reduce((a, g) => a + g.short, 0); + const underrunMs = gaps.reduce((a, g) => a + g.ms, 0); + + const tapped = window.filter((s) => s.quanta !== undefined && s.quiet !== undefined); + const firstTap = tapped[0]; + const lastTap = tapped.at(-1); + const quanta = (lastTap?.quanta ?? 0) - (firstTap?.quanta ?? 0); + const silence = quanta > 0 ? ((lastTap?.quiet ?? 0) - (firstTap?.quiet ?? 0)) / quanta : null; + if (!lastTap) notes.push("tap: never reported, so underruns and silence are unmeasured"); + + // ── the playhead ──────────────────────────────────────────────────────── + // The lag drifts when the media timeline does not run at wall rate, a property of the source + // rather than the player. Fitted and removed first, or it reads as a skip every few seconds. + // A least-squares fit would read a skip itself as drift and hide part of it, so the rate is the + // median slope across spans of two skip windows, which a step contaminates only a few of. + const playing = window.filter((s) => s.timestamp !== undefined && !s.stalled); + const reach = 2 * SKIP_WINDOW; + const mediaRate = + median( + playing.slice(reach).map((s, i) => { + const from = playing[i] as Sample; + return (s.at - (s.timestamp as Ms) - (from.at - (from.timestamp as Ms))) / (s.at - from.at); + }), + ) ?? null; + const base = playing[0]?.at ?? 0; + const lags = window.map((s) => + s.timestamp === undefined || s.stalled ? undefined : s.at - s.timestamp - (mediaRate ?? 0) * (s.at - base), + ); + const skips = skipAheads(lags); + const stalledShare = window.length > 0 ? window.filter((s) => s.stalled).length / window.length : null; + + // ── the target ────────────────────────────────────────────────────────── + const targets = window.flatMap((s) => (s.delay === undefined ? [] : [s.delay])); + // Measured backwards from the end: a target that settles and moves again has not converged. + const series = samples.filter((s) => first && s.at >= first.at && s.delay !== undefined); + const final = series.at(-1)?.delay; + let settled = series[0]; + for (let i = series.length - 1; i >= 0; i--) { + if (Math.abs((series[i]?.delay ?? 0) - (final ?? 0)) > BUCKET_MS) { + settled = series[i + 1]; + break; + } + } + const converge = first && settled ? settled.at - first.at : null; + + const loads = window.flatMap((s) => (s.renderLoad === undefined ? [] : [s.renderLoad])); + + const metrics: Record = Object.fromEntries(METRIC_KEYS.map((k) => [k, null])); + const perMin = (total: number) => round(total / minutes); + Object.assign(metrics, { + underruns_total: lastTap ? underruns : null, + underruns_per_min: lastTap ? perMin(underruns) : null, + short_quanta_total: lastTap ? short : null, + short_quanta_per_min: lastTap ? perMin(short) : null, + underrun_episodes_total: lastTap ? gaps.length : null, + underrun_episodes_per_min: lastTap ? perMin(gaps.length) : null, + underrun_ms_total: lastTap ? round(underrunMs) : null, + underrun_ms_per_min: lastTap ? perMin(underrunMs) : null, + underrun_ms_max: lastTap ? round(Math.max(0, ...gaps.map((g) => g.ms))) : null, + skip_aheads_total: skips.count, + skip_aheads_per_min: perMin(skips.count), + discarded_ms_total: round(skips.ms), + discarded_ms_per_min: perMin(skips.ms), + stalled_share: round(stalledShare, 3), + silence_share: round(silence, 3), + target_ms_p50: round(percentile(targets, 50)), + target_ms_p95: round(percentile(targets, 95)), + target_ms_max: round(percentile(targets, 100)), + converge_ms_last: round(converge), + render_load_max: round(percentile(loads, 100), 3), + }); + + // ── stages ────────────────────────────────────────────────────────────── + // Only the viewer's clocks are readable here, so only the receiver's stages are measured, and no + // end-to-end exists to sum them to. `publish_flush` is what the publisher declares, reported but + // never summed with viewer-clock spans. + const last = window.at(-1); + const span = (ms: Ms | undefined, source: StageSpan["source"]): StageSpan => + ms === undefined ? { ms: null, source: "unmeasured" } : { ms: round(ms), source }; + const known: Partial> = { + publish_flush: span(environment?.jitter, "declared"), + jitter_buffer: span(last?.delay, "measured"), + device: + last?.outputLatency === undefined + ? span(undefined, "unmeasured") + : span(last.outputLatency + (last.baseLatency ?? 0), "measured"), + }; + const stages = Object.fromEntries(STAGES.map((s) => [s, known[s] ?? span(undefined, "unmeasured")])) as Record< + Stage, + StageSpan + >; + + const drift: Drift[] = [ + { clock: "viewer", rate: 0, offset: 0 }, + { clock: "render", rate: round(renderRate === null ? null : renderRate - 1, 6), offset: round(renderOffset) }, + // The lag's slope is the media clock's rate against the viewer's, negated. + { clock: "media", rate: round(mediaRate === null ? null : -mediaRate, 6), offset: null }, + { clock: "publisher", rate: null, offset: null }, + { clock: "relay", rate: null, offset: null }, + ]; + + return { + version: 1, + row, + windowMs: round(t1 - t0) ?? 0, + warmupMs, + environment: environment ?? null, + voids, + metrics, + stages, + endToEnd: null, + drift, + shaper, + notes: notes.slice(0, 50), + }; +} diff --git a/test/audio-quality/clients/js/src/page.ts b/test/audio-quality/clients/js/src/page.ts new file mode 100644 index 0000000000..ee742da8cc --- /dev/null +++ b/test/audio-quality/clients/js/src/page.ts @@ -0,0 +1,43 @@ +/** + * The page under measurement: the demo's player and nothing else. + * + * The element and its registration imports are the ones `demo/web` uses, so a regression in the + * element's attribute handling or the worklet reaches these numbers the way it reaches a viewer. + * Everything comes from the query string, so one build serves every row: + * + * ?url=http://127.0.0.1:4499&broadcast=tone-opus.hang&delay=auto + * + * The driver drains the probe through `globalThis.audioQuality`. + * + * @module + */ +import "@moq/watch/element"; +import type MoqWatch from "@moq/watch/element"; +import { type Probe, probe } from "./probe.ts"; + +const params = new URLSearchParams(location.search); +const required = (name: string): string => { + const value = params.get(name); + if (!value) throw new Error(`missing ?${name}`); + return value; +}; + +const watch = document.createElement("moq-watch") as MoqWatch; +watch.setAttribute("url", required("url")); +watch.setAttribute("name", required("broadcast")); +// "auto" adapts, and is what every profile but the control runs. A fixed delay carries its unit: +// the element rejects `delay="250"`. +watch.setAttribute("delay", params.get("delay") ?? "auto"); +watch.appendChild(document.createElement("canvas")); +document.body.appendChild(watch); + +// Audio is the measurement: muted, nothing would reach the ring. Volume first, because un-muting +// restores the stashed volume. +watch.volume = 1; +watch.muted = false; +watch.paused = false; + +declare global { + var audioQuality: Probe; +} +globalThis.audioQuality = probe(watch); diff --git a/test/audio-quality/clients/js/src/probe.ts b/test/audio-quality/clients/js/src/probe.ts new file mode 100644 index 0000000000..2245170d70 --- /dev/null +++ b/test/audio-quality/clients/js/src/probe.ts @@ -0,0 +1,139 @@ +/** + * Samples a `` every 250 ms, from its public signals and a tap on its output. + * + * Nothing here patches the player or reaches past its `out` surface, so a number it reports is one + * any consumer could read. What the player does not publish (how full each render quantum was) is + * read off its output by the tap in `tap.ts`, every quantum rather than a sample of them. + * + * Adapted from the black-box probe in `debug-findings/analysis/blackbox.js` on the reporter's fork + * (`fperex/moq`, branch `debug/rt-audio`). See ../../../README.md. + * + * @module + */ +import type MoqWatch from "@moq/watch/element"; +import { type Environment, SAMPLE_INTERVAL_MS, type Sample, SILENCE_RMS, type Stall } from "./schema.ts"; +import { type Tap, tap } from "./tap.ts"; + +/** Chromium's render capacity surface, which is not in lib.dom. */ +type RenderCapacity = { + start(options: { updateInterval: number }): void; + addEventListener(type: "update", listener: (event: { averageLoad: number }) => void): void; +}; + +/** What the probe has collected, until someone drains it. */ +export type Probe = { + /** Samples taken since the last drain. */ + drain(): Sample[]; + /** What the page is, once the catalog says the session is up. */ + environment(): Environment | undefined; + /** Console warnings and errors so far. */ + notes(): string[]; + /** Whether the ring has a playhead and is not stalled. */ + playing(): boolean; +}; + +/** Start sampling `watch`, from before it has produced anything, so startup is on the record too. */ +export function probe(watch: MoqWatch): Probe { + const samples: Sample[] = []; + const notes: string[] = []; + const note = (line: string) => { + if (notes.length < 200) notes.push(line.slice(0, 200)); + }; + + const { audio, sync, broadcast } = watch.player; + + // Stall changes are taken as they happen: a re-stall shorter than the sample grid is exactly the + // one a 250 ms read would miss, and it is what tells a stall from an underrun. + let stalls: Stall[] = []; + audio.out.stalled.subscribe((stalled) => { + const context = audio.out.context.peek(); + if (context) stalls.push({ at: context.currentTime * 1000, stalled }); + }); + + // The tap follows the output node, which the player rebuilds with its graph. A tap left on the old + // node would report a perfect run as one long silence. + let tapped: AudioNode | undefined; + let current: Tap | undefined; + let renderLoad: number | undefined; + const attach = () => { + const root = audio.out.root.peek(); + if (!root || root === tapped) return; + tapped = root; + current?.close(); + current = undefined; + + tap(root, { floor: SILENCE_RMS }).then( + (t) => { + if (tapped === root) current = t; + else t.close(); + }, + (err: unknown) => note(`tap: ${err instanceof Error ? err.message : String(err)}`), + ); + + const capacity = (root.context as unknown as { renderCapacity?: RenderCapacity }).renderCapacity; + if (!capacity) { + note("renderCapacity: unavailable, render_load will be null"); + return; + } + capacity.addEventListener("update", (event) => { + renderLoad = event.averageLoad; + }); + // A whole second: shorter intervals are refused by some builds, and load is smooth anyway. + capacity.start({ updateInterval: 1 }); + }; + + const sample = (): Sample => { + attach(); + const context = audio.out.context.peek(); + const counts = current?.counts(); + const gaps = current?.take() ?? []; + const taken = stalls; + stalls = []; + return { + // Read back to back, so the pair calibrates the render clock against the viewer's. + at: performance.now(), + render: context ? context.currentTime * 1000 : undefined, + timestamp: audio.out.timestamp.peek(), + stalled: audio.out.stalled.peek(), + delay: sync.out.delay.peek(), + outputLatency: context ? context.outputLatency * 1000 : undefined, + baseLatency: context ? context.baseLatency * 1000 : undefined, + renderLoad, + quanta: counts?.quanta, + quiet: counts?.quiet, + gaps: gaps.length > 0 ? gaps : undefined, + stalls: taken.length > 0 ? taken : undefined, + }; + }; + + // Console output is evidence: a decoder reset shows up here before it shows up in a count. + for (const level of ["warn", "error"] as const) { + const original = console[level]; + console[level] = (...args: unknown[]) => { + note(`${level}: ${args.map(String).join(" ")}`); + original(...args); + }; + } + + setInterval(() => samples.push(sample()), SAMPLE_INTERVAL_MS); + + return { + drain: () => samples.splice(0, samples.length), + environment() { + const catalog = broadcast.out.catalog.peek(); + if (!catalog) return undefined; + // One rendition is published per broadcast here, so the first is the one playing. + const config = Object.values(catalog.audio?.renditions ?? {})[0]; + return { + crossOriginIsolated: globalThis.crossOriginIsolated === true, + transport: watch.connection.transport.peek(), + codec: config?.codec, + rate: config?.sampleRate, + jitter: config?.jitter, + contextRate: audio.out.context.peek()?.sampleRate, + }; + }, + notes: () => notes.slice(), + playing: () => audio.out.timestamp.peek() !== undefined && !audio.out.stalled.peek(), + }; +} diff --git a/test/audio-quality/clients/js/src/quantum.test.ts b/test/audio-quality/clients/js/src/quantum.test.ts new file mode 100644 index 0000000000..adf9d50d07 --- /dev/null +++ b/test/audio-quality/clients/js/src/quantum.test.ts @@ -0,0 +1,61 @@ +import { describe, expect, test } from "bun:test"; +import { classify, Ledger } from "./quantum.ts"; + +const quantum = (values: number[], channels = 2) => Array.from({ length: channels }, () => Float32Array.from(values)); +const tone = (length: number) => Array.from({ length }, (_, i) => 0.5 * Math.sin(i / 3 + 0.1)); + +describe("classify", () => { + test("a quantum the ring filled is full", () => { + expect(classify(quantum(tone(128))).fill).toBe("full"); + }); + + test("trailing zeros in every channel are the samples the ring did not have", () => { + const values = [...tone(100), ...new Array(28).fill(0)]; + expect(classify(quantum(values))).toMatchObject({ fill: "short", missing: 28 }); + }); + + test("one channel still playing means the ring supplied that sample", () => { + const left = Float32Array.from([...tone(100), ...new Array(28).fill(0)]); + const right = Float32Array.from(tone(128)); + expect(classify([left, right])).toMatchObject({ fill: "full", missing: 0 }); + }); + + test("all zeros is silent, however it came about", () => { + expect(classify(quantum(new Array(128).fill(0)))).toMatchObject({ fill: "silent", missing: 128 }); + expect(classify([])).toMatchObject({ fill: "silent", missing: 0 }); + }); +}); + +describe("Ledger", () => { + const RATE = 48000; + const full = classify(quantum(tone(128))); + const short = classify(quantum([...tone(64), ...new Array(64).fill(0)])); + const silent = classify(quantum(new Array(128).fill(0))); + + test("the tune-in before the first audible quantum is not a gap", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, silent); + ledger.add(128, 128, silent); + ledger.add(256, 128, full); + expect(ledger.counts).toEqual({ quanta: 1, quiet: 0 }); + expect(ledger.take()).toEqual([]); + }); + + test("consecutive unfilled quanta are one gap, closed by the next full one", () => { + const ledger = new Ledger(RATE, 0.001); + ledger.add(0, 128, full); + ledger.add(128, 128, short); + ledger.add(256, 128, silent); + ledger.add(384, 128, short); + expect(ledger.take()).toEqual([]); + + ledger.add(512, 128, full); + const [gap, ...rest] = ledger.take(); + expect(rest).toEqual([]); + // Starts at the first missing sample, half-way through the first short quantum. + expect(gap?.at).toBeCloseTo((192 / RATE) * 1000); + expect(gap?.ms).toBeCloseTo(((64 + 128 + 64) / RATE) * 1000); + expect(gap).toMatchObject({ quanta: 3, short: 2 }); + expect(ledger.counts).toEqual({ quanta: 5, quiet: 1 }); + }); +}); diff --git a/test/audio-quality/clients/js/src/quantum.ts b/test/audio-quality/clients/js/src/quantum.ts new file mode 100644 index 0000000000..e28ef166a3 --- /dev/null +++ b/test/audio-quality/clients/js/src/quantum.ts @@ -0,0 +1,112 @@ +/** + * What one render quantum at the player's output says about the ring behind it. + * + * The player's render worklet copies what its ring holds into the front of each output quantum and + * leaves the rest as the zeros Web Audio hands it. So a ring that ran dry shows up at the output as + * exact zeros at the end of a quantum across every channel, and an empty or stalled ring as a quantum + * of nothing but zeros. The published tone never lands on exact zero in every channel at once, which + * is what makes the output readable as the ring's fill level without a counter in the player. + * + * Free of worklet globals so the classification is unit tested rather than trusted. + * + * @module + */ +import type { Gap } from "./schema.ts"; + +/** How the ring filled one quantum. */ +export type Fill = "full" | "short" | "silent"; + +/** One quantum, classified. */ +export type Quantum = { + fill: Fill; + /** Samples per channel the ring did not supply: the trailing zeros, or the whole quantum. */ + missing: number; + /** Mean square over every sample of every channel. */ + power: number; +}; + +/** Classify one quantum of planar audio. No channels at all is a quantum that played nothing. */ +export function classify(channels: Float32Array[]): Quantum { + const length = channels[0]?.length ?? 0; + if (length === 0) return { fill: "silent", missing: 0, power: 0 }; + + let last = -1; + let sum = 0; + for (const channel of channels) { + for (let i = 0; i < channel.length; i++) { + const value = channel[i] ?? 0; + if (value !== 0 && i > last) last = i; + sum += value * value; + } + } + const power = sum / (length * channels.length); + if (last < 0) return { fill: "silent", missing: length, power }; + + const missing = length - 1 - last; + return { fill: missing > 0 ? "short" : "full", missing, power }; +} + +/** Cumulative counts since the first audible quantum. */ +export type Counts = { + /** Quanta rendered. */ + quanta: number; + /** Quanta under the silence floor, whatever the reason. */ + quiet: number; +}; + +/** + * Folds quanta into {@link Counts}, and closes a {@link Gap} each time the ring fills a quantum again. + * + * Nothing before the first non-silent quantum counts: the ring is filling for the first time, which + * is the tune-in, not a gap. + */ +export class Ledger { + readonly counts: Counts = { quanta: 0, quiet: 0 }; + readonly #rate: number; + readonly #floor: number; + #started = false; + #open: { frame: number; missing: number; quanta: number; short: number } | undefined; + #closed: Gap[] = []; + + /** `rate` is the context's sample rate; `floor` is the silence floor as an RMS level. */ + constructor(rate: number, floor: number) { + this.#rate = rate; + this.#floor = floor; + } + + /** Account for one quantum of `length` samples that started at `frame` on the render clock. */ + add(frame: number, length: number, quantum: Quantum): void { + if (!this.#started) { + if (quantum.fill === "silent") return; + this.#started = true; + } + + this.counts.quanta++; + if (Math.sqrt(quantum.power) < this.#floor) this.counts.quiet++; + if (quantum.fill === "full") { + this.#close(); + return; + } + + // A short quantum inside an open gap is a ring that got a little and ran out again: still one + // gap, since only a full quantum ends it. + this.#open ??= { frame: frame + length - quantum.missing, missing: 0, quanta: 0, short: 0 }; + const open = this.#open; + open.missing += quantum.missing; + open.quanta++; + if (quantum.fill === "short") open.short++; + } + + /** Gaps closed since the last call. A gap still open is reported once a full quantum ends it. */ + take(): Gap[] { + return this.#closed.splice(0, this.#closed.length); + } + + #close(): void { + if (!this.#open) return; + const ms = (samples: number) => (samples / this.#rate) * 1000; + const { frame, missing, quanta, short } = this.#open; + this.#closed.push({ at: ms(frame), ms: ms(missing), quanta, short }); + this.#open = undefined; + } +} diff --git a/test/audio-quality/clients/js/src/schema.ts b/test/audio-quality/clients/js/src/schema.ts new file mode 100644 index 0000000000..3e36798d1b --- /dev/null +++ b/test/audio-quality/clients/js/src/schema.ts @@ -0,0 +1,407 @@ +/** + * The metric contract: what the page samples, what a row reduces to, and what a budget grades. + * + * This file is the schema, not a description of one. The page emits {@link Sample}s, `analyze.ts` + * reduces them to a {@link Summary}, and `grade.ts` reads that against `budgets.json`. {@link METRICS} + * says for every graded number what it counts, which clock it sits on, and how a series became one + * value. Another lane (native playout, the latency ledger) that emits the same {@link Summary} is + * comparable to this one by construction. + * + * Conventions, stated once for everything below: + * + * - Every duration is milliseconds as a float. A sample count is converted at the rate it was + * counted at before it reaches a {@link Summary}. + * - Every count is a non-negative integer. Every share is a fraction of 1. + * - Every timestamp sits on a named {@link Clock} and is reduced to `viewer` before any two are + * subtracted. {@link Drift} records what each reduction took. + * + * @module + */ + +/** Milliseconds, as a float. The unit of every duration in this schema. */ +export type Ms = number; + +/** + * The clocks a timestamp can be taken on. + * + * - `viewer`: the page's `performance.now()`, monotonic. The reference every other clock is reduced + * to, so a stage span is always a difference of two `viewer` values. + * - `render`: the page's `AudioContext` frame clock, the audio device's oscillator. Same host, so it + * is reduced by an offset and a drift fitted over the run from paired readings (see + * {@link MAX_RENDER_DRIFT}). + * - `media`: the publisher's presentation timestamps, seen through the stream. Not a clock the viewer + * can read directly: its rate against `viewer` is fitted, its offset is unknowable from here. + * - `publisher`, `relay`: other processes, often other machines. A timestamp taken there is reduced + * to `viewer` by an offset measured over the same session, NTP style: the offset at the midpoint of + * a round trip, with half the round trip as its uncertainty. A stage whose uncertainty exceeds + * {@link identityTolerance} is reported unmeasured rather than guessed. The browser lane carries no + * such timestamp yet, so every stage that needs one is null here. + */ +export type Clock = "viewer" | "render" | "media" | "publisher" | "relay"; + +/** + * How a series became the one number a budget grades. + * + * - `total`: summed over the graded window. + * - `per_min`: `total` divided by the graded window's length in minutes. + * - `p50`, `p95`: nearest-rank percentile of the window's samples. + * - `max`: the largest value in the window. + * - `share`: the fraction of the window for which the condition held. + * - `last`: the value at the end of the window. + */ +export type Aggregation = "total" | "per_min" | "p50" | "p95" | "max" | "share" | "last"; + +/** What one metric is: enough to read a number off a summary without guessing. */ +export type MetricSpec = { + /** What the raw measurement is counted in, before aggregation. */ + unit: "ms" | "count" | "share"; + /** The clock its timestamps were taken on, before reduction to `viewer`. */ + clock: Clock; + /** The aggregations reported for it, and therefore the budget keys that can grade it. */ + aggregations: Aggregation[]; + /** One plain line saying what it measures. */ + description: string; +}; + +/** + * Every graded metric, keyed by its base name. + * + * A budget key is `_`: `underruns` graded per minute is `underruns_per_min`. Naming + * the aggregation in the key is what keeps "underruns: 3" and "underruns: 3/min" from being graded + * against each other. + */ +export const METRICS = { + underruns: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: + "Render quanta the ring could only partly fill, or not fill at all, while it was playing. A quantum rendered while the ring is stalled is not one.", + }, + short_quanta: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: "Of the underruns, the quanta the ring filled the front of and then ran dry part-way through.", + }, + underrun_episodes: { + unit: "count", + clock: "render", + aggregations: ["total", "per_min"], + description: "Maximal runs of consecutive underruns: one audible gap each, however many quanta it spanned.", + }, + underrun_ms: { + unit: "ms", + clock: "render", + aggregations: ["total", "per_min", "max"], + description: + "Audio the ring failed to supply while playing: the missing samples at the device rate. `max` is the longest episode.", + }, + skip_aheads: { + unit: "count", + clock: "viewer", + aggregations: ["total", "per_min"], + description: + "Re-anchors that discarded buffered audio: a forward step in the lag between the viewer clock and the playhead.", + }, + discarded_ms: { + unit: "ms", + clock: "media", + aggregations: ["total", "per_min"], + description: + "Media time the skip-aheads jumped over without playing. Audio dropped before it reached the ring moves no playhead and is not counted.", + }, + stalled: { + unit: "share", + clock: "viewer", + aggregations: ["share"], + description: + "Share of the window the ring spent stalled, refilling rather than playing: silence the player chose.", + }, + silence: { + unit: "share", + clock: "render", + aggregations: ["share"], + description: + "Share of rendered quanta under the silence floor at the player's output, whatever the cause. The published tone is never quiet, so any of it is a gap or a stall.", + }, + target_ms: { + unit: "ms", + clock: "viewer", + aggregations: ["p50", "p95", "max"], + description: "The playout delay the receiver resolved (`sync.out.delay`), sampled through the window.", + }, + converge_ms: { + unit: "ms", + clock: "viewer", + aggregations: ["last"], + description: + "From first audio to the last time the resolved target moved more than one bucket from its final value.", + }, + render_load: { + unit: "share", + clock: "render", + aggregations: ["max"], + description: + "Chromium's render capacity: the share of each quantum's budget the graph used. High says the runner, not the player, was short of time.", + }, +} as const satisfies Record; + +/** One of the {@link METRICS} names. */ +export type Metric = keyof typeof METRICS; + +/** Every `_` key, in schema order, so tables read the same every run. */ +export const METRIC_KEYS: string[] = Object.entries(METRICS).flatMap(([name, spec]) => + spec.aggregations.map((aggregation) => `${name}_${aggregation}`), +); + +/** A quantum whose RMS at the player's output is under this is quiet. About -60 dBFS. */ +export const SILENCE_RMS = 0.001; + +/** How often the page samples. */ +export const SAMPLE_INTERVAL_MS = 250; + +/** A resolved target within this of its final value has settled. One estimator bucket. */ +export const BUCKET_MS = 20; + +/** + * The render clock may run this far from the viewer's, as a fraction, before the row is void. + * + * Both oscillators are on one host, so real drift is parts per million. Past 1% the device is not + * running at wall rate at all, which is a throttled or stalled context, and every render-clock count + * would be read off a clock that was not moving. + */ +export const MAX_RENDER_DRIFT = 0.01; + +/** + * The stages an end-to-end delay splits into, in order, as exclusive spans. + * + * Exclusive is what makes the sum mean anything: each stage is defined by the two boundaries that + * end the previous stage and start the next, so no two can claim the same millisecond. + * + * - `capture`: sound at the publisher's input to its samples being readable. + * - `encode`: samples readable to the encoded frame existing. + * - `publish_flush`: frame existing to it being written to the transport, where a publisher that + * batches frames holds them. + * - `network`: written by the publisher to delivered to the viewer's container consumer, relay + * included. + * - `jitter_buffer`: delivered to its decoded samples leaving the ring for a render quantum. The + * decoder runs inside this span in a live pipeline, so it is not also `decode`. + * - `decode`: decode time that did not overlap the jitter buffer, which is zero while the ring holds + * anything. + * - `render`: leaving the ring to leaving the audio graph. + * - `device`: leaving the graph to the speaker (`outputLatency` plus `baseLatency`). + * - `unaccounted`: end-to-end minus the sum of the rest. Named because a large one is the finding. + * + * The identity is `sum(stages) == end-to-end`, within {@link identityTolerance}. + */ +export const STAGES = [ + "capture", + "encode", + "publish_flush", + "network", + "jitter_buffer", + "decode", + "render", + "device", + "unaccounted", +] as const; + +/** One of {@link STAGES}. */ +export type Stage = (typeof STAGES)[number]; + +/** A stage span and where its number came from. */ +export type StageSpan = { + /** The span, or null when this lane cannot measure it. */ + ms: Ms | null; + /** `measured` on the viewer clock, `declared` by the publisher, or `unmeasured`. */ + source: "measured" | "declared" | "unmeasured"; +}; + +/** The tolerance on the sum-to-end-to-end identity: the larger of 2 ms or 2% of the total. */ +export const identityTolerance = (endToEnd: Ms): Ms => Math.max(2, endToEnd * 0.02); + +/** What reducing one clock to `viewer` took. */ +export type Drift = { + /** Which clock. */ + clock: Clock; + /** Its rate against `viewer`, minus one: 0.001 runs 1 ms fast per second. Null when unmeasured. */ + rate: number | null; + /** What was added to put it on the viewer epoch. Null when unmeasured or unknowable. */ + offset: Ms | null; +}; + +// ── the row ───────────────────────────────────────────────────────────────── + +/** Which ring the page ran, decided by whether the document is cross-origin isolated. */ +export type Ring = "isolated" | "plain"; + +/** The audio codecs the matrix publishes. */ +export type Codec = "opus" | "aac"; + +/** + * One matrix cell. + * + * A budget is keyed by the whole thing. The codec, its rate, and the ring each move the expected + * floor as much as the profile does, so a profile-only key would grade one cell against another's + * threshold. + */ +export type Row = { + runtime: "chromium"; + codec: Codec; + /** Sample rate in Hz. */ + rate: number; + profile: string; + ring: Ring; +}; + +/** The row as a flat string: a file name and a table label. */ +export const rowKey = (row: Row): string => `${row.runtime}-${row.codec}-${row.rate}-${row.profile}-${row.ring}`; + +/** Parse {@link rowKey}. The profile is the only field that may contain a dash. */ +export function parseRow(key: string): Row { + const parts = key.split("-"); + const [runtime, codec, rate] = parts; + const ring = parts.at(-1); + const profile = parts.slice(3, -1).join("-"); + const hz = Number.parseInt(rate ?? "", 10); + if ( + runtime !== "chromium" || + (codec !== "opus" && codec !== "aac") || + (ring !== "isolated" && ring !== "plain") || + !Number.isFinite(hz) || + profile === "" + ) { + throw new Error(`not a row key: ${key}`); + } + return { runtime, codec, rate: hz, profile, ring }; +} + +// ── what the page emits ───────────────────────────────────────────────────── + +/** A run of consecutive quanta the ring did not fill, as the output tap saw it. */ +export type Gap = { + /** Where the first missing sample would have played, on the `render` clock. */ + at: Ms; + /** The missing audio, at the device rate. */ + ms: Ms; + /** Quanta in it the ring did not fill, short and silent alike. */ + quanta: number; + /** Of those, the quanta it filled the front of. */ + short: number; +}; + +/** A change in whether the ring is stalled, stamped on the `render` clock when the page saw it. */ +export type Stall = { + at: Ms; + stalled: boolean; +}; + +/** One probe sample: what the page could read at that instant, plus what the tap reported since. */ +export type Sample = { + /** `performance.now()`: the `viewer` clock. */ + at: Ms; + /** `AudioContext.currentTime`, read at the same instant: the `render` clock. */ + render?: Ms; + + /** `audio.out.timestamp`: the playhead on the `media` clock. Undefined while there is none. */ + timestamp?: Ms; + /** `audio.out.stalled`: the ring is refilling rather than playing. */ + stalled?: boolean; + /** `sync.out.delay`: the resolved playout target. */ + delay?: Ms; + + /** `AudioContext.outputLatency`. */ + outputLatency?: Ms; + /** `AudioContext.baseLatency`. */ + baseLatency?: Ms; + /** `AudioContext.renderCapacity` average load, Chromium only. */ + renderLoad?: number; + + /** The tap's cumulative quanta rendered since the first audible one. */ + quanta?: number; + /** The tap's cumulative quanta under {@link SILENCE_RMS}. */ + quiet?: number; + /** Gaps the tap closed since the previous sample. */ + gaps?: Gap[]; + /** Stall changes since the previous sample. */ + stalls?: Stall[]; +}; + +/** What the page reports once the session is up. */ +export type Environment = { + /** Whether the document is cross-origin isolated, and therefore which ring runs. */ + crossOriginIsolated: boolean; + /** The transport the session negotiated. Anything but WebTransport never crossed the UDP shaper. */ + transport?: string; + /** The audio rendition's codec string. */ + codec?: string; + /** The audio rendition's sample rate, Hz. */ + rate?: number; + /** The publisher's declared flush span: the `publish_flush` stage. */ + jitter?: Ms; + /** The AudioContext's rate, Hz, which the render clock counts in. */ + contextRate?: number; +}; + +// ── what a row reduces to ─────────────────────────────────────────────────── + +/** The shaper's counters for one direction, as its exit line prints them. */ +export type ShaperCounters = { + packets: number; + lost: number; + overflowed: number; + throttled: number; + delayed: number; + reordered: number; +}; + +/** The shaper's run: its seed, how it exited, and what it did. */ +export type Shaper = { + seed: number | null; + /** Its exit status. Nonzero means the profile never acted, or forwarding stopped. */ + status: number | null; + up: ShaperCounters | null; + down: ShaperCounters | null; +}; + +/** Why a row cannot be graded. A void row is reported, and fails an enforced run. */ +export type Void = { + /** The assertion that failed. */ + assertion: string; + /** What was seen instead. */ + detail: string; +}; + +/** Everything one row produced: the graded numbers and what makes them trustworthy. */ +export type Summary = { + /** Bumped when a metric's meaning changes, not when one is added. */ + version: 1; + row: Row; + /** Graded window, after the warmup. */ + windowMs: Ms; + /** Discarded after first audio, before grading. */ + warmupMs: Ms; + environment: Environment | null; + /** Empty means the row is gradeable. */ + voids: Void[]; + /** Every {@link METRIC_KEYS} entry. Null is unmeasured, never zero. */ + metrics: Record; + stages: Record; + /** The delay the stages sum to, or null when no clock spans the whole path. */ + endToEnd: Ms | null; + drift: Drift[]; + shaper: Shaper | null; + /** Console warnings and errors, truncated. */ + notes: string[]; +}; + +/** One budget: the whole row it applies to, and a ceiling per graded key. */ +export type Budget = Row & { [key: string]: number | string }; + +/** The checked-in budget file. */ +export type Budgets = { + /** How the ceilings were arrived at. */ + note: string; + rows: Budget[]; +}; diff --git a/test/audio-quality/clients/js/src/tap-worklet.ts b/test/audio-quality/clients/js/src/tap-worklet.ts new file mode 100644 index 0000000000..e5a02245fb --- /dev/null +++ b/test/audio-quality/clients/js/src/tap-worklet.ts @@ -0,0 +1,39 @@ +/** + * The harness's own AudioWorklet, fanned out from the player's output: it listens and never plays. + * + * It sees every render quantum rather than a 250 ms sample of them, which is what lets a gap of a few + * milliseconds between two probe samples be counted at all. See `quantum.ts` for how a quantum is + * read. + * + * @module + */ +import { classify, Ledger } from "./quantum.ts"; +import type { TapInit, TapReport } from "./tap.ts"; + +/** Quanta between reports: about 170 ms at 48 kHz, inside the probe's 250 ms. */ +const REPORT_EVERY = 64; + +class Tap extends AudioWorkletProcessor { + #ledger: Ledger; + #since = 0; + + constructor(options: AudioWorkletNodeOptions) { + super(); + const init = options.processorOptions as TapInit; + this.#ledger = new Ledger(sampleRate, init.floor); + } + + process(inputs: Float32Array[][]): boolean { + const input = inputs[0] ?? []; + this.#ledger.add(currentFrame, input[0]?.length ?? 128, classify(input)); + + if (++this.#since >= REPORT_EVERY) { + this.#since = 0; + const report: TapReport = { counts: { ...this.#ledger.counts }, gaps: this.#ledger.take() }; + this.port.postMessage(report); + } + return true; + } +} + +registerProcessor("audio-quality-tap", Tap); diff --git a/test/audio-quality/clients/js/src/tap.ts b/test/audio-quality/clients/js/src/tap.ts new file mode 100644 index 0000000000..7d0ac4dcc4 --- /dev/null +++ b/test/audio-quality/clients/js/src/tap.ts @@ -0,0 +1,70 @@ +/** + * Fan the tap worklet out from a player's output node, and collect what it reports. + * + * The tap reaches the destination through a zero gain, because Web Audio only renders what the + * destination pulls: left dangling it would never be asked for a quantum. The zero gain keeps it out + * of what the listener hears. + * + * @module + */ +import type { Counts } from "./quantum.ts"; +import type { Gap } from "./schema.ts"; +import TapWorklet from "./tap-worklet.ts?worklet"; + +/** What the worklet is constructed with. */ +export type TapInit = { + /** The silence floor, as an RMS level. */ + floor: number; +}; + +/** What the worklet posts every few dozen quanta. */ +export type TapReport = { + /** Cumulative since the first audible quantum. */ + counts: Counts; + /** Gaps closed since the last report. */ + gaps: Gap[]; +}; + +/** A running tap. */ +export type Tap = { + /** The latest cumulative counts, or undefined before the first report. */ + counts(): Counts | undefined; + /** Gaps closed since the last call. */ + take(): Gap[]; + close(): void; +}; + +/** Attach a tap to `root`. */ +export async function tap(root: AudioNode, init: TapInit): Promise { + const context = root.context as AudioContext; + await context.audioWorklet.addModule(TapWorklet); + + const node = new AudioWorkletNode(context, "audio-quality-tap", { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: init, + }); + const mute = new GainNode(context, { gain: 0 }); + root.connect(node); + node.connect(mute); + mute.connect(context.destination); + + let counts: Counts | undefined; + const gaps: Gap[] = []; + node.port.onmessage = (event: MessageEvent) => { + counts = event.data.counts; + gaps.push(...event.data.gaps); + }; + + return { + counts: () => counts, + take: () => gaps.splice(0, gaps.length), + close() { + node.port.onmessage = null; + root.disconnect(node); + node.disconnect(); + mute.disconnect(); + }, + }; +} diff --git a/test/audio-quality/clients/js/tsconfig.json b/test/audio-quality/clients/js/tsconfig.json new file mode 100644 index 0000000000..5a536d16f4 --- /dev/null +++ b/test/audio-quality/clients/js/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../../../js/tsconfig.json", + "compilerOptions": { + "types": ["audioworklet", "bun", "vite/client"] + }, + "include": ["analyze.ts", "driver.ts", "grade.ts", "src", "vite.config.ts", "../../../../js/common/worklet.d.ts"] +} diff --git a/test/audio-quality/clients/js/vite.config.ts b/test/audio-quality/clients/js/vite.config.ts new file mode 100644 index 0000000000..c971d84052 --- /dev/null +++ b/test/audio-quality/clients/js/vite.config.ts @@ -0,0 +1,19 @@ +/** + * Vite config for the audio quality page. `@moq/watch` is consumed as workspace source, so its render + * worklet is inlined as a blob URL by the same plugin its own build uses, along with the harness's tap + * worklet: a worklet fetched over the network would be one more thing that can stall inside the + * measurement. + * + * `base: "./"` because one build is served under both `/isolated/` and `/plain/`. + * + * @module + */ +import { defineConfig } from "vite"; +import { workletInline } from "../../../../js/common/vite-plugin-worklet"; + +/** esnext keeps WebCodecs / WebTransport syntax intact for headless Chromium. */ +export default defineConfig({ + base: "./", + plugins: [workletInline()], + build: { target: "esnext", outDir: "dist" }, +}); diff --git a/test/audio-quality/relay.toml b/test/audio-quality/relay.toml new file mode 100644 index 0000000000..5b8ed37b1d --- /dev/null +++ b/test/audio-quality/relay.toml @@ -0,0 +1,22 @@ +# Relay config for the audio quality harness. +# Anonymous access, self-signed localhost cert, QUIC + HTTP on 127.0.0.1:4443. +# +# run.sh rewrites the port into a copy in the run directory. The page reaches QUIC through the +# shaper on another port; the driver answers the page's /certificate.sha256 fetch with this relay's +# hash, so the shaper only carries UDP. + +[log] +level = "info" + +[listen] +# QUIC on UDP. 127.0.0.1 avoids IPv6 flakiness on CI runners. +bind = "127.0.0.1:4443" +tls.generate = ["localhost", "127.0.0.1"] + +[web.http] +# HTTP on TCP, serving /certificate.sha256 to the publishers and the driver. +listen = "127.0.0.1:4443" + +[auth] +# Allow anonymous access to everything. +public = "**" diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh new file mode 100755 index 0000000000..f50b59e2c8 --- /dev/null +++ b/test/audio-quality/run.sh @@ -0,0 +1,298 @@ +#!/usr/bin/env bash +# Plays a tone in headless Chromium over a seeded, impaired UDP path, and grades what it heard. +# +# One row is one page playing one codec through one shaper profile on one ring, for --duration +# seconds of audio. See README.md for the profiles, the metric contract, and the void rules. +set -euo pipefail + +AQ_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$AQ_DIR/../.." && pwd) +CLIENT="$AQ_DIR/clients/js" + +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$AQ_DIR/../lib/harness.sh" + +# Captured before the parse consumes it, so the rerun command carries every flag. +RERUN="$(harness_env AQ_PROFILE RELAY_BIN)just test audio-quality$(harness_argv "$@")" + +ALL_PROFILES=(near-zero mild wide fixed-250) +ALL_RINGS=(isolated plain) +ALL_CODECS=(opus aac) + +PROFILES=("${ALL_PROFILES[@]}") +RINGS=("${ALL_RINGS[@]}") +CODECS=("${ALL_CODECS[@]}") +DURATION=60 +SEED=7 +OUT="" +LIST=0 +ENFORCE=0 +PROFILE="${AQ_PROFILE:-debug}" + +split() { + local IFS=, + # shellcheck disable=SC2206 # splitting on commas is the point + SPLIT=($1) +} + +need() { + if [[ $2 -lt 2 || -z "${3:-}" || "${3:-}" == -* ]]; then + echo "error: $1 requires a value" >&2 + exit 2 + fi +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --profiles) + need "$1" $# "${2:-}" + split "$2" + PROFILES=("${SPLIT[@]}") + shift 2 + ;; + --rings) + need "$1" $# "${2:-}" + split "$2" + RINGS=("${SPLIT[@]}") + shift 2 + ;; + --codecs) + need "$1" $# "${2:-}" + split "$2" + CODECS=("${SPLIT[@]}") + shift 2 + ;; + --duration) + need "$1" $# "${2:-}" + DURATION="$2" + shift 2 + ;; + --seed) + need "$1" $# "${2:-}" + SEED="$2" + shift 2 + ;; + --out) + need "$1" $# "${2:-}" + OUT="$2" + shift 2 + ;; + --list) + LIST=1 + shift + ;; + --enforce) + ENFORCE=1 + shift + ;; + *) + echo "unknown arg: $1" >&2 + exit 2 + ;; + esac +done + +if [[ ! "$DURATION" =~ ^[0-9]+$ ]] || ((DURATION <= 5)); then + echo "error: --duration must be whole seconds, more than the 5s warmup (got '$DURATION')" >&2 + exit 2 +fi +if [[ ! "$SEED" =~ ^[0-9]+$ ]]; then + echo "error: --seed must be a non-negative integer (got '$SEED')" >&2 + exit 2 +fi + +valid() { + local want="$1" name="$2" + shift 2 + local known + for known in "$@"; do + [[ "$known" == "$want" ]] && return 0 + done + echo "error: unknown $name '$want' (known: $*)" >&2 + exit 2 +} +for p in "${PROFILES[@]}"; do valid "$p" profile "${ALL_PROFILES[@]}"; done +for r in "${RINGS[@]}"; do valid "$r" ring "${ALL_RINGS[@]}"; done +for c in "${CODECS[@]}"; do valid "$c" codec "${ALL_CODECS[@]}"; done + +if [[ $LIST -eq 0 && -n "$OUT" && -d "$OUT" ]] && [[ -n "$(ls -A "$OUT" 2>/dev/null)" ]]; then + echo "error: --out $OUT is not empty; remove it or name a new directory" >&2 + exit 2 +fi + +# The rate is part of the row key: a 44.1 kHz stream's frames do not land on a 48 kHz quantum. +rate_of() { + case "$1" in + opus) echo 48000 ;; + aac) echo 44100 ;; + esac +} + +# Every profile but the control adapts. The element requires a unit on a fixed delay. +delay_of() { + case "$1" in + fixed-250) echo "250ms" ;; + *) echo "auto" ;; + esac +} + +# The path treatment, as moq-shaper flags. Loss, reorder, and rate limits stay off: the buffer's job +# is absorbing arrival spread, and congestion response would make a failure hard to attribute. +shaper_of() { + case "$1" in + near-zero | fixed-250) ;; + mild) echo "--delay 5ms --jitter 5ms" ;; + wide) echo "--delay 40ms --jitter 40ms" ;; + esac +} + +ROWS=() +for codec in "${CODECS[@]}"; do + for profile in "${PROFILES[@]}"; do + for ring in "${RINGS[@]}"; do + ROWS+=("chromium-$codec-$(rate_of "$codec")-$profile-$ring") + done + done +done + +if [[ $LIST -eq 1 ]]; then + printf '%s\n' "${ROWS[@]}" + echo "${#ROWS[@]} rows at ${DURATION}s each, seed $SEED" >&2 + exit 0 +fi + +for tool in cargo bun ffmpeg; do + command -v "$tool" >/dev/null 2>&1 || { + echo "error: $tool not found; run inside 'nix develop'" >&2 + exit 1 + } +done + +harness_begin audio-quality "$RERUN" + +# ── build ─────────────────────────────────────────────────────────────────── +flag=() +[[ "$PROFILE" == "debug" ]] || flag=(--profile "$PROFILE") +echo "building moq-relay, moq, moq-shaper ($PROFILE)..." +(cd "$WORKSPACE" && cargo build --locked ${flag[@]+"${flag[@]}"} -p moq-relay -p moq-cli -p moq-shaper) +TARGET_BASE="${CARGO_TARGET_DIR:-$WORKSPACE/target}" +RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" +MOQ="$TARGET_BASE/$PROFILE/moq" +SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" + +echo "building the page..." +( + cd "$CLIENT" + bun install --frozen-lockfile + bunx playwright install chromium + bunx vite build +) + +# ── relay ─────────────────────────────────────────────────────────────────── +harness_port relay +RELAY_PORT="$HARNESS_PORT" +RELAY_URL="http://127.0.0.1:$RELAY_PORT" +sed "s/4443/$RELAY_PORT/g" "$AQ_DIR/relay.toml" >"$HARNESS_RUN/relay.toml" +harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" +if ! harness_ready "$RELAY_URL/certificate.sha256" 30 "$HARNESS_PID"; then + echo "relay never became ready" >&2 + sed 's/^/ relay: /' "$HARNESS_RUN/relay.log" >&2 || true + exit 1 +fi +harness_endpoint relay "$RELAY_URL" + +# ── publishers ────────────────────────────────────────────────────────────── +# A sine tone, never silent, so any quiet quantum at the player's output is a gap or a stall. The +# publishers talk to the relay directly: impairing the ingest would grade the receiver on a stream +# already damaged before it was published. The AAC arm keeps ffmpeg's default PES packing, whose +# multi-frame bursts are the flush-span shape the reporter measured on the public relay (#3477). +# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' +publish_opus() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=48000" \ + -ac 2 -c:a libopus -b:a 128k \ + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-opus.hang" import fmp4 +} + +# shellcheck disable=SC2329 # invoked indirectly via 'harness_spawn' +publish_aac() { + ffmpeg -hide_banner -v error -re -f lavfi -i "sine=frequency=440:sample_rate=44100" \ + -ac 2 -c:a aac -b:a 128k -f mpegts - | + "$MOQ" --connect "$RELAY_URL" --broadcast "tone-aac.hang" import ts +} + +for codec in "${CODECS[@]}"; do + harness_spawn "pub-$codec" "$HARNESS_RUN/pub-$codec.log" "publish_$codec" +done + +# ── rows ──────────────────────────────────────────────────────────────────── +# One shaper port for the run: rows play one at a time. +harness_port shaper +SHAPER_PORT="$HARNESS_PORT" + +echo "" +echo "running ${#ROWS[@]} rows at ${DURATION}s each (seed $SEED)" +failed=0 +for tag in "${ROWS[@]}"; do + IFS=- read -r _ codec _ rest <<<"$tag" + ring="${rest##*-}" + profile="${rest%-*}" + echo "" + echo "── $tag ──" + + # shellcheck disable=SC2046 # the profile's flags are meant to split + harness_spawn "shaper-$tag" "$HARNESS_RUN/shaper-$tag.log" "$SHAPER" \ + --listen "127.0.0.1:$SHAPER_PORT" --target "127.0.0.1:$RELAY_PORT" --seed "$SEED" $(shaper_of "$profile") + shaper_pid="$HARNESS_PID" + # It prints its seed and profile once bound; a page dialing before that would lose its handshake. + until grep -q '^shaper:' "$HARNESS_RUN/shaper-$tag.log" 2>/dev/null; do + if harness_exited "$shaper_pid"; then + echo "shaper exited before binding for $tag" >&2 + sed 's/^/ shaper: /' "$HARNESS_RUN/shaper-$tag.log" >&2 || true + exit 1 + fi + sleep 0.05 + done + + status=0 + harness_spawn "driver-$tag" "$HARNESS_RUN/driver-$tag.log" bun "$CLIENT/driver.ts" \ + --url "http://127.0.0.1:$SHAPER_PORT" \ + --fingerprint "$RELAY_URL/certificate.sha256" \ + --broadcast "tone-$codec.hang" \ + --page "$CLIENT/dist" \ + --ring "$ring" \ + --delay "$(delay_of "$profile")" \ + --duration "$DURATION" \ + --tag "$tag" \ + --out "$HARNESS_RUN" + harness_wait "$HARNESS_PID" || status=$? + if [[ $status -ne 0 ]]; then + failed=1 + sed 's/^/ driver: /' "$HARNESS_RUN/driver-$tag.log" >&2 || true + fi + + # SIGTERM, not the harness's SIGKILL: the shaper prints its counters and checks the profile + # acted on the way out, and exits nonzero if it did not. + kill -TERM -- -"$shaper_pid" 2>/dev/null || true + shaper_status=0 + harness_wait "$shaper_pid" || shaper_status=$? + echo "$shaper_status" >"$HARNESS_RUN/shaper-$tag.status" + + bun "$CLIENT/analyze.ts" --run "$HARNESS_RUN" --row "$tag" >"$HARNESS_RUN/$tag.summary.md" || { + echo "analyze failed for $tag" >&2 + failed=1 + } +done + +# ── grade ─────────────────────────────────────────────────────────────────── +echo "" +grade=(bun "$CLIENT/grade.ts" --run "$HARNESS_RUN" --budgets "$AQ_DIR/budgets.json") +[[ $ENFORCE -eq 0 ]] || grade+=(--enforce) +"${grade[@]}" || failed=1 + +if [[ -n "$OUT" ]]; then + mkdir -p "$OUT" + cp -R "$HARNESS_RUN/." "$OUT/" + echo "saved: $OUT" +fi +exit "$failed" diff --git a/test/justfile b/test/justfile index a9a777d330..def6dbbcf4 100644 --- a/test/justfile +++ b/test/justfile @@ -70,6 +70,16 @@ media *args: drill *args: just rs test -p moq-relay --test drills {{ args }} +# Plays a tone in headless Chromium through moq-shaper and grades what the listener +# heard (underruns, skip-aheads, silence, the resolved delay) against +# audio-quality/budgets.json. The full matrix is 16 rows at 60 s; nightly runs it with +# --enforce. Narrow it with `--profiles mild --codecs opus --rings plain --duration 20`. +# See audio-quality/README.md. + +# Grade browser audio playout over an impaired path. +audio-quality *args: + ./audio-quality/run.sh "$@" + # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. From 1a89a2d43a230fc051de6ba60ce957b968613350 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 17:25:12 -0700 Subject: [PATCH 3/4] test(audio-quality): record budgets, build the page per run, check it on PRs Ceilings come from two full local runs plus one short one: the fixed-250 control measured zero on every counter, and every auto row underruns on today's shallow target. The page builds into the run directory so concurrent runs cannot empty each other's dist/, the JS check scope covers the new workspace, and the nightly job sets MOQ_TEST_RUNS on the step, where the runner context is allowed. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 6 +- js/justfile | 2 +- test/audio-quality/budgets.json | 229 +++++++++++++++++++++++++++++++- test/audio-quality/run.sh | 6 +- 4 files changed, 235 insertions(+), 8 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f72da3d430..c08d9fa6c6 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -191,9 +191,6 @@ jobs: runs-on: ubuntu-latest # Sixteen rows at a minute each, plus the relay, CLI, and shaper built from source. timeout-minutes: 60 - env: - # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. - MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs steps: - name: Free disk space uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main @@ -221,6 +218,9 @@ jobs: - name: Audio quality run: nix develop --command just test audio-quality --enforce shell: bash -leo pipefail {0} + env: + # Somewhere upload-artifact can reach; the harness default is under $TMPDIR. + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs # The run directory is the diagnosis: each process's log, the shaper's # counters, the raw samples, and every row's summary. The harness keeps it # on a failure without being asked. diff --git a/js/justfile b/js/justfile index 4be426446e..93e433102c 100644 --- a/js/justfile +++ b/js/justfile @@ -9,7 +9,7 @@ set working-directory := '.' # Paths a JS check depends on: every bun workspace (they are not all under js/, # see the root package.json) plus the config shared by all of them. `check`, # `fix`, and `ci` all skip themselves when a diff misses this. -scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' +scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/(audio-quality|interop)/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' # Run all checks by default. default: diff --git a/test/audio-quality/budgets.json b/test/audio-quality/budgets.json index 327fd2b15e..0cdde6b25c 100644 --- a/test/audio-quality/budgets.json +++ b/test/audio-quality/budgets.json @@ -1,4 +1,229 @@ { - "note": "placeholder", - "rows": [] + "note": "Measured 2026-09-28 on a shared Linux workstation: two full 60 s runs and one 20 s run, seed 7. Each ceiling is twice the worse run, rounded up to two significant figures, with a floor (5 underruns/min, 20 gap ms/min, 1 skip/min, 100 discarded ms/min, 1% stalled, 1% silence) so one runner hiccup does not fail a clean row; target_ms_p95 is the worst measured plus one 20 ms bucket. The fixed-250 control measured zero on every counter. Every auto row records today's shallow target (#3477: 43 ms for Opus, underrunning thousands of quanta a minute), so its ceilings catch a regression past that floor and nothing better; the jitter target work is expected to tighten them. Not yet measured on the nightly runner, which has no audio device; re-record from its first runs.", + "rows": [ + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "fixed-250", + "ring": "isolated", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 440 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "fixed-250", + "ring": "plain", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 440 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "mild", + "ring": "isolated", + "underruns_per_min": 270, + "underrun_ms_per_min": 780, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 160, + "stalled_share": 0.01, + "silence_share": 0.012, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "mild", + "ring": "plain", + "underruns_per_min": 570, + "underrun_ms_per_min": 1700, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 120, + "stalled_share": 0.01, + "silence_share": 0.026, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "near-zero", + "ring": "isolated", + "underruns_per_min": 760, + "underrun_ms_per_min": 2200, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.028, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "near-zero", + "ring": "plain", + "underruns_per_min": 1500, + "underrun_ms_per_min": 4400, + "skip_aheads_per_min": 17, + "discarded_ms_per_min": 690, + "stalled_share": 0.01, + "silence_share": 0.066, + "target_ms_p95": 210 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 430, + "underrun_ms_per_min": 1200, + "skip_aheads_per_min": 6.6, + "discarded_ms_per_min": 330, + "stalled_share": 0.01, + "silence_share": 0.018, + "target_ms_p95": 250 + }, + { + "runtime": "chromium", + "codec": "aac", + "rate": 44100, + "profile": "wide", + "ring": "plain", + "underruns_per_min": 540, + "underrun_ms_per_min": 1500, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 260, + "stalled_share": 0.01, + "silence_share": 0.03, + "target_ms_p95": 250 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "fixed-250", + "ring": "isolated", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 300 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "fixed-250", + "ring": "plain", + "underruns_per_min": 5, + "underrun_ms_per_min": 20, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.01, + "target_ms_p95": 300 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "mild", + "ring": "isolated", + "underruns_per_min": 4600, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 1, + "discarded_ms_per_min": 100, + "stalled_share": 0.01, + "silence_share": 0.19, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "mild", + "ring": "plain", + "underruns_per_min": 4800, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 2.2, + "discarded_ms_per_min": 130, + "stalled_share": 0.028, + "silence_share": 0.22, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "near-zero", + "ring": "isolated", + "underruns_per_min": 6100, + "underrun_ms_per_min": 16000, + "skip_aheads_per_min": 8.0, + "discarded_ms_per_min": 340, + "stalled_share": 0.034, + "silence_share": 0.25, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "near-zero", + "ring": "plain", + "underruns_per_min": 4600, + "underrun_ms_per_min": 12000, + "skip_aheads_per_min": 4.4, + "discarded_ms_per_min": 290, + "stalled_share": 0.01, + "silence_share": 0.19, + "target_ms_p95": 63 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "wide", + "ring": "isolated", + "underruns_per_min": 8100, + "underrun_ms_per_min": 22000, + "skip_aheads_per_min": 29, + "discarded_ms_per_min": 1700, + "stalled_share": 0.56, + "silence_share": 0.73, + "target_ms_p95": 82 + }, + { + "runtime": "chromium", + "codec": "opus", + "rate": 48000, + "profile": "wide", + "ring": "plain", + "underruns_per_min": 6700, + "underrun_ms_per_min": 17000, + "skip_aheads_per_min": 18, + "discarded_ms_per_min": 1300, + "stalled_share": 0.49, + "silence_share": 0.75, + "target_ms_p95": 82 + } + ] } diff --git a/test/audio-quality/run.sh b/test/audio-quality/run.sh index f50b59e2c8..c10544597e 100755 --- a/test/audio-quality/run.sh +++ b/test/audio-quality/run.sh @@ -180,12 +180,14 @@ RELAY="${RELAY_BIN:-$TARGET_BASE/$PROFILE/moq-relay}" MOQ="$TARGET_BASE/$PROFILE/moq" SHAPER="$TARGET_BASE/$PROFILE/moq-shaper" +# Into the run directory: vite empties its output first, so a shared dist/ would vanish under a +# concurrent run's page loads. echo "building the page..." ( cd "$CLIENT" bun install --frozen-lockfile bunx playwright install chromium - bunx vite build + bunx vite build --outDir "$HARNESS_RUN/page" --emptyOutDir ) # ── relay ─────────────────────────────────────────────────────────────────── @@ -259,7 +261,7 @@ for tag in "${ROWS[@]}"; do --url "http://127.0.0.1:$SHAPER_PORT" \ --fingerprint "$RELAY_URL/certificate.sha256" \ --broadcast "tone-$codec.hang" \ - --page "$CLIENT/dist" \ + --page "$HARNESS_RUN/page" \ --ring "$ring" \ --delay "$(delay_of "$profile")" \ --duration "$DURATION" \ From 8dc1be3bf0db668e8529a8498176d87b8d5a0ec8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 17:30:05 -0700 Subject: [PATCH 4/4] ci(nightly): run audio quality on ARM with the shared Rust cache Matches wasm.yml, which moved its Playwright lane to the ARM runners that cache.yml warms. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c08d9fa6c6..eed4b612bc 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -184,11 +184,11 @@ jobs: MOQ_STRICT: 1 # Plays the matrix in headless Chromium through moq-shaper and grades what the - # listener heard against test/audio-quality/budgets.json. x86 rather than arm, - # like interop.yml and wasm.yml, because that is where Playwright's Chromium runs. + # listener heard against test/audio-quality/budgets.json. ARM64 like wasm.yml, + # the only architecture cache.yml warms a Rust store for. audio-quality: name: Audio quality - runs-on: ubuntu-latest + runs-on: ubuntu-24.04-arm # Sixteen rows at a minute each, plus the relay, CLI, and shaper built from source. timeout-minutes: 60 steps: @@ -207,9 +207,7 @@ jobs: extra-substituters = https://kixelated.cachix.org extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - cache-on-failure: true + uses: ./.github/actions/rust-cache # Playwright's Chromium isn't in the nix devShell; `--with-deps` installs # its apt runtime libs, and run.sh's own install then no-ops. - name: Playwright Chromium