From f8e5249cabf49bbe4b229a044327b24e28bde8e3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 08:35:31 -0700 Subject: [PATCH 1/7] chore: claim nightly wire compatibility quest From 5ddd43454db40647e910a55479e4db965d8fe8d9 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 2 Oct 2026 09:29:55 -0700 Subject: [PATCH 2/7] test: add released wire compatibility lanes Co-Authored-By: GPT-6 --- .github/workflows/interop.yml | 10 ++ test/interop/README.md | 43 +++++ test/interop/clients/js-native/subscribe.ts | 11 +- test/interop/compat.sh | 164 ++++++++++++++++++++ test/interop/compat/client.ts | 64 ++++++++ test/interop/compat/container.rs | 45 ++++++ test/interop/compat/planned-breaks.json | 10 ++ test/interop/compat/resolve.test.ts | 36 +++++ test/interop/compat/resolve.ts | 92 +++++++++++ test/interop/compat/transport.ts | 27 ++++ test/interop/interop.sh | 90 ++++++++++- test/justfile | 6 + 12 files changed, 590 insertions(+), 8 deletions(-) create mode 100755 test/interop/compat.sh create mode 100644 test/interop/compat/client.ts create mode 100644 test/interop/compat/container.rs create mode 100644 test/interop/compat/planned-breaks.json create mode 100644 test/interop/compat/resolve.test.ts create mode 100644 test/interop/compat/resolve.ts create mode 100644 test/interop/compat/transport.ts diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 74a22e98bd..fc5f5d23bd 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -100,6 +100,16 @@ jobs: # and a passing one is deleted as usual. MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + # Registry installs and the release matrix run only on main, nightly or + # on demand. A dev branch intentionally permits unpublished API breaks. + - name: Released wire compatibility + if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' + run: nix develop --command just test wire-compat + shell: bash -leo pipefail {0} + env: + GH_TOKEN: ${{ github.token }} + MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs + - name: Interop negative control run: nix develop --command just test interop-negative shell: bash -leo pipefail {0} diff --git a/test/interop/README.md b/test/interop/README.md index 8d862ba42e..05f65af8b0 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -92,6 +92,49 @@ Subscriber names: `rust`, `python`, `go`, `js` (browser), `js-native-node`, A client whose source build fails fails only its own matrix cells (see `mark_broken` in `interop.sh`); it never aborts the rest of the run. +## Released wire compatibility + +`just test wire-compat` compares this checkout with the newest stable, +non-yanked crates.io and npm releases. The existing Interop workflow runs it +nightly and on demand on `main`. Registry installation is omitted from PR runs; +resolver and removed-version regressions run through `just test harness`. + +The run records exact releases, npm's resolved lockfile, both CLI help outputs, +and each matrix cell in the harness artifacts. It uses checksummed GitHub +binaries when available on Linux x86\_64, with an exact-version `cargo install` +fallback. Run inside `nix develop` with GitHub CLI registry access. + +Four sources sign JWTs: current and published Rust CLI, and current and +published `@moq/auth`. All four verify every token; the JavaScript readers also +assert normalized permission scope. Current and released `hang` and +`@moq/hang` encode and decode one another's catalogs and legacy frame headers, +checking the exact timestamp and payload. These are format checks, independent +of a media decoder. + +The session lanes derive supported drafts from each CLI's `--connect-version` +choices. The relay offers only the cell's version, and JavaScript checks the +negotiated version. Current Rust media publishers feed released Rust and JS +readers, then released publishers feed current readers, through both relay +sources. Rust exports must decode to a video frame through ffmpeg. The existing +JS subscriber reconstructs the catalog and decodes the container. JS publishers +write one group after actual subscriber demand, and opposite-source Rust +readers verify its bytes through a live subscription. Both Rust versions also +fetch the same completed media group and compare every frame's exact payload. +FETCH support is derived from each Rust library's protocol capabilities; older +lite drafts without FETCH are logged and skipped. Losing a published FETCH +capability fails the run. JS IETF FETCH is unsupported in both directions and +is logged as a capability boundary; Rust covers this operation. + +Checkout-only versions are logged and omitted. Removing a released version +fails before sessions start. A maintainer-approved deliberate removal belongs +in `compat/planned-breaks.json` as a protocol name mapped to its approval, +reason, and the exact affected release versions. Every exception is logged. +Entries become errors once an affected release changes or no longer offers the +version, forcing removal or a fresh review. The measured lite-07-wip exception +was approved because the draft is unpublished; future WIP drafts receive no +automatic exception. `dev` does not run +this release comparison because its unpublished breaks are intentional. + ## Media QA The matrix asks "did bytes arrive and did a pixel light up". That passes on a diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 59ec48610a..8e4e1c5ff4 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -12,6 +12,7 @@ * * @module */ +import { writeFileSync } from "node:fs"; import { parseArgs } from "node:util"; import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; @@ -36,6 +37,7 @@ const { positionals, values } = parseArgs({ url: { type: "string" }, broadcast: { type: "string" }, timeout: { type: "string", default: "20" }, + "track-file": { type: "string" }, }, }); @@ -50,7 +52,10 @@ if (role !== "subscribe" || !url || !broadcast || !Number.isFinite(timeoutMs) || async function run(): Promise { const origin = new Moq.Origin.Producer(); - const connection = await Moq.Connection.connect({ url: new URL(url as string), consume: origin }); + const connect = process.env.INTEROP_COMPAT_TRANSPORT + ? (await import(process.env.INTEROP_COMPAT_TRANSPORT)).connect + : Moq.Connection.connect; + const connection = await connect({ url: new URL(url as string), consume: origin }); let requested: Moq.Origin.Requesting | undefined; try { const path = Moq.Path.from(broadcast as string); @@ -74,6 +79,10 @@ async function run(): Promise { } const [name, config] = video; + if (values["track-file"]) { + writeFileSync(values["track-file"], name); + return; + } let format: Container.Format; if (config.container.kind === "legacy") { format = new Container.Legacy.Format(config); diff --git a/test/interop/compat.sh b/test/interop/compat.sh new file mode 100755 index 0000000000..f418375c6d --- /dev/null +++ b/test/interop/compat.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# Compare this checkout with registry releases using the existing interop drivers. +set -euo pipefail +DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +WORKSPACE=$(cd "$DIR/../.." && pwd) +# shellcheck source-path=SCRIPTDIR source=../lib/harness.sh +source "$DIR/../lib/harness.sh" +harness_begin wire-compat "just test wire-compat$(harness_argv "$@")" +[[ $# == 0 ]] || { + echo "wire-compat takes no arguments" >&2 + exit 2 +} +cd "$WORKSPACE" +bun install --frozen-lockfile +bun "$DIR/compat/resolve.ts" resolve "$HARNESS_RUN/versions.json" +value() { python3 -c 'import json,sys; print(json.load(open(sys.argv[1]))[sys.argv[2]])' "$HARNESS_RUN/versions.json" "$1"; } + +# The binary asset and registry version must agree. A missing asset builds the +# exact published crate instead; a corrupt/mismatched asset is a hard failure. +release_binary() { + local crate="$1" binary="$2" version asset digest url + version=$(value "$crate") + mkdir -p "$HARNESS_RUN/released/bin" + if gh release view "$crate-v$version" --repo moq-dev/moq --json assets >"$HARNESS_RUN/$crate-assets.json" 2>"$HARNESS_RUN/$crate-assets.log"; then + asset=$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(next((a["url"] for a in d["assets"] if a["name"]==sys.argv[2]),""))' "$HARNESS_RUN/$crate-assets.json" "$crate-v$version-x86_64-unknown-linux-gnu") + else + asset="" + fi + if [[ -n "$asset" ]]; then + # Release assets are an optimization for Linux x86_64, the nightly host. + [[ "$(uname -sm)" == "Linux x86_64" ]] || asset="" + fi + if [[ -n "$asset" ]]; then + url="https://github.com/moq-dev/moq/releases/download/$crate-v$version/SHA256SUMS" + curl -fLS "$url" -o "$HARNESS_RUN/$crate-sums" + digest=$(awk -v name="$crate-v$version-x86_64-unknown-linux-gnu" '$2 == name { print $1 }' "$HARNESS_RUN/$crate-sums") + [[ "$digest" =~ ^[0-9a-f]{64}$ ]] || { + echo "release is missing its asset checksum" >&2 + exit 1 + } + curl -fLS "$asset" -o "$HARNESS_RUN/released/bin/$binary" + printf '%s %s\n' "$digest" "$HARNESS_RUN/released/bin/$binary" | sha256sum -c - + chmod +x "$HARNESS_RUN/released/bin/$binary" + else + cargo install --locked --version "$version" --root "$HARNESS_RUN/released" --target-dir "$WORKSPACE/target/compat-released" "$crate" + fi + "$HARNESS_RUN/released/bin/$binary" --version +} +release_binary moq-cli moq +release_binary moq-relay moq-relay +cargo build --locked -p moq-cli -p moq-relay +TARGET=$(cargo metadata --no-deps --format-version 1 | python3 -c 'import json,sys;print(json.load(sys.stdin)["target_directory"])') +CURRENT="$TARGET/debug/moq" +RELEASED="$HARNESS_RUN/released/bin/moq" +"$CURRENT" --help >"$HARNESS_RUN/current-help" +"$RELEASED" --help >"$HARNESS_RUN/released-help" +bun "$DIR/compat/resolve.ts" matrix "$HARNESS_RUN/current-help" "$HARNESS_RUN/released-help" "$DIR/compat/planned-breaks.json" "$HARNESS_RUN/shared" "$HARNESS_RUN/versions.json" + +# A separate package root prevents bun workspaces from replacing released deps +# with the checkout. Copy exactly the same clients into both environments. +mkdir -p "$HARNESS_RUN/js" +cp "$DIR/compat/client.ts" "$DIR/compat/transport.ts" "$DIR/clients/js-native/subscribe.ts" "$HARNESS_RUN/js/" +python3 - "$HARNESS_RUN/versions.json" "$HARNESS_RUN/js/package.json" <<'PY' +import json, sys +versions = json.load(open(sys.argv[1])) +deps = {name: versions[name] for name in ("@moq/net", "@moq/hang", "@moq/auth")} +deps.update({"@moq/web-transport": "^0.1.4", "@moq/json": "*", "tsx": "^4.23.15"}) +json.dump({"private": True, "type": "module", "dependencies": deps}, open(sys.argv[2], "w")) +PY +(cd "$HARNESS_RUN/js" && bun install) +cp "$HARNESS_RUN/js/bun.lock" "$HARNESS_RUN/resolved-js.lock" + +# JWT verification compares normalized permissions, not just signature validity. +"$CURRENT" auth generate --out "$HARNESS_RUN/key.jwk" +for source in current released js-current js-released; do + case "$source" in + current) "$CURRENT" auth sign --key "$HARNESS_RUN/key.jwk" --root compat --publish 'video/**' --subscribe '**' ;; + released) "$RELEASED" auth sign --key "$HARNESS_RUN/key.jwk" --root compat --publish 'video/**' --subscribe '**' ;; + js-current) bun "$DIR/compat/client.ts" sign "$HARNESS_RUN/key.jwk" ;; + js-released) bun "$HARNESS_RUN/js/client.ts" sign "$HARNESS_RUN/key.jwk" ;; + esac >"$HARNESS_RUN/$source.jwt" + for verifier in "$CURRENT" "$RELEASED"; do + "$verifier" auth verify --key "$HARNESS_RUN/key.jwk" --in "$HARNESS_RUN/$source.jwt" >"$HARNESS_RUN/claims.json" + done + bun "$DIR/compat/client.ts" verify "$HARNESS_RUN/key.jwk" "$HARNESS_RUN/$source.jwt" + bun "$HARNESS_RUN/js/client.ts" verify "$HARNESS_RUN/key.jwk" "$HARNESS_RUN/$source.jwt" + echo "PASS token signed by $source" +done + +# Exercise the actual hang library release, independently of the version bundled +# into moq-cli. Compile one small, identical adapter against either source. +for source in current released; do + mkdir -p "$HARNESS_RUN/hang-$source/src" + cp "$DIR/compat/container.rs" "$HARNESS_RUN/hang-$source/src/main.rs" + if [[ "$source" == current ]]; then dependency="{ path = \"$WORKSPACE/rs/hang\" }"; else dependency="\"=$(value hang)\""; fi + cat >"$HARNESS_RUN/hang-$source/Cargo.toml" < released clients and the reverse, +# through either relay. The relay offers only one version, so JS cannot silently +# negotiate its preferred draft instead of the draft this cell names. +while read -r version; do + current_fetch=$("$TARGET/compat-hang-current/debug/compat-container" fetch-supported "$version") + released_fetch=$("$TARGET/compat-hang-released/debug/compat-container" fetch-supported "$version") + if [[ "$released_fetch" == true && "$current_fetch" != true ]]; then + echo "checkout removed published FETCH capability for $version" >&2 + exit 1 + fi + for relay_source in current released; do + if [[ "$relay_source" == current ]]; then relay="$TARGET/debug/moq-relay"; else relay="$HARNESS_RUN/released/bin/moq-relay"; fi + for source in current released; do + if [[ "$source" == current ]]; then + pub="$CURRENT" + sub="$RELEASED" + js="$HARNESS_RUN/js" + transport="$js/transport.ts" + else + pub="$RELEASED" + sub="$CURRENT" + js="$DIR/clients/js-native" + transport="$DIR/compat/transport.ts" + fi + echo "=== $version relay=$relay_source publisher=$source consumers=opposite ===" + RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + INTEROP_NATIVE_CLIENT="$js" INTEROP_COMPAT_TRANSPORT="$transport" INTEROP_DECODE=1 bash "$DIR/interop.sh" --subscribers rust,js-native-node + if [[ "$source" == current ]]; then publisher_js="$DIR/compat"; else publisher_js="$HARNESS_RUN/js"; fi + # Drive JS publication from actual subscriber demand. The CLI's newest-group + # read uses SUBSCRIBE here, independently of one-shot FETCH support. + RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + INTEROP_JS_PUBLISH_CLIENT="$publisher_js" INTEROP_READ_CURRENT=1 bash "$DIR/interop.sh" --publishers js-native --subscribers rust + if [[ "$current_fetch" == true && "$released_fetch" == true ]]; then + # JS does not implement IETF FETCH in either direction. Both Rust + # sources fetch the same completed group and compare exact payloads. + echo "INFO JS IETF FETCH unsupported; exercising both Rust implementations" + RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + INTEROP_COMPAT_TRANSPORT="$DIR/compat/transport.ts" INTEROP_FETCH_TRACK=1 bash "$DIR/interop.sh" --publishers rust --subscribers rust + else + echo "SKIP $version FETCH: not offered by both protocol implementations" + fi + done + done +done <"$HARNESS_RUN/shared" +echo "wire compatibility: token, session, catalog, and container lanes passed" diff --git a/test/interop/compat/client.ts b/test/interop/compat/client.ts new file mode 100644 index 0000000000..1ebe1a2859 --- /dev/null +++ b/test/interop/compat/client.ts @@ -0,0 +1,64 @@ +import { readFileSync, writeFileSync } from "node:fs"; +import * as Auth from "@moq/auth"; +import * as Catalog from "@moq/hang/catalog"; +import * as Container from "@moq/hang/container"; +import * as Moq from "@moq/net"; +import { install } from "@moq/web-transport"; +import { connect } from "./transport"; + +const [command, ...args] = process.argv.slice(2); +if (command === "sign") { + const key = Auth.Key.parse(readFileSync(args[0], "utf8")); + console.log(await Auth.Key.sign(key, { root: "compat", publish: ["video/**"], subscribe: ["**"] })); +} else if (command === "verify") { + const key = Auth.Key.parse(readFileSync(args[0], "utf8")); + const claims = await Auth.Key.verify(key, readFileSync(args[1], "utf8").trim()); + if ( + claims.root !== "compat" || + JSON.stringify(claims.publish) !== '["video/**"]' || + JSON.stringify(claims.subscribe) !== '["**"]' + ) { + throw new Error(`token scope changed: ${JSON.stringify(claims)}`); + } +} else if (command === "decode") { + Catalog.RootSchema.parse(JSON.parse(readFileSync(args[0], "utf8"))); + const format = new Container.Legacy.Format("video"); + const decoded = format.decode(new Uint8Array(readFileSync(args[1]))); + if ( + decoded.length !== 1 || + decoded[0].timestamp !== 1000001 || + Buffer.from(decoded[0].payload).toString() !== "compat-frame" + ) { + throw new Error("legacy container decoded the wrong timestamp or payload"); + } +} else if (command === "encode") { + const catalog = Catalog.RootSchema.parse({ + video: { renditions: { video: { codec: "avc3.42001e", container: { kind: "legacy" } } } }, + audio: { renditions: {} }, + }); + writeFileSync(args[0], JSON.stringify(catalog)); + writeFileSync( + args[1], + Container.Legacy.encodeFrame(new TextEncoder().encode("compat-frame"), Moq.Time.Micro(1000001)), + ); +} else if (command === "publish") { + install(); + const origin = new Moq.Origin.Producer(); + const broadcast = origin.createBroadcast(Moq.Path.from(args[1])); + const track = broadcast.createTrack("data"); + broadcast.announce(); + const connection = await connect({ + url: new URL(args[0]), + publish: origin.consume(), + websocket: { enabled: false }, + }); + console.log("published"); + while (!track.used.peek()) await track.used.changed(); + const group = track.appendGroup(); + group.writeFrame({ + payload: new TextEncoder().encode("compat-fetch"), + timestamp: Moq.Time.Timestamp.fromMicros(Moq.Time.Micro(1)), + }); + group.close(); + await connection.closed; +} else throw new Error(`unknown compatibility client command: ${command}`); diff --git a/test/interop/compat/container.rs b/test/interop/compat/container.rs new file mode 100644 index 0000000000..c12e26a2bc --- /dev/null +++ b/test/interop/compat/container.rs @@ -0,0 +1,45 @@ +use hang::{ + catalog::Catalog, + container::{Frame, Timestamp}, +}; +use std::{env, fs}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = env::args().collect(); + match args[1].as_str() { + "fetch-supported" => { + let version: hang::moq_net::Version = args[2].parse()?; + let supported = match version { + hang::moq_net::Version::Lite(version) => version.has_track_stream(), + hang::moq_net::Version::Ietf(_) => true, + _ => return Err("unknown protocol family".into()), + }; + println!("{supported}"); + } + "encode" => { + let catalog = Catalog::<()>::from_str( + r#"{"video":{"renditions":{"video":{"codec":"avc3.42001e","container":{"kind":"legacy"}}}},"audio":{"renditions":{}}}"#, + )?; + fs::write(&args[2], catalog.to_json()?)?; + let frame = Frame { + timestamp: Timestamp::from_micros(1000001)?, + payload: "compat-frame".as_bytes().to_vec().into(), + }; + let mut bytes = bytes::BytesMut::new(); + frame.encode(&mut bytes)?; + fs::write(&args[3], bytes)?; + } + "decode" => { + let catalog = Catalog::<()>::from_str(&fs::read_to_string(&args[2])?)?; + assert_eq!( + catalog.video.renditions["video"].codec.to_string(), + "avc3.42001e" + ); + let frame = Frame::decode(bytes::Bytes::from(fs::read(&args[3])?))?; + assert_eq!(frame.timestamp, Timestamp::from_micros(1000001)?); + assert_eq!(frame.payload.as_ref(), b"compat-frame"); + } + _ => return Err("expected encode or decode".into()), + } + Ok(()) +} diff --git a/test/interop/compat/planned-breaks.json b/test/interop/compat/planned-breaks.json new file mode 100644 index 0000000000..0766109cc3 --- /dev/null +++ b/test/interop/compat/planned-breaks.json @@ -0,0 +1,10 @@ +{ + "moq-lite-07-wip": { + "reason": "Approved under /recommended: this draft is unpublished and its wire may change. Released @moq/net 0.4.1 cannot decode current relay announcements (long buffer) while the Rust reader succeeds. Compare stable lite and IETF drafts; reassess this measured exception when any affected release changes.", + "releases": { + "moq-cli": "0.13.0", + "moq-relay": "0.16.0", + "@moq/net": "0.4.1" + } + } +} diff --git a/test/interop/compat/resolve.test.ts b/test/interop/compat/resolve.test.ts new file mode 100644 index 0000000000..7cb0ca20e0 --- /dev/null +++ b/test/interop/compat/resolve.test.ts @@ -0,0 +1,36 @@ +import { expect, test } from "bun:test"; +import { matrix, newest, versions } from "./resolve"; + +test("a yanked or prerelease version cannot replace the installable release", () => { + expect( + newest([ + { version: "1.9.0" }, + { version: "1.10.0" }, + { version: "2.0.0", yanked: true }, + { version: "3.0.0-beta.1" }, + ]), + ).toBe("1.10.0"); +}); + +test("wrapped help adds drafts without a harness edit", () => { + expect( + versions("--connect-version value\n [possible values: moq-lite-06,\n moq-transport-99]\n--other value"), + ).toEqual(["moq-lite-06", "moq-transport-99"]); + expect(() => versions("--connect-version unrestricted")).toThrow(); +}); + +test("removing a published draft fails, while current-only drafts are logged", () => { + expect(() => matrix(["new"], ["old"], {})).toThrow("removed published"); + expect(matrix(["old", "new"], ["old"], {})).toEqual({ shared: ["old"], currentOnly: ["new"], planned: {} }); + const exception = { reason: "Approved migration", releases: { "moq-cli": "1.0.0" } }; + expect(matrix(["new"], ["old"], { old: exception }, { "moq-cli": "1.0.0" }).planned).toEqual({ old: exception }); + expect(() => matrix(["old"], ["old"], { old: exception }, { "moq-cli": "1.0.1" })).toThrow("stale"); +}); + +test("an approved wire break can retain its protocol name, but new drafts still fail", () => { + const exception = { reason: "Unpublished wire changed", releases: { "@moq/net": "0.4.1" } }; + expect(matrix(["wip"], ["wip"], { wip: exception }, { "@moq/net": "0.4.1" }).shared).toEqual([]); + expect(() => matrix(["wip"], ["wip", "unreviewed"], { wip: exception }, { "@moq/net": "0.4.1" })).toThrow( + "removed published", + ); +}); diff --git a/test/interop/compat/resolve.ts b/test/interop/compat/resolve.ts new file mode 100644 index 0000000000..1bfef7fcd1 --- /dev/null +++ b/test/interop/compat/resolve.ts @@ -0,0 +1,92 @@ +import { readFileSync, writeFileSync } from "node:fs"; + +/** Select registry releases that remain installable, excluding prereleases and yanked crates. */ +export function newest(versions: { version: string; yanked?: boolean }[]): string { + const candidates = versions.filter((v) => !v.yanked && /^\d+\.\d+\.\d+$/.test(v.version)); + candidates.sort((a, b) => Bun.semver.order(b.version, a.version)); + if (!candidates[0]) throw new Error("registry has no stable, non-yanked release"); + return candidates[0].version; +} + +/** Read the connect-version choices without maintaining a draft list. */ +export function versions(help: string): string[] { + const start = help.indexOf("--connect-version"); + if (start < 0) throw new Error("CLI has no --connect-version"); + const choices = help.slice(start).match(/\[possible values: ([\s\S]*?)\]/)?.[1]; + if (!choices) throw new Error("CLI does not enumerate its supported versions"); + const values = choices.split(",").map((v) => v.trim()); + if (!values.length || values.some((v) => !/^moq-(lite|transport)-\d+(?:-wip)?$/.test(v))) { + throw new Error(`malformed protocol choices: ${choices}`); + } + return values; +} + +/** One reviewed exception, bound to the releases whose incompatibility was measured. */ +export interface PlannedBreak { + reason: string; + releases: Record; +} + +/** Refuse removed released versions unless a maintainer acknowledged the exact releases. */ +export function matrix( + current: string[], + released: string[], + skipped: Record, + releases: Record = {}, +) { + for (const [version, exception] of Object.entries(skipped)) { + if (!released.includes(version) || !exception.reason?.trim() || !Object.keys(exception.releases).length) { + throw new Error(`invalid or stale planned break: ${version}`); + } + for (const [name, expected] of Object.entries(exception.releases)) { + if (releases[name] !== expected) + throw new Error( + `stale planned break: ${version}, ${name} changed from ${expected} to ${releases[name]}`, + ); + } + } + const missing = released.filter((v) => !current.includes(v) && !skipped[v]); + if (missing.length) throw new Error(`checkout removed published versions: ${missing.join(", ")}`); + return { + shared: released.filter((v) => current.includes(v) && !skipped[v]), + currentOnly: current.filter((v) => !released.includes(v)), + planned: skipped, + }; +} + +async function registry(url: string) { + const response = await fetch(url, { + headers: { "User-Agent": "moq-wire-compat (https://github.com/moq-dev/moq)" }, + }); + if (!response.ok) throw new Error(`registry ${url}: ${response.status}`); + return response.json(); +} + +if (import.meta.main) { + const [command, ...args] = process.argv.slice(2); + if (command === "resolve") { + const result: Record = {}; + for (const crate of ["moq-cli", "moq-relay", "hang"]) { + const data = await registry(`https://crates.io/api/v1/crates/${crate}`); + result[crate] = newest( + data.versions.map((v: { num: string; yanked: boolean }) => ({ version: v.num, yanked: v.yanked })), + ); + } + for (const name of ["@moq/net", "@moq/auth", "@moq/hang"]) { + const data = await registry(`https://registry.npmjs.org/${encodeURIComponent(name)}`); + result[name] = newest(Object.keys(data.versions).map((version) => ({ version }))); + } + writeFileSync(args[0], JSON.stringify(result, null, 2)); + console.log(JSON.stringify(result, null, 2)); + } else if (command === "matrix") { + const skipped = JSON.parse(readFileSync(args[2], "utf8")); + const result = matrix( + versions(readFileSync(args[0], "utf8")), + versions(readFileSync(args[1], "utf8")), + skipped, + JSON.parse(readFileSync(args[4], "utf8")), + ); + console.log(JSON.stringify(result, null, 2)); + writeFileSync(args[3], `${result.shared.join("\n")}\n`); + } else throw new Error("expected resolve or matrix"); +} diff --git a/test/interop/compat/transport.ts b/test/interop/compat/transport.ts new file mode 100644 index 0000000000..6b687e3680 --- /dev/null +++ b/test/interop/compat/transport.ts @@ -0,0 +1,27 @@ +import * as Moq from "@moq/net"; + +/** Pin the transport to one wire version, including drafts omitted from defaults. */ +export async function connect(props: Moq.Connection.ConnectProps) { + const version = process.env.INTEROP_VERSION; + if (!version) return Moq.Connection.connect(props); + const url = new URL(props.url); + const fingerprintUrl = new URL("/certificate.sha256", url); + const response = await fetch(fingerprintUrl); + if (!response.ok) throw new Error(`certificate endpoint: ${response.status}`); + const hash = (await response.text()).trim(); + if (!/^[a-f0-9]{64}$/i.test(hash)) throw new Error("malformed certificate fingerprint"); + url.protocol = "https:"; + // These three drafts share moql and select the precise version in SETUP. + const protocol = /^(moq-lite-0[12]|moq-transport-14)$/.test(version) ? "moql" : version; + const transport = new WebTransport(url, { + protocols: [protocol], + serverCertificateHashes: [{ algorithm: "sha-256", value: Uint8Array.from(Buffer.from(hash, "hex")) }], + }); + await transport.ready; + const connection = await Moq.Connection.connect({ ...props, transport }); + if (connection.version !== version) { + connection.close(); + throw new Error(`requested ${version}, negotiated ${connection.version}`); + } + return connection; +} diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 166257d7ae..216eb99742 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -24,7 +24,7 @@ source "$INTEROP_DIR/../lib/harness.sh" # Captured before the parse below consumes it, so the rerun command carries every # flag and every environment override this run was actually given. -RERUN="$(harness_env INTEROP_TIMEOUT INTEROP_FPS INTEROP_SIZE INTEROP_PORT INTEROP_PROFILE RELAY_BIN MOQ_BIN)just test interop$(harness_argv "$@")" +RERUN="$(harness_env INTEROP_TIMEOUT INTEROP_FPS INTEROP_SIZE INTEROP_PORT INTEROP_PROFILE RELAY_BIN MOQ_BIN INTEROP_SUB_MOQ INTEROP_VERSION INTEROP_NATIVE_CLIENT INTEROP_DECODE INTEROP_JS_PUBLISH_CLIENT INTEROP_COMPAT_TRANSPORT INTEROP_READ_CURRENT INTEROP_FETCH_TRACK)just test interop$(harness_argv "$@")" PUBLISHERS="rust" SUBSCRIBERS="rust" @@ -129,7 +129,7 @@ needs() { # True if any browser/native JS client is in play (they share one bun install). needs_js() { - needs js || needs js-native-node || needs js-native-bun + needs js || needs js-native || needs js-native-node || needs js-native-bun } harness_begin interop "$RERUN" @@ -182,6 +182,13 @@ require_tools() { # Build moq-relay + moq-cli from the workspace. The relay is the spine of the # test, so a failure here aborts rather than marking a single client broken. build_relay_cli() { + if [[ -n "$RELAY" && -n "$MOQ" ]]; then + [[ -x "$RELAY" && -x "$MOQ" ]] || { + echo "override binaries are not executable" >&2 + exit 1 + } + return + fi local flag=() [[ "$PROFILE" == "release" ]] && flag=(--release) echo "building moq-relay + moq-cli ($PROFILE)..." @@ -229,12 +236,26 @@ prepare_python() { # @moq/* packages resolve to this checkout's source) and build the browser page. prepare_js() { have bun || { - for v in js js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun not found"; done + for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun not found"; done return } + if [[ -n "${INTEROP_NATIVE_CLIENT:-}" ]]; then + [[ -f "$INTEROP_NATIVE_CLIENT/subscribe.ts" ]] || { + echo "missing staged native client" >&2 + exit 1 + } + return + fi + if [[ -n "${INTEROP_JS_PUBLISH_CLIENT:-}" ]]; then + [[ -f "$INTEROP_JS_PUBLISH_CLIENT/client.ts" ]] || { + echo "missing staged publisher" >&2 + exit 1 + } + return + fi echo "installing js clients (workspace @moq/* via bun)..." if ! (cd "$WORKSPACE" && bun install --frozen-lockfile) >"$HARNESS_RUN/js-install.log" 2>&1; then - for v in js js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done + for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun install failed"; done sed 's/^/ /' "$HARNESS_RUN/js-install.log" >&2 || true return fi @@ -409,6 +430,10 @@ echo "starting relay on 127.0.0.1:${PORT}..." # interop.toml is the source of truth; rewrite its port into a scratch copy so the # committed file never has to be edited for a run. sed "s/4443/${PORT}/g" "$INTEROP_DIR/interop.toml" >"$HARNESS_RUN/relay.toml" +if [[ -n "${INTEROP_VERSION:-}" ]]; then + # Version values come from the executable's advertised CLI choices. + sed -i "/\[listen\]/a version = [\"${INTEROP_VERSION}\"]" "$HARNESS_RUN/relay.toml" +fi harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" if ! harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"; then echo "relay never became ready" >&2 @@ -439,7 +464,7 @@ run_publisher() { local lang="$1" broadcast="$2" case "$lang" in rust) - ffmpeg_h264 | "$MOQ" --connect "$URL" --broadcast "$broadcast" import avc3 + ffmpeg_h264 | "$MOQ" --connect "$URL" ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} --broadcast "$broadcast" import avc3 ;; python) ffmpeg_h264 | "$PY" "$CLIENTS/python/interop.py" \ @@ -448,6 +473,9 @@ run_publisher() { go) ffmpeg_h264 | "$GO_INTEROP" publish --url "$URL" --broadcast "$broadcast" ;; + js-native) + bun "$INTEROP_JS_PUBLISH_CLIENT/client.ts" publish "$URL" "$broadcast" + ;; js) # Headless Chromium encodes its own H.264 from a fake camera via # WebCodecs (lazily, once a subscriber creates demand). @@ -477,7 +505,7 @@ start_publisher() { # shellcheck disable=SC2329 # reached from a function 'harness_spawn' invokes run_native() { local out - out=$( (cd "$CLIENTS/js-native" && "$@") 2>&1) || true + out=$( (cd "${INTEROP_NATIVE_CLIENT:-$CLIENTS/js-native}" && "$@") 2>&1) || true printf '%s\n' "$out" >&2 printf '%s\n' "$out" | grep -q '^received ' } @@ -489,8 +517,56 @@ run_subscriber() { rust) # moq-cli only handles SIGINT, so -k forces SIGKILL if it ignores the # SIGTERM that fires when no data arrives within the timeout. + if [[ "${INTEROP_FETCH_TRACK:-0}" == 1 ]]; then + # Discover the name from the decoded catalog, never today's naming convention. + (cd "$CLIENTS/js-native" && node --import tsx subscribe.ts subscribe --url "$URL" --broadcast "$broadcast" \ + --timeout "$TIMEOUT" --track-file "$HARNESS_RUN/$broadcast.track") || return + local track binary index=0 group_sequence="" + track=$(cat "$HARNESS_RUN/$broadcast.track") + for binary in "$MOQ" "${INTEROP_SUB_MOQ:-$MOQ}"; do + local group_flags=() + [[ -n "$group_sequence" ]] && group_flags=(--group "$group_sequence") + timeout -k 3 "$TIMEOUT" "$binary" --connect "$URL" ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} \ + --broadcast "$broadcast" fetch "$track" ${group_flags[@]+"${group_flags[@]}"} --json >"$HARNESS_RUN/$broadcast.$index.json" || return + # The first reader creates real demand and completes an actual group. + # The second must fetch that same retained group by its observed ID. + group_sequence=$(python3 -c 'import json,sys;print(json.loads(open(sys.argv[1]).readline())["group"])' "$HARNESS_RUN/$broadcast.$index.json") || return + index=$((index + 1)) + done + python3 - "$HARNESS_RUN/$broadcast.0.json" "$HARNESS_RUN/$broadcast.1.json" <<'PYCODE' +import base64, json, sys +outputs=[] +sequence=None +for path in sys.argv[1:]: + frames=[json.loads(line) for line in open(path)] + assert frames, "FETCH returned no frames" + payloads=[] + for index, frame in enumerate(frames): + payload=base64.b64decode(frame["payload"], validate=True) + if sequence is None: sequence=frame["group"] + assert frame["group"] == sequence and frame["frame"] == index and frame["size"] == len(payload), frame + payloads.append(payload) + outputs.append(payloads) +assert outputs[0] == outputs[1], "current/released FETCH changed the immutable group's payloads" +PYCODE + return + fi + if [[ "${INTEROP_READ_CURRENT:-0}" == 1 ]]; then + timeout -k 3 "$TIMEOUT" "${INTEROP_SUB_MOQ:-$MOQ}" --connect "$URL" \ + ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} --broadcast "$broadcast" fetch data >"$HARNESS_RUN/$broadcast.fetch" + [[ "$(cat "$HARNESS_RUN/$broadcast.fetch")" == "compat-fetch" ]] + return + fi + if [[ "${INTEROP_DECODE:-0}" == 1 ]]; then + # ffmpeg actually decodes the exported media; mux headers alone cannot pass. + (timeout -k 3 "$TIMEOUT" "${INTEROP_SUB_MOQ:-$MOQ}" --connect "$URL" \ + ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} --broadcast "$broadcast" export fmp4 || [[ "$?" == 141 ]]) | + ffmpeg -hide_banner -loglevel error -i - -frames:v 1 -f rawvideo -pix_fmt gray "$HARNESS_RUN/$broadcast.raw" + [[ -s "$HARNESS_RUN/$broadcast.raw" ]] + return + fi local n - n=$(timeout -k 3 "$TIMEOUT" "$MOQ" --connect "$URL" --broadcast "$broadcast" \ + n=$(timeout -k 3 "$TIMEOUT" "${INTEROP_SUB_MOQ:-$MOQ}" --connect "$URL" --broadcast "$broadcast" \ export fmp4 | head -c 1 | wc -c | tr -d ' ' || true) [[ "${n:-0}" -ge 1 ]] ;; diff --git a/test/justfile b/test/justfile index 1be6619e2c..40979b34d7 100644 --- a/test/justfile +++ b/test/justfile @@ -22,6 +22,7 @@ default: # which this recipe installs, as the interop and wasm harnesses do. harness: ./lib/harness.test.sh + cd .. && bun test test/interop/compat/resolve.test.ts cd .. && bun install --frozen-lockfile cd interop/clients/js && bunx playwright install chromium cd interop/clients/js && bun test ./harness.browser.ts @@ -39,6 +40,11 @@ interop *args: just rs test -p moq-net --lib varint_interop --run-ignored all ./interop/interop.sh "$@" +# Nightly main vs newest installable registry releases: tokens, pinned session +# versions, decoded catalogs/containers, and group FETCH. See interop/README.md. +wire-compat: + ./interop/compat.sh + # Negative control: no publisher, every subscriber must time out (proves the # harness can actually report failure). From 2ee92f9976563c1bec1c287e8ae874817a556e6b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 Oct 2026 10:35:47 -0700 Subject: [PATCH 3/7] test: make the released wire matrix report every break - Rust `auth verify` output is asserted against the expected root and scopes. - Every @moq package the clients import resolves to an exact release. - FETCH support per draft comes from the released relay, publisher, and CLI fetching from one another, replacing the per-family library probe. - The FETCH fixture fetches the group a live subscriber observed by its ID, discovered by the publisher's own JS, instead of the racy newest-group read. - IETF drafts 15+ connect with their `moqt-NN` ALPN. - The JS publisher fixture runs against both @moq/net APIs. - Every cell runs and the run lists each failing cell before failing. - The lite-07-wip exception is remeasured against the current releases. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/interop.yml | 2 +- test/interop/README.md | 27 +++--- test/interop/clients/js-native/subscribe.ts | 5 +- test/interop/compat.sh | 100 ++++++++++++++------ test/interop/compat/client.ts | 8 +- test/interop/compat/container.rs | 9 -- test/interop/compat/planned-breaks.json | 8 +- test/interop/compat/resolve.ts | 3 +- test/interop/compat/transport.ts | 7 +- test/interop/interop.sh | 17 ++-- 10 files changed, 111 insertions(+), 75 deletions(-) diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index c5f38065f8..6d0f55b894 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -107,7 +107,7 @@ jobs: MOQ_TEST_RUNS: ${{ runner.temp }}/moq-test-runs # Registry installs and the release matrix run only on main, nightly or - # on demand. A dev branch intentionally permits unpublished API breaks. + # on demand: a break only needs catching before the next release. - name: Released wire compatibility if: github.ref == 'refs/heads/main' && github.event_name != 'pull_request' run: nix develop --command just test wire-compat diff --git a/test/interop/README.md b/test/interop/README.md index 59824c3e5a..69879e0005 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -111,11 +111,11 @@ binaries when available on Linux x86\_64, with an exact-version `cargo install` fallback. Run inside `nix develop` with GitHub CLI registry access. Four sources sign JWTs: current and published Rust CLI, and current and -published `@moq/auth`. All four verify every token; the JavaScript readers also -assert normalized permission scope. Current and released `hang` and -`@moq/hang` encode and decode one another's catalogs and legacy frame headers, -checking the exact timestamp and payload. These are format checks, independent -of a media decoder. +published `@moq/auth`. Both Rust CLIs and both JavaScript packages verify every +token and assert its normalized root and permission scope. Current and released +`hang` and `@moq/hang` encode and decode one another's catalogs and legacy frame +headers, checking the exact timestamp and payload. These are format checks, +independent of a media decoder. The session lanes derive supported drafts from each CLI's `--connect-version` choices. The relay offers only the cell's version, and JavaScript checks the @@ -125,21 +125,20 @@ sources. Rust exports must decode to a video frame through ffmpeg. The existing JS subscriber reconstructs the catalog and decodes the container. JS publishers write one group after actual subscriber demand, and opposite-source Rust readers verify its bytes through a live subscription. Both Rust versions also -fetch the same completed media group and compare every frame's exact payload. -FETCH support is derived from each Rust library's protocol capabilities; older -lite drafts without FETCH are logged and skipped. Losing a published FETCH -capability fails the run. JS IETF FETCH is unsupported in both directions and -is logged as a capability boundary; Rust covers this operation. +FETCH the group a live subscriber is filling and compare every frame's exact +payload. The released relay, publisher, and CLI decide FETCH support per draft: +when they fetch from one another, every mixed cell must too; otherwise the +draft's FETCH is logged and skipped. JS IETF FETCH is unsupported in both +directions; Rust covers this operation. +Every cell runs, and the run lists each failing cell before it fails. Checkout-only versions are logged and omitted. Removing a released version fails before sessions start. A maintainer-approved deliberate removal belongs in `compat/planned-breaks.json` as a protocol name mapped to its approval, reason, and the exact affected release versions. Every exception is logged. Entries become errors once an affected release changes or no longer offers the -version, forcing removal or a fresh review. The measured lite-07-wip exception -was approved because the draft is unpublished; future WIP drafts receive no -automatic exception. `dev` does not run -this release comparison because its unpublished breaks are intentional. +version, forcing removal or a fresh review. Future WIP drafts receive no +automatic exception. ## Media QA diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 9673a786ed..5e7aecd6b6 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -80,7 +80,10 @@ async function run(): Promise { const [name, config] = video; if (values["track-file"]) { - writeFileSync(values["track-file"], name); + // Name a group that live demand is filling, so a FETCH of it never races eviction. + const group = await bc.track(name).subscribe({ priority: 0 }).recvGroup(); + if (!group) throw new Error(`${name} ended before its first group`); + writeFileSync(values["track-file"], `${name}\n${group.sequence}\n`); return; } let format: Container.Format; diff --git a/test/interop/compat.sh b/test/interop/compat.sh index f418375c6d..da0df8603b 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -63,8 +63,9 @@ cp "$DIR/compat/client.ts" "$DIR/compat/transport.ts" "$DIR/clients/js-native/su python3 - "$HARNESS_RUN/versions.json" "$HARNESS_RUN/js/package.json" <<'PY' import json, sys versions = json.load(open(sys.argv[1])) -deps = {name: versions[name] for name in ("@moq/net", "@moq/hang", "@moq/auth")} -deps.update({"@moq/web-transport": "^0.1.4", "@moq/json": "*", "tsx": "^4.23.15"}) +deps = {name: version for name, version in versions.items() if name.startswith("@moq/")} +# tsx only loads the TypeScript clients and never touches the wire. +deps["tsx"] = "^4.23.15" json.dump({"private": True, "type": "module", "dependencies": deps}, open(sys.argv[2], "w")) PY (cd "$HARNESS_RUN/js" && bun install) @@ -79,8 +80,21 @@ for source in current released js-current js-released; do js-current) bun "$DIR/compat/client.ts" sign "$HARNESS_RUN/key.jwk" ;; js-released) bun "$HARNESS_RUN/js/client.ts" sign "$HARNESS_RUN/key.jwk" ;; esac >"$HARNESS_RUN/$source.jwt" - for verifier in "$CURRENT" "$RELEASED"; do - "$verifier" auth verify --key "$HARNESS_RUN/key.jwk" --in "$HARNESS_RUN/$source.jwt" >"$HARNESS_RUN/claims.json" + for verifier in current released; do + if [[ "$verifier" == current ]]; then bin="$CURRENT"; else bin="$RELEASED"; fi + "$bin" auth verify --key "$HARNESS_RUN/key.jwk" --in "$HARNESS_RUN/$source.jwt" >"$HARNESS_RUN/$source.$verifier.claims" + # `auth verify` prints the claims' Rust Debug form, the CLI's only output. + python3 - "$HARNESS_RUN/$source.$verifier.claims" <<'PY' +import re, sys +text = re.sub(r"\s+", "", open(sys.argv[1]).read()) +def patterns(field): + block = re.search(field + r":Patterns\(\[(.*?)\]", text) + assert block, f"no {field} in {text}" + return re.findall(r'Pattern\("([^"]*)"\)', block.group(1)) +root = re.search(r'root:"([^"]*)"', text) +claims = (root and root.group(1), patterns("publish"), patterns("subscribe")) +assert claims == ("compat", ["video/**"], ["**"]), f"token scope changed: {claims}" +PY done bun "$DIR/compat/client.ts" verify "$HARNESS_RUN/key.jwk" "$HARNESS_RUN/$source.jwt" bun "$HARNESS_RUN/js/client.ts" verify "$HARNESS_RUN/key.jwk" "$HARNESS_RUN/$source.jwt" @@ -117,48 +131,72 @@ for source in current released js-current js-released; do echo "PASS catalog/container written by $source" done +# Every cell runs, so one nightly names every break; the run fails at the end. +failed=() +cell() { + local name="$1" + shift + echo "=== $name ===" + if "$@"; then + echo "PASS $name" + else + echo "FAIL $name" + failed+=("$name") + fi +} + +# FETCH support differs by draft and by build, so the released binaries alone +# decide it: when they fetch a group from one another, every mixed cell must too. +fetch() { + local relay="$1" pub="$2" sub="$3" js="$4" transport="$5" version="$6" + RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + INTEROP_NATIVE_CLIENT="$js" INTEROP_COMPAT_TRANSPORT="$transport" INTEROP_FETCH_TRACK=1 \ + bash "$DIR/interop.sh" --publishers rust --subscribers rust +} + # Existing media drivers: current publisher -> released clients and the reverse, # through either relay. The relay offers only one version, so JS cannot silently # negotiate its preferred draft instead of the draft this cell names. while read -r version; do - current_fetch=$("$TARGET/compat-hang-current/debug/compat-container" fetch-supported "$version") - released_fetch=$("$TARGET/compat-hang-released/debug/compat-container" fetch-supported "$version") - if [[ "$released_fetch" == true && "$current_fetch" != true ]]; then - echo "checkout removed published FETCH capability for $version" >&2 - exit 1 + released_relay="$HARNESS_RUN/released/bin/moq-relay" + if fetch "$released_relay" "$RELEASED" "$RELEASED" "$HARNESS_RUN/js" "$HARNESS_RUN/js/transport.ts" "$version" >"$HARNESS_RUN/$version.fetch-baseline.log" 2>&1; then + released_fetch=true + else + released_fetch=false + echo "SKIP $version FETCH: the released binaries do not fetch from one another ($HARNESS_RUN/$version.fetch-baseline.log)" fi for relay_source in current released; do - if [[ "$relay_source" == current ]]; then relay="$TARGET/debug/moq-relay"; else relay="$HARNESS_RUN/released/bin/moq-relay"; fi + if [[ "$relay_source" == current ]]; then relay="$TARGET/debug/moq-relay"; else relay="$released_relay"; fi for source in current released; do + # Each side's JS environment and the version-pinning transport beside it. if [[ "$source" == current ]]; then - pub="$CURRENT" - sub="$RELEASED" - js="$HARNESS_RUN/js" - transport="$js/transport.ts" + pub="$CURRENT" sub="$RELEASED" publisher_js="$DIR/compat" + own_js="$DIR/clients/js-native" own_transport="$DIR/compat/transport.ts" + other_js="$HARNESS_RUN/js" other_transport="$HARNESS_RUN/js/transport.ts" else - pub="$RELEASED" - sub="$CURRENT" - js="$DIR/clients/js-native" - transport="$DIR/compat/transport.ts" + pub="$RELEASED" sub="$CURRENT" publisher_js="$HARNESS_RUN/js" + own_js="$HARNESS_RUN/js" own_transport="$HARNESS_RUN/js/transport.ts" + other_js="$DIR/clients/js-native" other_transport="$DIR/compat/transport.ts" fi - echo "=== $version relay=$relay_source publisher=$source consumers=opposite ===" - RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ - INTEROP_NATIVE_CLIENT="$js" INTEROP_COMPAT_TRANSPORT="$transport" INTEROP_DECODE=1 bash "$DIR/interop.sh" --subscribers rust,js-native-node - if [[ "$source" == current ]]; then publisher_js="$DIR/compat"; else publisher_js="$HARNESS_RUN/js"; fi + name="$version relay=$relay_source publisher=$source" + cell "$name media" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + INTEROP_NATIVE_CLIENT="$other_js" INTEROP_COMPAT_TRANSPORT="$other_transport" INTEROP_DECODE=1 bash "$DIR/interop.sh" --subscribers rust,js-native-node # Drive JS publication from actual subscriber demand. The CLI's newest-group # read uses SUBSCRIBE here, independently of one-shot FETCH support. - RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + cell "$name js-publish" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ INTEROP_JS_PUBLISH_CLIENT="$publisher_js" INTEROP_READ_CURRENT=1 bash "$DIR/interop.sh" --publishers js-native --subscribers rust - if [[ "$current_fetch" == true && "$released_fetch" == true ]]; then - # JS does not implement IETF FETCH in either direction. Both Rust - # sources fetch the same completed group and compare exact payloads. - echo "INFO JS IETF FETCH unsupported; exercising both Rust implementations" - RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ - INTEROP_COMPAT_TRANSPORT="$DIR/compat/transport.ts" INTEROP_FETCH_TRACK=1 bash "$DIR/interop.sh" --publishers rust --subscribers rust - else - echo "SKIP $version FETCH: not offered by both protocol implementations" + if [[ "$released_fetch" == true ]]; then + # JS does not implement IETF FETCH in either direction, so both Rust + # sources fetch the same group and compare exact payloads. The + # publisher's own JS finds the group, so a catalog break fails only + # the media cell. + cell "$name fetch" fetch "$relay" "$pub" "$sub" "$own_js" "$own_transport" "$version" fi done done done <"$HARNESS_RUN/shared" +if ((${#failed[@]})); then + printf 'FAIL %s\n' "${failed[@]}" >&2 + exit 1 +fi echo "wire compatibility: token, session, catalog, and container lanes passed" diff --git a/test/interop/compat/client.ts b/test/interop/compat/client.ts index 1ebe1a2859..4c6e65a42f 100644 --- a/test/interop/compat/client.ts +++ b/test/interop/compat/client.ts @@ -45,7 +45,7 @@ if (command === "sign") { install(); const origin = new Moq.Origin.Producer(); const broadcast = origin.createBroadcast(Moq.Path.from(args[1])); - const track = broadcast.createTrack("data"); + const track = broadcast.createTrack("data", { timescale: Moq.Time.Timescale.MICRO }); broadcast.announce(); const connection = await connect({ url: new URL(args[0]), @@ -53,7 +53,11 @@ if (command === "sign") { websocket: { enabled: false }, }); console.log("published"); - while (!track.used.peek()) await track.used.changed(); + // `demand()` replaced the producer's own `used` after @moq/net 0.4.2, and this one + // client runs against both packages. + type Used = { peek(): boolean; changed(): Promise }; + const used: Used = "demand" in track ? track.demand().used : (track as unknown as { used: Used }).used; + while (!used.peek()) await used.changed(); const group = track.appendGroup(); group.writeFrame({ payload: new TextEncoder().encode("compat-fetch"), diff --git a/test/interop/compat/container.rs b/test/interop/compat/container.rs index c12e26a2bc..ab9752958b 100644 --- a/test/interop/compat/container.rs +++ b/test/interop/compat/container.rs @@ -7,15 +7,6 @@ use std::{env, fs}; fn main() -> Result<(), Box> { let args: Vec<_> = env::args().collect(); match args[1].as_str() { - "fetch-supported" => { - let version: hang::moq_net::Version = args[2].parse()?; - let supported = match version { - hang::moq_net::Version::Lite(version) => version.has_track_stream(), - hang::moq_net::Version::Ietf(_) => true, - _ => return Err("unknown protocol family".into()), - }; - println!("{supported}"); - } "encode" => { let catalog = Catalog::<()>::from_str( r#"{"video":{"renditions":{"video":{"codec":"avc3.42001e","container":{"kind":"legacy"}}}},"audio":{"renditions":{}}}"#, diff --git a/test/interop/compat/planned-breaks.json b/test/interop/compat/planned-breaks.json index 0766109cc3..e675bc4ef8 100644 --- a/test/interop/compat/planned-breaks.json +++ b/test/interop/compat/planned-breaks.json @@ -1,10 +1,10 @@ { "moq-lite-07-wip": { - "reason": "Approved under /recommended: this draft is unpublished and its wire may change. Released @moq/net 0.4.1 cannot decode current relay announcements (long buffer) while the Rust reader succeeds. Compare stable lite and IETF drafts; reassess this measured exception when any affected release changes.", + "reason": "Approved under /recommended: this draft is unpublished and its wire may change. Remeasured against these releases: released moq-cli and @moq/net reject current lite-07-wip sessions (long buffer, invalid value). Compare stable lite and IETF drafts; reassess this measured exception when any affected release changes.", "releases": { - "moq-cli": "0.13.0", - "moq-relay": "0.16.0", - "@moq/net": "0.4.1" + "moq-cli": "0.14.2", + "moq-relay": "0.17.2", + "@moq/net": "0.4.2" } } } diff --git a/test/interop/compat/resolve.ts b/test/interop/compat/resolve.ts index 1bfef7fcd1..34962ca2dd 100644 --- a/test/interop/compat/resolve.ts +++ b/test/interop/compat/resolve.ts @@ -72,7 +72,8 @@ if (import.meta.main) { data.versions.map((v: { num: string; yanked: boolean }) => ({ version: v.num, yanked: v.yanked })), ); } - for (const name of ["@moq/net", "@moq/auth", "@moq/hang"]) { + // Every package the clients import directly, so no range drifts between nightlies. + for (const name of ["@moq/net", "@moq/auth", "@moq/hang", "@moq/json", "@moq/web-transport"]) { const data = await registry(`https://registry.npmjs.org/${encodeURIComponent(name)}`); result[name] = newest(Object.keys(data.versions).map((version) => ({ version }))); } diff --git a/test/interop/compat/transport.ts b/test/interop/compat/transport.ts index 6b687e3680..37c5c7d9b2 100644 --- a/test/interop/compat/transport.ts +++ b/test/interop/compat/transport.ts @@ -11,8 +11,11 @@ export async function connect(props: Moq.Connection.ConnectProps) { const hash = (await response.text()).trim(); if (!/^[a-f0-9]{64}$/i.test(hash)) throw new Error("malformed certificate fingerprint"); url.protocol = "https:"; - // These three drafts share moql and select the precise version in SETUP. - const protocol = /^(moq-lite-0[12]|moq-transport-14)$/.test(version) ? "moql" : version; + // These three drafts share moql and select the precise version in SETUP. Later + // moq-transport drafts register `moqt-NN`, and later lite drafts their own name. + const protocol = /^(moq-lite-0[12]|moq-transport-14)$/.test(version) + ? "moql" + : version.replace(/^moq-transport-/, "moqt-"); const transport = new WebTransport(url, { protocols: [protocol], serverCertificateHashes: [{ algorithm: "sha-256", value: Uint8Array.from(Buffer.from(hash, "hex")) }], diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 52560f14c6..3a4408b8f6 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -519,19 +519,16 @@ run_subscriber() { # moq-cli only handles SIGINT, so -k forces SIGKILL if it ignores the # SIGTERM that fires when no data arrives within the timeout. if [[ "${INTEROP_FETCH_TRACK:-0}" == 1 ]]; then - # Discover the name from the decoded catalog, never today's naming convention. - (cd "$CLIENTS/js-native" && node --import tsx subscribe.ts subscribe --url "$URL" --broadcast "$broadcast" \ + # Discover the track from the decoded catalog, never today's naming + # convention, and a group live demand is filling. + (cd "${INTEROP_NATIVE_CLIENT:-$CLIENTS/js-native}" && node --import tsx subscribe.ts subscribe --url "$URL" --broadcast "$broadcast" \ --timeout "$TIMEOUT" --track-file "$HARNESS_RUN/$broadcast.track") || return - local track binary index=0 group_sequence="" - track=$(cat "$HARNESS_RUN/$broadcast.track") + local track group binary index=0 + { read -r track && read -r group; } <"$HARNESS_RUN/$broadcast.track" || return + # Both readers FETCH that group by its observed ID, which waits for it to finish. for binary in "$MOQ" "${INTEROP_SUB_MOQ:-$MOQ}"; do - local group_flags=() - [[ -n "$group_sequence" ]] && group_flags=(--group "$group_sequence") timeout -k 3 "$TIMEOUT" "$binary" --connect "$URL" ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} \ - --broadcast "$broadcast" fetch "$track" ${group_flags[@]+"${group_flags[@]}"} --json >"$HARNESS_RUN/$broadcast.$index.json" || return - # The first reader creates real demand and completes an actual group. - # The second must fetch that same retained group by its observed ID. - group_sequence=$(python3 -c 'import json,sys;print(json.loads(open(sys.argv[1]).readline())["group"])' "$HARNESS_RUN/$broadcast.$index.json") || return + --broadcast "$broadcast" fetch "$track" --group "$group" --json >"$HARNESS_RUN/$broadcast.$index.json" || return index=$((index + 1)) done python3 - "$HARNESS_RUN/$broadcast.0.json" "$HARNESS_RUN/$broadcast.1.json" <<'PYCODE' From 349c17ea9175db4bd64770af0fae2d21d371efd2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 Oct 2026 12:29:18 -0700 Subject: [PATCH 4/7] test: compare only moq-lite with releases and record known breaks - Drop IETF drafts from the released comparison: our clients and relays always prefer moq-lite with each other. - Planned breaks can scope to cells (lanes, versions, relay, publisher); each skipped cell is logged with the break's name. - Record the catalog `archive` reshape (#4034) for every media cell, and the lite-05/06 FETCH hang through a released relay from a current publisher. - The JS publisher lane FETCHes the group a live subscriber observed, instead of racing the newest group. Co-Authored-By: Claude Opus 5.5 --- test/interop/README.md | 42 ++++++----- test/interop/clients/js-native/subscribe.ts | 32 +++++---- test/interop/compat.sh | 44 +++++++----- test/interop/compat/planned-breaks.json | 25 ++++++- test/interop/compat/resolve.test.ts | 38 ++++++++-- test/interop/compat/resolve.ts | 78 +++++++++++++++++---- test/interop/compat/transport.ts | 8 +-- test/interop/interop.sh | 26 +++++-- test/justfile | 2 +- 9 files changed, 211 insertions(+), 84 deletions(-) diff --git a/test/interop/README.md b/test/interop/README.md index 69879e0005..d15c05dd1c 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -117,28 +117,32 @@ token and assert its normalized root and permission scope. Current and released headers, checking the exact timestamp and payload. These are format checks, independent of a media decoder. -The session lanes derive supported drafts from each CLI's `--connect-version` -choices. The relay offers only the cell's version, and JavaScript checks the -negotiated version. Current Rust media publishers feed released Rust and JS -readers, then released publishers feed current readers, through both relay -sources. Rust exports must decode to a video frame through ffmpeg. The existing -JS subscriber reconstructs the catalog and decodes the container. JS publishers -write one group after actual subscriber demand, and opposite-source Rust -readers verify its bytes through a live subscription. Both Rust versions also -FETCH the group a live subscriber is filling and compare every frame's exact -payload. The released relay, publisher, and CLI decide FETCH support per draft: -when they fetch from one another, every mixed cell must too; otherwise the -draft's FETCH is logged and skipped. JS IETF FETCH is unsupported in both -directions; Rust covers this operation. +The session lanes take the moq-lite drafts from each CLI's `--connect-version` +choices. IETF drafts are left out on purpose: our clients and relays always +prefer moq-lite with each other, and `just test interop` covers IETF. The relay +offers only the cell's version, and JavaScript checks the negotiated version. +Current Rust media publishers feed released Rust and JS readers, then released +publishers feed current readers, through both relay sources. Rust exports must +decode to a video frame through ffmpeg. The existing JS subscriber +reconstructs the catalog and decodes the container. + +The released relay, publisher, and CLI decide FETCH support per draft: when +they fetch from one another, every mixed cell must too; otherwise the draft's +FETCH lanes are logged and skipped. Both Rust versions FETCH the group the +publisher's own JS subscriber observed and compare every frame's exact +payload. JS publishers write one group after that subscriber's demand, and the +opposite-source Rust CLI FETCHes it by the same observed ID. Every cell runs, and the run lists each failing cell before it fails. Checkout-only versions are logged and omitted. Removing a released version -fails before sessions start. A maintainer-approved deliberate removal belongs -in `compat/planned-breaks.json` as a protocol name mapped to its approval, -reason, and the exact affected release versions. Every exception is logged. -Entries become errors once an affected release changes or no longer offers the -version, forcing removal or a fresh review. Future WIP drafts receive no -automatic exception. +fails before sessions start. A maintainer-approved break belongs in +`compat/planned-breaks.json` with its reason and the exact affected release +versions. Keyed by a protocol name it drops that version; with `cells` it +skips only the named lanes, optionally narrowed to versions, a relay source, +or a publisher source. Every skipped cell is logged with the break's name. +Entries become errors once an affected release changes or no longer offers a +listed version, forcing removal or a fresh review. Future WIP drafts receive +no automatic exception. ## Media QA diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 5e7aecd6b6..767bca5ab8 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -38,6 +38,7 @@ const { positionals, values } = parseArgs({ broadcast: { type: "string" }, timeout: { type: "string", default: "20" }, "track-file": { type: "string" }, + track: { type: "string" }, }, }); @@ -50,6 +51,20 @@ if (role !== "subscribe" || !url || !broadcast || !Number.isFinite(timeoutMs) || process.exit(2); } +// The .hang catalog lives on the "catalog.json" track. It's a @moq/json snapshot+delta +// value, reconstructed by Json.Snapshot.Consumer. A lazy publisher may announce video in a +// later update, so keep reading until one has it. +async function video(bc: Moq.Broadcast.Consumer): Promise<[string, Catalog.VideoConfig]> { + const track = bc.track("catalog.json").subscribe({ priority: Catalog.PRIORITY.catalog }); + const catalog = new Json.Snapshot.Consumer({ track, schema: Catalog.RootSchema }); + for (;;) { + const root = await catalog.next(); + if (!root) throw new Error("catalog ended without a video track"); + const first = Object.entries(root.video?.renditions ?? {})[0]; + if (first) return first; + } +} + async function run(): Promise { const origin = new Moq.Origin.Producer(); const connect = process.env.INTEROP_COMPAT_TRANSPORT @@ -66,26 +81,17 @@ async function run(): Promise { bc = requested.active.peek(); } - // The .hang catalog lives on the "catalog.json" track. It's a @moq/json - // snapshot+delta value, reconstructed by Json.Snapshot.Consumer. A lazy publisher may - // announce video in a later update, so keep reading until one has it. - const track = bc.track("catalog.json").subscribe({ priority: Catalog.PRIORITY.catalog }); - const catalog = new Json.Snapshot.Consumer({ track, schema: Catalog.RootSchema }); - let video: [string, Catalog.VideoConfig] | undefined; - while (!video) { - const root = await catalog.next(); - if (!root) throw new Error("catalog ended without a video track"); - video = Object.entries(root.video?.renditions ?? {})[0]; - } - - const [name, config] = video; if (values["track-file"]) { // Name a group that live demand is filling, so a FETCH of it never races eviction. + // The track is `--track`, or the catalog's first video rendition. + const name = values.track ?? (await video(bc))[0]; const group = await bc.track(name).subscribe({ priority: 0 }).recvGroup(); if (!group) throw new Error(`${name} ended before its first group`); writeFileSync(values["track-file"], `${name}\n${group.sequence}\n`); return; } + + const [name, config] = await video(bc); let format: Container.Format; if (config.container.kind === "legacy") { format = new Container.Legacy.Format(config); diff --git a/test/interop/compat.sh b/test/interop/compat.sh index da0df8603b..619f7d98dc 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -131,11 +131,18 @@ for source in current released js-current js-released; do echo "PASS catalog/container written by $source" done -# Every cell runs, so one nightly names every break; the run fails at the end. +# Every cell runs, so one nightly names every break; the run fails at the end. A +# cell a planned break covers is logged with that break's name instead. failed=() cell() { - local name="$1" - shift + local lane="$1" version="$2" relay_source="$3" source="$4" planned + shift 4 + local name="$version relay=$relay_source publisher=$source $lane" + planned=$(bun "$DIR/compat/resolve.ts" skip "$DIR/compat/planned-breaks.json" "$lane" "$version" "$relay_source" "$source") + if [[ -n "$planned" ]]; then + echo "SKIP $name: planned break $planned" + return + fi echo "=== $name ===" if "$@"; then echo "PASS $name" @@ -156,14 +163,16 @@ fetch() { # Existing media drivers: current publisher -> released clients and the reverse, # through either relay. The relay offers only one version, so JS cannot silently -# negotiate its preferred draft instead of the draft this cell names. +# negotiate its preferred draft instead of the draft this cell names. Only moq-lite +# drafts are listed: our clients and relays always prefer lite with each other, so +# IETF between them is not compared here (`just test interop` covers IETF). while read -r version; do released_relay="$HARNESS_RUN/released/bin/moq-relay" if fetch "$released_relay" "$RELEASED" "$RELEASED" "$HARNESS_RUN/js" "$HARNESS_RUN/js/transport.ts" "$version" >"$HARNESS_RUN/$version.fetch-baseline.log" 2>&1; then released_fetch=true else released_fetch=false - echo "SKIP $version FETCH: the released binaries do not fetch from one another ($HARNESS_RUN/$version.fetch-baseline.log)" + echo "SKIP $version fetch and js-publish: the released binaries do not fetch from one another ($HARNESS_RUN/$version.fetch-baseline.log)" fi for relay_source in current released; do if [[ "$relay_source" == current ]]; then relay="$TARGET/debug/moq-relay"; else relay="$released_relay"; fi @@ -178,20 +187,19 @@ while read -r version; do own_js="$HARNESS_RUN/js" own_transport="$HARNESS_RUN/js/transport.ts" other_js="$DIR/clients/js-native" other_transport="$DIR/compat/transport.ts" fi - name="$version relay=$relay_source publisher=$source" - cell "$name media" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + cell media "$version" "$relay_source" "$source" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ INTEROP_NATIVE_CLIENT="$other_js" INTEROP_COMPAT_TRANSPORT="$other_transport" INTEROP_DECODE=1 bash "$DIR/interop.sh" --subscribers rust,js-native-node - # Drive JS publication from actual subscriber demand. The CLI's newest-group - # read uses SUBSCRIBE here, independently of one-shot FETCH support. - cell "$name js-publish" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ - INTEROP_JS_PUBLISH_CLIENT="$publisher_js" INTEROP_READ_CURRENT=1 bash "$DIR/interop.sh" --publishers js-native --subscribers rust - if [[ "$released_fetch" == true ]]; then - # JS does not implement IETF FETCH in either direction, so both Rust - # sources fetch the same group and compare exact payloads. The - # publisher's own JS finds the group, so a catalog break fails only - # the media cell. - cell "$name fetch" fetch "$relay" "$pub" "$sub" "$own_js" "$own_transport" "$version" - fi + [[ "$released_fetch" == true ]] || continue + # Drive JS publication from actual subscriber demand: the publisher's own JS + # subscribes and observes the group, which the opposite Rust CLI then + # FETCHes by ID. Reading the newest group races its live delivery. + cell js-publish "$version" "$relay_source" "$source" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ + INTEROP_JS_PUBLISH_CLIENT="$publisher_js" INTEROP_NATIVE_CLIENT="$own_js" INTEROP_COMPAT_TRANSPORT="$own_transport" \ + INTEROP_FETCH_DATA=1 bash "$DIR/interop.sh" --publishers js-native --subscribers rust + # Both Rust sources fetch the same group and compare exact payloads. The + # publisher's own JS finds the group, so a catalog break fails only the + # media cell. + cell fetch "$version" "$relay_source" "$source" fetch "$relay" "$pub" "$sub" "$own_js" "$own_transport" "$version" done done done <"$HARNESS_RUN/shared" diff --git a/test/interop/compat/planned-breaks.json b/test/interop/compat/planned-breaks.json index e675bc4ef8..015f81d331 100644 --- a/test/interop/compat/planned-breaks.json +++ b/test/interop/compat/planned-breaks.json @@ -1,10 +1,33 @@ { "moq-lite-07-wip": { - "reason": "Approved under /recommended: this draft is unpublished and its wire may change. Remeasured against these releases: released moq-cli and @moq/net reject current lite-07-wip sessions (long buffer, invalid value). Compare stable lite and IETF drafts; reassess this measured exception when any affected release changes.", + "reason": "Approved under /recommended: this draft is unpublished and its wire may change. Remeasured against these releases: released moq-cli and @moq/net reject current lite-07-wip sessions (long buffer, invalid value). Compare stable lite drafts; reassess this measured exception when any affected release changes.", "releases": { "moq-cli": "0.14.2", "moq-relay": "0.17.2", "@moq/net": "0.4.2" } + }, + "catalog-archive": { + "reason": "Accepted by the maintainer (2026-10-09): #4034 reshaped the catalog `archive` entry in place (`timelines` replaced the required `track`), so released and current hang/@moq/hang reject each other's live catalogs (missing field `track` / `timelines`). Every media cell is cross-version.", + "releases": { + "moq-cli": "0.14.2", + "hang": "0.21.11", + "@moq/hang": "0.5.2" + }, + "cells": { + "lanes": ["media"] + } + }, + "lite-fetch-released-relay": { + "reason": "Known lite break, accepted to land the nightly (2026-10-09): a FETCH through the released relay from a current publisher hangs until the timeout on lite-05 and lite-06, while released-to-released and current-to-current pass. Follow-up quest planned.", + "releases": { + "moq-relay": "0.17.2" + }, + "cells": { + "lanes": ["fetch"], + "versions": ["moq-lite-05", "moq-lite-06"], + "relay": "released", + "publisher": "current" + } } } diff --git a/test/interop/compat/resolve.test.ts b/test/interop/compat/resolve.test.ts index 7cb0ca20e0..b182e2810d 100644 --- a/test/interop/compat/resolve.test.ts +++ b/test/interop/compat/resolve.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { matrix, newest, versions } from "./resolve"; +import { matrix, newest, planned, versions } from "./resolve"; test("a yanked or prerelease version cannot replace the installable release", () => { expect( @@ -19,18 +19,42 @@ test("wrapped help adds drafts without a harness edit", () => { expect(() => versions("--connect-version unrestricted")).toThrow(); }); +const OLD = "moq-lite-01"; +const NEW = "moq-lite-02"; + test("removing a published draft fails, while current-only drafts are logged", () => { - expect(() => matrix(["new"], ["old"], {})).toThrow("removed published"); - expect(matrix(["old", "new"], ["old"], {})).toEqual({ shared: ["old"], currentOnly: ["new"], planned: {} }); + expect(() => matrix([NEW], [OLD], {})).toThrow("removed published"); + expect(matrix([OLD, NEW], [OLD], {})).toEqual({ shared: [OLD], currentOnly: [NEW], planned: {} }); const exception = { reason: "Approved migration", releases: { "moq-cli": "1.0.0" } }; - expect(matrix(["new"], ["old"], { old: exception }, { "moq-cli": "1.0.0" }).planned).toEqual({ old: exception }); - expect(() => matrix(["old"], ["old"], { old: exception }, { "moq-cli": "1.0.1" })).toThrow("stale"); + expect(matrix([NEW], [OLD], { [OLD]: exception }, { "moq-cli": "1.0.0" }).planned).toEqual({ [OLD]: exception }); + expect(() => matrix([OLD], [OLD], { [OLD]: exception }, { "moq-cli": "1.0.1" })).toThrow("stale"); }); test("an approved wire break can retain its protocol name, but new drafts still fail", () => { const exception = { reason: "Unpublished wire changed", releases: { "@moq/net": "0.4.1" } }; - expect(matrix(["wip"], ["wip"], { wip: exception }, { "@moq/net": "0.4.1" }).shared).toEqual([]); - expect(() => matrix(["wip"], ["wip", "unreviewed"], { wip: exception }, { "@moq/net": "0.4.1" })).toThrow( + expect(matrix([OLD], [OLD], { [OLD]: exception }, { "@moq/net": "0.4.1" }).shared).toEqual([]); + expect(() => matrix([OLD], [OLD, NEW], { [OLD]: exception }, { "@moq/net": "0.4.1" })).toThrow( "removed published", ); }); + +test("IETF drafts never enter the released comparison", () => { + expect(matrix([OLD, "moq-transport-14"], [OLD, "moq-transport-14", "moq-transport-15"], {}).shared).toEqual([OLD]); +}); + +test("a cell-scoped break skips only its cells and expires with its releases", () => { + const breaks = { + hang: { + reason: "Fetch hangs", + releases: { "moq-relay": "1.0.0" }, + cells: { lanes: ["fetch"], versions: [OLD], relay: "released" as const }, + }, + }; + expect(matrix([OLD], [OLD], breaks, { "moq-relay": "1.0.0" }).shared).toEqual([OLD]); + expect(() => matrix([OLD], [OLD], breaks, { "moq-relay": "1.0.1" })).toThrow("stale"); + expect(() => matrix([NEW], [NEW], breaks, { "moq-relay": "1.0.0" })).toThrow("stale"); + const cell = { lane: "fetch", version: OLD, relay: "released", publisher: "current" }; + expect(planned(breaks, cell)).toBe("hang"); + expect(planned(breaks, { ...cell, relay: "current" })).toBeUndefined(); + expect(planned(breaks, { ...cell, lane: "media" })).toBeUndefined(); +}); diff --git a/test/interop/compat/resolve.ts b/test/interop/compat/resolve.ts index 34962ca2dd..ed22954e2a 100644 --- a/test/interop/compat/resolve.ts +++ b/test/interop/compat/resolve.ts @@ -21,39 +21,85 @@ export function versions(help: string): string[] { return values; } -/** One reviewed exception, bound to the releases whose incompatibility was measured. */ +/** The matrix cells a planned break covers; an omitted field matches every cell. */ +export interface Cells { + lanes: string[]; + versions?: string[]; + relay?: "current" | "released"; + publisher?: "current" | "released"; +} + +/** + * One reviewed exception, bound to the releases whose incompatibility was measured. Keyed + * by protocol version it drops that version; with `cells` it skips only those cells. + */ export interface PlannedBreak { reason: string; releases: Record; + cells?: Cells; } +/** One session cell of the matrix. */ +export interface Cell { + lane: string; + version: string; + relay: string; + publisher: string; +} + +// moq-lite is what our clients and relays prefer with each other, so released-vs-current +// compares only lite. IETF drafts are for third-party interop, covered by `just test interop`. +const lite = (version: string) => version.startsWith("moq-lite-"); + /** Refuse removed released versions unless a maintainer acknowledged the exact releases. */ export function matrix( - current: string[], - released: string[], + currentAll: string[], + releasedAll: string[], skipped: Record, releases: Record = {}, ) { - for (const [version, exception] of Object.entries(skipped)) { - if (!released.includes(version) || !exception.reason?.trim() || !Object.keys(exception.releases).length) { - throw new Error(`invalid or stale planned break: ${version}`); + const current = currentAll.filter(lite); + const released = releasedAll.filter(lite); + for (const [name, exception] of Object.entries(skipped)) { + const versions = exception.cells ? (exception.cells.versions ?? []) : [name]; + if ( + !exception.reason?.trim() || + !Object.keys(exception.releases).length || + (exception.cells && !exception.cells.lanes.length) || + versions.some((v) => !released.includes(v)) + ) { + throw new Error(`invalid or stale planned break: ${name}`); } - for (const [name, expected] of Object.entries(exception.releases)) { - if (releases[name] !== expected) - throw new Error( - `stale planned break: ${version}, ${name} changed from ${expected} to ${releases[name]}`, - ); + for (const [pkg, expected] of Object.entries(exception.releases)) { + if (releases[pkg] !== expected) + throw new Error(`stale planned break: ${name}, ${pkg} changed from ${expected} to ${releases[pkg]}`); } } - const missing = released.filter((v) => !current.includes(v) && !skipped[v]); + const dropped = (v: string) => !!skipped[v] && !skipped[v].cells; + const missing = released.filter((v) => !current.includes(v) && !dropped(v)); if (missing.length) throw new Error(`checkout removed published versions: ${missing.join(", ")}`); return { - shared: released.filter((v) => current.includes(v) && !skipped[v]), + shared: released.filter((v) => current.includes(v) && !dropped(v)), currentOnly: current.filter((v) => !released.includes(v)), planned: skipped, }; } +/** The planned break that skips `cell`, if any. */ +export function planned(skipped: Record, cell: Cell): string | undefined { + for (const [name, { cells }] of Object.entries(skipped)) { + if ( + cells?.lanes.includes(cell.lane) && + (!cells.versions || cells.versions.includes(cell.version)) && + (!cells.relay || cells.relay === cell.relay) && + (!cells.publisher || cells.publisher === cell.publisher) + ) { + return name; + } + } + return undefined; +} + async function registry(url: string) { const response = await fetch(url, { headers: { "User-Agent": "moq-wire-compat (https://github.com/moq-dev/moq)" }, @@ -89,5 +135,9 @@ if (import.meta.main) { ); console.log(JSON.stringify(result, null, 2)); writeFileSync(args[3], `${result.shared.join("\n")}\n`); - } else throw new Error("expected resolve or matrix"); + } else if (command === "skip") { + const [file, lane, version, relay, publisher] = args; + const name = planned(JSON.parse(readFileSync(file, "utf8")), { lane, version, relay, publisher }); + if (name) console.log(name); + } else throw new Error("expected resolve, matrix, or skip"); } diff --git a/test/interop/compat/transport.ts b/test/interop/compat/transport.ts index 37c5c7d9b2..67c25cf612 100644 --- a/test/interop/compat/transport.ts +++ b/test/interop/compat/transport.ts @@ -11,11 +11,9 @@ export async function connect(props: Moq.Connection.ConnectProps) { const hash = (await response.text()).trim(); if (!/^[a-f0-9]{64}$/i.test(hash)) throw new Error("malformed certificate fingerprint"); url.protocol = "https:"; - // These three drafts share moql and select the precise version in SETUP. Later - // moq-transport drafts register `moqt-NN`, and later lite drafts their own name. - const protocol = /^(moq-lite-0[12]|moq-transport-14)$/.test(version) - ? "moql" - : version.replace(/^moq-transport-/, "moqt-"); + // Lite-01 and lite-02 share moql and select the precise version in SETUP; later + // lite drafts register their own name. The released comparison runs only lite. + const protocol = /^moq-lite-0[12]$/.test(version) ? "moql" : version; const transport = new WebTransport(url, { protocols: [protocol], serverCertificateHashes: [{ algorithm: "sha-256", value: Uint8Array.from(Buffer.from(hash, "hex")) }], diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 3a4408b8f6..ca55a7eafc 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -25,7 +25,7 @@ source "$INTEROP_DIR/../lib/harness.sh" # Captured before the parse below consumes it, so the rerun command carries every # flag and every environment override this run was actually given. -RERUN="$(harness_env INTEROP_TIMEOUT INTEROP_FPS INTEROP_SIZE INTEROP_PORT INTEROP_PROFILE RELAY_BIN MOQ_BIN INTEROP_SUB_MOQ INTEROP_VERSION INTEROP_NATIVE_CLIENT INTEROP_DECODE INTEROP_JS_PUBLISH_CLIENT INTEROP_COMPAT_TRANSPORT INTEROP_READ_CURRENT INTEROP_FETCH_TRACK)just test interop$(harness_argv "$@")" +RERUN="$(harness_env INTEROP_TIMEOUT INTEROP_FPS INTEROP_SIZE INTEROP_PORT INTEROP_PROFILE RELAY_BIN MOQ_BIN INTEROP_SUB_MOQ INTEROP_VERSION INTEROP_NATIVE_CLIENT INTEROP_DECODE INTEROP_JS_PUBLISH_CLIENT INTEROP_COMPAT_TRANSPORT INTEROP_FETCH_DATA INTEROP_FETCH_TRACK)just test interop$(harness_argv "$@")" PUBLISHERS="rust" SUBSCRIBERS="rust" @@ -511,6 +511,16 @@ run_native() { printf '%s\n' "$out" | grep -q '^received ' } +# Write a track name and the group a live JS subscriber saw on it to +# $HARNESS_RUN/.track. Extra args go to subscribe.ts (e.g. --track). +# shellcheck disable=SC2329 # reached from run_subscriber, which 'harness_spawn' invokes +observe_group() { + local broadcast="$1" + shift + (cd "${INTEROP_NATIVE_CLIENT:-$CLIENTS/js-native}" && node --import tsx subscribe.ts subscribe --url "$URL" --broadcast "$broadcast" \ + --timeout "$TIMEOUT" --track-file "$HARNESS_RUN/$broadcast.track" "$@") +} + # shellcheck disable=SC2329 # reached from a function 'harness_spawn' invokes run_subscriber() { local lang="$1" broadcast="$2" publisher="${3:-}" @@ -521,8 +531,7 @@ run_subscriber() { if [[ "${INTEROP_FETCH_TRACK:-0}" == 1 ]]; then # Discover the track from the decoded catalog, never today's naming # convention, and a group live demand is filling. - (cd "${INTEROP_NATIVE_CLIENT:-$CLIENTS/js-native}" && node --import tsx subscribe.ts subscribe --url "$URL" --broadcast "$broadcast" \ - --timeout "$TIMEOUT" --track-file "$HARNESS_RUN/$broadcast.track") || return + observe_group "$broadcast" || return local track group binary index=0 { read -r track && read -r group; } <"$HARNESS_RUN/$broadcast.track" || return # Both readers FETCH that group by its observed ID, which waits for it to finish. @@ -549,9 +558,14 @@ assert outputs[0] == outputs[1], "current/released FETCH changed the immutable g PYCODE return fi - if [[ "${INTEROP_READ_CURRENT:-0}" == 1 ]]; then - timeout -k 3 "$TIMEOUT" "${INTEROP_SUB_MOQ:-$MOQ}" --connect "$URL" \ - ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} --broadcast "$broadcast" fetch data >"$HARNESS_RUN/$broadcast.fetch" + if [[ "${INTEROP_FETCH_DATA:-0}" == 1 ]]; then + # The JS fixture writes one group on `data` once a subscriber wants it. A + # live subscriber observes it; the reader then FETCHes it by that ID. + observe_group "$broadcast" --track data || return + local track group + { read -r track && read -r group; } <"$HARNESS_RUN/$broadcast.track" || return + timeout -k 3 "$TIMEOUT" "${INTEROP_SUB_MOQ:-$MOQ}" --connect "$URL" ${INTEROP_VERSION:+--connect-version "$INTEROP_VERSION"} \ + --broadcast "$broadcast" fetch "$track" --group "$group" >"$HARNESS_RUN/$broadcast.fetch" || return [[ "$(cat "$HARNESS_RUN/$broadcast.fetch")" == "compat-fetch" ]] return fi diff --git a/test/justfile b/test/justfile index 7c6e0230cc..0e44e758ff 100644 --- a/test/justfile +++ b/test/justfile @@ -41,7 +41,7 @@ interop *args: just rs test -p moq-net --lib varint_interop ietf_datagram_interop --run-ignored all ./interop/interop.sh "$@" -# Nightly main vs newest installable registry releases: tokens, pinned session +# Nightly main vs newest installable registry releases: tokens, moq-lite session # versions, decoded catalogs/containers, and group FETCH. See interop/README.md. wire-compat: ./interop/compat.sh From cdde8b1bcca93c77898f5fffb453d51f61148ca0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 Oct 2026 12:31:00 -0700 Subject: [PATCH 5/7] quest: wire compatibility lands Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/lite07-finalize.md | 1 - quest/m1/wire-compat.md | 61 ------------------------------------- 3 files changed, 63 deletions(-) delete mode 100644 quest/m1/wire-compat.md diff --git a/quest/m1/README.md b/quest/m1/README.md index cccd45e86e..7bda7155d3 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -68,7 +68,6 @@ were done or not worth their cost. - [FFI runtime](/quest/m1/ffi-runtime.md) - moq-ffi drives moq on a multi-thread runtime instead of one thread - [Audio group duration](/quest/m1/audio-group-duration.md) - audio groups span at least 20 ms by default, so small frames don't mint a group each - [Pipelined requests](/quest/m1/pipeline-requests/README.md) - SUBSCRIBE and the first FETCH go out with the track-info request at every hop, so first data arrives a round trip sooner per hop -- [Wire compatibility](/quest/m1/wire-compat.md) - a nightly run tests this checkout against the last published release for tokens, session wire, and catalog/container - [Kotlin wrapper POMs](/quest/m1/kt-ffi-pom.md) - Maven builds of `dev.moq:moq` resolve a published moq-ffi instead of the missing `0.0.0-dev` - [Installable moq-gst](/quest/m1/gst-brew-path.md) - `brew install moq-gst` installs the plugin from the tarball's `lib/gstreamer-1.0/`, and the Nix package carries `x264enc`, `avenc_aac`, and `avdec_h264` - [Same-epoch importers](/quest/m1/hop-aligned-import.md) - importers sharing one `--epoch` and fed one stream publish identical groups and timestamps, so failover between a redundant pair survives diff --git a/quest/m1/lite07-finalize.md b/quest/m1/lite07-finalize.md index f96f99fa69..3302f278f0 100644 --- a/quest/m1/lite07-finalize.md +++ b/quest/m1/lite07-finalize.md @@ -84,5 +84,4 @@ is published; older versions are unchanged. ## Related - [The moq.pro mesh runs lite-07](/quest/m3/lite07-mesh.md) - the deployment condition that follows this -- [Wire compatibility](/quest/m1/wire-compat.md) - the nightly covers lite-07 once a release carries it - [moq.pro: lite-07 on cluster dials](https://github.com/moq-dev/moq.pro/blob/main/quest/m2/lite07.md) - the mesh rollout that adopts the finalized version, and the customer rollout after it; moq.pro holds it in m2 until the cut is called diff --git a/quest/m1/wire-compat.md b/quest/m1/wire-compat.md deleted file mode 100644 index 3d5c5945c6..0000000000 --- a/quest/m1/wire-compat.md +++ /dev/null @@ -1,61 +0,0 @@ -# [M] Nightly wire compatibility against the last release - -## Goal - -A nightly run pits this checkout against the last published release and -fails when they stop understanding each other, before the break ships. -It covers, in both directions: - -- **Tokens:** the current `moq-auth` signs and the last published `moq-cli` - and `@moq/auth` verify, and the published side signs while the current - side verifies. -- **Session wire:** the current relay and clients against the last published - `moq-cli`, `moq-relay`, and `@moq/net`, on every lite and IETF version - both sides publish: publish, subscribe, announce, and fetch. -- **Catalog and container:** the last published `hang` and `@moq/hang` read - the current catalog and frames, and the current ones read theirs. - -## Plan - -Motivated by https://github.com/moq-dev/moq/pull/4190, where a token format -change broke every published credential and only moq-dev/smoke#49 noticed. -Neither existing harness asks this question: moq-dev/smoke tests published -against published, and `test/interop` builds every client from the checkout -(see `test/interop/README.md`). - -Decided by the maintainer: - -- **Nightly only**, not per PR. Installing released packages is slow, and - a break only needs catching before the next release. -- **Resolve the last release at run time** (the crates.io and npm registries' - newest non-yanked version), never a hand-bumped pin that goes stale. Log - the resolved versions in the run so a failure names both sides. -- **Not bindings.** Python, Go, Swift, Kotlin, and C stay with smoke and - `test/interop`. They wrap the same Rust, so the Rust lanes see their wire. - -Guidance: - -- Reuse `test/interop`'s relay and client drivers rather than build a second - harness. The new axis is which side comes from the registry, so a - "published" client source next to the checkout one may be enough. -- Derive the version matrix from what both sides accept (for example each - CLI's `--connect-version` choices), not a hand-kept list, so a new draft - joins the matrix and a dropped one leaves it without an edit. A version - only the checkout offers is skipped and logged. A version the last release - supports but the checkout no longer offers fails the run unless it is - acknowledged in the same skip list as planned breaks (decided with the - maintainer; silently dropping a published version is the regression this - exists to catch). -- Prefer released binaries (GitHub release assets) over `cargo install` of - the published crates if the build time threatens the nightly budget. -- Subscribers must decode the catalog and frames, not only see a non-empty - frame, or the container lane proves nothing. -- A deliberate break on `main` is expected to fail against the last release. - Run against `main` only, and document in the harness how a planned break - is acknowledged (for example a skip list that the next release clears). -- Wire the job into the existing nightly (`interop.yml` already has a - schedule) and document the recipe beside `just test interop`. - -## Related - -- [Track tail interop](/quest/m1/track-tail-interop.md) - another cross-language case in the same harness From 093516d08f258e5f6b0ce8ec7141d9734ce027a3 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 Oct 2026 12:53:17 -0700 Subject: [PATCH 6/7] test: harden the released matrix against stdin and config drift - Read the version list on fd 3 so a child reading stdin cannot end the loop. - Check every staged JS client, not only the first one given. - Fail when the relay config does not end up with exactly one pinned version. Co-Authored-By: Claude Opus 5.5 --- test/interop/compat.sh | 5 +++-- test/interop/interop.sh | 14 ++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/test/interop/compat.sh b/test/interop/compat.sh index 619f7d98dc..f07f4331e4 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -166,7 +166,8 @@ fetch() { # negotiate its preferred draft instead of the draft this cell names. Only moq-lite # drafts are listed: our clients and relays always prefer lite with each other, so # IETF between them is not compared here (`just test interop` covers IETF). -while read -r version; do +# Fd 3, so a child reading stdin (ffmpeg does) cannot consume the version list. +while read -r version <&3; do released_relay="$HARNESS_RUN/released/bin/moq-relay" if fetch "$released_relay" "$RELEASED" "$RELEASED" "$HARNESS_RUN/js" "$HARNESS_RUN/js/transport.ts" "$version" >"$HARNESS_RUN/$version.fetch-baseline.log" 2>&1; then released_fetch=true @@ -202,7 +203,7 @@ while read -r version; do cell fetch "$version" "$relay_source" "$source" fetch "$relay" "$pub" "$sub" "$own_js" "$own_transport" "$version" done done -done <"$HARNESS_RUN/shared" +done 3<"$HARNESS_RUN/shared" if ((${#failed[@]})); then printf 'FAIL %s\n' "${failed[@]}" >&2 exit 1 diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 29f9176901..a6c0c93fde 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -242,15 +242,13 @@ prepare_js() { for v in js js-native js-native-node js-native-bun; do needs "$v" && mark_broken "$v" "bun not found"; done return } - if [[ -n "${INTEROP_NATIVE_CLIENT:-}" ]]; then - [[ -f "$INTEROP_NATIVE_CLIENT/subscribe.ts" ]] || { + # Staged released-compat clients bring their own packages; check every one given. + if [[ -n "${INTEROP_NATIVE_CLIENT:-}${INTEROP_JS_PUBLISH_CLIENT:-}" ]]; then + [[ -z "${INTEROP_NATIVE_CLIENT:-}" || -f "$INTEROP_NATIVE_CLIENT/subscribe.ts" ]] || { echo "missing staged native client" >&2 exit 1 } - return - fi - if [[ -n "${INTEROP_JS_PUBLISH_CLIENT:-}" ]]; then - [[ -f "$INTEROP_JS_PUBLISH_CLIENT/client.ts" ]] || { + [[ -z "${INTEROP_JS_PUBLISH_CLIENT:-}" || -f "$INTEROP_JS_PUBLISH_CLIENT/client.ts" ]] || { echo "missing staged publisher" >&2 exit 1 } @@ -466,6 +464,10 @@ sed "s/4443/${PORT}/g" "$INTEROP_DIR/interop.toml" >"$HARNESS_RUN/relay.toml" if [[ -n "${INTEROP_VERSION:-}" ]]; then # Version values come from the executable's advertised CLI choices. sed -i "/\[listen\]/a version = [\"${INTEROP_VERSION}\"]" "$HARNESS_RUN/relay.toml" + [[ "$(grep -c '^version = ' "$HARNESS_RUN/relay.toml")" == 1 ]] || { + echo "relay.toml needs exactly one pinned [listen] version" >&2 + exit 1 + } fi harness_spawn relay "$HARNESS_RUN/relay.log" "$RELAY" "$HARNESS_RUN/relay.toml" if ! harness_ready "$URL/certificate.sha256" 30 "$HARNESS_PID"; then From 14de4698b44dd8413841ea83f91ac9660491dd2f Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 9 Oct 2026 13:15:57 -0700 Subject: [PATCH 7/7] test: take lite FETCH capability from each library, and count skips With IETF out of the released comparison, each moq-net library states FETCH support exactly (lite-05 onward), so a failing FETCH cell always fails the run instead of turning into a capability SKIP. The run also reports how many session cells ran and how many were skipped. Co-Authored-By: Claude Opus 5.5 --- test/interop/README.md | 9 +++++---- test/interop/compat.sh | 29 +++++++++++++++++++---------- test/interop/compat/container.rs | 10 +++++++++- 3 files changed, 33 insertions(+), 15 deletions(-) diff --git a/test/interop/README.md b/test/interop/README.md index 171dfa1379..3359d53444 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -133,14 +133,15 @@ publishers feed current readers, through both relay sources. Rust exports must decode to a video frame through ffmpeg. The existing JS subscriber reconstructs the catalog and decodes the container. -The released relay, publisher, and CLI decide FETCH support per draft: when -they fetch from one another, every mixed cell must too; otherwise the draft's -FETCH lanes are logged and skipped. Both Rust versions FETCH the group the +Each `moq-net` library states whether a draft has FETCH (lite-05 onward +today). A draft without it in both is logged and its FETCH lanes skipped; one +the release supports and the checkout dropped fails. Both Rust versions FETCH the group the publisher's own JS subscriber observed and compare every frame's exact payload. JS publishers write one group after that subscriber's demand, and the opposite-source Rust CLI FETCHes it by the same observed ID. -Every cell runs, and the run lists each failing cell before it fails. +Every session cell runs, and the run counts the cells that ran and were +skipped, then lists each failing cell before it fails. Checkout-only versions are logged and omitted. Removing a released version fails before sessions start. A maintainer-approved break belongs in `compat/planned-breaks.json` with its reason and the exact affected release diff --git a/test/interop/compat.sh b/test/interop/compat.sh index f07f4331e4..a16e1a2222 100755 --- a/test/interop/compat.sh +++ b/test/interop/compat.sh @@ -131,9 +131,11 @@ for source in current released js-current js-released; do echo "PASS catalog/container written by $source" done -# Every cell runs, so one nightly names every break; the run fails at the end. A -# cell a planned break covers is logged with that break's name instead. +# Every session cell runs, so one nightly names every break; the run fails at the +# end. A cell a planned break covers is logged with that break's name instead. failed=() +ran=0 +skipped=0 cell() { local lane="$1" version="$2" relay_source="$3" source="$4" planned shift 4 @@ -141,8 +143,10 @@ cell() { planned=$(bun "$DIR/compat/resolve.ts" skip "$DIR/compat/planned-breaks.json" "$lane" "$version" "$relay_source" "$source") if [[ -n "$planned" ]]; then echo "SKIP $name: planned break $planned" + skipped=$((skipped + 1)) return fi + ran=$((ran + 1)) echo "=== $name ===" if "$@"; then echo "PASS $name" @@ -152,8 +156,6 @@ cell() { fi } -# FETCH support differs by draft and by build, so the released binaries alone -# decide it: when they fetch a group from one another, every mixed cell must too. fetch() { local relay="$1" pub="$2" sub="$3" js="$4" transport="$5" version="$6" RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ @@ -169,11 +171,17 @@ fetch() { # Fd 3, so a child reading stdin (ffmpeg does) cannot consume the version list. while read -r version <&3; do released_relay="$HARNESS_RUN/released/bin/moq-relay" - if fetch "$released_relay" "$RELEASED" "$RELEASED" "$HARNESS_RUN/js" "$HARNESS_RUN/js/transport.ts" "$version" >"$HARNESS_RUN/$version.fetch-baseline.log" 2>&1; then - released_fetch=true - else - released_fetch=false - echo "SKIP $version fetch and js-publish: the released binaries do not fetch from one another ($HARNESS_RUN/$version.fetch-baseline.log)" + # Each moq-net library states whether the draft has FETCH, so a FETCH cell that + # fails is a failure, never a skip. Dropping a released capability fails too. + current_fetch=$("$TARGET/compat-hang-current/debug/compat-container" fetch-supported "$version") + released_fetch=$("$TARGET/compat-hang-released/debug/compat-container" fetch-supported "$version") + if [[ "$released_fetch" == true && "$current_fetch" != true ]]; then + echo "FAIL $version: the checkout removed published FETCH" + failed+=("$version FETCH removed") + fi + if [[ "$released_fetch" != true || "$current_fetch" != true ]]; then + echo "SKIP $version fetch and js-publish: no FETCH on this draft in both libraries" + skipped=$((skipped + 8)) fi for relay_source in current released; do if [[ "$relay_source" == current ]]; then relay="$TARGET/debug/moq-relay"; else relay="$released_relay"; fi @@ -190,7 +198,7 @@ while read -r version <&3; do fi cell media "$version" "$relay_source" "$source" env RELAY_BIN="$relay" MOQ_BIN="$pub" INTEROP_SUB_MOQ="$sub" INTEROP_VERSION="$version" \ INTEROP_NATIVE_CLIENT="$other_js" INTEROP_COMPAT_TRANSPORT="$other_transport" INTEROP_DECODE=1 bash "$DIR/interop.sh" --subscribers rust,js-native-node - [[ "$released_fetch" == true ]] || continue + [[ "$released_fetch" == true && "$current_fetch" == true ]] || continue # Drive JS publication from actual subscriber demand: the publisher's own JS # subscribes and observes the group, which the opposite Rust CLI then # FETCHes by ID. Reading the newest group races its live delivery. @@ -204,6 +212,7 @@ while read -r version <&3; do done done done 3<"$HARNESS_RUN/shared" +echo "session cells: $ran ran, $skipped skipped" if ((${#failed[@]})); then printf 'FAIL %s\n' "${failed[@]}" >&2 exit 1 diff --git a/test/interop/compat/container.rs b/test/interop/compat/container.rs index ab9752958b..0b1b8fa024 100644 --- a/test/interop/compat/container.rs +++ b/test/interop/compat/container.rs @@ -7,6 +7,14 @@ use std::{env, fs}; fn main() -> Result<(), Box> { let args: Vec<_> = env::args().collect(); match args[1].as_str() { + "fetch-supported" => { + // The library's own capability, so a failing FETCH cell is a failure, not a skip. + let supported = match args[2].parse::()? { + hang::moq_net::Version::Lite(version) => version.has_track_stream(), + _ => return Err("the released comparison runs only moq-lite".into()), + }; + println!("{supported}"); + } "encode" => { let catalog = Catalog::<()>::from_str( r#"{"video":{"renditions":{"video":{"codec":"avc3.42001e","container":{"kind":"legacy"}}}},"audio":{"renditions":{}}}"#, @@ -30,7 +38,7 @@ fn main() -> Result<(), Box> { assert_eq!(frame.timestamp, Timestamp::from_micros(1000001)?); assert_eq!(frame.payload.as_ref(), b"compat-frame"); } - _ => return Err("expected encode or decode".into()), + _ => return Err("expected fetch-supported, encode, or decode".into()), } Ok(()) }