diff --git a/.claude/skills/spawn-quests/SKILL.md b/.claude/skills/spawn-quests/SKILL.md index 80d7880e34..ce7afbe14b 100644 --- a/.claude/skills/spawn-quests/SKILL.md +++ b/.claude/skills/spawn-quests/SKILL.md @@ -18,6 +18,7 @@ Include a short summary and your recommendation. Spawn a background sub-agent for each /start-quest. Create a fresh worktree on the base `quest branch` prints, creating that line branch first if it is missing. +Agents share no writable files: each keeps its scratch files in its own worktree's `.scratch/`, and anything you hand every agent goes in its prompt, not a shared file. Limit the concurrency to at most N agents in parallel, where N is half the number of physical CPU cores. Monitor the sub-agents and report their final status, but do not monitor their PRs. diff --git a/.claude/skills/start-quest/SKILL.md b/.claude/skills/start-quest/SKILL.md index d6820efab3..35d5078967 100644 --- a/.claude/skills/start-quest/SKILL.md +++ b/.claude/skills/start-quest/SKILL.md @@ -13,4 +13,6 @@ If you are unsure on the best course of action, ask the user for direction. Confirm the quest is ready and unclaimed. Claim it as `quest/CLAUDE.md` describes: `quest branch` names the branch and its bases, missing line branches get a draft PR, and the quest branch gets an empty commit. Implement the quest until it is complete, or some blocker is hit, then create a PR against the base. +Keep scratch files (PR body, logs, notes) in the worktree's gitignored `.scratch/`. +Never write to or clean up a directory other agents share, such as a session scratchpad. Summarize the notable changes for the user. diff --git a/.github/scripts/package-binary.test.sh b/.github/scripts/package-binary.test.sh index 2b2554b6c1..b1390552f1 100755 --- a/.github/scripts/package-binary.test.sh +++ b/.github/scripts/package-binary.test.sh @@ -42,3 +42,31 @@ cmp "$binary" "$bare" cmp "$binary" "$tmp/extracted/$name/bin/moq-relay" echo "release assets package together without path collisions" + +# Without --binary the script builds the flake package named after the binary: +# `.#moq` for the moq-cli crate, since `.#moq-cli` is a stub refusing the old name. +cat >"$tmp/bin/nix" <<'NIX' +#!/usr/bin/env bash +set -euo pipefail +[[ "$1 $3" == "build --out-link" && "$2" == *"#moq" ]] || { + echo "unexpected: nix $*" >&2 + exit 1 +} +mkdir -p "$4/bin" +printf '#!/usr/bin/env sh\necho moq\n' >"$4/bin/moq" +chmod 0755 "$4/bin/moq" +NIX +chmod 0755 "$tmp/bin/nix" + +PATH="$tmp/bin:$PATH" "$WORKSPACE_DIR/rs/scripts/package-binary.sh" \ + --crate moq-cli \ + --bin moq \ + --version 0.12.2 \ + --target "$target" \ + --output "$tmp/dist" + +name="moq-cli-v0.12.2-$target" +tar -xzf "$tmp/dist/$name.tar.gz" -C "$tmp/extracted" +[[ "$("$tmp/extracted/$name/bin/moq")" == moq ]] + +echo "a nix build packages the flake output named after the binary" diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 639de6c3f0..f6a5e4d433 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -104,13 +104,13 @@ jobs: # and the test artifacts (codegen + linked test binaries). They share # little, and a PR usually needs both. - name: Check - run: nix develop --command just check --all + run: nix develop --command just ci check --all env: MOQ_STRICT: 1 - name: Test if: ${{ !cancelled() }} - run: nix develop --command just test all + run: nix develop --command just ci test --all env: MOQ_STRICT: 1 NEXTEST_PROFILE: ci diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index a1b03fd00c..4b26c72456 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -41,7 +41,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # Full history so `just check` can diff against origin/$GITHUB_BASE_REF. + # Full history so `just ci check` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main @@ -77,7 +77,7 @@ jobs: run: bash .claude/hooks/direnv.test.sh - name: Check - run: nix develop --command just check + run: nix develop --command just ci check env: MOQ_STRICT: 1 @@ -97,7 +97,7 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - # Full history so `just test` can diff against origin/$GITHUB_BASE_REF. + # Full history so `just ci test` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main @@ -120,7 +120,7 @@ jobs: # NEXTEST_PROFILE picks up the longer hang timeout in .config/nextest.toml; # without it a runner under load could trip the local one. - name: Test - run: nix develop --command just test + run: nix develop --command just ci test env: MOQ_STRICT: 1 NEXTEST_PROFILE: ci diff --git a/.gitignore b/.gitignore index 414686073c..ec8f692718 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ /.claude/tmp/ /.claude/worktrees/ +# Per-worktree agent scratch (PR bodies, logs, notes); see the start-quest skill. +/.scratch/ + # IDE .idea diff --git a/CLAUDE.md b/CLAUDE.md index a94be3ee39..504b71a14b 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -83,8 +83,7 @@ Use the Nix dev shell so tooling matches CI. direnv loads it automatically, but if not: `nix develop --command ...`. ```bash -just check # Lint and compile what the branch changed -just test # Test what the branch changed, same scope +just check # Lint, compile, and test what the branch changed just fix # Auto-fix lint/formatting, same scope ``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0bf08999dd..10c8135fdb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ When pushing additional commits to an existing PR, update the title and descript When taking over someone else's PR, push commits on top of theirs so they keep credit. Create a draft PR. -Switch it to "Ready for review" when you're finished and local `just check` and `just test` pass. +Switch it to "Ready for review" when you're finished and local `just check` passes. Fix any merge conflicts and failing CI checks. # AI diff --git a/Cargo.lock b/Cargo.lock index c648a8d0a6..4d2368190d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2835,7 +2835,7 @@ dependencies = [ [[package]] name = "hang" -version = "0.21.2" +version = "0.21.3" dependencies = [ "anyhow", "bytes", @@ -3454,9 +3454,9 @@ dependencies = [ [[package]] name = "iroh-metrics" -version = "1.0.1" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "291065721ad7c477b972e581bbc528df031dc8eb5e39fe1ff3300ae5dfb157ef" +checksum = "8ede55536349842337f7cdc63012ec33dc637945f3259c98a0763aac364cdf09" dependencies = [ "iroh-metrics-derive", "itoa", @@ -3884,7 +3884,7 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libmoq" -version = "0.6.2" +version = "0.6.3" dependencies = [ "anyhow", "bytes", @@ -4155,7 +4155,7 @@ dependencies = [ [[package]] name = "moq-archive" -version = "0.0.2" +version = "0.0.3" dependencies = [ "async-trait", "bytes", @@ -4172,7 +4172,7 @@ dependencies = [ [[package]] name = "moq-audio" -version = "0.1.1" +version = "0.1.2" dependencies = [ "block2 0.6.2", "bytes", @@ -4203,7 +4203,7 @@ dependencies = [ [[package]] name = "moq-auth" -version = "0.1.0" +version = "0.1.1" dependencies = [ "anyhow", "aws-lc-rs", @@ -4251,7 +4251,7 @@ dependencies = [ [[package]] name = "moq-binary" -version = "0.1.1" +version = "0.1.2" dependencies = [ "bytes", "kio 0.6.0", @@ -4263,7 +4263,7 @@ dependencies = [ [[package]] name = "moq-boy" -version = "0.5.2" +version = "0.5.3" dependencies = [ "anyhow", "boytacean", @@ -4283,7 +4283,7 @@ dependencies = [ [[package]] name = "moq-cli" -version = "0.12.2" +version = "0.12.3" dependencies = [ "anyhow", "axum", @@ -4321,7 +4321,7 @@ dependencies = [ [[package]] name = "moq-e2ee" -version = "0.0.2" +version = "0.0.3" dependencies = [ "aws-lc-rs", "base64 0.23.1", @@ -4340,7 +4340,7 @@ dependencies = [ [[package]] name = "moq-ffi" -version = "0.4.2" +version = "0.4.3" dependencies = [ "bytes", "getrandom 0.4.3", @@ -4365,7 +4365,7 @@ dependencies = [ [[package]] name = "moq-flate" -version = "0.1.3" +version = "0.2.0" dependencies = [ "bytes", "flate2", @@ -4374,10 +4374,11 @@ dependencies = [ [[package]] name = "moq-gst" -version = "0.4.2" +version = "0.4.3" dependencies = [ "anyhow", "bytes", + "futures", "gst-plugin-version-helper", "gstreamer", "hang", @@ -4392,7 +4393,7 @@ dependencies = [ [[package]] name = "moq-hls" -version = "0.5.2" +version = "0.5.3" dependencies = [ "axum", "bytes", @@ -4416,7 +4417,7 @@ dependencies = [ [[package]] name = "moq-json" -version = "0.4.2" +version = "0.5.0" dependencies = [ "bytes", "criterion", @@ -4433,7 +4434,7 @@ dependencies = [ [[package]] name = "moq-loc" -version = "0.2.10" +version = "0.2.11" dependencies = [ "bytes", "moq-net", @@ -4451,7 +4452,7 @@ dependencies = [ [[package]] name = "moq-mux" -version = "0.10.2" +version = "0.10.3" dependencies = [ "anyhow", "base64 0.23.1", @@ -4488,7 +4489,7 @@ version = "0.20.0" [[package]] name = "moq-net" -version = "0.3.1" +version = "0.3.2" dependencies = [ "arrayvec", "bytes", @@ -4605,7 +4606,7 @@ dependencies = [ [[package]] name = "moq-relay" -version = "0.15.2" +version = "0.15.3" dependencies = [ "anyhow", "axum", @@ -4648,7 +4649,7 @@ dependencies = [ [[package]] name = "moq-room" -version = "0.2.2" +version = "0.2.3" dependencies = [ "kio 0.6.0", "moq-auth", @@ -4662,7 +4663,7 @@ dependencies = [ [[package]] name = "moq-rtc" -version = "0.3.2" +version = "0.3.3" dependencies = [ "aws-lc-rs", "axum", @@ -4682,7 +4683,7 @@ dependencies = [ [[package]] name = "moq-rtmp" -version = "0.3.2" +version = "0.3.3" dependencies = [ "anyhow", "byteorder", @@ -4730,7 +4731,7 @@ dependencies = [ [[package]] name = "moq-srt" -version = "0.3.2" +version = "0.3.3" dependencies = [ "bytes", "futures", @@ -4746,7 +4747,7 @@ dependencies = [ [[package]] name = "moq-stats" -version = "0.2.2" +version = "0.2.3" dependencies = [ "futures", "moq-json", @@ -4761,7 +4762,7 @@ dependencies = [ [[package]] name = "moq-tokio" -version = "0.19.13" +version = "0.19.14" dependencies = [ "anyhow", "bytes", @@ -4814,7 +4815,7 @@ dependencies = [ [[package]] name = "moq-transcode" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "bytes", @@ -4833,7 +4834,7 @@ dependencies = [ [[package]] name = "moq-uring" -version = "0.0.3" +version = "0.0.4" dependencies = [ "anyhow", "bytes", @@ -4890,7 +4891,7 @@ dependencies = [ [[package]] name = "moq-video" -version = "0.1.1" +version = "0.1.2" dependencies = [ "anyhow", "ash", @@ -7429,9 +7430,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier" -version = "0.7.0" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0" +checksum = "1167586491e2b18b8bfbb293e8180ec17c201c4f076d7cb3070ca964e7598f98" dependencies = [ "core-foundation 0.10.1", "core-foundation-sys", @@ -7450,9 +7451,9 @@ dependencies = [ [[package]] name = "rustls-platform-verifier-android" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" +checksum = "eec689c0bc40ff2458a5977b6619cb718087084a18e02a131c599b62d05e1a5f" [[package]] name = "rustls-webpki" diff --git a/Cargo.toml b/Cargo.toml index 6751e2b493..3e25cae5ee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -118,10 +118,10 @@ criterion = "0.8" # (`fallback-dynamic-loading`), so nothing links against libcuda at build time. cudarc = { version = "0.19", default-features = false, features = ["driver", "fallback-dynamic-loading", "cuda-12020"] } dispatch2 = "0.3.1" -flate2 = "1.1" +flate2 = { version = "1.1", default-features = false } futures = "0.3" getrandom = { version = "0.4", features = ["wasm_js"] } -hang = { version = "0.21.2", path = "rs/hang" } +hang = { version = "0.21.3", path = "rs/hang" } hex = "0.4" # HMAC-SHA256 for the mDNS membership proofs (moq-tokio's `mdns` feature). hmac = "0.13" @@ -139,16 +139,16 @@ loom = { version = "0.7.2", features = ["futures"] } # DNS-SD advertisement and browsing for LAN peer discovery (moq-tokio's `mdns` feature). # `async` awaits the event channel instead of blocking a thread on it. mdns-sd = { version = "0.21", features = ["async"] } -moq-audio = { version = "0.1.1", path = "rs/moq-audio", default-features = false } -moq-auth = { version = "0.1.0", path = "rs/moq-auth" } -moq-binary = { version = "0.1.1", path = "rs/moq-binary" } -moq-flate = { version = "0.1.3", path = "rs/moq-flate" } -moq-hls = { version = "0.5.2", path = "rs/moq-hls", default-features = false } -moq-json = { version = "0.4.2", path = "rs/moq-json" } -moq-loc = { version = "0.2.10", path = "rs/moq-loc" } +moq-audio = { version = "0.1.2", path = "rs/moq-audio", default-features = false } +moq-auth = { version = "0.1.1", path = "rs/moq-auth" } +moq-binary = { version = "0.1.2", path = "rs/moq-binary" } +moq-flate = { version = "0.2.0", path = "rs/moq-flate" } +moq-hls = { version = "0.5.3", path = "rs/moq-hls", default-features = false } +moq-json = { version = "0.5.0", path = "rs/moq-json" } +moq-loc = { version = "0.2.11", path = "rs/moq-loc" } moq-msf = { version = "0.5.0", path = "rs/moq-msf" } -moq-mux = { version = "0.10.2", path = "rs/moq-mux" } -moq-net = { version = "0.3.1", path = "rs/moq-net" } +moq-mux = { version = "0.10.3", path = "rs/moq-mux" } +moq-net = { version = "0.3.2", path = "rs/moq-net" } # The MoQ fork of noq (moq-dev/noq). iroh keeps upstream noq, so a build with the # iroh feature carries both stacks. moq-noq-proto = { version = "1.3", default-features = false } @@ -158,23 +158,23 @@ moq-noq-udp = "1.3" # used by moq-video on Linux. moq-nvenc = { version = "0.1.0", path = "rs/moq-nvenc" } moq-pattern = { version = "0.1.0", path = "rs/moq-pattern" } -moq-relay = { version = "0.15.2", path = "rs/moq-relay", default-features = false } -moq-rtc = { version = "0.3.2", path = "rs/moq-rtc" } -moq-rtmp = { version = "0.3.2", path = "rs/moq-rtmp" } +moq-relay = { version = "0.15.3", path = "rs/moq-relay", default-features = false } +moq-rtc = { version = "0.3.3", path = "rs/moq-rtc" } +moq-rtmp = { version = "0.3.3", path = "rs/moq-rtmp" } moq-sock = { version = "0.1.0", path = "rs/moq-sock" } -moq-srt = { version = "0.3.2", path = "rs/moq-srt" } -moq-stats = { version = "0.2.2", path = "rs/moq-stats" } -moq-tokio = { version = "0.19.13", path = "rs/moq-tokio", default-features = false } +moq-srt = { version = "0.3.3", path = "rs/moq-srt" } +moq-stats = { version = "0.2.3", path = "rs/moq-stats" } +moq-tokio = { version = "0.19.14", path = "rs/moq-tokio", default-features = false } # Default features off on moq-transcode and moq-video so each workspace consumer # chooses native codecs, OpenH264, and rendering explicitly. Both crates still # provide working native plus software defaults when depended on directly. # VAAPI is opt-in everywhere; its decoder is hardware-validated, while its # encoder is not yet. -moq-transcode = { version = "0.1.1", path = "rs/moq-transcode", default-features = false } +moq-transcode = { version = "0.1.2", path = "rs/moq-transcode", default-features = false } # default-features off (the noq backend) so the consumer picks which QUIC # stack the io_uring path compiles; cargo features are additive, so a default-on # backend could not be opted out of. -moq-uring = { version = "0.0.3", path = "rs/moq-uring", default-features = false } +moq-uring = { version = "0.0.4", path = "rs/moq-uring", default-features = false } # In-tree fork of `v4l` with the videodev2.h bindings checked in, so moq-video's # `capture` and `v4l2` need no libclang or kernel headers. Linux only; an empty # stub elsewhere. @@ -187,7 +187,7 @@ moq-vaapi = "0.1.0" # `features = ["capture"]` to a consumer that ships in those bindings pulls the # whole device graph into every one of them. Codec features are independent of # that argument, so moq-ffi and libmoq opt NVIDIA, OpenH264, and VAAPI back in. -moq-video = { version = "0.1.1", path = "rs/moq-video", default-features = false } +moq-video = { version = "0.1.2", path = "rs/moq-video", default-features = false } nix = { version = "0.31.3", features = ["net", "socket", "uio"] } # Upstream noq-proto, only for iroh's controller factory types. noq-proto = { version = "1.2", default-features = false } diff --git a/README.md b/README.md index af82693845..1da6f96877 100644 --- a/README.md +++ b/README.md @@ -122,22 +122,18 @@ just # Build everything just build -# Lint and compile what your branch changed +# Lint, compile, and test what your branch changed just check -# Test what your branch changed, same scope -just test - # Automatically fix some linting errors, same scope just fix # Same as the above, over every package just check --all -just test all just fix --all ``` -CI runs these same two recipes, so they cover the same ground locally. It sets two things you don't: `MOQ_STRICT=1`, which turns a missing tool into an error instead of a skipped check, and `NEXTEST_PROFILE=ci`, which allows a longer hang timeout. +CI runs `check` as two parallel jobs, `just ci check` and `just ci test`, so it covers the same ground as your local run. It sets two things you don't: `MOQ_STRICT=1`, which turns a missing tool into an error instead of a skipped check, and `NEXTEST_PROFILE=ci`, which allows a longer hang timeout. See the [development guide](https://doc.moq.dev/setup/dev) and the [justfile](justfile) for more. diff --git a/bun.lock b/bun.lock index b53786bfe9..49a1ec1627 100644 --- a/bun.lock +++ b/bun.lock @@ -99,6 +99,7 @@ "dependencies": { "@moq/flate": "workspace:^", "@moq/net": "workspace:^", + "@moq/signals": "workspace:^", }, "devDependencies": { "@types/bun": "^1.4.2", @@ -655,6 +656,10 @@ "@moq/hang": ["@moq/hang@workspace:js/hang"], + "@moq/interop-browser": ["@moq/interop-browser@workspace:test/interop/clients/js"], + + "@moq/interop-native": ["@moq/interop-native@workspace:test/interop/clients/js-native"], + "@moq/json": ["@moq/json@workspace:js/json"], "@moq/loc": ["@moq/loc@workspace:js/loc"], @@ -673,10 +678,6 @@ "@moq/signals": ["@moq/signals@workspace:js/signals"], - "@moq/interop-browser": ["@moq/interop-browser@workspace:test/interop/clients/js"], - - "@moq/interop-native": ["@moq/interop-native@workspace:test/interop/clients/js-native"], - "@moq/wasm": ["@moq/wasm@workspace:js/wasm"], "@moq/wasm-test": ["@moq/wasm-test@workspace:test/wasm"], diff --git a/cpp/obs/src/moq-output.cpp b/cpp/obs/src/moq-output.cpp index 97ee873f98..bbe39e6879 100644 --- a/cpp/obs/src/moq-output.cpp +++ b/cpp/obs/src/moq-output.cpp @@ -538,12 +538,19 @@ void MoQOutput::AudioData(struct encoder_packet *packet) // Audio has no keyframes, so it has no group boundary of its own: without this the whole // stream is one group. Cut per frame, which is one QUIC stream per packet forwarded without // waiting for the next, the right trade for live. Video groups at its own keyframes. + // Cut before observing the flush so a failed observation never leaves the group open. result = moq_publish_media_cut(handle); if (result < 0) { LOG_ERROR("Failed to cut audio group: %d", result); return; } + result = moq_publish_media_flush(handle, pts_us); + if (result < 0) { + LOG_ERROR("Failed to observe audio encoder flush: %d", result); + return; + } + total_bytes_sent += packet->size; } @@ -576,6 +583,12 @@ void MoQOutput::VideoData(struct encoder_packet *packet) return; } + result = moq_publish_media_flush(handle, pts_us); + if (result < 0) { + LOG_ERROR("Failed to observe video encoder flush: %d", result); + return; + } + total_bytes_sent += packet->size; } diff --git a/cpp/obs/test/moq-output-test.cpp b/cpp/obs/test/moq-output-test.cpp index 8aec31a46a..e28039ea60 100644 --- a/cpp/obs/test/moq-output-test.cpp +++ b/cpp/obs/test/moq-output-test.cpp @@ -49,6 +49,8 @@ std::atomic g_begin_capture{0}; std::atomic g_settings_ok{true}; std::string g_rate_control = "CBR"; moq_video_hint g_video_hint{}; +std::atomic g_video_flushes{0}; +std::atomic g_audio_flushes{0}; // Lets a test run something inside obs_output_signal_stop, standing in for a // frontend that stops the output straight from the signal handler. std::function g_on_signal; @@ -139,9 +141,9 @@ bool obs_encoder_get_extra_data(const obs_encoder_t *, uint8_t **, size_t *) return false; } -const char *obs_encoder_get_codec(const obs_encoder_t *) +const char *obs_encoder_get_codec(const obs_encoder_t *encoder) { - return "h264"; + return encoder == reinterpret_cast(0x3) ? "opus" : "h264"; } // VideoInit reads coded size and CBR from the encoder. Any new libobs call in @@ -244,7 +246,7 @@ int32_t moq_publish_video(uint32_t, const moq_video_init *config) int32_t moq_publish_audio(uint32_t, const moq_audio_init *) { - return 7; + return 8; } int32_t moq_publish_media_finish(uint32_t) @@ -257,6 +259,15 @@ int32_t moq_publish_media_frame(uint32_t, const uint8_t *, uintptr_t, uint64_t) return 0; } +int32_t moq_publish_media_flush(uint32_t handle, uint64_t) +{ + if (handle == 7) + g_video_flushes++; + else if (handle == 8) + g_audio_flushes++; + return 0; +} + int32_t moq_publish_media_cut(uint32_t) { return 0; @@ -403,6 +414,8 @@ void reset() g_settings_ok = true; g_rate_control = "CBR"; g_video_hint = {}; + g_video_flushes = 0; + g_audio_flushes = 0; g_start_gate = nullptr; g_connect_fires_terminal = false; g_connect_fires_terminal_threaded = false; @@ -438,6 +451,14 @@ int main() packet.timebase_num = 1; packet.timebase_den = 30; o.Data(&packet); + CHECK(g_video_flushes == 1); + encoder_packet audio{}; + audio.type = OBS_ENCODER_AUDIO; + audio.encoder = reinterpret_cast(0x3); + audio.timebase_num = 1; + audio.timebase_den = 48000; + o.Data(&audio); + CHECK(g_audio_flushes == 1); CHECK(g_video_hint.has_coded); CHECK(g_video_hint.has_bitrate == (g_rate_control == "CBR")); if (g_video_hint.has_bitrate) diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 1f43856f01..8727301bc0 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -43,7 +43,7 @@ typedef BroadcastRequest = MoqBroadcastRequest; /// A stream of route announcements and retractions under a prefix. typedef AnnounceConsumer = MoqAnnounceConsumer; -/// A literal prefix plus an optional relative pattern for announcement discovery. +/// A literal prefix, an optional relative pattern, and the hidden-path opt-in for announcement discovery. typedef AnnounceConfig = MoqAnnounceConfig; /// A pending wait for a route to cover a specific path. diff --git a/dart/moq/lib/src/client.dart b/dart/moq/lib/src/client.dart index 3a821799d8..714fda2829 100644 --- a/dart/moq/lib/src/client.dart +++ b/dart/moq/lib/src/client.dart @@ -8,9 +8,13 @@ final class AnnounceOptions { /// Pattern relative to [prefix], or null for every path beneath it. final String? filter; - const AnnounceOptions({this.prefix = '', this.filter}); + /// Also list paths with a segment starting with `.` below [prefix]. + final bool hidden; - AnnounceConfig get _ffi => AnnounceConfig(prefix: prefix, filter: filter); + const AnnounceOptions({this.prefix = '', this.filter, this.hidden = false}); + + AnnounceConfig get _ffi => + AnnounceConfig(prefix: prefix, filter: filter, hidden: hidden); } /// Everything [Moq.connect] can be told beyond the URL. diff --git a/dart/moq_ffi/dart_test.yaml b/dart/moq_ffi/dart_test.yaml new file mode 100644 index 0000000000..d11c7b216b --- /dev/null +++ b/dart/moq_ffi/dart_test.yaml @@ -0,0 +1,3 @@ +# Suites share one process, so a concurrent suite would add its allocations to +# the resident memory that leak_test.dart measures. +concurrency: 1 diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 92e68c91cb..d4635f31c0 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -500,7 +500,13 @@ class MoqAudioInit { final MoqAudioFormat format; final Uint8List data; final String? label; - MoqAudioInit({required this.format, required this.data, this.label = null}); + final String? track; + MoqAudioInit({ + required this.format, + required this.data, + this.label = null, + this.track = null, + }); } class FfiConverterMoqAudioInit { @@ -525,8 +531,13 @@ class FfiConverterMoqAudioInit { ); final label = label_lifted.value; new_offset += label_lifted.bytesRead; + final track_lifted = FfiConverterOptionalString.read( + Uint8List.view(buf.buffer, new_offset), + ); + final track = track_lifted.value; + new_offset += track_lifted.bytesRead; return LiftRetVal( - MoqAudioInit(format: format, data: data, label: label), + MoqAudioInit(format: format, data: data, label: label, track: track), new_offset - buf.offsetInBytes, ); } @@ -536,6 +547,7 @@ class FfiConverterMoqAudioInit { FfiConverterMoqAudioFormat.allocationSize(value.format) + FfiConverterUint8List.allocationSize(value.data) + FfiConverterOptionalString.allocationSize(value.label) + + FfiConverterOptionalString.allocationSize(value.track) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -556,6 +568,10 @@ class FfiConverterMoqAudioInit { value.label, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterOptionalString.write( + value.track, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } @@ -563,6 +579,7 @@ class FfiConverterMoqAudioInit { return FfiConverterMoqAudioFormat.allocationSize(value.format) + FfiConverterUint8List.allocationSize(value.data) + FfiConverterOptionalString.allocationSize(value.label) + + FfiConverterOptionalString.allocationSize(value.track) + 0; } } @@ -1308,11 +1325,13 @@ class MoqVideoInit { final Uint8List data; final String? label; final MoqVideoHint? hint; + final String? track; MoqVideoInit({ required this.format, required this.data, this.label = null, this.hint = null, + this.track = null, }); } @@ -1343,8 +1362,19 @@ class FfiConverterMoqVideoInit { ); final hint = hint_lifted.value; new_offset += hint_lifted.bytesRead; + final track_lifted = FfiConverterOptionalString.read( + Uint8List.view(buf.buffer, new_offset), + ); + final track = track_lifted.value; + new_offset += track_lifted.bytesRead; return LiftRetVal( - MoqVideoInit(format: format, data: data, label: label, hint: hint), + MoqVideoInit( + format: format, + data: data, + label: label, + hint: hint, + track: track, + ), new_offset - buf.offsetInBytes, ); } @@ -1355,6 +1385,7 @@ class FfiConverterMoqVideoInit { FfiConverterUint8List.allocationSize(value.data) + FfiConverterOptionalString.allocationSize(value.label) + FfiConverterOptionalMoqVideoHint.allocationSize(value.hint) + + FfiConverterOptionalString.allocationSize(value.track) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -1379,6 +1410,10 @@ class FfiConverterMoqVideoInit { value.hint, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterOptionalString.write( + value.track, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } @@ -1387,6 +1422,7 @@ class FfiConverterMoqVideoInit { FfiConverterUint8List.allocationSize(value.data) + FfiConverterOptionalString.allocationSize(value.label) + FfiConverterOptionalMoqVideoHint.allocationSize(value.hint) + + FfiConverterOptionalString.allocationSize(value.track) + 0; } } @@ -1469,7 +1505,12 @@ class FfiConverterMoqVideoProperties { class MoqAnnounceConfig { final String prefix; final String? filter; - MoqAnnounceConfig({this.prefix = '', this.filter = null}); + final bool hidden; + MoqAnnounceConfig({ + this.prefix = '', + this.filter = null, + this.hidden = false, + }); } class FfiConverterMoqAnnounceConfig { @@ -1489,8 +1530,13 @@ class FfiConverterMoqAnnounceConfig { ); final filter = filter_lifted.value; new_offset += filter_lifted.bytesRead; + final hidden_lifted = FfiConverterBool.read( + Uint8List.view(buf.buffer, new_offset), + ); + final hidden = hidden_lifted.value; + new_offset += hidden_lifted.bytesRead; return LiftRetVal( - MoqAnnounceConfig(prefix: prefix, filter: filter), + MoqAnnounceConfig(prefix: prefix, filter: filter, hidden: hidden), new_offset - buf.offsetInBytes, ); } @@ -1499,6 +1545,7 @@ class FfiConverterMoqAnnounceConfig { final total_length = FfiConverterString.allocationSize(value.prefix) + FfiConverterOptionalString.allocationSize(value.filter) + + FfiConverterBool.allocationSize(value.hidden) + 0; final buf = Uint8List(total_length); write(value, buf); @@ -1515,12 +1562,17 @@ class FfiConverterMoqAnnounceConfig { value.filter, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterBool.write( + value.hidden, + Uint8List.view(buf.buffer, new_offset), + ); return new_offset - buf.offsetInBytes; } static int allocationSize(MoqAnnounceConfig value) { return FfiConverterString.allocationSize(value.prefix) + FfiConverterOptionalString.allocationSize(value.filter) + + FfiConverterBool.allocationSize(value.hidden) + 0; } } @@ -6561,6 +6613,7 @@ abstract class MoqMediaProducerInterface { void cut(); MoqTrackDemand demand(); void finish(); + void flush({required int timestampUs}); String name(); void seek({required int sequence}); Future unused(); @@ -6620,6 +6673,16 @@ class MoqMediaProducer implements MoqMediaProducerInterface { }, moqExceptionErrorHandler); } + void flush({required int timestampUs}) { + return rustCall((status) { + uniffi_moq_ffi_fn_method_moqmediaproducer_flush( + uniffiClonePointer(), + FfiConverterUInt64.lower(timestampUs), + status, + ); + }, moqExceptionErrorHandler); + } + String name() { return rustCallWithLifter( (status) => uniffi_moq_ffi_fn_method_moqmediaproducer_name( @@ -8116,14 +8179,9 @@ class FfiConverterOptionalBool { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalBool.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalBool.allocationSize(value)); FfiConverterOptionalBool.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(bool? value, Uint8List buf) { @@ -8166,14 +8224,9 @@ class FfiConverterOptionalDouble64 { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalDouble64.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalDouble64.allocationSize(value)); FfiConverterOptionalDouble64.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(double? value, Uint8List buf) { @@ -8216,14 +8269,11 @@ class FfiConverterOptionalMoqAnnounceUpdate { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqAnnounceUpdate.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqAnnounceUpdate.allocationSize(value), + ); FfiConverterOptionalMoqAnnounceUpdate.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqAnnounceUpdate? value, Uint8List buf) { @@ -8266,14 +8316,9 @@ class FfiConverterOptionalMoqCatalog { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqCatalog.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalMoqCatalog.allocationSize(value)); FfiConverterOptionalMoqCatalog.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqCatalog? value, Uint8List buf) { @@ -8316,14 +8361,11 @@ class FfiConverterOptionalMoqDatagram { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqDatagram.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqDatagram.allocationSize(value), + ); FfiConverterOptionalMoqDatagram.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqDatagram? value, Uint8List buf) { @@ -8366,14 +8408,11 @@ class FfiConverterOptionalMoqDimensions { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqDimensions.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqDimensions.allocationSize(value), + ); FfiConverterOptionalMoqDimensions.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqDimensions? value, Uint8List buf) { @@ -8421,16 +8460,11 @@ class FfiConverterOptionalMoqFetchGroupOptions { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqFetchGroupOptions.allocationSize( - value, + final buf = Uint8List( + FfiConverterOptionalMoqFetchGroupOptions.allocationSize(value), ); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); FfiConverterOptionalMoqFetchGroupOptions.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqFetchGroupOptions? value, Uint8List buf) { @@ -8473,14 +8507,9 @@ class FfiConverterOptionalMoqFrame { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqFrame.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalMoqFrame.allocationSize(value)); FfiConverterOptionalMoqFrame.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqFrame? value, Uint8List buf) { @@ -8523,14 +8552,11 @@ class FfiConverterOptionalMoqGroupConsumer { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqGroupConsumer.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqGroupConsumer.allocationSize(value), + ); FfiConverterOptionalMoqGroupConsumer.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqGroupConsumer? value, Uint8List buf) { @@ -8573,14 +8599,11 @@ class FfiConverterOptionalMoqMediaFrame { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqMediaFrame.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqMediaFrame.allocationSize(value), + ); FfiConverterOptionalMoqMediaFrame.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqMediaFrame? value, Uint8List buf) { @@ -8623,14 +8646,11 @@ class FfiConverterOptionalMoqOriginProducer { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqOriginProducer.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqOriginProducer.allocationSize(value), + ); FfiConverterOptionalMoqOriginProducer.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqOriginProducer? value, Uint8List buf) { @@ -8673,14 +8693,9 @@ class FfiConverterOptionalMoqRequest { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqRequest.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalMoqRequest.allocationSize(value)); FfiConverterOptionalMoqRequest.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqRequest? value, Uint8List buf) { @@ -8723,14 +8738,11 @@ class FfiConverterOptionalMoqSubscription { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqSubscription.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqSubscription.allocationSize(value), + ); FfiConverterOptionalMoqSubscription.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqSubscription? value, Uint8List buf) { @@ -8773,14 +8785,11 @@ class FfiConverterOptionalMoqTrackInfo { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqTrackInfo.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqTrackInfo.allocationSize(value), + ); FfiConverterOptionalMoqTrackInfo.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqTrackInfo? value, Uint8List buf) { @@ -8823,14 +8832,11 @@ class FfiConverterOptionalMoqVideoHint { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalMoqVideoHint.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalMoqVideoHint.allocationSize(value), + ); FfiConverterOptionalMoqVideoHint.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(MoqVideoHint? value, Uint8List buf) { @@ -8873,14 +8879,11 @@ class FfiConverterOptionalSequenceString { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalSequenceString.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List( + FfiConverterOptionalSequenceString.allocationSize(value), + ); FfiConverterOptionalSequenceString.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(List? value, Uint8List buf) { @@ -8923,14 +8926,9 @@ class FfiConverterOptionalString { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalString.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalString.allocationSize(value)); FfiConverterOptionalString.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(String? value, Uint8List buf) { @@ -8973,14 +8971,9 @@ class FfiConverterOptionalUInt64 { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalUInt64.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalUInt64.allocationSize(value)); FfiConverterOptionalUInt64.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(int? value, Uint8List buf) { @@ -9023,14 +9016,9 @@ class FfiConverterOptionalUint8List { if (value == null) { return toRustBuffer(Uint8List.fromList([0])); } - final length = FfiConverterOptionalUint8List.allocationSize(value); - final Pointer frameData = calloc(length); - final buf = frameData.asTypedList(length); + final buf = Uint8List(FfiConverterOptionalUint8List.allocationSize(value)); FfiConverterOptionalUint8List.write(value, buf); - final bytes = calloc(); - bytes.ref.len = length; - bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + return toRustBuffer(buf); } static int write(Uint8List? value, Uint8List buf) { @@ -9232,7 +9220,7 @@ class FfiConverterUint8List { static LiftRetVal read(Uint8List buf) { final length = buf.buffer.asByteData(buf.offsetInBytes).getInt32(0); - final bytes = Uint8List.view(buf.buffer, buf.offsetInBytes + 4, length); + final bytes = buf.sublist(4, 4 + length); return LiftRetVal(bytes, length + 4); } @@ -10575,6 +10563,15 @@ external void uniffi_moq_ffi_fn_method_moqmediaproducer_finish( Pointer uniffiStatus, ); +@Native, Uint64, Pointer)>( + assetId: _uniffiAssetId, +) +external void uniffi_moq_ffi_fn_method_moqmediaproducer_flush( + Pointer ptr, + int timestamp_us, + Pointer uniffiStatus, +); + @Native, Pointer)>( assetId: _uniffiAssetId, ) @@ -11938,6 +11935,9 @@ external int uniffi_moq_ffi_checksum_method_moqmediaproducer_demand(); @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqmediaproducer_finish(); +@Native(assetId: _uniffiAssetId) +external int uniffi_moq_ffi_checksum_method_moqmediaproducer_flush(); + @Native(assetId: _uniffiAssetId) external int uniffi_moq_ffi_checksum_method_moqmediaproducer_name(); @@ -12409,7 +12409,7 @@ void _checkApiChecksums() { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqbroadcastproducer_publish_audio() != - 47444) { + 31691) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } if (uniffi_moq_ffi_checksum_method_moqbroadcastproducer_publish_audio_on_track() != @@ -12512,6 +12512,9 @@ void _checkApiChecksums() { if (uniffi_moq_ffi_checksum_method_moqmediaproducer_finish() != 38480) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } + if (uniffi_moq_ffi_checksum_method_moqmediaproducer_flush() != 10235) { + throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); + } if (uniffi_moq_ffi_checksum_method_moqmediaproducer_name() != 7199) { throw UniffiInternalError.panicked("UniFFI API checksum mismatch"); } diff --git a/dart/moq_ffi/lib/src/uniffi_runtime.dart b/dart/moq_ffi/lib/src/uniffi_runtime.dart index ef3dc04b16..ab87a64ed3 100644 --- a/dart/moq_ffi/lib/src/uniffi_runtime.dart +++ b/dart/moq_ffi/lib/src/uniffi_runtime.dart @@ -71,11 +71,11 @@ void checkCallStatus( if (status.ref.code == CALL_SUCCESS) { return; } else if (status.ref.code == CALL_ERROR) { - throw errorHandler.lift(status.ref.errorBuf); + throw liftAndFree(status.ref.errorBuf, errorHandler.lift); } else if (status.ref.code == CALL_UNEXPECTED_ERROR) { if (status.ref.errorBuf.len > 0) { throw UniffiInternalError.panicked( - FfiConverterString.lift(status.ref.errorBuf), + liftAndFree(status.ref.errorBuf, FfiConverterString.lift), ); } else { throw UniffiInternalError.panicked("Rust panic"); @@ -101,6 +101,16 @@ T rustCall( } } +T liftAndFree(F raw, T Function(F) lifter) { + try { + return lifter(raw); + } finally { + if (raw is RustBuffer) { + raw.free(); + } + } +} + T rustCallWithLifter( F Function(Pointer) ffiCall, T Function(F) lifter, [ @@ -110,7 +120,7 @@ T rustCallWithLifter( try { final rawResult = ffiCall(status); checkCallStatus(errorHandler ?? NullRustCallStatusErrorHandler(), status); - return lifter(rawResult); + return liftAndFree(rawResult, lifter); } finally { calloc.free(status); } @@ -119,7 +129,6 @@ T rustCallWithLifter( class NullRustCallStatusErrorHandler extends UniffiRustCallStatusErrorHandler { @override Exception lift(RustBuffer errorBuf) { - errorBuf.free(); return UniffiInternalError.panicked("Unexpected CALL_ERROR"); } } @@ -175,7 +184,12 @@ RustBuffer toRustBuffer(Uint8List data) { final bytes = calloc(); bytes.ref.len = length; bytes.ref.data = frameData; - return RustBuffer.fromBytes(bytes.ref); + try { + return RustBuffer.fromBytes(bytes.ref); + } finally { + calloc.free(frameData); + calloc.free(bytes); + } } ForeignBytes lowerForeignBytes(Uint8List data) { @@ -306,7 +320,7 @@ Future uniffiRustCallAsync( try { final result = completeFunc(rustFuture, status); checkCallStatus(errorHandler ?? NullRustCallStatusErrorHandler(), status); - return liftFunc(result); + return liftAndFree(result, liftFunc); } finally { calloc.free(status); } diff --git a/dart/moq_ffi/test/leak_test.dart b/dart/moq_ffi/test/leak_test.dart new file mode 100644 index 0000000000..fe8ce19431 --- /dev/null +++ b/dart/moq_ffi/test/leak_test.dart @@ -0,0 +1,91 @@ +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:moq_ffi/moq_ffi.dart'; +import 'package:test/test.dart'; + +// Each call moves `size` bytes across the FFI boundary, so a leak of that +// buffer grows resident memory by `iterations * size`. Growth under a quarter of +// that leaves room for allocator and Dart heap noise without hiding a leak. +const size = 64 * 1024; +const iterations = 2000; +const leaked = size * iterations; + +void main() { + test('a returned String is released', () { + final track = MoqBroadcastProducer().publishTrack( + name: 'x' * size, + info: null, + ); + // Warm up so one-time allocations do not count as growth. + for (var i = 0; i < 100; i++) { + track.name(); + } + + final before = ProcessInfo.currentRss; + for (var i = 0; i < iterations; i++) { + track.name(); + } + final growth = ProcessInfo.currentRss - before; + + expect(growth, lessThan(leaked ~/ 4)); + }); + + test('a non-null optional argument is released', () async { + // A local broadcast has no origin to resolve against, so each call lowers + // the optional String and then throws, also covering the error buffer. + final consumer = MoqBroadcastProducer().consume(); + final reference = 'x' * size; + Future call() => expectLater( + consumer.resolve(reference: reference), + throwsA(isA()), + ); + + for (var i = 0; i < 100; i++) { + await call(); + } + + final before = ProcessInfo.currentRss; + for (var i = 0; i < iterations; i++) { + await call(); + } + final growth = ProcessInfo.currentRss - before; + + expect(growth, lessThan(leaked ~/ 4)); + }); + + test('an async return is released', () async { + final payload = Uint8List(size); + + // Handles are released deterministically so the only growth left is what + // the bindings leak, not frames a live track still holds. + Future roundTrip() async { + final broadcast = MoqBroadcastProducer(); + final track = broadcast.publishTrack(name: 'frames', info: null); + final consumer = track.consume(subscription: null); + final producer = track.appendGroup(); + producer.writeFrame(frame: MoqFrame(payload: payload)); + producer.finish(); + final group = await consumer.nextGroup(); + final frame = await group!.readFrame(); + expect(frame!.payload.length, size); + group.dispose(); + producer.dispose(); + consumer.dispose(); + track.dispose(); + broadcast.dispose(); + } + + for (var i = 0; i < 100; i++) { + await roundTrip(); + } + + final before = ProcessInfo.currentRss; + for (var i = 0; i < iterations; i++) { + await roundTrip(); + } + final growth = ProcessInfo.currentRss - before; + + expect(growth, lessThan(leaked ~/ 4)); + }); +} diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 1a32851e4a..2c5f49c738 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -272,6 +272,12 @@ See [Authentication](/bin/relay/auth). groups fetchable, which the [HLS gateway](/bin/hls) depends on. `export --max-age` (default 500 ms) is how long *this* consumer waits for a stalled group before skipping. Raising the first never delays playback. +For `export ts`, `--max-age` also bounds how long the muxer holds a leading +track for a lagging one. Frames go out in media-time order across all tracks, +not arrival order, so two exporters of one broadcast emit them in one order. A +track quiet for longer is muxed around until it catches up; a sparse track +(SCTE-35) costs that wait once per cue. `--max-age 0` keeps arrival order. + ## Debugging `RUST_LOG=debug` prints the negotiated version and every subscription. diff --git a/doc/bin/obs.md b/doc/bin/obs.md index c8c3518b9a..e22801d357 100644 --- a/doc/bin/obs.md +++ b/doc/bin/obs.md @@ -33,6 +33,10 @@ OBS Studio install. **About** lists plugin and libmoq versions, documentation links, and available video encoders. +OBS reports each locally encoded packet's handoff to libmoq against the shared +broadcast media clock. Each track's catalog `jitter` is the largest measured +delay above that track's own recent minimum, rounded up to milliseconds. + ## Source quality and moq-transcode OBS publishes **one** hang mezzanine. It does not encode a viewer ladder inside diff --git a/doc/bin/rtmp.md b/doc/bin/rtmp.md index da3b3e3e8d..dee1c0f39d 100644 --- a/doc/bin/rtmp.md +++ b/doc/bin/rtmp.md @@ -30,6 +30,6 @@ publish or play request to accept, map to a path, or reject. The CLI listener is unauthenticated; firewall it. Implemented in pure Rust (no librtmp). The CLI speaks plaintext `rtmp://` -only; the library adds RTMPS when the embedder supplies a TLS config. FLAC and -MP3 enhanced-audio payloads are dropped because hang has no catalog codec for -them. +only; the library adds RTMPS on the same port when the embedder supplies a TLS +config. FLAC and MP3 enhanced-audio payloads are dropped because hang has no +catalog codec for them. diff --git a/doc/concept/audio-jitter.md b/doc/concept/audio-jitter.md index bb316d007f..eb4c490763 100644 --- a/doc/concept/audio-jitter.md +++ b/doc/concept/audio-jitter.md @@ -441,4 +441,4 @@ import outside this directory rather than leaving it to review. longer match it, which is what stops a file being edited by hand to make a failing implementation pass. It compares rather than rewrites; regenerating is the `bun doc/concept/audio-jitter/corpus.ts` command above. It runs in -`just test` and `just check`. +`just check`. diff --git a/doc/concept/hang.md b/doc/concept/hang.md index b8623c5f49..22699656f8 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -54,7 +54,7 @@ A few things the catalog can express beyond decoder config: - **Labels.** Any rendition may carry a human-readable `label` for a track picker. The map key stays the track name used to subscribe, so labels need not be unique and renaming one doesn't rename the track. - **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. -- **Jitter.** A rendition can say how long the publisher holds a frame before flushing it, in whole milliseconds rounded up: one frame for a track flushed immediately, the B-frame depth for a reordered one, the fragment for a segmented one. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. +- **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Stalled renditions.** A publisher can flag a rendition as temporarily bad so players prefer another one without the track disappearing. First-party video publishers set this flag after more than three frame intervals of source silence or encoding lag while subscribed, and clear it after three on-time completed frames or when idle. Browser and native capture poll while waiting; FLV and MPEG-TS importers observe video silence as container data arrives. The shared detector is `hang::catalog::stalled::Detector` in Rust and `Catalog.Stalled.Detector` in JavaScript. It is a playback diagnostic, not an authorization or routing signal. - **Archive.** A broadcast may advertise an `archive` entry naming its timeline track (a small index of each complete aligned segment) and, if recorded, the replay MoQ path, object-store URL, and format version. The timeline is what lets the [HLS gateway](/bin/hls) build playlists without subscribing to media. - **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a segment index. @@ -112,7 +112,8 @@ In Rust the catalog owns the lifetime: `catalog.json_stream(track, config)` (or `json_snapshot` / `binary_snapshot` / `binary_stream`) writes the entry and retracts it when the producer drops. Read the config from `catalog.json.tracks` or `catalog.binary.tracks`, then pair its name and config with -`moq_mux::catalog::Entry::new` to subscribe. In the browser, read the same map, +`moq_mux::catalog::Entry::new` to subscribe. In C, `moq_publish_json_*` and +`moq_publish_binary_*` do the same, retracting on `_finish`. In the browser, read the same map, subscribe by name, and hand the track to `@moq/json` or `@moq/binary`. ## Container diff --git a/doc/concept/moq-lite.md b/doc/concept/moq-lite.md index 1423170ed6..2235506c52 100644 --- a/doc/concept/moq-lite.md +++ b/doc/concept/moq-lite.md @@ -29,8 +29,8 @@ implement in an afternoon. The wire spec is A dedicated ALPN selects the wire version for moq-lite 03 and newer. The legacy `moql` ALPN negotiates moq-lite 01 or 02 via `SETUP`. In moq-lite 05 and newer, each side also sends a `SETUP` message with its capabilities. -Rust and TypeScript speak moq-lite 01 through 06 and moq-transport drafts -14 through 22. Clients offer `moq-lite-06` first by default. +Rust and TypeScript speak moq-lite 01 through 07 and moq-transport drafts +14 through 22. Clients offer `moq-lite-07` first by default. ## Discovery @@ -63,6 +63,35 @@ stops new requests from resolving through it but leaves subscriptions already in flight alone: each track runs to its own end, the publisher's FIN or reset. moq-transport sessions behave the same when a namespace is withdrawn. +### Hidden broadcasts + +A path segment starting with `.` hides a route from discovery, the way a +dotfile hides from `ls`. A platform publishes its own broadcasts there (relay +stats under `.stats/`, cluster gossip under `.internal/`) without them turning +up in an app that lists everything and plays what it finds. Only segments +below the requested prefix count: listing the root skips `.stats/node`, but +listing `.stats` shows `node`. A `.` elsewhere in a segment (`catalog.pro`) is +part of the name. + +Hiding narrows discovery and nothing else. Subscribing to a hidden path by +name works without asking, and tokens authorize it like any other path. To +list hidden routes too, opt in per announce request: + +```rust +let announced = origin.consume().with_hidden(true).announced(); +``` + +```typescript +const announced = connection.announced(Path.Pattern.all(), { hidden: true }); +``` + +On the wire, moq-lite 07 carries the opt-in on each announce request, and +moq-transport carries it as a `SUBSCRIBE_NAMESPACE` parameter once the peer's +`SETUP` says it understands one ([hidden](/draft/moq-hidden)). An older peer +never opts in, so it never discovers hidden routes. Rust sessions always opt in +on the wire and filter per local reader, so a relay mirrors everything and +each consumer decides. + ## Path patterns Rust's `moq_net::Pattern` and TypeScript's `Path.Pattern` from `@moq/net` diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 4ccc33e257..f2c7e9929e 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -26,6 +26,11 @@ maps everything else to "not supported" or a harmless equivalent. The [moq-lite page](/concept/moq-lite#what-moq-lite-leaves-out) lists the differences. +An IETF publisher declares the track's default priority in `SUBSCRIBE_OK` or +`PUBLISH` when that draft carries track properties. Groups without a priority +flag inherit it. If the property is absent, the IETF wire default of 128 maps +to model priority 127, where higher values are served first. + On drafts 14–19, the Rust publisher serves relative joining `FETCH` requests with offset zero for `NextObject` subscriptions. The fetch delivers the saved current-group prefix, and the subscription delivers later objects. Standalone, @@ -38,7 +43,8 @@ tracks waiting for a group that never arrives. Several project drafts extend the IETF wire without breaking it, since `SETUP` ignores unknown parameters: [cluster](/draft/moq-cluster) routing hop lists, -[solicit](/draft/moq-solicit) to make announcements opt-in, and +[solicit](/draft/moq-solicit) to make announcements opt-in, +[hidden](/draft/moq-hidden) to keep `.`-named namespaces out of discovery, and [probe](/draft/moq-probe) for bandwidth estimation. [moq-e2ee](/draft/moq-e2ee) is not a transport extension: it encrypts application payloads so relays still forward named tracks they cannot read. diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index 10e4e093ff..a32dcc4aca 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -43,7 +43,7 @@ and `target/include/moq.h`. - **Server.** `moq_server_listen` binds before it returns (a bad address or certificate fails there) and hands each incoming session to `on_request` as a request handle. Read `moq_session_request_path` and `_query` to route and authenticate, then `moq_session_request_accept` (a session handle, with origins like `moq_session_connect`) or `moq_session_request_reject` with an HTTP-style code (401 and 403 become the protocol's unauthorized close). An accepted session reports `1` once SETUP completes and never reconnects. `moq_server_addr` reports an ephemeral port and `moq_server_fingerprints` the hashes a client pins for a `tls_generate` certificate. `moq_server_close` stops listening; its terminal callback fires once the sockets are released. - **Demand.** A watcher on a published track (`moq_publish_track_demand`, `moq_publish_media_demand`, `moq_encode_video_demand`, `moq_encode_audio_demand`) calls `on_demand` with `MOQ_DEMAND_USED` or `MOQ_DEMAND_UNUSED` right away and again on every change, so an encoder on a battery-powered device runs only while someone is watching. The first call is the current state, so a track that went unused before the watcher existed still reports it. `moq_publish_demand_cancel` stops it; the terminal callback still fires. A container has no single demand and is refused. Demand follows the last real subscriber: an origin that served the track drops its source copy on the unused edge and keeps only the finished groups it already cached warm for 30 seconds, so the cache linger does not delay the unused edge. - **Requests.** `moq_publish_dynamic` serves subscriptions to tracks the broadcast never declared: each arrives as a request handle, read its name with `moq_track_request_name`, then `moq_track_request_accept` (a raw track handle), `moq_track_request_video` / `_audio` (the media handle `moq_publish_video` / `_audio` return), or `moq_track_request_abort` with an application code the subscriber sees. Without a live handler an unknown name is refused. `moq_publish_track_dynamic` does the same for fetches of groups a track no longer has cached, delivered as `moq_group_request_*` (`sequence`, `priority`, `frame_start`); `moq_group_request_accept` starts the producer at `frame_start` so written frames keep their group indices. Register it with `moq_track_request_dynamic` before accepting a track that was itself requested by a fetch, so that pending group survives the transition. Both handlers stop with `moq_publish_dynamic_cancel`. -- **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams, JSON snapshot and stream tracks, group fetch, catalog sections, shared video properties, and stalled hints. The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a literal prefix and an optional relative pattern filter; `moq_announce_update.prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. +- **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams, JSON and binary data tracks (snapshot or stream, each advertised in the catalog for as long as it lives), group fetch, catalog sections, shared video properties, and stalled hints. The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a literal prefix and an optional relative pattern filter; `moq_announce_update.prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts); name the dot segment in `prefix` to list them. ```c moq_client_config config; @@ -58,6 +58,8 @@ if (session < 0) return fail(moq_error()); ``` +For a locally encoded media track, call `moq_publish_media_flush(media, timestamp_us)` after `moq_publish_media_frame` with the same broadcast-clock PTS. The monotonic handoff time is sampled inside libmoq. Do not call it for file, pipe, or network imports; those remain clock-free. Invalid handles and unrepresentable timestamps return a negative error code. + ## Connection stats Every field in `moq_connection_stats` carries a matching `_valid` flag, diff --git a/doc/lib/dart/index.md b/doc/lib/dart/index.md index c3dfd82dae..564d4e06ed 100644 --- a/doc/lib/dart/index.md +++ b/doc/lib/dart/index.md @@ -71,7 +71,8 @@ every path beneath it (`''` for everything; Dart spells the origin method claim should stay advertised, and reject the requests you will not serve. A route is a capability, not an inventory. `announcements(options:)` takes a literal prefix plus an optional relative pattern; `announcement.prefix()` -stays origin-relative and `captures()` reports the wildcard matches. +stays origin-relative and `captures()` reports the wildcard matches. Paths with +a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts) unless `hidden: true`. Sessions reconnect with backoff when the transport drops and re-announce local broadcasts. `Moq.connect` and `Server.listen` take a `ConnectOptions` / @@ -106,6 +107,8 @@ catalog and container types are there, so already-encoded frames flow through `MoqMediaProducer`/`MoqMediaConsumer`, but encoding is up to `package:camera`, platform channels, or another codec package. +`MediaProducer.flush(timestampUs: ...)` records the handoff of a locally encoded frame on the broadcast media clock. Call it after `writeFrame` only for live encoder output; file, pipe, and network imports stay clock-free. `MediaProducer` aliases the generated FFI object, so its method is available directly. + ## Connection stats `session.stats()` returns a `ConnectionStats` snapshot. Each field is `null` diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 9351c09202..3d8e10bff6 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -69,6 +69,8 @@ _ = broadcast.Announce(moq.Route{}) broadcast.Finish() // keep the producer reachable while publishing, then finish explicitly ``` +For locally encoded media, call `MediaProducer.Flush(timestampUs)` after `WriteFrame` with the same broadcast-clock PTS. It measures catalog jitter at the transport handoff. File, pipe, and network imports should omit `Flush`; built-in encoders observe their own output. + The three advertising operations: `client.CreateBroadcast(path)` (or `origin.CreateBroadcast`) returns an unannounced producer, invisible to everyone; `broadcast.Announce(route)` / `broadcast.Unannounce()` own that exact-path @@ -78,6 +80,8 @@ while the claim should stay advertised, and reject the requests you will not serve. A route is a capability, not an inventory. `Announced(options)` combines a literal prefix with an optional relative pattern; `ann.Prefix()` stays relative to the origin and `ann.Captures()` reports the wildcard matches. +Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts) unless +`Hidden: true`. Every call that can block takes a `context.Context` first. Cancelling it returns `ctx.Err()` promptly and tears the in-flight native work down, so a diff --git a/doc/lib/js/net.md b/doc/lib/js/net.md index 0ea3b7c819..3b6b3c693a 100644 --- a/doc/lib/js/net.md +++ b/doc/lib/js/net.md @@ -104,7 +104,9 @@ match (otherwise `undefined`), `kind` is `"announced"`, `"updated"` (a reprice in place), or `"retracted"`, and `route` carries hops and cost (on a retraction, its last values). The consumer is an async iterable. A prefix is not a broadcast name; the scope filters locally while sessions request its -literal head on the wire. +literal head on the wire. Paths with a `.`-prefixed segment below that head +are [hidden](/concept/moq-lite#hidden-broadcasts) unless `announced(scope, { hidden: true })` opts in; +`broadcasts(scope, { hidden: true })` takes the same option. Examples in [`js/net/examples/`](https://github.com/moq-dev/moq/tree/main/js/net/examples). diff --git a/doc/lib/js/publish.md b/doc/lib/js/publish.md index 2af452cb20..6c9c1dd0c5 100644 --- a/doc/lib/js/publish.md +++ b/doc/lib/js/publish.md @@ -54,6 +54,21 @@ framerate, and bitrate are tunable through `el.video.config`; the audio encoder exposes its codec and volume. For simulcast or several renditions, drop the element and register your own encoders on a `Publish.Broadcast`. +The video and audio encoders measure how far their output falls behind the media +clock when they flush frames. Catalog jitter is the spread above each +rendition's own recent minimum lateness, so a constant encoder delay is not jitter. +The advertised value only rises; frame duration alone does not set it. + +## Clock + +Every timestamp the publisher writes is `performance.now()` in microseconds, +so camera, microphone, screen, and file sources share one timeline. The catalog +advertises that mapping as its root `clock` from the first snapshot, with PTS +zero at `performance.timeOrigin`, so a viewer or an HLS export can name any +frame's wall time. The mapping is fixed for the page: a system-clock +adjustment never retimes the broadcast. Stamp your own tracks (e.g. text cues) +on the same timeline to stay in sync. + ## Custom tracks `broadcast.net` is the underlying `Moq.Broadcast.Producer`, so an application @@ -120,7 +135,9 @@ new Publish.Audio.Encoder("audio", { broadcast, capture: audioCapture, enabled: Standalone components start enabled unless you pass `enabled: false` (or a signal). Camera and microphone sources may prompt for permission on construction, so build an enabled screen source inside the user gesture that -authorizes screen capture. +authorizes screen capture. Audio capture that starts before the page's first +click or keypress waits for one: browsers suspend Web Audio until then, and the +audio rendition stays out of the catalog until samples flow. Every input and output is a signal from [`@moq/signals`](/lib/js/signals). Load from a CDN (`https://esm.sh/@moq/publish/element`) for a no-build embed. diff --git a/doc/lib/js/signals.md b/doc/lib/js/signals.md index 515db35a9a..54b59838f6 100644 --- a/doc/lib/js/signals.md +++ b/doc/lib/js/signals.md @@ -38,7 +38,8 @@ The rules that differ from other signal libraries: - **Nothing is tracked implicitly.** `effect.get(signal)` subscribes; `signal.peek()` doesn't. - **Writes coalesce per microtask** and only notify on a real change (deep for plain objects, identity for class instances). -- **Effects own their resources.** `effect.timer`, `interval`, `animate`, `event`, `spawn`, and `run` (a nested effect) all clean up on rerun or close, so never call `setTimeout` or `addEventListener` inside one directly. A rerun waits for the previous run's `spawn` tasks to settle, and `effect.abort`/`effect.cancel` tell them to stop. +- **Effects own their resources.** `effect.timer`, `interval`, `animate`, `event`, `spawn`, and `run` (a nested effect) all clean up on rerun or close, so never call `setTimeout` or `addEventListener` inside one directly. A rerun waits for the previous run's `spawn` tasks to settle, and `effect.abort`/`effect.race` tell them to stop. +- **Race with `race`, not `Promise.race`.** `Promise.race` leaves a listener on every value that loses, so racing a long-lived one (a `closed`, a run's teardown) once per frame grows the heap. `race([...])` accepts promises and `Once` values and drops its listeners when it settles; `effect.race(promise)` also resolves `undefined` once the run is torn down. - **Dev builds warn** about effects that tracked nothing, effects garbage-collected without `close()`, and signals leaking subscribers. Components follow one shape: `in` (wired inputs), `out` (read-only derived diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index 6a5a79fee3..5ee0252dfa 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -106,7 +106,7 @@ const dispose = el.signals.run((effect) => { const consumer = new Json.Snapshot.Consumer({ track }); effect.spawn(async () => { for (;;) { - const value = await Promise.race([effect.cancel, consumer.next()]); + const value = await effect.race(consumer.next()); if (value === undefined) break; console.log("metadata", value); } diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index 96c03ecf9d..c696279adc 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -51,6 +51,8 @@ Moq.connect("https://relay.example.com").use { moq -> } ``` +`MediaProducer.flush(timestampUs)` records a locally encoded frame's transport handoff on the broadcast media clock. Call it after `writeFrame` for live encoder output; omit it for file, pipe, and network imports. `MediaProducer` is a typealias, so the generated method is available directly. + The three advertising operations: `moq.createBroadcast(path)` (or `origin.createBroadcast`) returns an unannounced producer, invisible to everyone; `broadcast.announce(route)` / `broadcast.unannounce()` own that exact-path @@ -59,7 +61,8 @@ path beneath it (`""` for everything). Hold the returned `OriginDynamic` while the claim should stay advertised, and reject the requests you will not serve. A route is a capability, not an inventory. `announcements(config)` takes a literal prefix plus an optional relative pattern; `announcement.prefix()` -stays origin-relative and `captures()` reports the wildcard matches. +stays origin-relative and `captures()` reports the wildcard matches. Paths with +a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts) unless `hidden = true`. Sessions reconnect with backoff when the transport drops and re-announce local broadcasts. `moq.epoch()` counts the connections, 1 on the first, pairing with diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index a3560a3e41..28e51ca85c 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -69,6 +69,8 @@ async def main(): asyncio.run(main()) ``` +For already-encoded live output, call `audio.flush(timestamp_us)` after each `audio.write_frame` with the same broadcast-clock PTS. It samples the transport handoff for catalog jitter. File, pipe, and network imports should omit `flush`; raw-pixel and PCM encoders inside the binding measure their own output. + The three advertising operations, as the other bindings spell them: `client.create_broadcast(path)` (or `OriginProducer.create_broadcast`) returns an unannounced producer, invisible to everyone; `broadcast.announce(route)` / @@ -79,6 +81,8 @@ advertised, and reject the requests you will not serve. A route is a capability, not an inventory. `announced(prefix, filter=...)` combines a literal root with an optional relative pattern; each announcement `.prefix` stays relative to the origin and `.captures` reports what the pattern wildcards matched. +Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts) unless +`hidden=True`. Sessions reconnect with backoff when the transport drops and re-announce local broadcasts. `session.epoch()` counts the connections, 1 on the first, pairing diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index fa08f1debf..613f9e2b52 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -38,11 +38,14 @@ Each catalog track constructor returns one `container::Producer` that owns the media stream and its catalog entry. `set` publishes or replaces its config, `modify` edits the published config through a guard, and dropping the producer retires the entry. Calling `modify` before the first `set` returns -`Error::NotPublished`. Container writes measure bitrate and jitter and publish -the estimate automatically when groups are cut or finished. Invalid jitter is -rejected before the edit is retained, including while the initial catalog is -reserved. Codec importers propagate catalog and media errors through their -configuration and frame-writing methods. +`Error::NotPublished`. Container writes measure bitrate; importers can also +measure batch span or reorder delay for jitter. Locally encoded frames call +`container::Producer::flush(timestamp, Instant::now())`; jitter is the spread +above that track's own recent minimum lateness, published as soon as it rises. +Generic imports remain clock-free. Invalid or decreasing jitter is rejected +before the edit is retained, including while the initial catalog is reserved. +Codec importers propagate catalog and media errors through their configuration +and frame-writing methods. ```bash cargo add moq-mux diff --git a/doc/lib/rs/moq-net.md b/doc/lib/rs/moq-net.md index 0a3dd1b1bc..2f56c399b6 100644 --- a/doc/lib/rs/moq-net.md +++ b/doc/lib/rs/moq-net.md @@ -135,7 +135,9 @@ most specific matching scope member's wildcards stood for when the prefix pins them, and `route` carries hops and cost (on a retraction, its last values). The consumer is also a `futures::Stream`. A prefix is not a broadcast name; sessions request each scope member's literal head and filter -locally. +locally. Routes with a `.`-prefixed segment below that head are [hidden](/concept/moq-lite#hidden-broadcasts) +unless `with_hidden(true)` opts the consumer in. Sessions always ask the peer +for hidden routes, so each local consumer decides. ## Limiting reads diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index 9ef9eb8dd2..0af912a441 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -55,6 +55,8 @@ try broadcast.announce() session.shutdown() ``` +For already-encoded live output, call `audio.flush(timestampUs:)` after `writeFrame` with the same broadcast-clock PTS. It measures catalog jitter at the transport handoff. File, pipe, and network imports should omit `flush`; built-in encoders observe their own output. + The three advertising operations: `session.publish.createBroadcast(path:)` returns an unannounced producer, invisible to everyone; `broadcast.announce(route:)` / `broadcast.unannounce()` own that exact-path advertisement; @@ -64,6 +66,8 @@ claim should stay advertised, and reject the requests you will not serve. A route is a capability, not an inventory. `announced(prefix:filter:)` combines a literal root with an optional relative pattern; `announcement.prefix` stays relative to the origin and `captures` reports what the wildcards matched. +Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts) unless +`hidden: true`. For a self-signed relay on your own test network, `try client.setTlsVerify(false)` accepts any certificate; prefer `setTlsRoots` or a fingerprint anywhere else. diff --git a/doc/setup/dev.md b/doc/setup/dev.md index 9aaf57bad6..b7a3de2ffa 100644 --- a/doc/setup/dev.md +++ b/doc/setup/dev.md @@ -14,9 +14,8 @@ match CI. | `just` | Start the local relay, test publisher, and web demo. | | `just --list` | List every recipe. | | `just fix` | Format and lint the packages this branch changed. | -| `just check` | Compile and lint the same scope. This is what CI runs. | -| `just test` | Run tests for the same scope. | -| `just fix --all`, `just check --all`, `just test all` | The same, over every package. | +| `just check` | Compile, lint, and test the same scope. This is what CI runs. | +| `just fix --all`, `just check --all` | The same, over every package. | | `just pub bbb ` | Publish Big Buck Bunny (also `tos`, `clock`, `gst`, `hls`). | | `just sub gst bbb ` | Play a broadcast through GStreamer. | | `just relay` | Run a local relay on its own. | @@ -60,7 +59,6 @@ taskkill /IM moq.exe /F ```bash just fix just check -just test ``` See [CONTRIBUTING.md](https://github.com/moq-dev/moq/blob/main/CONTRIBUTING.md) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index a3a574a8f7..ed250cf3fa 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -502,13 +502,14 @@ The container used to frame this rendition's media, as described in {{container} If absent, it defaults to `{ "kind": "legacy" }`. ### jitter {#field-jitter} -The maximum delay, in milliseconds, between a frame being ready and the publisher flushing it. +The maximum delay, in milliseconds, that a publisher has measured before handing a frame to the transport. A consumer's jitter buffer SHOULD be at least this large to avoid stalling. -If absent, a consumer SHOULD assume each frame is flushed immediately. +If absent, a consumer SHOULD assume no publisher delay has been measured. -It is measured at the publisher: how far behind the media clock a frame is when the publisher hands it to the transport, whether an encoder, a reorder buffer, or a segmenter held it. -An importer can estimate this delay from the media span of a batch, such as a run of audio PES packets between video packets in TS or an fMP4 fragment, without measuring the time spent waiting for input. -It is never a measurement of the network, which a consumer observes for itself and which no two consumers of the same broadcast would agree on. +An encoder compares each frame's flush time with its media timestamp and subtracts the smallest lateness recently observed on the same rendition. +The field is the spread above that minimum, so neither a constant encoder delay, a fixed clock offset, nor long-term drift counts as jitter. +A container importer instead estimates the delay from the media span of a batch, such as a run of audio PES packets between video packets in TS or an fMP4 fragment, without measuring the time spent waiting for input. +A publisher does not count ingest network delay in this field; a consumer measures network delay separately. A publisher MUST round the value up to a whole number of milliseconds, so a consumer sizing a buffer against it is never handed a bound below the real one. A publisher MUST NOT advertise `0`; a track that flushes each frame immediately omits the field instead. @@ -517,13 +518,10 @@ A publisher MUST NOT lower a previously advertised value, since a burst it emitt For example: -- If each frame is flushed immediately, a video track's `jitter` is `1000/framerate` rounded up: 34 at 30 fps. -- If up to 3 B-frames may be emitted in a row, it is `3 * 1000/framerate`. -- If frames are buffered into 2 second segments, it is `2000`. -- If frames are flushed several at a time, it is the media span of the whole burst, not of one frame. - -An audio frame's duration is codec dependent. -AAC often uses 1024 samples per frame, so at 44100Hz an immediately-flushed track's `jitter` is 24. +- A frame flushed without extra delay contributes no `jitter`, regardless of frame rate. +- An encoder that consistently flushes 200 milliseconds late contributes no `jitter`; only variation above its own minimum counts. +- A fragment or packet batch contributes the media span between its earliest timestamp and flush point. +- Reordered frames contribute the delay they were held before flushing, without treating a decode-order presentation timestamp gap as delay by itself. # Container {#container} Audio, video, and text tracks use a container to encapsulate the media payload. @@ -1060,9 +1058,10 @@ This document has no IANA actions. ## moq-hang-03 {:numbered="false"} +- Defined encoder `jitter` as flush lateness above the rendition's own recent minimum, replacing fixed frame-duration hints; container batches retain media-span estimates. - Clarified that CMAF audio samples are sync samples independently of publisher group boundaries. - Clarified that container importers can estimate jitter from batch media spans without measuring input wait time. -- Specified the `jitter` field's computation: the publisher's own structure rather than the network, rounded up to whole milliseconds, never `0` (a consumer treats `0` as absent), and never lowered once advertised. The 30 fps and 44.1 kHz AAC examples became 34 and 24. +- Specified the `jitter` field's computation: the publisher's own structure rather than the network, rounded up to whole milliseconds, never `0` (a consumer treats `0` as absent), and never lowered once advertised. - For video, an empty codec payload is the exclusive end of the frame before it. A video publisher SHOULD end each group with one when the exclusive end is known. Audio retains its terminal-trimming marker before codec drain packets. A publisher MAY estimate an unknown final duration from the frame cadence, but MUST NOT use batching or reorder delay as that duration. A consumer skips it and does not submit it to a decoder. Audio terminal-packet trimming is unchanged. - Specified version 1 recording objects: JSON track properties and binary group/frame tables with ascending, delta-encoded group sequences. diff --git a/drafts/draft-lcurley-moq-hidden.md b/drafts/draft-lcurley-moq-hidden.md new file mode 100644 index 0000000000..ee1c14720e --- /dev/null +++ b/drafts/draft-lcurley-moq-hidden.md @@ -0,0 +1,139 @@ +--- +title: "MoQ Hidden Extension" +abbrev: "moq-hidden" +category: info + +docname: draft-lcurley-moq-hidden-latest +submissiontype: IETF # also: "independent", "editorial", "IAB", or "IRTF" +number: +date: +v: 3 +area: wit +workgroup: moq + +author: + - + fullname: Luke Curley + email: kixelated@gmail.com + +normative: + moqt: I-D.ietf-moq-transport + +informative: + +--- abstract + +This document defines an extension for MoQ Transport {{moqt}} that hides namespaces from discovery. +A namespace with a field starting with `.` below the prefix a subscriber asked for is left out of the advertisements it receives, unless its SUBSCRIBE_NAMESPACE opts in. +A platform can then add internal namespaces, such as statistics, without them turning up in applications that list everything and use what they find. + +--- note_Note_to_Readers + +This document was generated by an AI model from the implementation at [github.com/moq-dev/moq](https://github.com/moq-dev/moq) and is maintained alongside it. +Submit an [issue](https://github.com/moq-dev/moq/issues) or [PR](https://github.com/moq-dev/moq/pulls) if this spec sucks and you want to fix anything. + +--- middle + +# Conventions and Definitions +{::boilerplate bcp14-tagged} + +An endpoint **advertises** a namespace by sending PUBLISH_NAMESPACE, or NAMESPACE in response to a SUBSCRIBE_NAMESPACE. + +A namespace is **hidden** from a subscription when one of its fields beyond the subscription's Track Namespace Prefix starts with the byte 0x2E (`.`). +A field inside the prefix never hides anything, so a prefix that names the hidden field itself lists what is under it, and a namespace at or above the prefix has no field beyond it. +Only the first byte counts: `catalog.v2` is not hidden. + + +# Introduction +Discovery in {{moqt}} is all or nothing: a subscriber that asks for a prefix is told every namespace beneath it. +An application that asks for the empty prefix and plays what it finds breaks the moment its platform publishes anything else under the same root, like a relay's own statistics or internal routing state. + +This extension reserves a leading `.` for such namespaces, as file systems do for hidden files. +A hidden namespace is still published, routed, and subscribed to like any other; it is only left out of discovery by default. +A subscriber that wants hidden namespaces too says so on the SUBSCRIBE_NAMESPACE that would list them. + +The name alone decides: there is no publisher-side flag, so a namespace cannot be hidden from one subscriber and listed to another under the same prefix. + + +# Setup Negotiation + +An endpoint declares that it understands the HIDDEN parameter ({{parameter}}) with the following Setup Option ({{moqt}} Section 10.3): + +~~~ +HIDDEN Setup Option { + Option Key (vi64) = 0x40B5C + Option Value (vi64) = 1 +} +~~~ + +A receiver MUST ignore the value. +An endpoint MUST NOT send the HIDDEN parameter to a peer that did not declare this option, because an unknown parameter fails decoding. +A subscriber that wants hidden namespaces therefore waits for the peer's SETUP before sending SUBSCRIBE_NAMESPACE. + +The rest of this extension applies whether or not the peer declared the option: a peer that never heard of it never opts in, so it is never told about hidden namespaces. + + +# Opting In {#parameter} + +A subscriber opts in to hidden namespaces with the following parameter on SUBSCRIBE_NAMESPACE: + +~~~ +HIDDEN Parameter { + Type (vi64) = 0x40B5E + Value (vi64) = 0 or 1 +} +~~~ + +A value of 1 opts in; 0 or an absent parameter does not. +A receiver MUST close the session with a PROTOCOL_VIOLATION on any other value. + + +# Advertising {#advertising} + +A publisher SHOULD NOT advertise a hidden namespace in response to a SUBSCRIBE_NAMESPACE that did not opt in. +Hiding is a convenience for discovery, not access control, so a publisher MAY treat a subscriber it trusts, such as another relay in its own cluster, as opted in. + +An unsolicited PUBLISH_NAMESPACE answers no prefix, so it is measured against the empty one: a publisher SHOULD NOT send one for a hidden namespace. +When unsolicited advertisements are live, a SUBSCRIBE_NAMESPACE is answered with only the namespaces they left out, which is to say those hidden from the empty prefix, that the subscription may see. +That covers both an opt-in and a prefix that names a hidden field itself, and no namespace is advertised twice. + +Hiding narrows discovery and nothing else. +A SUBSCRIBE, FETCH, or TRACK_STATUS for a track in a hidden namespace is served exactly as it would be without this extension. + + +# Security Considerations + +A hidden namespace is not a secret. +Anyone who learns its name can subscribe to it, and a subscriber can opt in at will, so a publisher MUST apply the same authorization to hidden namespaces as to any other. + + +# IANA Considerations + +This document requests the following registrations. +High, distinctive values are requested to avoid the low ranges reserved by {{moqt}} and to minimize collisions with provisional registrations by other extensions. + +## MOQT Setup Options + +This document requests one registration in the "MOQT Setup Options" registry ({{moqt}} Section 15.4), whose policy is Specification Required. + +| Value | Name | Reference | +|:--------|:-------|:--------------| +| 0x40B5C | HIDDEN | This Document | + +## MOQT Message Parameters + +This document requests one registration in the "MOQT Message Parameters" registry ({{moqt}} Section 15.7). + +| Value | Name | Carried In | Reference | +|:--------|:-------|:--------------------|:--------------| +| 0x40B5E | HIDDEN | SUBSCRIBE_NAMESPACE | This Document | + +Both values are even, so each is a bare varint. + + +--- back + +# Acknowledgments +{:numbered="false"} + +This document was drafted with the assistance of Claude, an AI assistant by Anthropic. diff --git a/drafts/draft-lcurley-moq-lite.md b/drafts/draft-lcurley-moq-lite.md index dcac539e93..8c5c464c08 100644 --- a/drafts/draft-lcurley-moq-lite.md +++ b/drafts/draft-lcurley-moq-lite.md @@ -94,7 +94,7 @@ A Session consists of a connection between a client and a server. There is currently no P2P support within QUIC so it's out of scope for moq-lite. The moq-lite version identifier is `moq-lite-xx` where `xx` is the two-digit draft version. -The identifier for this draft is `moq-lite-06`. +The identifier for this draft is `moq-lite-07`. For bare QUIC, this is negotiated as an ALPN token during the QUIC handshake. For WebTransport over HTTP/3, the QUIC ALPN remains `h3` and the moq-lite version is advertised via the `WT-Available-Protocols` and `WT-Protocol` CONNECT headers. @@ -383,6 +383,15 @@ A route covers a path when its prefix is a leading run of the path's segments; m A publisher answering a request stream presents each of its routes clamped to the intersection with the requested prefix: a route above the request's prefix appears as the request prefix itself (an empty suffix), which is exactly the covered set the subscriber may see. There MAY be multiple Announce Streams, potentially containing overlapping prefixes, that get their own ANNOUNCE_OK + announcements. +#### Hidden Paths {#hidden} +A route is hidden from a request when a segment of its path below the requested prefix starts with `.` (0x2E). +A segment inside the prefix never hides anything, so a request that names the hidden segment itself (`.stats`) lists what is under it, and a route at or above the prefix has no segment below it. +Only the first byte counts: `catalog.v2` is not hidden. + +A publisher SHOULD NOT announce a hidden route unless the ANNOUNCE_REQUEST set `Hidden`. +Hiding is a convenience for discovery, not access control: a publisher MAY treat a subscriber it trusts, such as another relay in its own cluster, as opted in, and MUST authorize hidden paths like any other. +SUBSCRIBE, FETCH, and TRACK resolve a hidden path exactly as any other. + #### Routing {#routing} Each advertisement carries the path of Hop IDs it traversed and an accumulated Warm and Cold Route Cost (see [ANNOUNCE_START](#announce-start)), which relays use to build a loop-free mesh. @@ -801,12 +810,17 @@ A subscriber sends an ANNOUNCE_REQUEST message to indicate it wants to receive a ANNOUNCE_REQUEST Message { Message Length (i) Broadcast Path Prefix (s), + Hidden (8), } ~~~ **Broadcast Path Prefix**: Indicate interest for any broadcasts with a path that starts with this prefix. +**Hidden**: +1 to also receive hidden routes (see [Hidden Paths](#hidden)), 0 otherwise. +Any other value is a PROTOCOL_VIOLATION. + The publisher MUST respond with an ANNOUNCE_OK message followed by ANNOUNCE_START messages for any matching routes, followed by ANNOUNCE_START, ANNOUNCE_END, and ANNOUNCE_UPDATE messages for any future updates, subject to [Routing](#routing). Implementations SHOULD consider reasonable limits on the number of matching broadcasts to prevent resource exhaustion. @@ -1314,6 +1328,11 @@ The `Message Length` describes the payload size on the wire. # Appendix A: Changelog +## moq-lite-07 + +- Assigned `moq-lite-07` as this draft's protocol identifier. +- Hid routes with a `.`-prefixed segment below the requested prefix from announce discovery, and added the ANNOUNCE_REQUEST `Hidden` field to opt in. + ## moq-lite-06 - Assigned `moq-lite-06` as this draft's protocol identifier. diff --git a/flake.nix b/flake.nix index 9cee6d70e4..028422fd1b 100644 --- a/flake.nix +++ b/flake.nix @@ -304,17 +304,18 @@ ]; # uniffi-bindgen-dart renders rs/moq-ffi into dart/moq_ffi. The fork - # carries the uniffi 0.32 port and library-mode CLI while those changes - # remain open upstream. + # carries the uniffi 0.32 port, library-mode CLI, and RustBuffer leak + # fixes while those changes remain open upstream. Its tags add a + # `-kixelated.N` pre-release so they never collide with upstream's. uniffi-bindgen-dart = pkgs.rustPlatform.buildRustPackage rec { pname = "uniffi-bindgen-dart"; - version = "0.3.0+v0.32.0"; + version = "0.3.1-kixelated.4+v0.32.0"; src = pkgs.fetchFromGitHub { owner = "kixelated"; repo = "uniffi-dart"; rev = "v${version}"; - hash = "sha256-jvVEZVZLorj+GPUXL6Y4riCLsbJcWWbQgIIUoK/ZSEo="; + hash = "sha256-BCIooajAp0Wqt7LeanFSdmS/GT0uYo+d8Qv2jGWCJD8="; }; # The upstream repository ignores Cargo.lock so cargo installs test diff --git a/go/scripts/stage.sh b/go/scripts/stage.sh index ca72d92c67..3fcde90410 100755 --- a/go/scripts/stage.sh +++ b/go/scripts/stage.sh @@ -63,8 +63,8 @@ command -v uniffi-bindgen-go >/dev/null 2>&1 || { HOST_TARGET=$(rustc -vV | awk '/^host:/ {print $2}') # Debug by default. This is a compile-and-test gate, not a benchmark, and a -# release build of moq-ffi shares no artifacts with the debug ones `just check` -# and `just test` already produce, so it was a third full compile of the +# release build of moq-ffi shares no artifacts with the debug ones `just ci check` +# and `just ci test` already produce, so it was a third full compile of the # dependency tree (~5 min of CI on its own, plus a whole target/release tree on # a runner that was already tight on disk). Set MOQ_FFI_PROFILE=release for an # optimized cdylib; the shipped artifacts are built by rs/moq-ffi/build.sh, diff --git a/go/wrapper/README.md b/go/wrapper/README.md index 52ed3ce69d..cf241671b5 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -77,6 +77,10 @@ for media tracks whose timescale should be selected by the importer. `WithVideoHint(moq.VideoHint{...})` for video catalog fields that are known before the stream reveals them. +`WithAudioTrack(name)` / `WithVideoTrack(name)` name the track instead of +deriving a unique name from the format. A duplicate name fails, and the +`OnTrack` variants refuse it because the request already names the track. + JSON tracks are available in two modes. `PublishJSONSnapshot` / `SubscribeJSONSnapshot` carry lossy latest state, while `PublishJSONStream` / `SubscribeJSONStream` carry every record in order. Producers accept any `encoding/json` value; consumers return diff --git a/go/wrapper/origin.go b/go/wrapper/origin.go index cb5c1915f0..67eb77e040 100644 --- a/go/wrapper/origin.go +++ b/go/wrapper/origin.go @@ -125,6 +125,8 @@ type AnnounceOptions struct { Prefix string // Filter is a pattern relative to Prefix. Nil matches every path beneath it. Filter *string + // Hidden also lists paths with a segment starting with "." below Prefix. + Hidden bool } // Announced streams routes under a literal prefix matching an optional pattern filter. @@ -132,6 +134,7 @@ func (o *OriginConsumer) Announced(options AnnounceOptions) (*AnnounceConsumer, inner, err := o.inner.Announced(ffi.MoqAnnounceConfig{ Prefix: options.Prefix, Filter: options.Filter, + Hidden: options.Hidden, }) if err != nil { return nil, err diff --git a/go/wrapper/publish.go b/go/wrapper/publish.go index 29ef6ad19c..6eccc6d8f6 100644 --- a/go/wrapper/publish.go +++ b/go/wrapper/publish.go @@ -30,6 +30,22 @@ func WithVideoLabel(label string) VideoOption { } } +// WithAudioTrack names the track instead of deriving a unique name from the +// format. A requested track already has a name, so the OnTrack variant refuses it. +func WithAudioTrack(track string) AudioOption { + return func(init *ffi.MoqAudioInit) { + init.Track = &track + } +} + +// WithVideoTrack names the track instead of deriving a unique name from the +// format. A requested track already has a name, so the OnTrack variant refuses it. +func WithVideoTrack(track string) VideoOption { + return func(init *ffi.MoqVideoInit) { + init.Track = &track + } +} + // WithVideoHint seeds catalog fields that a video stream cannot reveal itself. func WithVideoHint(hint VideoHint) VideoOption { return func(init *ffi.MoqVideoInit) { @@ -343,6 +359,12 @@ func (m *MediaProducer) WriteFrame(frame Frame) error { return m.inner.WriteFrame(frame) } +// Flush records a local encoder's frame handoff on the broadcast media clock. +// Call after WriteFrame only for local encoder output, not file or network imports. +func (m *MediaProducer) Flush(timestampUs uint64) error { + return m.inner.Flush(timestampUs) +} + // Cut draws a group boundary here. // // Audio has no boundary of its own (every packet is independently decodable), so this is diff --git a/js/CLAUDE.md b/js/CLAUDE.md index c565e540eb..9e0bf43c2a 100644 --- a/js/CLAUDE.md +++ b/js/CLAUDE.md @@ -13,6 +13,7 @@ The spine of the JS code; read `signals/src/index.ts` before touching reactive c - `Signal` writes coalesce per microtask and notify only on change. Equality is deep for plain data but identity for class instances; `set(v, true)` forces a notify. `peek` reads without subscribing. - `Computed`: derived, `undefined` until first run and after `close()`. Standalone ones must be closed; `effect.computed()` closes with its parent. - `Effect`: reruns when a signal read via `effect.get(signal)` changes. Register teardown with `effect.cleanup(fn)`; it runs before the next run and on `close()`. A rerun waits for every `effect.spawn` task from the previous run to settle, so register teardown unconditionally. +- Never `Promise.race` a value that outlives the call, such as a `closed`; use `race` or `effect.race`, which release their listeners. - Use the scoped helpers (`effect.interval`, `timer`, `timeout`, `animate`, `event`, `subscribe`, `set`, `proxy`, `run`) instead of raw timers or listeners, so cleanup is automatic. Prefer nested `effect.run` over one giant effect. # Producer / consumer diff --git a/js/binary/package.json b/js/binary/package.json index 499de5a656..849e5f012a 100644 --- a/js/binary/package.json +++ b/js/binary/package.json @@ -21,7 +21,8 @@ }, "dependencies": { "@moq/flate": "workspace:^", - "@moq/net": "workspace:^" + "@moq/net": "workspace:^", + "@moq/signals": "workspace:^" }, "devDependencies": { "@types/bun": "^1.4.2", diff --git a/js/binary/src/stream/consumer.ts b/js/binary/src/stream/consumer.ts index 47d9ee635d..f4cc5cf441 100644 --- a/js/binary/src/stream/consumer.ts +++ b/js/binary/src/stream/consumer.ts @@ -1,5 +1,6 @@ import { Decoder as Flate } from "@moq/flate"; import type * as Moq from "@moq/net"; +import { race } from "@moq/signals"; import { isDeflate } from "../compression.ts"; import type { Config as CodecConfig } from "./producer.ts"; @@ -102,7 +103,7 @@ export class Consumer { if (buffered) return buffered; const frame = group.readFrame(); - const winner = await Promise.race([frame.then((frame) => ({ frame }) as const), this.#recvGroup()]); + const winner = await race([frame.then((frame) => ({ frame }) as const), this.#recvGroup()]); if ("frame" in winner) return winner.frame; if (winner.group) { diff --git a/js/binary/src/stream/stream.test.ts b/js/binary/src/stream/stream.test.ts index b1f547c3a3..b8745147fb 100644 --- a/js/binary/src/stream/stream.test.ts +++ b/js/binary/src/stream/stream.test.ts @@ -1,3 +1,4 @@ +import { heapStats } from "bun:jsc"; import { expect, test } from "bun:test"; import { DEFAULT_MAX_FRAME_SIZE } from "@moq/flate"; import { Time, Track } from "@moq/net"; @@ -146,3 +147,32 @@ test("an undecodable payload ends the log for a reader already inside the group" // Surfaces the terminal error rather than hanging on the still-open group. await expect(consumer.next()).rejects.toThrow("limit"); }); + +// A blocked read races the frame against the track's next group, which stays pending for the whole +// log. Racing it per payload must not leave a reaction behind on it each time. +test("blocked reads leave nothing behind on the pending group read", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track }); + const subscriber = track.subscribe(); + const consumer = new Consumer({ track: subscriber }); + const promises = () => { + Bun.gc(true); + return heapStats().objectTypeCounts.Promise ?? 0; + }; + + const read = async (from: number, count: number) => { + for (let n = from; n < from + count; n++) { + const next = consumer.next(); + producer.append(new Uint8Array([n & 0xff])); + expect((await next)?.[0]).toBe(n & 0xff); + } + }; + + await read(0, 50); + const before = promises(); + await read(50, 1000); + expect(promises() - before).toBeLessThan(100); + + subscriber.close(); + producer.finish(); +}); diff --git a/js/hang/src/catalog/text.ts b/js/hang/src/catalog/text.ts index a8330ab028..3bd4e3dc21 100644 --- a/js/hang/src/catalog/text.ts +++ b/js/hang/src/catalog/text.ts @@ -52,7 +52,12 @@ export const TextConfigSchema = z.object({ // The maximum jitter before the next cue is flushed, in milliseconds. The player's jitter buffer // should be at least this large; absent means each cue is flushed immediately. - jitter: z.optional(u53Schema), + jitter: z.optional( + z.pipe( + u53Schema, + z.transform((value) => (value === 0 ? undefined : value)), + ), + ), }); /** Schema for the catalog text section: a map of track name to rendition config. */ diff --git a/js/watch/src/audio/unlock.test.ts b/js/hang/src/util/gesture.test.ts similarity index 89% rename from js/watch/src/audio/unlock.test.ts rename to js/hang/src/util/gesture.test.ts index 1dba2f807e..f65087dbb3 100644 --- a/js/watch/src/audio/unlock.test.ts +++ b/js/hang/src/util/gesture.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import { Effect } from "@moq/signals"; -import { unlockOnGesture } from "./unlock"; +import { unlock } from "./gesture"; // Minimal AudioContext stand-in: an EventTarget with a mutable `state` and a counting // `resume()`. `transition` mirrors a real context firing `statechange` when its state moves. @@ -38,7 +38,7 @@ const asContext = (ctx: MockContext) => ctx as unknown as AudioContext; test("retries resume() on a user gesture until the context is running", async () => { const ctx = new MockContext(); const effect = new Effect(); - unlockOnGesture(effect, asContext(ctx)); + unlock(effect, asContext(ctx)); await flush(); // The at-load attempt fires once. Browsers requiring a gesture reject it, but we still @@ -53,11 +53,15 @@ test("retries resume() on a user gesture until the context is running", async () document.dispatchEvent(new Event("keydown")); expect(ctx.resumeCalls).toBe(3); + // Touch and pen only grant activation on pointerup, so a tap must retry there too. + document.dispatchEvent(new Event("pointerup")); + expect(ctx.resumeCalls).toBe(4); + // Once the context is actually running, stop retrying: further gestures are no-ops. ctx.transition("running"); await flush(); document.dispatchEvent(new Event("pointerdown")); - expect(ctx.resumeCalls).toBe(3); + expect(ctx.resumeCalls).toBe(4); effect.close(); }); @@ -65,7 +69,7 @@ test("retries resume() on a user gesture until the context is running", async () test("re-arms when Safari drops the context to interrupted", async () => { const ctx = new MockContext(); const effect = new Effect(); - unlockOnGesture(effect, asContext(ctx)); + unlock(effect, asContext(ctx)); await flush(); ctx.transition("running"); @@ -87,7 +91,7 @@ test("re-arms when Safari drops the context to interrupted", async () => { test("stops resuming after the effect closes", async () => { const ctx = new MockContext(); const effect = new Effect(); - unlockOnGesture(effect, asContext(ctx)); + unlock(effect, asContext(ctx)); await flush(); effect.close(); diff --git a/js/hang/src/util/gesture.ts b/js/hang/src/util/gesture.ts new file mode 100644 index 0000000000..e92b72998d --- /dev/null +++ b/js/hang/src/util/gesture.ts @@ -0,0 +1,38 @@ +import { type Effect, type Getter, Signal } from "@moq/signals"; + +/** + * Resume a suspended {@link AudioContext} from a real user gesture, returning whether it is running. + * + * A context built before any user activation starts suspended in browsers that gate audio on a + * gesture, and a `resume()` made then is rejected. A single unconditional attempt would fire once, + * be rejected, and never retry, leaving the graph silent. This instead attempts `resume()` + * immediately (for autoplay-permissive browsers like Chrome with prior engagement), then retries on + * every gesture until the context is actually running, dropping the listeners once it is. A mouse + * grants activation on `pointerdown` but touch and pen only on `pointerup`, so both are listened to, + * plus `keydown`. + * + * Safari also reports an "interrupted" state (a WebKit-only value outside the + * suspended/running/closed set) and can leave it on its own; mirroring `statechange` into the + * returned signal picks that up so the listeners are re-armed or dropped as the state moves. + * + * Scoped to `effect`: the listeners are removed when the effect reruns or closes. + */ +export function unlock(effect: Effect, context: AudioContext): Getter { + const running = new Signal(context.state === "running"); + effect.event(context, "statechange", () => running.set(context.state === "running")); + + effect.run((inner) => { + if (inner.get(running)) return; + + const resume = () => { + context.resume().catch(() => {}); + }; + + resume(); + inner.event(document, "pointerdown", resume); + inner.event(document, "pointerup", resume); + inner.event(document, "keydown", resume); + }); + + return running; +} diff --git a/js/hang/src/util/index.ts b/js/hang/src/util/index.ts index cef64e9f45..a09394873b 100644 --- a/js/hang/src/util/index.ts +++ b/js/hang/src/util/index.ts @@ -1,11 +1,12 @@ /** * Miscellaneous helpers for the hang media layer: AAC and Opus codec constraints, hex encoding, - * and the libav/WebCodecs polyfill. + * the libav/WebCodecs polyfill, and unlocking Web Audio on a user gesture. * * @module */ export * as Aac from "./aac"; +export * as Gesture from "./gesture"; export * as Hacks from "./hacks"; export * as Hex from "./hex"; export * as Libav from "./libav"; diff --git a/js/json/src/stream/consumer.ts b/js/json/src/stream/consumer.ts index 0d93d8dd10..6ca14024be 100644 --- a/js/json/src/stream/consumer.ts +++ b/js/json/src/stream/consumer.ts @@ -1,4 +1,5 @@ import type * as Moq from "@moq/net"; +import { race } from "@moq/signals"; import { Decoder } from "./decoder.ts"; import type { Config as CodecConfig } from "./encoder.ts"; @@ -101,7 +102,7 @@ export class Consumer { if (buffered) return buffered; const frame = group.readFrame(); - const winner = await Promise.race([frame.then((frame) => ({ frame }) as const), this.#recvGroup()]); + const winner = await race([frame.then((frame) => ({ frame }) as const), this.#recvGroup()]); if ("frame" in winner) return winner.frame; if (winner.group) { diff --git a/js/json/src/stream/stream.test.ts b/js/json/src/stream/stream.test.ts index 5e93b54724..35512bac88 100644 --- a/js/json/src/stream/stream.test.ts +++ b/js/json/src/stream/stream.test.ts @@ -1,3 +1,4 @@ +import { heapStats } from "bun:jsc"; import { expect, test } from "bun:test"; import { Time, Track } from "@moq/net"; import { Consumer, Producer, Rolled } from "./index.ts"; @@ -110,3 +111,32 @@ test("a second concurrent read is refused rather than served the first one's gro expect(() => consumer.next()).toThrow("multiple calls to next not supported"); expect(await first).toEqual({ n: 0 }); }); + +// A blocked read races the frame against the track's next group, which stays pending for the whole +// log. Racing it per record must not leave a reaction behind on it each time. +test("blocked reads leave nothing behind on the pending group read", async () => { + const track = new Track.Producer("test"); + const producer = new Producer({ track }); + const subscriber = track.subscribe(); + const consumer = new Consumer({ track: subscriber }); + const promises = () => { + Bun.gc(true); + return heapStats().objectTypeCounts.Promise ?? 0; + }; + + const read = async (from: number, count: number) => { + for (let n = from; n < from + count; n++) { + const next = consumer.next(); + producer.append({ n }); + expect((await next)?.n).toBe(n); + } + }; + + await read(0, 50); + const before = promises(); + await read(50, 1000); + expect(promises() - before).toBeLessThan(100); + + subscriber.close(); + producer.finish(); +}); diff --git a/js/moq-boy/src/element.tsx b/js/moq-boy/src/element.tsx index b776fd8e52..48346b6de1 100644 --- a/js/moq-boy/src/element.tsx +++ b/js/moq-boy/src/element.tsx @@ -135,7 +135,7 @@ export default class MoqBoy extends HTMLElement { effect.spawn(async () => { for (;;) { - const entry = await Promise.race([effect.cancel, announced.next()]); + const entry = await effect.race(announced.next()); if (!entry) break; // A broad route that cannot pin the game id names nothing to open. diff --git a/js/moq-boy/src/game.ts b/js/moq-boy/src/game.ts index c9bec3f4d5..9d2a03d39d 100644 --- a/js/moq-boy/src/game.ts +++ b/js/moq-boy/src/game.ts @@ -266,7 +266,7 @@ export class Game { const consumer = new Json.Snapshot.Consumer({ track: statusTrack, schema: GameStatusSchema }); // Closing the track on cleanup unblocks a pending next() (it returns undefined), so the loop - // ends without racing effect.cancel. + // ends without racing the teardown. effect.spawn(async () => { for (;;) { let status: GameStatus | undefined; diff --git a/js/net/examples/wait.ts b/js/net/examples/wait.ts index 5bf4d8170a..79adf1babf 100644 --- a/js/net/examples/wait.ts +++ b/js/net/examples/wait.ts @@ -28,7 +28,7 @@ async function main() { effect.spawn(async () => { for (;;) { - const group = await Promise.race([effect.cancel, track.recvGroup()]); + const group = await effect.race(track.recvGroup()); if (!group) break; console.log("received:", await group.readString()); } diff --git a/js/net/src/announced.ts b/js/net/src/announced.ts index 8c0f25527f..01056ae153 100644 --- a/js/net/src/announced.ts +++ b/js/net/src/announced.ts @@ -38,6 +38,21 @@ export interface Update { route: Route; } +/** + * Options for an announcement stream. + * + * @public + */ +export interface Options { + /** + * Also report hidden routes: those with a path segment starting with `.` below the + * scope's literal head. Hidden routes are left out by default, so a platform can add + * `.`-named broadcasts (stats, internal routes) without them turning up in apps that + * list everything. Subscribing to a hidden path by name works either way. + */ + hidden?: boolean; +} + /** Whether a route covers the path after an update of this {@link Kind}. */ export function isActive(kind: Kind): boolean { return kind !== "retracted"; diff --git a/js/net/src/connection/accept.ts b/js/net/src/connection/accept.ts index c048ec5768..0430892187 100644 --- a/js/net/src/connection/accept.ts +++ b/js/net/src/connection/accept.ts @@ -87,6 +87,8 @@ async function acceptInner( return acceptSetup(transport, url, Ietf.Version.DRAFT_16, wiring); } else if (protocol === Ietf.ALPN.DRAFT_15) { return acceptSetup(transport, url, Ietf.Version.DRAFT_15, wiring); + } else if (protocol === Lite.ALPN_07) { + return new Lite.Connection({ url, quic: transport, version: Lite.Version.DRAFT_07, ...wiring }); } else if (protocol === Lite.ALPN_06) { return new Lite.Connection({ url, quic: transport, version: Lite.Version.DRAFT_06, ...wiring }); } else if (protocol === Lite.ALPN_05) { @@ -112,7 +114,7 @@ async function acceptAlpn( version: Ietf.IetfVersion, wiring: SessionProps, ): Promise { - const { control, solicit, cluster } = await exchangeSetup(transport, version, "moq-lite-js"); + const { control, solicit, hidden, cluster } = await exchangeSetup(transport, version, "moq-lite-js"); return new Ietf.Connection({ ...wiring, @@ -121,6 +123,7 @@ async function acceptAlpn( quic: transport, control, solicit, + hidden, cluster, // v17+ uses NativeSession which manages its own request IDs; maxRequestId is unused. maxRequestId: 0n, @@ -156,6 +159,7 @@ async function acceptSetup( params.setVarint(Ietf.SetupOption.MaxRequestId, 42069n); params.setBytes(Ietf.SetupOption.Implementation, encoder.encode("moq-lite-js")); Ietf.solicitIntoSetup(params); + Ietf.hiddenIntoSetup(params); const server = new Ietf.ServerSetup({ version, parameters: params }); await server.encode(stream.writer, version); @@ -171,6 +175,7 @@ async function acceptSetup( maxRequestId, version, solicit: Ietf.solicitFromSetup(client.parameters), + hidden: Ietf.hiddenFromSetup(client.parameters), }); } @@ -214,6 +219,7 @@ async function acceptNegotiated( params.setVarint(Ietf.SetupOption.MaxRequestId, 42069n); params.setBytes(Ietf.SetupOption.Implementation, encoder.encode("moq-lite-js")); Ietf.solicitIntoSetup(params); + Ietf.hiddenIntoSetup(params); const server = new Ietf.ServerSetup({ version: selectedVersion, parameters: params }); await server.encode(stream.writer, setupVersion); @@ -237,6 +243,7 @@ async function acceptNegotiated( maxRequestId, version: selectedVersion as Ietf.IetfVersion, solicit: Ietf.solicitFromSetup(client.parameters), + hidden: Ietf.hiddenFromSetup(client.parameters), }); } else { throw new Error(`unsupported version: ${selectedVersion.toString(16)}`); diff --git a/js/net/src/connection/connect.test.ts b/js/net/src/connection/connect.test.ts index 22dc73115b..02b8e6d175 100644 --- a/js/net/src/connection/connect.test.ts +++ b/js/net/src/connection/connect.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { ALPN_05, ALPN_06 } from "../lite/version.ts"; +import { ALPN_05, ALPN_07 } from "../lite/version.ts"; import { createMockTransportPair } from "../mock.ts"; import { type ConnectProps, connect as connectSession } from "./connect.ts"; @@ -54,8 +54,8 @@ function stubWebTransport(transport: WebTransport): () => void { }; } -test("WebTransport offers lite-06 first by default", async () => { - const pair = createMockTransportPair(ALPN_06); +test("WebTransport offers lite-07 first by default", async () => { + const pair = createMockTransportPair(ALPN_07); const original = globalThis.WebTransport; let protocols: string[] | undefined; @@ -72,7 +72,7 @@ test("WebTransport offers lite-06 first by default", async () => { globalThis.WebTransport = original; } - expect(protocols?.[0]).toBe("moq-lite-06"); + expect(protocols?.[0]).toBe("moq-lite-07"); }); test("connect logs the relay URL without its credentials", async () => { diff --git a/js/net/src/connection/connect.ts b/js/net/src/connection/connect.ts index 185cd4fecc..b0fcbc62b1 100644 --- a/js/net/src/connection/connect.ts +++ b/js/net/src/connection/connect.ts @@ -281,6 +281,8 @@ async function negotiate(url: URL, session: WebTransport, wiring: SessionProps): setupVersion = Ietf.Version.DRAFT_16; } else if (protocol === Ietf.ALPN.DRAFT_15) { setupVersion = Ietf.Version.DRAFT_15; + } else if (protocol === Lite.ALPN_07) { + return new Lite.Connection({ url, quic: session, version: Lite.Version.DRAFT_07, ...wiring }); } else if (protocol === Lite.ALPN_06) { return new Lite.Connection({ url, quic: session, version: Lite.Version.DRAFT_06, ...wiring }); } else if (protocol === Lite.ALPN_05) { @@ -304,6 +306,7 @@ async function negotiate(url: URL, session: WebTransport, wiring: SessionProps): params.setVarint(Ietf.SetupOption.MaxRequestId, 42069n); params.setBytes(Ietf.SetupOption.Implementation, encoder.encode("moq-lite-js")); Ietf.solicitIntoSetup(params); + Ietf.hiddenIntoSetup(params); const client = new Ietf.ClientSetup({ versions: @@ -342,6 +345,7 @@ async function negotiate(url: URL, session: WebTransport, wiring: SessionProps): maxRequestId, version: server.version as Ietf.IetfVersion, solicit: Ietf.solicitFromSetup(server.parameters), + hidden: Ietf.hiddenFromSetup(server.parameters), }); } else { throw new Error(`unsupported server version: ${server.version.toString()}`); @@ -358,7 +362,7 @@ async function handshakeAlpn( version: Ietf.IetfVersion, wiring: SessionProps, ): Promise { - const { control, solicit, cluster } = await exchangeSetup(session, version, "moq-lite-js"); + const { control, solicit, hidden, cluster } = await exchangeSetup(session, version, "moq-lite-js"); return new Ietf.Connection({ ...wiring, @@ -367,6 +371,7 @@ async function handshakeAlpn( quic: session, control, solicit, + hidden, cluster, // v17+ uses NativeSession which manages its own request IDs; maxRequestId is unused. maxRequestId: 0n, @@ -433,6 +438,7 @@ async function connectWebTransport( allowPooling: false, congestionControl: "low-latency", protocols: [ + Lite.ALPN_07, Lite.ALPN_06, Lite.ALPN_05, Lite.ALPN_04, @@ -519,6 +525,7 @@ async function connectWebSocket(url: URL, delay: number, cancel: Promise): // advertises every QMux draft it knows about and the server picks one. // Insertion order is the negotiation preference on the wire. const versions = { + [Lite.ALPN_07]: null, [Lite.ALPN_06]: null, [Lite.ALPN_05]: null, [Lite.ALPN_04]: null, diff --git a/js/net/src/connection/established.ts b/js/net/src/connection/established.ts index c8d7278217..f796892762 100644 --- a/js/net/src/connection/established.ts +++ b/js/net/src/connection/established.ts @@ -37,8 +37,9 @@ export interface Established { * Subscribe to broadcast announcements matching `scope`, any pattern (`foo/**` * for a subtree, `room/* /chat` for each room's chat, default `**`). Paths are * relative to the session; captures report what the scope's wildcards stood for. + * Hidden routes are left out unless `options.hidden` opts in. */ - announced(scope?: Path.Pattern): announce.Consumer; + announced(scope?: Path.Pattern, options?: announce.Options): announce.Consumer; /** * Snapshot the transport's counters, querying it fresh on each call. diff --git a/js/net/src/connection/forward.ts b/js/net/src/connection/forward.ts index ff977a61c1..66745398a9 100644 --- a/js/net/src/connection/forward.ts +++ b/js/net/src/connection/forward.ts @@ -3,7 +3,7 @@ * * @module */ -import type { Dispose } from "@moq/signals"; +import { type Dispose, race } from "@moq/signals"; import { isActive } from "../announced.ts"; import type { Dynamic, Producer as OriginProducer, RequestSlot } from "../origin.ts"; import type * as Path from "../path.ts"; @@ -47,7 +47,8 @@ export function forwardAnnounced(conn: Established, origin: OriginProducer): voi return; } - const announced = conn.announced(); + // Hidden routes are mirrored too; each local reader opts in on its own. + const announced = conn.announced(undefined, { hidden: true }); const inserted = new Map(); // End the stream the moment the session closes rather than waiting for the wire to @@ -159,7 +160,7 @@ async function serveRequests(conn: Established, origin: OriginProducer): Promise // Woken by the table too, not just the requests: a path that stops being routed needs // the blind answer this loop skipped while it was. - await Promise.race([table.changed(), closed]); + await race([table.changed(), closed]); } // Session gone: withdraw our answers, waking a standby session to provide fresh ones. diff --git a/js/net/src/connection/handshake.ts b/js/net/src/connection/handshake.ts index ae7154674e..05cd214c8b 100644 --- a/js/net/src/connection/handshake.ts +++ b/js/net/src/connection/handshake.ts @@ -19,11 +19,12 @@ export async function exchangeSetup( transport: WebTransport, version: Ietf.IetfVersion, implementation: string, -): Promise<{ control: Stream; solicit: boolean | undefined; cluster: Ietf.Cluster.Hops }> { +): Promise<{ control: Stream; solicit: boolean | undefined; hidden: boolean; cluster: Ietf.Cluster.Hops }> { const encoder = new TextEncoder(); const params = new Ietf.SetupOptions(); params.setBytes(Ietf.SetupOption.Implementation, encoder.encode(implementation)); Ietf.solicitIntoSetup(params); + Ietf.hiddenIntoSetup(params); // One id per session, like the moq-lite connection: nothing in this process forwards // between sessions, so there is nothing for a shared id to detect. @@ -40,6 +41,7 @@ export async function exchangeSetup( return { control: new Stream({ writer, reader: received.reader }), solicit: received.solicit, + hidden: received.hidden, cluster: { self, peer: received.cluster }, }; } @@ -56,7 +58,7 @@ async function sendSetup(transport: WebTransport, version: Ietf.IetfVersion, set async function receiveSetup( transport: WebTransport, version: Ietf.IetfVersion, -): Promise<{ reader: Reader; solicit: boolean | undefined; cluster: Hop | undefined }> { +): Promise<{ reader: Reader; solicit: boolean | undefined; hidden: boolean; cluster: Hop | undefined }> { const uniReader = transport.incomingUnidirectionalStreams.getReader() as ReadableStreamDefaultReader< ReadableStream >; @@ -75,6 +77,7 @@ async function receiveSetup( return { reader, solicit: Ietf.solicitFromSetup(setup.parameters), + hidden: Ietf.hiddenFromSetup(setup.parameters), cluster: Ietf.Cluster.fromSetup(setup.parameters, version), }; } diff --git a/js/net/src/connection/pool.ts b/js/net/src/connection/pool.ts index 20e4d5d18e..959697ca09 100644 --- a/js/net/src/connection/pool.ts +++ b/js/net/src/connection/pool.ts @@ -265,7 +265,7 @@ export class Connection { * and URL switches: a switch retracts everything from * the old relay's origin, then the new one's arrivals stream in. */ - announced(scope: Path.Pattern = Path.Pattern.all()): Announce.Consumer { + announced(scope: Path.Pattern = Path.Pattern.all(), options?: Announce.Options): Announce.Consumer { const producer = new Announce.Producer(); const consumer = producer.consume(); @@ -283,7 +283,7 @@ export class Connection { const origin = effect.get(this.#origin); if (!origin) return; - const upstream = origin.announced(scope); + const upstream = origin.announced(scope, options); effect.cleanup(() => upstream.close()); // Track what this origin announced so a URL switch retracts it; the last @@ -293,7 +293,7 @@ export class Connection { effect.spawn(async () => { try { for (;;) { - const entry = await Promise.race([effect.cancel, upstream.next()]); + const entry = await effect.race(upstream.next()); if (!entry) break; if (Announce.isActive(entry.kind)) active.set(entry.prefix, entry); else active.delete(entry.prefix); diff --git a/js/net/src/connection/reload.ts b/js/net/src/connection/reload.ts index 5063fbff8d..b7b77bcaef 100644 --- a/js/net/src/connection/reload.ts +++ b/js/net/src/connection/reload.ts @@ -255,7 +255,7 @@ export class Reload { if (pending) return; pending = true; try { - const stats = await Promise.race([effect.cancel, connection.stats()]); + const stats = await effect.race(connection.stats()); if (stats) this.#estimate.set(stats.estimatedSendRate); } finally { pending = false; @@ -341,7 +341,7 @@ export class Reload { // A cancelled effect resolves undefined, so the sentinel tells the session // closing (null for clean, an Error otherwise) apart from this run being // torn down. - const closed = await Promise.race([effect.cancel, connection.closed]); + const closed = await effect.race(connection.closed); if (closed === undefined) return; console.warn("connection closed, reconnecting"); @@ -443,10 +443,10 @@ export class Reload { * * Stays empty while the relay lacks {@link Established.discovery}. */ - announced(scope: Path.Pattern = Path.Pattern.all()): Announce.Consumer { + announced(scope: Path.Pattern = Path.Pattern.all(), options?: Announce.Options): Announce.Consumer { // With a consume origin the table already spans reconnects (the forwarder retracts // a dead session's entries), so its stream is the same thing with less machinery. - if (this.consume) return this.consume.announced(scope); + if (this.consume) return this.consume.announced(scope, options); const producer = new Announce.Producer(); const consumer = producer.consume(); @@ -460,7 +460,7 @@ export class Reload { // consumer empty rather than opening a subscription that can't be answered. if (!conn.discovery) return; - const upstream = conn.announced(scope); + const upstream = conn.announced(scope, options); effect.cleanup(() => upstream.close()); // Track what this connection announced so we can retract it if the connection @@ -470,7 +470,7 @@ export class Reload { effect.spawn(async () => { try { for (;;) { - const entry = await Promise.race([effect.cancel, upstream.next()]); + const entry = await effect.race(upstream.next()); if (!entry) break; if (Announce.isActive(entry.kind)) active.set(entry.prefix, entry); else active.delete(entry.prefix); diff --git a/js/net/src/ietf/adapter.test.ts b/js/net/src/ietf/adapter.test.ts index 5f7ea4c06b..1945cef825 100644 --- a/js/net/src/ietf/adapter.test.ts +++ b/js/net/src/ietf/adapter.test.ts @@ -8,6 +8,18 @@ import { PublishNamespace, PublishNamespaceCancel, PublishNamespaceDone } from " import { RequestError } from "./request.ts"; import { ALPN, Version } from "./version.ts"; +test("draft-14 TRACK_STATUS_OK cannot be routed as NAMESPACE_DONE", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_14); + const control = await Stream.open(pair.server, { version: Version.DRAFT_14 }); + const adapter = new ControlStreamAdapter(pair.server, control, Version.DRAFT_14, 100n, true); + const running = adapter.run(); + const peer = await Stream.accept(pair.client, Version.DRAFT_14); + if (!peer) throw new Error("no control stream"); + await peer.writer.u53(0x0e); + await peer.writer.u16(0); + await expect(running).rejects.toThrow("unexpected message 0x0e"); +}); + // Draft-15 is the interesting one: it names its namespace withdrawals instead of // numbering them, so the adapter has to resolve them through a map it keeps itself. const VERSION = Version.DRAFT_15; diff --git a/js/net/src/ietf/adapter.ts b/js/net/src/ietf/adapter.ts index f0a766370a..11d11dff5a 100644 --- a/js/net/src/ietf/adapter.ts +++ b/js/net/src/ietf/adapter.ts @@ -636,7 +636,10 @@ export class ControlStreamAdapter implements Session { return { route: Route.FollowUp, requestId: subNs08 }; } case 0x0e: { - // v15: NamespaceDone entry (no requestId) — route to SubscribeNamespace stream + if (this.version === Version.DRAFT_14 || this.version === Version.DRAFT_15) { + throw new Error("unexpected message 0x0e"); + } + // v16+: NamespaceDone entry (no requestId) — route to SubscribeNamespace stream const subNs0e = this.#subscribeNamespaces.values().next().value; if (subNs0e === undefined) throw new Error("unexpected message 0x0e: no SubscribeNamespace stream"); return { route: Route.FollowUp, requestId: subNs0e }; diff --git a/js/net/src/ietf/connection.ts b/js/net/src/ietf/connection.ts index d4d24eb416..e1d22f9bf8 100644 --- a/js/net/src/ietf/connection.ts +++ b/js/net/src/ietf/connection.ts @@ -85,6 +85,7 @@ export class Connection implements Established { discovery = true, publish, solicit, + hidden = false, cluster, }: { url: URL; @@ -102,6 +103,8 @@ export class Connection implements Established { * nothing, which is the one case where announcing at us unasked is not a bug. */ solicit?: boolean; + /** Whether the peer understands the HIDDEN parameter (MoQ Hidden). */ + hidden?: boolean; /** * The Hop IDs this session declared (MoQ Cluster). `undefined` on a version that * cannot negotiate the extension, as is a `peer` the peer never declared. @@ -138,7 +141,7 @@ export class Connection implements Established { }); this.#solicit = solicit; this.#cluster = cluster; - this.#subscriber = new Subscriber({ session: this.#session, cluster }); + this.#subscriber = new Subscriber({ session: this.#session, cluster, hidden }); registerWire(this, { consume: (path) => this.#subscriber.consume(path) }); void this.#run(); @@ -179,8 +182,8 @@ export class Connection implements Established { } /** Gets an announced reader for `scope`; see {@link Established.announced}. */ - announced(scope?: Path.Pattern): announce.Consumer { - return this.#subscriber.announced(scope); + announced(scope?: Path.Pattern, options?: announce.Options): announce.Consumer { + return this.#subscriber.announced(scope, options); } /** @@ -219,7 +222,11 @@ export class Connection implements Established { } case SubscribeNamespaceLegacy.id: { const legacy = await SubscribeNamespaceLegacy.decode(stream.reader, this.#session.version); - const msg = new SubscribeNamespace({ requestId: legacy.requestId, namespace: legacy.namespace }); + const msg = new SubscribeNamespace({ + requestId: legacy.requestId, + namespace: legacy.namespace, + hidden: legacy.hidden, + }); await this.#publisher.runSubscribeNamespace(msg, stream); break; } diff --git a/js/net/src/ietf/error.test.ts b/js/net/src/ietf/error.test.ts index 0c84f1d0dc..dbee25afa3 100644 --- a/js/net/src/ietf/error.test.ts +++ b/js/net/src/ietf/error.test.ts @@ -11,9 +11,18 @@ const ALL: IetfVersion[] = [ Version.DRAFT_18, Version.DRAFT_19, Version.DRAFT_20, + Version.DRAFT_21, + Version.DRAFT_22, ]; -const KINDS: RequestKind[] = ["subscribe", "fetch", "publish", "publish_namespace", "subscribe_namespace"]; +const KINDS: RequestKind[] = [ + "subscribe", + "fetch", + "publish", + "publish_namespace", + "subscribe_namespace", + "track_status", +]; /** Every condition a rejection can carry. A new one belongs here. */ const CONDITIONS: RequestCondition[] = [ @@ -22,6 +31,8 @@ const CONDITIONS: RequestCondition[] = [ "timeout", "not_supported", "does_not_exist", + "invalid_range", + "invalid_joining_request_id", "uninterested", "malformed_track", "going_away", @@ -45,6 +56,7 @@ function registered(kind: RequestKind, version: IetfVersion): number[] { case "publish_namespace": return [0x0, 0x1, 0x2, 0x3, 0x4, 0x10, 0x12]; case "subscribe_namespace": + case "track_status": return [0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x10, 0x12]; } } @@ -58,6 +70,19 @@ function registered(kind: RequestKind, version: IetfVersion): number[] { ]; } +test("fetch refusal codes follow each draft registry", () => { + for (const version of ALL) { + const range = version === Version.DRAFT_14 ? 0x5 : 0x11; + expect(toRequestCode("invalid_range", "fetch", version)).toBe(range); + expect(fromRequestCode(range, "fetch", version)).toBe("invalid_range"); + const joining = version === Version.DRAFT_14 ? 0x7 : version <= Version.DRAFT_19 ? 0x32 : undefined; + expect(toRequestCode("invalid_joining_request_id", "fetch", version)).toBe(joining ?? 0); + if (joining !== undefined) + expect(fromRequestCode(joining, "fetch", version)).toBe("invalid_joining_request_id"); + else expect(fromRequestCode(0x32, "fetch", version)).toBeUndefined(); + } +}); + test("only registered codes reach the wire", () => { for (const version of ALL) { for (const kind of KINDS) { diff --git a/js/net/src/ietf/error.ts b/js/net/src/ietf/error.ts index 886594f703..21a5a0b526 100644 --- a/js/net/src/ietf/error.ts +++ b/js/net/src/ietf/error.ts @@ -66,7 +66,13 @@ export function sharedStreamCode(code: number, version: IetfVersion): boolean { * * @internal */ -export type RequestKind = "subscribe" | "fetch" | "publish" | "publish_namespace" | "subscribe_namespace"; +export type RequestKind = + | "subscribe" + | "fetch" + | "publish" + | "publish_namespace" + | "subscribe_namespace" + | "track_status"; /** * What a rejection reports, before the draft picks the number for it. @@ -86,6 +92,8 @@ export type RequestCondition = /** This endpoint does not implement the request at all. */ | "not_supported" /** The broadcast or track the peer asked for is not here. */ + | "invalid_range" + | "invalid_joining_request_id" | "does_not_exist" /** The content the peer offered is not wanted here, so it should stop offering it. */ | "uninterested" @@ -103,6 +111,23 @@ const NOT_SUPPORTED = 0x3; /** A draining session. Draft-17 and later. */ const GOING_AWAY = 0x6; +/** FETCH-only errors. INVALID_JOINING_REQUEST_ID was removed in draft-20. */ +const INVALID_RANGE_14 = 0x5; +const INVALID_RANGE = 0x11; +const INVALID_JOINING_REQUEST_ID_14 = 0x7; +const INVALID_JOINING_REQUEST_ID = 0x32; + +function invalidRange(kind: RequestKind, version: IetfVersion): number | undefined { + if (kind !== "fetch") return undefined; + return version === Version.DRAFT_14 ? INVALID_RANGE_14 : INVALID_RANGE; +} + +function invalidJoiningRequestId(kind: RequestKind, version: IetfVersion): number | undefined { + if (kind !== "fetch") return undefined; + if (version === Version.DRAFT_14) return INVALID_JOINING_REQUEST_ID_14; + return version <= Version.DRAFT_19 ? INVALID_JOINING_REQUEST_ID : undefined; +} + /** Draft-14 calls it TRACK_DOES_NOT_EXIST; draft-15 renamed it and moved it off 0x4. */ const DOES_NOT_EXIST_14 = 0x4; const DOES_NOT_EXIST = 0x10; @@ -118,7 +143,7 @@ const MALFORMED_TRACK = 0x12; /** The value for "the thing you asked for is not here", or undefined where none is assigned. */ function doesNotExist(kind: RequestKind, version: IetfVersion): number | undefined { if (version !== Version.DRAFT_14) return DOES_NOT_EXIST; - return kind === "subscribe" || kind === "fetch" ? DOES_NOT_EXIST_14 : undefined; + return kind === "subscribe" || kind === "fetch" || kind === "track_status" ? DOES_NOT_EXIST_14 : undefined; } /** The value for "we do not want this", or undefined where none is assigned. */ @@ -159,13 +184,17 @@ export function toRequestCode(condition: RequestCondition, kind: RequestKind, ve return TIMEOUT; case "not_supported": return NOT_SUPPORTED; + case "invalid_range": + return invalidRange(kind, version) ?? INTERNAL_ERROR; + case "invalid_joining_request_id": + return invalidJoiningRequestId(kind, version) ?? INTERNAL_ERROR; case "does_not_exist": return doesNotExist(kind, version) ?? INTERNAL_ERROR; // A subscriber that asked for content cannot act on "we do not want it": what it needs // to know is that we do not have it, which is the same refusal from its side. Only the // requests that offer content say UNINTERESTED. case "uninterested": - return kind === "subscribe" || kind === "fetch" + return kind === "subscribe" || kind === "fetch" || kind === "track_status" ? (doesNotExist(kind, version) ?? INTERNAL_ERROR) : (uninterested(kind, version) ?? INTERNAL_ERROR); case "malformed_track": @@ -194,6 +223,8 @@ export function fromRequestCode(code: number, kind: RequestKind, version: IetfVe } if (code === doesNotExist(kind, version)) return "does_not_exist"; + if (code === invalidRange(kind, version)) return "invalid_range"; + if (code === invalidJoiningRequestId(kind, version)) return "invalid_joining_request_id"; if (code === uninterested(kind, version)) return "uninterested"; if (code === malformedTrack(kind, version)) return "malformed_track"; if (code === goingAway(version)) return "going_away"; diff --git a/js/net/src/ietf/hidden.ts b/js/net/src/ietf/hidden.ts new file mode 100644 index 0000000000..cd22aff02f --- /dev/null +++ b/js/net/src/ietf/hidden.ts @@ -0,0 +1,31 @@ +import { SetupOption, type SetupOptions } from "./parameters.ts"; + +/** + * The MoQ Hidden extension (draft-lcurley-moq-hidden-00). + * + * A namespace with a field starting with `.` below the prefix a subscription asked for is + * left out of discovery unless the SUBSCRIBE_NAMESPACE opts in with the HIDDEN parameter. + * An unknown parameter fails decoding, so the parameter is only sent to a peer whose SETUP + * carried the HIDDEN option. + * + * @module + * @internal + */ + +/** + * Whether the peer understands the HIDDEN parameter. + * + * @internal + */ +export function hiddenFromSetup(params: SetupOptions): boolean { + return params.getVarint(SetupOption.Hidden) !== undefined; +} + +/** + * Declare that we understand the HIDDEN parameter. + * + * @internal + */ +export function hiddenIntoSetup(params: SetupOptions) { + params.setVarint(SetupOption.Hidden, 1n); +} diff --git a/js/net/src/ietf/ietf.test.ts b/js/net/src/ietf/ietf.test.ts index 949bd5a3cd..ba648c6162 100644 --- a/js/net/src/ietf/ietf.test.ts +++ b/js/net/src/ietf/ietf.test.ts @@ -90,6 +90,77 @@ async function encodeFetchFrameVersioned( return concatChunks(written); } +test("DEFAULT_PUBLISHER_PRIORITY has exact SUBSCRIBE_OK bytes per draft", async () => { + for (const version of [ + Version.DRAFT_14, + Version.DRAFT_15, + Version.DRAFT_16, + Version.DRAFT_17, + Version.DRAFT_18, + Version.DRAFT_19, + Version.DRAFT_20, + Version.DRAFT_21, + Version.DRAFT_22, + ] as const) { + const requestId = version <= Version.DRAFT_16 ? 7n : undefined; + const baseline = await encodeVersioned(new Subscribe.SubscribeOk({ requestId, trackAlias: 42n }), version); + const encoded = await encodeVersioned( + new Subscribe.SubscribeOk({ requestId, trackAlias: 42n, properties: { priority: 37 } }), + version, + ); + if (version <= Version.DRAFT_16) { + expect(Array.from(encoded)).toEqual(Array.from(baseline)); + } else { + expect(Array.from(encoded)).toEqual([baseline[0], baseline[1] + 2, ...baseline.slice(2), 0x0e, 37]); + const decoded = await decodeVersioned(encoded, Subscribe.SubscribeOk.decode, version); + expect(decoded.properties.priority).toBe(37); + } + } +}); + +test("DEFAULT_PUBLISHER_PRIORITY has exact PUBLISH bytes per draft", async () => { + for (const version of [ + Version.DRAFT_14, + Version.DRAFT_15, + Version.DRAFT_16, + Version.DRAFT_17, + Version.DRAFT_18, + Version.DRAFT_19, + Version.DRAFT_20, + Version.DRAFT_21, + Version.DRAFT_22, + ] as const) { + const fields = { + requestId: 1n, + trackNamespace: Path.from("ns"), + trackName: "video", + trackAlias: 42n, + groupOrder: 2, + contentExists: false, + largest: undefined, + forward: true, + }; + const baseline = await encodeVersioned(new Publish(fields), version); + const encoded = await encodeVersioned(new Publish({ ...fields, priority: 37 }), version); + if (version <= Version.DRAFT_16) { + expect(Array.from(encoded)).toEqual(Array.from(baseline)); + } else { + // The new property precedes GROUP_ORDER, so its delta changes 0x22 to 0x14. + expect(Array.from(encoded)).toEqual([ + baseline[0], + baseline[1] + 2, + ...baseline.slice(2, -2), + 0x0e, + 37, + 0x14, + 2, + ]); + const decoded = await decodeVersioned(encoded, Publish.decode, version); + expect(decoded.priority).toBe(37); + } + } +}); + test("Message Parameters: uint8 wire encoding changes in draft 17", async () => { const params = new Parameters(); params.subscriberPriority = 255; diff --git a/js/net/src/ietf/index.ts b/js/net/src/ietf/index.ts index 9fd3ec8474..e1ebb8199d 100644 --- a/js/net/src/ietf/index.ts +++ b/js/net/src/ietf/index.ts @@ -4,6 +4,7 @@ export * from "./connection.ts"; export * from "./control.ts"; export * from "./fetch.ts"; export * from "./goaway.ts"; +export * from "./hidden.ts"; export * from "./object.ts"; export * from "./parameters.ts"; export * from "./publish.ts"; diff --git a/js/net/src/ietf/parameters.ts b/js/net/src/ietf/parameters.ts index 5d9762b6da..f2a6a3120b 100644 --- a/js/net/src/ietf/parameters.ts +++ b/js/net/src/ietf/parameters.ts @@ -16,6 +16,8 @@ export const SetupOption = { RelayCost: 0x40b56n, /** SOLICIT, from the MoQ Solicit extension. See `solicit.ts`. */ Solicit: 0x40b5an, + /** HIDDEN, from the MoQ Hidden extension. See `hidden.ts`. */ + Hidden: 0x40b5cn, } as const; /// Setup Options — used in SETUP messages. @@ -202,6 +204,8 @@ const MSG_PARAM_SUBSCRIBER_PRIORITY = 0x20n; const MSG_PARAM_GROUP_ORDER = 0x22n; /// ROUTE_COST, from the MoQ Cluster extension. See `cluster.ts`. const MSG_PARAM_ROUTE_COST = 0x40b58n; +/// HIDDEN, from the MoQ Hidden extension. See `hidden.ts`. +const MSG_PARAM_HIDDEN = 0x40b5en; // Bytes parameter IDs (odd) const MSG_PARAM_LARGEST_OBJECT = 0x09n; @@ -225,6 +229,7 @@ function getMessageParamKind(id: bigint): MessageParamKind { case MSG_PARAM_MAX_CACHE_DURATION: case MSG_PARAM_EXPIRES: case MSG_PARAM_ROUTE_COST: + case MSG_PARAM_HIDDEN: return "varint"; case MSG_PARAM_PUBLISHER_PRIORITY: case MSG_PARAM_SUBSCRIBER_PRIORITY: @@ -341,6 +346,19 @@ export class Parameters { this.vars.set(MSG_PARAM_MAX_CACHE_DURATION, v); } + /** HIDDEN (MoQ Hidden): also advertise hidden namespaces. Absent and 0 both mean no. */ + get hidden(): boolean { + const v = this.vars.get(MSG_PARAM_HIDDEN); + if (v === undefined || v === 0n) return false; + if (v === 1n) return true; + throw new Error(`invalid HIDDEN parameter: ${v}`); + } + + set hidden(v: boolean) { + if (v) this.vars.set(MSG_PARAM_HIDDEN, 1n); + else this.vars.delete(MSG_PARAM_HIDDEN); + } + // --- Bytes accessors --- get largest(): MessageLocation | undefined { diff --git a/js/net/src/ietf/properties.ts b/js/net/src/ietf/properties.ts index 2a4c2b06ab..1941361ced 100644 --- a/js/net/src/ietf/properties.ts +++ b/js/net/src/ietf/properties.ts @@ -11,6 +11,7 @@ const TIMESCALE = 0x08n; // It shares its number with the GROUP_ORDER *message parameter*, which is a different // registry: that one is only legal in SUBSCRIBE, PUBLISH_OK, and FETCH, where the // subscriber states its own preference. +const DEFAULT_PUBLISHER_PRIORITY = 0x0en; const DEFAULT_PUBLISHER_GROUP_ORDER = 0x22n; /** @@ -26,6 +27,9 @@ export interface Properties { /// `undefined` declares no timeline, so the subscriber times objects by arrival. timescale?: Timescale; + /// Publisher priority for a group header without its priority flag. The wire default is 128. + priority?: number; + /// The publisher's preference for prioritizing groups within a subscription, /// Ascending (0x1) or Descending (0x2). /// @@ -54,6 +58,15 @@ export async function encode(w: Writer, properties: Properties, version: IetfVer prevType = TIMESCALE; } + if (properties.priority !== undefined) { + if (!Number.isInteger(properties.priority) || properties.priority < 0 || properties.priority > 255) { + throw new RangeError(`invalid publisher priority: ${properties.priority}`); + } + await w.u62(DEFAULT_PUBLISHER_PRIORITY - prevType); + await w.u62(BigInt(properties.priority)); + prevType = DEFAULT_PUBLISHER_PRIORITY; + } + if (properties.groupOrder !== undefined) { await w.u62(DEFAULT_PUBLISHER_GROUP_ORDER - prevType); await w.u62(BigInt(properties.groupOrder)); @@ -94,6 +107,9 @@ export async function decode(r: Reader, version: IetfVersion): Promise 255n) throw new Error(`invalid publisher priority: ${value}`); + properties.priority = Number(value); } else if (abs === DEFAULT_PUBLISHER_GROUP_ORDER) { // Only Ascending (0x1) and Descending (0x2) are defined here. Unlike the draft-14 // fields, 0x0 has no "publisher decides" meaning to fall back on. diff --git a/js/net/src/ietf/publish.ts b/js/net/src/ietf/publish.ts index 1b7b7d520e..200d19cf78 100644 --- a/js/net/src/ietf/publish.ts +++ b/js/net/src/ietf/publish.ts @@ -16,6 +16,7 @@ export class Publish { trackName: string; trackAlias: bigint; groupOrder: number; + priority?: number; contentExists: boolean; largest: { groupId: bigint; objectId: bigint } | undefined; forward: boolean; @@ -26,6 +27,7 @@ export class Publish { trackName, trackAlias, groupOrder, + priority, contentExists, largest, forward, @@ -35,6 +37,7 @@ export class Publish { trackName: string; trackAlias: bigint; groupOrder: number; + priority?: number; contentExists: boolean; largest: { groupId: bigint; objectId: bigint } | undefined; forward: boolean; @@ -44,6 +47,7 @@ export class Publish { this.trackName = trackName; this.trackAlias = trackAlias; this.groupOrder = groupOrder; + this.priority = priority; this.contentExists = contentExists; this.largest = largest; this.forward = forward; @@ -90,7 +94,7 @@ export class Publish { await params.encode(w, version); // Track Properties are the final field, so nothing may follow. - await Properties.encode(w, { groupOrder: this.groupOrder }, version); + await Properties.encode(w, { groupOrder: this.groupOrder, priority: this.priority }, version); } } @@ -142,6 +146,7 @@ export class Publish { trackName, trackAlias, groupOrder, + priority: properties.priority, contentExists: !!largest, largest, forward, diff --git a/js/net/src/ietf/publisher.test.ts b/js/net/src/ietf/publisher.test.ts index ad5cedf578..0d0908fb2f 100644 --- a/js/net/src/ietf/publisher.test.ts +++ b/js/net/src/ietf/publisher.test.ts @@ -21,6 +21,7 @@ import { Publisher } from "./publisher.ts"; import { RequestError, RequestOk } from "./request.ts"; import { Subscribe, SubscribeOk } from "./subscribe.ts"; import { SubscribeNamespace } from "./subscribe_namespace.ts"; +import { TrackStatusRequest } from "./track.ts"; import { ALPN, type IetfVersion, Version } from "./version.ts"; function publish(origin: OriginProducer, path: Path.Valid) { @@ -129,6 +130,50 @@ function publisher( }; } +test("TRACK_STATUS gets exact NOT_SUPPORTED refusal bytes on every draft", async () => { + const phrase = new TextEncoder().encode("TRACK_STATUS is not supported"); + for (const version of [ + Version.DRAFT_14, + Version.DRAFT_15, + Version.DRAFT_16, + Version.DRAFT_17, + Version.DRAFT_18, + Version.DRAFT_19, + Version.DRAFT_20, + Version.DRAFT_21, + Version.DRAFT_22, + ] as const) { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, version, true); + const { pub, origin } = publisher(pair.server, { session }); + const written: Uint8Array[] = []; + const stream = new Stream({ + readable: new ReadableStream(), + writable: new WritableStream({ + write: (chunk) => { + written.push(new Uint8Array(chunk)); + }, + }), + version, + }); + await pub.runTrackStatusRequest( + new TrackStatusRequest({ requestId: 7n, trackNamespace: Path.from("test"), trackName: "video" }), + stream, + ); + await stream.writer.closed; + const body = [ + ...(version <= Version.DRAFT_16 ? [7] : []), + 3, + ...(version >= Version.DRAFT_16 ? [0] : []), + phrase.length, + ...phrase, + ]; + const expected = [version === Version.DRAFT_14 ? 0x0f : 0x05, 0, body.length, ...body]; + expect(written.flatMap((chunk) => Array.from(chunk))).toEqual(expected); + origin.close(); + } +}); + // The header is part of the group's lifetime too. If it blocks on flow control, advancing // the live edge must reset the stream without waiting for that write to finish. test("a blocked group header is reset when the group expires", async () => { diff --git a/js/net/src/ietf/publisher.ts b/js/net/src/ietf/publisher.ts index fc358b342a..0074eef517 100644 --- a/js/net/src/ietf/publisher.ts +++ b/js/net/src/ietf/publisher.ts @@ -1,9 +1,9 @@ -import { type Dispose, type Getter, Signal } from "@moq/signals"; +import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import type * as broadcast from "../broadcast.ts"; import { controlTimeout, error, reason, StreamCode, StreamError } from "../error.ts"; import type * as group from "../group.ts"; import { type Route, routesEqual } from "../hop.ts"; -import { hooks } from "../internal.ts"; +import { hiddenBelow, hooks } from "../internal.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import * as Path from "../path.ts"; import { type Stream, Writer } from "../stream.ts"; @@ -30,7 +30,7 @@ import { SubscribeNamespaceEntryDone, SubscribeNamespaceOk, } from "./subscribe_namespace.ts"; -import { TrackStatus, type TrackStatusRequest } from "./track.ts"; +import type { TrackStatusRequest } from "./track.ts"; import { type IetfVersion, Version } from "./version.ts"; /** First wait before re-offering a namespace the peer refused or we couldn't open for. */ @@ -325,7 +325,7 @@ export class Publisher { ? // Declaring the timescale is what opts the track into timestamps; every // object Timestamp below is in these units. We serve the newest group // first, matching moq-lite. - { timescale, groupOrder: Properties.DESCENDING } + { timescale, priority: publisherPriority, groupOrder: Properties.DESCENDING } : // INCLUDE_PROPERTIES=0. The block stays present but empty, which also means // the track opts out of timestamps for this subscriber. {}, @@ -394,7 +394,7 @@ export class Publisher { let publishError: Error | undefined; try { - await Promise.race([Promise.all([serving, filling]), stream.reader.closed]); + await race([Promise.all([serving, filling]), stream.reader.closed]); } catch (err: unknown) { publishError = error(err); } @@ -495,7 +495,7 @@ export class Publisher { // Reading from the filter's start drops the objects below it: they are outside // the requested range, so skipping them is not a gap. - const read = await Promise.race([hooks.readGroupFrame(group, slice.skip), stream.closed]); + const read = await race([hooks.readGroupFrame(group, slice.skip), stream.closed]); if (!read) break; next = read.sequence + 1; if (slice.until !== undefined && read.sequence >= slice.until) { @@ -608,11 +608,7 @@ export class Publisher { if (fill.until !== undefined && next >= fill.until) break; // Reading from the fill's start drops everything below it; see the same read in #runGroup. - const frame = await Promise.race([ - group.readFrameSequence({ from: Number(fill.skip) }), - stream.closed, - cancelled, - ]); + const frame = await race([group.readFrameSequence({ from: Number(fill.skip) }), stream.closed, cancelled]); if (left) throw new Error("unsubscribed before the fill finished"); if (!frame) break; next = BigInt(frame.sequence) + 1n; @@ -632,10 +628,10 @@ export class Publisher { /** * Handles an incoming SUBSCRIBE_NAMESPACE on a bidi stream. * - * This carries the advertisements only when the peer asked to be told on request - * (MoQ Solicit); otherwise {@link runPublishNamespaces} has already announced - * everything and repeating it here would leave the peer holding two sources for one - * broadcast. Draft-16+ streams Namespace entries inline; draft-14/15 predate those + * This carries the advertisements when the peer asked to be told on request (MoQ + * Solicit); otherwise {@link runPublishNamespaces} has already announced everything + * visible and repeating it here would leave the peer holding two sources for one + * broadcast, so only the hidden namespaces it may see ride here (MoQ Hidden). Draft-16+ streams Namespace entries inline; draft-14/15 predate those * messages, so each advertisement is a PUBLISH_NAMESPACE request of its own. * * @internal @@ -662,12 +658,12 @@ export class Publisher { await ok.encode(stream.writer, version); } - if (!this.#requiresSolicitation) { - // Already announced, unasked. Hold the stream open until the peer is done. - await stream.reader.closed; - stream.close(); - return; - } + // Hidden namespaces are left out unless the peer opted in (MoQ Hidden). Unless the + // peer asked to be told only on request, it has already heard everything visible + // from the empty prefix unasked, so this stream carries only what that hid. + const carries = (covered: Path.Valid) => + (msg.hidden || !hiddenBelow(prefix, covered)) && + (this.#requiresSolicitation || hiddenBelow(Path.empty(), covered)); // Reports whether the peer now holds the namespace: an inline entry always // lands, but a PUBLISH_NAMESPACE request can be declined. @@ -718,7 +714,7 @@ export class Publisher { const updated = new Map(); for (const [covered, snap] of advertised) { const suffix = Path.stripPrefix(prefix, covered); - if (suffix === null) continue; + if (suffix === null || !carries(covered)) continue; updated.set(suffix, snap); } @@ -767,8 +763,8 @@ export class Publisher { // Wait for the next change, or for the peer to unsubscribe. const next = await (retry - ? Promise.race([changed, stream.reader.closed, retryAfter(retry).then(() => advertised)]) - : Promise.race([changed, stream.reader.closed])); + ? race([changed, stream.reader.closed, retryAfter(retry).then(() => advertised)]) + : race([changed, stream.reader.closed])); dispose(); if (!next) break; } @@ -847,6 +843,8 @@ export class Publisher { const updated = new Map(); for (const [covered, snap] of advertised) { + // Unasked, a hidden namespace stays off the wire (MoQ Hidden). + if (hiddenBelow(Path.empty(), covered)) continue; updated.set(covered, snap); } @@ -893,8 +891,8 @@ export class Publisher { // Wait for the next change, which has already fired if one landed above. const next = await (retry - ? Promise.race([changed, closed, retryAfter(retry).then(() => advertised)]) - : Promise.race([changed, closed])); + ? race([changed, closed, retryAfter(retry).then(() => advertised)]) + : race([changed, closed])); dispose?.(); if (!next) break; } @@ -1061,25 +1059,22 @@ export class Publisher { */ async runTrackStatusRequest(msg: TrackStatusRequest, stream: Stream) { const version = this.#session.version; - + const errorCode = toRequestCode("not_supported", "track_status", version); if (version === Version.DRAFT_14) { - // v14: respond with TrackStatus (0x0E = TRACK_STATUS_OK) - await stream.writer.u53(TrackStatus.id); - const status = new TrackStatus({ - trackNamespace: msg.trackNamespace, - trackName: msg.trackName, - statusCode: TrackStatus.STATUS_NOT_FOUND, - lastGroupId: 0n, - lastObjectId: 0n, - }); - await status.encode(stream.writer, version); + // TRACK_STATUS_ERROR shares the SUBSCRIBE_ERROR body on draft-14. + await stream.writer.u53(0x0f); + await new SubscribeError({ + requestId: msg.requestId, + errorCode, + reasonPhrase: "TRACK_STATUS is not supported", + }).encode(stream.writer, version); } else { - // v15+: respond with RequestOk (0x07) - await stream.writer.u53(RequestOk.id); - const ok = new RequestOk({ + await stream.writer.u53(RequestError.id); + await new RequestError({ requestId: version === Version.DRAFT_15 || version === Version.DRAFT_16 ? msg.requestId : undefined, - }); - await ok.encode(stream.writer, version); + errorCode, + reasonPhrase: "TRACK_STATUS is not supported", + }).encode(stream.writer, version); } stream.close(); } diff --git a/js/net/src/ietf/subscribe_namespace.ts b/js/net/src/ietf/subscribe_namespace.ts index 6d7944726f..25f5054a71 100644 --- a/js/net/src/ietf/subscribe_namespace.ts +++ b/js/net/src/ietf/subscribe_namespace.ts @@ -30,10 +30,17 @@ export class SubscribeNamespace { namespace: Path.Valid; requestId: bigint; + /** MoQ Hidden: also advertise hidden namespaces. Only sent to a peer that declared it. */ + hidden: boolean; - constructor({ namespace, requestId }: { namespace: Path.Valid; requestId: bigint }) { + constructor({ + namespace, + requestId, + hidden = false, + }: { namespace: Path.Valid; requestId: bigint; hidden?: boolean }) { this.namespace = namespace; this.requestId = requestId; + this.hidden = hidden; } async #encode(w: Writer, version: IetfVersion): Promise { @@ -42,7 +49,9 @@ export class SubscribeNamespace { } await w.u62(this.requestId); await Namespace.encode(w, this.namespace); - await new Parameters().encode(w, version); + const params = new Parameters(); + params.hidden = this.hidden; + await params.encode(w, version); } async encode(w: Writer, version: IetfVersion): Promise { @@ -59,9 +68,9 @@ export class SubscribeNamespace { } const requestId = await r.u62(); const namespace = await Namespace.decode(r); - await Parameters.decode(r, version); + const params = await Parameters.decode(r, version); - return new SubscribeNamespace({ namespace, requestId }); + return new SubscribeNamespace({ namespace, requestId, hidden: params.hidden }); } } @@ -77,19 +86,24 @@ export class SubscribeNamespaceLegacy { namespace: Path.Valid; requestId: bigint; subscribeOptions: number; // v16/v17: default 0x01 (NAMESPACE only) + /** MoQ Hidden: see {@link SubscribeNamespace.hidden}. */ + hidden: boolean; constructor({ namespace, requestId, subscribeOptions = 1, + hidden = false, }: { namespace: Path.Valid; requestId: bigint; subscribeOptions?: number; + hidden?: boolean; }) { this.namespace = namespace; this.requestId = requestId; this.subscribeOptions = subscribeOptions; + this.hidden = hidden; } async #encode(w: Writer, version: IetfVersion): Promise { @@ -104,7 +118,9 @@ export class SubscribeNamespaceLegacy { if (version === Version.DRAFT_16 || version === Version.DRAFT_17) { await w.u53(this.subscribeOptions); } - await new Parameters().encode(w, version); + const params = new Parameters(); + params.hidden = this.hidden; + await params.encode(w, version); } async encode(w: Writer, version: IetfVersion): Promise { @@ -128,9 +144,9 @@ export class SubscribeNamespaceLegacy { if (version === Version.DRAFT_16 || version === Version.DRAFT_17) { subscribeOptions = await r.u53(); } - await Parameters.decode(r, version); + const params = await Parameters.decode(r, version); - return new SubscribeNamespaceLegacy({ namespace, requestId, subscribeOptions }); + return new SubscribeNamespaceLegacy({ namespace, requestId, subscribeOptions, hidden: params.hidden }); } } diff --git a/js/net/src/ietf/subscriber.test.ts b/js/net/src/ietf/subscriber.test.ts index 6e40f92103..a0f560d42a 100644 --- a/js/net/src/ietf/subscriber.test.ts +++ b/js/net/src/ietf/subscriber.test.ts @@ -840,6 +840,82 @@ async function subscribeTrack(): Promise<{ subscriber: Subscriber; track: track. return { subscriber, track }; } +test("older peer without priority property inherits wire priority 128", async () => { + const { subscriber, track } = await subscribeTrack(); + expect((await track.info()).priority).toBe(0xff - 128); + const group = new GroupMessage({ + trackAlias: ALIAS, + groupId: 3, + subGroupId: 0, + publisherPriority: 0, + flags: { ...groupFlags(true), hasPriority: false }, + }); + await subscriber.handleGroup(group, new Reader(undefined, encodeObjects([0]), VERSION)); + expect(group.publisherPriority).toBe(128); + track.close(); +}); + +test("an info-only lookup waits for SUBSCRIBE_OK instead of abandoning", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session }); + const info = subscriber.consume(Path.from("room")).track("video").info(); + const peer = await nextStream(pair.client); + if (!peer) throw new Error("missing SUBSCRIBE stream"); + expect(await peer.reader.u53()).toBe(Subscribe.id); + const request = await Subscribe.decode(peer.reader, VERSION); + + await peer.writer.u53(SubscribeOk.id); + await new SubscribeOk({ + requestId: request.requestId, + trackAlias: ALIAS, + properties: { priority: 37 }, + }).encode(peer.writer, VERSION); + expect((await info).priority).toBe(0xff - 37); +}); + +test("early group waits for SUBSCRIBE_OK priority before track acceptance", async () => { + const pair = createMockTransportPair(ALPN.DRAFT_19); + const session = new NativeSession(pair.server, VERSION, true); + const subscriber = new Subscriber({ session }); + const track = subscriber.consume(Path.from("room")).track("video").subscribe(); + const peer = await nextStream(pair.client); + if (!peer) throw new Error("missing SUBSCRIBE stream"); + expect(await peer.reader.u53()).toBe(Subscribe.id); + const request = await Subscribe.decode(peer.reader, VERSION); + + const flags = { ...groupFlags(true), hasPriority: false }; + const group = new GroupMessage({ trackAlias: ALIAS, groupId: 3, subGroupId: 0, publisherPriority: 0, flags }); + const arriving = subscriber.handleGroup(group, new Reader(undefined, encodeObjects([0]), VERSION)); + const pending = await Promise.race([arriving.then(() => false), Promise.resolve(true)]); + expect(pending).toBe(true); + + await peer.writer.u53(SubscribeOk.id); + await new SubscribeOk({ + requestId: request.requestId, + trackAlias: ALIAS, + properties: { priority: 37 }, + }).encode(peer.writer, VERSION); + await arriving; + expect((await track.info()).priority).toBe(0xff - 37); + expect(group.publisherPriority).toBe(37); + const ordered = track.ordered(); + expect((await ordered.nextGroup())?.sequence).toBe(3); + const explicit = new GroupMessage({ + trackAlias: ALIAS, + groupId: 4, + subGroupId: 0, + publisherPriority: 9, + flags: groupFlags(true), + }); + await subscriber.handleGroup(explicit, new Reader(undefined, encodeObjects([0]), VERSION)); + expect(explicit.publisherPriority).toBe(9); + expect((await track.info()).priority).toBe(0xff - 37); + expect((await ordered.nextGroup())?.sequence).toBe(4); + ordered.close(); + track.close(); +}); + /** * A group is the unit an application resyncs on, so one served from partway through is * unusable: the objects on the stream do not decode without the head the filter excluded, diff --git a/js/net/src/ietf/subscriber.ts b/js/net/src/ietf/subscriber.ts index 24dc891ec6..33e8595b92 100644 --- a/js/net/src/ietf/subscriber.ts +++ b/js/net/src/ietf/subscriber.ts @@ -1,10 +1,11 @@ +import { race, Signal } from "@moq/signals"; import * as announce from "../announced.ts"; import * as broadcast from "../broadcast.ts"; import { BroadcastCache } from "../consume.ts"; import { controlTimeout, error, ProtocolViolation, reason } from "../error.ts"; import * as netGroup from "../group.ts"; import { Cost, type Route, routesEqual, UNKNOWN_HOP } from "../hop.ts"; -import { scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; +import { hiddenBelow, hooks, scopeCaptures, scopeHead, scopeOverlaps } from "../internal.ts"; import * as Path from "../path.ts"; import type { Reader, Stream } from "../stream.ts"; import { type Timescale, Timestamp } from "../time.ts"; @@ -16,7 +17,7 @@ import { DuplicateTrackAlias, RetiredTrackAlias, TrackAliases } from "./aliases. import * as Cluster from "./cluster.ts"; import { requestReason, toRequestCode } from "./error.ts"; import { Frame, type Group as GroupMessage } from "./object.ts"; -import { toWire } from "./priority.ts"; +import { fromWire, toWire } from "./priority.ts"; import { type Publish, PublishError } from "./publish.ts"; import { type PublishNamespace, @@ -69,6 +70,14 @@ type SubscribeSetupState = { rejected?: boolean; }; +/** A local announce reader's filter: its scope, the prefix it asked for, and its hidden opt-in. */ +type Filter = { scope: Path.Pattern; prefix: Path.Valid; hidden: boolean }; + +/** Whether a reader with `filter` sees an announcement at `path`. */ +function sees(filter: Filter, path: Path.Valid): boolean { + return scopeOverlaps(filter.scope, path) && (filter.hidden || !hiddenBelow(filter.prefix, path)); +} + /** * Handles subscribing to broadcasts using moq-transport protocol. * Uses the stream-per-request pattern (real bidi streams for v17, virtual for v14-v16). @@ -109,7 +118,10 @@ export class Subscriber { #announced = new Map(); // Any consumers that want each new announcement, keyed by their local filter. - #announcedConsumers = new Map(); + #announcedConsumers = new Map(); + + // Whether the peer understands the HIDDEN parameter (MoQ Hidden). + #hidden: boolean; /** * Creates a new Subscriber instance. @@ -119,14 +131,18 @@ export class Subscriber { constructor({ session, cluster, + hidden = false, }: { /** The session abstraction for bidi streams and request IDs. */ session: Session; /** The Hop IDs the SETUP exchange settled (MoQ Cluster). */ cluster?: Cluster.Hops; + /** Whether the peer understands the HIDDEN parameter (MoQ Hidden). */ + hidden?: boolean; }) { this.#session = session; this.#cluster = cluster; + this.#hidden = hidden; } /** @@ -154,13 +170,19 @@ export class Subscriber { * The peer is asked with SUBSCRIBE_NAMESPACE regardless of what it declared, and an * unsolicited PUBLISH_NAMESPACE lands here too, so a peer that only tells and one * that only answers are both discovered. + * + * Hidden routes (a `.`-prefixed segment below the scope's head) are left out unless + * `options.hidden` opts in. The opt-in rides the SUBSCRIBE_NAMESPACE when the peer + * understands it (MoQ Hidden); the rule is also applied here, since an unsolicited + * PUBLISH_NAMESPACE or a peer that never heard of it hides nothing. */ - announced(scope: Path.Pattern = Path.Pattern.all()): announce.Consumer { + announced(scope: Path.Pattern = Path.Pattern.all(), options?: announce.Options): announce.Consumer { // The wire speaks announce interest by prefix. const prefix = scopeHead(scope); + const filter = { scope, prefix, hidden: options?.hidden ?? false }; const announced = new announce.Producer(); for (const [active, info] of this.#announced) { - if (!scopeOverlaps(scope, active)) continue; + if (!sees(filter, active)) continue; announced.append({ prefix: active, captures: scopeCaptures(scope, active), @@ -168,9 +190,9 @@ export class Subscriber { route: info.route, }); } - this.#announcedConsumers.set(announced, scope); + this.#announcedConsumers.set(announced, filter); - void this.#runAnnounced(announced, prefix).finally(() => { + void this.#runAnnounced(announced, prefix, filter.hidden && this.#hidden).finally(() => { this.#announcedConsumers.delete(announced); announced.close(); }); @@ -191,8 +213,9 @@ export class Subscriber { this.#announced.set(path, { count: 1, route }); console.debug(`announced: broadcast=${path} active=true`); - for (const [consumer, scope] of this.#announcedConsumers) { - if (!scopeOverlaps(scope, path)) continue; + for (const [consumer, filter] of this.#announcedConsumers) { + if (!sees(filter, path)) continue; + const scope = filter.scope; consumer.append({ prefix: path, captures: scopeCaptures(scope, path), kind: "announced", route }); } } @@ -207,8 +230,9 @@ export class Subscriber { if (existing === undefined || routesEqual(existing.route, route)) return; existing.route = route; console.debug(`announced: broadcast=${path} rerouted`); - for (const [consumer, scope] of this.#announcedConsumers) { - if (!scopeOverlaps(scope, path)) continue; + for (const [consumer, filter] of this.#announcedConsumers) { + if (!sees(filter, path)) continue; + const scope = filter.scope; consumer.append({ prefix: path, captures: scopeCaptures(scope, path), kind: "updated", route }); } } @@ -231,8 +255,9 @@ export class Subscriber { this.#consumes.evict(path); console.debug(`announced: broadcast=${path} active=false`); - for (const [consumer, scope] of this.#announcedConsumers) { - if (!scopeOverlaps(scope, path)) continue; + for (const [consumer, filter] of this.#announcedConsumers) { + if (!sees(filter, path)) continue; + const scope = filter.scope; try { consumer.append({ prefix: path, @@ -246,7 +271,7 @@ export class Subscriber { } } - async #runAnnounced(announced: announce.Producer, prefix: Path.Valid) { + async #runAnnounced(announced: announce.Producer, prefix: Path.Valid, hidden: boolean) { const version = this.#session.version; // Suffixes live on this stream, so a repeat is recognized as an update to the @@ -283,10 +308,16 @@ export class Subscriber { version === Version.DRAFT_17 ) { await stream.writer.u53(SubscribeNamespaceLegacy.id); - await new SubscribeNamespaceLegacy({ namespace: prefix, requestId }).encode(stream.writer, version); + await new SubscribeNamespaceLegacy({ namespace: prefix, requestId, hidden }).encode( + stream.writer, + version, + ); } else { await stream.writer.u53(SubscribeNamespace.id); - await new SubscribeNamespace({ namespace: prefix, requestId }).encode(stream.writer, version); + await new SubscribeNamespace({ namespace: prefix, requestId, hidden }).encode( + stream.writer, + version, + ); } console.debug(`subscribe_namespace written: requestId=${requestId}`); @@ -365,7 +396,7 @@ export class Subscriber { }); // Wait for either the read loop or the announced to close - await Promise.race([readLoop, announced.closed]); + await race([readLoop, announced.closed]); // For v14/v15: send UnsubscribeNamespace before closing if (version === Version.DRAFT_14 || version === Version.DRAFT_15) { @@ -447,12 +478,9 @@ export class Subscriber { console.debug(`subscribe start: id=${requestId} broadcast=${broadcast} track=${request.name}`); - // IETF negotiates group order in SUBSCRIBE_OK; this implementation only supports - // descending (newest-first), which is what moq-lite fixes group order to, so the - // mapping needs nothing here. (There's no per-frame timescale either, so every - // property stays at its default.) This resolves the consumer's track.info() and - // gives us the write side that incoming object streams are routed into. - const producer = request.accept({}); + // Keep the request pending until SUBSCRIBE_OK supplies immutable track metadata. + // Group streams already wait on the alias, so early data stays behind this response. + const producer = hooks.pendingTrackProducer(request); // Open the stream and wait for SUBSCRIBE_OK under a timeout. State // flows back via `state` so the timeout path can clean up the stream @@ -464,6 +492,11 @@ export class Subscriber { // would miss the local side going away and leave it serving a track nobody reads. // Demand returning before we commit is not abandonment, matching the serving loop. const waitAbandoned = async (): Promise => { + // An info-only lookup attaches no subscriber yet still waits on SUBSCRIBE_OK for + // the track info, so only demand that arrived and then left is abandonment. + while (!producer.used.peek() && producer.closed.peek() === undefined) { + await Signal.race(producer.used, producer.closed); + } for (;;) { await producer.unused(); if (producer.closed.peek() !== undefined || !producer.used.peek()) return null; @@ -473,7 +506,7 @@ export class Subscriber { let stream: Stream; let trackAlias: bigint; try { - const result = await Promise.race([ + const result = await race([ withTimeout( setup, SUBSCRIBE_OK_TIMEOUT_MS, @@ -490,7 +523,7 @@ export class Subscriber { } catch (err) { // A control request that timed out is not late content, so it carries its own code. const e = err instanceof TimeoutError ? controlTimeout(err) : error(err); - producer.close(e); + request.reject(e); console.warn( `subscribe error: id=${requestId} broadcast=${broadcast} track=${request.name} error=${reason(e)}`, ); @@ -544,7 +577,7 @@ export class Subscriber { // Terminal conditions settle at most once (stream close = PublishDone, track close = // local unsubscribe); race them once so the demand loop doesn't re-subscribe each pass. - const done = Promise.race([ + const done = race([ stream.reader.closed.then(() => publisherEnded), producer.closed.then(() => localEnded), ]); @@ -554,7 +587,7 @@ export class Subscriber { // down resumes on the same stream. let terminal = localEnded; for (;;) { - const reason = await Promise.race([done, producer.unused().then(() => idle)]); + const reason = await race([done, producer.unused().then(() => idle)]); if (reason === idle && producer.closed.peek() === undefined && producer.used.peek()) continue; terminal = reason; break; @@ -669,6 +702,8 @@ export class Subscriber { } const ok = await SubscribeOk.decode(state.stream.reader, version); + if (state.cancelled) throw new Error("subscribe cancelled before acceptance"); + request.accept({ priority: fromWire(ok.properties.priority ?? 128) }); try { this.#aliases.set(ok.trackAlias, producer, { broadcast, name: request.name }); @@ -921,11 +956,14 @@ export class Subscriber { try { // The control message establishing this alias can arrive after the data stream. const track = await this.#aliases.get(group.trackAlias); + // The alias binds after SUBSCRIBE_OK commits the track property; an omitted + // header priority inherits it (draft-21 section 10.4). + if (!group.flags.hasPriority) group.publisherPriority = toWire((await track.info()).priority); track.writeGroup(producer); for (;;) { - const done = await Promise.race([stream.done(), producer.closed, track.closed]); + const done = await race([stream.done(), producer.closed, track.closed]); if (done !== false) break; const frame = await Frame.decode( diff --git a/js/net/src/integration.test.ts b/js/net/src/integration.test.ts index f6ca858b41..2a98a9a8bc 100644 --- a/js/net/src/integration.test.ts +++ b/js/net/src/integration.test.ts @@ -407,6 +407,61 @@ test("integration: lite draft-06 announce lifecycle", async () => { server.close(); }); +/** Collect announced prefixes until `until` arrives. */ +async function announcedUntil(announced: { next(): Promise<{ prefix: Path.Valid } | undefined> }, until: string) { + const seen: string[] = []; + while (!seen.includes(until)) { + const entry = await withTimeout(announced.next(), 1000, `waiting for ${until}`); + if (!entry) throw new Error("announcements ended"); + seen.push(entry.prefix); + } + return seen; +} + +// A `.`-named broadcast is left out of discovery unless the request opts in or names the +// dot segment. lite-06 cannot carry the opt-in, so its peer never lists the hidden path. +for (const [protocol, carriesOptIn] of [ + [Lite.ALPN_07, true], + [Lite.ALPN_06, false], + [Ietf.ALPN.DRAFT_19, true], + [Ietf.ALPN.DRAFT_16, true], +] as const) { + test(`integration: ${protocol} hides dot paths from discovery`, async () => { + const pair = createMockTransportPair(protocol); + const origin = new OriginProducer(); + const [client, server] = await Promise.all([ + connect(url, { transport: pair.client }), + accept(pair.server, url, { publish: origin.consume() }), + ]); + + // Published first, so a reader that may see it lists it before `visible`. + const hidden = publish(origin, Path.from(".x/y")); + const visible = publish(origin, Path.from("visible")); + + const plain = client.announced(); + expect(await announcedUntil(plain, "visible")).toEqual(["visible"]); + + // An IETF reader shares the session's table, so `visible` may land before the + // opted-in request's own answer; wait on the hidden path itself where it is due. + const opted = client.announced(undefined, { hidden: true }); + if (carriesOptIn) await announcedUntil(opted, ".x/y"); + else expect(await announcedUntil(opted, "visible")).toEqual(["visible"]); + + if (carriesOptIn) { + const named = client.announced(Path.Pattern.parse(".x/**")); + expect(await announcedUntil(named, ".x/y")).toEqual([".x/y"]); + named.close(); + } + + plain.close(); + opted.close(); + hidden.close(); + visible.close(); + client.close(); + server.close(); + }); +} + test("integration: lite draft-05 datagram delivery", async () => { const enc = new TextEncoder(); const dec = new TextDecoder(); diff --git a/js/net/src/internal.ts b/js/net/src/internal.ts index 77316e00c8..3f235938cd 100644 --- a/js/net/src/internal.ts +++ b/js/net/src/internal.ts @@ -37,6 +37,15 @@ export function scopeHead(scope: Path.Pattern): Path.Valid { return Path.from(scope.head); } +/** + * Whether a segment of `path` below `prefix` starts with `.`, which hides it from announce + * discovery unless the request opts in. A path at or above the prefix never hides. + */ +export function hiddenBelow(prefix: Path.Valid, path: Path.Valid): boolean { + const below = Path.stripPrefix(prefix, path); + return below !== null && Path.parts(below).some((part) => part.startsWith(".")); +} + /** Whether the announced prefix's subtree overlaps `scope`. */ export function scopeOverlaps(scope: Path.Pattern, prefix: Path.Valid): boolean { return scope.overlaps(Path.Pattern.subtree(prefix)); @@ -97,6 +106,8 @@ export interface TrackRequestOptions { export const hooks: { /** Mint a track {@link Request}; assigned by `track.ts`. */ makeRequest: (options: TrackRequestOptions) => Request; + /** Access the existing producer while a request awaits immutable wire metadata. */ + pendingTrackProducer: (request: Request) => Producer; /** * Take the next group the subscriber's cursor allows, without waiting; assigned by `track.ts`. * @@ -141,6 +152,9 @@ export const hooks: { makeRequest: () => { throw new Error("track.ts not loaded"); }, + pendingTrackProducer: () => { + throw new Error("track.ts not loaded"); + }, tryRecvGroup: () => { throw new Error("track.ts not loaded"); }, diff --git a/js/net/src/lite/announce.test.ts b/js/net/src/lite/announce.test.ts index c0efde17c5..4a6acc7dfc 100644 --- a/js/net/src/lite/announce.test.ts +++ b/js/net/src/lite/announce.test.ts @@ -142,6 +142,15 @@ test("AnnounceRequest drops excludeHop on draft-06", async () => { expect(with06.byteLength).toBeLessThan(with05.byteLength); }); +// Draft07 carries the hidden opt-in; every earlier version decodes as not opted in. +test("AnnounceRequest carries hidden from draft-07", async () => { + for (const hidden of [false, true]) { + const msg = new AnnounceRequest(Path.from("room/"), 0n, hidden); + expect((await requestRoundTrip(msg, Version.DRAFT_07)).hidden).toBe(hidden); + expect((await requestRoundTrip(msg, Version.DRAFT_06)).hidden).toBe(false); + } +}); + // The draft reserves Hop ID 0 for a responder that was never assigned an id, or that // withholds it to obscure its routing. Rejecting it tore down the announce stream of a // conforming publisher. diff --git a/js/net/src/lite/announce.ts b/js/net/src/lite/announce.ts index aca95eeaad..d9d6bf7874 100644 --- a/js/net/src/lite/announce.ts +++ b/js/net/src/lite/announce.ts @@ -3,7 +3,7 @@ import { type Cost, type Hop, HopSchema, MAX_HOPS, UNKNOWN_HOP } from "../hop.ts import * as Path from "../path.ts"; import type { Reader, Writer } from "../stream.ts"; import * as Message from "./message.ts"; -import { hasAnnounceId, hasAnnounceOk, hasExcludeHop, hasRouteCost, Version } from "./version.ts"; +import { hasAnnounceId, hasAnnounceOk, hasExcludeHop, hasHidden, hasRouteCost, Version } from "./version.ts"; // Pre-lite-06 inner status values, carried inside the single ANNOUNCE_BROADCAST body. const STATUS_ENDED = 0; @@ -266,10 +266,15 @@ export class AnnounceRequest { * * Must be a bigint: peer origins are up to 62 bits and overflow u53. */ excludeHop: bigint; + /** Lite07+: also announce routes with a `.`-prefixed segment below the prefix. Not on + * the wire earlier, so a value set here is ignored when encoding for an older version + * and decodes as false. */ + hidden: boolean; - constructor(prefix: Path.Valid, excludeHop: bigint = 0n) { + constructor(prefix: Path.Valid, excludeHop: bigint = 0n, hidden = false) { this.prefix = prefix; this.excludeHop = excludeHop; + this.hidden = hidden; } async #encode(w: Writer, version: Version) { @@ -277,12 +282,16 @@ export class AnnounceRequest { if (hasExcludeHop(version)) { await w.u62(this.excludeHop); } + if (hasHidden(version)) { + await w.bool(this.hidden); + } } static async #decode(r: Reader, version: Version): Promise { const prefix = Path.decode(await r.string()); const excludeHop = hasExcludeHop(version) ? await r.u62() : 0n; - return new AnnounceRequest(prefix, excludeHop); + const hidden = hasHidden(version) ? await r.bool() : false; + return new AnnounceRequest(prefix, excludeHop, hidden); } async encode(w: Writer, version: Version): Promise { diff --git a/js/net/src/lite/connection.ts b/js/net/src/lite/connection.ts index 3aca065ab6..75e681d6f8 100644 --- a/js/net/src/lite/connection.ts +++ b/js/net/src/lite/connection.ts @@ -170,8 +170,8 @@ export class Connection implements Established { } } - announced(scope?: Path.Pattern): announce.Consumer { - return this.#subscriber.announced(scope); + announced(scope?: Path.Pattern, options?: announce.Options): announce.Consumer { + return this.#subscriber.announced(scope, options); } async #runSession() { diff --git a/js/net/src/lite/publisher.test.ts b/js/net/src/lite/publisher.test.ts index 2410765e41..d8092a083f 100644 --- a/js/net/src/lite/publisher.test.ts +++ b/js/net/src/lite/publisher.test.ts @@ -1088,6 +1088,14 @@ test("lite draft-06: a subscription starting mid-group skips the head", async () expect(served).toEqual([{ sequence: 0, frameStart: 2, payloads: ["c", "d"] }]); }); +// A start at the group's final frame count is a valid, empty range: FIN, don't reset. +// A relay resuming a parked track asks for exactly this. +test("lite draft-06: a subscription starting at the end of a group serves it empty", async () => { + const { start, served } = await serve({ 0: ["a", "b"] }, { startGroup: 0, startFrame: 2 }); + expect(start).toBe(0); + expect(served).toEqual([{ sequence: 0, frameStart: 2, payloads: [] }]); +}); + // The end bound is inclusive. test("lite draft-06: a subscription capped mid-group stops at the end frame", async () => { const { served } = await serve( diff --git a/js/net/src/lite/publisher.ts b/js/net/src/lite/publisher.ts index bb2230ed82..bff615b0e9 100644 --- a/js/net/src/lite/publisher.ts +++ b/js/net/src/lite/publisher.ts @@ -1,9 +1,9 @@ -import { type Dispose, type Getter, Signal } from "@moq/signals"; +import { type Dispose, type Getter, race, Signal } from "@moq/signals"; import type * as broadcast from "../broadcast.ts"; import { error, NotFound, reason, StreamCode, StreamError } from "../error.ts"; import type * as group from "../group.ts"; import { type Hop, type Route, routesEqual } from "../hop.ts"; -import { hooks } from "../internal.ts"; +import { hiddenBelow, hooks } from "../internal.ts"; import type { Consumer as OriginConsumer } from "../origin.ts"; import * as Path from "../path.ts"; import { type Reader, type Stream, Writer } from "../stream.ts"; @@ -32,7 +32,11 @@ import { hasAnnounceId, hasAnnounceOk, hasDatagrams, hasProbeRtt, resolvesStart, // Where each originated route lands under the requested prefix: its suffix beneath // the prefix, or the empty suffix for a route above it, where the most specific // such route wins the way a request through the prefix would resolve. -function presented(prefix: Path.Valid, table: ReadonlyMap): Map { +function presented( + prefix: Path.Valid, + table: ReadonlyMap, + hidden: boolean, +): Map { const out = new Map(); let rootLen = -1; for (const [covered, snap] of table) { @@ -42,6 +46,8 @@ function presented(prefix: Path.Valid, table: ReadonlyMap