From 157e541950fc31f6d9f4289eb9e77e7794e60907 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 5 Oct 2026 23:45:03 -0700 Subject: [PATCH 01/12] quest: claim m1/catalog-enabled Co-Authored-By: Claude Opus 5.5 From f8d4cc49c2bbc7edd315828cc465ddee5edb8dda Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 00:37:27 -0700 Subject: [PATCH 02/12] feat(hang)!: one enabled flag replaces stalled Every audio and video rendition carries an optional `enabled` (default true, written only when false); a disabled rendition has no frames coming and must not be selected. Writers stop writing `stalled`, readers ignore it, and the flapping encoder-lag detector is deleted in JS and Rust. @moq/publish keeps a paused rendition in the catalog with `enabled: false` and its last config, encoding one black keyframe before disabling video. @moq/watch deselects disabled renditions. moq-transcode, moq-hls, moq-ffi, the wrappers, and C (`moq_consume_video_enabled`) follow. Co-Authored-By: Claude Opus 5.5 --- dart/moq/lib/src/aliases.dart | 4 +- dart/moq_ffi/lib/src/moq.dart | 32 +- doc/concept/hang.md | 2 +- doc/concept/standard.md | 1 - doc/lib/c/index.md | 2 +- doc/lib/rs/hang.md | 2 +- drafts/draft-lcurley-moq-hang.md | 24 +- go/wrapper/types.go | 4 +- js/hang/src/catalog/audio.test.ts | 12 + js/hang/src/catalog/audio.ts | 4 + js/hang/src/catalog/index.ts | 1 - js/hang/src/catalog/stalled.test.ts | 122 ------- js/hang/src/catalog/stalled.ts | 93 ------ js/hang/src/catalog/video.test.ts | 19 +- js/hang/src/catalog/video.ts | 6 +- js/msf/src/catalog.test.ts | 21 -- js/msf/src/catalog.ts | 2 - js/publish/src/audio/encoder.test.ts | 22 ++ js/publish/src/audio/encoder.ts | 34 +- js/publish/src/video/encoder.test.ts | 158 +++++++--- js/publish/src/video/encoder.ts | 144 ++++----- js/watch/src/audio/decoder.test.ts | 31 ++ js/watch/src/audio/source.ts | 5 +- js/watch/src/msf.test.ts | 16 - js/watch/src/msf.ts | 1 - js/watch/src/video/config.test.ts | 2 +- js/watch/src/video/source.test.ts | 45 ++- js/watch/src/video/source.ts | 22 +- .../kotlin/dev/moq/Aliases.kt | 4 +- py/moq-rs/README.md | 4 +- ...bandwidth-grant-in-moq-audio-instead-of.md | 7 +- quest/m1/README.md | 1 - quest/m1/catalog-enabled.md | 66 ---- quest/m1/rendition-preference.md | 6 +- quest/m3/ladder/README.md | 3 +- quest/m3/ladder/controller.md | 1 - rs/hang/src/catalog/audio/mod.rs | 21 ++ rs/hang/src/catalog/mod.rs | 11 +- rs/hang/src/catalog/root.rs | 3 +- rs/hang/src/catalog/stalled.rs | 297 ------------------ rs/hang/src/catalog/video/mod.rs | 33 +- rs/moq-c/README.md | 2 +- rs/moq-c/src/api.rs | 12 +- rs/moq-c/src/consume.rs | 6 +- rs/moq-c/src/test.rs | 34 +- rs/moq-ffi/src/audio.rs | 1 + rs/moq-ffi/src/media.rs | 24 +- rs/moq-ffi/src/test.rs | 1 + rs/moq-ffi/src/video.rs | 3 +- rs/moq-hls/src/export/rendition.rs | 3 +- rs/moq-msf/src/lib.rs | 10 - rs/moq-mux/src/catalog/msf/consumer.rs | 3 - rs/moq-mux/src/catalog/producer.rs | 3 - rs/moq-mux/src/catalog/tracks.rs | 2 +- rs/moq-mux/src/codec/av1/import.rs | 19 -- rs/moq-mux/src/codec/h264/import.rs | 63 ---- rs/moq-mux/src/codec/h265/import.rs | 19 -- rs/moq-mux/src/codec/video.rs | 148 +-------- rs/moq-mux/src/codec/vp8/import.rs | 19 -- rs/moq-mux/src/codec/vp9/import.rs | 19 -- rs/moq-mux/src/container/flv/import.rs | 74 +---- rs/moq-mux/src/container/ts/import.rs | 18 -- rs/moq-rtc/src/codec/av1.rs | 5 - rs/moq-rtc/src/codec/h264.rs | 5 - rs/moq-rtc/src/codec/h265.rs | 5 - rs/moq-rtc/src/codec/mod.rs | 26 -- rs/moq-rtc/src/codec/vp8.rs | 5 - rs/moq-rtc/src/codec/vp9.rs | 5 - rs/moq-rtc/src/ingest.rs | 4 - rs/moq-rtc/src/session.rs | 19 +- rs/moq-transcode/src/catalog.rs | 24 +- rs/moq-transcode/src/pipeline.rs | 10 +- rs/moq-video/src/encode/producer.rs | 76 +---- swift/Sources/Moq/Aliases.swift | 6 +- 74 files changed, 536 insertions(+), 1425 deletions(-) delete mode 100644 js/hang/src/catalog/stalled.test.ts delete mode 100644 js/hang/src/catalog/stalled.ts delete mode 100644 quest/m1/catalog-enabled.md delete mode 100644 rs/hang/src/catalog/stalled.rs diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 742fbe54f4..93929a19fd 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -152,7 +152,7 @@ typedef Frame = MoqFrame; /// A media [Frame] whose keyframe flag marks group starts or video keyframes; audio flags only group starts. typedef MediaFrame = MoqMediaFrame; -/// The catalog description of a video track, including whether the publisher recommends temporarily avoiding it. +/// The catalog description of a video track, including whether it is enabled (a disabled one has no frames coming). typedef Video = MoqVideo; /// Caller-provided catalog fields for a video track. @@ -167,7 +167,7 @@ typedef VideoProperties = MoqVideoProperties; /// A single video codec an importer can parse. typedef VideoFormat = MoqVideoFormat; -/// The catalog description of an audio track: codec, sample rate, channels, and container. +/// The catalog description of an audio track: codec, sample rate, channels, whether it is enabled, and container. typedef Audio = MoqAudio; /// An audio codec, its required init bytes, and an optional label. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index 3e5e353de9..08789e7834 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -414,6 +414,7 @@ class MoqAudio { final int sampleRate; final int channelCount; final int? bitrate; + final bool enabled; final MoqContainer container; MoqAudio({ this.label = null, @@ -423,6 +424,7 @@ class MoqAudio { required this.sampleRate, required this.channelCount, this.bitrate, + this.enabled = true, required this.container, }); } @@ -469,6 +471,11 @@ class FfiConverterMoqAudio { ); final bitrate = bitrate_lifted.value; new_offset += bitrate_lifted.bytesRead; + final enabled_lifted = FfiConverterBool.read( + Uint8List.view(buf.buffer, new_offset), + ); + final enabled = enabled_lifted.value; + new_offset += enabled_lifted.bytesRead; final container_lifted = FfiConverterMoqContainer.read( Uint8List.view(buf.buffer, new_offset), ); @@ -483,6 +490,7 @@ class FfiConverterMoqAudio { sampleRate: sampleRate, channelCount: channelCount, bitrate: bitrate, + enabled: enabled, container: container, ), new_offset - buf.offsetInBytes, @@ -498,6 +506,7 @@ class FfiConverterMoqAudio { FfiConverterUInt32.allocationSize(value.sampleRate) + FfiConverterUInt32.allocationSize(value.channelCount) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + 0; final buf = Uint8List(total_length); @@ -535,6 +544,10 @@ class FfiConverterMoqAudio { value.bitrate, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterBool.write( + value.enabled, + Uint8List.view(buf.buffer, new_offset), + ); new_offset += FfiConverterMoqContainer.write( value.container, Uint8List.view(buf.buffer, new_offset), @@ -550,6 +563,7 @@ class FfiConverterMoqAudio { FfiConverterUInt32.allocationSize(value.sampleRate) + FfiConverterUInt32.allocationSize(value.channelCount) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + 0; } @@ -1103,7 +1117,7 @@ class MoqVideo { final MoqDimensions? coded; final MoqDimensions? displayAspect; final int? bitrate; - final bool stalled; + final bool enabled; final double? framerate; final MoqContainer container; MoqVideo({ @@ -1114,7 +1128,7 @@ class MoqVideo { this.coded, this.displayAspect, this.bitrate, - this.stalled = false, + this.enabled = true, this.framerate, required this.container, }); @@ -1162,11 +1176,11 @@ class FfiConverterMoqVideo { ); final bitrate = bitrate_lifted.value; new_offset += bitrate_lifted.bytesRead; - final stalled_lifted = FfiConverterBool.read( + final enabled_lifted = FfiConverterBool.read( Uint8List.view(buf.buffer, new_offset), ); - final stalled = stalled_lifted.value; - new_offset += stalled_lifted.bytesRead; + final enabled = enabled_lifted.value; + new_offset += enabled_lifted.bytesRead; final framerate_lifted = FfiConverterOptionalDouble64.read( Uint8List.view(buf.buffer, new_offset), ); @@ -1186,7 +1200,7 @@ class FfiConverterMoqVideo { coded: coded, displayAspect: displayAspect, bitrate: bitrate, - stalled: stalled, + enabled: enabled, framerate: framerate, container: container, ), @@ -1203,7 +1217,7 @@ class FfiConverterMoqVideo { FfiConverterOptionalMoqDimensions.allocationSize(value.coded) + FfiConverterOptionalMoqDimensions.allocationSize(value.displayAspect) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + - FfiConverterBool.allocationSize(value.stalled) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + 0; @@ -1243,7 +1257,7 @@ class FfiConverterMoqVideo { Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterBool.write( - value.stalled, + value.enabled, Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterOptionalDouble64.write( @@ -1265,7 +1279,7 @@ class FfiConverterMoqVideo { FfiConverterOptionalMoqDimensions.allocationSize(value.coded) + FfiConverterOptionalMoqDimensions.allocationSize(value.displayAspect) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + - FfiConverterBool.allocationSize(value.stalled) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + 0; diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 1a7bfb40f6..280e4ebc6b 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -56,7 +56,7 @@ A few things the catalog can express beyond decoder config: - **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. -- **Stalled renditions.** A publisher can flag a rendition as temporarily bad so players prefer another one without the track disappearing. First-party video publishers set this flag after more than three frame intervals of source silence or encoding lag while subscribed, and clear it after three on-time completed frames or when idle. Browser and native capture poll while waiting; FLV and MPEG-TS importers observe video silence as container data arrives. The shared detector is `hang::catalog::stalled::Detector` in Rust and `Catalog.Stalled.Detector` in JavaScript. It is a playback diagnostic, not an authorization or routing signal. +- **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. - **Archive.** A broadcast may advertise an `archive` entry naming its timeline track (a small index of each complete aligned segment) and, if recorded, the replay MoQ path, object-store URL, and format version. The timeline is what lets the [HLS gateway](/bin/hls) build playlists without subscribing to media. - **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a segment index. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 9b4730a795..d96c6d2dbb 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -113,7 +113,6 @@ The MoQ Streaming Format is a catalog, playing the role HLS playlists and SDP do elsewhere. It overlaps with the [hang catalog](/concept/hang) and the two will likely converge. The tools track draft-01 and hide the version on the wire, so draft-00 catalogs still decode and init data always arrives inline. -The `stalled` rendition hint is shared between the two formats. ## LOC diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index f49b96850e..e18616f3c9 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -52,7 +52,7 @@ pkg-config file included. - **Server.** `moq_server_listen` binds before it returns (a bad address or certificate fails there) and hands each incoming session to `on_request` as a request handle. Read `moq_session_request_path` and `_query` to route and authenticate, then `moq_session_request_accept` (a session handle, with origins like `moq_session_connect`) or `moq_session_request_reject` with an HTTP-style code (401 and 403 become the protocol's unauthorized close). An accepted session reports `1` once SETUP completes and never reconnects. `moq_server_addr` reports an ephemeral port and `moq_server_fingerprints` the hashes a client pins for a `tls_generate` certificate. `moq_server_close` stops listening; its terminal callback fires once the sockets are released. - **Demand.** A watcher on a published track (`moq_publish_track_demand`, `moq_publish_media_demand`, `moq_encode_video_demand`, `moq_encode_audio_demand`) calls `on_demand` with `MOQ_DEMAND_USED` or `MOQ_DEMAND_UNUSED` right away and again on every change, so an encoder on a battery-powered device runs only while someone is watching. The first call is the current state, so a track that went unused before the watcher existed still reports it. `moq_publish_demand_cancel` stops it; the terminal callback still fires. A container has no single demand and is refused. Demand follows the last real subscriber: an origin that served the track drops its source copy on the unused edge, so a cache linger never delays it. Only a relay keeps what it already delivered warm for 30 seconds, and a returning subscriber is served that cache only once the publisher confirms it is still current. - **Requests.** `moq_publish_dynamic` serves subscriptions to tracks the broadcast never declared: each arrives as a request handle, read its name with `moq_track_request_name`, then `moq_track_request_accept` (a raw track handle), `moq_track_request_video` / `_audio` (the media handle `moq_publish_video` / `_audio` return), or `moq_track_request_abort` with an application code the subscriber sees. Without a live handler an unknown name is refused. `moq_publish_track_dynamic` does the same for fetches of groups a track no longer has cached, delivered as `moq_group_request_*` (`sequence`, `priority`, `frame_start`); `moq_group_request_accept` starts the producer at `frame_start` so written frames keep their group indices. Register it with `moq_track_request_dynamic` before accepting a track that was itself requested by a fetch, so that pending group survives the transition. Both handlers stop with `moq_publish_dynamic_cancel`. -- **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams, JSON and binary data tracks (snapshot or stream, each advertised in the catalog for as long as it lives), group fetch, catalog sections, shared video properties, and stalled hints. The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), `moq_publish_close` (ends the broadcast for good and releases its handle), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a `moq_announce_config` (NULL or zeroed for everything): a literal prefix and an optional relative pattern filter; each `moq_announce_event` has a `kind` (`MOQ_ANNOUNCE_KIND_START`, `_UPDATE`, `_END`, or `_LIVE` once every route live when the listener started has been delivered, with no prefix or captures); `prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts); set `hidden` or name the dot segment in `prefix` to list them. +- **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams, JSON and binary data tracks (snapshot or stream, each advertised in the catalog for as long as it lives), group fetch, catalog sections, shared video properties, and whether a video rendition is enabled (`moq_consume_video_enabled`). The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), `moq_publish_close` (ends the broadcast for good and releases its handle), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a `moq_announce_config` (NULL or zeroed for everything): a literal prefix and an optional relative pattern filter; each `moq_announce_event` has a `kind` (`MOQ_ANNOUNCE_KIND_START`, `_UPDATE`, `_END`, or `_LIVE` once every route live when the listener started has been delivered, with no prefix or captures); `prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts); set `hidden` or name the dot segment in `prefix` to list them. ```c moq_client_config config; diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index b25a36a24a..5ed1a3b05b 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -12,7 +12,7 @@ The [hang media format](/concept/hang) on top of `moq-net`: a live catalog describing renditions with WebCodecs-style decoder configs, and containers that carry a timestamp with every frame. -- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), stalled hints, cross-broadcast rendition references, and the `archive` entry (timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. +- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), per-rendition `enabled` flags, cross-broadcast rendition references, and the `archive` entry (timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. - **Containers.** `legacy`, `cmaf`, and `loc`, decoded for you; unknown kinds pass through untouched. - **`Ordered`** reads a media track as timestamped frames, reorders groups, and skips ones that fall past your max age. - **Codecs described**: H.264, H.265, VP8, VP9, AV1, AAC, Opus, PCM. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 52d5060e68..dc5128ec79 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -192,7 +192,7 @@ In addition to the WebCodecs fields, each rendition MAY carry the common renditi type VideoDecoderConfigExtensions = { "displayAspectWidth": number | undefined, "displayAspectHeight": number | undefined, - "stalled": boolean | undefined, + "enabled": boolean | undefined, } ~~~ @@ -200,10 +200,7 @@ type VideoDecoderConfigExtensions = { A consumer that understands neither field MUST assume square pixels, a 1:1 ratio. Both MUST be present together; a consumer that sees only one MUST ignore it. -`stalled` indicates that the publisher recommends temporarily avoiding the rendition. -The track remains available when `stalled` is true. -A consumer SHOULD select an unstalled rendition when it supports one, but MAY select a stalled rendition when no unstalled rendition is suitable. -If absent, `stalled` defaults to false. +`enabled` is defined in {{field-enabled}}. For example: @@ -217,7 +214,7 @@ For example: "codedWidth": 1280, "codedHeight": 720, "bitrate": 6000000, - "stalled": true, + "enabled": false, "framerate": 30.0, "jitter": 34 }, @@ -240,6 +237,14 @@ For example: } ~~~ +### enabled {#field-enabled} +The `enabled` field says whether a consumer may select an audio or video rendition. +When `enabled` is false, no frames are coming, and a consumer MUST NOT select the rendition. +If absent, `enabled` defaults to true; a publisher SHOULD only write it when false. + +Earlier versions defined a video `stalled` field instead. +A publisher MUST NOT write `stalled`, and a consumer MUST ignore it. + ## Audio An audio track contains the necessary information to decode an audio stream. @@ -254,7 +259,7 @@ The `renditions` field contains a map of track names to audio decoder configurat See the [WebCodecs specification](https://www.w3.org/TR/webcodecs/#audio-decoder-config) for specifics and registered codecs. Any field carrying raw bytes, notably `description`, is a hex string ({{binary}}). -In addition to the WebCodecs fields, each rendition MAY carry the common rendition fields ({{common}}). +In addition to the WebCodecs fields, each rendition MAY carry the common rendition fields ({{common}}) and `enabled` ({{field-enabled}}). ### PCM {#audio-pcm} @@ -1088,6 +1093,11 @@ This document has no IANA actions. # Appendix A: Changelog {:numbered="false"} +## moq-hang-04 +{:numbered="false"} + +- Replaced the video `stalled` field with an optional `enabled` field on audio and video renditions. A consumer MUST NOT select a disabled rendition and ignores `stalled`. A consumer that predates `enabled` keeps selecting a disabled rendition. + ## moq-hang-03 {:numbered="false"} diff --git a/go/wrapper/types.go b/go/wrapper/types.go index 85c6274c4a..bdd643aad7 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -6,7 +6,7 @@ import ffi "moq.dev/moq-ffi/moq" // These are plain data, so type aliases are exact: a moq.AudioFrame is an // ffi.MoqAudioFrame, constructible and comparable across the boundary. type ( - // Audio describes one audio rendition in a broadcast catalog: codec, sample rate, channel count, and container. + // Audio describes one audio rendition in a broadcast catalog: codec, sample rate, channel count, whether it is enabled, and container. Audio = ffi.MoqAudio // AudioCodec selects the audio encoder codec. Build one with OpusAudioCodec or AacAudioCodec; // adding a codec later adds a constructor, not a breaking enum change. @@ -67,7 +67,7 @@ type ( // TrackInfo holds publisher-side track properties: priority, ordering, max age, and timescale. // A zero Priority is the least urgent, not the default; set 127 for the midpoint a nil TrackInfo uses. TrackInfo = ffi.MoqTrackInfo - // Video describes one catalog rendition, including whether the publisher recommends temporarily avoiding it. + // Video describes one catalog rendition, including whether it is enabled (a disabled one has no frames coming). Video = ffi.MoqVideo // VideoHint supplies catalog fields a video stream can't reveal itself, such as bitrate, filling only the gaps. VideoHint = ffi.MoqVideoHint diff --git a/js/hang/src/catalog/audio.test.ts b/js/hang/src/catalog/audio.test.ts index 9ad31d8365..e3bc0d5b4a 100644 --- a/js/hang/src/catalog/audio.test.ts +++ b/js/hang/src/catalog/audio.test.ts @@ -24,3 +24,15 @@ test("audio config accepts a human-readable label", () => { expect(config.label).toBe("English"); }); + +test("audio config accepts optional enabled state", () => { + const config = AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48_000, + numberOfChannels: 2, + enabled: false, + }); + + expect(config.enabled).toBe(false); +}); diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index f897c9125a..451f7be337 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -44,6 +44,10 @@ export const AudioConfigSchema = z.object({ // TODO: Support up to Number.MAX_SAFE_INTEGER bitrate: z.optional(u53Schema), + // Whether this rendition may be selected. When false, no frames are coming and a consumer + // must not select it. Default: true, so publishers only write it when false. + enabled: z.optional(z.boolean()), + // The maximum delay between a frame being ready and the publisher flushing it, in whole // milliseconds rounded up. The player's jitter buffer should be larger than this value. // If not provided, the player should assume each frame is flushed immediately. diff --git a/js/hang/src/catalog/index.ts b/js/hang/src/catalog/index.ts index ceb52fd14f..4a59edca7b 100644 --- a/js/hang/src/catalog/index.ts +++ b/js/hang/src/catalog/index.ts @@ -20,7 +20,6 @@ export * from "./mode"; export * from "./path"; export * from "./priority"; export * from "./root"; -export * as Stalled from "./stalled"; export * from "./text"; export * from "./track"; export * from "./video"; diff --git a/js/hang/src/catalog/stalled.test.ts b/js/hang/src/catalog/stalled.test.ts deleted file mode 100644 index fbdef4ceff..0000000000 --- a/js/hang/src/catalog/stalled.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { expect, test } from "bun:test"; -import { Time } from "@moq/net"; -import { CLEAR_FRAMES, DEFAULT_INTERVAL, Detector, intervalFromFps, SET_INTERVALS } from "./stalled"; - -const FRAME = Time.Micro.fromMilli(33 as Time.Milli); - -function sample(lag: Time.Micro): Parameters[0] { - return { - frame: true, - mediaLag: lag, - quiet: 0 as Time.Micro, - interval: FRAME, - demand: true, - idle: false, - }; -} - -test("idle is never stalled", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - idle: true, - demand: true, - mediaLag: Time.Micro.fromSecond(10 as Time.Second), - quiet: Time.Micro.fromSecond(10 as Time.Second), - interval: FRAME, - }), - ).toBe(false); - expect(stalled.stalled).toBe(false); -}); - -test("no demand is never stalled", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - demand: false, - idle: false, - mediaLag: Time.Micro.fromSecond(10 as Time.Second), - quiet: Time.Micro.fromSecond(10 as Time.Second), - interval: FRAME, - }), - ).toBe(false); - expect(stalled.stalled).toBe(false); -}); - -test("lag past the threshold sets the flag", () => { - const stalled = new Detector(); - expect(stalled.observe(sample((FRAME * SET_INTERVALS) as Time.Micro))).toBe(false); - expect(stalled.observe(sample((FRAME * SET_INTERVALS + 1) as Time.Micro))).toBe(true); - expect(stalled.flag()).toBe(true); -}); - -test("a quiet source sets the flag", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - mediaLag: 0 as Time.Micro, - quiet: (FRAME * SET_INTERVALS + 1) as Time.Micro, - interval: FRAME, - demand: true, - idle: false, - }), - ).toBe(true); - expect(stalled.stalled).toBe(true); -}); - -test("clearing needs a run of on-time frames", () => { - const stalled = new Detector(); - stalled.observe(sample((FRAME * 10) as Time.Micro)); - for (let i = 0; i < CLEAR_FRAMES - 1; i++) { - expect(stalled.observe(sample(10 as Time.Micro))).toBe(false); - expect(stalled.stalled).toBe(true); - } - expect(stalled.observe(sample(10 as Time.Micro))).toBe(true); - expect(stalled.stalled).toBe(false); - expect(stalled.flag()).toBeUndefined(); -}); - -test("dropping demand clears immediately", () => { - const stalled = new Detector(); - stalled.observe(sample((FRAME * 10) as Time.Micro)); - expect(stalled.observe({ ...sample((FRAME * 10) as Time.Micro), demand: false })).toBe(true); - expect(stalled.stalled).toBe(false); -}); - -test("a zero interval uses the default", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - mediaLag: (DEFAULT_INTERVAL * SET_INTERVALS + 1) as Time.Micro, - quiet: 0 as Time.Micro, - interval: 0 as Time.Micro, - demand: true, - idle: false, - }), - ).toBe(true); -}); - -test("intervalFromFps falls back", () => { - expect(intervalFromFps(undefined)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(0)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(Number.NaN)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(50)).toBe(Time.Micro.fromMilli(20 as Time.Milli)); -}); - -test("polls do not count as recovery frames", () => { - const stalled = new Detector(); - stalled.observe(sample((FRAME * 10) as Time.Micro)); - for (let i = 0; i < 10; i++) { - expect(stalled.observe({ ...sample(0 as Time.Micro), frame: false })).toBe(false); - } - expect(stalled.stalled).toBe(true); -}); - -test("unrepresentable frame intervals use the default", () => { - expect(intervalFromFps(Number.MIN_VALUE)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(Number.MAX_VALUE)).toBe(DEFAULT_INTERVAL); -}); diff --git a/js/hang/src/catalog/stalled.ts b/js/hang/src/catalog/stalled.ts deleted file mode 100644 index dfec96753f..0000000000 --- a/js/hang/src/catalog/stalled.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Time } from "@moq/net"; - -/** How many frame intervals of lag mark a rendition stalled. */ -export const SET_INTERVALS = 3; - -/** Consecutive on-time frames required to clear a stall. */ -export const CLEAR_FRAMES = 3; - -/** Frame interval used when the catalog does not advertise a framerate. */ -export const DEFAULT_INTERVAL = Time.Micro.fromMilli(33 as Time.Milli); - -/** One observation of a rendition's source versus what the transport has accepted. */ -export interface Sample { - /** This observation completes a newly accepted frame. */ - frame: boolean; - /** Newest captured timestamp minus newest timestamp handed to the transport. */ - mediaLag: Time.Micro; - /** Wall time since the source last delivered a frame, in microseconds. */ - quiet: Time.Micro; - /** One frame interval. The set threshold is {@link SET_INTERVALS} of these. */ - interval: Time.Micro; - /** Someone is subscribed to this rendition. */ - demand: boolean; - /** Camera released / no live source. Never stalled. */ - idle: boolean; -} - -/** A positive frame interval, falling back to {@link DEFAULT_INTERVAL}. */ -export function interval(value: Time.Micro): Time.Micro { - return value > 0 ? value : DEFAULT_INTERVAL; -} - -/** Frame interval implied by a catalog framerate, or {@link DEFAULT_INTERVAL}. */ -export function intervalFromFps(fps: number | undefined): Time.Micro { - if (fps === undefined || !Number.isFinite(fps) || fps <= 0) return DEFAULT_INTERVAL; - const value = Time.Micro.fromSecond((1 / fps) as Time.Second); - return Number.isFinite(value) && value >= 0.0005 && value < 2 ** 64 * 1_000_000 ? value : DEFAULT_INTERVAL; -} - -/** - * Hysteresis around the catalog `stalled` bit. - * - * Set once lag exceeds a few frame intervals; clear only after a run of on-time - * frames. Idle or undemanded broadcasts are never stalled. - */ -export class Detector { - #on = false; - #recover = 0; - - /** Whether the rendition is currently stalled. */ - get stalled(): boolean { - return this.#on; - } - - /** The catalog value: `true` while stalled, omitted otherwise. */ - flag(): true | undefined { - return this.#on ? true : undefined; - } - - /** Feed one sample. Returns whether the catalog flag changed. */ - observe(sample: Sample): boolean { - if (sample.idle || !sample.demand) return this.#clear(); - - const frame = interval(sample.interval); - const lag = Math.max(sample.mediaLag, sample.quiet) as Time.Micro; - const over = lag > frame * SET_INTERVALS; - const onTime = lag <= frame; - - if (!this.#on) { - if (!over) return false; - this.#on = true; - this.#recover = 0; - return true; - } - - if (!onTime) { - this.#recover = 0; - return false; - } - - if (!sample.frame) return false; - this.#recover++; - if (this.#recover < CLEAR_FRAMES) return false; - return this.#clear(); - } - - #clear(): boolean { - this.#recover = 0; - if (!this.#on) return false; - this.#on = false; - return true; - } -} diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 2c8138c1e0..c84789add2 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -15,15 +15,26 @@ test("video config accepts canonical display aspect fields", () => { expect("displayRatioHeight" in parsed).toBe(false); }); -test("video config accepts optional stalled state", () => { +test("video config accepts optional enabled state", () => { const active = VideoConfigSchema.parse({ codec: "avc1.64001f", container: { kind: "legacy" }, }); - const stalled = VideoConfigSchema.parse({ ...active, stalled: true }); + const disabled = VideoConfigSchema.parse({ ...active, enabled: false }); - expect(active.stalled).toBeUndefined(); - expect(stalled.stalled).toBe(true); + expect(active.enabled).toBeUndefined(); + expect(disabled.enabled).toBe(false); +}); + +test("video config ignores a legacy stalled flag", () => { + const parsed = VideoConfigSchema.parse({ + codec: "avc1.64001f", + container: { kind: "legacy" }, + stalled: true, + }); + + expect("stalled" in parsed).toBe(false); + expect(parsed.enabled).toBeUndefined(); }); test("video config accepts a human-readable label", () => { diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 53c9946a7f..fd272cd67c 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -48,9 +48,9 @@ export const VideoConfigSchema = z.object({ // TODO: Support up to Number.MAX_SAFE_INTEGER bitrate: z.optional(u53Schema), - // Whether the publisher recommends temporarily avoiding this rendition. - // The track remains available and may still be selected as a fallback. - stalled: z.optional(z.boolean()), + // Whether this rendition may be selected. When false, no frames are coming and a consumer + // must not select it. Default: true, so publishers only write it when false. + enabled: z.optional(z.boolean()), // If true, the decoder will optimize for latency. // Default: true diff --git a/js/msf/src/catalog.test.ts b/js/msf/src/catalog.test.ts index a6e16a222b..7501ed5f5b 100644 --- a/js/msf/src/catalog.test.ts +++ b/js/msf/src/catalog.test.ts @@ -209,27 +209,6 @@ test("preserves delay through decode and encode", () => { expect(wireTracks[0].delay).toBe(200); }); -test.each([ - ["omitted", undefined], - ["false", false], - ["true", true], -] as const)("preserves %s stalled state through decode and encode", (_name, stalled) => { - const track = { - name: "video0", - packaging: "loc", - isLive: true, - role: "video", - codec: "av01.0.08M.10.0.110.09", - ...(stalled === undefined ? {} : { stalled }), - }; - const catalog = decode(encodeJson({ version: "draft-01", tracks: [track] })); - - expect(catalog.tracks[0].stalled).toBe(stalled); - const wire = decodeJson(encode(catalog)); - const tracks = wire.tracks as { stalled?: boolean }[]; - expect(tracks[0].stalled).toBe(stalled); -}); - test("carries extension root members through decode and encode", () => { // An extension section (here `mpegts`) rides the catalog root untouched, and a // section this build has never heard of survives the same way. diff --git a/js/msf/src/catalog.ts b/js/msf/src/catalog.ts index 9596adbee0..b3b3bb9224 100644 --- a/js/msf/src/catalog.ts +++ b/js/msf/src/catalog.ts @@ -35,8 +35,6 @@ const trackShape = { samplerate: z.optional(z.number()), channelConfig: z.optional(z.string()), bitrate: z.optional(z.number()), - // Non-standard: whether the publisher recommends temporarily avoiding this track. - stalled: z.optional(z.boolean()), /** Resolved base64 initialization data (draft-01's initRef indirection is resolved away). */ initData: z.optional(z.string()), renderGroup: z.optional(z.number()), diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index ffd4db52f9..bd4b342915 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -342,6 +342,28 @@ test("disabling with a subscriber attached marks where submitted audio ends", as ]); }); +// A muted rendition stays in the catalog, so a viewer deselects it with a one-field delta instead of +// seeing it removed and re-added. +test("disabling keeps the rendition in the catalog with enabled: false", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(); + const { encoder, enabled, rendition } = env; + + const before = encoder.out.catalog.peek(); + expect(before).toBeDefined(); + expect(before?.enabled).toBeUndefined(); + + enabled.set(false); + await settle(); + expect(encoder.out.catalog.peek()).toEqual({ ...before, enabled: false } as never); + expect(rendition.config.peek()).toEqual({ ...before, enabled: false } as never); + expect(encoder.out.active.peek()).toBe(false); + + enabled.set(true); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(before); +}); + // Closing tears down the subscription and the pipeline, which both end the epoch; cleanups run // last-in, first-out, so the marker lands once and before the rendition closes the track. test("closing with a subscriber attached marks the end once before the track closes", async () => { diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index f57b069d35..14459c7d72 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -80,8 +80,9 @@ export interface Stats { // Signals the encoder reads. export type EncoderInput = { - // Whether to publish (and encode) this rendition. Defaults to true. When false the rendition drops out of the - // catalog and stops encoding, ending the epoch, but stays registered so a subscriber still gets an idle track. + // Whether to encode this rendition. Defaults to true. When false it stops encoding, ending the epoch, + // but stays in the catalog with `enabled: false` and its last config, and stays registered so a + // subscriber still gets an idle track. enabled: Getter; // The broadcast to register the rendition on. Undefined resolves the config but has nowhere to publish. @@ -110,7 +111,8 @@ export type EncoderProps = Inputs & { }; type EncoderOutput = { - // The catalog config published for this rendition, or undefined while there's no capture. + // The catalog config published for this rendition, `enabled: false` while disabled, or undefined + // while there's no capture. catalog: Signal; // The head of the capture graph, so callers can tap the raw capture. Volume is applied to the // PCM rather than in the graph, so this is pre-gain. Undefined for a source that isn't a track. @@ -209,6 +211,10 @@ export class Encoder { #signals = new Effect(); #estimator = new Estimator(); + // The estimator's jitter and delay, republished whenever either rises. + #estimate = new Signal({}); + // The last config published while enabled, which a disabled rendition keeps advertising. + #last?: Catalog.AudioConfig; constructor(name: string, props?: EncoderProps) { // `source` moved to Audio.Capture, which renditions share. TypeScript catches this, but a @@ -381,8 +387,8 @@ export class Encoder { // Derive the encoder config from the captured format and the codec. Re-runs whenever either changes, so a // codec update (bitrate, frame duration) reconfigures without waiting for a channel-count change. // - // Gated on `enabled` the same way the video encoder is: a disabled rendition has to drop out of - // the catalog, and a sample source keeps its format while muted rather than tearing down. + // Gated on `enabled` the same way the video encoder is: a disabled rendition stops resolving a + // config, and a sample source keeps its format while muted rather than tearing down. #runConfig(effect: Effect): void { const fade = effect.get(this.fade); if (!Number.isFinite(fade) || fade < 0) @@ -400,8 +406,15 @@ export class Encoder { } // Publish the config immediately so a consumer can request the demand-gated track. Once encoding - // starts, republish Opus with the exact decoder description reported for that encoder config. + // starts, republish Opus with the exact decoder description reported for that encoder config. A + // disabled rendition keeps its last config, since muting also releases the capture. #runCatalog(effect: Effect): void { + const estimate = effect.get(this.#estimate); + if (!effect.get(this.in.enabled)) { + effect.set(this.#out.catalog, this.#last && { ...this.#last, ...estimate, enabled: false }); + return; + } + const config = effect.get(this.#config)?.catalog; if (!config) { effect.set(this.#out.catalog, undefined); @@ -410,10 +423,8 @@ export class Encoder { const decoder = effect.get(this.#decoderDescription); const catalog = decoder?.config === config ? { ...config, description: decoder.description } : config; - effect.set(this.#out.catalog, { - ...catalog, - ...this.#estimator.estimate, - }); + this.#last = { ...catalog, ...estimate }; + effect.set(this.#out.catalog, this.#last); } // Collect the encode-only Opus knobs that are set, reading the codec through the effect so the @@ -495,8 +506,7 @@ export class Encoder { if (this.#floor !== undefined && frame.timestamp < this.#floor) return; live.producer.encode(frame, frame.timestamp as Time.Micro, true); if (this.#estimator.flush(frame.timestamp, baseline)) { - const catalog = this.#out.catalog.peek(); - if (catalog) this.#out.catalog.set({ ...catalog, ...this.#estimator.estimate }); + this.#estimate.set({ ...this.#estimator.estimate }); } }, error: (err) => { diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 60f7ea5010..0599a53edb 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -1,4 +1,5 @@ import { expect, spyOn, test } from "bun:test"; +import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Moq from "@moq/net"; import { Signal } from "@moq/signals"; @@ -488,83 +489,150 @@ test("frame sources retain their dimensions and nominal frame rate", async () => } }); -test.each(["encoder lag", "quiet startup"])("marks a rendition stalled for %s", async (reason) => { - const clock = spyOn(performance, "now").mockReturnValue(0); - // The fake holds every live output, like a throttled encoder that never hands frames to the session. - using _videoEncoder = installFakeVideoEncoder(); +// A viewer released before `enabled` keeps selecting a disabled rendition, so the last picture it +// decodes has to be black rather than a frozen frame. The rendition stays in the catalog with its last +// config, even once the capture is released. +test("disabling encodes one black keyframe and keeps the rendition with enabled: false", async () => { + const encoded: { timestamp: number; keyFrame: boolean; black: boolean }[] = []; + + class RecordingVideoFrame { + readonly timestamp: number; + readonly black: boolean; + constructor(data: unknown, init?: { timestamp?: number }) { + this.timestamp = init?.timestamp ?? 0; + this.black = data instanceof Uint8Array && data[0] === 0x10; + } + close(): void {} + } + + class RecordingVideoEncoder { + state: CodecState = "unconfigured"; + #output: VideoEncoderInit["output"]; + #codec?: string; + #last = 0; + + constructor(init: VideoEncoderInit) { + this.#output = init.output; + } + + static async isConfigSupported(config: VideoEncoderConfig): Promise<{ supported: boolean }> { + return { supported: config.codec.startsWith("avc1") }; + } + + configure(config: VideoEncoderConfig): void { + this.state = "configured"; + this.#codec = config.codec; + } + + encode(frame: { timestamp: number; black?: boolean }, options?: VideoEncoderEncodeOptions): void { + this.#last = frame.timestamp; + encoded.push({ timestamp: frame.timestamp, keyFrame: !!options?.keyFrame, black: !!frame.black }); + } + + // Emits the newest frame, and reports the configured codec as Chrome does. + async flush(): Promise { + const chunk = { type: "key", timestamp: this.#last, byteLength: 1, copyTo: () => {} }; + this.#output(chunk as never, { decoderConfig: { codec: this.#codec } } as never); + } + + close(): void { + this.state = "closed"; + } + } + + const original = Object.getOwnPropertyDescriptor(globalThis, "VideoEncoder"); + Object.defineProperty(globalThis, "VideoEncoder", { + configurable: true, + value: RecordingVideoEncoder, + writable: true, + }); + const videoFrame = Object.getOwnPropertyDescriptor(globalThis, "VideoFrame"); + Object.defineProperty(globalThis, "VideoFrame", { + configurable: true, + value: RecordingVideoFrame, + writable: true, + }); class Frame { - codedWidth = 640; - codedHeight = 480; - timestamp: number; - closed = false; + readonly timestamp: number; constructor(timestamp: number) { this.timestamp = timestamp; } clone(): Frame { return new Frame(this.timestamp); } - close(): void { - this.closed = true; - } + close(): void {} } const { Fanout } = await import("../fanout"); let controller!: ReadableStreamDefaultController; - const stream = new ReadableStream({ - start: (c) => { - controller = c; - }, - }); - const fanout = new Fanout(stream, { - clone: (frame) => frame.clone(), - release: (frame) => frame.close(), - }); + const fanout = new Fanout( + new ReadableStream({ + start: (c) => { + controller = c; + }, + }), + { clone: (frame) => frame.clone(), release: (frame) => frame.close() }, + ); - const track = new Moq.Track.Producer("video/hd").accept(); + const track = new Moq.Track.Producer("video").accept(); const rendition = { config: new Signal(undefined), track: new Signal(track), close: () => track.close(), }; + const source = new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }); const capture = { - in: { - source: new Signal({ - getSettings: () => ({ frameRate: 30 }), - getConstraints: () => ({}), - } as never), - }, - out: { - display: new Signal({ width: 640, height: 480 }), - frames: new Signal(fanout), - }, + in: { source }, + out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(fanout) }, }; - const encoder = new Encoder("video/hd", { - enabled: true, + const enabled = new Signal(true); + const encoder = new Encoder("video", { + enabled, broadcast: { video: () => rendition, baseline: new Baseline() } as never, capture: capture as never, }); try { await settle(); - expect(encoder.out.catalog.peek()?.stalled).toBeUndefined(); + const before = encoder.out.catalog.peek(); + expect(before?.codedWidth).toBe(Catalog.u53(640)); + expect(before?.enabled).toBeUndefined(); - if (reason === "encoder lag") { - // Five frames at 30fps exceed three frame intervals of unaccepted capture. - for (let i = 0; i < 5; i++) { - controller.enqueue(new Frame(i * 33_333) as unknown as VideoFrame); - await new Promise((resolve) => setTimeout(resolve, 10)); - } - } else { - clock.mockReturnValue(200); - await new Promise((resolve) => setTimeout(resolve, 60)); + for (const millis of [0, 33, 66]) { + controller.enqueue(new Frame(millis * 1000) as unknown as VideoFrame); + await new Promise((resolve) => setTimeout(resolve, 5)); } + expect(encoded.filter((frame) => frame.black)).toEqual([]); + + enabled.set(false); + await settle(); + expect(encoded.filter((frame) => frame.black)).toEqual([ + { timestamp: 66_000 + 33_333, keyFrame: true, black: true }, + ]); + expect(encoder.out.active.peek()).toBe(false); + + // Muting releases the capture; the rendition keeps advertising what it last published. + source.set(undefined); + await settle(); + const disabled = encoder.out.catalog.peek(); + expect(disabled?.enabled).toBe(false); + expect(disabled?.codec).toBe(before?.codec); + expect(disabled?.codedWidth).toBe(before?.codedWidth); + + source.set({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }); + enabled.set(true); await settle(); - expect(encoder.out.catalog.peek()?.stalled).toBe(true); + expect(encoder.out.catalog.peek()?.enabled).toBeUndefined(); + expect(encoded.filter((frame) => frame.black)).toHaveLength(1); } finally { encoder.close(); fanout.close(); - clock.mockRestore(); + track.close(); + if (original) Object.defineProperty(globalThis, "VideoEncoder", original); + else Reflect.deleteProperty(globalThis, "VideoEncoder"); + if (videoFrame) Object.defineProperty(globalThis, "VideoFrame", videoFrame); + else Reflect.deleteProperty(globalThis, "VideoFrame"); } }); diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index b690588b58..8f7a804aac 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -63,8 +63,9 @@ export interface Config { // Signals the encoder reads. export type EncoderInput = { - // Whether to publish (and encode) this rendition. Defaults to true. When false the rendition drops out of the - // catalog and stops encoding, but stays registered so a subscriber still gets an idle track. + // Whether to encode this rendition. Defaults to true. When false it stops encoding but stays in the + // catalog with `enabled: false` and its last config, and stays registered so a subscriber still gets + // an idle track. enabled: Getter; // The broadcast to register the rendition on. Undefined resolves the config for a local preview @@ -86,7 +87,8 @@ export type EncoderProps = Inputs & { }; type EncoderOutput = { - // The catalog config published for this rendition, or undefined while disabled. + // The catalog config published for this rendition, `enabled: false` while disabled, or undefined + // until one resolves. catalog: Signal; // The resolved WebCodecs config (codec, bitrate, dimensions), available even with no subscriber. // Exposed so a local preview can re-encode with identical settings to mirror the wire output. @@ -166,12 +168,11 @@ export class Encoder { readonly settled: Computed; #signals = new Effect(); - #stalled = new Catalog.Stalled.Detector(); - #firstCaptured?: Time.Micro; - #lastCaptured?: Time.Micro; - #lastAccepted?: Time.Micro; - #lastCaptureWall?: number; #estimator = new Estimator(); + // The estimator's jitter and delay, republished whenever either rises. + #estimate = new Signal({}); + // The last config published while enabled, which a disabled rendition keeps advertising. + #last?: Catalog.VideoConfig; constructor(name: string, props?: EncoderProps) { this.name = name; @@ -233,7 +234,7 @@ export class Encoder { const rendition = broadcast.video(this.name); effect.cleanup(() => rendition.close()); - // Publish the resolved catalog config; undefined (while disabled) drops it from the catalog. + // Publish the resolved catalog config; undefined (until one resolves) leaves it out of the catalog. effect.proxy(rendition.config, this.out.catalog); // Encode only while enabled and a subscriber is attached (the demand gate). @@ -241,10 +242,7 @@ export class Encoder { const enabled = effect.get(this.in.enabled); const track = effect.get(rendition.track); effect.set(this.#out.active, enabled && !!track, false); - if (!enabled || !track) { - this.#observe({ demand: false, idle: true }); - return; - } + if (!enabled || !track) return; this.#encode(track, broadcast.baseline, effect); }); @@ -279,22 +277,9 @@ export class Encoder { // Encode captured frames into the track producer, reconfiguring when the resolved config changes. #encode(track: Moq.Track.Producer, baseline: Baseline, effect: Effect): void { const capture = effect.get(this.in.capture); - if (!capture) { - this.#observe({ demand: true, idle: true }); - return; - } - - this.#observe({ demand: false, idle: true }); - this.#lastCaptureWall = performance.now(); + if (!capture) return; const producer = new Container.Legacy.Producer(track, new Container.Legacy.Format("video")); - // The broadcast owns this static track across demand gaps. When demand disappears, close the - // current group, marking the break so a later subscriber resumes on the same track without - // the pre-gap group reading as live. A fatal encoder error still aborts the track through - // producer.close(err) below. - effect.cleanup(() => { - if (track.closed.peek() === undefined) producer.discontinuity(); - }); let lastKeyframe: Time.Micro | undefined; let lastEncoded: Time.Micro | undefined; @@ -315,24 +300,23 @@ export class Encoder { producer.encode(frame, frame.timestamp as Time.Micro, key); if (this.#estimator.flush(frame.timestamp, baseline)) { - const catalog = this.#out.catalog.peek(); - if (catalog) this.#out.catalog.set({ ...catalog, ...this.#estimator.estimate }); + this.#estimate.set({ ...this.#estimator.estimate }); } - this.#lastAccepted = frame.timestamp as Time.Micro; - this.#observe({ demand: true, idle: false, frame: true }); }, error: (err: Error) => { producer.close(err); }, }); - effect.cleanup(() => encoder.close()); - + // What the encoder was last configured with. The resolved config clears on disable, before + // the black keyframe below is encoded. + let configured: VideoEncoderConfig | undefined; effect.run((effect) => { const config = effect.get(this.out.resolved); if (!config) return; encoder.configure(config); + configured = config; }); effect.run((effect) => { @@ -368,11 +352,6 @@ export class Encoder { if (frame.timestamp - lastEncoded < minGap - minGap / 2) continue; } lastEncoded = frame.timestamp as Time.Micro; - const captured = frame.timestamp as Time.Micro; - this.#firstCaptured ??= captured; - this.#lastCaptured = captured; - this.#lastCaptureWall = performance.now(); - this.#observe({ demand: true, idle: false }); const interval = config?.keyframeInterval ?? Time.Milli.fromSecond(2 as Time.Second); @@ -396,49 +375,52 @@ export class Encoder { } }); }); - }); - effect.interval(() => this.#observe({ demand: true, idle: false }), 50); + // Teardown is ours rather than a cleanup, since disabling first encodes one black keyframe and + // has to wait for it. A viewer that predates `enabled` keeps selecting a disabled rendition, + // and shows black instead of a frozen picture. + await new Promise((resolve) => effect.abort.addEventListener("abort", resolve, { once: true })); + try { + const config = configured; + if ( + !this.in.enabled.peek() && + config && + lastEncoded !== undefined && + encoder.state === "configured" && + track.closed.peek() === undefined + ) { + // One frame interval after the last picture, in whole microseconds like any VideoFrame. + const interval = Time.Micro.fromSecond((1 / (config.framerate ?? 30)) as Time.Second); + const frame = black(config, Math.round(lastEncoded + interval)); + try { + encoder.encode(frame, { keyFrame: true }); + } finally { + frame.close(); + } + await encoder.flush(); + } + } finally { + if (encoder.state !== "closed") encoder.close(); + // The broadcast owns this static track across demand gaps. Close the current group, marking + // the break so a later subscriber resumes on the same track without the pre-gap group reading + // as live. A fatal encoder error still aborts the track through producer.close(err) above. + if (track.closed.peek() === undefined) producer.discontinuity(); + } + }); } - #observe(state: { demand: boolean; idle: boolean; frame?: boolean }): void { - if (state.idle) { - this.#firstCaptured = undefined; - this.#lastCaptured = undefined; - this.#lastAccepted = undefined; - this.#lastCaptureWall = undefined; - } - const catalog = this.#out.catalog.peek(); - const mediaLag = ((): Time.Micro => { - if (this.#lastCaptured === undefined || this.#firstCaptured === undefined) return 0 as Time.Micro; - const accepted = this.#lastAccepted ?? this.#firstCaptured; - return Math.max(0, this.#lastCaptured - accepted) as Time.Micro; - })(); - const quiet = - this.#lastCaptureWall === undefined - ? (0 as Time.Micro) - : Time.Micro.fromMilli((performance.now() - this.#lastCaptureWall) as Time.Milli); - if ( - !this.#stalled.observe({ - frame: state.frame ?? false, - mediaLag, - quiet, - interval: Catalog.Stalled.intervalFromFps(catalog?.framerate ?? this.out.resolved.peek()?.framerate), - demand: state.demand, - idle: state.idle, - }) - ) { + // Publishes the catalog for the configured settings. A disabled rendition keeps its last config, + // since muting also releases the capture it would resolve a new one from. + #runCatalog(effect: Effect): void { + const enabled = effect.get(this.in.enabled); + const estimate = effect.get(this.#estimate); + if (!enabled) { + effect.set(this.#out.catalog, this.#last && { ...this.#last, ...estimate, enabled: false }); return; } - if (!catalog) return; - this.#out.catalog.set({ ...catalog, stalled: this.#stalled.flag() }); - } - // Returns the catalog for the configured settings, or undefined while disabled / unresolved. - #runCatalog(effect: Effect): void { - const enabled = effect.get(this.in.enabled); const live = effect.get(this.#live); - if (!enabled || !live) { + if (!live) { effect.set(this.#out.catalog, undefined); return; } @@ -454,10 +436,10 @@ export class Encoder { codedHeight: Catalog.u53(config.height), optimizeForLatency: true, container: { kind: "legacy" } as const, - ...this.#estimator.estimate, - stalled: this.#stalled.flag(), + ...estimate, }; + this.#last = catalog; effect.set(this.#out.catalog, catalog); } @@ -722,6 +704,16 @@ function ceiling(codec: string, dimensions: { width: number; height: number }, t return capped; } +// One black I420 picture the size of `config`, in limited range like most encoders expect. +function black(config: VideoEncoderConfig, timestamp: number): VideoFrame { + const { width, height } = config; + const luma = width * height; + const data = new Uint8Array((luma * 3) / 2); + data.fill(0x10, 0, luma); + data.fill(0x80, luma); + return new VideoFrame(data, { format: "I420", codedWidth: width, codedHeight: height, timestamp }); +} + // Encode one frame with a throwaway encoder and return the codec string it reports, like // moq-video's `Config::probe`. The encoder picks the profile and level it actually writes, so reading // them back lets the rendition be advertised before a subscriber starts the real encoder, without a diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index a3858d2f05..4deddee37b 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -219,6 +219,13 @@ async function play(initial: Delay) { play, // Remove the rendition from the catalog, then restore it. remove: () => catalog.set({}), + // Keep the rendition in the catalog with `enabled: false`, as a muted publisher does. + disable: () => + catalog.set({ + audio: { renditions: { audio: Catalog.AudioConfigSchema.parse({ ...audio, enabled: false }) } }, + }), + // The rendition the source selected. + track: () => source.out.track.peek(), restore: (patch: Record = {}) => catalog.set({ audio: { renditions: { audio: Catalog.AudioConfigSchema.parse({ ...audio, ...patch }) } } }), // Serve another broadcast from the next subscription, its timeline starting over. @@ -353,6 +360,30 @@ describe("Decoder across a rendition's absence", () => { } }); + it("deselects a disabled rendition and keeps one graph across disable and enable", async () => { + const playback = await play(Time.Milli(100)); + try { + await playback.play(12); + expect(playback.track()).toBe("audio"); + + playback.disable(); + await microtasks(); + expect(playback.track()).toBeUndefined(); + expect(contexts).toHaveLength(1); + + playback.restore(); + await microtasks(); + expect(playback.track()).toBe("audio"); + await playback.play(6); + + expect(contexts).toHaveLength(1); + expect(contexts[0].calls.close).toBe(0); + expect(playback.subscriptions()).toBe(2); + } finally { + playback.close(); + } + }); + it("plays a different broadcast whose timeline starts behind the playhead", async () => { const playback = await play(Time.Milli(100)); try { diff --git a/js/watch/src/audio/source.ts b/js/watch/src/audio/source.ts index 4791946583..9f83e0e008 100644 --- a/js/watch/src/audio/source.ts +++ b/js/watch/src/audio/source.ts @@ -104,7 +104,10 @@ export class Source { } #runSelected(effect: Effect): void { - const available = effect.get(this.#out.available); + // A disabled rendition has no frames coming, so it is never selected, even by name. + const available = Object.fromEntries( + Object.entries(effect.get(this.#out.available)).filter(([, config]) => config.enabled !== false), + ); if (Object.keys(available).length === 0) return; const target = effect.get(this.in.target); diff --git a/js/watch/src/msf.test.ts b/js/watch/src/msf.test.ts index 2a58e4d815..0b8ab50cda 100644 --- a/js/watch/src/msf.test.ts +++ b/js/watch/src/msf.test.ts @@ -62,22 +62,6 @@ test("rounds a fractional MSF delay up, and drops zero", () => { expect(hang.audio?.renditions.early?.delay).toBeUndefined(); }); -test("preserves stalled video renditions", () => { - const catalog: Msf.Catalog = { - tracks: [ - { - name: "video", - packaging: "loc", - role: "video", - codec: "vp09.00.10.08", - stalled: true, - }, - ], - }; - - expect(toHang(catalog).video?.renditions.video?.stalled).toBe(true); -}); - test("keeps loc packaging as the loc container", () => { const catalog: Msf.Catalog = { tracks: [ diff --git a/js/watch/src/msf.ts b/js/watch/src/msf.ts index a78a396ec5..d084293f3c 100644 --- a/js/watch/src/msf.ts +++ b/js/watch/src/msf.ts @@ -67,7 +67,6 @@ function toVideoConfig(track: Msf.Track): Catalog.VideoConfig | undefined { codedHeight: track.height != null ? u53(track.height) : undefined, framerate: track.framerate, bitrate: track.bitrate != null ? u53(track.bitrate) : undefined, - stalled: track.stalled, jitter: track.jitter != null ? u53(track.jitter) : undefined, delay: track.delay != null ? delayMillis(track.delay) : undefined, }; diff --git a/js/watch/src/video/config.test.ts b/js/watch/src/video/config.test.ts index feb8dcf0b7..38e0533fb7 100644 --- a/js/watch/src/video/config.test.ts +++ b/js/watch/src/video/config.test.ts @@ -31,7 +31,7 @@ test("metadata changes do not change the playback identity", async () => { rendition.set( config({ bitrate: 2_000_000, - stalled: true, + enabled: false, codedWidth: 1280, codedHeight: 720, framerate: 30, diff --git a/js/watch/src/video/source.test.ts b/js/watch/src/video/source.test.ts index 869a8930f6..acb946f074 100644 --- a/js/watch/src/video/source.test.ts +++ b/js/watch/src/video/source.test.ts @@ -173,8 +173,8 @@ describe("Source error signal", () => { }); }); -describe("Source stalled rendition selection", () => { - it("moves to an unstalled rendition when the selected one is throttled", async () => { +describe("Source disabled rendition selection", () => { + it("moves to an enabled rendition when the selected one is disabled, and back", async () => { const state = mutableBroadcast({ low: config("avc1.64001e", { bitrate: 1_000_000 }), high: config("avc1.640028", { bitrate: 2_000_000 }), @@ -191,56 +191,67 @@ describe("Source stalled rendition selection", () => { video: { renditions: { low: config("avc1.64001e", { bitrate: 1_000_000 }), - high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), + high: config("avc1.640028", { bitrate: 2_000_000, enabled: false }), }, }, }); await settle(); expect(source.out.track.peek()).toBe("low"); + + state.catalog.set({ + video: { + renditions: { + low: config("avc1.64001e", { bitrate: 1_000_000 }), + high: config("avc1.640028", { bitrate: 2_000_000 }), + }, + }, + }); + await settle(); + expect(source.out.track.peek()).toBe("high"); source.close(); }); - it("keeps a stalled manual target while an unstalled rendition exists", async () => { + it("never selects a disabled manual target", async () => { const source = new Source({ broadcast: mockBroadcast({ low: config("avc1.64001e", { bitrate: 1_000_000 }), - high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), + high: config("avc1.640028", { bitrate: 2_000_000, enabled: false }), }), target: { name: "high" }, supported: async () => true, }); await settle(); - expect(source.out.track.peek()).toBe("high"); - expect(Object.keys(source.out.available.peek())).toEqual(["low", "high"]); + expect(source.out.track.peek()).toBe("low"); source.close(); }); - it("selects the lowest rendition when every supported rendition is stalled", async () => { + it("selects nothing when every rendition is disabled", async () => { const source = new Source({ broadcast: mockBroadcast({ - high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), - low: config("avc1.64001e", { bitrate: 1_000_000, stalled: true }), + high: config("avc1.640028", { bitrate: 2_000_000, enabled: false }), + low: config("avc1.64001e", { bitrate: 1_000_000, enabled: false }), }), supported: async () => true, }); await settle(); - expect(source.out.track.peek()).toBe("low"); + expect(source.out.track.peek()).toBeUndefined(); + expect(source.out.config.peek()).toBeUndefined(); source.close(); }); - it("uses coded dimensions when stalled renditions omit bitrate", async () => { + it("ignores a legacy stalled flag", async () => { const source = new Source({ broadcast: mockBroadcast({ - high: config("avc1.640028", { codedWidth: 1920, codedHeight: 1080, stalled: true }), - low: config("avc1.64001e", { codedWidth: 854, codedHeight: 480, stalled: true }), + low: config("avc1.64001e", { bitrate: 1_000_000 }), + high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), }), supported: async () => true, }); await settle(); - expect(source.out.track.peek()).toBe("low"); + expect(source.out.track.peek()).toBe("high"); source.close(); }); @@ -265,7 +276,7 @@ describe("Source stalled rendition selection", () => { renditions: { high: config("avc1.640028", { bitrate: 1_500_000, - stalled: true, + label: "Main camera", codedWidth: 1280, codedHeight: 720, }), @@ -276,7 +287,7 @@ describe("Source stalled rendition selection", () => { expect(probes).toBe(1); expect(Number(source.out.config.peek()?.bitrate)).toBe(1_500_000); - expect(source.out.config.peek()?.stalled).toBe(true); + expect(source.out.config.peek()?.label).toBe("Main camera"); source.close(); }); diff --git a/js/watch/src/video/source.ts b/js/watch/src/video/source.ts index b6e4ac831e..9448bf56fa 100644 --- a/js/watch/src/video/source.ts +++ b/js/watch/src/video/source.ts @@ -213,16 +213,9 @@ function bestRendition(entries: [string, Catalog.VideoConfig][]): string { return best[0]; } -/** Return unstalled renditions, or the lowest bitrate or resolution when every option is stalled. */ -function selectableRenditions(renditions: Record): Record { - const active = Object.entries(renditions).filter(([, config]) => !config.stalled); - if (active.length > 0) return Object.fromEntries(active); - - const entries = Object.entries(renditions); - if (entries.length === 0) return {}; - const byRate = byBitrate(0)(entries); - const lowest = byRate.length === 1 ? byRate[0] : byDimensions(0, 0)(entries)[0]; - return { [lowest]: renditions[lowest] }; +/** Return the renditions that may be selected: a disabled one has no frames coming. */ +function enabledRenditions(renditions: Record): Record { + return Object.fromEntries(Object.entries(renditions).filter(([, config]) => config.enabled !== false)); } /** @@ -335,19 +328,16 @@ export class Source { } #runSelected(effect: Effect): void { - const supported = effect.get(this.#out.available); + const available = enabledRenditions(effect.get(this.#out.available)); const target = effect.get(this.in.target); - // A manual choice stays selected while stalled. `stalled` steers automatic adaptation; it - // must not silently override an explicit user selection. - if (target?.name && target.name in supported) { - const config = supported[target.name]; + if (target?.name && target.name in available) { + const config = available[target.name]; effect.set(this.#out.track, target.name); effect.set(this.#out.config, config); return; } - const available = selectableRenditions(supported); if (Object.keys(available).length === 0) return; // Auto-select: use recv bandwidth if no explicit bitrate target. diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 3d237b19e4..472c7bf554 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -117,7 +117,7 @@ typealias Datagram = uniffi.moq.MoqDatagram typealias Frame = uniffi.moq.MoqFrame /** A media [Frame] whose keyframe flag marks group starts or video keyframes; audio flags only group starts. */ typealias MediaFrame = uniffi.moq.MoqMediaFrame -/** The catalog description of a video track, including whether the publisher recommends temporarily avoiding it. */ +/** The catalog description of a video track, including whether it is enabled (a disabled one has no frames coming). */ typealias Video = uniffi.moq.MoqVideo /** Caller-provided catalog fields for a video track. */ typealias VideoHint = uniffi.moq.MoqVideoHint @@ -135,7 +135,7 @@ typealias AudioInit = uniffi.moq.MoqAudioInit typealias VideoInit = uniffi.moq.MoqVideoInit /** A container format and its leading bytes. */ typealias ContainerInit = uniffi.moq.MoqContainerInit -/** The catalog description of an audio track: codec, sample rate, channels, and container. */ +/** The catalog description of an audio track: codec, sample rate, channels, whether it is enabled, and container. */ typealias Audio = uniffi.moq.MoqAudio /** A width and height pair, in pixels. */ typealias Dimensions = uniffi.moq.MoqDimensions diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 69339fe6ce..4b9e7af7de 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -215,8 +215,8 @@ Every handle whose cleanup is `cancel()` is an async context manager, so exiting - **`Frame`**. `.payload: bytes`, `.timestamp_us: int`. The unit of every write and every raw read. - **`MediaFrame`**. `.payload: bytes`, `.timestamp_us: int`, `.keyframe: bool`. Returned by media subscriptions. `keyframe` marks a group start or video keyframe; for audio it is true only at a group start. - **`Datagram`**. `.sequence: int`, `.timestamp_us: int`, `.payload: bytes`. Delivered only on datagram-capable transports with lite-05 or newer moq-lite, or moq-transport. -- **`Audio`**. `.codec`, `.sample_rate`, `.channel_count`, `.bitrate`, `.description`. -- **`Video`**. `.codec`, `.coded: Dimensions`, `.display_aspect`, `.bitrate`, `.stalled`, `.framerate`, `.description`. A true `.stalled` recommends temporarily avoiding the rendition without making it unavailable. +- **`Audio`**. `.codec`, `.sample_rate`, `.channel_count`, `.bitrate`, `.enabled`, `.description`. +- **`Video`**. `.codec`, `.coded: Dimensions`, `.display_aspect`, `.bitrate`, `.enabled`, `.framerate`, `.description`. A false `.enabled` means no frames are coming, so don't select the rendition. - **`Subscription`**. Subscriber delivery preferences: priority, staleness, and optional group range. - **`TrackInfo`**. Publisher track properties: priority, cache window, and timescale. - **`Dimensions`**. `.width: int`, `.height: int`. diff --git a/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md b/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md index 1bc8f4232a..d49a72fb2c 100644 --- a/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md +++ b/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md @@ -7,8 +7,7 @@ share of the connection's estimate: a grant below the configured bitrate lowers the encoder at once, room coming back raises it gradually, and a link too small for the configured audio rate sheds audio bits instead of stalling. The reservation's ceiling stays the configured bitrate. A grant below the -codec floor disables the rendition (`enabled: false`, -[enabled flag](/quest/m1/catalog-enabled.md)) until it recovers. PCM keeps reserve-only +codec floor disables the rendition (catalog `enabled: false`) until it recovers. PCM keeps reserve-only usage: it claims its fixed rate and never reads the grant. The rate policy has one home shared by every sender, so audio and video back @@ -65,10 +64,6 @@ returns; a grant below the Opus floor disables the rendition, and recovered bandwidth enables it again even after the last viewer unsubscribed; a PCM Producer ignores every grant. -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - defines the `enabled` field a disabled rendition sets - ## Closes - [#2848](https://github.com/moq-dev/moq/issues/2848) - close this issue when the quest finishes diff --git a/quest/m1/README.md b/quest/m1/README.md index 5cd69305a1..493e2732cd 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -50,7 +50,6 @@ blocks. The quests that gated m0 lines moved under them. - [fMP4 init from the catalog](/quest/m1/fmp4-catalog-init.md) - fMP4 export writes avc3/hev1 entries from the catalog, so an Annex-B H.264 or H.265 init no longer waits for the first keyframe - [Delete the live marker](/quest/m1/announce-live-removal.md) - announce streams yield only route events; the `Live` marker and its hold machinery are gone - [WebTransport close backends](/quest/m1/wt-close-backends.md) - web-transport-noq and -quinn keep the session alive while closing, and iroh's client reads a capsule close -- [Enabled flag](/quest/m1/catalog-enabled.md) - one `enabled` rendition flag replaces `stalled` and pausing, and the flapping encoder-lag detector is deleted - [DTX timestamps](/quest/m1/opus-dtx-timestamps.md) - Opus DTX keeps the capture timeline, so voice enables it by default again - [Spent capture budget errors](/quest/m1/capture-exhausted-error.md) - a camera or microphone that runs out of retries sets `out.error` instead of failing silently - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it diff --git a/quest/m1/catalog-enabled.md b/quest/m1/catalog-enabled.md deleted file mode 100644 index 704af18d4f..0000000000 --- a/quest/m1/catalog-enabled.md +++ /dev/null @@ -1,66 +0,0 @@ -# [L] One enabled flag says whether to use a rendition - -## Goal - -A hang rendition carries `enabled`, default true. `enabled: false` means no -frames are coming, and a consumer MUST NOT select it. It replaces both the -encoder-lag `stalled` flag and a separate pause flag: whoever decides a -rendition should not be used (an app pausing it, or the bandwidth allocator) -writes the same field, and a publisher mute is a one-field catalog delta -instead of a removal. - -## Plan - -Decided (2026-10-04): - -- Wire: an optional `enabled` boolean on every rendition, audio and video, - written only when false. Writers stop writing `stalled`, and readers ignore - it: released publishers still flap it, and following it would keep #4772 - alive for them. Update - `drafts/draft-lcurley-moq-hang.md` (replacing the `stalled` section, with a - changelog entry, since `stalled` shipped in -03), `doc/concept/hang.md`, and - run `just drafts check`. MSF stops mirroring `stalled` and does not mirror - `enabled`. -- Delete the encoder-lag detector: `Stalled.Detector` in - `js/hang/src/catalog/stalled.ts` and `rs/hang/src/catalog/stalled.rs`, its - producers in `js/publish/src/video/encoder.ts` and - `rs/moq-mux/src/codec/video.rs`, and its importer hooks. It is what flaps - (#4772, #4776): frame-count hysteresis, plus clearing on demand loss. No - replacement adapts an overloaded encoder; that waits for a consumer. -- Publisher pause: `enabled` false on `@moq/publish` keeps the rendition in - the catalog with `enabled: false` instead of removing it, and the encoder - stops. The encoder keeps the last config it published, since muting also - releases the capture. Before disabling a video rendition it encodes one - black keyframe, so a viewer released before this field shows black rather - than a frozen picture. Older viewers otherwise keep selecting a disabled - rendition; that degradation is accepted and noted in the changelog. -- Viewer: `@moq/watch` deselects a disabled rendition, and its audio graph - outlives the absence as it does for a removed one. -- Bandwidth: a rendition is enabled only once its reservation is granted, and - disabled, with encoding stopped, when the grant falls below its floor, - until the grant recovers. A disabled rendition loses its subscribers, and - the allocator grants nothing to an undemanded track, so recovery evaluates - a hypothetical share against the current estimate instead of waiting for a - grant. The [ladder](/quest/m3/ladder/README.md) and - [audio grant following](/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md) - adopt that rule; this quest only defines the field. -- Everything else that names `stalled` moves to `enabled`: moq-transcode's - `inherit_stalled`, the moq-ffi field and its wrappers, and the C - `moq_consume_video_stalled`, which becomes `moq_consume_video_enabled`. - Selection in `js/watch/src/video/source.ts` filters on it, as - [rendition preference](/quest/m1/rendition-preference.md) expects. - -Tests: disabling an audio rendition publishes a catalog with the same -rendition and `enabled: false`, and a viewer deselects it and keeps one -AudioContext across disable and enable; a legacy `stalled: true` changes -nothing; nothing in the tree writes `stalled`. - -## Closes - -- [#4772](https://github.com/moq-dev/moq/issues/4772) - close this issue when the quest finishes -- [#4776](https://github.com/moq-dev/moq/issues/4776) - close this issue when the quest finishes - -## Related - -- [Ladder](/quest/m3/ladder/README.md) - disables a rung its grant cannot sustain -- [Rendition preference](/quest/m1/rendition-preference.md) - the other per-rendition selection field diff --git a/quest/m1/rendition-preference.md b/quest/m1/rendition-preference.md index 80ae4490e9..df2f4b09fc 100644 --- a/quest/m1/rendition-preference.md +++ b/quest/m1/rendition-preference.md @@ -41,7 +41,7 @@ Decided in planning interviews on 2026-10-01: - Video only. Optional on the wire, omitted when 0 (as `enabled` is omitted at its default). Additive, so older players ignore it. - Selection order in `js/watch/src/video/source.ts`: drop disabled - renditions ([enabled flag](/quest/m1/catalog-enabled.md)), then decode + renditions (catalog `enabled: false`), then decode support, then keep the highest preference among supported renditions, then the existing target and bitrate pick within what is left. Preference is about decodability only. @@ -71,10 +71,6 @@ Decided in planning interviews on 2026-10-01: the field. moq-transcode producing same-size codec fallbacks; the consumer publishes its own. -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - the `enabled` field selection filters on first - ## Related - [JS rendition ranking](/quest/m1/js-ranked.md) - mirrors `Video::ranked` in `@moq/hang`, which sorts by preference first once this quest lands diff --git a/quest/m3/ladder/README.md b/quest/m3/ladder/README.md index bf2572440f..2583b9a2d1 100644 --- a/quest/m3/ladder/README.md +++ b/quest/m3/ladder/README.md @@ -18,8 +18,7 @@ state. Deferred in the 2026-09-30 audit and moved to m3 in the 2026-10-05 audit: no named consumer for the publisher-side ladder. -The catalog and player half is the rendition `enabled` flag -([enabled flag](/quest/m1/catalog-enabled.md)), which replaces the `stalled` +The catalog and player half is the rendition `enabled` flag, which replaced the `stalled` state shipped in [moq#2865](https://github.com/moq-dev/moq/pull/2865): `enabled: false` means no frames are coming and a viewer must not select the rendition. Routing, decoder, and presentation identities are split so a diff --git a/quest/m3/ladder/controller.md b/quest/m3/ladder/controller.md index 92853267a9..d1342cc28a 100644 --- a/quest/m3/ladder/controller.md +++ b/quest/m3/ladder/controller.md @@ -73,7 +73,6 @@ priority where the subscriber's order wins, and a custom ladder order. ## Required - [Scope track priority](/quest/m1/track-priority-scope.md) - settles the priority ranking the controller assigns before the encoder side reads it -- [Enabled flag](/quest/m1/catalog-enabled.md) - the `enabled` field the controller publishes for rungs ## Related diff --git a/rs/hang/src/catalog/audio/mod.rs b/rs/hang/src/catalog/audio/mod.rs index 7c013c94a9..63f00fa266 100644 --- a/rs/hang/src/catalog/audio/mod.rs +++ b/rs/hang/src/catalog/audio/mod.rs @@ -98,6 +98,14 @@ pub struct AudioConfig { #[serde(default)] pub bitrate: Option, + /// Whether this rendition may be selected. When false, no frames are coming and a consumer + /// must not select it. Only written when false. + #[serde( + default = "crate::catalog::enabled_default", + skip_serializing_if = "crate::catalog::enabled_skip" + )] + pub enabled: bool, + // Some codecs include a description so the decoder can be initialized without extra data. // If not provided, there may be in-band metadata (marginally higher overhead). #[serde(default)] @@ -152,6 +160,7 @@ impl AudioConfig { sample_rate, channel_count, bitrate: None, + enabled: true, description: None, container: Container::default(), jitter: None, @@ -174,4 +183,16 @@ mod test { let decoded: AudioConfig = serde_json::from_value(encoded).expect("failed to decode"); assert_eq!(decoded.label.as_deref(), Some("English")); } + + #[test] + fn enabled_is_written_only_when_false() { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + assert!(serde_json::to_value(&config).unwrap().get("enabled").is_none()); + + config.enabled = false; + let encoded = serde_json::to_value(&config).expect("failed to encode"); + assert_eq!(encoded["enabled"], false); + let decoded: AudioConfig = serde_json::from_value(encoded).expect("failed to decode"); + assert!(!decoded.enabled); + } } diff --git a/rs/hang/src/catalog/mod.rs b/rs/hang/src/catalog/mod.rs index 109a973603..74a7a0f575 100644 --- a/rs/hang/src/catalog/mod.rs +++ b/rs/hang/src/catalog/mod.rs @@ -18,7 +18,6 @@ mod millis; mod mode; mod priority; mod root; -pub mod stalled; mod text; mod video; @@ -36,6 +35,16 @@ pub use root::*; pub use text::*; pub use video::*; +/// Serde default for a rendition's `enabled` field. +pub(crate) fn enabled_default() -> bool { + true +} + +/// Skip serializing `enabled` unless it is false, so only a disabled rendition writes it. +pub(crate) fn enabled_skip(enabled: &bool) -> bool { + *enabled +} + pub(crate) fn deserialize_timescale_or_default<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 540b928748..27b9000f5c 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -351,7 +351,7 @@ mod test { display_aspect_width: None, display_aspect_height: None, bitrate: None, - stalled: None, + enabled: true, framerate: None, optimize_for_latency: None, container: Container::Legacy, @@ -370,6 +370,7 @@ mod test { sample_rate: 48_000, channel_count: 2, bitrate: None, + enabled: true, description: None, container: Container::Legacy, jitter: Some(std::time::Duration::from_millis(40)), diff --git a/rs/hang/src/catalog/stalled.rs b/rs/hang/src/catalog/stalled.rs deleted file mode 100644 index ed2ed927cf..0000000000 --- a/rs/hang/src/catalog/stalled.rs +++ /dev/null @@ -1,297 +0,0 @@ -//! Publisher-side hysteresis for the hang catalog `stalled` flag. -//! -//! The wire already carries [`super::VideoConfig::stalled`]; this is the one -//! detector every first-party catalog producer uses to decide when to set it. -//! A rendition is stalled while demand is active and it is not keeping up: the -//! newest source frame has pulled more than a few frame intervals ahead of the -//! newest frame handed to the transport, or the source has gone quiet for that -//! long. An idle broadcast (camera released, no source) is never stalled. -//! -//! The flag flips through an ordinary catalog update, not per frame: it is set -//! once lag exceeds the threshold and cleared only after a run of on-time -//! frames. - -use std::time::Duration; - -/// How many frame intervals of lag mark a rendition stalled. -pub const SET_INTERVALS: u32 = 3; - -/// Consecutive on-time frames required to clear a stall. -pub const CLEAR_FRAMES: u32 = 3; - -/// Frame interval used when the catalog does not advertise a framerate. -pub const DEFAULT_INTERVAL: Duration = Duration::from_millis(33); - -/// One observation of a rendition's source versus what the transport has accepted. -#[derive(Clone, Copy, Debug)] -pub struct Sample { - /// This observation completes a newly accepted frame. - pub frame: bool, - /// Newest captured/source timestamp minus newest timestamp handed to the transport. - pub media_lag: Duration, - /// Wall time since the source last delivered a frame for this rendition. - pub quiet: Duration, - /// One frame interval. The set threshold is [`SET_INTERVALS`] of these. - pub interval: Duration, - /// Someone is subscribed to this rendition. - pub demand: bool, - /// Camera released / no live source. Never stalled. - pub idle: bool, -} - -/// Hysteresis around the catalog `stalled` bit. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Detector { - on: bool, - recover: u32, -} - -impl Detector { - /// A detector that starts unstalled. - pub fn new() -> Self { - Self::default() - } - - /// Whether the rendition is currently stalled. - pub fn get(&self) -> bool { - self.on - } - - /// The catalog value: `Some(true)` while stalled, omitted otherwise. - pub fn flag(&self) -> Option { - self.on.then_some(true) - } - - /// Feed one sample. Returns whether the catalog flag changed. - pub fn observe(&mut self, sample: Sample) -> bool { - if sample.idle || !sample.demand { - return self.clear(); - } - - let interval = interval(sample.interval); - let lag = sample.media_lag.max(sample.quiet); - let over = lag > interval.saturating_mul(SET_INTERVALS); - let on_time = lag <= interval; - - if !self.on { - if !over { - return false; - } - self.on = true; - self.recover = 0; - return true; - } - - if !on_time { - self.recover = 0; - return false; - } - - if !sample.frame { - return false; - } - self.recover = self.recover.saturating_add(1); - if self.recover < CLEAR_FRAMES { - return false; - } - self.clear() - } - - fn clear(&mut self) -> bool { - self.recover = 0; - if !self.on { - return false; - } - self.on = false; - true - } -} - -/// A positive frame interval, falling back to [`DEFAULT_INTERVAL`]. -pub fn interval(value: Duration) -> Duration { - if value.is_zero() { DEFAULT_INTERVAL } else { value } -} - -/// Frame interval implied by a catalog framerate, or [`DEFAULT_INTERVAL`]. -pub fn interval_from_fps(fps: Option) -> Duration { - let Some(fps) = fps.filter(|fps| fps.is_finite() && *fps > 0.0) else { - return DEFAULT_INTERVAL; - }; - Duration::try_from_secs_f64(1.0 / fps) - .ok() - .filter(|value| !value.is_zero()) - .unwrap_or(DEFAULT_INTERVAL) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sample(lag: Duration) -> Sample { - Sample { - frame: true, - media_lag: lag, - quiet: Duration::ZERO, - interval: Duration::from_millis(33), - demand: true, - idle: false, - } - } - - fn quiet(gap: Duration) -> Sample { - Sample { - frame: true, - media_lag: Duration::ZERO, - quiet: gap, - interval: Duration::from_millis(33), - demand: true, - idle: false, - } - } - - #[test] - fn idle_is_never_stalled() { - let mut stalled = Detector::new(); - assert!(!stalled.observe(Sample { - frame: true, - idle: true, - demand: true, - media_lag: Duration::from_secs(10), - quiet: Duration::from_secs(10), - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn no_demand_is_never_stalled() { - let mut stalled = Detector::new(); - assert!(!stalled.observe(Sample { - frame: true, - demand: false, - idle: false, - media_lag: Duration::from_secs(10), - quiet: Duration::from_secs(10), - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn lag_past_the_threshold_sets_the_flag() { - let mut stalled = Detector::new(); - assert!(!stalled.observe(sample(Duration::from_millis(99)))); - assert!(stalled.observe(sample(Duration::from_millis(100)))); - assert_eq!(stalled.flag(), Some(true)); - } - - #[test] - fn a_quiet_source_sets_the_flag() { - let mut stalled = Detector::new(); - assert!(stalled.observe(quiet(Duration::from_millis(100)))); - assert!(stalled.get()); - } - - #[test] - fn clearing_needs_a_run_of_on_time_frames() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(stalled.get()); - - assert!(stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.get()); - assert_eq!(stalled.flag(), None); - } - - #[test] - fn a_late_frame_resets_recovery() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(200)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(stalled.get()); - assert!(stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.get()); - } - - #[test] - fn dropping_demand_clears_immediately() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - assert!(stalled.observe(Sample { - frame: true, - demand: false, - idle: false, - media_lag: Duration::from_millis(200), - quiet: Duration::ZERO, - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn releasing_the_camera_clears_immediately() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - assert!(stalled.observe(Sample { - frame: true, - idle: true, - demand: true, - media_lag: Duration::from_millis(200), - quiet: Duration::ZERO, - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn on_time_frames_do_not_flip_a_healthy_rendition() { - let mut stalled = Detector::new(); - for _ in 0..10 { - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - } - assert!(!stalled.get()); - } - - #[test] - fn a_zero_interval_uses_the_default() { - let mut stalled = Detector::new(); - assert!(stalled.observe(Sample { - frame: true, - media_lag: DEFAULT_INTERVAL.saturating_mul(SET_INTERVALS) + Duration::from_millis(1), - quiet: Duration::ZERO, - interval: Duration::ZERO, - demand: true, - idle: false, - })); - assert!(stalled.get()); - } - - #[test] - fn polls_do_not_count_as_recovery_frames() { - let mut stalled = Detector::new(); - stalled.observe(sample(Duration::from_millis(200))); - let mut poll = sample(Duration::ZERO); - poll.frame = false; - for _ in 0..10 { - assert!(!stalled.observe(poll)); - } - assert!(stalled.get()); - } - - #[test] - fn interval_from_fps_falls_back() { - assert_eq!(interval_from_fps(Some(f64::MAX)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(f64::MIN_POSITIVE)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(None), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(0.0)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(f64::NAN)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(50.0)), Duration::from_millis(20)); - } -} diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index f4bba68565..7f2a80d7ec 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -209,12 +209,13 @@ pub struct VideoConfig { #[serde(default)] pub bitrate: Option, - /// Whether the publisher recommends temporarily avoiding this rendition. - /// - /// The track remains available. Consumers may still select it when no - /// unstalled rendition is suitable. - #[serde(default)] - pub stalled: Option, + /// Whether this rendition may be selected. When false, no frames are coming and a consumer + /// must not select it. Only written when false. + #[serde( + default = "crate::catalog::enabled_default", + skip_serializing_if = "crate::catalog::enabled_skip" + )] + pub enabled: bool, /// The frame rate of the video track, if known. #[serde(default)] @@ -280,7 +281,7 @@ impl VideoConfig { display_aspect_width: None, display_aspect_height: None, bitrate: None, - stalled: None, + enabled: true, framerate: None, optimize_for_latency: None, container: Container::default(), @@ -370,17 +371,25 @@ mod test { } #[test] - fn stalled_is_optional_and_round_trips() { + fn enabled_is_written_only_when_false() { let mut config = VideoConfig::new(VideoCodec::VP8); let encoded = serde_json::to_value(&config).expect("failed to encode"); - assert!(encoded.get("stalled").is_none()); + assert!(encoded.get("enabled").is_none()); - config.stalled = Some(true); + config.enabled = false; let encoded = serde_json::to_value(&config).expect("failed to encode"); - assert_eq!(encoded["stalled"], true); + assert_eq!(encoded["enabled"], false); let decoded: VideoConfig = serde_json::from_value(encoded).expect("failed to decode"); - assert_eq!(decoded.stalled, Some(true)); + assert!(!decoded.enabled); + } + + #[test] + fn legacy_stalled_is_ignored() { + let json = serde_json::json!({ "codec": "vp8", "stalled": true }); + let config: VideoConfig = serde_json::from_value(json).expect("failed to decode"); + assert!(config.enabled); + assert!(serde_json::to_value(&config).unwrap().get("stalled").is_none()); } #[test] diff --git a/rs/moq-c/README.md b/rs/moq-c/README.md index 5d70b64d35..f1b61ea8a3 100644 --- a/rs/moq-c/README.md +++ b/rs/moq-c/README.md @@ -117,7 +117,7 @@ int32_t moq_consume_catalog(uint32_t broadcast, moq_status_callback on_catalog, int32_t moq_consume_catalog_cancel(uint32_t catalog); int32_t moq_consume_catalog_free(uint32_t catalog); int32_t moq_consume_video_config(uint32_t catalog, uint32_t index, moq_video_config *dst); -int32_t moq_consume_video_stalled(uint32_t catalog, uint32_t index, bool *dst); +int32_t moq_consume_video_enabled(uint32_t catalog, uint32_t index, bool *dst); int32_t moq_consume_audio_config(uint32_t catalog, uint32_t index, moq_audio_config *dst); // Consuming: Video diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index 4f74abdaf4..7ab9f03527 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -3286,25 +3286,25 @@ pub unsafe extern "C" fn moq_consume_video_config(catalog: u32, index: u32, dst: }) } -/// Query whether the publisher recommends temporarily avoiding a video rendition. +/// Query whether a video rendition may be selected. /// -/// The track remains available. A false value also covers catalogs that omit the -/// optional field. +/// False means no frames are coming and the rendition must not be selected. A catalog that +/// omits the optional field reads as true. /// /// Returns zero on success, or a negative code on failure. /// /// # Safety /// - The caller must ensure that `dst` points to properly aligned, writable storage for a `bool`. #[unsafe(no_mangle)] -pub unsafe extern "C" fn moq_consume_video_stalled(catalog: u32, index: u32, dst: *mut bool) -> i32 { +pub unsafe extern "C" fn moq_consume_video_enabled(catalog: u32, index: u32, dst: *mut bool) -> i32 { ffi::enter(move || { let catalog = ffi::parse_id(catalog)?; if dst.is_null() { return Err(Error::InvalidPointer); } - let stalled = State::lock().consume.video_stalled(catalog, index as usize)?; - unsafe { dst.write(stalled) }; + let enabled = State::lock().consume.video_enabled(catalog, index as usize)?; + unsafe { dst.write(enabled) }; Ok(()) }) } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index 01167c9c79..4803a03afb 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -270,8 +270,8 @@ impl Consume { Ok(()) } - /// Return whether the publisher recommends temporarily avoiding a video rendition. - pub fn video_stalled(&self, catalog: Id, index: usize) -> Result { + /// Return whether a video rendition may be selected. + pub fn video_enabled(&self, catalog: Id, index: usize) -> Result { let consume = self.catalog.get(catalog).ok_or(Error::CatalogNotFound)?; let (_, config) = consume .catalog @@ -280,7 +280,7 @@ impl Consume { .iter() .nth(index) .ok_or(Error::NoIndex)?; - Ok(config.stalled.unwrap_or(false)) + Ok(config.enabled) } /// Fill `dst` with the properties shared by every video rendition. diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index 00539f4ec1..3a074d6814 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -760,10 +760,10 @@ fn publish_catalog_roundtrip() { container: moq_container::default(), }; assert_eq!(unsafe { moq_publish_video_config(broadcast, &video) }, 0); - let stalled_video_name = "video-stalled"; - let stalled_video = moq_video_config { - name: stalled_video_name.as_ptr() as *const c_char, - name_len: stalled_video_name.len(), + let disabled_video_name = "video-disabled"; + let disabled_video = moq_video_config { + name: disabled_video_name.as_ptr() as *const c_char, + name_len: disabled_video_name.len(), label: std::ptr::null(), label_len: 0, codec: video_codec.as_ptr() as *const c_char, @@ -774,7 +774,7 @@ fn publish_catalog_roundtrip() { coded_height: height, container: moq_container::default(), }; - assert_eq!(unsafe { moq_publish_video_config(broadcast, &stalled_video) }, 0); + assert_eq!(unsafe { moq_publish_video_config(broadcast, &disabled_video) }, 0); { let mut state = State::lock(); let (_, catalog) = state.publish.pair_mut(Id::try_from(broadcast).unwrap()).unwrap(); @@ -783,9 +783,9 @@ fn publish_catalog_roundtrip() { .unwrap() .video .renditions - .get_mut(stalled_video_name) + .get_mut(disabled_video_name) .unwrap() - .stalled = Some(true); + .enabled = false; } let properties = moq_video_properties { display_width: 1080, @@ -851,30 +851,30 @@ fn publish_catalog_roundtrip() { ); assert_eq!(video_cfg.coded_width, 1920); assert_eq!(video_cfg.coded_height, 1080); - let mut stalled = std::mem::MaybeUninit::::uninit(); + let mut enabled = std::mem::MaybeUninit::::uninit(); assert_eq!( - unsafe { moq_consume_video_stalled(catalog_id, 0, stalled.as_mut_ptr()) }, + unsafe { moq_consume_video_enabled(catalog_id, 0, enabled.as_mut_ptr()) }, 0 ); - assert!(!unsafe { stalled.assume_init() }); + assert!(unsafe { enabled.assume_init() }); - let mut stalled = std::mem::MaybeUninit::::uninit(); + let mut enabled = std::mem::MaybeUninit::::uninit(); assert_eq!( - unsafe { moq_consume_video_stalled(catalog_id, 1, stalled.as_mut_ptr()) }, + unsafe { moq_consume_video_enabled(catalog_id, 1, enabled.as_mut_ptr()) }, 0 ); - assert!(unsafe { stalled.assume_init() }); + assert!(!unsafe { enabled.assume_init() }); assert_eq!( - unsafe { moq_consume_video_stalled(catalog_id, 0, std::ptr::null_mut()) }, + unsafe { moq_consume_video_enabled(catalog_id, 0, std::ptr::null_mut()) }, -6, - "null stalled pointer should return InvalidPointer (-6)" + "null enabled pointer should return InvalidPointer (-6)" ); assert_eq!( unsafe { moq_publish_video_remove( broadcast, - stalled_video_name.as_ptr() as *const c_char, - stalled_video_name.len(), + disabled_video_name.as_ptr() as *const c_char, + disabled_video_name.len(), ) }, 0 diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 22c3ba4742..193a176c6b 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -436,6 +436,7 @@ mod tests { sample_rate: 48_000, channel_count: 2, bitrate: None, + enabled: true, container: MoqContainer::Legacy, } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 5042c51083..2bbd26c7e5 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -91,9 +91,9 @@ pub struct MoqVideo { pub coded: Option, pub display_aspect: Option, pub bitrate: Option, - /// Whether the publisher recommends temporarily avoiding this rendition. - #[uniffi(default = false)] - pub stalled: bool, + /// Whether this rendition may be selected; when false, no frames are coming. + #[uniffi(default = true)] + pub enabled: bool, pub framerate: Option, pub container: MoqContainer, } @@ -112,6 +112,9 @@ pub struct MoqAudio { pub sample_rate: u32, pub channel_count: u32, pub bitrate: Option, + /// Whether this rendition may be selected; when false, no frames are coming. + #[uniffi(default = true)] + pub enabled: bool, pub container: MoqContainer, } @@ -379,7 +382,7 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog None, }, bitrate: config.bitrate, - stalled: config.stalled.unwrap_or(false), + enabled: config.enabled, framerate: config.framerate, container: MoqContainer::from_catalog(&config.container)?, }, @@ -402,6 +405,7 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog crate::media::MoqAudio { sample_rate: 48_000, channel_count: 2, bitrate: None, + enabled: true, container: MoqContainer::Legacy, } } diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 413ef5e5e1..d5f36d7245 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -730,7 +730,6 @@ fn video_config(catalog_video: crate::media::MoqVideo) -> Result VideoConfig { config.jitter = None; config.delay = None; config.label = None; - config.stalled = None; + config.enabled = true; // Publishers estimate it and republish once it settles. The rendition keeps the fMP4 // timescale it was built with, and the framerate is refreshed in place like the bitrate. config.framerate = None; @@ -205,6 +205,7 @@ fn normalize_audio(config: &AudioConfig) -> AudioConfig { config.jitter = None; config.delay = None; config.label = None; + config.enabled = true; config } diff --git a/rs/moq-msf/src/lib.rs b/rs/moq-msf/src/lib.rs index 83087ca201..9f7cf1fc31 100644 --- a/rs/moq-msf/src/lib.rs +++ b/rs/moq-msf/src/lib.rs @@ -144,11 +144,6 @@ pub struct Track { /// Bitrate in bits per second. pub bitrate: Option, - /// Whether the publisher recommends temporarily avoiding this track. - /// - /// This is a non-standard extension shared with the hang catalog. - pub stalled: Option, - /// Resolved base64 initialization data. /// /// On the wire this is carried indirectly through draft-01's `initDataList` + @@ -481,7 +476,6 @@ impl Track { samplerate: None, channel_config: None, bitrate: None, - stalled: None, init_data: None, init_ref: None, render_group: None, @@ -633,7 +627,6 @@ mod test { samplerate: None, channel_config: None, bitrate: Some(6_000_000), - stalled: Some(true), init_data: None, init_ref: None, render_group: Some(1), @@ -658,7 +651,6 @@ mod test { samplerate: Some(48_000), channel_config: Some("2".to_string()), bitrate: Some(128_000), - stalled: None, init_data: None, init_ref: None, render_group: Some(1), @@ -683,7 +675,6 @@ mod test { samplerate: None, channel_config: None, bitrate: Some(5_000_000), - stalled: None, init_data: None, init_ref: None, render_group: Some(1), @@ -714,7 +705,6 @@ mod test { assert!(track.get("maxObjSapStartingType").is_none()); assert!(track.get("jitter").is_none()); assert!(track.get("delay").is_none()); - assert_eq!(track["stalled"], true); } #[test] diff --git a/rs/moq-mux/src/catalog/msf/consumer.rs b/rs/moq-mux/src/catalog/msf/consumer.rs index 58374e78ec..320e70f5d0 100644 --- a/rs/moq-mux/src/catalog/msf/consumer.rs +++ b/rs/moq-mux/src/catalog/msf/consumer.rs @@ -233,7 +233,6 @@ fn video_config_from_msf(track: &moq_msf::Track) -> Result> config.coded_width = track.width; config.coded_height = track.height; config.bitrate = track.bitrate; - config.stalled = track.stalled; config.framerate = track.framerate; config.container = container; config.jitter = track.jitter; @@ -408,7 +407,6 @@ mod test { track.height = Some(1080); track.framerate = Some(30.0); track.bitrate = Some(5_000_000); - track.stalled = Some(true); track.init_data = init_data.map(str::to_string); track.render_group = Some(1); track @@ -446,7 +444,6 @@ mod test { assert_eq!(video.coded_height, Some(1080)); assert_eq!(video.framerate, Some(30.0)); assert_eq!(video.bitrate, Some(5_000_000)); - assert_eq!(video.stalled, Some(true)); } #[test] diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index ae3c14e2f4..1c39901f76 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -1033,7 +1033,6 @@ fn to_msf_media(catalog: &hang::Catalog) -> moq_msf::Catalog { track.height = config.coded_height; track.framerate = config.framerate; track.bitrate = config.bitrate; - track.stalled = config.stalled; track.init_data = init_data; track.render_group = Some(1); track.alt_group = if has_multiple_video { Some(1) } else { None }; @@ -1552,7 +1551,6 @@ mod test { video_config.coded_width = Some(1280); video_config.coded_height = Some(720); video_config.bitrate = Some(6_000_000); - video_config.stalled = Some(true); video_config.framerate = Some(30.0); video_config.container = Container::Legacy; @@ -1583,7 +1581,6 @@ mod test { assert_eq!(video.height, Some(720)); assert_eq!(video.framerate, Some(30.0)); assert_eq!(video.bitrate, Some(6_000_000)); - assert_eq!(video.stalled, Some(true)); assert!(video.init_data.is_none()); // H.264 may carry B-frames, so SAP starting type is 2 (leading pictures allowed). assert_eq!(video.max_grp_sap_starting_type, Some(2)); diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index a1292745f5..cf3a34be87 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -224,7 +224,7 @@ impl From for VideoHint { /// /// Total by construction: every field the hint can hold is taken from the config, so there is no /// per-field copy for a caller to forget. Fields with no hint slot (`broadcast`, `description`, - /// `stalled`) are set through the catalog directly. + /// `enabled`) are set through the catalog directly. fn from(config: hang::catalog::VideoConfig) -> Self { Self { label: config.label, diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index adabb67728..017d89ff7e 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -253,29 +253,10 @@ impl Import { // A pre-keyframe delta has no group to anchor it: the producer returns // MissingKeyframe, which a caller joining mid-stream skips. self.track.write(frame)?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; } Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// Publish split frames, resolving the config from the first keyframe's inline /// sequence header and refining the catalog jitter. pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index c5ce239469..b2fe8a3164 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -210,8 +210,6 @@ impl Import { } self.track.write(frame)?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; } Ok(()) } @@ -221,23 +219,6 @@ impl Import { pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { self.write_frames(frames) } - - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } } /// The catalog config a stream's codec bytes describe, resolved without publishing anything: an @@ -497,48 +478,4 @@ mod tests { "no config yet, so no catalog" ); } - - /// Encode lag past a few frame intervals marks the rendition stalled, and - /// releasing the source clears it. Demand is required; an idle import is not stalled. - #[tokio::test(start_paused = true)] - async fn encode_lag_marks_the_rendition_stalled() { - let sps: &[u8] = &[ - 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, - 0x01, 0xd4, 0xc0, 0x80, - ]; - let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; - let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; - let mut annexb = BytesMut::new(); - for nal in [sps, pps, idr] { - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(nal); - } - - let mut split = Split::new(); - let (track, catalog) = setup("video"); - let _demand = track.subscribe(None); - let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); - let pts = moq_net::Timestamp::from_micros(0).unwrap(); - let mut frames = split.decode(&annexb, pts).expect("split keyframe"); - frames.extend(split.flush(pts).expect("flush keyframe")); - import.decode(frames).expect("decode keyframe"); - assert_eq!( - catalog.snapshot().video.renditions.get("video").and_then(|c| c.stalled), - None - ); - - import - .observe_lag(std::time::Duration::from_millis(200)) - .expect("observe lag"); - assert_eq!( - catalog.snapshot().video.renditions.get("video").and_then(|c| c.stalled), - Some(true) - ); - - import.idle().expect("idle"); - assert_eq!( - catalog.snapshot().video.renditions.get("video").and_then(|c| c.stalled), - None - ); - } } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index a46634d931..f3755515dc 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -212,29 +212,10 @@ impl Import { // A pre-keyframe delta has no group to anchor it: the producer returns // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. self.track.write(frame)?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; } Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// Publish split frames, resolving the config from the first keyframe's inline /// SPS and refining the catalog jitter as it goes. pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 219da9d09d..58011bc49e 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -3,13 +3,8 @@ //! Every video importer resolves its [`VideoConfig`](hang::catalog::VideoConfig) lazily from the //! bitstream and re-publishes it whenever the stream reveals a change. [`Catalog`] owns the part of //! that which is identical across codecs: overlay the caller's [`VideoHint`], skip a publish -//! that matches the last one, and drive the shared [`hang::catalog::stalled::Detector`] detector so a -//! lagging rendition is marked in the catalog. [`Reorder`] is what a sequence header declares about -//! frame reordering, read by the exporters that author a decode clock. - -use std::time::{Duration, Instant}; - -use hang::catalog::stalled::{Detector, Sample}; +//! that matches the last one. [`Reorder`] is what a sequence header declares about frame +//! reordering, read by the exporters that author a decode clock. use crate::catalog::VideoHint; @@ -29,29 +24,20 @@ pub(crate) struct Reorder { /// The catalog-publishing state a video importer overlays onto every config it resolves. /// -/// Holds the caller's hint, the last config published (to dedupe re-publishes), and the -/// stalled detector. Not generic over the extension: none of that depends on it. +/// Holds the caller's hint and the last config published (to dedupe re-publishes). Not generic +/// over the extension: neither depends on it. pub(crate) struct Catalog { /// Overlaid onto every config, so a hinted field counts as supplied and is never overwritten by /// the rendition's detector. hint: VideoHint, /// The last config published, so an unchanged re-resolve doesn't re-mirror the rendition. last: Option, - stalled: Detector, - last_source: Option, - lag: Duration, } impl Catalog { /// Hold `hint` for every publish. pub(crate) fn new(hint: VideoHint) -> Self { - Self { - hint, - last: None, - stalled: Detector::new(), - last_source: None, - lag: Duration::ZERO, - } + Self { hint, last: None } } /// The config the hint alone resolves to, for importers that publish the catalog before parsing @@ -68,11 +54,9 @@ impl Catalog { /// Overlay the hint onto `config` and publish it to `rendition`, unless it matches the last /// publish. A changed config just re-mirrors the rendition; there are no fixed tracks to - /// reject a reconfiguration. The detector's current flag is applied so a bitstream republish - /// cannot clear a stall. + /// reject a reconfiguration. pub(crate) fn publish(&mut self, track: &mut Track, mut config: hang::catalog::VideoConfig) -> crate::Result<()> { self.hint.apply(&mut config); - config.stalled = self.stalled.flag(); if self.last.as_ref() == Some(&config) { return Ok(()); } @@ -81,124 +65,4 @@ impl Catalog { self.last = Some(config); Ok(()) } - - /// A frame was handed to the transport, completing one recovery observation. - pub(crate) fn on_frame(&mut self, track: &mut Track, demand: bool) -> crate::Result<()> { - self.last_source = Some(Instant::now()); - self.publish_stalled(track, demand, true, self.lag) - } - - /// Re-evaluate stall from silence: the source has not delivered since the last frame. - pub(crate) fn tick(&mut self, track: &mut Track, demand: bool) -> crate::Result<()> { - self.publish_stalled(track, demand, false, Duration::ZERO) - } - - /// Record extra delay (a slow encode) without treating it as a new source frame. - pub(crate) fn observe_lag(&mut self, track: &mut Track, demand: bool, lag: Duration) -> crate::Result<()> { - self.lag = lag; - self.publish_stalled(track, demand, false, lag) - } - - /// The source is gone (camera released). Never stalled. - pub(crate) fn idle(&mut self, track: &mut Track) -> crate::Result<()> { - self.last_source = None; - self.lag = Duration::ZERO; - self.publish_stalled(track, false, false, Duration::ZERO) - } - - fn publish_stalled( - &mut self, - track: &mut Track, - demand: bool, - frame: bool, - extra_lag: Duration, - ) -> crate::Result<()> { - if demand { - self.last_source.get_or_insert_with(Instant::now); - } else { - self.last_source = None; - } - let media_lag = extra_lag; - let quiet = self - .last_source - .map(|at| Instant::now().saturating_duration_since(at)) - .unwrap_or(Duration::ZERO); - let interval = hang::catalog::stalled::interval_from_fps(self.last.as_ref().and_then(|c| c.framerate)); - if !self.stalled.observe(Sample { - frame, - media_lag, - quiet, - interval, - demand, - idle: !demand, - }) { - return Ok(()); - } - let flag = self.stalled.flag(); - if let Some(last) = self.last.as_mut() { - last.stalled = flag; - } - let mut config = track.modify()?; - config.stalled = flag; - config.commit() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn quiet_startup_tracks_demand() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let reserved = catalog.reserve(); - let net = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let mut rendition = reserved - .video( - net, - crate::catalog::hang::Container::Legacy(crate::container::Kind::Video), - None, - ) - .unwrap(); - let mut state = Catalog::new(VideoHint::default()); - state - .publish( - &mut rendition, - hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8), - ) - .unwrap(); - - state.tick(&mut rendition, true).unwrap(); - let start = state - .last_source - .expect("demand starts the quiet clock before the first frame"); - state.last_source = Some(start - Duration::from_millis(200)); - state.tick(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, Some(true)); - - state.tick(&mut rendition, false).unwrap(); - assert!(state.last_source.is_none()); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, None); - state.tick(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, None); - - state - .observe_lag(&mut rendition, true, Duration::from_millis(200)) - .unwrap(); - state.on_frame(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, Some(true)); - state.observe_lag(&mut rendition, true, Duration::ZERO).unwrap(); - for _ in 0..2 { - state.on_frame(&mut rendition, true).unwrap(); - for _ in 0..10 { - state.tick(&mut rendition, true).unwrap(); - } - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, Some(true)); - } - state.on_frame(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, None); - } } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index 40bdb5c23b..dde79d1f31 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -88,29 +88,10 @@ impl Import { keyframe: header.keyframe, duration: None, })?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// A watch-only handle to this track's subscriber demand. pub fn demand(&self) -> moq_net::track::Demand { self.track.track().demand() diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index 477175b4ad..c873265943 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -88,29 +88,10 @@ impl Import { keyframe: header.keyframe, duration: None, })?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// A watch-only handle to this track's subscriber demand. pub fn demand(&self) -> moq_net::track::Demand { self.track.track().demand() diff --git a/rs/moq-mux/src/container/flv/import.rs b/rs/moq-mux/src/container/flv/import.rs index 363495b1a2..0e56bb97de 100644 --- a/rs/moq-mux/src/container/flv/import.rs +++ b/rs/moq-mux/src/container/flv/import.rs @@ -21,7 +21,6 @@ //! enhanced audio, and any other codec, are logged and dropped. use std::collections::BTreeMap; -use std::time::{Duration, Instant}; use anyhow::Context; use bytes::{Buf, Bytes, BytesMut}; @@ -87,8 +86,6 @@ pub struct Import { struct VideoStream { track: crate::container::Producer, config: VideoConfig, - stalled: hang::catalog::stalled::Detector, - last_source: Option, } /// The demuxed audio track plus its current catalog config. @@ -181,7 +178,6 @@ impl Import { } } - self.tick_stalled()?; Ok(()) } @@ -473,26 +469,16 @@ impl Import { // A media frame means every sequence header has arrived (FLV sends config before data), so // the track set is declared; release the reservation to publish. self.initial_reservation = None; - let written = { - let stream = self.video.get_mut(&track_id).expect("checked above"); - match stream.track.write(Frame { - timestamp, - duration: None, - payload: Bytes::copy_from_slice(data), - keyframe, - }) { - Ok(()) => { - stream.last_source = Some(Instant::now()); - true - } - Err(crate::Error::MissingKeyframe(_)) => false, - Err(e) => return Err(e.into()), - } - }; - if written { - self.publish_stalled(track_id, true)?; + let stream = self.video.get_mut(&track_id).expect("checked above"); + match stream.track.write(Frame { + timestamp, + duration: None, + payload: Bytes::copy_from_slice(data), + keyframe, + }) { + Ok(()) | Err(crate::Error::MissingKeyframe(_)) => Ok(()), + Err(e) => Err(e.into()), } - Ok(()) } /// Write one audio frame as its own group, so the relay can forward it immediately. @@ -541,8 +527,6 @@ impl Import { // site (the producer reports MissingKeyframe), so a mid-GOP join works. track: media, config, - stalled: hang::catalog::stalled::Detector::new(), - last_source: None, }, ); Ok(()) @@ -568,46 +552,6 @@ impl Import { Ok(()) } - fn tick_stalled(&mut self) -> anyhow::Result<()> { - let ids: Vec = self.video.keys().copied().collect(); - for id in ids { - self.publish_stalled(id, false)?; - } - Ok(()) - } - - fn publish_stalled(&mut self, track_id: u8, frame: bool) -> anyhow::Result<()> { - let Some(stream) = self.video.get_mut(&track_id) else { - return Ok(()); - }; - let demand = stream.track.demand().is_used(); - if demand { - stream.last_source.get_or_insert_with(Instant::now); - } else { - stream.last_source = None; - } - let quiet = stream - .last_source - .map(|at| Instant::now().saturating_duration_since(at)) - .unwrap_or(Duration::ZERO); - let interval = hang::catalog::stalled::interval_from_fps(stream.config.framerate); - if !stream.stalled.observe(hang::catalog::stalled::Sample { - frame, - media_lag: Duration::ZERO, - quiet, - interval, - demand, - idle: false, - }) { - return Ok(()); - } - let flag = stream.stalled.flag(); - let mut config = stream.track.modify()?; - config.stalled = flag; - config.commit()?; - Ok(()) - } - /// Drop any existing video track `track_id` (finishing it and clearing its /// catalog rendition) and allocate a fresh one. fn replace_video(&mut self, track_id: u8) -> anyhow::Result { diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 2a6d26dbe0..9f4a77c730 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -334,10 +334,6 @@ impl Import { // decode call (plus the store's own host-clock debounce) coalesces a junction's // burst of sub-table commits into few groups instead of one per commit. self.si.flush(self.last_pts.unwrap_or(Timestamp::ZERO), false)?; - // Video PIDs that went quiet while the mux kept flowing mark themselves stalled. - for stream in self.streams.values_mut() { - stream.tick()?; - } Ok(()) } @@ -1914,20 +1910,6 @@ impl Stream { /// the cut are whole and correct on their own. False where it carries exactly one: half /// an access unit is a picture with missing slices, and half a keyframe stays wrong for /// every picture that references it. Verbatim payloads are all-or-nothing the same way. - fn tick(&mut self) -> anyhow::Result<()> { - match self { - Stream::H264 { import, .. } => import.tick()?, - Stream::H265 { import, .. } => import.tick()?, - Stream::Aac(_) - | Stream::Opus(_) - | Stream::Legacy(_) - | Stream::Verbatim(_) - | Stream::Clock - | Stream::Ignored => {} - } - Ok(()) - } - fn salvages_partial_pes(&self) -> bool { match self { Stream::Aac(_) | Stream::Legacy(_) => true, diff --git a/rs/moq-rtc/src/codec/av1.rs b/rs/moq-rtc/src/codec/av1.rs index b16898b421..b01bc9847e 100644 --- a/rs/moq-rtc/src/codec/av1.rs +++ b/rs/moq-rtc/src/codec/av1.rs @@ -32,11 +32,6 @@ impl codec::Bridge for Bridge { Ok(()) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 961a692255..6ecca1d033 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -31,11 +31,6 @@ impl codec::Bridge for Bridge { Ok(()) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/h265.rs b/rs/moq-rtc/src/codec/h265.rs index 606cad6af3..f04c1c5a3e 100644 --- a/rs/moq-rtc/src/codec/h265.rs +++ b/rs/moq-rtc/src/codec/h265.rs @@ -33,11 +33,6 @@ impl codec::Bridge for Bridge { Ok(()) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index d4b7b23f9b..773196859a 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -44,12 +44,6 @@ pub struct Frame { pub trait Bridge: Send { fn push(&mut self, frame: Frame) -> Result<()>; - /// Re-evaluate stall from source silence. Video bridges mark a quiet - /// rendition stalled; audio is a no-op. - fn tick(&mut self) -> Result<()> { - Ok(()) - } - /// Abort the published track with `err` so subscribers see the real cause /// (the peer disconnected, an ICE failure) rather than a bare `Error::Dropped`. /// @@ -65,9 +59,6 @@ pub(crate) trait DeferredImport: Send + Sized { /// Decode one complete codec frame. fn decode(&mut self, frame: Bytes, pts: moq_net::Timestamp) -> moq_mux::Result<()>; - /// Re-evaluate stall from source silence. - fn tick(&mut self) -> moq_mux::Result<()>; - /// Abort the active media track. fn abort(self, err: moq_net::Error); } @@ -81,10 +72,6 @@ impl DeferredImport for moq_mux::codec::vp8::Import { moq_mux::codec::vp8::Import::decode(self, frame, Some(pts)) } - fn tick(&mut self) -> moq_mux::Result<()> { - moq_mux::codec::vp8::Import::tick(self) - } - fn abort(self, err: moq_net::Error) { moq_mux::codec::vp8::Import::abort(self, err); } @@ -99,10 +86,6 @@ impl DeferredImport for moq_mux::codec::vp9::Import { moq_mux::codec::vp9::Import::decode(self, frame, Some(pts)) } - fn tick(&mut self) -> moq_mux::Result<()> { - moq_mux::codec::vp9::Import::tick(self) - } - fn abort(self, err: moq_net::Error) { moq_mux::codec::vp9::Import::abort(self, err); } @@ -163,15 +146,6 @@ impl DeferredVideo { import.decode(frame, pts).map_err(Into::into) } - /// Re-evaluate stall from source silence once the importer exists. - pub fn tick(&mut self) -> Result<()> { - if let DeferredState::Active(import) = &mut self.state { - import.tick().map_err(Into::into) - } else { - Ok(()) - } - } - /// Abort the media track in either lifecycle state. pub fn abort(self, err: moq_net::Error) { match self.state { diff --git a/rs/moq-rtc/src/codec/vp8.rs b/rs/moq-rtc/src/codec/vp8.rs index 4cede5be7e..6ff4376128 100644 --- a/rs/moq-rtc/src/codec/vp8.rs +++ b/rs/moq-rtc/src/codec/vp8.rs @@ -25,11 +25,6 @@ impl codec::Bridge for Bridge { self.import.decode(frame.payload, pts) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/vp9.rs b/rs/moq-rtc/src/codec/vp9.rs index 0bd213df9c..e70dae9828 100644 --- a/rs/moq-rtc/src/codec/vp9.rs +++ b/rs/moq-rtc/src/codec/vp9.rs @@ -25,11 +25,6 @@ impl codec::Bridge for Bridge { self.import.decode(frame.payload, pts) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/ingest.rs b/rs/moq-rtc/src/ingest.rs index 3dec2cd57a..ab5213d391 100644 --- a/rs/moq-rtc/src/ingest.rs +++ b/rs/moq-rtc/src/ingest.rs @@ -68,10 +68,6 @@ impl session::MediaSink for IngestSink { self.bridges.push(mid, frame) } - fn tick(&mut self) -> Result<()> { - self.bridges.tick() - } - fn abort(&mut self, err: moq_net::Error) { self.bridges.abort(err); } diff --git a/rs/moq-rtc/src/session.rs b/rs/moq-rtc/src/session.rs index e0dd59c288..e2f815aecf 100644 --- a/rs/moq-rtc/src/session.rs +++ b/rs/moq-rtc/src/session.rs @@ -73,11 +73,6 @@ pub trait MediaSink: Send { /// loop has already converted the timestamp to microseconds. fn on_frame(&mut self, mid: str0m::media::Mid, frame: codec::Frame) -> Result<()>; - /// Re-evaluate stall from source silence on every ingest track. - fn tick(&mut self) -> Result<()> { - Ok(()) - } - /// Called once when the session ends with a genuine failure, so the sink can /// abort its tracks with the real cause instead of a bare `Error::Dropped`. fn abort(&mut self, err: moq_net::Error); @@ -232,12 +227,7 @@ impl Session { } let timeout = match self.rtc.poll_output().map_err(Error::rtc)? { - Output::Timeout(t) => { - if let MediaRole::Ingest(sink) = &mut self.role { - sink.tick()?; - } - t - } + Output::Timeout(t) => t, Output::Transmit(t) => { let dst = crate::net::to_family(t.destination, socket_v6); if let Err(err) = self.socket.send_to(&t.contents, dst).await { @@ -592,13 +582,6 @@ impl Bridges { Ok(()) } - pub fn tick(&mut self) -> Result<()> { - for bridge in self.inner.values_mut() { - bridge.tick()?; - } - Ok(()) - } - /// Abort every bridge's track with `err` so subscribers see the real cause /// rather than a bare `Error::Dropped`. /// diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index 8f741798cf..cacfb64e61 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -327,10 +327,11 @@ pub(crate) async fn rung_entry( Ok(entry) } -/// Rungs inherit the state of the rendition the pipeline actually decodes. -pub(crate) fn inherit_stalled(rungs: &mut [Published], source: &VideoConfig) { +/// Rungs are enabled exactly when the rendition the pipeline decodes is: a disabled source sends +/// no frames, so neither can its rungs. +pub(crate) fn inherit_enabled(rungs: &mut [Published], source: &VideoConfig) { for published in rungs { - published.entry.stalled = source.stalled; + published.entry.enabled = source.enabled; } } @@ -415,10 +416,10 @@ mod tests { } #[test] - fn rungs_inherit_a_stalled_source() { + fn rungs_inherit_a_disabled_source() { let mut source_catalog = moq_mux::catalog::hang::Catalog::default(); let mut src = source(1280, 720, Some(2_500_000)); - src.stalled = Some(true); + src.enabled = false; source_catalog.video.insert("video", src).unwrap(); let mut published = [Published { @@ -432,20 +433,17 @@ mod tests { entry: source(640, 360, Some(600_000)), }]; - inherit_stalled(&mut published, &source_catalog.video.renditions["video"]); + inherit_enabled(&mut published, &source_catalog.video.renditions["video"]); let mut out = moq_mux::catalog::hang::Catalog::default(); populate(&mut out, &source_catalog, &published, None).unwrap(); - assert_eq!( - out.video.renditions.get("video/360p").and_then(|c| c.stalled), - Some(true) - ); + assert!(!out.video.renditions["video/360p"].enabled); - // A different local rendition may stay stalled after the selected input recovers. + // Following an enabled input re-enables the rungs. let healthy = source(1920, 1080, Some(5_000_000)); source_catalog.video.insert("healthy", healthy.clone()).unwrap(); - inherit_stalled(&mut published, &healthy); + inherit_enabled(&mut published, &healthy); populate(&mut out, &source_catalog, &published, None).unwrap(); - assert_eq!(out.video.renditions["video/360p"].stalled, None); + assert!(out.video.renditions["video/360p"].enabled); } #[test] diff --git a/rs/moq-transcode/src/pipeline.rs b/rs/moq-transcode/src/pipeline.rs index a81becc9fe..ea33202fd3 100644 --- a/rs/moq-transcode/src/pipeline.rs +++ b/rs/moq-transcode/src/pipeline.rs @@ -18,12 +18,12 @@ use std::collections::HashMap; use hang::catalog::{Video, VideoConfig}; -/// Picture identity for ladder follow: a stall bit flipping is not a source change. +/// Picture identity for ladder follow: enabling or disabling a rendition is not a source change. fn same_picture(a: &VideoConfig, b: &VideoConfig) -> bool { let mut a = a.clone(); let mut b = b.clone(); - a.stalled = None; - b.stalled = None; + a.enabled = true; + b.enabled = true; a == b } @@ -135,7 +135,7 @@ impl Pipeline { }; published.push(Published { rung, entry }); } - catalog::inherit_stalled(&mut published, source); + catalog::inherit_enabled(&mut published, source); Ok(published) } @@ -179,7 +179,7 @@ impl Pipeline { } }; if name == self.name && same_picture(&rendition, &self.rendition) { - catalog::inherit_stalled(&mut self.rungs, &rendition); + catalog::inherit_enabled(&mut self.rungs, &rendition); self.rendition = rendition; return Ok(()); } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index b08542be82..016befe8b6 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -177,33 +177,6 @@ impl Producer { Ok(()) } - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> Result<(), Error> { - match &mut self.codecs { - Codecs::H264 { import, .. } => import.observe_lag(lag)?, - Codecs::H265 { import, .. } => import.observe_lag(lag)?, - } - Ok(()) - } - - /// Re-evaluate stall from source silence while waiting for the next frame. - pub fn tick(&mut self) -> Result<(), Error> { - match &mut self.codecs { - Codecs::H264 { import, .. } => import.tick()?, - Codecs::H265 { import, .. } => import.tick()?, - } - Ok(()) - } - - /// The camera is released; this rendition is never stalled while idle. - pub fn idle(&mut self) -> Result<(), Error> { - match &mut self.codecs { - Codecs::H264 { import, .. } => import.idle()?, - Codecs::H265 { import, .. } => import.idle()?, - } - Ok(()) - } - /// Mark a break in the published timeline: whatever is published next does not continue /// what came before. /// @@ -648,29 +621,22 @@ fn capture_stopped(producer: &mut Producer) -> Result<(), Erro producer.discontinuity() } -// Keep observing silence while source setup or an encode is pending. The work -// future stays pinned across ticks, so a slow operation is never restarted. +// Race source setup or an encode against the last viewer leaving, so an unwatched track +// stops waiting promptly. #[cfg(feature = "capture")] -async fn wait_capture( - producer: &mut Producer, +async fn wait_capture( demand: &moq_net::track::Demand, work: impl std::future::Future>, ) -> Result, Error> { - let mut work = std::pin::pin!(work); - let mut timer = tokio::time::interval(hang::catalog::stalled::DEFAULT_INTERVAL); - loop { - tokio::select! { - biased; - res = demand.unused() => { - if let Err(err) = res { - log_track_ended(err); - } - producer.idle()?; - return Ok(None); + tokio::select! { + biased; + res = demand.unused() => { + if let Err(err) = res { + log_track_ended(err); } - _ = timer.tick() => producer.tick()?, - res = &mut work => return res.map(Some), + Ok(None) } + res = work => res.map(Some), } } @@ -710,7 +676,7 @@ async fn capture_loop( } // Open the camera and an encoder sized to its negotiated mode. - let Some(mut camera) = wait_capture(producer, demand, source.open(capture)).await? else { + let Some(mut camera) = wait_capture(demand, source.open(capture)).await? else { continue; }; // Capture timestamps use a private monotonic timeline. Sample both clocks @@ -733,7 +699,7 @@ async fn capture_loop( // No cut on reopen: a fresh encoder opens with a keyframe on every backend, // so the viewer whose subscription reopened the camera can decode from the // first frame regardless, and a backend that cannot cut still captures. - let Some(mut encoder) = wait_capture(producer, demand, Sink::open(&encoder_config)).await? else { + let Some(mut encoder) = wait_capture(demand, Sink::open(&encoder_config)).await? else { continue; }; // A reopen can land on another backend than the probe did, so the controls @@ -761,7 +727,6 @@ async fn capture_loop( // Race the next frame against the last viewer leaving so we release the // camera promptly when demand drops. `biased` checks demand first so an // unwatched track stops before reading another frame. - let interval = hang::catalog::stalled::interval_from_fps(Some(framerate.as_f64())); let frame = tokio::select! { biased; res = demand.unused() => { @@ -779,14 +744,7 @@ async fn capture_loop( } // A read error is terminal for this selection (the source is gone // or was refused); `None` just ends the stream, so reopen below. - // Timing out is a quiet camera: mark the rendition stalled and wait again. - frame = tokio::time::timeout(interval, camera.read()) => match frame { - Ok(frame) => frame?, - Err(_) => { - producer.tick()?; - continue; - } - }, + frame = camera.read() => frame?, }; let Some(mut frame) = frame else { break }; @@ -796,12 +754,9 @@ async fn capture_loop( // Never refused: `Control::cut` stops counting once this encoder is known to. encoder.cut().await?; } - let started = Instant::now(); - let Some(encoded) = wait_capture(producer, demand, encoder.encode(frame)).await? else { + let Some(encoded) = wait_capture(demand, encoder.encode(frame)).await? else { break; }; - let lag = started.elapsed(); - producer.observe_lag(lag)?; for unit in encoded.iter().filter(|unit| unit.keyframe) { forced.keyframe(unit.timestamp); } @@ -811,7 +766,6 @@ async fn capture_loop( // Drop the camera (LED off) and encoder before waiting for the next viewer. drop(camera); drop(encoder); - producer.idle()?; capture_stopped(producer)?; tracing::info!("capture stopped; released source"); } @@ -1511,7 +1465,7 @@ mod tests { use moq_mux::container::Container as _; // Off Apple, encoding waits on an OS thread. Keep Tokio from auto-advancing - // the paused clock to a stall deadline while that thread answers. + // the paused clock to a timer deadline while that thread answers. let (hold, released) = tokio::sync::oneshot::channel::<()>(); let clock = tokio::task::spawn_blocking(move || { let _ = released.blocking_recv(); diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 11e670dedf..a91e00b692 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -15,8 +15,8 @@ public typealias MediaFrame = MoqFFI.MoqMediaFrame /// The JSON manifest describing a broadcast's tracks: video and audio /// renditions, display geometry, and untyped application sections. public typealias Catalog = MoqFFI.MoqCatalog -/// A video rendition in the catalog: codec, dimensions, bitrate, temporary -/// avoidance recommendation, framerate, and container. +/// A video rendition in the catalog: codec, dimensions, bitrate, whether it is +/// enabled, framerate, and container. public typealias Video = MoqFFI.MoqVideo /// Caller-provided catalog fields for a video track. public typealias VideoHint = MoqFFI.MoqVideoHint @@ -30,7 +30,7 @@ public typealias ContainerFormat = MoqFFI.MoqContainerFormat /// that property from the next catalog snapshot. public typealias VideoProperties = MoqFFI.MoqVideoProperties /// An audio rendition in the catalog: codec, sample rate, channel count, -/// bitrate, and container. +/// bitrate, whether it is enabled, and container. public typealias Audio = MoqFFI.MoqAudio /// One raw-audio frame: PCM samples in the configured layout plus a /// presentation timestamp. From b9dec437a6de1110bb49b7d60d61d540966e6b29 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 01:12:53 -0700 Subject: [PATCH 03/12] fix(watch): show black when no video rendition is selectable A player whose every rendition is disabled deselected the track but kept presenting the last decoded picture. Clear it, as going offline already does; a paused or hidden player still holds its frame. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/video/decoder.test.ts | 19 +++++++++++++++++++ js/watch/src/video/decoder.ts | 16 ++++++++-------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index b1565a597d..7ca407d056 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -228,6 +228,7 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[]) { return { decoder, enabled, + catalog, submitted, close() { decoder.close(); @@ -265,6 +266,24 @@ it("promoting from no active track holds its picture and timestamp until a frame } }); +it("clears the picture once no rendition is selectable", async () => { + const playback = await guardedPlayback("legacy", [sample(10, 1000)]); + try { + const held = playback.decoder.out.frame.peek(); + expect(held).toBeDefined(); + + const video = playback.catalog.peek().video?.renditions.video; + if (!video) throw new Error("missing rendition"); + playback.catalog.set({ video: { renditions: { video: { ...video, enabled: false } } } }); + await microtasks(); + expect(playback.decoder.out.frame.peek()).toBeUndefined(); + expect(playback.decoder.out.timestamp.peek()).toBeUndefined(); + expect((held as unknown as Picture).closed).toBe(true); + } finally { + playback.close(); + } +}); + // A late marker from older history raises a discontinuity without making older groups newer. function marker(group: number, end: number, discontinuity: number): Read { return { group, discontinuity, continuous: false, frame: undefined, end: Time.Micro(end) }; diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 56134653c2..b41005e644 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -160,19 +160,19 @@ export class Decoder { } #runPending(effect: Effect): void { - const values = effect.getAll([ - this.in.enabled, - this.source.in.broadcast, - this.source.out.track, - this.#identity, - ]); - if (!values) { + const enabled = effect.get(this.in.enabled); + const broadcast = effect.get(this.source.in.broadcast); + const track = effect.get(this.source.out.track); + const identity = effect.get(this.#identity); + if (!enabled || !broadcast || !track || !identity) { // Close the active track when disabled (e.g. paused or not visible). // The pending cleanup won't do this because it was already promoted to #active. this.#active.set(undefined); + // A paused or hidden player keeps its picture, but with nothing selectable (every rendition + // disabled or removed) it shows black rather than a frozen frame. + if (enabled && broadcast && !track) this.#clearCurrentFrame(); return; } - const [_, broadcast, track, identity] = values; // Honor a per-rendition `broadcast` override: subscribe on the resolved source // broadcast instead of the catalog's own broadcast. From 20aa72b49e6424c8ae30f1ceb916235240c60fd5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 01:12:53 -0700 Subject: [PATCH 04/12] test(interop): disable and re-enable the fixture's video rendition The media lane mutes the fixture's video, checks the rendition stays published with enabled: false, the player goes black instead of freezing, and playback resumes in sync once re-enabled. Co-Authored-By: Claude Opus 5.5 --- test/interop/clients/js/media.ts | 40 ++++++++++++++++++++++++- test/interop/clients/js/src/contract.ts | 6 ++++ test/interop/clients/js/src/fixture.ts | 9 +++++- test/interop/clients/js/src/setup.ts | 2 ++ 4 files changed, 55 insertions(+), 2 deletions(-) diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index f4e9577d3a..3172d205d7 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -55,7 +55,7 @@ import { import * as Pattern from "./src/pattern"; /** Cases beyond the mandatory capability probe, publisher readiness, and cold start. */ -const CASES = ["capture-denial", "pause", "rejoin", "detach", "republish", "late-join"] as const; +const CASES = ["capture-denial", "pause", "disable", "rejoin", "detach", "republish", "late-join"] as const; type Case = (typeof CASES)[number]; const { values } = parseArgs({ @@ -568,6 +568,44 @@ try { assertMedia(await collect(player, playerErrors, WINDOW_MS), "after resume"); } + // ── disable and re-enable the video rendition ──────────────────────────── + // A disabled rendition stays in the catalog with `enabled: false`. The player deselects it and + // shows black instead of freezing on the last picture, then picks it up again once re-enabled. + if (wants("disable")) { + console.error("=== disable and enable video ==="); + const before = await readPlayerState(player); + await command(publisher, "disableVideo"); + await waitForFixture(publisher, publisherErrors, { + deadline: Date.now() + SETTLE_MS, + assertion: "disable keeps the rendition", + description: "the fixture to publish its video rendition with enabled: false and stop encoding", + predicate: (state) => !state.videoEnabled && !state.videoActive, + }); + await waitForState(player, playerErrors, { + deadline: Date.now() + SETTLE_MS, + assertion: "disable blanks the picture", + description: `the canvas to go black instead of holding frame ${before.frameId}`, + predicate: (state) => !state.painted, + }); + const blank = await collect(player, playerErrors, HELD_MS); + const lit = blank.filter((s) => s.painted); + check( + lit.length === 0, + "disable blanks the picture", + () => `the canvas showed a picture again while disabled: ${JSON.stringify(lit[0])}`, + ); + + await command(publisher, "enableVideo"); + const resumed = await waitForState(player, playerErrors, { + deadline: Date.now() + timeoutMs, + assertion: "enable resumes playback", + description: `the presented frame to move past the ${before.frameId} showing before the disable`, + predicate: (state) => (state.frameId ?? 0) > (before.frameId ?? 0), + }); + console.error(` black while disabled, then resumed at frame ${resumed.frameId}`); + assertMedia(await collect(player, playerErrors, WINDOW_MS), "after enable"); + } + // ── unsubscribe and rejoin ─────────────────────────────────────────────── if (wants("rejoin")) { console.error("=== unsubscribe and rejoin ==="); diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index d9c0f2eafa..cb5562dcb6 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -41,6 +41,8 @@ export type FixtureState = { audioActive: boolean; /** Frames the encoder has produced. */ encodedFrames: number; + /** Whether the video rendition is published as enabled; false while {@link InteropControl.disableVideo} holds. */ + videoEnabled: boolean; }; /** The camera publisher state mirrored onto its element for Playwright. */ @@ -180,6 +182,10 @@ export type InteropControl = { reattach(): void; /** Connect a second player and leave it behind for the leaked-session negative control. */ startLeak(): void; + /** Keep the fixture's video rendition in the catalog with `enabled: false`, stopping its frames. */ + disableVideo(): void; + /** Re-enable the fixture's video rendition. */ + enableVideo(): void; }; /** The `window` property the commands are published on. */ diff --git a/test/interop/clients/js/src/fixture.ts b/test/interop/clients/js/src/fixture.ts index 17f6faef55..cf3f52ca66 100644 --- a/test/interop/clients/js/src/fixture.ts +++ b/test/interop/clients/js/src/fixture.ts @@ -50,6 +50,7 @@ export class Fixture { readonly #audio: AudioContext; readonly #frameId = new Signal(-1); readonly #audioState = new Signal("suspended"); + readonly #videoEnabled = new Signal(true); #timer: number | undefined; constructor(host: HTMLElement, url: string, name: string, fault: Fault) { @@ -98,7 +99,7 @@ export class Fixture { const video = new Publish.Video.Encoder("video", { broadcast, capture, - enabled: true, + enabled: this.#videoEnabled, config: { frameRate: Pattern.FPS, keyframeInterval: KEYFRAME_INTERVAL, maxBitrate: MAX_BITRATE }, }); this.#signals.cleanup(() => video.close()); @@ -160,6 +161,7 @@ export class Fixture { videoActive: effect.get(video.out.active), audioActive: effect.get(audio.out.active), encodedFrames: effect.get(video.out.stats).frames, + videoEnabled: effect.get(video.out.catalog)?.enabled !== false, }; host.dataset.interopFixture = JSON.stringify(state); }); @@ -210,6 +212,11 @@ export class Fixture { }); } + /** Enable or disable the video rendition, which stays in the catalog either way. */ + setVideo(enabled: boolean): void { + this.#videoEnabled.set(enabled); + } + /** Stop publishing and release the capture, audio graph, and session. */ close(): void { this.#signals.close(); diff --git a/test/interop/clients/js/src/setup.ts b/test/interop/clients/js/src/setup.ts index 0992884671..46bc44b22f 100644 --- a/test/interop/clients/js/src/setup.ts +++ b/test/interop/clients/js/src/setup.ts @@ -65,6 +65,8 @@ if (role === "publish") { fixture?.close(); fixture = new Fixture(host, url, broadcast, fault); }, + disableVideo: () => fixture?.setVideo(false), + enableVideo: () => fixture?.setVideo(true), }); } else if (role === "subscribe") { await customElements.whenDefined("moq-watch"); From 107fde0839c23af9750149dd180e64fee9237de7 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 02:07:33 -0700 Subject: [PATCH 05/12] fix(transcode): prefer an enabled source rendition A disabled rendition sends no frames, yet the transcoder could pick it over a lower-ranked enabled one and disable every rung. Rank enabled renditions first, and let a disabled source give way only to an enabled one, so an all-disabled source still passes enabled: false to its rungs. Co-Authored-By: Claude Opus 5.5 --- rs/moq-transcode/src/catalog.rs | 74 +++++++++++++++++++++++++++++---- 1 file changed, 66 insertions(+), 8 deletions(-) diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index cacfb64e61..80c08704ae 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -159,18 +159,25 @@ impl Decoders { } /// Pick the rendition to transcode from: the best [ranked](Video::ranked) rendition local -/// to the source broadcast that this host can decode. +/// to the source broadcast that this host can decode, enabled ones first. /// /// [`Error::NoSource`] means wait for a later snapshot. Any other error means /// nothing on offer can be decoded here, and is why the best one refused. pub(crate) async fn choose_source(video: &Video, decoders: &mut Decoders) -> Result<(String, VideoConfig), Error> { // Best first. A rendition without dimensions ranks after every one with them: // it can't be chosen yet, but it can still keep the transcoder waiting. - let candidates = video - .ranked() - // A rendition that itself lives in another broadcast can't be subscribed - // through this one; composing relative references is a follow-up. - .filter(|(_, config)| config.broadcast.is_none()) + let ranked = |enabled: bool| { + video + .ranked() + .filter(move |(_, config)| config.enabled == enabled) + // A rendition that itself lives in another broadcast can't be subscribed + // through this one; composing relative references is a follow-up. + .filter(|(_, config)| config.broadcast.is_none()) + }; + // A disabled rendition sends no frames. It is the source only when nothing enabled is, + // so the rungs inherit `enabled: false` rather than the ladder vanishing. + let candidates = ranked(true) + .chain(ranked(false)) .filter_map(|(name, config)| Some((name, config, codec(config)?))); let mut refused = None; @@ -216,6 +223,7 @@ pub(crate) async fn follow_source( current: &str, decoders: &mut Decoders, ) -> Result<(String, VideoConfig), Error> { + let mut kept = None; if let Some(config) = video.renditions.get(current) && config.broadcast.is_none() && dimensions(config).is_some() @@ -223,9 +231,18 @@ pub(crate) async fn follow_source( // The name can stay while the codec changes under it. && decoders.probe(codec, config).await { - return Ok((current.to_string(), config.clone())); + if config.enabled { + return Ok((current.to_string(), config.clone())); + } + kept = Some(config); + } + + // A disabled source gives way only to an enabled one. + match (kept, choose_source(video, decoders).await) { + (Some(config), Ok((_, chosen))) if !chosen.enabled => Ok((current.to_string(), config.clone())), + (Some(config), Err(_)) => Ok((current.to_string(), config.clone())), + (_, chosen) => chosen, } - choose_source(video, decoders).await } /// Whether a rung decoding `old` can keep decoding `new` untouched. @@ -821,6 +838,47 @@ mod tests { assert_eq!(name, "tall"); } + /// A disabled rendition sends no frames, so an enabled one wins even when it ranks lower, + /// and a disabled one is chosen only when nothing enabled is on offer. + #[tokio::test] + async fn chooses_an_enabled_source_first() { + let mut video = Video::default(); + let mut high = source(1920, 1080, None); + high.enabled = false; + video.insert("high", high).unwrap(); + video.insert("low", source(640, 360, None)).unwrap(); + + let (name, _) = choose_source(&video, &mut decodes_all()).await.unwrap(); + assert_eq!(name, "low"); + + video.renditions.get_mut("low").unwrap().enabled = false; + let (name, config) = choose_source(&video, &mut decodes_all()).await.unwrap(); + assert_eq!(name, "high"); + assert!(!config.enabled); + } + + /// A source that is disabled gives way to an enabled rendition, but stays put when every + /// rendition is disabled, so its rungs inherit `enabled: false` instead of switching. + #[tokio::test] + async fn follow_leaves_a_disabled_source_only_for_an_enabled_one() { + let mut video = Video::default(); + let mut low = source(640, 360, None); + low.enabled = false; + video.insert("low", low).unwrap(); + let mut high = source(1920, 1080, None); + high.enabled = false; + video.insert("high", high).unwrap(); + let mut decoders = decodes_all(); + + let (name, config) = follow_source(&video, "low", &mut decoders).await.unwrap(); + assert_eq!(name, "low"); + assert!(!config.enabled); + + video.renditions.get_mut("high").unwrap().enabled = true; + let (name, _) = follow_source(&video, "low", &mut decoders).await.unwrap(); + assert_eq!(name, "high"); + } + /// Probing opens a real decoder, so each codec is opened once per transcoder /// rather than once per catalog edit. #[tokio::test] From cbbd5f7fa18d6231796c38e3bbb32c7a24f146b8 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 04:50:20 -0700 Subject: [PATCH 06/12] fix(hang): rank enabled video renditions first Video::ranked() feeds every first-party Rust selector (transcode, rtc egress, rtmp, FLV export), so ordering disabled renditions last keeps them all off a rendition with no frames coming while an enabled sibling exists. moq-transcode drops its local copy of the ordering. Co-Authored-By: Claude Opus 5.5 --- rs/hang/src/catalog/video/mod.rs | 14 ++++++++++---- rs/moq-transcode/src/catalog.rs | 22 +++++++++------------- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 7f2a80d7ec..2d4693b8cb 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -106,16 +106,17 @@ impl Video { self.renditions.remove(name) } - /// Iterate the renditions best first: largest picture, then highest bitrate. + /// Iterate the renditions best first: enabled, then largest picture, then highest bitrate. /// /// A consumer that carries one rendition takes the first it supports, so the - /// picture doesn't depend on how the tracks are named. Unknown dimensions or - /// bitrate rank below known ones, and exact ties keep name order. + /// picture doesn't depend on how the tracks are named. A disabled rendition sends + /// no frames, so it ranks below every enabled one. Unknown dimensions or bitrate + /// rank below known ones, and exact ties keep name order. pub fn ranked(&self) -> impl Iterator { let mut ranked: Vec<_> = self.renditions.iter().collect(); ranked.sort_by_key(|(_, config)| { let area = u64::from(config.coded_width.unwrap_or(0)) * u64::from(config.coded_height.unwrap_or(0)); - std::cmp::Reverse((area, config.bitrate)) + std::cmp::Reverse((config.enabled, area, config.bitrate)) }); ranked.into_iter() } @@ -324,6 +325,11 @@ mod test { let names: Vec<_> = video.ranked().map(|(name, _)| name.as_str()).collect(); assert_eq!(names, ["f", "d", "e", "c", "b", "a"]); + + // A disabled rendition sends no frames, so it ranks below every enabled one. + video.renditions.get_mut("f").unwrap().enabled = false; + let names: Vec<_> = video.ranked().map(|(name, _)| name.as_str()).collect(); + assert_eq!(names, ["d", "e", "c", "b", "a", "f"]); } #[test] diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index 80c08704ae..505986a9a1 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -166,18 +166,13 @@ impl Decoders { pub(crate) async fn choose_source(video: &Video, decoders: &mut Decoders) -> Result<(String, VideoConfig), Error> { // Best first. A rendition without dimensions ranks after every one with them: // it can't be chosen yet, but it can still keep the transcoder waiting. - let ranked = |enabled: bool| { - video - .ranked() - .filter(move |(_, config)| config.enabled == enabled) - // A rendition that itself lives in another broadcast can't be subscribed - // through this one; composing relative references is a follow-up. - .filter(|(_, config)| config.broadcast.is_none()) - }; - // A disabled rendition sends no frames. It is the source only when nothing enabled is, - // so the rungs inherit `enabled: false` rather than the ladder vanishing. - let candidates = ranked(true) - .chain(ranked(false)) + // A disabled rendition ranks last, so it is the source only when nothing enabled is, + // and the rungs inherit `enabled: false` rather than the ladder vanishing. + let candidates = video + .ranked() + // A rendition that itself lives in another broadcast can't be subscribed + // through this one; composing relative references is a follow-up. + .filter(|(_, config)| config.broadcast.is_none()) .filter_map(|(name, config)| Some((name, config, codec(config)?))); let mut refused = None; @@ -237,7 +232,8 @@ pub(crate) async fn follow_source( kept = Some(config); } - // A disabled source gives way only to an enabled one. + // A disabled source gives way only to an enabled one. An enabled source is never + // swapped for a better one, even its own predecessor re-enabled, to avoid churning the ladder. match (kept, choose_source(video, decoders).await) { (Some(config), Ok((_, chosen))) if !chosen.enabled => Ok((current.to_string(), config.clone())), (Some(config), Err(_)) => Ok((current.to_string(), config.clone())), From cfa4b9e970cba3c186864cd36efb460e9f2d99eb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 04:50:20 -0700 Subject: [PATCH 07/12] fix(publish): keep a re-enabled rendition disabled until it resolves Unmuting dropped the rendition from the catalog until the reopened capture resolved a config, so a mute round trip still looked like remove and re-add. Keep advertising the last config with enabled: false until then. A failed black keyframe during teardown no longer closes the track the broadcast owns. Co-Authored-By: Claude Opus 5.5 --- js/publish/src/audio/encoder.test.ts | 10 +++++++++- js/publish/src/audio/encoder.ts | 19 ++++++++++-------- js/publish/src/video/encoder.test.ts | 6 +++++- js/publish/src/video/encoder.ts | 29 +++++++++++++++++----------- 4 files changed, 43 insertions(+), 21 deletions(-) diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index d672699184..e9bda7361d 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -253,6 +253,7 @@ async function setup(baseline = new Baseline(), codec?: Codec) { config, encoder, enabled, + capture, track, rendition, feed, @@ -347,7 +348,7 @@ test("disabling with a subscriber attached marks where submitted audio ends", as test("disabling keeps the rendition in the catalog with enabled: false", async () => { using _webcodecs = installFakeWebCodecs(); using env = await setup(); - const { encoder, enabled, rendition } = env; + const { encoder, enabled, capture, rendition } = env; const before = encoder.out.catalog.peek(); expect(before).toBeDefined(); @@ -359,8 +360,15 @@ test("disabling keeps the rendition in the catalog with enabled: false", async ( expect(rendition.config.peek()).toEqual({ ...before, enabled: false } as never); expect(encoder.out.active.peek()).toBe(false); + // Muting released the microphone, so re-enabling waits on its format without dropping the rendition. + const format = capture.out.format.peek(); + capture.out.format.set(undefined as never); enabled.set(true); await settle(); + expect(encoder.out.catalog.peek()).toEqual({ ...before, enabled: false } as never); + + capture.out.format.set(format); + await settle(); expect(encoder.out.catalog.peek()).toEqual(before); }); diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 14459c7d72..87cc4fb86c 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -215,6 +215,9 @@ export class Encoder { #estimate = new Signal({}); // The last config published while enabled, which a disabled rendition keeps advertising. #last?: Catalog.AudioConfig; + // Whether the rendition was disabled since a config last resolved, so it keeps advertising `#last` + // as disabled until the re-enabled capture resolves a new one. + #paused = false; constructor(name: string, props?: EncoderProps) { // `source` moved to Audio.Capture, which renditions share. TypeScript catches this, but a @@ -407,19 +410,19 @@ export class Encoder { // Publish the config immediately so a consumer can request the demand-gated track. Once encoding // starts, republish Opus with the exact decoder description reported for that encoder config. A - // disabled rendition keeps its last config, since muting also releases the capture. + // disabled rendition keeps its last config, since muting also releases the capture, and stays + // disabled after re-enabling until the reopened capture reports its format. #runCatalog(effect: Effect): void { const estimate = effect.get(this.#estimate); - if (!effect.get(this.in.enabled)) { - effect.set(this.#out.catalog, this.#last && { ...this.#last, ...estimate, enabled: false }); - return; - } - - const config = effect.get(this.#config)?.catalog; + const enabled = effect.get(this.in.enabled); + if (!enabled) this.#paused = true; + const config = enabled ? effect.get(this.#config)?.catalog : undefined; if (!config) { - effect.set(this.#out.catalog, undefined); + const last = this.#paused ? this.#last : undefined; + effect.set(this.#out.catalog, last && { ...last, ...estimate, enabled: false }); return; } + this.#paused = false; const decoder = effect.get(this.#decoderDescription); const catalog = decoder?.config === config ? { ...config, description: decoder.description } : config; diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index 0599a53edb..716df42bc5 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -620,9 +620,13 @@ test("disabling encodes one black keyframe and keeps the rendition with enabled: expect(disabled?.codec).toBe(before?.codec); expect(disabled?.codedWidth).toBe(before?.codedWidth); - source.set({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }); + // Re-enabled before the capture reopens: still the same rendition, still disabled. enabled.set(true); await settle(); + expect(encoder.out.catalog.peek()).toEqual(disabled); + + source.set({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }); + await settle(); expect(encoder.out.catalog.peek()?.enabled).toBeUndefined(); expect(encoded.filter((frame) => frame.black)).toHaveLength(1); } finally { diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index 8f7a804aac..27105cccb1 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -173,6 +173,9 @@ export class Encoder { #estimate = new Signal({}); // The last config published while enabled, which a disabled rendition keeps advertising. #last?: Catalog.VideoConfig; + // Whether the rendition was disabled since a config last resolved, so it keeps advertising `#last` + // as disabled until the re-enabled capture resolves a new one. + #paused = false; constructor(name: string, props?: EncoderProps) { this.name = name; @@ -283,6 +286,7 @@ export class Encoder { let lastKeyframe: Time.Micro | undefined; let lastEncoded: Time.Micro | undefined; + let tearingDown = false; effect.spawn(async () => { const encoder = new VideoEncoder({ @@ -304,7 +308,9 @@ export class Encoder { } }, error: (err: Error) => { - producer.close(err); + // The black keyframe is cosmetic; failing it must not end the track the broadcast owns. + if (tearingDown) console.warn("video encoder failed while stopping:", err); + else producer.close(err); }, }); @@ -380,6 +386,7 @@ export class Encoder { // has to wait for it. A viewer that predates `enabled` keeps selecting a disabled rendition, // and shows black instead of a frozen picture. await new Promise((resolve) => effect.abort.addEventListener("abort", resolve, { once: true })); + tearingDown = true; try { const config = configured; if ( @@ -397,7 +404,8 @@ export class Encoder { } finally { frame.close(); } - await encoder.flush(); + // A rejected flush already reported through the error callback. + await encoder.flush().catch(() => {}); } } finally { if (encoder.state !== "closed") encoder.close(); @@ -410,20 +418,19 @@ export class Encoder { } // Publishes the catalog for the configured settings. A disabled rendition keeps its last config, - // since muting also releases the capture it would resolve a new one from. + // since muting also releases the capture it would resolve a new one from. It stays disabled after + // re-enabling until the reopened capture resolves, so an unmute is one field changing too. #runCatalog(effect: Effect): void { - const enabled = effect.get(this.in.enabled); const estimate = effect.get(this.#estimate); - if (!enabled) { - effect.set(this.#out.catalog, this.#last && { ...this.#last, ...estimate, enabled: false }); - return; - } - - const live = effect.get(this.#live); + const enabled = effect.get(this.in.enabled); + if (!enabled) this.#paused = true; + const live = enabled ? effect.get(this.#live) : undefined; if (!live) { - effect.set(this.#out.catalog, undefined); + const last = this.#paused ? this.#last : undefined; + effect.set(this.#out.catalog, last && { ...last, ...estimate, enabled: false }); return; } + this.#paused = false; // Advertise the codec string the probe's encoder reported rather than the one we configured, // so it names the profile and level the bitstream actually carries. From dda7e5b7aab08a727ff6dde97c10114694e34d09 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 04:50:20 -0700 Subject: [PATCH 08/12] docs(hang): define enabled with the common rendition fields Also clear the buffered ranges when the watch decoder blanks for lack of a selectable rendition. Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-hang.md | 22 ++++++++++------------ js/watch/src/video/decoder.ts | 5 ++++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index dc5128ec79..b21edbbf1e 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -192,7 +192,6 @@ In addition to the WebCodecs fields, each rendition MAY carry the common renditi type VideoDecoderConfigExtensions = { "displayAspectWidth": number | undefined, "displayAspectHeight": number | undefined, - "enabled": boolean | undefined, } ~~~ @@ -200,8 +199,6 @@ type VideoDecoderConfigExtensions = { A consumer that understands neither field MUST assume square pixels, a 1:1 ratio. Both MUST be present together; a consumer that sees only one MUST ignore it. -`enabled` is defined in {{field-enabled}}. - For example: ~~~ @@ -237,14 +234,6 @@ For example: } ~~~ -### enabled {#field-enabled} -The `enabled` field says whether a consumer may select an audio or video rendition. -When `enabled` is false, no frames are coming, and a consumer MUST NOT select the rendition. -If absent, `enabled` defaults to true; a publisher SHOULD only write it when false. - -Earlier versions defined a video `stalled` field instead. -A publisher MUST NOT write `stalled`, and a consumer MUST ignore it. - ## Audio An audio track contains the necessary information to decode an audio stream. @@ -259,7 +248,7 @@ The `renditions` field contains a map of track names to audio decoder configurat See the [WebCodecs specification](https://www.w3.org/TR/webcodecs/#audio-decoder-config) for specifics and registered codecs. Any field carrying raw bytes, notably `description`, is a hex string ({{binary}}). -In addition to the WebCodecs fields, each rendition MAY carry the common rendition fields ({{common}}) and `enabled` ({{field-enabled}}). +In addition to the WebCodecs fields, each rendition MAY carry the common rendition fields ({{common}}). ### PCM {#audio-pcm} @@ -494,6 +483,7 @@ type CommonExtensions = { "container": Container, "jitter": number | undefined, "delay": number | undefined, + "enabled": boolean | undefined, } ~~~ @@ -558,6 +548,14 @@ A consumer MUST NOT subtract one rendition's `delay` from another's: each is a m For example, a video encoder that flushes 200 milliseconds after the audio encoder for the same media time advertises a video `delay` of 200 and no audio `delay`. +### enabled {#field-enabled} +The `enabled` field says whether a consumer may select an audio or video rendition; a text rendition does not carry it. +When `enabled` is false, no frames are coming, and a consumer MUST NOT select the rendition. +If absent, `enabled` defaults to true; a publisher SHOULD only write it when false. + +Earlier versions defined a video `stalled` field instead. +A publisher MUST NOT write `stalled`, and a consumer MUST ignore it. + # Container {#container} Audio, video, and text tracks use a container to encapsulate the media payload. A rendition declares its container via the `container` field of its catalog entry ({{common}}): diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index b41005e644..0f0de82945 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -170,7 +170,10 @@ export class Decoder { this.#active.set(undefined); // A paused or hidden player keeps its picture, but with nothing selectable (every rendition // disabled or removed) it shows black rather than a frozen frame. - if (enabled && broadcast && !track) this.#clearCurrentFrame(); + if (enabled && broadcast && !track) { + this.#clearCurrentFrame(); + this.#out.buffered.set([]); + } return; } From 92d377e372fc529457955b97fbdd496c17159beb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 06:09:14 -0700 Subject: [PATCH 09/12] docs(hang): note text renditions skip enabled; a capture that never reopens stays disabled Co-Authored-By: Claude Opus 5.5 --- drafts/draft-lcurley-moq-hang.md | 2 +- js/publish/src/audio/encoder.ts | 3 ++- js/publish/src/video/encoder.ts | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 24bc613d67..eb49df92a0 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -302,7 +302,7 @@ type TextConfig = { "format": "vtt" | "ttml" | "utf8" | string, "role": "subtitle" | "caption" | string | undefined, "lang": string | undefined, - // plus the common rendition fields + // plus the common rendition fields, except `enabled` } ~~~ diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 87cc4fb86c..5adcd9ac88 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -216,7 +216,8 @@ export class Encoder { // The last config published while enabled, which a disabled rendition keeps advertising. #last?: Catalog.AudioConfig; // Whether the rendition was disabled since a config last resolved, so it keeps advertising `#last` - // as disabled until the re-enabled capture resolves a new one. + // as disabled until the re-enabled capture resolves a new one. A capture that never reopens leaves + // it disabled, which is accurate: no frames are coming. #paused = false; constructor(name: string, props?: EncoderProps) { diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index 27105cccb1..b502c22926 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -174,7 +174,8 @@ export class Encoder { // The last config published while enabled, which a disabled rendition keeps advertising. #last?: Catalog.VideoConfig; // Whether the rendition was disabled since a config last resolved, so it keeps advertising `#last` - // as disabled until the re-enabled capture resolves a new one. + // as disabled until the re-enabled capture resolves a new one. A capture that never reopens leaves + // it disabled, which is accurate: no frames are coming. #paused = false; constructor(name: string, props?: EncoderProps) { From a5a7c21f317b56b6faee3cf7048b31776ab19ce4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 22:35:14 -0700 Subject: [PATCH 10/12] quest: remove completed catalog-enabled prerequisites Co-Authored-By: GPT-6 --- quest/m1/hang-changelog-04.md | 4 ---- quest/m2/native-enabled.md | 4 ---- 2 files changed, 8 deletions(-) diff --git a/quest/m1/hang-changelog-04.md b/quest/m1/hang-changelog-04.md index 476085ce53..af0905425c 100644 --- a/quest/m1/hang-changelog-04.md +++ b/quest/m1/hang-changelog-04.md @@ -13,7 +13,3 @@ Compare every -03 entry against the published including its body and changelog; do not rely on an illustrative list of late entries. The root `clock` section already shipped in -03 and stays there. Run `just drafts check`. This corrects attribution, not the format. - -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - land after #4915's draft changelog edit to avoid conflicting changes diff --git a/quest/m2/native-enabled.md b/quest/m2/native-enabled.md index 2741fc1eb3..b5a3fb4469 100644 --- a/quest/m2/native-enabled.md +++ b/quest/m2/native-enabled.md @@ -19,7 +19,3 @@ Test mixed and all-disabled catalogs, plus disable/re-enable transitions. A C `moq_consume_audio_enabled` getter is added only if a C consumer needs it; any FFI/C API change updates all wrappers and their docs in the same PR. Update native consumer docs for the selection behavior. - -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - #4915 provides enabled fields and enabled-first video ranking From d2e2dad24d6e9bc9fe24248c78567f90519397f2 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 23:11:04 -0700 Subject: [PATCH 11/12] fix(watch): release the audio graph when its broadcast is disabled Co-Authored-By: GPT-6 --- doc/lib/js/watch.md | 3 ++- js/watch/src/audio/decoder.test.ts | 27 +++++++++++++++++++++++++++ js/watch/src/audio/decoder.ts | 3 +++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/doc/lib/js/watch.md b/doc/lib/js/watch.md index ded3988fe7..c4dd05b56b 100644 --- a/doc/lib/js/watch.md +++ b/doc/lib/js/watch.md @@ -130,7 +130,8 @@ const dispose = el.signals.run((effect) => { Call `dispose()` from your framework's unmount cleanup when this subscription is no longer needed. Removing the element disables playback but keeps its -effects open so the same node can reconnect. +effects open so the same node can reconnect. Its audio graph is released and +rebuilt on return; pausing or muting keeps the graph warm. The effect re-runs whenever the catalog or the active broadcast changes, so a reconnect resubscribes on its own. A publisher that rewrites its catalog often diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 4deddee37b..73276c30f9 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -191,7 +191,9 @@ async function play(initial: Delay) { numberOfChannels: 2, }); const catalog = new Signal({ audio: { renditions: { audio } } }); + const enabled = new Signal(true); const broadcast = new Signal({ + in: { enabled }, out: { catalog }, relativeBroadcast, } as unknown as Broadcast); @@ -216,6 +218,8 @@ async function play(initial: Delay) { return { delay, + enabled, + context: decoder.out.context, play, // Remove the rendition from the catalog, then restore it. remove: () => catalog.set({}), @@ -254,6 +258,29 @@ async function play(initial: Delay) { }; } +describe("Decoder across a broadcast disable", () => { + it("releases the graph when its broadcast is disabled and rebuilds on return", async () => { + const playback = await play(Time.Milli(100)); + try { + await playback.play(); + const [context] = contexts; + + playback.enabled.set(false); + await microtasks(); + expect(context.calls.close).toBe(1); + expect(playback.context.peek()).toBeUndefined(); + + playback.enabled.set(true); + await microtasks(); + expect(contexts).toHaveLength(2); + expect(contexts[1].state).toBe("running"); + expect(playback.context.peek()).toBe(contexts[1] as unknown as AudioContext); + } finally { + playback.close(); + } + }); +}); + describe("Decoder across a delay change", () => { for (const initial of [Time.Milli(100), "auto"] as const) { it(`keeps the subscription and ring when ${initial} becomes a number`, async () => { diff --git a/js/watch/src/audio/decoder.ts b/js/watch/src/audio/decoder.ts index cfcf455365..c16c681954 100644 --- a/js/watch/src/audio/decoder.ts +++ b/js/watch/src/audio/decoder.ts @@ -207,6 +207,9 @@ export class Decoder { // This is less efficient for video-only playback but makes muting/unmuting instant. const shape = effect.get(this.#shape); if (!shape) return; + // Rendition absence keeps the graph warm, but a disabled broadcast releases its resources. + const broadcast = effect.get(this.source.in.broadcast); + if (!broadcast || !effect.get(broadcast.in.enabled)) return; const { sampleRate, channels: channelCount } = shape; From ccfa8259738d69ec24fa1a89b595b6b1e4ca57e4 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 6 Oct 2026 23:28:08 -0700 Subject: [PATCH 12/12] Preserve early-demand regression after catalog completion Co-Authored-By: GPT-6 --- quest/m1/README.md | 1 + quest/m1/catalog-early-demand.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) create mode 100644 quest/m1/catalog-early-demand.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 9ea80579de..48f1ab7adc 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -49,6 +49,7 @@ blocks. The quests that gated m0 lines moved under them. - [Bump mp4-atom for avc3](/quest/m1/mp4-atom-avc3-bump.md) - this repository depends on mp4-atom 0.16.2, which reads and writes an `avc3` sample entry - [fMP4 init from the catalog](/quest/m1/fmp4-catalog-init.md) - fMP4 export writes avc3/hev1 entries from the catalog, so an Annex-B H.264 or H.265 init no longer waits for the first keyframe - [WebTransport close backends](/quest/m1/wt-close-backends.md) - iroh's client reads a peer's capsule close, and moq-uring's HTTP/3 close task keeps the session alive until the capsule is sent +- [Early video demand](/quest/m1/catalog-early-demand.md) - a regression guards demand before the first keyframe after the detector removal - [Hang changelog](/quest/m1/hang-changelog-04.md) - the hang draft lists under -03 only what -03 published - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it - [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - one dynamic producer per track name in both languages, with the sequence namespace surviving a replacement diff --git a/quest/m1/catalog-early-demand.md b/quest/m1/catalog-early-demand.md new file mode 100644 index 0000000000..5e724c5d2a --- /dev/null +++ b/quest/m1/catalog-early-demand.md @@ -0,0 +1,29 @@ +# [XS] Guard video demand before the first keyframe + +## Goal + +A regression test proves that demand on a video track before its first +keyframe does not end the import or modify an unpublished rendition. +It guards the importer failure reported in #4945. + +## Plan + +The enabled-flag change in [#4915](https://github.com/moq-dev/moq/pull/4915) +removes `publish_stalled` and the importer detector hooks that called +`track.modify()` before a configuration was published. The catalog now +publishes a resolved configuration through `track.set()`; it has no +mutation driven by early demand. The relay half of the report is fixed by +[#4942](https://github.com/moq-dev/moq/pull/4942)'s publisher epochs. + +The source fixes are present; the before-first-keyframe regression from +the imported issue remains to be added. Exercise a video importer with +track demand before it resolves its configuration, then deliver its first +keyframe and require the import to continue. Use mocked time if needed, +and keep the test in the existing moq-mux Check/Test suite. Do not restore +the removed detector or add an availability mechanism. + +Public API: none. Wire: none. + +## Closes + +- [#4945](https://github.com/moq-dev/moq/issues/4945) - early video demand does not end a replacement import with "rendition is not published"