diff --git a/dart/moq/lib/src/aliases.dart b/dart/moq/lib/src/aliases.dart index 39accb60e6..d5884af22e 100644 --- a/dart/moq/lib/src/aliases.dart +++ b/dart/moq/lib/src/aliases.dart @@ -149,7 +149,7 @@ typedef Frame = MoqFrame; /// A media [Frame] whose keyframe flag marks group starts or video keyframes; audio flags only group starts. typedef MediaFrame = MoqMediaFrame; -/// The catalog description of a video track, including whether the publisher recommends temporarily avoiding it. +/// The catalog description of a video track, including whether it is enabled (a disabled one has no frames coming). typedef Video = MoqVideo; /// Caller-provided catalog fields for a video track. @@ -164,7 +164,7 @@ typedef VideoProperties = MoqVideoProperties; /// A single video codec an importer can parse. typedef VideoFormat = MoqVideoFormat; -/// The catalog description of an audio track: codec, sample rate, channels, and container. +/// The catalog description of an audio track: codec, sample rate, channels, whether it is enabled, and container. typedef Audio = MoqAudio; /// An audio codec, its required init bytes, and an optional label. diff --git a/dart/moq_ffi/lib/src/moq.dart b/dart/moq_ffi/lib/src/moq.dart index ed6fc8b261..e362a8d99c 100644 --- a/dart/moq_ffi/lib/src/moq.dart +++ b/dart/moq_ffi/lib/src/moq.dart @@ -412,6 +412,7 @@ class MoqAudio { final int sampleRate; final int channelCount; final int? bitrate; + final bool enabled; final MoqContainer container; MoqAudio({ this.label = null, @@ -421,6 +422,7 @@ class MoqAudio { required this.sampleRate, required this.channelCount, this.bitrate, + this.enabled = true, required this.container, }); } @@ -467,6 +469,11 @@ class FfiConverterMoqAudio { ); final bitrate = bitrate_lifted.value; new_offset += bitrate_lifted.bytesRead; + final enabled_lifted = FfiConverterBool.read( + Uint8List.view(buf.buffer, new_offset), + ); + final enabled = enabled_lifted.value; + new_offset += enabled_lifted.bytesRead; final container_lifted = FfiConverterMoqContainer.read( Uint8List.view(buf.buffer, new_offset), ); @@ -481,6 +488,7 @@ class FfiConverterMoqAudio { sampleRate: sampleRate, channelCount: channelCount, bitrate: bitrate, + enabled: enabled, container: container, ), new_offset - buf.offsetInBytes, @@ -496,6 +504,7 @@ class FfiConverterMoqAudio { FfiConverterUInt32.allocationSize(value.sampleRate) + FfiConverterUInt32.allocationSize(value.channelCount) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + 0; final buf = Uint8List(total_length); @@ -533,6 +542,10 @@ class FfiConverterMoqAudio { value.bitrate, Uint8List.view(buf.buffer, new_offset), ); + new_offset += FfiConverterBool.write( + value.enabled, + Uint8List.view(buf.buffer, new_offset), + ); new_offset += FfiConverterMoqContainer.write( value.container, Uint8List.view(buf.buffer, new_offset), @@ -548,6 +561,7 @@ class FfiConverterMoqAudio { FfiConverterUInt32.allocationSize(value.sampleRate) + FfiConverterUInt32.allocationSize(value.channelCount) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterMoqContainer.allocationSize(value.container) + 0; } @@ -1105,7 +1119,7 @@ class MoqVideo { final MoqDimensions? coded; final MoqDimensions? displayAspect; final int? bitrate; - final bool stalled; + final bool enabled; final double? framerate; final MoqContainer container; MoqVideo({ @@ -1116,7 +1130,7 @@ class MoqVideo { this.coded, this.displayAspect, this.bitrate, - this.stalled = false, + this.enabled = true, this.framerate, required this.container, }); @@ -1164,11 +1178,11 @@ class FfiConverterMoqVideo { ); final bitrate = bitrate_lifted.value; new_offset += bitrate_lifted.bytesRead; - final stalled_lifted = FfiConverterBool.read( + final enabled_lifted = FfiConverterBool.read( Uint8List.view(buf.buffer, new_offset), ); - final stalled = stalled_lifted.value; - new_offset += stalled_lifted.bytesRead; + final enabled = enabled_lifted.value; + new_offset += enabled_lifted.bytesRead; final framerate_lifted = FfiConverterOptionalDouble64.read( Uint8List.view(buf.buffer, new_offset), ); @@ -1188,7 +1202,7 @@ class FfiConverterMoqVideo { coded: coded, displayAspect: displayAspect, bitrate: bitrate, - stalled: stalled, + enabled: enabled, framerate: framerate, container: container, ), @@ -1205,7 +1219,7 @@ class FfiConverterMoqVideo { FfiConverterOptionalMoqDimensions.allocationSize(value.coded) + FfiConverterOptionalMoqDimensions.allocationSize(value.displayAspect) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + - FfiConverterBool.allocationSize(value.stalled) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + 0; @@ -1245,7 +1259,7 @@ class FfiConverterMoqVideo { Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterBool.write( - value.stalled, + value.enabled, Uint8List.view(buf.buffer, new_offset), ); new_offset += FfiConverterOptionalDouble64.write( @@ -1267,7 +1281,7 @@ class FfiConverterMoqVideo { FfiConverterOptionalMoqDimensions.allocationSize(value.coded) + FfiConverterOptionalMoqDimensions.allocationSize(value.displayAspect) + FfiConverterOptionalUInt64.allocationSize(value.bitrate) + - FfiConverterBool.allocationSize(value.stalled) + + FfiConverterBool.allocationSize(value.enabled) + FfiConverterOptionalDouble64.allocationSize(value.framerate) + FfiConverterMoqContainer.allocationSize(value.container) + 0; diff --git a/doc/concept/hang.md b/doc/concept/hang.md index 51ddd0d346..e6a5e25792 100644 --- a/doc/concept/hang.md +++ b/doc/concept/hang.md @@ -56,7 +56,7 @@ A few things the catalog can express beyond decoder config: - **Renditions in another broadcast.** A rendition may point at a relative broadcast path, so a transcoder can publish a ladder that adds low rungs and references the source's original rendition without re-publishing its bytes. The path resolves against where the consumer found the catalog, so a reference that escapes above the root names nothing and the catalog is rejected. - **Jitter.** A rendition can say how far its frames fell behind the media clock before the publisher flushed them, in whole milliseconds rounded up. Encoders report the spread of lateness above each rendition's own recent minimum, so a constant encoder delay is not jitter; container imports estimate batch spans without counting ingest delay. It describes the publisher, never the network, only grows over the life of a stream, and a player sizes its buffer to at least this much. A `0` is read as absent. - **Delay.** A rendition can also say how far its frames reach the transport behind the broadcast's earliest rendition, measured the same way from each rendition's minimum lateness, so a video encoder running 200 ms behind audio advertises `delay: 200` on video. It follows the same rules as jitter. A player holds the largest `delay + jitter` among the renditions it subscribes to, and never subtracts one rendition's `delay` from another's. -- **Stalled renditions.** A publisher can flag a rendition as temporarily bad so players prefer another one without the track disappearing. First-party video publishers set this flag after more than three frame intervals of source silence or encoding lag while subscribed, and clear it after three on-time completed frames or when idle. Browser and native capture poll while waiting; FLV and MPEG-TS importers observe video silence as container data arrives. The shared detector is `hang::catalog::stalled::Detector` in Rust and `Catalog.Stalled.Detector` in JavaScript. It is a playback diagnostic, not an authorization or routing signal. +- **Disabled renditions.** An audio or video rendition with `enabled: false` has no frames coming, and a player must not select it. Pausing a rendition in `@moq/publish` keeps it in the catalog this way rather than removing it, so a mute is a one-field update; a disabled video rendition encodes one black keyframe first, so a player that predates the field shows black instead of a frozen picture. The field is written only when false. The older video `stalled` field is no longer written, and readers ignore it. - **Archive.** A broadcast may advertise an `archive` entry mapping each track, the catalog included, to its own timeline track (a small index of the track's spans: timing plus group and frame positions) and, if recorded, the replay MoQ path, object-store URL, and format version. Every track is cut by one rule, at group boundaries between 2s and 10s (a zero minimum for sparse data such as the catalog), and tracks commit and expire independently. The timelines are what let the [HLS gateway](/bin/hls) build playlists without subscribing to media. - **Clock.** The optional root `clock` maps PTS zero to wall time so every media track and the archive index share one fixed epoch after timescale conversion. It is independent of `archive`, so a live-only publisher can expose wall-clock timing without creating a timeline. - **Extensions.** The root is a loose object. Applications add their own sections (`scte35`, for example) next to the ones hang defines, optionally naming a track that carries the data. Every library exposes a way to write your section without clobbering the built-in ones, and readers ignore what they don't know. diff --git a/doc/concept/standard.md b/doc/concept/standard.md index 4757aaa99e..6b4f008598 100644 --- a/doc/concept/standard.md +++ b/doc/concept/standard.md @@ -125,7 +125,6 @@ The MoQ Streaming Format is a catalog, playing the role HLS playlists and SDP do elsewhere. It overlaps with the [hang catalog](/concept/hang) and the two will likely converge. The tools track draft-01 and hide the version on the wire, so draft-00 catalogs still decode and init data always arrives inline. -The `stalled` rendition hint is shared between the two formats. ## LOC diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index cdd05db93e..90c4eae199 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -52,7 +52,7 @@ pkg-config file included. - **Server.** `moq_server_listen` binds before it returns (a bad address or certificate fails there) and hands each incoming session to `on_request` as a request handle. Read `moq_session_request_path` and `_query` to route and authenticate, then `moq_session_request_accept` (a session handle, with origins like `moq_session_connect`) or `moq_session_request_reject` with an HTTP-style code (401 and 403 become the protocol's unauthorized close). An accepted session reports `1` once SETUP completes and never reconnects. `moq_server_addr` reports an ephemeral port and `moq_server_fingerprints` the hashes a client pins for a `tls_generate` certificate. `moq_server_close` stops listening; its terminal callback fires once the sockets are released. - **Demand.** A watcher on a published track (`moq_publish_track_demand`, `moq_publish_media_demand`, `moq_encode_video_demand`, `moq_encode_audio_demand`) calls `on_demand` with `MOQ_DEMAND_USED` or `MOQ_DEMAND_UNUSED` right away and again on every change, so an encoder on a battery-powered device runs only while someone is watching. The first call is the current state, so a track that went unused before the watcher existed still reports it. `moq_publish_demand_cancel` stops it; the terminal callback still fires. A container has no single demand and is refused. Demand follows the last real subscriber: an origin that served the track drops its source copy on the unused edge, so a cache linger never delays it. Only a relay keeps what it already delivered warm for 30 seconds, and a returning subscriber is served that cache only once the publisher confirms it is still current. - **Requests.** `moq_publish_dynamic` serves subscriptions to tracks the broadcast never declared: each arrives as a request handle, read its name with `moq_track_request_name`, then `moq_track_request_accept` (a raw track handle), `moq_track_request_video` / `_audio` (the media handle `moq_publish_video` / `_audio` return), or `moq_track_request_abort` with an application code the subscriber sees. Without a live handler an unknown name is refused. `moq_publish_track_dynamic` does the same for fetches of groups a track no longer has cached, delivered as `moq_group_request_*` (`sequence`, `priority`, `frame_start`); `moq_group_request_accept` starts the producer at `frame_start` so written frames keep their group indices. Register it with `moq_track_request_dynamic` before accepting a track that was itself requested by a fetch, so that pending group survives the transition. Both handlers stop with `moq_publish_dynamic_cancel`. -- **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams (one read from an untimed track has `timestamp_present` false), JSON and binary data tracks (snapshot or stream, each advertised in the catalog for as long as it lives), group fetch, catalog sections, shared video properties, and stalled hints. The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), `moq_publish_close` (ends the broadcast for good and releases its handle), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a `moq_announce_config` (NULL or zeroed for everything): a literal prefix and an optional relative pattern filter; each `moq_announce_event` has a `kind` (`MOQ_ANNOUNCE_KIND_START`, `_UPDATE`, or `_END`); `prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts); set `hidden` or name the dot segment in `prefix` to list them. +- **Everything the bindings can do** ([list](/lib/#what-every-binding-can-do)): media publish and consume with the catalog managed for you, raw pixels and PCM with the codec inside (`moq_encode_video`, `moq_encode_audio`, and the `moq_decode_*` mirrors), raw tracks with timestamps and datagrams (one read from an untimed track has `timestamp_present` false), JSON and binary data tracks (snapshot or stream, each advertised in the catalog for as long as it lives), group fetch, catalog sections, shared video properties, and whether a video rendition is enabled (`moq_consume_video_enabled`). The three advertising operations are `moq_origin_create_broadcast` (unannounced producer, invisible to everyone), `moq_publish_announce` / `moq_publish_unannounce` (exact-path advertisement), `moq_publish_close` (ends the broadcast for good and releases its handle), and `moq_origin_dynamic` (a claim over a path prefix and everything beneath it; `""` for everything). A route is a capability, not an inventory. `moq_origin_announced` takes a `moq_announce_config` (NULL or zeroed for everything): a literal prefix and an optional relative pattern filter; each `moq_announce_event` has a `kind` (`MOQ_ANNOUNCE_KIND_START`, `_UPDATE`, or `_END`); `prefix` stays relative to the origin, while `captures` reports what each wildcard matched when `has_captures` is true. Paths with a `.`-prefixed segment below the prefix are [hidden](/concept/moq-lite#hidden-broadcasts); set `hidden` or name the dot segment in `prefix` to list them. ```c moq_client_config config; diff --git a/doc/lib/rs/hang.md b/doc/lib/rs/hang.md index 7ad276472f..4c54739967 100644 --- a/doc/lib/rs/hang.md +++ b/doc/lib/rs/hang.md @@ -12,7 +12,7 @@ The [hang media format](/concept/hang) on top of `moq-net`: a live catalog describing renditions with WebCodecs-style decoder configs, and containers that carry a timestamp with every frame. -- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), stalled hints, cross-broadcast rendition references, and the `archive` entry (each track's timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. +- **Catalog producer and consumer.** Typed `VideoConfig`, `AudioConfig` and text renditions, the `json` and `binary` data track sections, shared video properties (display size, rotation, flip), per-rendition `enabled` flags, cross-broadcast rendition references, and the `archive` entry (each track's timeline track, optional replay path, store URL, and format version). Extend it with your own sections through a lock or `#[serde(flatten)]`, or read unknown sections as raw JSON. - **Containers.** `legacy`, `cmaf`, and `loc`, decoded for you; unknown kinds pass through untouched. - **`Ordered`** reads a media track as timestamped frames, reorders groups, and skips ones that fall past your max delay. - **Codecs described**: H.264, H.265, VP8, VP9, AV1, AAC, Opus, PCM. diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 38dc856715..eb49df92a0 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -192,7 +192,6 @@ In addition to the WebCodecs fields, each rendition MAY carry the common renditi type VideoDecoderConfigExtensions = { "displayAspectWidth": number | undefined, "displayAspectHeight": number | undefined, - "stalled": boolean | undefined, } ~~~ @@ -200,11 +199,6 @@ type VideoDecoderConfigExtensions = { A consumer that understands neither field MUST assume square pixels, a 1:1 ratio. Both MUST be present together; a consumer that sees only one MUST ignore it. -`stalled` indicates that the publisher recommends temporarily avoiding the rendition. -The track remains available when `stalled` is true. -A consumer SHOULD select an unstalled rendition when it supports one, but MAY select a stalled rendition when no unstalled rendition is suitable. -If absent, `stalled` defaults to false. - For example: ~~~ @@ -217,7 +211,7 @@ For example: "codedWidth": 1280, "codedHeight": 720, "bitrate": 6000000, - "stalled": true, + "enabled": false, "framerate": 30.0, "jitter": 34 }, @@ -308,7 +302,7 @@ type TextConfig = { "format": "vtt" | "ttml" | "utf8" | string, "role": "subtitle" | "caption" | string | undefined, "lang": string | undefined, - // plus the common rendition fields + // plus the common rendition fields, except `enabled` } ~~~ @@ -489,6 +483,7 @@ type CommonExtensions = { "container": Container, "jitter": number | undefined, "delay": number | undefined, + "enabled": boolean | undefined, } ~~~ @@ -553,6 +548,14 @@ A consumer MUST NOT subtract one rendition's `delay` from another's: each is a m For example, a video encoder that flushes 200 milliseconds after the audio encoder for the same media time advertises a video `delay` of 200 and no audio `delay`. +### enabled {#field-enabled} +The `enabled` field says whether a consumer may select an audio or video rendition; a text rendition does not carry it. +When `enabled` is false, no frames are coming, and a consumer MUST NOT select the rendition. +If absent, `enabled` defaults to true; a publisher SHOULD only write it when false. + +Earlier versions defined a video `stalled` field instead. +A publisher MUST NOT write `stalled`, and a consumer MUST ignore it. + # Container {#container} Audio, video, and text tracks use a container to encapsulate the media payload. A rendition declares its container via the `container` field of its catalog entry ({{common}}): @@ -1063,6 +1066,7 @@ This document has no IANA actions. ## moq-hang-04 {:numbered="false"} +- Replaced the video `stalled` field with an optional `enabled` field on audio and video renditions. A consumer MUST NOT select a disabled rendition and ignores `stalled`. A consumer that predates `enabled` keeps selecting a disabled rendition. - Replaced the broadcast's one aligned timeline with one timeline per track: the catalog `archive` entry's `track` became a `timelines` map from each indexed track, the catalog included, to its timeline track. - Replaced the segment record with a per-track record: `sequence`, `pts`, `duration`, and a `start`/`end` range of group and frame positions, dropping cross-track pacing and completeness. - One cutting rule for every track: a record ends at the first group boundary past a minimum (2 seconds RECOMMENDED, zero for sparse data such as a catalog) and splits a group between frames at a maximum (10 seconds RECOMMENDED), so a group that never closes is indexed as it grows and `durationMax` bounds every record. diff --git a/go/wrapper/types.go b/go/wrapper/types.go index c7d4fdaff7..35a3d2696b 100644 --- a/go/wrapper/types.go +++ b/go/wrapper/types.go @@ -6,7 +6,7 @@ import ffi "moq.dev/moq-ffi/moq" // These are plain data, so type aliases are exact: a moq.AudioFrame is an // ffi.MoqAudioFrame, constructible and comparable across the boundary. type ( - // Audio describes one audio rendition in a broadcast catalog: codec, sample rate, channel count, and container. + // Audio describes one audio rendition in a broadcast catalog: codec, sample rate, channel count, whether it is enabled, and container. Audio = ffi.MoqAudio // AudioCodec selects the audio encoder codec. Build one with OpusAudioCodec or AacAudioCodec; // adding a codec later adds a constructor, not a breaking enum change. @@ -64,7 +64,7 @@ type ( // TrackInfo holds publisher-side track properties: priority, ordering, max age, and timescale. // A zero Priority is the least urgent, not the default; set 127 for the midpoint a nil TrackInfo uses. TrackInfo = ffi.MoqTrackInfo - // Video describes one catalog rendition, including whether the publisher recommends temporarily avoiding it. + // Video describes one catalog rendition, including whether it is enabled (a disabled one has no frames coming). Video = ffi.MoqVideo // VideoHint supplies catalog fields a video stream can't reveal itself, such as bitrate, filling only the gaps. VideoHint = ffi.MoqVideoHint diff --git a/js/hang/src/catalog/audio.test.ts b/js/hang/src/catalog/audio.test.ts index 9ad31d8365..e3bc0d5b4a 100644 --- a/js/hang/src/catalog/audio.test.ts +++ b/js/hang/src/catalog/audio.test.ts @@ -24,3 +24,15 @@ test("audio config accepts a human-readable label", () => { expect(config.label).toBe("English"); }); + +test("audio config accepts optional enabled state", () => { + const config = AudioConfigSchema.parse({ + codec: "opus", + container: { kind: "legacy" }, + sampleRate: 48_000, + numberOfChannels: 2, + enabled: false, + }); + + expect(config.enabled).toBe(false); +}); diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index f897c9125a..451f7be337 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -44,6 +44,10 @@ export const AudioConfigSchema = z.object({ // TODO: Support up to Number.MAX_SAFE_INTEGER bitrate: z.optional(u53Schema), + // Whether this rendition may be selected. When false, no frames are coming and a consumer + // must not select it. Default: true, so publishers only write it when false. + enabled: z.optional(z.boolean()), + // The maximum delay between a frame being ready and the publisher flushing it, in whole // milliseconds rounded up. The player's jitter buffer should be larger than this value. // If not provided, the player should assume each frame is flushed immediately. diff --git a/js/hang/src/catalog/index.ts b/js/hang/src/catalog/index.ts index ceb52fd14f..4a59edca7b 100644 --- a/js/hang/src/catalog/index.ts +++ b/js/hang/src/catalog/index.ts @@ -20,7 +20,6 @@ export * from "./mode"; export * from "./path"; export * from "./priority"; export * from "./root"; -export * as Stalled from "./stalled"; export * from "./text"; export * from "./track"; export * from "./video"; diff --git a/js/hang/src/catalog/stalled.test.ts b/js/hang/src/catalog/stalled.test.ts deleted file mode 100644 index fbdef4ceff..0000000000 --- a/js/hang/src/catalog/stalled.test.ts +++ /dev/null @@ -1,122 +0,0 @@ -import { expect, test } from "bun:test"; -import { Time } from "@moq/net"; -import { CLEAR_FRAMES, DEFAULT_INTERVAL, Detector, intervalFromFps, SET_INTERVALS } from "./stalled"; - -const FRAME = Time.Micro.fromMilli(33 as Time.Milli); - -function sample(lag: Time.Micro): Parameters[0] { - return { - frame: true, - mediaLag: lag, - quiet: 0 as Time.Micro, - interval: FRAME, - demand: true, - idle: false, - }; -} - -test("idle is never stalled", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - idle: true, - demand: true, - mediaLag: Time.Micro.fromSecond(10 as Time.Second), - quiet: Time.Micro.fromSecond(10 as Time.Second), - interval: FRAME, - }), - ).toBe(false); - expect(stalled.stalled).toBe(false); -}); - -test("no demand is never stalled", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - demand: false, - idle: false, - mediaLag: Time.Micro.fromSecond(10 as Time.Second), - quiet: Time.Micro.fromSecond(10 as Time.Second), - interval: FRAME, - }), - ).toBe(false); - expect(stalled.stalled).toBe(false); -}); - -test("lag past the threshold sets the flag", () => { - const stalled = new Detector(); - expect(stalled.observe(sample((FRAME * SET_INTERVALS) as Time.Micro))).toBe(false); - expect(stalled.observe(sample((FRAME * SET_INTERVALS + 1) as Time.Micro))).toBe(true); - expect(stalled.flag()).toBe(true); -}); - -test("a quiet source sets the flag", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - mediaLag: 0 as Time.Micro, - quiet: (FRAME * SET_INTERVALS + 1) as Time.Micro, - interval: FRAME, - demand: true, - idle: false, - }), - ).toBe(true); - expect(stalled.stalled).toBe(true); -}); - -test("clearing needs a run of on-time frames", () => { - const stalled = new Detector(); - stalled.observe(sample((FRAME * 10) as Time.Micro)); - for (let i = 0; i < CLEAR_FRAMES - 1; i++) { - expect(stalled.observe(sample(10 as Time.Micro))).toBe(false); - expect(stalled.stalled).toBe(true); - } - expect(stalled.observe(sample(10 as Time.Micro))).toBe(true); - expect(stalled.stalled).toBe(false); - expect(stalled.flag()).toBeUndefined(); -}); - -test("dropping demand clears immediately", () => { - const stalled = new Detector(); - stalled.observe(sample((FRAME * 10) as Time.Micro)); - expect(stalled.observe({ ...sample((FRAME * 10) as Time.Micro), demand: false })).toBe(true); - expect(stalled.stalled).toBe(false); -}); - -test("a zero interval uses the default", () => { - const stalled = new Detector(); - expect( - stalled.observe({ - frame: true, - mediaLag: (DEFAULT_INTERVAL * SET_INTERVALS + 1) as Time.Micro, - quiet: 0 as Time.Micro, - interval: 0 as Time.Micro, - demand: true, - idle: false, - }), - ).toBe(true); -}); - -test("intervalFromFps falls back", () => { - expect(intervalFromFps(undefined)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(0)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(Number.NaN)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(50)).toBe(Time.Micro.fromMilli(20 as Time.Milli)); -}); - -test("polls do not count as recovery frames", () => { - const stalled = new Detector(); - stalled.observe(sample((FRAME * 10) as Time.Micro)); - for (let i = 0; i < 10; i++) { - expect(stalled.observe({ ...sample(0 as Time.Micro), frame: false })).toBe(false); - } - expect(stalled.stalled).toBe(true); -}); - -test("unrepresentable frame intervals use the default", () => { - expect(intervalFromFps(Number.MIN_VALUE)).toBe(DEFAULT_INTERVAL); - expect(intervalFromFps(Number.MAX_VALUE)).toBe(DEFAULT_INTERVAL); -}); diff --git a/js/hang/src/catalog/stalled.ts b/js/hang/src/catalog/stalled.ts deleted file mode 100644 index dfec96753f..0000000000 --- a/js/hang/src/catalog/stalled.ts +++ /dev/null @@ -1,93 +0,0 @@ -import { Time } from "@moq/net"; - -/** How many frame intervals of lag mark a rendition stalled. */ -export const SET_INTERVALS = 3; - -/** Consecutive on-time frames required to clear a stall. */ -export const CLEAR_FRAMES = 3; - -/** Frame interval used when the catalog does not advertise a framerate. */ -export const DEFAULT_INTERVAL = Time.Micro.fromMilli(33 as Time.Milli); - -/** One observation of a rendition's source versus what the transport has accepted. */ -export interface Sample { - /** This observation completes a newly accepted frame. */ - frame: boolean; - /** Newest captured timestamp minus newest timestamp handed to the transport. */ - mediaLag: Time.Micro; - /** Wall time since the source last delivered a frame, in microseconds. */ - quiet: Time.Micro; - /** One frame interval. The set threshold is {@link SET_INTERVALS} of these. */ - interval: Time.Micro; - /** Someone is subscribed to this rendition. */ - demand: boolean; - /** Camera released / no live source. Never stalled. */ - idle: boolean; -} - -/** A positive frame interval, falling back to {@link DEFAULT_INTERVAL}. */ -export function interval(value: Time.Micro): Time.Micro { - return value > 0 ? value : DEFAULT_INTERVAL; -} - -/** Frame interval implied by a catalog framerate, or {@link DEFAULT_INTERVAL}. */ -export function intervalFromFps(fps: number | undefined): Time.Micro { - if (fps === undefined || !Number.isFinite(fps) || fps <= 0) return DEFAULT_INTERVAL; - const value = Time.Micro.fromSecond((1 / fps) as Time.Second); - return Number.isFinite(value) && value >= 0.0005 && value < 2 ** 64 * 1_000_000 ? value : DEFAULT_INTERVAL; -} - -/** - * Hysteresis around the catalog `stalled` bit. - * - * Set once lag exceeds a few frame intervals; clear only after a run of on-time - * frames. Idle or undemanded broadcasts are never stalled. - */ -export class Detector { - #on = false; - #recover = 0; - - /** Whether the rendition is currently stalled. */ - get stalled(): boolean { - return this.#on; - } - - /** The catalog value: `true` while stalled, omitted otherwise. */ - flag(): true | undefined { - return this.#on ? true : undefined; - } - - /** Feed one sample. Returns whether the catalog flag changed. */ - observe(sample: Sample): boolean { - if (sample.idle || !sample.demand) return this.#clear(); - - const frame = interval(sample.interval); - const lag = Math.max(sample.mediaLag, sample.quiet) as Time.Micro; - const over = lag > frame * SET_INTERVALS; - const onTime = lag <= frame; - - if (!this.#on) { - if (!over) return false; - this.#on = true; - this.#recover = 0; - return true; - } - - if (!onTime) { - this.#recover = 0; - return false; - } - - if (!sample.frame) return false; - this.#recover++; - if (this.#recover < CLEAR_FRAMES) return false; - return this.#clear(); - } - - #clear(): boolean { - this.#recover = 0; - if (!this.#on) return false; - this.#on = false; - return true; - } -} diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index 513d433044..be03b285a6 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -70,15 +70,26 @@ test("video config accepts canonical display aspect fields", () => { expect("displayRatioHeight" in parsed).toBe(false); }); -test("video config accepts optional stalled state", () => { +test("video config accepts optional enabled state", () => { const active = VideoConfigSchema.parse({ codec: "avc1.64001f", container: { kind: "legacy" }, }); - const stalled = VideoConfigSchema.parse({ ...active, stalled: true }); + const disabled = VideoConfigSchema.parse({ ...active, enabled: false }); - expect(active.stalled).toBeUndefined(); - expect(stalled.stalled).toBe(true); + expect(active.enabled).toBeUndefined(); + expect(disabled.enabled).toBe(false); +}); + +test("video config ignores a legacy stalled flag", () => { + const parsed = VideoConfigSchema.parse({ + codec: "avc1.64001f", + container: { kind: "legacy" }, + stalled: true, + }); + + expect("stalled" in parsed).toBe(false); + expect(parsed.enabled).toBeUndefined(); }); test("video config accepts a human-readable label", () => { diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 42bc727816..1eb1bd072e 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -48,9 +48,9 @@ export const VideoConfigSchema = z.object({ // TODO: Support up to Number.MAX_SAFE_INTEGER bitrate: z.optional(u53Schema), - // Whether the publisher recommends temporarily avoiding this rendition. - // The track remains available and may still be selected as a fallback. - stalled: z.optional(z.boolean()), + // Whether this rendition may be selected. When false, no frames are coming and a consumer + // must not select it. Default: true, so publishers only write it when false. + enabled: z.optional(z.boolean()), // If true, the decoder will optimize for latency. // Default: true diff --git a/js/msf/src/catalog.test.ts b/js/msf/src/catalog.test.ts index a6e16a222b..7501ed5f5b 100644 --- a/js/msf/src/catalog.test.ts +++ b/js/msf/src/catalog.test.ts @@ -209,27 +209,6 @@ test("preserves delay through decode and encode", () => { expect(wireTracks[0].delay).toBe(200); }); -test.each([ - ["omitted", undefined], - ["false", false], - ["true", true], -] as const)("preserves %s stalled state through decode and encode", (_name, stalled) => { - const track = { - name: "video0", - packaging: "loc", - isLive: true, - role: "video", - codec: "av01.0.08M.10.0.110.09", - ...(stalled === undefined ? {} : { stalled }), - }; - const catalog = decode(encodeJson({ version: "draft-01", tracks: [track] })); - - expect(catalog.tracks[0].stalled).toBe(stalled); - const wire = decodeJson(encode(catalog)); - const tracks = wire.tracks as { stalled?: boolean }[]; - expect(tracks[0].stalled).toBe(stalled); -}); - test("carries extension root members through decode and encode", () => { // An extension section (here `mpegts`) rides the catalog root untouched, and a // section this build has never heard of survives the same way. diff --git a/js/msf/src/catalog.ts b/js/msf/src/catalog.ts index 9596adbee0..b3b3bb9224 100644 --- a/js/msf/src/catalog.ts +++ b/js/msf/src/catalog.ts @@ -35,8 +35,6 @@ const trackShape = { samplerate: z.optional(z.number()), channelConfig: z.optional(z.string()), bitrate: z.optional(z.number()), - // Non-standard: whether the publisher recommends temporarily avoiding this track. - stalled: z.optional(z.boolean()), /** Resolved base64 initialization data (draft-01's initRef indirection is resolved away). */ initData: z.optional(z.string()), renderGroup: z.optional(z.number()), diff --git a/js/publish/src/audio/encoder.test.ts b/js/publish/src/audio/encoder.test.ts index c8415751b1..9d2d78e90d 100644 --- a/js/publish/src/audio/encoder.test.ts +++ b/js/publish/src/audio/encoder.test.ts @@ -260,6 +260,7 @@ async function setup(baseline = new Baseline(), codec?: Codec, groupDuration?: T config, encoder, enabled, + capture, track, rendition, feed, @@ -351,6 +352,35 @@ test("disabling with a subscriber attached marks where submitted audio ends", as ]); }); +// A muted rendition stays in the catalog, so a viewer deselects it with a one-field delta instead of +// seeing it removed and re-added. +test("disabling keeps the rendition in the catalog with enabled: false", async () => { + using _webcodecs = installFakeWebCodecs(); + using env = await setup(); + const { encoder, enabled, capture, rendition } = env; + + const before = encoder.out.catalog.peek(); + expect(before).toBeDefined(); + expect(before?.enabled).toBeUndefined(); + + enabled.set(false); + await settle(); + expect(encoder.out.catalog.peek()).toEqual({ ...before, enabled: false } as never); + expect(rendition.config.peek()).toEqual({ ...before, enabled: false } as never); + expect(encoder.out.active.peek()).toBe(false); + + // Muting released the microphone, so re-enabling waits on its format without dropping the rendition. + const format = capture.out.format.peek(); + capture.out.format.set(undefined as never); + enabled.set(true); + await settle(); + expect(encoder.out.catalog.peek()).toEqual({ ...before, enabled: false } as never); + + capture.out.format.set(format); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(before); +}); + // Closing tears down the subscription and the pipeline, which both end the epoch; cleanups run // last-in, first-out, so the marker lands once and before the rendition closes the track. test("closing with a subscriber attached marks the end once before the track closes", async () => { diff --git a/js/publish/src/audio/encoder.ts b/js/publish/src/audio/encoder.ts index 8b09f6289f..f8c9b145d9 100644 --- a/js/publish/src/audio/encoder.ts +++ b/js/publish/src/audio/encoder.ts @@ -76,8 +76,9 @@ export interface Stats { // Signals the encoder reads. export type EncoderInput = { - // Whether to publish (and encode) this rendition. Defaults to true. When false the rendition drops out of the - // catalog and stops encoding, ending the epoch, but stays registered so a subscriber still gets an idle track. + // Whether to encode this rendition. Defaults to true. When false it stops encoding, ending the epoch, + // but stays in the catalog with `enabled: false` and its last config, and stays registered so a + // subscriber still gets an idle track. enabled: Getter; // The broadcast to register the rendition on. Undefined resolves the config but has nowhere to publish. @@ -109,7 +110,8 @@ export type EncoderProps = Inputs & { }; type EncoderOutput = { - // The catalog config published for this rendition, or undefined while there's no capture. + // The catalog config published for this rendition, `enabled: false` while disabled, or undefined + // while there's no capture. catalog: Signal; // The head of the capture graph, so callers can tap the raw capture. Volume is applied to the // PCM rather than in the graph, so this is pre-gain. Undefined for a source that isn't a track. @@ -218,6 +220,14 @@ export class Encoder { #signals = new Effect(); #estimator = new Estimator(); + // The estimator's jitter and delay, republished whenever either rises. + #estimate = new Signal({}); + // The last config published while enabled, which a disabled rendition keeps advertising. + #last?: Catalog.AudioConfig; + // Whether the rendition was disabled since a config last resolved, so it keeps advertising `#last` + // as disabled until the re-enabled capture resolves a new one. A capture that never reopens leaves + // it disabled, which is accurate: no frames are coming. + #paused = false; constructor(name: string, props?: EncoderProps) { // `source` moved to Audio.Capture, which renditions share. TypeScript catches this, but a @@ -391,8 +401,8 @@ export class Encoder { // Derive the encoder config from the captured format and the codec. Re-runs whenever either changes, so a // codec update (bitrate, frame duration) reconfigures without waiting for a channel-count change. // - // Gated on `enabled` the same way the video encoder is: a disabled rendition has to drop out of - // the catalog, and a sample source keeps its format while muted rather than tearing down. + // Gated on `enabled` the same way the video encoder is: a disabled rendition stops resolving a + // config, and a sample source keeps its format while muted rather than tearing down. #runConfig(effect: Effect): void { const fade = effect.get(this.fade); if (!Number.isFinite(fade) || fade < 0) @@ -415,20 +425,25 @@ export class Encoder { } // Publish the config immediately so a consumer can request the demand-gated track. Once encoding - // starts, republish Opus with the exact decoder description reported for that encoder config. + // starts, republish Opus with the exact decoder description reported for that encoder config. A + // disabled rendition keeps its last config, since muting also releases the capture, and stays + // disabled after re-enabling until the reopened capture reports its format. #runCatalog(effect: Effect): void { - const config = effect.get(this.#config)?.catalog; + const estimate = effect.get(this.#estimate); + const enabled = effect.get(this.in.enabled); + if (!enabled) this.#paused = true; + const config = enabled ? effect.get(this.#config)?.catalog : undefined; if (!config) { - effect.set(this.#out.catalog, undefined); + const last = this.#paused ? this.#last : undefined; + effect.set(this.#out.catalog, last && { ...last, ...estimate, enabled: false }); return; } + this.#paused = false; const decoder = effect.get(this.#decoderDescription); const catalog = decoder?.config === config ? { ...config, description: decoder.description } : config; - effect.set(this.#out.catalog, { - ...catalog, - ...this.#estimator.estimate, - }); + this.#last = { ...catalog, ...estimate }; + effect.set(this.#out.catalog, this.#last); } // Collect the encode-only Opus knobs that are set, reading the codec through the effect so the @@ -516,8 +531,7 @@ export class Encoder { if (keyframe) live.start = timestamp; live.producer.encode(frame, timestamp, keyframe); if (this.#estimator.flush(frame.timestamp, baseline)) { - const catalog = this.#out.catalog.peek(); - if (catalog) this.#out.catalog.set({ ...catalog, ...this.#estimator.estimate }); + this.#estimate.set({ ...this.#estimator.estimate }); } }, error: (err) => { diff --git a/js/publish/src/video/encoder.test.ts b/js/publish/src/video/encoder.test.ts index fe66ce0eb7..354970c571 100644 --- a/js/publish/src/video/encoder.test.ts +++ b/js/publish/src/video/encoder.test.ts @@ -1,4 +1,5 @@ import { expect, spyOn, test } from "bun:test"; +import * as Catalog from "@moq/hang/catalog"; import * as Container from "@moq/hang/container"; import * as Moq from "@moq/net"; import { Signal } from "@moq/signals"; @@ -488,83 +489,154 @@ test("frame sources retain their dimensions and nominal frame rate", async () => } }); -test.each(["encoder lag", "quiet startup"])("marks a rendition stalled for %s", async (reason) => { - const clock = spyOn(performance, "now").mockReturnValue(0); - // The fake holds every live output, like a throttled encoder that never hands frames to the session. - using _videoEncoder = installFakeVideoEncoder(); +// A viewer released before `enabled` keeps selecting a disabled rendition, so the last picture it +// decodes has to be black rather than a frozen frame. The rendition stays in the catalog with its last +// config, even once the capture is released. +test("disabling encodes one black keyframe and keeps the rendition with enabled: false", async () => { + const encoded: { timestamp: number; keyFrame: boolean; black: boolean }[] = []; + + class RecordingVideoFrame { + readonly timestamp: number; + readonly black: boolean; + constructor(data: unknown, init?: { timestamp?: number }) { + this.timestamp = init?.timestamp ?? 0; + this.black = data instanceof Uint8Array && data[0] === 0x10; + } + close(): void {} + } + + class RecordingVideoEncoder { + state: CodecState = "unconfigured"; + #output: VideoEncoderInit["output"]; + #codec?: string; + #last = 0; + + constructor(init: VideoEncoderInit) { + this.#output = init.output; + } + + static async isConfigSupported(config: VideoEncoderConfig): Promise<{ supported: boolean }> { + return { supported: config.codec.startsWith("avc1") }; + } + + configure(config: VideoEncoderConfig): void { + this.state = "configured"; + this.#codec = config.codec; + } + + encode(frame: { timestamp: number; black?: boolean }, options?: VideoEncoderEncodeOptions): void { + this.#last = frame.timestamp; + encoded.push({ timestamp: frame.timestamp, keyFrame: !!options?.keyFrame, black: !!frame.black }); + } + + // Emits the newest frame, and reports the configured codec as Chrome does. + async flush(): Promise { + const chunk = { type: "key", timestamp: this.#last, byteLength: 1, copyTo: () => {} }; + this.#output(chunk as never, { decoderConfig: { codec: this.#codec } } as never); + } + + close(): void { + this.state = "closed"; + } + } + + const original = Object.getOwnPropertyDescriptor(globalThis, "VideoEncoder"); + Object.defineProperty(globalThis, "VideoEncoder", { + configurable: true, + value: RecordingVideoEncoder, + writable: true, + }); + const videoFrame = Object.getOwnPropertyDescriptor(globalThis, "VideoFrame"); + Object.defineProperty(globalThis, "VideoFrame", { + configurable: true, + value: RecordingVideoFrame, + writable: true, + }); class Frame { - codedWidth = 640; - codedHeight = 480; - timestamp: number; - closed = false; + readonly timestamp: number; constructor(timestamp: number) { this.timestamp = timestamp; } clone(): Frame { return new Frame(this.timestamp); } - close(): void { - this.closed = true; - } + close(): void {} } const { Fanout } = await import("../fanout"); let controller!: ReadableStreamDefaultController; - const stream = new ReadableStream({ - start: (c) => { - controller = c; - }, - }); - const fanout = new Fanout(stream, { - clone: (frame) => frame.clone(), - release: (frame) => frame.close(), - }); + const fanout = new Fanout( + new ReadableStream({ + start: (c) => { + controller = c; + }, + }), + { clone: (frame) => frame.clone(), release: (frame) => frame.close() }, + ); - const track = new Moq.Track.Producer("video/hd").accept({ timescale: Moq.Time.Timescale.MILLI }); + const track = new Moq.Track.Producer("video").accept({ timescale: Moq.Time.Timescale.MILLI }); const rendition = { config: new Signal(undefined), track: new Signal(track), close: () => track.close(), }; + const source = new Signal({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }); const capture = { - in: { - source: new Signal({ - getSettings: () => ({ frameRate: 30 }), - getConstraints: () => ({}), - } as never), - }, - out: { - display: new Signal({ width: 640, height: 480 }), - frames: new Signal(fanout), - }, + in: { source }, + out: { display: new Signal({ width: 640, height: 480 }), frames: new Signal(fanout) }, }; - const encoder = new Encoder("video/hd", { - enabled: true, + const enabled = new Signal(true); + const encoder = new Encoder("video", { + enabled, broadcast: { video: () => rendition, baseline: new Baseline() } as never, capture: capture as never, }); try { await settle(); - expect(encoder.out.catalog.peek()?.stalled).toBeUndefined(); + const before = encoder.out.catalog.peek(); + expect(before?.codedWidth).toBe(Catalog.u53(640)); + expect(before?.enabled).toBeUndefined(); - if (reason === "encoder lag") { - // Five frames at 30fps exceed three frame intervals of unaccepted capture. - for (let i = 0; i < 5; i++) { - controller.enqueue(new Frame(i * 33_333) as unknown as VideoFrame); - await new Promise((resolve) => setTimeout(resolve, 10)); - } - } else { - clock.mockReturnValue(200); - await new Promise((resolve) => setTimeout(resolve, 60)); + for (const millis of [0, 33, 66]) { + controller.enqueue(new Frame(millis * 1000) as unknown as VideoFrame); + await new Promise((resolve) => setTimeout(resolve, 5)); } + expect(encoded.filter((frame) => frame.black)).toEqual([]); + + enabled.set(false); await settle(); - expect(encoder.out.catalog.peek()?.stalled).toBe(true); + expect(encoded.filter((frame) => frame.black)).toEqual([ + { timestamp: 66_000 + 33_333, keyFrame: true, black: true }, + ]); + expect(encoder.out.active.peek()).toBe(false); + + // Muting releases the capture; the rendition keeps advertising what it last published. + source.set(undefined); + await settle(); + const disabled = encoder.out.catalog.peek(); + expect(disabled?.enabled).toBe(false); + expect(disabled?.codec).toBe(before?.codec); + expect(disabled?.codedWidth).toBe(before?.codedWidth); + + // Re-enabled before the capture reopens: still the same rendition, still disabled. + enabled.set(true); + await settle(); + expect(encoder.out.catalog.peek()).toEqual(disabled); + + source.set({ getSettings: () => ({ frameRate: 30 }), getConstraints: () => ({}) }); + await settle(); + expect(encoder.out.catalog.peek()?.enabled).toBeUndefined(); + expect(encoded.filter((frame) => frame.black)).toHaveLength(1); } finally { encoder.close(); fanout.close(); - clock.mockRestore(); + track.close(); + if (original) Object.defineProperty(globalThis, "VideoEncoder", original); + else Reflect.deleteProperty(globalThis, "VideoEncoder"); + if (videoFrame) Object.defineProperty(globalThis, "VideoFrame", videoFrame); + else Reflect.deleteProperty(globalThis, "VideoFrame"); } }); diff --git a/js/publish/src/video/encoder.ts b/js/publish/src/video/encoder.ts index b690588b58..b502c22926 100644 --- a/js/publish/src/video/encoder.ts +++ b/js/publish/src/video/encoder.ts @@ -63,8 +63,9 @@ export interface Config { // Signals the encoder reads. export type EncoderInput = { - // Whether to publish (and encode) this rendition. Defaults to true. When false the rendition drops out of the - // catalog and stops encoding, but stays registered so a subscriber still gets an idle track. + // Whether to encode this rendition. Defaults to true. When false it stops encoding but stays in the + // catalog with `enabled: false` and its last config, and stays registered so a subscriber still gets + // an idle track. enabled: Getter; // The broadcast to register the rendition on. Undefined resolves the config for a local preview @@ -86,7 +87,8 @@ export type EncoderProps = Inputs & { }; type EncoderOutput = { - // The catalog config published for this rendition, or undefined while disabled. + // The catalog config published for this rendition, `enabled: false` while disabled, or undefined + // until one resolves. catalog: Signal; // The resolved WebCodecs config (codec, bitrate, dimensions), available even with no subscriber. // Exposed so a local preview can re-encode with identical settings to mirror the wire output. @@ -166,12 +168,15 @@ export class Encoder { readonly settled: Computed; #signals = new Effect(); - #stalled = new Catalog.Stalled.Detector(); - #firstCaptured?: Time.Micro; - #lastCaptured?: Time.Micro; - #lastAccepted?: Time.Micro; - #lastCaptureWall?: number; #estimator = new Estimator(); + // The estimator's jitter and delay, republished whenever either rises. + #estimate = new Signal({}); + // The last config published while enabled, which a disabled rendition keeps advertising. + #last?: Catalog.VideoConfig; + // Whether the rendition was disabled since a config last resolved, so it keeps advertising `#last` + // as disabled until the re-enabled capture resolves a new one. A capture that never reopens leaves + // it disabled, which is accurate: no frames are coming. + #paused = false; constructor(name: string, props?: EncoderProps) { this.name = name; @@ -233,7 +238,7 @@ export class Encoder { const rendition = broadcast.video(this.name); effect.cleanup(() => rendition.close()); - // Publish the resolved catalog config; undefined (while disabled) drops it from the catalog. + // Publish the resolved catalog config; undefined (until one resolves) leaves it out of the catalog. effect.proxy(rendition.config, this.out.catalog); // Encode only while enabled and a subscriber is attached (the demand gate). @@ -241,10 +246,7 @@ export class Encoder { const enabled = effect.get(this.in.enabled); const track = effect.get(rendition.track); effect.set(this.#out.active, enabled && !!track, false); - if (!enabled || !track) { - this.#observe({ demand: false, idle: true }); - return; - } + if (!enabled || !track) return; this.#encode(track, broadcast.baseline, effect); }); @@ -279,25 +281,13 @@ export class Encoder { // Encode captured frames into the track producer, reconfiguring when the resolved config changes. #encode(track: Moq.Track.Producer, baseline: Baseline, effect: Effect): void { const capture = effect.get(this.in.capture); - if (!capture) { - this.#observe({ demand: true, idle: true }); - return; - } - - this.#observe({ demand: false, idle: true }); - this.#lastCaptureWall = performance.now(); + if (!capture) return; const producer = new Container.Legacy.Producer(track, new Container.Legacy.Format("video")); - // The broadcast owns this static track across demand gaps. When demand disappears, close the - // current group, marking the break so a later subscriber resumes on the same track without - // the pre-gap group reading as live. A fatal encoder error still aborts the track through - // producer.close(err) below. - effect.cleanup(() => { - if (track.closed.peek() === undefined) producer.discontinuity(); - }); let lastKeyframe: Time.Micro | undefined; let lastEncoded: Time.Micro | undefined; + let tearingDown = false; effect.spawn(async () => { const encoder = new VideoEncoder({ @@ -315,24 +305,25 @@ export class Encoder { producer.encode(frame, frame.timestamp as Time.Micro, key); if (this.#estimator.flush(frame.timestamp, baseline)) { - const catalog = this.#out.catalog.peek(); - if (catalog) this.#out.catalog.set({ ...catalog, ...this.#estimator.estimate }); + this.#estimate.set({ ...this.#estimator.estimate }); } - this.#lastAccepted = frame.timestamp as Time.Micro; - this.#observe({ demand: true, idle: false, frame: true }); }, error: (err: Error) => { - producer.close(err); + // The black keyframe is cosmetic; failing it must not end the track the broadcast owns. + if (tearingDown) console.warn("video encoder failed while stopping:", err); + else producer.close(err); }, }); - effect.cleanup(() => encoder.close()); - + // What the encoder was last configured with. The resolved config clears on disable, before + // the black keyframe below is encoded. + let configured: VideoEncoderConfig | undefined; effect.run((effect) => { const config = effect.get(this.out.resolved); if (!config) return; encoder.configure(config); + configured = config; }); effect.run((effect) => { @@ -368,11 +359,6 @@ export class Encoder { if (frame.timestamp - lastEncoded < minGap - minGap / 2) continue; } lastEncoded = frame.timestamp as Time.Micro; - const captured = frame.timestamp as Time.Micro; - this.#firstCaptured ??= captured; - this.#lastCaptured = captured; - this.#lastCaptureWall = performance.now(); - this.#observe({ demand: true, idle: false }); const interval = config?.keyframeInterval ?? Time.Milli.fromSecond(2 as Time.Second); @@ -396,52 +382,56 @@ export class Encoder { } }); }); - }); - - effect.interval(() => this.#observe({ demand: true, idle: false }), 50); - } - #observe(state: { demand: boolean; idle: boolean; frame?: boolean }): void { - if (state.idle) { - this.#firstCaptured = undefined; - this.#lastCaptured = undefined; - this.#lastAccepted = undefined; - this.#lastCaptureWall = undefined; - } - const catalog = this.#out.catalog.peek(); - const mediaLag = ((): Time.Micro => { - if (this.#lastCaptured === undefined || this.#firstCaptured === undefined) return 0 as Time.Micro; - const accepted = this.#lastAccepted ?? this.#firstCaptured; - return Math.max(0, this.#lastCaptured - accepted) as Time.Micro; - })(); - const quiet = - this.#lastCaptureWall === undefined - ? (0 as Time.Micro) - : Time.Micro.fromMilli((performance.now() - this.#lastCaptureWall) as Time.Milli); - if ( - !this.#stalled.observe({ - frame: state.frame ?? false, - mediaLag, - quiet, - interval: Catalog.Stalled.intervalFromFps(catalog?.framerate ?? this.out.resolved.peek()?.framerate), - demand: state.demand, - idle: state.idle, - }) - ) { - return; - } - if (!catalog) return; - this.#out.catalog.set({ ...catalog, stalled: this.#stalled.flag() }); + // Teardown is ours rather than a cleanup, since disabling first encodes one black keyframe and + // has to wait for it. A viewer that predates `enabled` keeps selecting a disabled rendition, + // and shows black instead of a frozen picture. + await new Promise((resolve) => effect.abort.addEventListener("abort", resolve, { once: true })); + tearingDown = true; + try { + const config = configured; + if ( + !this.in.enabled.peek() && + config && + lastEncoded !== undefined && + encoder.state === "configured" && + track.closed.peek() === undefined + ) { + // One frame interval after the last picture, in whole microseconds like any VideoFrame. + const interval = Time.Micro.fromSecond((1 / (config.framerate ?? 30)) as Time.Second); + const frame = black(config, Math.round(lastEncoded + interval)); + try { + encoder.encode(frame, { keyFrame: true }); + } finally { + frame.close(); + } + // A rejected flush already reported through the error callback. + await encoder.flush().catch(() => {}); + } + } finally { + if (encoder.state !== "closed") encoder.close(); + // The broadcast owns this static track across demand gaps. Close the current group, marking + // the break so a later subscriber resumes on the same track without the pre-gap group reading + // as live. A fatal encoder error still aborts the track through producer.close(err) above. + if (track.closed.peek() === undefined) producer.discontinuity(); + } + }); } - // Returns the catalog for the configured settings, or undefined while disabled / unresolved. + // Publishes the catalog for the configured settings. A disabled rendition keeps its last config, + // since muting also releases the capture it would resolve a new one from. It stays disabled after + // re-enabling until the reopened capture resolves, so an unmute is one field changing too. #runCatalog(effect: Effect): void { + const estimate = effect.get(this.#estimate); const enabled = effect.get(this.in.enabled); - const live = effect.get(this.#live); - if (!enabled || !live) { - effect.set(this.#out.catalog, undefined); + if (!enabled) this.#paused = true; + const live = enabled ? effect.get(this.#live) : undefined; + if (!live) { + const last = this.#paused ? this.#last : undefined; + effect.set(this.#out.catalog, last && { ...last, ...estimate, enabled: false }); return; } + this.#paused = false; // Advertise the codec string the probe's encoder reported rather than the one we configured, // so it names the profile and level the bitstream actually carries. @@ -454,10 +444,10 @@ export class Encoder { codedHeight: Catalog.u53(config.height), optimizeForLatency: true, container: { kind: "legacy" } as const, - ...this.#estimator.estimate, - stalled: this.#stalled.flag(), + ...estimate, }; + this.#last = catalog; effect.set(this.#out.catalog, catalog); } @@ -722,6 +712,16 @@ function ceiling(codec: string, dimensions: { width: number; height: number }, t return capped; } +// One black I420 picture the size of `config`, in limited range like most encoders expect. +function black(config: VideoEncoderConfig, timestamp: number): VideoFrame { + const { width, height } = config; + const luma = width * height; + const data = new Uint8Array((luma * 3) / 2); + data.fill(0x10, 0, luma); + data.fill(0x80, luma); + return new VideoFrame(data, { format: "I420", codedWidth: width, codedHeight: height, timestamp }); +} + // Encode one frame with a throwaway encoder and return the codec string it reports, like // moq-video's `Config::probe`. The encoder picks the profile and level it actually writes, so reading // them back lets the rendition be advertised before a subscriber starts the real encoder, without a diff --git a/js/watch/src/audio/decoder.test.ts b/js/watch/src/audio/decoder.test.ts index 4c0f2b5247..73276c30f9 100644 --- a/js/watch/src/audio/decoder.test.ts +++ b/js/watch/src/audio/decoder.test.ts @@ -223,6 +223,13 @@ async function play(initial: Delay) { play, // Remove the rendition from the catalog, then restore it. remove: () => catalog.set({}), + // Keep the rendition in the catalog with `enabled: false`, as a muted publisher does. + disable: () => + catalog.set({ + audio: { renditions: { audio: Catalog.AudioConfigSchema.parse({ ...audio, enabled: false }) } }, + }), + // The rendition the source selected. + track: () => source.out.track.peek(), restore: (patch: Record = {}) => catalog.set({ audio: { renditions: { audio: Catalog.AudioConfigSchema.parse({ ...audio, ...patch }) } } }), // Serve another broadcast from the next subscription, its timeline starting over. @@ -380,6 +387,30 @@ describe("Decoder across a rendition's absence", () => { } }); + it("deselects a disabled rendition and keeps one graph across disable and enable", async () => { + const playback = await play(Time.Milli(100)); + try { + await playback.play(12); + expect(playback.track()).toBe("audio"); + + playback.disable(); + await microtasks(); + expect(playback.track()).toBeUndefined(); + expect(contexts).toHaveLength(1); + + playback.restore(); + await microtasks(); + expect(playback.track()).toBe("audio"); + await playback.play(6); + + expect(contexts).toHaveLength(1); + expect(contexts[0].calls.close).toBe(0); + expect(playback.subscriptions()).toBe(2); + } finally { + playback.close(); + } + }); + it("plays a different broadcast whose timeline starts behind the playhead", async () => { const playback = await play(Time.Milli(100)); try { diff --git a/js/watch/src/audio/source.ts b/js/watch/src/audio/source.ts index 4791946583..9f83e0e008 100644 --- a/js/watch/src/audio/source.ts +++ b/js/watch/src/audio/source.ts @@ -104,7 +104,10 @@ export class Source { } #runSelected(effect: Effect): void { - const available = effect.get(this.#out.available); + // A disabled rendition has no frames coming, so it is never selected, even by name. + const available = Object.fromEntries( + Object.entries(effect.get(this.#out.available)).filter(([, config]) => config.enabled !== false), + ); if (Object.keys(available).length === 0) return; const target = effect.get(this.in.target); diff --git a/js/watch/src/msf.test.ts b/js/watch/src/msf.test.ts index 2a58e4d815..0b8ab50cda 100644 --- a/js/watch/src/msf.test.ts +++ b/js/watch/src/msf.test.ts @@ -62,22 +62,6 @@ test("rounds a fractional MSF delay up, and drops zero", () => { expect(hang.audio?.renditions.early?.delay).toBeUndefined(); }); -test("preserves stalled video renditions", () => { - const catalog: Msf.Catalog = { - tracks: [ - { - name: "video", - packaging: "loc", - role: "video", - codec: "vp09.00.10.08", - stalled: true, - }, - ], - }; - - expect(toHang(catalog).video?.renditions.video?.stalled).toBe(true); -}); - test("keeps loc packaging as the loc container", () => { const catalog: Msf.Catalog = { tracks: [ diff --git a/js/watch/src/msf.ts b/js/watch/src/msf.ts index a78a396ec5..d084293f3c 100644 --- a/js/watch/src/msf.ts +++ b/js/watch/src/msf.ts @@ -67,7 +67,6 @@ function toVideoConfig(track: Msf.Track): Catalog.VideoConfig | undefined { codedHeight: track.height != null ? u53(track.height) : undefined, framerate: track.framerate, bitrate: track.bitrate != null ? u53(track.bitrate) : undefined, - stalled: track.stalled, jitter: track.jitter != null ? u53(track.jitter) : undefined, delay: track.delay != null ? delayMillis(track.delay) : undefined, }; diff --git a/js/watch/src/video/config.test.ts b/js/watch/src/video/config.test.ts index feb8dcf0b7..38e0533fb7 100644 --- a/js/watch/src/video/config.test.ts +++ b/js/watch/src/video/config.test.ts @@ -31,7 +31,7 @@ test("metadata changes do not change the playback identity", async () => { rendition.set( config({ bitrate: 2_000_000, - stalled: true, + enabled: false, codedWidth: 1280, codedHeight: 720, framerate: 30, diff --git a/js/watch/src/video/decoder.test.ts b/js/watch/src/video/decoder.test.ts index b83e69143d..2df963a73e 100644 --- a/js/watch/src/video/decoder.test.ts +++ b/js/watch/src/video/decoder.test.ts @@ -260,6 +260,7 @@ async function guardedPlayback(kind: "legacy" | "cmaf", reads: Read[], hold: rea return { decoder, enabled, + catalog, submitted, outputs, get resets() { @@ -301,6 +302,24 @@ it("promoting from no active track holds its picture and timestamp until a frame } }); +it("clears the picture once no rendition is selectable", async () => { + const playback = await guardedPlayback("legacy", [sample(10, 1000)]); + try { + const held = playback.decoder.out.frame.peek(); + expect(held).toBeDefined(); + + const video = playback.catalog.peek().video?.renditions.video; + if (!video) throw new Error("missing rendition"); + playback.catalog.set({ video: { renditions: { video: { ...video, enabled: false } } } }); + await microtasks(); + expect(playback.decoder.out.frame.peek()).toBeUndefined(); + expect(playback.decoder.out.timestamp.peek()).toBeUndefined(); + expect((held as unknown as Picture).closed).toBe(true); + } finally { + playback.close(); + } +}); + // A late marker from older history raises a discontinuity without making older groups newer. function marker(group: number, end: number, discontinuity: number): Read { return { group, discontinuity, continuous: false, frame: undefined, end: Time.Micro(end) }; diff --git a/js/watch/src/video/decoder.ts b/js/watch/src/video/decoder.ts index 57a819ed15..fdccf2d5a8 100644 --- a/js/watch/src/video/decoder.ts +++ b/js/watch/src/video/decoder.ts @@ -160,19 +160,22 @@ export class Decoder { } #runPending(effect: Effect): void { - const values = effect.getAll([ - this.in.enabled, - this.source.in.broadcast, - this.source.out.track, - this.#identity, - ]); - if (!values) { + const enabled = effect.get(this.in.enabled); + const broadcast = effect.get(this.source.in.broadcast); + const track = effect.get(this.source.out.track); + const identity = effect.get(this.#identity); + if (!enabled || !broadcast || !track || !identity) { // Close the active track when disabled (e.g. paused or not visible). // The pending cleanup won't do this because it was already promoted to #active. this.#active.set(undefined); + // A paused or hidden player keeps its picture, but with nothing selectable (every rendition + // disabled or removed) it shows black rather than a frozen frame. + if (enabled && broadcast && !track) { + this.#clearCurrentFrame(); + this.#out.buffered.set([]); + } return; } - const [_, broadcast, track, identity] = values; // Honor a per-rendition `broadcast` override: subscribe on the resolved source // broadcast instead of the catalog's own broadcast. diff --git a/js/watch/src/video/source.test.ts b/js/watch/src/video/source.test.ts index 03a23602aa..5e41f86c55 100644 --- a/js/watch/src/video/source.test.ts +++ b/js/watch/src/video/source.test.ts @@ -173,8 +173,8 @@ describe("Source error signal", () => { }); }); -describe("Source stalled rendition selection", () => { - it("moves to an unstalled rendition when the selected one is throttled", async () => { +describe("Source disabled rendition selection", () => { + it("moves to an enabled rendition when the selected one is disabled, and back", async () => { const state = mutableBroadcast({ low: config("avc1.64001e", { bitrate: 1_000_000 }), high: config("avc1.640028", { bitrate: 2_000_000 }), @@ -191,56 +191,67 @@ describe("Source stalled rendition selection", () => { video: { renditions: { low: config("avc1.64001e", { bitrate: 1_000_000 }), - high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), + high: config("avc1.640028", { bitrate: 2_000_000, enabled: false }), }, }, }); await settle(); expect(source.out.track.peek()).toBe("low"); + + state.catalog.set({ + video: { + renditions: { + low: config("avc1.64001e", { bitrate: 1_000_000 }), + high: config("avc1.640028", { bitrate: 2_000_000 }), + }, + }, + }); + await settle(); + expect(source.out.track.peek()).toBe("high"); source.close(); }); - it("keeps a stalled manual target while an unstalled rendition exists", async () => { + it("never selects a disabled manual target", async () => { const source = new Source({ broadcast: mockBroadcast({ low: config("avc1.64001e", { bitrate: 1_000_000 }), - high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), + high: config("avc1.640028", { bitrate: 2_000_000, enabled: false }), }), target: { name: "high" }, supported: async () => true, }); await settle(); - expect(source.out.track.peek()).toBe("high"); - expect(Object.keys(source.out.available.peek())).toEqual(["low", "high"]); + expect(source.out.track.peek()).toBe("low"); source.close(); }); - it("selects the lowest rendition when every supported rendition is stalled", async () => { + it("selects nothing when every rendition is disabled", async () => { const source = new Source({ broadcast: mockBroadcast({ - high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), - low: config("avc1.64001e", { bitrate: 1_000_000, stalled: true }), + high: config("avc1.640028", { bitrate: 2_000_000, enabled: false }), + low: config("avc1.64001e", { bitrate: 1_000_000, enabled: false }), }), supported: async () => true, }); await settle(); - expect(source.out.track.peek()).toBe("low"); + expect(source.out.track.peek()).toBeUndefined(); + expect(source.out.config.peek()).toBeUndefined(); source.close(); }); - it("uses coded dimensions when stalled renditions omit bitrate", async () => { + it("ignores a legacy stalled flag", async () => { const source = new Source({ broadcast: mockBroadcast({ - high: config("avc1.640028", { codedWidth: 1920, codedHeight: 1080, stalled: true }), - low: config("avc1.64001e", { codedWidth: 854, codedHeight: 480, stalled: true }), + low: config("avc1.64001e", { bitrate: 1_000_000 }), + high: config("avc1.640028", { bitrate: 2_000_000, stalled: true }), }), supported: async () => true, }); await settle(); - expect(source.out.track.peek()).toBe("low"); + expect(source.out.track.peek()).toBe("high"); source.close(); }); @@ -265,7 +276,7 @@ describe("Source stalled rendition selection", () => { renditions: { high: config("avc1.640028", { bitrate: 1_500_000, - stalled: true, + label: "Main camera", codedWidth: 1280, codedHeight: 720, }), @@ -276,7 +287,7 @@ describe("Source stalled rendition selection", () => { expect(probes).toBe(1); expect(Number(source.out.config.peek()?.bitrate)).toBe(1_500_000); - expect(source.out.config.peek()?.stalled).toBe(true); + expect(source.out.config.peek()?.label).toBe("Main camera"); source.close(); }); diff --git a/js/watch/src/video/source.ts b/js/watch/src/video/source.ts index 97aa2cd011..2a01e582f4 100644 --- a/js/watch/src/video/source.ts +++ b/js/watch/src/video/source.ts @@ -185,16 +185,9 @@ function byBitrate(target: number): RenditionFilter { }; } -/** Return unstalled renditions, or the lowest bitrate or resolution when every option is stalled. */ -function selectableRenditions(renditions: Record): Record { - const active = Object.entries(renditions).filter(([, config]) => !config.stalled); - if (active.length > 0) return Object.fromEntries(active); - - const entries = Object.entries(renditions); - if (entries.length === 0) return {}; - const byRate = byBitrate(0)(entries); - const lowest = byRate.length === 1 ? byRate[0] : byDimensions(0, 0)(entries)[0]; - return { [lowest]: renditions[lowest] }; +/** Return the renditions that may be selected: a disabled one has no frames coming. */ +function enabledRenditions(renditions: Record): Record { + return Object.fromEntries(Object.entries(renditions).filter(([, config]) => config.enabled !== false)); } /** @@ -307,19 +300,16 @@ export class Source { } #runSelected(effect: Effect): void { - const supported = effect.get(this.#out.available); + const available = enabledRenditions(effect.get(this.#out.available)); const target = effect.get(this.in.target); - // A manual choice stays selected while stalled. `stalled` steers automatic adaptation; it - // must not silently override an explicit user selection. - if (target?.name && target.name in supported) { - const config = supported[target.name]; + if (target?.name && target.name in available) { + const config = available[target.name]; effect.set(this.#out.track, target.name); effect.set(this.#out.config, config); return; } - const available = selectableRenditions(supported); if (Object.keys(available).length === 0) return; // Auto-select: use recv bandwidth if no explicit bitrate target. diff --git a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt index 349e8ec8c4..bda99fe0f6 100644 --- a/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt +++ b/kt/moq/src/jvmAndAndroidMain/kotlin/dev/moq/Aliases.kt @@ -115,7 +115,7 @@ typealias Datagram = uniffi.moq.MoqDatagram typealias Frame = uniffi.moq.MoqFrame /** A media [Frame] whose keyframe flag marks group starts or video keyframes; audio flags only group starts. */ typealias MediaFrame = uniffi.moq.MoqMediaFrame -/** The catalog description of a video track, including whether the publisher recommends temporarily avoiding it. */ +/** The catalog description of a video track, including whether it is enabled (a disabled one has no frames coming). */ typealias Video = uniffi.moq.MoqVideo /** Caller-provided catalog fields for a video track. */ typealias VideoHint = uniffi.moq.MoqVideoHint @@ -133,7 +133,7 @@ typealias AudioInit = uniffi.moq.MoqAudioInit typealias VideoInit = uniffi.moq.MoqVideoInit /** A container format and its leading bytes. */ typealias ContainerInit = uniffi.moq.MoqContainerInit -/** The catalog description of an audio track: codec, sample rate, channels, and container. */ +/** The catalog description of an audio track: codec, sample rate, channels, whether it is enabled, and container. */ typealias Audio = uniffi.moq.MoqAudio /** A width and height pair, in pixels. */ typealias Dimensions = uniffi.moq.MoqDimensions diff --git a/py/moq-rs/README.md b/py/moq-rs/README.md index 9ae5871ee4..9514890c3a 100644 --- a/py/moq-rs/README.md +++ b/py/moq-rs/README.md @@ -215,8 +215,8 @@ Every handle whose cleanup is `cancel()` is an async context manager, so exiting - **`Frame`**. `.payload: bytes`, `.timestamp_us: int`. The unit of every write and every raw read. - **`MediaFrame`**. `.payload: bytes`, `.timestamp_us: int`, `.keyframe: bool`. Returned by media subscriptions. `keyframe` marks a group start or video keyframe; for audio it is true only at a group start. - **`Datagram`**. `.sequence: int`, `.timestamp_us: int`, `.payload: bytes`. Delivered only on datagram-capable transports with lite-05 or newer moq-lite, or moq-transport. -- **`Audio`**. `.codec`, `.sample_rate`, `.channel_count`, `.bitrate`, `.description`. -- **`Video`**. `.codec`, `.coded: Dimensions`, `.display_aspect`, `.bitrate`, `.stalled`, `.framerate`, `.description`. A true `.stalled` recommends temporarily avoiding the rendition without making it unavailable. +- **`Audio`**. `.codec`, `.sample_rate`, `.channel_count`, `.bitrate`, `.enabled`, `.description`. +- **`Video`**. `.codec`, `.coded: Dimensions`, `.display_aspect`, `.bitrate`, `.enabled`, `.framerate`, `.description`. A false `.enabled` means no frames are coming, so don't select the rendition. - **`Subscription`**. Subscriber delivery preferences: priority, staleness, and optional group range. - **`TrackInfo`**. Publisher track properties: priority, cache window, and timescale. - **`Dimensions`**. `.width: int`, `.height: int`. diff --git a/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md b/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md index 1bc8f4232a..d49a72fb2c 100644 --- a/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md +++ b/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md @@ -7,8 +7,7 @@ share of the connection's estimate: a grant below the configured bitrate lowers the encoder at once, room coming back raises it gradually, and a link too small for the configured audio rate sheds audio bits instead of stalling. The reservation's ceiling stays the configured bitrate. A grant below the -codec floor disables the rendition (`enabled: false`, -[enabled flag](/quest/m1/catalog-enabled.md)) until it recovers. PCM keeps reserve-only +codec floor disables the rendition (catalog `enabled: false`) until it recovers. PCM keeps reserve-only usage: it claims its fixed rate and never reads the grant. The rate policy has one home shared by every sender, so audio and video back @@ -65,10 +64,6 @@ returns; a grant below the Opus floor disables the rendition, and recovered bandwidth enables it again even after the last viewer unsubscribed; a PCM Producer ignores every grant. -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - defines the `enabled` field a disabled rendition sets - ## Closes - [#2848](https://github.com/moq-dev/moq/issues/2848) - close this issue when the quest finishes diff --git a/quest/m1/README.md b/quest/m1/README.md index 3c42d30a98..dd1d645ee9 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -45,7 +45,7 @@ blocks. The quests that gated m0 lines moved under them. - [Binding audio delay](/quest/m1/binding-surface.md) - moq-ffi and every wrapper configure and observe audio playout delay - [fMP4 init from the catalog](/quest/m1/fmp4-catalog-init.md) - fMP4 export writes avc3/hev1 entries from the catalog, so an Annex-B H.264 or H.265 init no longer waits for the first keyframe - [Bump web-transport-iroh for the capsule close](/quest/m1/iroh-capsule-bump.md) - condition: moq-dev/web-transport#419 ships in a release, then moq's iroh HTTP/3 client reports a peer's close capsule -- [Enabled flag](/quest/m1/catalog-enabled.md) - one `enabled` rendition flag replaces `stalled` and pausing, and the flapping encoder-lag detector is deleted +- [Early video demand](/quest/m1/catalog-early-demand.md) - a regression guards demand before the first keyframe after the detector removal - [Hang changelog](/quest/m1/hang-changelog-04.md) - the hang draft lists under -03 only what -03 published - [SUBSCRIBE_DROP](/quest/m1/subscribe-drop.md) - every stream group in a lite subscription arrives or is dropped by name, and lite-07 drops its stream count for it - [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - one dynamic producer per track name in both languages, with the sequence namespace surviving a replacement diff --git a/quest/m1/catalog-early-demand.md b/quest/m1/catalog-early-demand.md new file mode 100644 index 0000000000..5e724c5d2a --- /dev/null +++ b/quest/m1/catalog-early-demand.md @@ -0,0 +1,29 @@ +# [XS] Guard video demand before the first keyframe + +## Goal + +A regression test proves that demand on a video track before its first +keyframe does not end the import or modify an unpublished rendition. +It guards the importer failure reported in #4945. + +## Plan + +The enabled-flag change in [#4915](https://github.com/moq-dev/moq/pull/4915) +removes `publish_stalled` and the importer detector hooks that called +`track.modify()` before a configuration was published. The catalog now +publishes a resolved configuration through `track.set()`; it has no +mutation driven by early demand. The relay half of the report is fixed by +[#4942](https://github.com/moq-dev/moq/pull/4942)'s publisher epochs. + +The source fixes are present; the before-first-keyframe regression from +the imported issue remains to be added. Exercise a video importer with +track demand before it resolves its configuration, then deliver its first +keyframe and require the import to continue. Use mocked time if needed, +and keep the test in the existing moq-mux Check/Test suite. Do not restore +the removed detector or add an availability mechanism. + +Public API: none. Wire: none. + +## Closes + +- [#4945](https://github.com/moq-dev/moq/issues/4945) - early video demand does not end a replacement import with "rendition is not published" diff --git a/quest/m1/catalog-enabled.md b/quest/m1/catalog-enabled.md deleted file mode 100644 index 1847fd27c6..0000000000 --- a/quest/m1/catalog-enabled.md +++ /dev/null @@ -1,75 +0,0 @@ -# [L] One enabled flag says whether to use a rendition - -## Goal - -A hang rendition carries `enabled`, default true. `enabled: false` means no -frames are coming, and a consumer MUST NOT select it. It replaces both the -encoder-lag `stalled` flag and a separate pause flag: whoever decides a -rendition should not be used (an app pausing it, or the bandwidth allocator) -writes the same field, and a publisher mute is a one-field catalog delta -instead of a removal. - -## Plan - -Decided (2026-10-04): - -- Wire: an optional `enabled` boolean on every rendition, audio and video, - written only when false. Writers stop writing `stalled`, and readers ignore - it: released publishers still flap it, and following it would keep #4772 - alive for them. Update - `drafts/draft-lcurley-moq-hang.md` (replacing the `stalled` section, with a - changelog entry, since `stalled` shipped in -03), `doc/concept/hang.md`, and - run `just drafts check`. MSF stops mirroring `stalled` and does not mirror - `enabled`. -- Delete the encoder-lag detector: `Stalled.Detector` in - `js/hang/src/catalog/stalled.ts` and `rs/hang/src/catalog/stalled.rs`, its - producers in `js/publish/src/video/encoder.ts` and - `rs/moq-mux/src/codec/video.rs`, and its importer hooks. It is what flaps - (#4772, #4776): frame-count hysteresis, plus clearing on demand loss. No - replacement adapts an overloaded encoder; that waits for a consumer. -- Publisher pause: `enabled` false on `@moq/publish` keeps the rendition in - the catalog with `enabled: false` instead of removing it, and the encoder - stops. The encoder keeps the last config it published, since muting also - releases the capture. Before disabling a video rendition it encodes one - black keyframe, so a viewer released before this field shows black rather - than a frozen picture. Older viewers otherwise keep selecting a disabled - rendition; that degradation is accepted and noted in the changelog. -- Viewer: `@moq/watch` deselects a disabled rendition, and its audio graph - outlives the absence as it does for a removed one. -- Bandwidth: a rendition is enabled only once its reservation is granted, and - disabled, with encoding stopped, when the grant falls below its floor, - until the grant recovers. A disabled rendition loses its subscribers, and - the allocator grants nothing to an undemanded track, so recovery evaluates - a hypothetical share against the current estimate instead of waiting for a - grant. The [ladder](/quest/m3/ladder/README.md) and - [audio grant following](/quest/m1/2848-follow-the-bandwidth-grant-in-moq-audio-instead-of.md) - adopt that rule; this quest only defines the field. -- Everything else that names `stalled` moves to `enabled`: moq-transcode's - `inherit_stalled`, the moq-ffi field and its wrappers, and the C - `moq_consume_video_stalled`, which becomes `moq_consume_video_enabled`. - Selection in `js/watch/src/video/source.ts` filters on it, as - [rendition preference](/quest/m1/rendition-preference.md) expects. - -- Nothing edits a rendition before its config is published (decided - 2026-10-06, #4945). Today demand on a video track before its first - keyframe lets `publish_stalled` (`rs/moq-mux/src/codec/video.rs`) call - `track.modify()` on an unpublished rendition, and `moq import ts` exits - with `NotPublished`. Deleting the detector removes that path; whatever - writes `enabled` must not bring it back. - -Tests: demand on a video track before its first keyframe does not end -the import. Disabling an audio rendition publishes a catalog with the same -rendition and `enabled: false`, and a viewer deselects it and keeps one -AudioContext across disable and enable; a legacy `stalled: true` changes -nothing; nothing in the tree writes `stalled`. - -## Closes - -- [#4772](https://github.com/moq-dev/moq/issues/4772) - close this issue when the quest finishes -- [#4776](https://github.com/moq-dev/moq/issues/4776) - close this issue when the quest finishes -- [#4945](https://github.com/moq-dev/moq/issues/4945) - a replacement `moq import ts` exits with "rendition is not published"; the relay half is fixed by #4942's epochs - -## Related - -- [Ladder](/quest/m3/ladder/README.md) - disables a rung its grant cannot sustain -- [Rendition preference](/quest/m1/rendition-preference.md) - the other per-rendition selection field diff --git a/quest/m1/hang-changelog-04.md b/quest/m1/hang-changelog-04.md index 476085ce53..af0905425c 100644 --- a/quest/m1/hang-changelog-04.md +++ b/quest/m1/hang-changelog-04.md @@ -13,7 +13,3 @@ Compare every -03 entry against the published including its body and changelog; do not rely on an illustrative list of late entries. The root `clock` section already shipped in -03 and stays there. Run `just drafts check`. This corrects attribution, not the format. - -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - land after #4915's draft changelog edit to avoid conflicting changes diff --git a/quest/m1/rendition-preference.md b/quest/m1/rendition-preference.md index 3198cca0dc..08ae538789 100644 --- a/quest/m1/rendition-preference.md +++ b/quest/m1/rendition-preference.md @@ -41,7 +41,7 @@ Decided in planning interviews on 2026-10-01: - Video only. Optional on the wire, omitted when 0 (as `enabled` is omitted at its default). Additive, so older players ignore it. - Selection order in `js/watch/src/video/source.ts`: drop disabled - renditions ([enabled flag](/quest/m1/catalog-enabled.md)), then decode + renditions (catalog `enabled: false`), then decode support, then keep the highest preference among supported renditions, then the existing target and bitrate pick within what is left. Preference is about decodability only. @@ -71,10 +71,6 @@ Decided in planning interviews on 2026-10-01: the field. moq-transcode producing same-size codec fallbacks; the consumer publishes its own. -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - the `enabled` field selection filters on first - ## Related - [Audio rendition pick](/quest/m1/audio-ranked.md) - audio ranking, where `preference` could join later diff --git a/quest/m2/native-enabled.md b/quest/m2/native-enabled.md index 2741fc1eb3..b5a3fb4469 100644 --- a/quest/m2/native-enabled.md +++ b/quest/m2/native-enabled.md @@ -19,7 +19,3 @@ Test mixed and all-disabled catalogs, plus disable/re-enable transitions. A C `moq_consume_audio_enabled` getter is added only if a C consumer needs it; any FFI/C API change updates all wrappers and their docs in the same PR. Update native consumer docs for the selection behavior. - -## Required - -- [Enabled flag](/quest/m1/catalog-enabled.md) - #4915 provides enabled fields and enabled-first video ranking diff --git a/quest/m3/ladder/README.md b/quest/m3/ladder/README.md index 47694c066c..98e0b147fc 100644 --- a/quest/m3/ladder/README.md +++ b/quest/m3/ladder/README.md @@ -18,8 +18,7 @@ state. Deferred in the 2026-09-30 audit and moved to m3 in the 2026-10-05 audit: no named consumer for the publisher-side ladder. -The catalog and player half is the rendition `enabled` flag -([enabled flag](/quest/m1/catalog-enabled.md)), which replaces the `stalled` +The catalog and player half is the rendition `enabled` flag, which replaced the `stalled` state shipped in [moq#2865](https://github.com/moq-dev/moq/pull/2865): `enabled: false` means no frames are coming and a viewer must not select the rendition. Routing, decoder, and presentation identities are split so a diff --git a/quest/m3/ladder/controller.md b/quest/m3/ladder/controller.md index 3252e41ada..28a92a9bd3 100644 --- a/quest/m3/ladder/controller.md +++ b/quest/m3/ladder/controller.md @@ -70,7 +70,6 @@ priority where the subscriber's order wins, and a custom ladder order. ## Required - [Scope track priority](/quest/m1/track-priority-scope.md) - settles the priority ranking the controller assigns before the encoder side reads it -- [Enabled flag](/quest/m1/catalog-enabled.md) - the `enabled` field the controller publishes for rungs ## Related diff --git a/rs/hang/src/catalog/audio/mod.rs b/rs/hang/src/catalog/audio/mod.rs index 6c32bdf0c3..bd6fd6064b 100644 --- a/rs/hang/src/catalog/audio/mod.rs +++ b/rs/hang/src/catalog/audio/mod.rs @@ -103,6 +103,14 @@ pub struct AudioConfig { #[serde(default)] pub bitrate: Option, + /// Whether this rendition may be selected. When false, no frames are coming and a consumer + /// must not select it. Only written when false. + #[serde( + default = "crate::catalog::enabled_default", + skip_serializing_if = "crate::catalog::enabled_skip" + )] + pub enabled: bool, + // Some codecs include a description so the decoder can be initialized without extra data. // If not provided, there may be in-band metadata (marginally higher overhead). #[serde(default)] @@ -157,6 +165,7 @@ impl AudioConfig { sample_rate, channel_count, bitrate: None, + enabled: true, description: None, container: Container::default(), jitter: None, @@ -179,4 +188,16 @@ mod test { let decoded: AudioConfig = serde_json::from_value(encoded).expect("failed to decode"); assert_eq!(decoded.label.as_deref(), Some("English")); } + + #[test] + fn enabled_is_written_only_when_false() { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + assert!(serde_json::to_value(&config).unwrap().get("enabled").is_none()); + + config.enabled = false; + let encoded = serde_json::to_value(&config).expect("failed to encode"); + assert_eq!(encoded["enabled"], false); + let decoded: AudioConfig = serde_json::from_value(encoded).expect("failed to decode"); + assert!(!decoded.enabled); + } } diff --git a/rs/hang/src/catalog/mod.rs b/rs/hang/src/catalog/mod.rs index 109a973603..74a7a0f575 100644 --- a/rs/hang/src/catalog/mod.rs +++ b/rs/hang/src/catalog/mod.rs @@ -18,7 +18,6 @@ mod millis; mod mode; mod priority; mod root; -pub mod stalled; mod text; mod video; @@ -36,6 +35,16 @@ pub use root::*; pub use text::*; pub use video::*; +/// Serde default for a rendition's `enabled` field. +pub(crate) fn enabled_default() -> bool { + true +} + +/// Skip serializing `enabled` unless it is false, so only a disabled rendition writes it. +pub(crate) fn enabled_skip(enabled: &bool) -> bool { + *enabled +} + pub(crate) fn deserialize_timescale_or_default<'de, D>(deserializer: D) -> Result where D: serde::Deserializer<'de>, diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index 0c68fa32c3..b54196d93c 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -353,7 +353,7 @@ mod test { display_aspect_width: None, display_aspect_height: None, bitrate: None, - stalled: None, + enabled: true, framerate: None, optimize_for_latency: None, container: Container::Legacy, @@ -372,6 +372,7 @@ mod test { sample_rate: 48_000, channel_count: 2, bitrate: None, + enabled: true, description: None, container: Container::Legacy, jitter: Some(std::time::Duration::from_millis(40)), diff --git a/rs/hang/src/catalog/stalled.rs b/rs/hang/src/catalog/stalled.rs deleted file mode 100644 index ed2ed927cf..0000000000 --- a/rs/hang/src/catalog/stalled.rs +++ /dev/null @@ -1,297 +0,0 @@ -//! Publisher-side hysteresis for the hang catalog `stalled` flag. -//! -//! The wire already carries [`super::VideoConfig::stalled`]; this is the one -//! detector every first-party catalog producer uses to decide when to set it. -//! A rendition is stalled while demand is active and it is not keeping up: the -//! newest source frame has pulled more than a few frame intervals ahead of the -//! newest frame handed to the transport, or the source has gone quiet for that -//! long. An idle broadcast (camera released, no source) is never stalled. -//! -//! The flag flips through an ordinary catalog update, not per frame: it is set -//! once lag exceeds the threshold and cleared only after a run of on-time -//! frames. - -use std::time::Duration; - -/// How many frame intervals of lag mark a rendition stalled. -pub const SET_INTERVALS: u32 = 3; - -/// Consecutive on-time frames required to clear a stall. -pub const CLEAR_FRAMES: u32 = 3; - -/// Frame interval used when the catalog does not advertise a framerate. -pub const DEFAULT_INTERVAL: Duration = Duration::from_millis(33); - -/// One observation of a rendition's source versus what the transport has accepted. -#[derive(Clone, Copy, Debug)] -pub struct Sample { - /// This observation completes a newly accepted frame. - pub frame: bool, - /// Newest captured/source timestamp minus newest timestamp handed to the transport. - pub media_lag: Duration, - /// Wall time since the source last delivered a frame for this rendition. - pub quiet: Duration, - /// One frame interval. The set threshold is [`SET_INTERVALS`] of these. - pub interval: Duration, - /// Someone is subscribed to this rendition. - pub demand: bool, - /// Camera released / no live source. Never stalled. - pub idle: bool, -} - -/// Hysteresis around the catalog `stalled` bit. -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct Detector { - on: bool, - recover: u32, -} - -impl Detector { - /// A detector that starts unstalled. - pub fn new() -> Self { - Self::default() - } - - /// Whether the rendition is currently stalled. - pub fn get(&self) -> bool { - self.on - } - - /// The catalog value: `Some(true)` while stalled, omitted otherwise. - pub fn flag(&self) -> Option { - self.on.then_some(true) - } - - /// Feed one sample. Returns whether the catalog flag changed. - pub fn observe(&mut self, sample: Sample) -> bool { - if sample.idle || !sample.demand { - return self.clear(); - } - - let interval = interval(sample.interval); - let lag = sample.media_lag.max(sample.quiet); - let over = lag > interval.saturating_mul(SET_INTERVALS); - let on_time = lag <= interval; - - if !self.on { - if !over { - return false; - } - self.on = true; - self.recover = 0; - return true; - } - - if !on_time { - self.recover = 0; - return false; - } - - if !sample.frame { - return false; - } - self.recover = self.recover.saturating_add(1); - if self.recover < CLEAR_FRAMES { - return false; - } - self.clear() - } - - fn clear(&mut self) -> bool { - self.recover = 0; - if !self.on { - return false; - } - self.on = false; - true - } -} - -/// A positive frame interval, falling back to [`DEFAULT_INTERVAL`]. -pub fn interval(value: Duration) -> Duration { - if value.is_zero() { DEFAULT_INTERVAL } else { value } -} - -/// Frame interval implied by a catalog framerate, or [`DEFAULT_INTERVAL`]. -pub fn interval_from_fps(fps: Option) -> Duration { - let Some(fps) = fps.filter(|fps| fps.is_finite() && *fps > 0.0) else { - return DEFAULT_INTERVAL; - }; - Duration::try_from_secs_f64(1.0 / fps) - .ok() - .filter(|value| !value.is_zero()) - .unwrap_or(DEFAULT_INTERVAL) -} - -#[cfg(test)] -mod tests { - use super::*; - - fn sample(lag: Duration) -> Sample { - Sample { - frame: true, - media_lag: lag, - quiet: Duration::ZERO, - interval: Duration::from_millis(33), - demand: true, - idle: false, - } - } - - fn quiet(gap: Duration) -> Sample { - Sample { - frame: true, - media_lag: Duration::ZERO, - quiet: gap, - interval: Duration::from_millis(33), - demand: true, - idle: false, - } - } - - #[test] - fn idle_is_never_stalled() { - let mut stalled = Detector::new(); - assert!(!stalled.observe(Sample { - frame: true, - idle: true, - demand: true, - media_lag: Duration::from_secs(10), - quiet: Duration::from_secs(10), - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn no_demand_is_never_stalled() { - let mut stalled = Detector::new(); - assert!(!stalled.observe(Sample { - frame: true, - demand: false, - idle: false, - media_lag: Duration::from_secs(10), - quiet: Duration::from_secs(10), - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn lag_past_the_threshold_sets_the_flag() { - let mut stalled = Detector::new(); - assert!(!stalled.observe(sample(Duration::from_millis(99)))); - assert!(stalled.observe(sample(Duration::from_millis(100)))); - assert_eq!(stalled.flag(), Some(true)); - } - - #[test] - fn a_quiet_source_sets_the_flag() { - let mut stalled = Detector::new(); - assert!(stalled.observe(quiet(Duration::from_millis(100)))); - assert!(stalled.get()); - } - - #[test] - fn clearing_needs_a_run_of_on_time_frames() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(stalled.get()); - - assert!(stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.get()); - assert_eq!(stalled.flag(), None); - } - - #[test] - fn a_late_frame_resets_recovery() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(200)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - assert!(stalled.get()); - assert!(stalled.observe(sample(Duration::from_millis(10)))); - assert!(!stalled.get()); - } - - #[test] - fn dropping_demand_clears_immediately() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - assert!(stalled.observe(Sample { - frame: true, - demand: false, - idle: false, - media_lag: Duration::from_millis(200), - quiet: Duration::ZERO, - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn releasing_the_camera_clears_immediately() { - let mut stalled = Detector::new(); - assert!(stalled.observe(sample(Duration::from_millis(200)))); - assert!(stalled.observe(Sample { - frame: true, - idle: true, - demand: true, - media_lag: Duration::from_millis(200), - quiet: Duration::ZERO, - interval: Duration::from_millis(33), - })); - assert!(!stalled.get()); - } - - #[test] - fn on_time_frames_do_not_flip_a_healthy_rendition() { - let mut stalled = Detector::new(); - for _ in 0..10 { - assert!(!stalled.observe(sample(Duration::from_millis(10)))); - } - assert!(!stalled.get()); - } - - #[test] - fn a_zero_interval_uses_the_default() { - let mut stalled = Detector::new(); - assert!(stalled.observe(Sample { - frame: true, - media_lag: DEFAULT_INTERVAL.saturating_mul(SET_INTERVALS) + Duration::from_millis(1), - quiet: Duration::ZERO, - interval: Duration::ZERO, - demand: true, - idle: false, - })); - assert!(stalled.get()); - } - - #[test] - fn polls_do_not_count_as_recovery_frames() { - let mut stalled = Detector::new(); - stalled.observe(sample(Duration::from_millis(200))); - let mut poll = sample(Duration::ZERO); - poll.frame = false; - for _ in 0..10 { - assert!(!stalled.observe(poll)); - } - assert!(stalled.get()); - } - - #[test] - fn interval_from_fps_falls_back() { - assert_eq!(interval_from_fps(Some(f64::MAX)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(f64::MIN_POSITIVE)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(None), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(0.0)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(f64::NAN)), DEFAULT_INTERVAL); - assert_eq!(interval_from_fps(Some(50.0)), Duration::from_millis(20)); - } -} diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 5d8b796464..7812909760 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -113,16 +113,17 @@ impl Video { self.renditions.is_empty() } - /// Iterate the renditions best first: largest picture, then highest bitrate. + /// Iterate the renditions best first: enabled, then largest picture, then highest bitrate. /// /// A consumer that carries one rendition takes the first it supports, so the - /// picture doesn't depend on how the tracks are named. Unknown dimensions or - /// bitrate rank below known ones, and exact ties keep name order. + /// picture doesn't depend on how the tracks are named. A disabled rendition sends + /// no frames, so it ranks below every enabled one. Unknown dimensions or bitrate + /// rank below known ones, and exact ties keep name order. pub fn ranked(&self) -> impl Iterator { let mut ranked: Vec<_> = self.renditions.iter().collect(); ranked.sort_by_key(|(_, config)| { let area = u64::from(config.coded_width.unwrap_or(0)) * u64::from(config.coded_height.unwrap_or(0)); - std::cmp::Reverse((area, config.bitrate)) + std::cmp::Reverse((config.enabled, area, config.bitrate)) }); ranked.into_iter() } @@ -216,12 +217,13 @@ pub struct VideoConfig { #[serde(default)] pub bitrate: Option, - /// Whether the publisher recommends temporarily avoiding this rendition. - /// - /// The track remains available. Consumers may still select it when no - /// unstalled rendition is suitable. - #[serde(default)] - pub stalled: Option, + /// Whether this rendition may be selected. When false, no frames are coming and a consumer + /// must not select it. Only written when false. + #[serde( + default = "crate::catalog::enabled_default", + skip_serializing_if = "crate::catalog::enabled_skip" + )] + pub enabled: bool, /// The frame rate of the video track, if known. #[serde(default)] @@ -287,7 +289,7 @@ impl VideoConfig { display_aspect_width: None, display_aspect_height: None, bitrate: None, - stalled: None, + enabled: true, framerate: None, optimize_for_latency: None, container: Container::default(), @@ -330,6 +332,11 @@ mod test { let names: Vec<_> = video.ranked().map(|(name, _)| name.as_str()).collect(); assert_eq!(names, ["f", "d", "e", "c", "b", "a"]); + + // A disabled rendition sends no frames, so it ranks below every enabled one. + video.renditions.get_mut("f").unwrap().enabled = false; + let names: Vec<_> = video.ranked().map(|(name, _)| name.as_str()).collect(); + assert_eq!(names, ["d", "e", "c", "b", "a", "f"]); } #[test] @@ -377,17 +384,25 @@ mod test { } #[test] - fn stalled_is_optional_and_round_trips() { + fn enabled_is_written_only_when_false() { let mut config = VideoConfig::new(VideoCodec::VP8); let encoded = serde_json::to_value(&config).expect("failed to encode"); - assert!(encoded.get("stalled").is_none()); + assert!(encoded.get("enabled").is_none()); - config.stalled = Some(true); + config.enabled = false; let encoded = serde_json::to_value(&config).expect("failed to encode"); - assert_eq!(encoded["stalled"], true); + assert_eq!(encoded["enabled"], false); let decoded: VideoConfig = serde_json::from_value(encoded).expect("failed to decode"); - assert_eq!(decoded.stalled, Some(true)); + assert!(!decoded.enabled); + } + + #[test] + fn legacy_stalled_is_ignored() { + let json = serde_json::json!({ "codec": "vp8", "stalled": true }); + let config: VideoConfig = serde_json::from_value(json).expect("failed to decode"); + assert!(config.enabled); + assert!(serde_json::to_value(&config).unwrap().get("stalled").is_none()); } #[test] diff --git a/rs/moq-c/README.md b/rs/moq-c/README.md index efc5fd9ebf..b4eabd8fff 100644 --- a/rs/moq-c/README.md +++ b/rs/moq-c/README.md @@ -117,7 +117,7 @@ int32_t moq_consume_catalog(uint32_t broadcast, moq_status_callback on_catalog, int32_t moq_consume_catalog_cancel(uint32_t catalog); int32_t moq_consume_catalog_free(uint32_t catalog); int32_t moq_consume_video_config(uint32_t catalog, uint32_t index, moq_video_config *dst); -int32_t moq_consume_video_stalled(uint32_t catalog, uint32_t index, bool *dst); +int32_t moq_consume_video_enabled(uint32_t catalog, uint32_t index, bool *dst); int32_t moq_consume_audio_config(uint32_t catalog, uint32_t index, moq_audio_config *dst); // Consuming: Video diff --git a/rs/moq-c/src/api.rs b/rs/moq-c/src/api.rs index f49bbe38b4..fac8db501f 100644 --- a/rs/moq-c/src/api.rs +++ b/rs/moq-c/src/api.rs @@ -3291,25 +3291,25 @@ pub unsafe extern "C" fn moq_consume_video_config(catalog: u32, index: u32, dst: }) } -/// Query whether the publisher recommends temporarily avoiding a video rendition. +/// Query whether a video rendition may be selected. /// -/// The track remains available. A false value also covers catalogs that omit the -/// optional field. +/// False means no frames are coming and the rendition must not be selected. A catalog that +/// omits the optional field reads as true. /// /// Returns zero on success, or a negative code on failure. /// /// # Safety /// - The caller must ensure that `dst` points to properly aligned, writable storage for a `bool`. #[unsafe(no_mangle)] -pub unsafe extern "C" fn moq_consume_video_stalled(catalog: u32, index: u32, dst: *mut bool) -> i32 { +pub unsafe extern "C" fn moq_consume_video_enabled(catalog: u32, index: u32, dst: *mut bool) -> i32 { ffi::enter(move || { let catalog = ffi::parse_id(catalog)?; if dst.is_null() { return Err(Error::InvalidPointer); } - let stalled = State::lock().consume.video_stalled(catalog, index as usize)?; - unsafe { dst.write(stalled) }; + let enabled = State::lock().consume.video_enabled(catalog, index as usize)?; + unsafe { dst.write(enabled) }; Ok(()) }) } diff --git a/rs/moq-c/src/consume.rs b/rs/moq-c/src/consume.rs index a4e9bdb453..aed2e64588 100644 --- a/rs/moq-c/src/consume.rs +++ b/rs/moq-c/src/consume.rs @@ -278,8 +278,8 @@ impl Consume { Ok(()) } - /// Return whether the publisher recommends temporarily avoiding a video rendition. - pub fn video_stalled(&self, catalog: Id, index: usize) -> Result { + /// Return whether a video rendition may be selected. + pub fn video_enabled(&self, catalog: Id, index: usize) -> Result { let consume = self.catalog.get(catalog).ok_or(Error::CatalogNotFound)?; let (_, config) = consume .catalog @@ -288,7 +288,7 @@ impl Consume { .iter() .nth(index) .ok_or(Error::NoIndex)?; - Ok(config.stalled.unwrap_or(false)) + Ok(config.enabled) } /// Fill `dst` with the properties shared by every video rendition. diff --git a/rs/moq-c/src/test.rs b/rs/moq-c/src/test.rs index f656d9b263..a573c3bb39 100644 --- a/rs/moq-c/src/test.rs +++ b/rs/moq-c/src/test.rs @@ -754,10 +754,10 @@ fn publish_catalog_roundtrip() { container: moq_container::default(), }; assert_eq!(unsafe { moq_publish_video_config(broadcast, &video) }, 0); - let stalled_video_name = "video-stalled"; - let stalled_video = moq_video_config { - name: stalled_video_name.as_ptr() as *const c_char, - name_len: stalled_video_name.len(), + let disabled_video_name = "video-disabled"; + let disabled_video = moq_video_config { + name: disabled_video_name.as_ptr() as *const c_char, + name_len: disabled_video_name.len(), label: std::ptr::null(), label_len: 0, codec: video_codec.as_ptr() as *const c_char, @@ -768,7 +768,7 @@ fn publish_catalog_roundtrip() { coded_height: height, container: moq_container::default(), }; - assert_eq!(unsafe { moq_publish_video_config(broadcast, &stalled_video) }, 0); + assert_eq!(unsafe { moq_publish_video_config(broadcast, &disabled_video) }, 0); { let mut state = State::lock(); let (_, catalog) = state.publish.pair_mut(Id::try_from(broadcast).unwrap()).unwrap(); @@ -777,9 +777,9 @@ fn publish_catalog_roundtrip() { .unwrap() .video .renditions - .get_mut(stalled_video_name) + .get_mut(disabled_video_name) .unwrap() - .stalled = Some(true); + .enabled = false; } let properties = moq_video_properties { display_width: 1080, @@ -845,30 +845,30 @@ fn publish_catalog_roundtrip() { ); assert_eq!(video_cfg.coded_width, 1920); assert_eq!(video_cfg.coded_height, 1080); - let mut stalled = std::mem::MaybeUninit::::uninit(); + let mut enabled = std::mem::MaybeUninit::::uninit(); assert_eq!( - unsafe { moq_consume_video_stalled(catalog_id, 0, stalled.as_mut_ptr()) }, + unsafe { moq_consume_video_enabled(catalog_id, 0, enabled.as_mut_ptr()) }, 0 ); - assert!(!unsafe { stalled.assume_init() }); + assert!(unsafe { enabled.assume_init() }); - let mut stalled = std::mem::MaybeUninit::::uninit(); + let mut enabled = std::mem::MaybeUninit::::uninit(); assert_eq!( - unsafe { moq_consume_video_stalled(catalog_id, 1, stalled.as_mut_ptr()) }, + unsafe { moq_consume_video_enabled(catalog_id, 1, enabled.as_mut_ptr()) }, 0 ); - assert!(unsafe { stalled.assume_init() }); + assert!(!unsafe { enabled.assume_init() }); assert_eq!( - unsafe { moq_consume_video_stalled(catalog_id, 0, std::ptr::null_mut()) }, + unsafe { moq_consume_video_enabled(catalog_id, 0, std::ptr::null_mut()) }, -6, - "null stalled pointer should return InvalidPointer (-6)" + "null enabled pointer should return InvalidPointer (-6)" ); assert_eq!( unsafe { moq_publish_video_remove( broadcast, - stalled_video_name.as_ptr() as *const c_char, - stalled_video_name.len(), + disabled_video_name.as_ptr() as *const c_char, + disabled_video_name.len(), ) }, 0 diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 88ed3e866f..73d32bc35d 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -436,6 +436,7 @@ mod tests { sample_rate: 48_000, channel_count: 2, bitrate: None, + enabled: true, container: MoqContainer::Legacy, } } diff --git a/rs/moq-ffi/src/media.rs b/rs/moq-ffi/src/media.rs index 84213a118b..43c501dccc 100644 --- a/rs/moq-ffi/src/media.rs +++ b/rs/moq-ffi/src/media.rs @@ -91,9 +91,9 @@ pub struct MoqVideo { pub coded: Option, pub display_aspect: Option, pub bitrate: Option, - /// Whether the publisher recommends temporarily avoiding this rendition. - #[uniffi(default = false)] - pub stalled: bool, + /// Whether this rendition may be selected; when false, no frames are coming. + #[uniffi(default = true)] + pub enabled: bool, pub framerate: Option, pub container: MoqContainer, } @@ -112,6 +112,9 @@ pub struct MoqAudio { pub sample_rate: u32, pub channel_count: u32, pub bitrate: Option, + /// Whether this rendition may be selected; when false, no frames are coming. + #[uniffi(default = true)] + pub enabled: bool, pub container: MoqContainer, } @@ -381,7 +384,7 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog None, }, bitrate: config.bitrate, - stalled: config.stalled.unwrap_or(false), + enabled: config.enabled, framerate: config.framerate, container: MoqContainer::from_catalog(&config.container)?, }, @@ -404,6 +407,7 @@ pub(crate) fn convert_catalog(catalog: &moq_mux::catalog::hang::Catalog crate::media::MoqAudio { sample_rate: 48_000, channel_count: 2, bitrate: None, + enabled: true, container: MoqContainer::Legacy, } } diff --git a/rs/moq-ffi/src/video.rs b/rs/moq-ffi/src/video.rs index 5aef0c6c9c..163ebfd8b1 100644 --- a/rs/moq-ffi/src/video.rs +++ b/rs/moq-ffi/src/video.rs @@ -730,7 +730,6 @@ fn video_config(catalog_video: crate::media::MoqVideo) -> Result VideoConfig { config.jitter = None; config.delay = None; config.label = None; - config.stalled = None; + config.enabled = true; // Publishers estimate it and republish once it settles. The rendition keeps the fMP4 // timescale it was built with, and the framerate is refreshed in place like the bitrate. config.framerate = None; @@ -206,6 +206,7 @@ fn normalize_audio(config: &AudioConfig) -> AudioConfig { config.jitter = None; config.delay = None; config.label = None; + config.enabled = true; config } diff --git a/rs/moq-msf/src/lib.rs b/rs/moq-msf/src/lib.rs index 83087ca201..9f7cf1fc31 100644 --- a/rs/moq-msf/src/lib.rs +++ b/rs/moq-msf/src/lib.rs @@ -144,11 +144,6 @@ pub struct Track { /// Bitrate in bits per second. pub bitrate: Option, - /// Whether the publisher recommends temporarily avoiding this track. - /// - /// This is a non-standard extension shared with the hang catalog. - pub stalled: Option, - /// Resolved base64 initialization data. /// /// On the wire this is carried indirectly through draft-01's `initDataList` + @@ -481,7 +476,6 @@ impl Track { samplerate: None, channel_config: None, bitrate: None, - stalled: None, init_data: None, init_ref: None, render_group: None, @@ -633,7 +627,6 @@ mod test { samplerate: None, channel_config: None, bitrate: Some(6_000_000), - stalled: Some(true), init_data: None, init_ref: None, render_group: Some(1), @@ -658,7 +651,6 @@ mod test { samplerate: Some(48_000), channel_config: Some("2".to_string()), bitrate: Some(128_000), - stalled: None, init_data: None, init_ref: None, render_group: Some(1), @@ -683,7 +675,6 @@ mod test { samplerate: None, channel_config: None, bitrate: Some(5_000_000), - stalled: None, init_data: None, init_ref: None, render_group: Some(1), @@ -714,7 +705,6 @@ mod test { assert!(track.get("maxObjSapStartingType").is_none()); assert!(track.get("jitter").is_none()); assert!(track.get("delay").is_none()); - assert_eq!(track["stalled"], true); } #[test] diff --git a/rs/moq-mux/src/catalog/msf/consumer.rs b/rs/moq-mux/src/catalog/msf/consumer.rs index a7582392a7..11019e41e6 100644 --- a/rs/moq-mux/src/catalog/msf/consumer.rs +++ b/rs/moq-mux/src/catalog/msf/consumer.rs @@ -233,7 +233,6 @@ fn video_config_from_msf(track: &moq_msf::Track) -> Result> config.coded_width = track.width; config.coded_height = track.height; config.bitrate = track.bitrate; - config.stalled = track.stalled; config.framerate = track.framerate; config.container = container; config.jitter = track.jitter; @@ -408,7 +407,6 @@ mod test { track.height = Some(1080); track.framerate = Some(30.0); track.bitrate = Some(5_000_000); - track.stalled = Some(true); track.init_data = init_data.map(str::to_string); track.render_group = Some(1); track @@ -446,7 +444,6 @@ mod test { assert_eq!(video.coded_height, Some(1080)); assert_eq!(video.framerate, Some(30.0)); assert_eq!(video.bitrate, Some(5_000_000)); - assert_eq!(video.stalled, Some(true)); } #[test] diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index def17fb64c..dd4ebdb9fb 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -1049,7 +1049,6 @@ fn to_msf_media(catalog: &hang::Catalog) -> moq_msf::Catalog { track.height = config.coded_height; track.framerate = config.framerate; track.bitrate = config.bitrate; - track.stalled = config.stalled; track.init_data = init_data; track.render_group = Some(1); track.alt_group = if has_multiple_video { Some(1) } else { None }; @@ -1572,7 +1571,6 @@ mod test { video_config.coded_width = Some(1280); video_config.coded_height = Some(720); video_config.bitrate = Some(6_000_000); - video_config.stalled = Some(true); video_config.framerate = Some(30.0); video_config.container = Container::Legacy; @@ -1603,7 +1601,6 @@ mod test { assert_eq!(video.height, Some(720)); assert_eq!(video.framerate, Some(30.0)); assert_eq!(video.bitrate, Some(6_000_000)); - assert_eq!(video.stalled, Some(true)); assert!(video.init_data.is_none()); // H.264 may carry B-frames, so SAP starting type is 2 (leading pictures allowed). assert_eq!(video.max_grp_sap_starting_type, Some(2)); diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index e680f64849..15e6cc57ca 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -224,7 +224,7 @@ impl From for VideoHint { /// /// Total by construction: every field the hint can hold is taken from the config, so there is no /// per-field copy for a caller to forget. Fields with no hint slot (`broadcast`, `description`, - /// `stalled`) are set through the catalog directly. + /// `enabled`) are set through the catalog directly. fn from(config: hang::catalog::VideoConfig) -> Self { Self { label: config.label, diff --git a/rs/moq-mux/src/codec/av1/import.rs b/rs/moq-mux/src/codec/av1/import.rs index adabb67728..017d89ff7e 100644 --- a/rs/moq-mux/src/codec/av1/import.rs +++ b/rs/moq-mux/src/codec/av1/import.rs @@ -253,29 +253,10 @@ impl Import { // A pre-keyframe delta has no group to anchor it: the producer returns // MissingKeyframe, which a caller joining mid-stream skips. self.track.write(frame)?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; } Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// Publish split frames, resolving the config from the first keyframe's inline /// sequence header and refining the catalog jitter. pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index c5ce239469..b2fe8a3164 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -210,8 +210,6 @@ impl Import { } self.track.write(frame)?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; } Ok(()) } @@ -221,23 +219,6 @@ impl Import { pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { self.write_frames(frames) } - - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } } /// The catalog config a stream's codec bytes describe, resolved without publishing anything: an @@ -497,48 +478,4 @@ mod tests { "no config yet, so no catalog" ); } - - /// Encode lag past a few frame intervals marks the rendition stalled, and - /// releasing the source clears it. Demand is required; an idle import is not stalled. - #[tokio::test(start_paused = true)] - async fn encode_lag_marks_the_rendition_stalled() { - let sps: &[u8] = &[ - 0x67, 0x42, 0xc0, 0x1f, 0xda, 0x01, 0x40, 0x16, 0xe9, 0xb8, 0x08, 0x08, 0x0a, 0x00, 0x00, 0x07, 0xd0, 0x00, - 0x01, 0xd4, 0xc0, 0x80, - ]; - let pps: &[u8] = &[0x68, 0xce, 0x3c, 0x80]; - let idr: &[u8] = &[0x65, 0x88, 0x84, 0x21]; - let mut annexb = BytesMut::new(); - for nal in [sps, pps, idr] { - annexb.extend_from_slice(&[0, 0, 0, 1]); - annexb.extend_from_slice(nal); - } - - let mut split = Split::new(); - let (track, catalog) = setup("video"); - let _demand = track.subscribe(None); - let mut import = Import::new(track, catalog.reserve(), Default::default()).unwrap(); - let pts = moq_net::Timestamp::from_micros(0).unwrap(); - let mut frames = split.decode(&annexb, pts).expect("split keyframe"); - frames.extend(split.flush(pts).expect("flush keyframe")); - import.decode(frames).expect("decode keyframe"); - assert_eq!( - catalog.snapshot().video.renditions.get("video").and_then(|c| c.stalled), - None - ); - - import - .observe_lag(std::time::Duration::from_millis(200)) - .expect("observe lag"); - assert_eq!( - catalog.snapshot().video.renditions.get("video").and_then(|c| c.stalled), - Some(true) - ); - - import.idle().expect("idle"); - assert_eq!( - catalog.snapshot().video.renditions.get("video").and_then(|c| c.stalled), - None - ); - } } diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index a46634d931..f3755515dc 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -212,29 +212,10 @@ impl Import { // A pre-keyframe delta has no group to anchor it: the producer returns // MissingKeyframe, which the caller (e.g. a TS mid-stream join) skips. self.track.write(frame)?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; } Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// Publish split frames, resolving the config from the first keyframe's inline /// SPS and refining the catalog jitter as it goes. pub fn decode(&mut self, frames: impl IntoIterator) -> Result<()> { diff --git a/rs/moq-mux/src/codec/video.rs b/rs/moq-mux/src/codec/video.rs index 219da9d09d..58011bc49e 100644 --- a/rs/moq-mux/src/codec/video.rs +++ b/rs/moq-mux/src/codec/video.rs @@ -3,13 +3,8 @@ //! Every video importer resolves its [`VideoConfig`](hang::catalog::VideoConfig) lazily from the //! bitstream and re-publishes it whenever the stream reveals a change. [`Catalog`] owns the part of //! that which is identical across codecs: overlay the caller's [`VideoHint`], skip a publish -//! that matches the last one, and drive the shared [`hang::catalog::stalled::Detector`] detector so a -//! lagging rendition is marked in the catalog. [`Reorder`] is what a sequence header declares about -//! frame reordering, read by the exporters that author a decode clock. - -use std::time::{Duration, Instant}; - -use hang::catalog::stalled::{Detector, Sample}; +//! that matches the last one. [`Reorder`] is what a sequence header declares about frame +//! reordering, read by the exporters that author a decode clock. use crate::catalog::VideoHint; @@ -29,29 +24,20 @@ pub(crate) struct Reorder { /// The catalog-publishing state a video importer overlays onto every config it resolves. /// -/// Holds the caller's hint, the last config published (to dedupe re-publishes), and the -/// stalled detector. Not generic over the extension: none of that depends on it. +/// Holds the caller's hint and the last config published (to dedupe re-publishes). Not generic +/// over the extension: neither depends on it. pub(crate) struct Catalog { /// Overlaid onto every config, so a hinted field counts as supplied and is never overwritten by /// the rendition's detector. hint: VideoHint, /// The last config published, so an unchanged re-resolve doesn't re-mirror the rendition. last: Option, - stalled: Detector, - last_source: Option, - lag: Duration, } impl Catalog { /// Hold `hint` for every publish. pub(crate) fn new(hint: VideoHint) -> Self { - Self { - hint, - last: None, - stalled: Detector::new(), - last_source: None, - lag: Duration::ZERO, - } + Self { hint, last: None } } /// The config the hint alone resolves to, for importers that publish the catalog before parsing @@ -68,11 +54,9 @@ impl Catalog { /// Overlay the hint onto `config` and publish it to `rendition`, unless it matches the last /// publish. A changed config just re-mirrors the rendition; there are no fixed tracks to - /// reject a reconfiguration. The detector's current flag is applied so a bitstream republish - /// cannot clear a stall. + /// reject a reconfiguration. pub(crate) fn publish(&mut self, track: &mut Track, mut config: hang::catalog::VideoConfig) -> crate::Result<()> { self.hint.apply(&mut config); - config.stalled = self.stalled.flag(); if self.last.as_ref() == Some(&config) { return Ok(()); } @@ -81,124 +65,4 @@ impl Catalog { self.last = Some(config); Ok(()) } - - /// A frame was handed to the transport, completing one recovery observation. - pub(crate) fn on_frame(&mut self, track: &mut Track, demand: bool) -> crate::Result<()> { - self.last_source = Some(Instant::now()); - self.publish_stalled(track, demand, true, self.lag) - } - - /// Re-evaluate stall from silence: the source has not delivered since the last frame. - pub(crate) fn tick(&mut self, track: &mut Track, demand: bool) -> crate::Result<()> { - self.publish_stalled(track, demand, false, Duration::ZERO) - } - - /// Record extra delay (a slow encode) without treating it as a new source frame. - pub(crate) fn observe_lag(&mut self, track: &mut Track, demand: bool, lag: Duration) -> crate::Result<()> { - self.lag = lag; - self.publish_stalled(track, demand, false, lag) - } - - /// The source is gone (camera released). Never stalled. - pub(crate) fn idle(&mut self, track: &mut Track) -> crate::Result<()> { - self.last_source = None; - self.lag = Duration::ZERO; - self.publish_stalled(track, false, false, Duration::ZERO) - } - - fn publish_stalled( - &mut self, - track: &mut Track, - demand: bool, - frame: bool, - extra_lag: Duration, - ) -> crate::Result<()> { - if demand { - self.last_source.get_or_insert_with(Instant::now); - } else { - self.last_source = None; - } - let media_lag = extra_lag; - let quiet = self - .last_source - .map(|at| Instant::now().saturating_duration_since(at)) - .unwrap_or(Duration::ZERO); - let interval = hang::catalog::stalled::interval_from_fps(self.last.as_ref().and_then(|c| c.framerate)); - if !self.stalled.observe(Sample { - frame, - media_lag, - quiet, - interval, - demand, - idle: !demand, - }) { - return Ok(()); - } - let flag = self.stalled.flag(); - if let Some(last) = self.last.as_mut() { - last.stalled = flag; - } - let mut config = track.modify()?; - config.stalled = flag; - config.commit() - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn quiet_startup_tracks_demand() { - let mut broadcast = moq_net::broadcast::Info::new().produce(); - let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); - let reserved = catalog.reserve(); - let net = broadcast - .create_track("video", hang::container::track_info(hang::catalog::PRIORITY.video)) - .unwrap(); - let mut rendition = reserved - .video( - net, - crate::catalog::hang::Container::Legacy(crate::container::Kind::Video), - None, - ) - .unwrap(); - let mut state = Catalog::new(VideoHint::default()); - state - .publish( - &mut rendition, - hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8), - ) - .unwrap(); - - state.tick(&mut rendition, true).unwrap(); - let start = state - .last_source - .expect("demand starts the quiet clock before the first frame"); - state.last_source = Some(start - Duration::from_millis(200)); - state.tick(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, Some(true)); - - state.tick(&mut rendition, false).unwrap(); - assert!(state.last_source.is_none()); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, None); - state.tick(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, None); - - state - .observe_lag(&mut rendition, true, Duration::from_millis(200)) - .unwrap(); - state.on_frame(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, Some(true)); - state.observe_lag(&mut rendition, true, Duration::ZERO).unwrap(); - for _ in 0..2 { - state.on_frame(&mut rendition, true).unwrap(); - for _ in 0..10 { - state.tick(&mut rendition, true).unwrap(); - } - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, Some(true)); - } - state.on_frame(&mut rendition, true).unwrap(); - assert_eq!(catalog.snapshot().video.renditions["video"].stalled, None); - } } diff --git a/rs/moq-mux/src/codec/vp8/import.rs b/rs/moq-mux/src/codec/vp8/import.rs index 40bdb5c23b..dde79d1f31 100644 --- a/rs/moq-mux/src/codec/vp8/import.rs +++ b/rs/moq-mux/src/codec/vp8/import.rs @@ -88,29 +88,10 @@ impl Import { keyframe: header.keyframe, duration: None, })?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// A watch-only handle to this track's subscriber demand. pub fn demand(&self) -> moq_net::track::Demand { self.track.track().demand() diff --git a/rs/moq-mux/src/codec/vp9/import.rs b/rs/moq-mux/src/codec/vp9/import.rs index 477175b4ad..c873265943 100644 --- a/rs/moq-mux/src/codec/vp9/import.rs +++ b/rs/moq-mux/src/codec/vp9/import.rs @@ -88,29 +88,10 @@ impl Import { keyframe: header.keyframe, duration: None, })?; - let demand = self.track.demand().is_used(); - self.catalog.on_frame(&mut self.track, demand)?; Ok(()) } - /// Re-evaluate stall from source silence. - pub fn tick(&mut self) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.tick(&mut self.track, demand) - } - - /// The source is gone; this rendition is never stalled while idle. - pub fn idle(&mut self) -> crate::Result<()> { - self.catalog.idle(&mut self.track) - } - - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> crate::Result<()> { - let demand = self.track.demand().is_used(); - self.catalog.observe_lag(&mut self.track, demand, lag) - } - /// A watch-only handle to this track's subscriber demand. pub fn demand(&self) -> moq_net::track::Demand { self.track.track().demand() diff --git a/rs/moq-mux/src/container/flv/import.rs b/rs/moq-mux/src/container/flv/import.rs index 363495b1a2..0e56bb97de 100644 --- a/rs/moq-mux/src/container/flv/import.rs +++ b/rs/moq-mux/src/container/flv/import.rs @@ -21,7 +21,6 @@ //! enhanced audio, and any other codec, are logged and dropped. use std::collections::BTreeMap; -use std::time::{Duration, Instant}; use anyhow::Context; use bytes::{Buf, Bytes, BytesMut}; @@ -87,8 +86,6 @@ pub struct Import { struct VideoStream { track: crate::container::Producer, config: VideoConfig, - stalled: hang::catalog::stalled::Detector, - last_source: Option, } /// The demuxed audio track plus its current catalog config. @@ -181,7 +178,6 @@ impl Import { } } - self.tick_stalled()?; Ok(()) } @@ -473,26 +469,16 @@ impl Import { // A media frame means every sequence header has arrived (FLV sends config before data), so // the track set is declared; release the reservation to publish. self.initial_reservation = None; - let written = { - let stream = self.video.get_mut(&track_id).expect("checked above"); - match stream.track.write(Frame { - timestamp, - duration: None, - payload: Bytes::copy_from_slice(data), - keyframe, - }) { - Ok(()) => { - stream.last_source = Some(Instant::now()); - true - } - Err(crate::Error::MissingKeyframe(_)) => false, - Err(e) => return Err(e.into()), - } - }; - if written { - self.publish_stalled(track_id, true)?; + let stream = self.video.get_mut(&track_id).expect("checked above"); + match stream.track.write(Frame { + timestamp, + duration: None, + payload: Bytes::copy_from_slice(data), + keyframe, + }) { + Ok(()) | Err(crate::Error::MissingKeyframe(_)) => Ok(()), + Err(e) => Err(e.into()), } - Ok(()) } /// Write one audio frame as its own group, so the relay can forward it immediately. @@ -541,8 +527,6 @@ impl Import { // site (the producer reports MissingKeyframe), so a mid-GOP join works. track: media, config, - stalled: hang::catalog::stalled::Detector::new(), - last_source: None, }, ); Ok(()) @@ -568,46 +552,6 @@ impl Import { Ok(()) } - fn tick_stalled(&mut self) -> anyhow::Result<()> { - let ids: Vec = self.video.keys().copied().collect(); - for id in ids { - self.publish_stalled(id, false)?; - } - Ok(()) - } - - fn publish_stalled(&mut self, track_id: u8, frame: bool) -> anyhow::Result<()> { - let Some(stream) = self.video.get_mut(&track_id) else { - return Ok(()); - }; - let demand = stream.track.demand().is_used(); - if demand { - stream.last_source.get_or_insert_with(Instant::now); - } else { - stream.last_source = None; - } - let quiet = stream - .last_source - .map(|at| Instant::now().saturating_duration_since(at)) - .unwrap_or(Duration::ZERO); - let interval = hang::catalog::stalled::interval_from_fps(stream.config.framerate); - if !stream.stalled.observe(hang::catalog::stalled::Sample { - frame, - media_lag: Duration::ZERO, - quiet, - interval, - demand, - idle: false, - }) { - return Ok(()); - } - let flag = stream.stalled.flag(); - let mut config = stream.track.modify()?; - config.stalled = flag; - config.commit()?; - Ok(()) - } - /// Drop any existing video track `track_id` (finishing it and clearing its /// catalog rendition) and allocate a fresh one. fn replace_video(&mut self, track_id: u8) -> anyhow::Result { diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 07cc91409f..1728195c3a 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -336,10 +336,6 @@ impl Import { // decode call (plus the store's own host-clock debounce) coalesces a junction's // burst of sub-table commits into few groups instead of one per commit. self.si.flush(self.last_pts.unwrap_or(Timestamp::ZERO), false)?; - // Video PIDs that went quiet while the mux kept flowing mark themselves stalled. - for stream in self.streams.values_mut() { - stream.tick()?; - } Ok(()) } @@ -1805,20 +1801,6 @@ impl Stream { /// the cut are whole and correct on their own. False where it carries exactly one: half /// an access unit is a picture with missing slices, and half a keyframe stays wrong for /// every picture that references it. Verbatim payloads are all-or-nothing the same way. - fn tick(&mut self) -> anyhow::Result<()> { - match self { - Stream::H264 { import, .. } => import.tick()?, - Stream::H265 { import, .. } => import.tick()?, - Stream::Aac(_) - | Stream::Opus(_) - | Stream::Legacy(_) - | Stream::Verbatim(_) - | Stream::Clock - | Stream::Ignored => {} - } - Ok(()) - } - fn salvages_partial_pes(&self) -> bool { match self { Stream::Aac(_) | Stream::Legacy(_) => true, diff --git a/rs/moq-rtc/src/codec/av1.rs b/rs/moq-rtc/src/codec/av1.rs index b16898b421..b01bc9847e 100644 --- a/rs/moq-rtc/src/codec/av1.rs +++ b/rs/moq-rtc/src/codec/av1.rs @@ -32,11 +32,6 @@ impl codec::Bridge for Bridge { Ok(()) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/h264.rs b/rs/moq-rtc/src/codec/h264.rs index 961a692255..6ecca1d033 100644 --- a/rs/moq-rtc/src/codec/h264.rs +++ b/rs/moq-rtc/src/codec/h264.rs @@ -31,11 +31,6 @@ impl codec::Bridge for Bridge { Ok(()) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/h265.rs b/rs/moq-rtc/src/codec/h265.rs index 606cad6af3..f04c1c5a3e 100644 --- a/rs/moq-rtc/src/codec/h265.rs +++ b/rs/moq-rtc/src/codec/h265.rs @@ -33,11 +33,6 @@ impl codec::Bridge for Bridge { Ok(()) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/mod.rs b/rs/moq-rtc/src/codec/mod.rs index d4b7b23f9b..773196859a 100644 --- a/rs/moq-rtc/src/codec/mod.rs +++ b/rs/moq-rtc/src/codec/mod.rs @@ -44,12 +44,6 @@ pub struct Frame { pub trait Bridge: Send { fn push(&mut self, frame: Frame) -> Result<()>; - /// Re-evaluate stall from source silence. Video bridges mark a quiet - /// rendition stalled; audio is a no-op. - fn tick(&mut self) -> Result<()> { - Ok(()) - } - /// Abort the published track with `err` so subscribers see the real cause /// (the peer disconnected, an ICE failure) rather than a bare `Error::Dropped`. /// @@ -65,9 +59,6 @@ pub(crate) trait DeferredImport: Send + Sized { /// Decode one complete codec frame. fn decode(&mut self, frame: Bytes, pts: moq_net::Timestamp) -> moq_mux::Result<()>; - /// Re-evaluate stall from source silence. - fn tick(&mut self) -> moq_mux::Result<()>; - /// Abort the active media track. fn abort(self, err: moq_net::Error); } @@ -81,10 +72,6 @@ impl DeferredImport for moq_mux::codec::vp8::Import { moq_mux::codec::vp8::Import::decode(self, frame, Some(pts)) } - fn tick(&mut self) -> moq_mux::Result<()> { - moq_mux::codec::vp8::Import::tick(self) - } - fn abort(self, err: moq_net::Error) { moq_mux::codec::vp8::Import::abort(self, err); } @@ -99,10 +86,6 @@ impl DeferredImport for moq_mux::codec::vp9::Import { moq_mux::codec::vp9::Import::decode(self, frame, Some(pts)) } - fn tick(&mut self) -> moq_mux::Result<()> { - moq_mux::codec::vp9::Import::tick(self) - } - fn abort(self, err: moq_net::Error) { moq_mux::codec::vp9::Import::abort(self, err); } @@ -163,15 +146,6 @@ impl DeferredVideo { import.decode(frame, pts).map_err(Into::into) } - /// Re-evaluate stall from source silence once the importer exists. - pub fn tick(&mut self) -> Result<()> { - if let DeferredState::Active(import) = &mut self.state { - import.tick().map_err(Into::into) - } else { - Ok(()) - } - } - /// Abort the media track in either lifecycle state. pub fn abort(self, err: moq_net::Error) { match self.state { diff --git a/rs/moq-rtc/src/codec/vp8.rs b/rs/moq-rtc/src/codec/vp8.rs index 9f458a7e0c..b418f9e802 100644 --- a/rs/moq-rtc/src/codec/vp8.rs +++ b/rs/moq-rtc/src/codec/vp8.rs @@ -25,11 +25,6 @@ impl codec::Bridge for Bridge { self.import.decode(frame.payload, pts) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/codec/vp9.rs b/rs/moq-rtc/src/codec/vp9.rs index 0bd213df9c..e70dae9828 100644 --- a/rs/moq-rtc/src/codec/vp9.rs +++ b/rs/moq-rtc/src/codec/vp9.rs @@ -25,11 +25,6 @@ impl codec::Bridge for Bridge { self.import.decode(frame.payload, pts) } - fn tick(&mut self) -> Result<()> { - self.import.tick()?; - Ok(()) - } - fn abort(self: Box, err: moq_net::Error) { self.import.abort(err); } diff --git a/rs/moq-rtc/src/ingest.rs b/rs/moq-rtc/src/ingest.rs index 3dec2cd57a..ab5213d391 100644 --- a/rs/moq-rtc/src/ingest.rs +++ b/rs/moq-rtc/src/ingest.rs @@ -68,10 +68,6 @@ impl session::MediaSink for IngestSink { self.bridges.push(mid, frame) } - fn tick(&mut self) -> Result<()> { - self.bridges.tick() - } - fn abort(&mut self, err: moq_net::Error) { self.bridges.abort(err); } diff --git a/rs/moq-rtc/src/session.rs b/rs/moq-rtc/src/session.rs index e0dd59c288..e2f815aecf 100644 --- a/rs/moq-rtc/src/session.rs +++ b/rs/moq-rtc/src/session.rs @@ -73,11 +73,6 @@ pub trait MediaSink: Send { /// loop has already converted the timestamp to microseconds. fn on_frame(&mut self, mid: str0m::media::Mid, frame: codec::Frame) -> Result<()>; - /// Re-evaluate stall from source silence on every ingest track. - fn tick(&mut self) -> Result<()> { - Ok(()) - } - /// Called once when the session ends with a genuine failure, so the sink can /// abort its tracks with the real cause instead of a bare `Error::Dropped`. fn abort(&mut self, err: moq_net::Error); @@ -232,12 +227,7 @@ impl Session { } let timeout = match self.rtc.poll_output().map_err(Error::rtc)? { - Output::Timeout(t) => { - if let MediaRole::Ingest(sink) = &mut self.role { - sink.tick()?; - } - t - } + Output::Timeout(t) => t, Output::Transmit(t) => { let dst = crate::net::to_family(t.destination, socket_v6); if let Err(err) = self.socket.send_to(&t.contents, dst).await { @@ -592,13 +582,6 @@ impl Bridges { Ok(()) } - pub fn tick(&mut self) -> Result<()> { - for bridge in self.inner.values_mut() { - bridge.tick()?; - } - Ok(()) - } - /// Abort every bridge's track with `err` so subscribers see the real cause /// rather than a bare `Error::Dropped`. /// diff --git a/rs/moq-transcode/src/catalog.rs b/rs/moq-transcode/src/catalog.rs index 559b8ac1b9..bc275e712b 100644 --- a/rs/moq-transcode/src/catalog.rs +++ b/rs/moq-transcode/src/catalog.rs @@ -159,13 +159,15 @@ impl Decoders { } /// Pick the rendition to transcode from: the best [ranked](Video::ranked) rendition local -/// to the source broadcast that this host can decode. +/// to the source broadcast that this host can decode, enabled ones first. /// /// [`Error::NoSource`] means wait for a later snapshot. Any other error means /// nothing on offer can be decoded here, and is why the best one refused. pub(crate) async fn choose_source(video: &Video, decoders: &mut Decoders) -> Result<(String, VideoConfig), Error> { // Best first. A rendition without dimensions ranks after every one with them: // it can't be chosen yet, but it can still keep the transcoder waiting. + // A disabled rendition ranks last, so it is the source only when nothing enabled is, + // and the rungs inherit `enabled: false` rather than the ladder vanishing. let candidates = video .ranked() // A rendition that itself lives in another broadcast can't be subscribed @@ -216,6 +218,7 @@ pub(crate) async fn follow_source( current: &str, decoders: &mut Decoders, ) -> Result<(String, VideoConfig), Error> { + let mut kept = None; if let Some(config) = video.renditions.get(current) && config.broadcast.is_none() && dimensions(config).is_some() @@ -223,9 +226,19 @@ pub(crate) async fn follow_source( // The name can stay while the codec changes under it. && decoders.probe(codec, config).await { - return Ok((current.to_string(), config.clone())); + if config.enabled { + return Ok((current.to_string(), config.clone())); + } + kept = Some(config); + } + + // A disabled source gives way only to an enabled one. An enabled source is never + // swapped for a better one, even its own predecessor re-enabled, to avoid churning the ladder. + match (kept, choose_source(video, decoders).await) { + (Some(config), Ok((_, chosen))) if !chosen.enabled => Ok((current.to_string(), config.clone())), + (Some(config), Err(_)) => Ok((current.to_string(), config.clone())), + (_, chosen) => chosen, } - choose_source(video, decoders).await } /// Whether a rung decoding `old` can keep decoding `new` untouched. @@ -327,10 +340,11 @@ pub(crate) async fn rung_entry( Ok(entry) } -/// Rungs inherit the state of the rendition the pipeline actually decodes. -pub(crate) fn inherit_stalled(rungs: &mut [Published], source: &VideoConfig) { +/// Rungs are enabled exactly when the rendition the pipeline decodes is: a disabled source sends +/// no frames, so neither can its rungs. +pub(crate) fn inherit_enabled(rungs: &mut [Published], source: &VideoConfig) { for published in rungs { - published.entry.stalled = source.stalled; + published.entry.enabled = source.enabled; } } @@ -415,10 +429,10 @@ mod tests { } #[test] - fn rungs_inherit_a_stalled_source() { + fn rungs_inherit_a_disabled_source() { let mut source_catalog = moq_mux::catalog::hang::Catalog::default(); let mut src = source(1280, 720, Some(2_500_000)); - src.stalled = Some(true); + src.enabled = false; source_catalog.video.insert("video", src).unwrap(); let mut published = [Published { @@ -432,20 +446,17 @@ mod tests { entry: source(640, 360, Some(600_000)), }]; - inherit_stalled(&mut published, &source_catalog.video.renditions["video"]); + inherit_enabled(&mut published, &source_catalog.video.renditions["video"]); let mut out = moq_mux::catalog::hang::Catalog::default(); populate(&mut out, &source_catalog, &published, None).unwrap(); - assert_eq!( - out.video.renditions.get("video/360p").and_then(|c| c.stalled), - Some(true) - ); + assert!(!out.video.renditions["video/360p"].enabled); - // A different local rendition may stay stalled after the selected input recovers. + // Following an enabled input re-enables the rungs. let healthy = source(1920, 1080, Some(5_000_000)); source_catalog.video.insert("healthy", healthy.clone()).unwrap(); - inherit_stalled(&mut published, &healthy); + inherit_enabled(&mut published, &healthy); populate(&mut out, &source_catalog, &published, None).unwrap(); - assert_eq!(out.video.renditions["video/360p"].stalled, None); + assert!(out.video.renditions["video/360p"].enabled); } #[test] @@ -823,6 +834,47 @@ mod tests { assert_eq!(name, "tall"); } + /// A disabled rendition sends no frames, so an enabled one wins even when it ranks lower, + /// and a disabled one is chosen only when nothing enabled is on offer. + #[tokio::test] + async fn chooses_an_enabled_source_first() { + let mut video = Video::default(); + let mut high = source(1920, 1080, None); + high.enabled = false; + video.insert("high", high).unwrap(); + video.insert("low", source(640, 360, None)).unwrap(); + + let (name, _) = choose_source(&video, &mut decodes_all()).await.unwrap(); + assert_eq!(name, "low"); + + video.renditions.get_mut("low").unwrap().enabled = false; + let (name, config) = choose_source(&video, &mut decodes_all()).await.unwrap(); + assert_eq!(name, "high"); + assert!(!config.enabled); + } + + /// A source that is disabled gives way to an enabled rendition, but stays put when every + /// rendition is disabled, so its rungs inherit `enabled: false` instead of switching. + #[tokio::test] + async fn follow_leaves_a_disabled_source_only_for_an_enabled_one() { + let mut video = Video::default(); + let mut low = source(640, 360, None); + low.enabled = false; + video.insert("low", low).unwrap(); + let mut high = source(1920, 1080, None); + high.enabled = false; + video.insert("high", high).unwrap(); + let mut decoders = decodes_all(); + + let (name, config) = follow_source(&video, "low", &mut decoders).await.unwrap(); + assert_eq!(name, "low"); + assert!(!config.enabled); + + video.renditions.get_mut("high").unwrap().enabled = true; + let (name, _) = follow_source(&video, "low", &mut decoders).await.unwrap(); + assert_eq!(name, "high"); + } + /// Probing opens a real decoder, so each codec is opened once per transcoder /// rather than once per catalog edit. #[tokio::test] diff --git a/rs/moq-transcode/src/pipeline.rs b/rs/moq-transcode/src/pipeline.rs index a81becc9fe..ea33202fd3 100644 --- a/rs/moq-transcode/src/pipeline.rs +++ b/rs/moq-transcode/src/pipeline.rs @@ -18,12 +18,12 @@ use std::collections::HashMap; use hang::catalog::{Video, VideoConfig}; -/// Picture identity for ladder follow: a stall bit flipping is not a source change. +/// Picture identity for ladder follow: enabling or disabling a rendition is not a source change. fn same_picture(a: &VideoConfig, b: &VideoConfig) -> bool { let mut a = a.clone(); let mut b = b.clone(); - a.stalled = None; - b.stalled = None; + a.enabled = true; + b.enabled = true; a == b } @@ -135,7 +135,7 @@ impl Pipeline { }; published.push(Published { rung, entry }); } - catalog::inherit_stalled(&mut published, source); + catalog::inherit_enabled(&mut published, source); Ok(published) } @@ -179,7 +179,7 @@ impl Pipeline { } }; if name == self.name && same_picture(&rendition, &self.rendition) { - catalog::inherit_stalled(&mut self.rungs, &rendition); + catalog::inherit_enabled(&mut self.rungs, &rendition); self.rendition = rendition; return Ok(()); } diff --git a/rs/moq-video/src/encode/producer.rs b/rs/moq-video/src/encode/producer.rs index d6fe58ad7b..44be26c103 100644 --- a/rs/moq-video/src/encode/producer.rs +++ b/rs/moq-video/src/encode/producer.rs @@ -177,33 +177,6 @@ impl Producer { Ok(()) } - /// Record the encode duration before publishing its frames so the catalog can report a stall. - pub fn observe_lag(&mut self, lag: std::time::Duration) -> Result<(), Error> { - match &mut self.codecs { - Codecs::H264 { import, .. } => import.observe_lag(lag)?, - Codecs::H265 { import, .. } => import.observe_lag(lag)?, - } - Ok(()) - } - - /// Re-evaluate stall from source silence while waiting for the next frame. - pub fn tick(&mut self) -> Result<(), Error> { - match &mut self.codecs { - Codecs::H264 { import, .. } => import.tick()?, - Codecs::H265 { import, .. } => import.tick()?, - } - Ok(()) - } - - /// The camera is released; this rendition is never stalled while idle. - pub fn idle(&mut self) -> Result<(), Error> { - match &mut self.codecs { - Codecs::H264 { import, .. } => import.idle()?, - Codecs::H265 { import, .. } => import.idle()?, - } - Ok(()) - } - /// Mark a break in the published timeline: whatever is published next does not continue /// what came before. /// @@ -648,29 +621,22 @@ fn capture_stopped(producer: &mut Producer) -> Result<(), Erro producer.discontinuity() } -// Keep observing silence while source setup or an encode is pending. The work -// future stays pinned across ticks, so a slow operation is never restarted. +// Race source setup or an encode against the last viewer leaving, so an unwatched track +// stops waiting promptly. #[cfg(feature = "capture")] -async fn wait_capture( - producer: &mut Producer, +async fn wait_capture( demand: &moq_net::track::Demand, work: impl std::future::Future>, ) -> Result, Error> { - let mut work = std::pin::pin!(work); - let mut timer = tokio::time::interval(hang::catalog::stalled::DEFAULT_INTERVAL); - loop { - tokio::select! { - biased; - res = demand.unused() => { - if let Err(err) = res { - log_track_ended(err); - } - producer.idle()?; - return Ok(None); + tokio::select! { + biased; + res = demand.unused() => { + if let Err(err) = res { + log_track_ended(err); } - _ = timer.tick() => producer.tick()?, - res = &mut work => return res.map(Some), + Ok(None) } + res = work => res.map(Some), } } @@ -710,7 +676,7 @@ async fn capture_loop( } // Open the camera and an encoder sized to its negotiated mode. - let Some(mut camera) = wait_capture(producer, demand, source.open(capture)).await? else { + let Some(mut camera) = wait_capture(demand, source.open(capture)).await? else { continue; }; // Capture timestamps use a private monotonic timeline. Sample both clocks @@ -733,7 +699,7 @@ async fn capture_loop( // No cut on reopen: a fresh encoder opens with a keyframe on every backend, // so the viewer whose subscription reopened the camera can decode from the // first frame regardless, and a backend that cannot cut still captures. - let Some(mut encoder) = wait_capture(producer, demand, Sink::open(&encoder_config)).await? else { + let Some(mut encoder) = wait_capture(demand, Sink::open(&encoder_config)).await? else { continue; }; // A reopen can land on another backend than the probe did, so the controls @@ -761,7 +727,6 @@ async fn capture_loop( // Race the next frame against the last viewer leaving so we release the // camera promptly when demand drops. `biased` checks demand first so an // unwatched track stops before reading another frame. - let interval = hang::catalog::stalled::interval_from_fps(Some(framerate.as_f64())); let frame = tokio::select! { biased; res = demand.unused() => { @@ -779,14 +744,7 @@ async fn capture_loop( } // A read error is terminal for this selection (the source is gone // or was refused); `None` just ends the stream, so reopen below. - // Timing out is a quiet camera: mark the rendition stalled and wait again. - frame = tokio::time::timeout(interval, camera.read()) => match frame { - Ok(frame) => frame?, - Err(_) => { - producer.tick()?; - continue; - } - }, + frame = camera.read() => frame?, }; let Some(mut frame) = frame else { break }; @@ -796,12 +754,9 @@ async fn capture_loop( // Never refused: `Control::cut` stops counting once this encoder is known to. encoder.cut().await?; } - let started = Instant::now(); - let Some(encoded) = wait_capture(producer, demand, encoder.encode(frame)).await? else { + let Some(encoded) = wait_capture(demand, encoder.encode(frame)).await? else { break; }; - let lag = started.elapsed(); - producer.observe_lag(lag)?; for unit in encoded.iter().filter(|unit| unit.keyframe) { forced.keyframe(unit.timestamp); } @@ -811,7 +766,6 @@ async fn capture_loop( // Drop the camera (LED off) and encoder before waiting for the next viewer. drop(camera); drop(encoder); - producer.idle()?; capture_stopped(producer)?; tracing::info!("capture stopped; released source"); } @@ -1511,7 +1465,7 @@ mod tests { use moq_mux::container::Container as _; // Off Apple, encoding waits on an OS thread. Keep Tokio from auto-advancing - // the paused clock to a stall deadline while that thread answers. + // the paused clock to a timer deadline while that thread answers. let (hold, released) = tokio::sync::oneshot::channel::<()>(); let clock = tokio::task::spawn_blocking(move || { let _ = released.blocking_recv(); diff --git a/swift/Sources/Moq/Aliases.swift b/swift/Sources/Moq/Aliases.swift index 6189bf6c54..8188c1205e 100644 --- a/swift/Sources/Moq/Aliases.swift +++ b/swift/Sources/Moq/Aliases.swift @@ -15,8 +15,8 @@ public typealias MediaFrame = MoqFFI.MoqMediaFrame /// The JSON manifest describing a broadcast's tracks: video and audio /// renditions, display geometry, and untyped application sections. public typealias Catalog = MoqFFI.MoqCatalog -/// A video rendition in the catalog: codec, dimensions, bitrate, temporary -/// avoidance recommendation, framerate, and container. +/// A video rendition in the catalog: codec, dimensions, bitrate, whether it is +/// enabled, framerate, and container. public typealias Video = MoqFFI.MoqVideo /// Caller-provided catalog fields for a video track. public typealias VideoHint = MoqFFI.MoqVideoHint @@ -30,7 +30,7 @@ public typealias ContainerFormat = MoqFFI.MoqContainerFormat /// that property from the next catalog snapshot. public typealias VideoProperties = MoqFFI.MoqVideoProperties /// An audio rendition in the catalog: codec, sample rate, channel count, -/// bitrate, and container. +/// bitrate, whether it is enabled, and container. public typealias Audio = MoqFFI.MoqAudio /// One raw-audio frame: PCM samples in the configured layout plus a /// presentation timestamp. diff --git a/test/interop/clients/js/media.ts b/test/interop/clients/js/media.ts index d73259a50e..2d24122eb4 100644 --- a/test/interop/clients/js/media.ts +++ b/test/interop/clients/js/media.ts @@ -55,7 +55,7 @@ import { import * as Pattern from "./src/pattern"; /** Cases beyond the mandatory capability probe, publisher readiness, and cold start. */ -const CASES = ["capture-denial", "pause", "rejoin", "detach", "republish", "late-join"] as const; +const CASES = ["capture-denial", "pause", "disable", "rejoin", "detach", "republish", "late-join"] as const; type Case = (typeof CASES)[number]; const { values } = parseArgs({ @@ -565,6 +565,44 @@ try { assertMedia(await collect(player, playerErrors, WINDOW_MS), "after resume"); } + // ── disable and re-enable the video rendition ──────────────────────────── + // A disabled rendition stays in the catalog with `enabled: false`. The player deselects it and + // shows black instead of freezing on the last picture, then picks it up again once re-enabled. + if (wants("disable")) { + console.error("=== disable and enable video ==="); + const before = await readPlayerState(player); + await command(publisher, "disableVideo"); + await waitForFixture(publisher, publisherErrors, { + deadline: Date.now() + SETTLE_MS, + assertion: "disable keeps the rendition", + description: "the fixture to publish its video rendition with enabled: false and stop encoding", + predicate: (state) => !state.videoEnabled && !state.videoActive, + }); + await waitForState(player, playerErrors, { + deadline: Date.now() + SETTLE_MS, + assertion: "disable blanks the picture", + description: `the canvas to go black instead of holding frame ${before.frameId}`, + predicate: (state) => !state.painted, + }); + const blank = await collect(player, playerErrors, HELD_MS); + const lit = blank.filter((s) => s.painted); + check( + lit.length === 0, + "disable blanks the picture", + () => `the canvas showed a picture again while disabled: ${JSON.stringify(lit[0])}`, + ); + + await command(publisher, "enableVideo"); + const resumed = await waitForState(player, playerErrors, { + deadline: Date.now() + timeoutMs, + assertion: "enable resumes playback", + description: `the presented frame to move past the ${before.frameId} showing before the disable`, + predicate: (state) => (state.frameId ?? 0) > (before.frameId ?? 0), + }); + console.error(` black while disabled, then resumed at frame ${resumed.frameId}`); + assertMedia(await collect(player, playerErrors, WINDOW_MS), "after enable"); + } + // ── unsubscribe and rejoin ─────────────────────────────────────────────── if (wants("rejoin")) { console.error("=== unsubscribe and rejoin ==="); diff --git a/test/interop/clients/js/src/contract.ts b/test/interop/clients/js/src/contract.ts index ae44d2ae99..1211e7d087 100644 --- a/test/interop/clients/js/src/contract.ts +++ b/test/interop/clients/js/src/contract.ts @@ -43,6 +43,8 @@ export type FixtureState = { audioActive: boolean; /** Frames the encoder has produced. */ encodedFrames: number; + /** Whether the video rendition is published as enabled; false while {@link InteropControl.disableVideo} holds. */ + videoEnabled: boolean; }; /** The newest published video's GOP, sampled independently of canvas capture and encoding. */ @@ -214,6 +216,10 @@ export type InteropControl = { reattach(): void; /** Connect a second player and leave it behind for the leaked-session negative control. */ startLeak(): void; + /** Keep the fixture's video rendition in the catalog with `enabled: false`, stopping its frames. */ + disableVideo(): void; + /** Re-enable the fixture's video rendition. */ + enableVideo(): void; }; /** The `window` property the commands are published on. */ diff --git a/test/interop/clients/js/src/fixture.ts b/test/interop/clients/js/src/fixture.ts index a3d3ff1a78..d8f141a78f 100644 --- a/test/interop/clients/js/src/fixture.ts +++ b/test/interop/clients/js/src/fixture.ts @@ -51,6 +51,7 @@ export class Fixture { readonly #audio: AudioContext; readonly #frameId = new Signal(-1); readonly #audioState = new Signal("suspended"); + readonly #videoEnabled = new Signal(true); #timer: number | undefined; constructor(host: HTMLElement, url: string, name: string, fault: Fault) { @@ -101,7 +102,7 @@ export class Fixture { const video = new Publish.Video.Encoder("video", { broadcast, capture, - enabled: true, + enabled: this.#videoEnabled, config: { frameRate: Pattern.FPS, keyframeInterval: KEYFRAME_INTERVAL, maxBitrate: MAX_BITRATE }, }); this.#signals.cleanup(() => video.close()); @@ -163,6 +164,7 @@ export class Fixture { videoActive: effect.get(video.out.active), audioActive: effect.get(audio.out.active), encodedFrames: effect.get(video.out.stats).frames, + videoEnabled: effect.get(video.out.catalog)?.enabled !== false, }; host.dataset.interopFixture = JSON.stringify(state); }); @@ -213,6 +215,11 @@ export class Fixture { }); } + /** Enable or disable the video rendition, which stays in the catalog either way. */ + setVideo(enabled: boolean): void { + this.#videoEnabled.set(enabled); + } + /** Sample the current published keyframe before the viewer closes its subscription. */ async liveGop() { const broadcast = this.#broadcast.net.peek(); diff --git a/test/interop/clients/js/src/setup.ts b/test/interop/clients/js/src/setup.ts index 389e0850de..c6525e4ae1 100644 --- a/test/interop/clients/js/src/setup.ts +++ b/test/interop/clients/js/src/setup.ts @@ -69,6 +69,8 @@ if (role === "publish") { fixture?.close(); fixture = new Fixture(host, url, broadcast, fault); }, + disableVideo: () => fixture?.setVideo(false), + enableVideo: () => fixture?.setVideo(true), }); } else if (role === "subscribe") { await customElements.whenDefined("moq-watch");