diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index 7f95bc3083..b64d7e3523 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -536,6 +536,17 @@ For example: - A fragment or packet batch contributes the media span between its earliest timestamp and flush point. - Reordered frames contribute the delay they were held before flushing, without treating a decode-order presentation timestamp gap as delay by itself. +### warmup {#field-warmup} + +An audio or video rendition MAY declare `warmup`, an integer duration in milliseconds after a group start before its decoded output is presentable without decoding the previous group. +A publisher MUST round a fractional-millisecond warmup up to the next whole millisecond. +Absent or `0` means every group start is immediately presentable. +After a non-continuous join, a consumer MUST decode from the group start and MUST NOT present frames stamped before the group's start plus `warmup`. +It SHOULD join that much earlier to obtain the requested presentation time. + +For video, `warmup` can describe a refresh cycle during which decoding converges without an initial keyframe. +For audio, it describes decoder pre-roll, for example 80 milliseconds for Opus, and does not change the group-start rule. + ### delay {#field-delay} The maximum amount, in milliseconds, by which a rendition's minimum flush lateness ({{field-jitter}}) has trailed the smallest minimum among the broadcast's renditions that measure it. If absent, a consumer SHOULD assume the rendition does not trail the others. @@ -571,9 +582,11 @@ type Container = The `kind` field selects the framing; a consumer MUST ignore a rendition whose `kind` it does not recognize. Every container shares the same group rules: -Each moq-lite group MUST start with a keyframe, except a group that contains only a discontinuity marker. +A video group MUST start at a random access point: a keyframe unless the rendition declares `warmup` ({{field-warmup}}), in which case it MAY start at a picture from which decoding converges within that duration. +Other groups MUST start with a keyframe. +A group containing only a discontinuity marker is exempt from these rules. If the codec does not support delta frames (e.g. audio), a group MAY consist of multiple keyframes. -Otherwise, a group MUST consist of a single keyframe followed by zero or more delta frames. +Otherwise, a group MUST consist of its random access picture followed by zero or more delta frames. A group with no decodable frames is a walk-now discontinuity: one empty codec payload and no media. A consumer MUST NOT submit the marker to a decoder. @@ -1074,6 +1087,7 @@ This document has no IANA actions. ## moq-hang-04 {:numbered="false"} +- Added optional audio and video `warmup` durations, rounded up to whole milliseconds, and allowed video groups to start at convergent random access pictures. - Defined encoder `jitter` as flush lateness above the rendition's own recent minimum, replacing fixed frame-duration hints; container batches retain media-span estimates. - Timed CMAF samples from the frame timestamp, which is the fragment's earliest presentation time; `tfdt` is only relative within the fragment, and places the samples of an untimed frame. A timed CMAF track counts in its `mdhd` ticks. - Allowed a DVR to delete timeline objects no checkpoint recovery needs, oldest first. diff --git a/js/hang/src/catalog/audio.test.ts b/js/hang/src/catalog/audio.test.ts index e3bc0d5b4a..e81523d1d8 100644 --- a/js/hang/src/catalog/audio.test.ts +++ b/js/hang/src/catalog/audio.test.ts @@ -36,3 +36,16 @@ test("audio config accepts optional enabled state", () => { expect(config.enabled).toBe(false); }); + +test("audio warmup round trips as optional integer milliseconds", () => { + const base = { codec: "opus", container: { kind: "legacy" }, sampleRate: 48_000, numberOfChannels: 2 }; + expect(AudioConfigSchema.parse(base).warmup).toBeUndefined(); + for (const warmup of [0, 80, 1_000]) { + const config = AudioConfigSchema.parse({ ...base, warmup }); + expect(Number(config.warmup)).toBe(warmup); + expect(AudioConfigSchema.parse(JSON.parse(JSON.stringify(config)))).toEqual(config); + } + for (const warmup of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => AudioConfigSchema.parse({ ...base, warmup })).toThrow(); + } +}); diff --git a/js/hang/src/catalog/audio.ts b/js/hang/src/catalog/audio.ts index 451f7be337..c40d4bd49d 100644 --- a/js/hang/src/catalog/audio.ts +++ b/js/hang/src/catalog/audio.ts @@ -64,6 +64,10 @@ export const AudioConfigSchema = z.object({ ), ), + // After a non-continuous join, decode from the group start, present at start plus warmup, + // and join that many milliseconds earlier. + warmup: z.optional(u53Schema), + // How far this rendition's frames reach the transport behind the broadcast's earliest // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only diff --git a/js/hang/src/catalog/video.test.ts b/js/hang/src/catalog/video.test.ts index be03b285a6..69db8fc0a1 100644 --- a/js/hang/src/catalog/video.test.ts +++ b/js/hang/src/catalog/video.test.ts @@ -124,3 +124,16 @@ test("legacy video arrays derive display size from display aspect fields", () => expect(Number(parsed.renditions.video?.displayAspectWidth)).toBe(16); expect(Number(parsed.renditions.video?.displayAspectHeight)).toBe(9); }); + +test("video warmup round trips as optional integer milliseconds", () => { + const base = { codec: "vp8", container: { kind: "legacy" } }; + expect(VideoConfigSchema.parse(base).warmup).toBeUndefined(); + for (const warmup of [0, 80, 1_000]) { + const config = VideoConfigSchema.parse({ ...base, warmup }); + expect(Number(config.warmup)).toBe(warmup); + expect(VideoConfigSchema.parse(JSON.parse(JSON.stringify(config)))).toEqual(config); + } + for (const warmup of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + expect(() => VideoConfigSchema.parse({ ...base, warmup })).toThrow(); + } +}); diff --git a/js/hang/src/catalog/video.ts b/js/hang/src/catalog/video.ts index 1eb1bd072e..570d5a31a9 100644 --- a/js/hang/src/catalog/video.ts +++ b/js/hang/src/catalog/video.ts @@ -74,6 +74,10 @@ export const VideoConfigSchema = z.object({ ), ), + // After a non-continuous join, decode from the group start, present at start plus warmup, + // and join that many milliseconds earlier. + warmup: z.optional(u53Schema), + // How far this rendition's frames reach the transport behind the broadcast's earliest // rendition, in whole milliseconds rounded up. A player holds `delay + jitter` for it and never // subtracts one rendition's `delay` from another's. Absent on the earliest rendition. It only diff --git a/quest/m1/README.md b/quest/m1/README.md index 25c59174d5..26d2ec87fb 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -171,7 +171,6 @@ ladders. - [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust container consumer reports a frame after a subscribe or discontinuity as non-continuous, like JS, for the tune-in and warmup trims - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - a viewer joining at a recovery point drops the leading pictures it cannot decode; continuous viewers keep them - [Watch decode errors](/quest/m1/watch-decode-error.md) - a WebCodecs error ends the subscription and the element reports it -- [Catalog warmup](/quest/m1/catalog-warmup.md) - `warmup` on video and audio renditions, in the catalog and the draft - [Audio warmup](/quest/m1/audio-warmup.md) - a viewer joining an Opus rendition mid-stream never hears the unconverged first 80 ms - [#3021](/quest/m1/3021-moq-gst-anchor-generated-media-timelines-to-wall-clock.md) - moq-gst picks the broadcast wall epoch; a restarted source is a new epoch, not a forward re-anchor - [TS passthrough export](/quest/m1/ts-passthrough-export.md) - `export ts --passthrough` writes the `m2ts` track back byte-identical (less late drops) on a fixed delay diff --git a/quest/m1/audio-warmup.md b/quest/m1/audio-warmup.md index fc23444dad..9160f0e82d 100644 --- a/quest/m1/audio-warmup.md +++ b/quest/m1/audio-warmup.md @@ -28,7 +28,6 @@ frames decode independently and set nothing; HE-AAC is out of scope. ## Required -- [Catalog warmup](/quest/m1/catalog-warmup.md) - the field this reads and writes - [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the signal the trim keys on ## Related diff --git a/quest/m1/catalog-warmup.md b/quest/m1/catalog-warmup.md deleted file mode 100644 index a51fdc4a90..0000000000 --- a/quest/m1/catalog-warmup.md +++ /dev/null @@ -1,32 +0,0 @@ -# [S] Catalog warmup on video and audio renditions - -## Goal - -A video or audio rendition can declare `warmup`, a duration in milliseconds -after a group start during which decoded output is not presentable to a viewer -that did not decode the previous group. Absent means every group start is -presentable at once, which is what every rendition says today. Both catalog -implementations parse and emit it, and the hang draft specifies it. Nothing -honours it yet; the consumer, import, and encoder quests do. - -## Plan - -- `rs/hang/src/catalog/video/mod.rs` `VideoConfig` and - `rs/hang/src/catalog/audio/mod.rs` `AudioConfig` gain - `warmup: Option`, serialized as integer milliseconds exactly like - `jitter` (`DurationMilliSeconds`), with a one-line doc that says what a - consumer does with it: decode from the group start, present nothing stamped - before start plus `warmup` after a non-continuous join, and join that much - further back. Mirror in `js/hang/src/catalog/video.ts` and `audio.ts` with - the `u53Schema` optional used for `jitter`, plus round-trip tests beside the - existing catalog tests in both languages. -- `drafts/draft-lcurley-moq-hang.md`: add a `warmup` field section next to - `jitter` for both rendition types, with the video (refresh cycle) and audio - (Opus pre-roll, 80 ms) examples. Relax the group rule at the "Each moq-lite - group MUST start with a keyframe" text for video only: a video group MUST - start at a random access point, which is a keyframe unless the rendition - declares `warmup`, in which case it is a picture from which decoding - converges within `warmup`. Audio keeps its group rule; there `warmup` is - pre-roll and says nothing about group starts. Run `just drafts check`. -- Keep the field out of every producer and consumer in this quest; the - catalog change lands alone so the dependents can proceed in parallel. diff --git a/quest/m2/intra-refresh/README.md b/quest/m2/intra-refresh/README.md index 3d5055f1a3..17a54f4ae1 100644 --- a/quest/m2/intra-refresh/README.md +++ b/quest/m2/intra-refresh/README.md @@ -51,4 +51,3 @@ Decisions the quests share: - [Audio warmup](/quest/m1/audio-warmup.md) - Opus convergence after a mid-stream join uses the same `warmup` field - [Open-GOP leading pictures](/quest/m1/open-gop-leading-pictures.md) - frames stamped before the group's keyframe are the other tune-in trim -- [Catalog warmup](/quest/m1/catalog-warmup.md) - the generic `warmup` field this line reads, kept in m1 for audio and open-GOP tune-in diff --git a/quest/m2/intra-refresh/consumer-warmup.md b/quest/m2/intra-refresh/consumer-warmup.md index 03ad02019c..536638fcad 100644 --- a/quest/m2/intra-refresh/consumer-warmup.md +++ b/quest/m2/intra-refresh/consumer-warmup.md @@ -60,7 +60,6 @@ replaces both: the rule is timestamp arithmetic on the group start. ## Required -- [Catalog warmup](/quest/m1/catalog-warmup.md) - the field this reads - [Rust non-continuous signal](/quest/m1/rust-continuous.md) - the Rust signal this keys on ## Related diff --git a/quest/m2/intra-refresh/export-sync-flags.md b/quest/m2/intra-refresh/export-sync-flags.md index ff4d597363..8874b8217e 100644 --- a/quest/m2/intra-refresh/export-sync-flags.md +++ b/quest/m2/intra-refresh/export-sync-flags.md @@ -24,5 +24,4 @@ Audio `warmup` is Opus pre-roll and changes no export flag. ## Required -- [Catalog warmup](/quest/m1/catalog-warmup.md) - the field exports read - [Consumer warmup](/quest/m2/intra-refresh/consumer-warmup.md) - the IDR check this reuses diff --git a/quest/m2/intra-refresh/h264-import.md b/quest/m2/intra-refresh/h264-import.md index 33b2fa3fef..ea803af20d 100644 --- a/quest/m2/intra-refresh/h264-import.md +++ b/quest/m2/intra-refresh/h264-import.md @@ -37,7 +37,3 @@ were immediately decodable. length-prefixed path; and a real clip generated with x264 `intra-refresh=1` added as a second round-trip in `test/ts/run.sh` next to the closed-GOP one, asserting the catalog `warmup` and the group count. - -## Required - -- [Catalog warmup](/quest/m1/catalog-warmup.md) - the field import writes diff --git a/rs/hang/src/catalog/audio/mod.rs b/rs/hang/src/catalog/audio/mod.rs index 1320e762fc..5d4b24d14e 100644 --- a/rs/hang/src/catalog/audio/mod.rs +++ b/rs/hang/src/catalog/audio/mod.rs @@ -152,6 +152,12 @@ pub struct AudioConfig { #[serde(default)] pub jitter: Option, + /// After a non-continuous join, decode from the group start, present at start plus warmup, and join that much earlier. + /// Serialized as whole milliseconds, rounded up so presentation never starts early. + #[serde_as(as = "Option")] + #[serde(default)] + pub warmup: Option, + /// How far this rendition's frames reach the transport behind the broadcast's earliest /// rendition, measured at the publisher from each rendition's minimum flush lateness. /// Absent on the earliest rendition and on any rendition the publisher did not measure. @@ -183,6 +189,7 @@ impl AudioConfig { description: None, container: Container::default(), jitter: None, + warmup: None, delay: None, } } @@ -192,6 +199,31 @@ impl AudioConfig { mod test { use super::*; + #[test] + fn warmup_round_trips() { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + assert!(serde_json::to_value(&config).unwrap().get("warmup").is_none()); + for millis in [0, 80, 1_000] { + config.warmup = Some(std::time::Duration::from_millis(millis)); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["warmup"], millis); + let decoded: AudioConfig = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded.warmup, config.warmup); + } + } + + #[test] + fn warmup_rounds_up() { + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + for (nanos, millis) in [(1, 1), (499_999, 1), (333_200_000, 334)] { + config.warmup = Some(std::time::Duration::from_nanos(nanos)); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["warmup"], millis); + let decoded: AudioConfig = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded.warmup, Some(std::time::Duration::from_millis(millis))); + } + } + #[test] fn ranked_orders_by_enabled_then_bitrate_then_rate_then_channels() { fn rendition(sample_rate: u32, channels: u32, bitrate: Option) -> AudioConfig { diff --git a/rs/hang/src/catalog/millis.rs b/rs/hang/src/catalog/millis.rs index b691dd8d52..5f5ab4701f 100644 --- a/rs/hang/src/catalog/millis.rs +++ b/rs/hang/src/catalog/millis.rs @@ -7,18 +7,32 @@ use serde::ser::Error as _; use serde::{Deserialize, Serialize, Serializer}; use serde_with::{DeserializeAs, SerializeAs}; -/// A [`serde_with`] adapter encoding an optional [`Duration`] as whole milliseconds, rounded up. +/// A [`serde_with`] adapter encoding a [`Duration`] as whole milliseconds, rounded up. /// /// Every duration on the wire here is an upper bound a consumer sizes a buffer against, so /// rounding down is the one direction that breaks it. A 44.1 kHz AAC frame is 23.2 ms, and /// truncating advertises 23; anything under a millisecond truncates to 0, which reads as /// "flushed immediately" rather than "a little". /// -/// So `0` is not a value the field carries: a track that flushes each frame immediately omits it. -/// Reading one is how a publisher that truncated says "immediately", so it decodes as absent; -/// writing one is a caller stating something the field can't mean, so it's refused. +/// The optional implementation is for jitter and delay, where zero decodes as absent and +/// cannot be written. Fields such as warmup, where zero is meaningful, use `Option`. pub(crate) struct MillisCeil; +impl SerializeAs for MillisCeil { + fn serialize_as(source: &Duration, serializer: S) -> Result { + let millis = source.as_nanos().div_ceil(1_000_000); + u64::try_from(millis) + .map_err(|_| S::Error::custom("a duration too long to express in milliseconds"))? + .serialize(serializer) + } +} + +impl<'de> DeserializeAs<'de, Duration> for MillisCeil { + fn deserialize_as>(deserializer: D) -> Result { + u64::deserialize(deserializer).map(Duration::from_millis) + } +} + impl SerializeAs> for MillisCeil { fn serialize_as(source: &Option, serializer: S) -> Result { let Some(duration) = source else { @@ -31,10 +45,7 @@ impl SerializeAs> for MillisCeil { )); } - let millis = duration.as_nanos().div_ceil(1_000_000); - u64::try_from(millis) - .map_err(|_| S::Error::custom("a duration too long to express in milliseconds"))? - .serialize(serializer) + Self::serialize_as(duration, serializer) } } diff --git a/rs/hang/src/catalog/root.rs b/rs/hang/src/catalog/root.rs index f8fd16d0be..2602cc365f 100644 --- a/rs/hang/src/catalog/root.rs +++ b/rs/hang/src/catalog/root.rs @@ -363,6 +363,7 @@ mod test { framerate: None, optimize_for_latency: None, container: Container::Legacy, + warmup: None, jitter: Some(std::time::Duration::from_millis(100)), delay: Some(std::time::Duration::from_millis(200)), }, @@ -381,6 +382,7 @@ mod test { enabled: true, description: None, container: Container::Legacy, + warmup: None, jitter: Some(std::time::Duration::from_millis(40)), delay: None, }, diff --git a/rs/hang/src/catalog/video/mod.rs b/rs/hang/src/catalog/video/mod.rs index 7812909760..11e74adc61 100644 --- a/rs/hang/src/catalog/video/mod.rs +++ b/rs/hang/src/catalog/video/mod.rs @@ -259,6 +259,12 @@ pub struct VideoConfig { #[serde(default)] pub jitter: Option, + /// After a non-continuous join, decode from the group start, present at start plus warmup, and join that much earlier. + /// Serialized as whole milliseconds, rounded up so presentation never starts early. + #[serde_as(as = "Option")] + #[serde(default)] + pub warmup: Option, + /// How far this rendition's frames reach the transport behind the broadcast's earliest /// rendition, measured at the publisher from each rendition's minimum flush lateness. /// Absent on the earliest rendition and on any rendition the publisher did not measure. @@ -294,6 +300,7 @@ impl VideoConfig { optimize_for_latency: None, container: Container::default(), jitter: None, + warmup: None, delay: None, } } @@ -305,6 +312,31 @@ mod test { use super::*; + #[test] + fn warmup_round_trips() { + let mut config = VideoConfig::new(VideoCodec::VP8); + assert!(serde_json::to_value(&config).unwrap().get("warmup").is_none()); + for millis in [0, 80, 1_000] { + config.warmup = Some(std::time::Duration::from_millis(millis)); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["warmup"], millis); + let decoded: VideoConfig = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded.warmup, config.warmup); + } + } + + #[test] + fn warmup_rounds_up() { + let mut config = VideoConfig::new(VideoCodec::VP8); + for (nanos, millis) in [(1, 1), (499_999, 1), (333_200_000, 334)] { + config.warmup = Some(std::time::Duration::from_nanos(nanos)); + let encoded = serde_json::to_value(&config).unwrap(); + assert_eq!(encoded["warmup"], millis); + let decoded: VideoConfig = serde_json::from_value(encoded).unwrap(); + assert_eq!(decoded.warmup, Some(std::time::Duration::from_millis(millis))); + } + } + #[test] fn ranked_orders_by_picture_then_bitrate() { fn rendition(size: Option<(u32, u32)>, bitrate: Option) -> VideoConfig {