From d44a4bf12d256633c0a0bd5facd13c5ddf781871 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 20:44:55 -0700 Subject: [PATCH 1/2] feat(moq-mux): estimate the catalog framerate A video rendition's framerate is now measured like its bitrate and jitter: frames per second of advance in the highest timestamp, sampled at cuts so reordered and open GOPs measure exactly, snapped to a standard rate and peak-held. A supplied framerate (a VideoHint, or an SPS that fixes the rate) still wins. H.264 now supplies the rate when fixed_frame_rate_flag is set. HEVC supplies it only when its HRD fixes the picture rate, instead of publishing the VUI tick, which is only a ceiling. Co-Authored-By: Claude Opus 5.5 --- rs/moq-mux/src/catalog/estimate.rs | 226 +++++++++++++++++++++++++++- rs/moq-mux/src/catalog/tracks.rs | 12 ++ rs/moq-mux/src/codec/h264/import.rs | 20 +++ rs/moq-mux/src/codec/h264/mod.rs | 44 +++++- rs/moq-mux/src/codec/h265/import.rs | 23 +-- rs/moq-mux/src/codec/h265/mod.rs | 35 ++--- 6 files changed, 325 insertions(+), 35 deletions(-) diff --git a/rs/moq-mux/src/catalog/estimate.rs b/rs/moq-mux/src/catalog/estimate.rs index 9455626547..52830505b8 100644 --- a/rs/moq-mux/src/catalog/estimate.rs +++ b/rs/moq-mux/src/catalog/estimate.rs @@ -4,8 +4,29 @@ use std::time::{Duration, Instant}; use moq_net::Timestamp; -/// The window over which bitrate is averaged before it is reported. -const BITRATE_WINDOW: Duration = Duration::from_secs(1); +/// The window over which bitrate and framerate are averaged before they are reported. +const RATE_WINDOW: Duration = Duration::from_secs(1); + +/// The frame rates a measurement snaps to. Anything else (a variable-rate source's average) +/// publishes nothing, since consumers size clocks from the value (an fMP4 timescale of +/// `framerate * 1000`). +const FRAMERATES: [f64; 11] = [ + 24_000.0 / 1001.0, + 24.0, + 25.0, + 30_000.0 / 1001.0, + 30.0, + 48.0, + 50.0, + 60_000.0 / 1001.0, + 60.0, + 100.0, + 120.0, +]; + +/// How far a measurement may sit from the rate it snaps to: half the 0.1% gap between an NTSC +/// rate and its integer neighbour, so no measurement is ambiguous between them. +const FRAMERATE_TOLERANCE: f64 = 0.0005; const JITTER_WINDOW: Duration = Duration::from_secs(10); /// The catalog fields an [`Estimator`] can measure from the frames fed to it. @@ -22,6 +43,8 @@ pub struct Estimate { pub bitrate: Option, /// The most this track's minimum flush lateness has trailed the broadcast's earliest track. pub delay: Option, + /// The maximum frame rate in frames per second, when it is a standard one. + pub framerate: Option, } impl Estimate { @@ -42,9 +65,15 @@ impl Estimate { self.delay = delay.into(); self } + + /// Set the frame rate in frames per second (or clear it with `None`). + pub fn with_framerate(mut self, framerate: impl Into>) -> Self { + self.framerate = framerate.into(); + self + } } -/// Measures the catalog jitter and bitrate of one track from the frames written to it. +/// Measures the catalog jitter, bitrate, and framerate of one track from the frames written to it. /// /// A [`container::Producer`](crate::container::Producer) created through the catalog owns one, /// feeds it as you write, and publishes the result automatically. Local encoders also call @@ -85,6 +114,7 @@ impl Estimate { pub struct Estimator { jitter: Jitter, bitrate: Bitrate, + framerate: Framerate, /// This track's recent minimum flush lateness. baseline: Window, /// The recent minimum across every track sharing it. A standalone estimator's is its own, so @@ -113,6 +143,7 @@ impl Estimator { pub fn write(&mut self, timestamp: Timestamp, bytes: usize) { let timestamp = nanos(timestamp); self.bitrate.write(timestamp, bytes); + self.framerate.write(timestamp); } /// Measure when an encoder handed a frame to the transport. Only locally encoded frames should @@ -134,6 +165,7 @@ impl Estimator { /// frame) leaves the span open to fold into the next one, so nothing is lost by cutting often. pub fn cut(&mut self, end: Option) { self.bitrate.cut(end.map(nanos)); + self.framerate.cut(); } /// Discard the open bitrate span and the flush baselines, so nothing is measured across a break @@ -143,6 +175,7 @@ impl Estimator { /// pre-pause minimum would otherwise read as a delay until it left the window. pub fn discontinuity(&mut self) { self.bitrate.discontinuity(); + self.framerate.discontinuity(); self.baseline = Window::default(); self.broadcast.discontinuity(); } @@ -168,6 +201,7 @@ impl Estimator { jitter: self.jitter.current(), bitrate: self.bitrate.current(), delay: (!self.delay.is_zero()).then_some(self.delay), + framerate: self.framerate.max, } } } @@ -195,7 +229,7 @@ fn elapsed(start: u128, end: u128) -> Option { /// Tracks the maximum bitrate in bits per second, averaged over whole spans. /// /// A span's bytes are only counted once it is closed, and the average is taken over at least -/// [`BITRATE_WINDOW`] of media so a lone keyframe doesn't spike the reported value. +/// [`RATE_WINDOW`] of media so a lone keyframe doesn't spike the reported value. #[derive(Default)] struct Bitrate { span: Option, @@ -237,7 +271,7 @@ impl Bitrate { self.window_bytes = self.window_bytes.saturating_add(span.bytes); self.window_duration += duration; - if self.window_duration < BITRATE_WINDOW { + if self.window_duration < RATE_WINDOW { return; } @@ -259,6 +293,69 @@ impl Bitrate { } } +/// Tracks the maximum framerate: frames written per second of advance in the highest timestamp, +/// snapped to one of [`FRAMERATES`]. +/// +/// Not per span like [`Bitrate`]: a reordered group spans more than its own frames (an open GOP's +/// leading pictures present before its keyframe), while the highest timestamp advances one period +/// per frame whatever the order. Sampled at cuts, so every window starts and ends at the same point +/// of the group structure. +#[derive(Default)] +struct Framerate { + /// The highest timestamp written, in scale-free nanoseconds. + high: Option, + /// The highest timestamp when the window opened. + open: Option, + /// Frames written since the window opened. + frames: u64, + max: Option, +} + +impl Framerate { + fn write(&mut self, ts: u128) { + self.high = Some(self.high.map_or(ts, |high| high.max(ts))); + self.frames += 1; + } + + fn cut(&mut self) { + let Some(high) = self.high else { + return; + }; + // The first window opens at a cut too, not at the first frame, which may be a join that + // starts partway into the group structure. + let Some(open) = self.open else { + self.open = Some(high); + self.frames = 0; + return; + }; + let Some(duration) = elapsed(open, high).filter(|duration| *duration >= RATE_WINDOW) else { + return; + }; + let measured = self.frames as f64 / duration.as_secs_f64(); + let snapped = FRAMERATES + .into_iter() + .map(|rate| (rate, (measured - rate).abs() / rate)) + .filter(|(_, error)| *error <= FRAMERATE_TOLERANCE) + .min_by(|a, b| a.1.total_cmp(&b.1)) + .map(|(rate, _)| rate); + if let Some(framerate) = snapped + && self.max.is_none_or(|max| framerate > max) + { + self.max = Some(framerate); + } + self.open = Some(high); + self.frames = 0; + } + + /// Time nothing across a break in the timeline. + fn discontinuity(&mut self) { + *self = Self { + max: self.max, + ..Self::default() + }; + } +} + /// An open run of frames: everything written since the last boundary that could be timed. struct Span { /// Scale-free nanoseconds, per [`nanos`]. @@ -562,6 +659,107 @@ mod tests { assert_eq!(estimate.jitter, None); } + /// Write groups `groups` of `order.len()` frames at 25 fps, in decode order with `order` giving + /// each frame's display offset from its keyframe, cutting at every keyframe like + /// `container::Producer::write`. Group 0 starts a second in, so leading pictures stay positive. + fn feed_groups(estimator: &mut Estimator, groups: std::ops::Range, order: &[i64]) { + for group in groups { + let keyframe = 25 + (group * order.len() as u64) as i64; + for (i, &offset) in order.iter().enumerate() { + let ts = micros(((keyframe + offset) as u64) * 40_000); + if i == 0 { + estimator.cut(Some(ts)); + } + estimator.write(ts, 1_000); + } + } + } + + #[test] + fn framerate_waits_for_the_window() { + let mut estimator = Estimator::new(); + // Half a second of 25 fps. + let order: Vec = (0..12).collect(); + feed_groups(&mut estimator, 0..1, &order); + estimator.cut(None); + assert_eq!(estimator.estimate().framerate, None); + + feed_groups(&mut estimator, 1..5, &order); + assert_eq!(estimator.estimate().framerate, Some(25.0)); + } + + /// B-frames in decode order: every group presents a frame below the one written before it. + #[test] + fn framerate_counts_reordered_groups() { + let mut estimator = Estimator::new(); + // IPBB in a 24-frame closed GOP. + let mut order = vec![0]; + for quad in 0..(23 / 3) { + let base = quad * 3; + order.extend([base + 3, base + 1, base + 2]); + } + order.extend(22..24); + assert_eq!(order.len(), 24); + feed_groups(&mut estimator, 0..5, &order); + assert_eq!(estimator.estimate().framerate, Some(25.0)); + } + + /// An open GOP's leading pictures present before their keyframe, so a group spans more than + /// its own frames: timing groups by their span would read 24 frames over 27 periods. + #[test] + fn framerate_counts_leading_pictures() { + let mut estimator = Estimator::new(); + // Keyframe, then three leading B-frames presented before it, then the rest. + let mut order = vec![0, -3, -2, -1]; + order.extend(1..21); + assert_eq!(order.len(), 24); + feed_groups(&mut estimator, 0..5, &order); + assert_eq!(estimator.estimate().framerate, Some(25.0)); + } + + /// 30000/1001 fps on a 90 kHz clock: no period is a whole number of nanoseconds, and the + /// measurement is 0.1% from 30 fps. + #[test] + fn framerate_snaps_to_ntsc() { + let mut estimator = Estimator::new(); + for i in 0..300u64 { + let ts = Timestamp::from_scale(i * 3003, 90_000).unwrap(); + estimator.cut(Some(ts)); + estimator.write(ts, 1_000); + estimator.cut(None); + } + assert_eq!(estimator.estimate().framerate, Some(30_000.0 / 1001.0)); + } + + /// A variable-rate source whose windows average no standard rate publishes nothing. + #[test] + fn framerate_skips_nonstandard_rates() { + let mut estimator = Estimator::new(); + // 27 fps: 37 ms apart, give or take. + for i in 0..100u64 { + let ts = micros(i * 37_037); + estimator.cut(Some(ts)); + estimator.write(ts, 1_000); + } + assert_eq!(estimator.estimate().framerate, None); + } + + /// The highest rate over any window is reported, and a slower one doesn't lower it. + #[test] + fn framerate_reports_the_maximum() { + let mut estimator = Estimator::new(); + let order: Vec = (0..25).collect(); + feed_groups(&mut estimator, 0..3, &order); + assert_eq!(estimator.estimate().framerate, Some(25.0)); + + for i in 0..40u64 { + let ts = micros(4_000_000 + i * 41_667); + estimator.cut(Some(ts)); + estimator.write(ts, 1_000); + } + assert_eq!(estimator.estimate().framerate, Some(25.0), "24 fps doesn't lower it"); + } + /// A break in the timeline discards the open span rather than timing it across the gap, and /// resets the frame spacing so the gap is never mistaken for a frame duration. #[test] @@ -582,4 +780,22 @@ mod tests { ); assert_eq!(estimator.estimate().jitter, None, "the gap is not a frame duration"); } + + /// Frames on either side of a break are never timed across it. + #[test] + fn discontinuity_resets_the_framerate_window() { + let mut estimator = Estimator::new(); + for i in 0..10u64 { + estimator.write(micros(i * 40_000), 1_000); + } + estimator.discontinuity(); + + // Resumed 40 minutes later at 25 fps. Timed across the gap it would read ~0 fps. + for i in 0..60u64 { + let ts = micros(2_400_000_000 + i * 40_000); + estimator.cut(Some(ts)); + estimator.write(ts, 1_000); + } + assert_eq!(estimator.estimate().framerate, Some(25.0)); + } } diff --git a/rs/moq-mux/src/catalog/tracks.rs b/rs/moq-mux/src/catalog/tracks.rs index 2e040a17ba..80e29e2fb9 100644 --- a/rs/moq-mux/src/catalog/tracks.rs +++ b/rs/moq-mux/src/catalog/tracks.rs @@ -296,11 +296,13 @@ impl RenditionConfig for hang::catalog::VideoConfig { .with_jitter(self.jitter) .with_bitrate(self.bitrate) .with_delay(self.delay) + .with_framerate(self.framerate) } fn set_estimate(&mut self, estimate: Estimate) { self.jitter = estimate.jitter; self.bitrate = estimate.bitrate; self.delay = estimate.delay; + self.framerate = estimate.framerate; } } @@ -598,6 +600,9 @@ impl> Rendition { if estimate.delay.is_none() { estimate.delay = detected.delay; } + if estimate.framerate.is_none() { + estimate.framerate = detected.framerate; + } estimate } @@ -920,6 +925,7 @@ mod tests { let config = snapshot.video.renditions.get("v").unwrap(); assert!(config.jitter.is_some(), "absent jitter should be auto-detected"); assert!(config.bitrate.is_some(), "absent bitrate should be auto-detected"); + assert_eq!(config.framerate, Some(25.0), "absent framerate should be auto-detected"); } #[test] @@ -989,6 +995,7 @@ mod tests { let hint = VideoHint { bitrate: Some(456), + framerate: Some(30.0), ..Default::default() }; let mut config = config(None, None); @@ -1000,6 +1007,11 @@ mod tests { let snapshot = catalog.snapshot(); let config = snapshot.video.renditions.get("v").unwrap(); assert_eq!(config.bitrate, Some(456), "a hinted bitrate must not be overwritten"); + assert_eq!( + config.framerate, + Some(30.0), + "a hinted framerate must not be overwritten" + ); assert!(config.jitter.is_some(), "the unhinted jitter should still be detected"); } diff --git a/rs/moq-mux/src/codec/h264/import.rs b/rs/moq-mux/src/codec/h264/import.rs index dc1ed46da5..45787d5521 100644 --- a/rs/moq-mux/src/codec/h264/import.rs +++ b/rs/moq-mux/src/codec/h264/import.rs @@ -273,6 +273,7 @@ fn config_from_avcc(avcc_bytes: &[u8]) -> Result { }); config.coded_width = avcc.coded_width; config.coded_height = avcc.coded_height; + config.framerate = avcc.sps.first().and_then(|sps| super::sps_framerate(sps)); config.description = Some(Bytes::copy_from_slice(avcc_bytes)); Ok(config) } @@ -290,6 +291,7 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { }); config.coded_width = Some(sps.coded_width); config.coded_height = Some(sps.coded_height); + config.framerate = super::sps_framerate(sps_nal); Ok(config) } @@ -325,6 +327,24 @@ mod tests { (track, catalog) } + /// Only a fixed-rate SPS states the frame rate; without `fixed_frame_rate_flag` its tick is a + /// ceiling, left for the catalog estimator to measure. + #[test] + fn config_takes_only_a_fixed_framerate() { + use crate::codec::h264::fixtures; + + let fixed = super::super::build_avcc( + &[Bytes::from_static(fixtures::SPS_IPB)], + &[Bytes::from_static(fixtures::PPS)], + ) + .unwrap(); + assert_eq!(config(&fixed).unwrap().framerate, Some(25.0)); + + let mut inline = vec![0, 0, 0, 1]; + inline.extend_from_slice(fixtures::SPS_IPB_VARIABLE); + assert_eq!(config(&inline).unwrap().framerate, None); + } + /// An avcC initializer resolves a config with the avcC stored as `description`. #[tokio::test(start_paused = true)] async fn initialize_avc1_lands_in_catalog() { diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 4340b7c02b..257f184a4e 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -149,6 +149,31 @@ impl Sps { /// /// `None` when the SPS carries no VUI or no bitstream restriction, or fails to parse. pub(crate) fn sps_reorder(nal: &[u8]) -> Option { + let vui = sps_vui(nal)?; + Some(crate::codec::video::Reorder { + depth: vui.depth?, + period: vui.period, + }) +} + +/// The frame rate an SPS NAL unit fixes with `fixed_frame_rate_flag`. +/// +/// Without the flag the VUI tick only bounds the rate from above, so it is left to measurement. +pub(crate) fn sps_framerate(nal: &[u8]) -> Option { + let (units, scale) = sps_vui(nal)?.period?; + Some(scale as f64 / units as f64) +} + +/// What an SPS VUI declares about picture timing. +struct Vui { + /// One frame's duration as `(units, scale)` when `fixed_frame_rate_flag` is set. + period: Option<(u64, u64)>, + /// `max_num_reorder_frames`, when the bitstream restriction is present. + depth: Option, +} + +/// `None` when the SPS carries no VUI or fails to parse. +fn sps_vui(nal: &[u8]) -> Option { if nal.len() < 4 { return None; } @@ -157,13 +182,13 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { if !sps.vui_parameters_present_flag { return None; } - vui_reorder(&rbsp).ok().flatten() + vui(&rbsp).ok().flatten() } /// Walk an SPS RBSP to its VUI (ITU-T H.264 7.3.2.1.1, E.1.1). `h264_parser` stops at /// `vui_parameters_present_flag` without exposing its position, so this re-reads the fields /// before it. -fn vui_reorder(rbsp: &[u8]) -> h264_parser::Result> { +fn vui(rbsp: &[u8]) -> h264_parser::Result> { use h264_parser::bitreader::BitReader; use h264_parser::eg::{read_se, read_ue}; @@ -271,14 +296,17 @@ fn vui_reorder(rbsp: &[u8]) -> h264_parser::Result Bytes { let mut buf = BytesMut::new(); for nal in nals { diff --git a/rs/moq-mux/src/codec/h265/import.rs b/rs/moq-mux/src/codec/h265/import.rs index a5607b3155..13ff53a150 100644 --- a/rs/moq-mux/src/codec/h265/import.rs +++ b/rs/moq-mux/src/codec/h265/import.rs @@ -278,7 +278,7 @@ fn config_from_sps(sps_nal: &[u8]) -> Result { }); config.coded_width = Some(sps.rbsp.cropped_width() as u32); config.coded_height = Some(sps.rbsp.cropped_height() as u32); - config.framerate = vui_data.framerate; + config.framerate = super::sps_period(&sps.rbsp).map(|(units, scale)| scale as f64 / units as f64); config.display_aspect_width = vui_data.display_ratio_width; config.display_aspect_height = vui_data.display_ratio_height; Ok(config) @@ -303,19 +303,12 @@ fn find_sps(payload: &[u8]) -> Option { #[derive(Default)] struct VuiData { - framerate: Option, display_ratio_width: Option, display_ratio_height: Option, } impl VuiData { fn new(vui: &scuffle_h265::VuiParameters) -> Self { - // FPS = time_scale / num_units_in_tick - let framerate = vui - .vui_timing_info - .as_ref() - .map(|t| t.time_scale.get() as f64 / t.num_units_in_tick.get() as f64); - let (display_ratio_width, display_ratio_height) = match &vui.aspect_ratio_info { // Extended SAR has explicit arbitrary values for width and height. scuffle_h265::AspectRatioInfo::ExtendedSar { sar_width, sar_height } => { @@ -328,7 +321,6 @@ impl VuiData { }; VuiData { - framerate, display_ratio_width, display_ratio_height, } @@ -375,6 +367,19 @@ mod tests { (track, catalog) } + /// Only an SPS whose HRD fixes the picture rate states it; the VUI tick alone is a ceiling, + /// left for the catalog estimator to measure. + #[test] + fn config_takes_only_a_fixed_framerate() { + let mut fixed = vec![0, 0, 0, 1]; + fixed.extend_from_slice(fixtures::SPS_PYRAMID); + assert_eq!(config(&fixed).unwrap().framerate, Some(25.0)); + + let mut ceiling = vec![0, 0, 0, 1]; + ceiling.extend_from_slice(fixtures::SPS); + assert_eq!(config(&ceiling).unwrap().framerate, None); + } + /// An hvcC initializer resolves a config with the hvcC stored as `description`. /// The hvc1 analogue of the avcC test in [`crate::codec::h264`]'s importer. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/codec/h265/mod.rs b/rs/moq-mux/src/codec/h265/mod.rs index ff5d51d49f..d0f420e274 100644 --- a/rs/moq-mux/src/codec/h265/mod.rs +++ b/rs/moq-mux/src/codec/h265/mod.rs @@ -225,28 +225,29 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { let highest = sps.rbsp.sps_max_sub_layers_minus1 as usize; let reorder = &sps.rbsp.sub_layer_ordering_info.sps_max_num_reorder_pics; let depth = *reorder.get(highest).or(reorder.last())?; - let period = sps - .rbsp - .vui_parameters - .as_ref() - .and_then(|vui| vui.vui_timing_info.as_ref()) - .and_then(|timing| { - let sub_layers = &timing.hrd_parameters.as_ref()?.sub_layers; - let elemental = sub_layers - .get(highest) - .or(sub_layers.last())? - .elemental_duration_in_tc_minus1?; - let units = elemental - .checked_add(1)? - .checked_mul(u64::from(timing.num_units_in_tick.get()))?; - Some((units, u64::from(timing.time_scale.get()))) - }); Some(crate::codec::video::Reorder { depth: u32::try_from(depth).ok()?, - period, + period: sps_period(&sps.rbsp), }) } +/// One picture's duration at the highest sub-layer as `(units, scale)`, when the VUI's HRD +/// parameters fix the picture rate within the CVS (ITU-T H.265 E.3.2). Without that the VUI tick +/// only bounds the rate from above. +pub(crate) fn sps_period(sps: &scuffle_h265::SpsRbsp) -> Option<(u64, u64)> { + let highest = sps.sps_max_sub_layers_minus1 as usize; + let timing = sps.vui_parameters.as_ref()?.vui_timing_info.as_ref()?; + let sub_layers = &timing.hrd_parameters.as_ref()?.sub_layers; + let elemental = sub_layers + .get(highest) + .or(sub_layers.last())? + .elemental_duration_in_tc_minus1?; + let units = elemental + .checked_add(1)? + .checked_mul(u64::from(timing.num_units_in_tick.get()))?; + Some((units, u64::from(timing.time_scale.get()))) +} + /// Annex-B → length-prefixed transmuxer; the H.265 analogue of /// [`crate::codec::h264::Avc1`]. /// From 471f146cd817c9998ac8bf4c959c2f2fd540902d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 21:46:02 -0700 Subject: [PATCH 2/2] fix(moq-mux): read an H.264 framerate only from a frame-only SPS fixed_frame_rate_flag says the tick is constant, not that a frame is two ticks. A field-capable SPS leaves pic_struct to the slice header, and E.2.1 Table E-6 makes a tick a field or a frame depending on it, so an interlaced stream publishes twice its real rate. A supplied value wins in Rendition::resolved, so the estimator could never correct it. sps_reorder keeps the period: it is a buffer hint, not a published rate. HEVC needs no guard, since it codes interlaced content as frames and the HRD's elemental_duration_in_tc_minus1 is a picture duration either way. SPS_FIELD is an x264 interlaced encode with fixed_frame_rate_flag forced on, so it carries the same 50 Hz tick as the progressive fixtures. It reads Some(25.0) without the guard. Co-Authored-By: Space Bunny Free --- rs/moq-mux/src/codec/h264/mod.rs | 37 +++++++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/rs/moq-mux/src/codec/h264/mod.rs b/rs/moq-mux/src/codec/h264/mod.rs index 257f184a4e..bddbe19963 100644 --- a/rs/moq-mux/src/codec/h264/mod.rs +++ b/rs/moq-mux/src/codec/h264/mod.rs @@ -159,13 +159,21 @@ pub(crate) fn sps_reorder(nal: &[u8]) -> Option { /// The frame rate an SPS NAL unit fixes with `fixed_frame_rate_flag`. /// /// Without the flag the VUI tick only bounds the rate from above, so it is left to measurement. +/// A field-capable SPS is left out too: `pic_struct` in the slice header then decides whether a +/// tick counts a field or a frame (ITU-T H.264 E.2.1, Table E-6), and only measurement settles it. pub(crate) fn sps_framerate(nal: &[u8]) -> Option { - let (units, scale) = sps_vui(nal)?.period?; + let vui = sps_vui(nal)?; + if !vui.frame_only { + return None; + } + let (units, scale) = vui.period?; Some(scale as f64 / units as f64) } /// What an SPS VUI declares about picture timing. struct Vui { + /// `frame_mbs_only_flag`: every picture is a frame, so two ticks are one frame. + frame_only: bool, /// One frame's duration as `(units, scale)` when `fixed_frame_rate_flag` is set. period: Option<(u64, u64)>, /// `max_num_reorder_frames`, when the bitstream restriction is present. @@ -244,7 +252,8 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { r.skip_bits(1)?; // gaps_in_frame_num_value_allowed_flag read_ue(&mut r)?; // pic_width_in_mbs_minus1 read_ue(&mut r)?; // pic_height_in_map_units_minus1 - if !r.read_flag()? { + let frame_only = r.read_flag()?; // frame_mbs_only_flag + if !frame_only { r.skip_bits(1)?; // mb_adaptive_frame_field_flag } r.skip_bits(1)?; // direct_8x8_inference_flag @@ -296,7 +305,11 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } r.skip_bits(1)?; // pic_struct_present_flag if !r.read_flag()? { - return Ok(Some(Vui { period, depth: None })); + return Ok(Some(Vui { + frame_only, + period, + depth: None, + })); } r.skip_bits(1)?; // motion_vectors_over_pic_boundaries_flag for _ in 0..4 { @@ -304,6 +317,7 @@ fn vui(rbsp: &[u8]) -> h264_parser::Result> { } let depth = read_ue(&mut r)?; // max_num_reorder_frames Ok(Some(Vui { + frame_only, period, depth: Some(depth), })) @@ -672,6 +686,12 @@ pub(crate) mod fixtures { 0x67, 0x64, 0x00, 0x0a, 0xac, 0xe4, 0x10, 0x9b, 0x01, 0x10, 0x00, 0x00, 0x03, 0x00, 0x10, 0x00, 0x00, 0x03, 0x03, 0x20, 0xf1, 0x22, 0x51, 0x20, ]; + /// `interlaced=1:tff=1:bframes=0` at 320x240, with `fixed_frame_rate_flag` forced on: the same + /// 50 Hz tick, but `frame_mbs_only_flag` 0 leaves `pic_struct` to the slice header. + pub(crate) const SPS_FIELD: &[u8] = &[ + 0x67, 0xf4, 0x00, 0x15, 0x91, 0x9d, 0x02, 0x82, 0x1f, 0x89, 0xc0, 0x44, 0x00, 0x00, 0x03, 0x00, 0x04, 0x00, + 0x00, 0x03, 0x00, 0xca, 0x7c, 0x50, 0xaa, 0x80, + ]; pub(crate) const PPS: &[u8] = &[0x68, 0xeb, 0xe3, 0xcb, 0x22, 0xc0]; } @@ -704,6 +724,15 @@ mod tests { sps_reorder(fixtures::SPS_IPB_VARIABLE), Some(Reorder { depth: 1, period: None }) ); + // The field-capable SPS still declares its depth and tick; only the frame rate needs + // `pic_struct` to read it. + assert_eq!( + sps_reorder(fixtures::SPS_FIELD), + Some(Reorder { + depth: 0, + period: Some((2, 50)) + }) + ); // Baseline with no VUI declares nothing. assert_eq!(sps_reorder(&[0x67, 0x42, 0xc0, 0x1f, 0xde]), None); // A VUI cut short is no declaration rather than a wrong one. @@ -715,6 +744,8 @@ mod tests { assert_eq!(sps_framerate(fixtures::SPS_IPB), Some(25.0)); // The same 50 Hz tick without `fixed_frame_rate_flag` is only a ceiling. assert_eq!(sps_framerate(fixtures::SPS_IPB_VARIABLE), None); + // A field-capable SPS makes the tick a field or a frame depending on `pic_struct`. + assert_eq!(sps_framerate(fixtures::SPS_FIELD), None); assert_eq!(sps_framerate(&[0x67, 0x42, 0xc0, 0x1f, 0xde]), None); }