diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 830328b03e..84a4c4cc26 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -171,6 +171,10 @@ track set is fixed: a rendition that leaves and returns with the same codec configuration is written under its original track, while a new rendition, a changed configuration, or a return that replays media already written ends the export with an error naming it. Restart the export to pick up a new rendition. +An Opus rendition declared without its OpusHead gets a guessed pre-skip. A +head that arrives later with the same channel count, decode rate, and gain is +accepted even if its pre-skip and input rate differ. An init already written +keeps the guess, and later heads must match the first one. ## Play diff --git a/doc/lib/rs/moq-mux.md b/doc/lib/rs/moq-mux.md index e8dc5b3faa..249e949b64 100644 --- a/doc/lib/rs/moq-mux.md +++ b/doc/lib/rs/moq-mux.md @@ -49,7 +49,10 @@ then fixed. A rendition that returns with the same sample entry reuses its track id; `fmp4::Error::TrackAdded`, `TrackChanged`, and `TrackRewound` end the export for a new rendition, a changed sample entry, or a replay of media already written, and `TrackUndescribed` names a track that never delivered its codec -configuration. +configuration. An Opus entry synthesized without a catalog `description` guesses +its pre-skip and input sample rate, so a later OpusHead that agrees on everything +else settles it instead of changing it. An OpusHead whose channel count +contradicts its catalog entry fails with `fmp4::Error::OpusChannelCount`. Each catalog track constructor returns one `container::Producer` that owns the media stream and its catalog entry. `set` publishes or replaces its config, diff --git a/rs/moq-mux/src/container/fmp4/export.rs b/rs/moq-mux/src/container/fmp4/export.rs index 079c6a166f..4f5e2b3d58 100644 --- a/rs/moq-mux/src/container/fmp4/export.rs +++ b/rs/moq-mux/src/container/fmp4/export.rs @@ -181,6 +181,9 @@ struct Declaration { trex: mp4_atom::Trex, /// The source init's ftyp, for a CMAF rendition. ftyp: Option, + /// An Opus sample entry synthesized without the catalog's OpusHead, so its pre-skip + /// and input sample rate are guesses a later description may correct. + provisional: bool, } struct Fmp4Track { @@ -190,7 +193,7 @@ struct Fmp4Track { config: Rendition, /// The `trak` from the first time the track could be described. A later description - /// must keep its sample entry. + /// must keep its sample entry, except to settle a provisional one. declaration: Option, /// The current subscription has been checked against `declaration`. Until then its @@ -301,11 +304,19 @@ impl Fmp4Track { let Some(declaration) = self.declaration()? else { return Ok(()); }; - match &self.declaration { - Some(declared) if sample_entry(&declared.trak) != sample_entry(&declaration.trak) => { - return Err(Error::TrackChanged(name.to_string()).into()); + match &mut self.declaration { + Some(declared) => { + let guessed = declared.provisional || declaration.provisional; + if sample_entry(&declared.trak, guessed) != sample_entry(&declaration.trak, guessed) { + return Err(Error::TrackChanged(name.to_string()).into()); + } + // The OpusHead settles a guessed entry. An init already written keeps its + // guessed pre-skip: a few milliseconds of untrimmed encoder delay cost less + // than restarting the file mid-stream. + if declared.provisional && !declaration.provisional { + *declared = declaration; + } } - Some(_) => {} None => self.declaration = Some(declaration), } self.described = true; @@ -346,7 +357,9 @@ impl Fmp4Track { Container::Cmaf { init, .. } => init, Container::Legacy | Container::Loc => { let trak = super::synthesize_audio_trak(self.track_id, self.timescale, config)?; - return Ok(Some(Declaration::synthesized(trak))); + let mut declaration = Declaration::synthesized(trak); + declaration.provisional = matches!(config.codec, AudioCodec::Opus) && config.description.is_none(); + return Ok(Some(declaration)); } Container::Unknown(unknown) => return Err(crate::Error::unsupported_container(unknown)), }, @@ -365,7 +378,12 @@ impl Fmp4Track { // init for the same rendition may not share. trak.mdia.mdhd.timescale = super::mdhd_timescale(self.timescale)?; let trex = trexs.pop().unwrap_or_else(|| default_trex(self.track_id)); - Ok(Some(Declaration { trak, trex, ftyp })) + Ok(Some(Declaration { + trak, + trex, + ftyp, + provisional: false, + })) } } @@ -375,19 +393,30 @@ impl Declaration { trex: default_trex(trak.tkhd.track_id), trak, ftyp: None, + provisional: false, } } } /// The `stsd` a returning or reconfigured rendition must keep, without the AAC bitrate -/// hints a catalog's `bitrate` churns: decoders ignore them. -fn sample_entry(trak: &mp4_atom::Trak) -> mp4_atom::Stsd { +/// hints a catalog's `bitrate` churns: decoders ignore them. With `guessed`, one side's +/// Opus pre-skip and input rate (metadata) were made up without an OpusHead, so only the +/// rest must agree. Its gain was not: a wrong gain changes every sample, so a nonzero one +/// fails. +fn sample_entry(trak: &mp4_atom::Trak, guessed: bool) -> mp4_atom::Stsd { let mut stsd = trak.mdia.minf.stbl.stsd.clone(); for codec in &mut stsd.codecs { - if let mp4_atom::Codec::Mp4a(mp4a) = codec { - mp4a.esds.es_desc.dec_config.max_bitrate = 0; - mp4a.esds.es_desc.dec_config.avg_bitrate = 0; - mp4a.btrt = None; + match codec { + mp4_atom::Codec::Mp4a(mp4a) => { + mp4a.esds.es_desc.dec_config.max_bitrate = 0; + mp4a.esds.es_desc.dec_config.avg_bitrate = 0; + mp4a.btrt = None; + } + mp4_atom::Codec::Opus(opus) if guessed => { + opus.dops.pre_skip = 0; + opus.dops.input_sample_rate = 0; + } + _ => {} } } stsd diff --git a/rs/moq-mux/src/container/fmp4/export_test.rs b/rs/moq-mux/src/container/fmp4/export_test.rs index c544d13f34..34b91435b5 100644 --- a/rs/moq-mux/src/container/fmp4/export_test.rs +++ b/rs/moq-mux/src/container/fmp4/export_test.rs @@ -873,6 +873,19 @@ fn synthesize_opus_trak_preserves_pre_skip() { assert_eq!(opus.dops.pre_skip, 312); } +/// An OpusHead that contradicts the catalog's channel count is refused rather than +/// written beside it, so the sample entry and dOps can't disagree. +#[test] +fn synthesize_opus_trak_refuses_a_contradicting_head() { + use hang::catalog::{AudioCodec, AudioConfig}; + + let mut config = AudioConfig::new(AudioCodec::Opus, 48_000, 2); + config.description = Some(crate::codec::opus::Config::new(48_000, 1).encode().unwrap()); + + let err = super::synthesize_audio_trak(1, 48_000, &config).unwrap_err(); + assert!(matches!(err, super::Error::OpusChannelCount { catalog: 2, head: 1 })); +} + /// dOps has nowhere to put a channel mapping table, so a family 1 head is refused rather /// than written as family 0 without it. #[test] @@ -1460,6 +1473,154 @@ async fn a_changed_sample_entry_fails() { ); } +/// An OpusHead for `channels` channels with `pre_skip`, as a catalog description. Its +/// input rate is 44.1 kHz: metadata the 48 kHz guess has no way to know. +fn opus_head(channels: u32, pre_skip: u16) -> Bytes { + crate::codec::opus::Config::new(44_100, channels) + .with_pre_skip(pre_skip) + .encode() + .unwrap() +} + +/// Set the catalog description of the audio rendition `name`. +fn describe_audio(live: &mut Live, name: &str, description: Bytes) { + live.catalog + .modify() + .unwrap() + .audio + .renditions + .get_mut(name) + .unwrap() + .description = Some(description); +} + +/// The pre-skip of the Opus sample entry in `init`. +fn init_pre_skip(init: &Bytes) -> u16 { + let mut cursor = Cursor::new(init.as_ref()); + while let Some(atom) = mp4_atom::Any::decode_maybe(&mut cursor).expect("decode init") { + let mp4_atom::Any::Moov(moov) = atom else { + continue; + }; + for trak in &moov.trak { + if let mp4_atom::Codec::Opus(opus) = &trak.mdia.minf.stbl.stsd.codecs[0] { + return opus.dops.pre_skip; + } + } + } + panic!("no Opus trak in the init"); +} + +/// A browser declares Opus before its encoder reports the OpusHead. The head that follows +/// settles the guessed pre-skip rather than failing the export, and binds it from then on. +#[tokio::test(start_paused = true)] +async fn a_later_opus_head_settles_the_guess() { + let (mut live, mut audio) = live_av(); + let name = audio.name().to_string(); + live.track.write(video_frame(0, true)).unwrap(); + write_audio(&mut audio, 0, 6); + let mut exporter = crate::container::fmp4::Export::new(live.source(), live.catalog_stream().await); + let init = chunk_now(&mut exporter).await.init().expect("init"); + assert_eq!(init_pre_skip(&init), 0, "the init carries the guess"); + drain_now(&mut exporter).await; + + describe_audio(&mut live, &name, opus_head(2, 312)); + write_audio(&mut audio, 120_000, 5); + assert!(!drain_now(&mut exporter).await.is_empty(), "the audio keeps flowing"); + + describe_audio(&mut live, &name, opus_head(2, 120)); + let err = error_now(&mut exporter).await; + assert!( + matches!(&err, crate::Error::Cmaf(crate::container::fmp4::Error::TrackChanged(changed)) if *changed == name), + "{err:?}" + ); +} + +/// An OpusHead that settles the guess before the init is written lands in the init. +#[tokio::test(start_paused = true)] +async fn an_opus_head_before_the_init_is_written() { + let (mut live, mut audio) = live_av(); + let name = audio.name().to_string(); + write_audio(&mut audio, 0, 6); + let mut exporter = crate::container::fmp4::Export::new(live.source(), live.catalog_stream().await); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(1), exporter.next_chunk()) + .await + .is_err(), + "the init waits for the video" + ); + + describe_audio(&mut live, &name, opus_head(2, 312)); + live.track.write(video_frame(0, true)).unwrap(); + let init = chunk_now(&mut exporter).await.init().expect("init"); + assert_eq!(init_pre_skip(&init), 312); +} + +/// An OpusHead with another channel count or gain is a different track, not a settled +/// guess: the gain changes every sample, so a written init can't keep a guessed one. +#[tokio::test(start_paused = true)] +async fn an_opus_head_that_disagrees_fails() { + an_opus_head_fails(1, opus_head(1, 312)).await; + an_opus_head_fails(2, opus_head_gain(-1536)).await; +} + +/// A browser that reconnects returns without its head, then sends the same head again. +/// The settled entry holds through both. +#[tokio::test(start_paused = true)] +async fn a_settled_opus_head_survives_a_return() { + let (mut live, mut audio) = live_av(); + let name = audio.name().to_string(); + live.track.write(video_frame(0, true)).unwrap(); + write_audio(&mut audio, 0, 6); + let mut exporter = crate::container::fmp4::Export::new(live.source(), live.catalog_stream().await); + chunk_now(&mut exporter).await.init().expect("init"); + describe_audio(&mut live, &name, opus_head(2, 312)); + drain_now(&mut exporter).await; + + let mut config = live.catalog.modify().unwrap().audio.renditions.remove(&name).unwrap(); + drain_now(&mut exporter).await; + write_audio(&mut audio, 1_000_000, 5); + config.description = None; + live.catalog + .modify() + .unwrap() + .audio + .renditions + .insert(name.clone(), config); + drain_now(&mut exporter).await; + describe_audio(&mut live, &name, opus_head(2, 312)); + write_audio(&mut audio, 1_100_000, 5); + assert!(!drain_now(&mut exporter).await.is_empty(), "the audio keeps flowing"); +} + +/// An OpusHead for stereo with `output_gain`. +fn opus_head_gain(output_gain: i16) -> Bytes { + let mut head = crate::codec::opus::Config::new(44_100, 2).with_pre_skip(312); + head.output_gain = output_gain; + head.encode().unwrap() +} + +/// Declare stereo Opus without a head, write the init, then fail on `head` for `channels`. +async fn an_opus_head_fails(channels: u32, head: Bytes) { + let (mut live, mut audio) = live_av(); + let name = audio.name().to_string(); + live.track.write(video_frame(0, true)).unwrap(); + write_audio(&mut audio, 0, 6); + let mut exporter = crate::container::fmp4::Export::new(live.source(), live.catalog_stream().await); + chunk_now(&mut exporter).await.init().expect("init"); + drain_now(&mut exporter).await; + + let mut catalog = live.catalog.modify().unwrap(); + let config = catalog.audio.renditions.get_mut(&name).unwrap(); + config.channel_count = channels; + config.description = Some(head); + drop(catalog); + let err = error_now(&mut exporter).await; + assert!( + matches!(&err, crate::Error::Cmaf(crate::container::fmp4::Error::TrackChanged(changed)) if *changed == name), + "{err:?}" + ); +} + /// An AAC catalog's `bitrate` lands in the esds as a hint decoders ignore, so changing it /// keeps the track. #[tokio::test(start_paused = true)] diff --git a/rs/moq-mux/src/container/fmp4/mod.rs b/rs/moq-mux/src/container/fmp4/mod.rs index 2403ef3008..5fee8e32f8 100644 --- a/rs/moq-mux/src/container/fmp4/mod.rs +++ b/rs/moq-mux/src/container/fmp4/mod.rs @@ -155,6 +155,10 @@ pub enum Error { #[error("audio codec {0} needs a description (AudioSpecificConfig) to synthesize a CMAF init")] MissingAudioDescription(String), + /// An Opus catalog entry whose OpusHead contradicts its channel count. + #[error("Opus head has {head} channels but the catalog declares {catalog}")] + OpusChannelCount { catalog: u32, head: u32 }, + #[error("multi-sample fragment has a non-final sample with no duration; DTS is unrecoverable")] MissingSampleDuration, @@ -738,6 +742,12 @@ pub(crate) fn synthesize_audio_trak(track_id: u32, timescale: u64, config: &Audi if head.mapping_family != 0 { return Err(crate::codec::opus::Error::UnsupportedMappingFamily(head.mapping_family).into()); } + if head.channel_count != config.channel_count { + return Err(Error::OpusChannelCount { + catalog: config.channel_count, + head: head.channel_count, + }); + } head } None => crate::codec::opus::Config::new(config.sample_rate, config.channel_count),