diff --git a/quest/m1/audio-codecs/README.md b/quest/m1/audio-codecs/README.md index 9b3763d7f1..fdeca61705 100644 --- a/quest/m1/audio-codecs/README.md +++ b/quest/m1/audio-codecs/README.md @@ -43,7 +43,6 @@ its own decode and encode quest so verification stays per host. ## Quests -- [TS export PCE](/quest/m1/audio-codecs/ts-export-pce.md) - a TS export of a PCE-described AAC track writes channel_config 0 and the PCE instead of a count-derived config - [AudioToolbox decode](/quest/m1/audio-codecs/decode-audiotoolbox.md) - macOS and iOS decode HE-AAC, multichannel AAC, and what else the framework offers - [Opus surround](/quest/m1/audio-codecs/opus-surround.md) - mapping family 1 decodes on every host through the multistream decoder - [Encode seam](/quest/m1/audio-codecs/encode-backend.md) - `encode::backend` and `Codec::Aac`, so a native publisher can produce AAC-LC diff --git a/quest/m1/audio-codecs/ts-export-pce.md b/quest/m1/audio-codecs/ts-export-pce.md deleted file mode 100644 index 31e9b76b25..0000000000 --- a/quest/m1/audio-codecs/ts-export-pce.md +++ /dev/null @@ -1,17 +0,0 @@ -# [S] TS export carries an AAC track's program config element - -## Goal - -`moq-mux`'s MPEG-TS export writes an AAC track described by a program config -element (channelConfiguration 0) as ADTS with channel_config 0 and the PCE at -the start of the first raw data block, so a TS round trip keeps the layout. -Today the export derives the ADTS channel_config from the channel count, which -mislabels such a track. - -## Plan - -Take the PCE from the track's AudioSpecificConfig description with the parser -that reads it on import, and write it as ffmpeg does: once, leading the first -raw data block. Tracks with a nonzero channelConfiguration are unchanged. Test -a round trip of the quad fixture from the PCE import (`aac_quad.ts`): import, -export, and import again, asserting the same description and channel count. diff --git a/rs/moq-ffi/src/binary.rs b/rs/moq-ffi/src/binary.rs index 8e343bd2ad..cf9e400094 100644 --- a/rs/moq-ffi/src/binary.rs +++ b/rs/moq-ffi/src/binary.rs @@ -8,6 +8,7 @@ use std::sync::Arc; +use moq_mux::binary::Config; use moq_mux::catalog::hang::Extra; use crate::error::MoqError; @@ -25,9 +26,9 @@ pub struct MoqBinaryConfig { pub mime: Option, } -impl From for moq_mux::binary::Config { +impl From for Config { fn from(config: MoqBinaryConfig) -> Self { - let mut out = moq_mux::binary::Config::default().with_compression(config.compression); + let mut out = Config::default().with_compression(config.compression); if let Some(mime) = config.mime { out = out.with_mime(mime); } @@ -48,7 +49,7 @@ impl MoqBroadcastProducer { let _guard = crate::ffi::enter(); self.with_state(|state| { let track = state.broadcast.create_track(name, None)?; - let producer = state.catalog.binary_snapshot(track, config.into())?; + let producer = state.catalog.binary_snapshot(track, Config::from(config))?; Ok(Arc::new(MoqBinarySnapshotProducer { inner: std::sync::Mutex::new(Some(producer)), })) @@ -66,7 +67,7 @@ impl MoqBroadcastProducer { let _guard = crate::ffi::enter(); self.with_state(|state| { let track = state.broadcast.create_track(name, None)?; - let producer = state.catalog.binary_stream(track, config.into())?; + let producer = state.catalog.binary_stream(track, Config::from(config))?; Ok(Arc::new(MoqBinaryStreamProducer { inner: std::sync::Mutex::new(Some(producer)), })) diff --git a/rs/moq-mux/src/codec/aac/mod.rs b/rs/moq-mux/src/codec/aac/mod.rs index 6d7f7cb67a..5ccb24a110 100644 --- a/rs/moq-mux/src/codec/aac/mod.rs +++ b/rs/moq-mux/src/codec/aac/mod.rs @@ -70,53 +70,10 @@ impl Config { /// that follows, and a reserved one is refused. Any SBR/PS extension bits after the /// core fields are consumed. pub fn parse(buf: &mut T) -> Result { - if buf.remaining() < 2 { - return Err(Error::ConfigTooShort); - } - let mut reader = BitReader::new(buf); - let object_type = read_object_type(&mut reader)?; - - // samplingFrequencyIndex: 4 bits; index 15 means an explicit 24-bit rate follows. - let freq_index = reader.read(4, Error::IncompleteConfig)? as u8; - let sample_rate = if freq_index == 15 { - reader.read(24, Error::ExplicitSampleRateTooShort)? - } else { - *SAMPLE_RATES - .get(freq_index as usize) - .ok_or(Error::UnsupportedSampleRateIndex(freq_index))? - }; - - // channelConfiguration: 4 bits, immediately after the (possibly explicit) rate. - let channel_config = reader.read(4, Error::IncompleteConfig)? as u8; + let (object_type, sample_rate, channel_config) = read_header(&mut reader)?; let channel_count = match channel_config { - 0 => { - // Explicit SBR and PS name their core object type after an extension rate; the - // GASpecificConfig carrying the program config element follows that core type. - let mut core = object_type; - if matches!(object_type, 5 | 29) { - if reader.read(4, Error::IncompleteConfig)? == 15 { - reader.read(24, Error::IncompleteConfig)?; - } - core = read_object_type(&mut reader)?; - if core == 22 { - // extensionChannelConfiguration, only for ER BSAC. - reader.read(4, Error::IncompleteConfig)?; - } - } - if !GENERAL_AUDIO.contains(&core) { - return Err(Error::ProgramConfigUnsupported(core)); - } - - // GASpecificConfig: frameLengthFlag, dependsOnCoreCoder (then a 14-bit - // coreCoderDelay), and extensionFlag precede the element. - reader.read(1, Error::IncompleteConfig)?; - if reader.read(1, Error::IncompleteConfig)? == 1 { - reader.read(14, Error::IncompleteConfig)?; - } - reader.read(1, Error::IncompleteConfig)?; - program_config(&mut reader)? - } + 0 => program_config(&mut reader)?, _ => channel_count_from_config(channel_config)?, }; @@ -178,13 +135,65 @@ impl Config { } } +/// Read an AudioSpecificConfig up to its channels: the audioObjectType, sample rate, and +/// channelConfiguration. For a channelConfiguration of 0 the reader stops at the program config +/// element. +fn read_header(reader: &mut BitReader) -> Result<(u8, u32, u8)> { + if reader.buf.remaining() < 2 { + return Err(Error::ConfigTooShort); + } + let object_type = read_object_type(reader)?; + + // samplingFrequencyIndex: 4 bits; index 15 means an explicit 24-bit rate follows. + let freq_index = reader.read(4, Error::IncompleteConfig)? as u8; + let sample_rate = if freq_index == 15 { + reader.read(24, Error::ExplicitSampleRateTooShort)? + } else { + *SAMPLE_RATES + .get(freq_index as usize) + .ok_or(Error::UnsupportedSampleRateIndex(freq_index))? + }; + + // channelConfiguration: 4 bits, immediately after the (possibly explicit) rate. + let channel_config = reader.read(4, Error::IncompleteConfig)? as u8; + if channel_config != 0 { + return Ok((object_type, sample_rate, channel_config)); + } + + // Explicit SBR and PS name their core object type after an extension rate; the + // GASpecificConfig carrying the program config element follows that core type. + let mut core = object_type; + if matches!(object_type, 5 | 29) { + if reader.read(4, Error::IncompleteConfig)? == 15 { + reader.read(24, Error::IncompleteConfig)?; + } + core = read_object_type(reader)?; + if core == 22 { + // extensionChannelConfiguration, only for ER BSAC. + reader.read(4, Error::IncompleteConfig)?; + } + } + if !GENERAL_AUDIO.contains(&core) { + return Err(Error::ProgramConfigUnsupported(core)); + } + + // GASpecificConfig: frameLengthFlag, dependsOnCoreCoder (then a 14-bit + // coreCoderDelay), and extensionFlag precede the element. + reader.read(1, Error::IncompleteConfig)?; + if reader.read(1, Error::IncompleteConfig)? == 1 { + reader.read(14, Error::IncompleteConfig)?; + } + reader.read(1, Error::IncompleteConfig)?; + Ok((object_type, sample_rate, 0)) +} + /// Build the AudioSpecificConfig for a stream that signals its fields per frame, as ADTS does. /// /// A `channel_config` of 0 means a program config element leads `block`, the stream's first raw -/// data block; it moves into the config, so the config describes the channels on its own. Any -/// other value leaves `block` unread. An element placed anywhere else is refused: reaching past -/// the channel data needs a full Huffman decode. -pub(crate) fn in_band_config(profile: u8, sample_rate: u32, channel_config: u8, block: &[u8]) -> Result { +/// data block; it moves into the config, so the config describes the channels on its own, and +/// `block` advances past it. Any other value leaves `block` untouched. An element placed anywhere +/// else is refused: reaching past the channel data needs a full Huffman decode. +pub(crate) fn in_band_config(profile: u8, sample_rate: u32, channel_config: u8, block: &mut &[u8]) -> Result { let mut out = BitWriter::default(); out.write(5, u32::from(profile & 0x1F)); match SAMPLE_RATES.iter().position(|&rate| rate == sample_rate) { @@ -200,8 +209,8 @@ pub(crate) fn in_band_config(profile: u8, sample_rate: u32, channel_config: u8, // GASpecificConfig: frameLengthFlag, dependsOnCoreCoder, and extensionFlag, all clear. out.write(3, 0); - let mut block = block; - let mut reader = BitReader::new(&mut block); + // The element ends byte aligned to the block, so the reader stops exactly past it. + let mut reader = BitReader::new(block); if reader.read(3, Error::ProgramConfigMissing)? != ID_PCE { return Err(Error::ProgramConfigMissing); } @@ -213,6 +222,27 @@ pub(crate) fn in_band_config(profile: u8, sample_rate: u32, channel_config: u8, Ok(Bytes::from(out.bytes)) } +/// Split an AudioSpecificConfig into the channel signaling of a stream that carries it per frame, +/// as ADTS does; the inverse of [`in_band_config`]. +/// +/// Returns the channelConfiguration and, when that is 0, the program config element framed to +/// lead a raw data block: its element ID, then the element aligned to the block. +pub(crate) fn in_band_channels(asc: &[u8]) -> Result<(u8, Option)> { + let mut asc = asc; + let mut reader = BitReader::new(&mut asc); + let (_, _, channel_config) = read_header(&mut reader)?; + if channel_config != 0 { + return Ok((channel_config, None)); + } + + let mut out = BitWriter::default(); + out.write(3, ID_PCE); + reader.record = Some(out); + program_config(&mut reader)?; + let out = reader.record.take().expect("recording set above"); + Ok((0, Some(Bytes::from(out.bytes)))) +} + /// The raw data block element ID of a program config element (ISO 14496-3 Table 4.85). const ID_PCE: u32 = 5; @@ -597,23 +627,48 @@ mod tests { let mut block = BitWriter::default(); block.write(3, ID_PCE); write_pce(&mut block, &[false, true], &[], &[true], 1); + let pce = block.bytes.clone(); // The channel elements that follow are never read. block.write(8, 0xFF); - let asc = in_band_config(2, 48_000, 0, &block.bytes).unwrap(); + let mut rest = block.bytes.as_slice(); + let asc = in_band_config(2, 48_000, 0, &mut rest).unwrap(); assert_eq!(asc, pce_asc(2, &[false, true], &[], &[true], 1)); assert_eq!(Config::parse(&mut asc.as_ref()).unwrap().channel_count, 6); + assert_eq!(rest, [0xFF], "the element moves out of the block"); + + // And back: the element leads the block again, byte for byte. + assert_eq!(in_band_channels(&asc).unwrap(), (0, Some(Bytes::from(pce)))); + } + + #[test] + fn in_band_channels_of_ffmpeg_program_config_element() { + // Round trip ffmpeg's own element through a raw data block and back into a config. The + // trailing SBR sync extension is not part of the element, so it drops. + let (config, pce) = in_band_channels(&FFMPEG_QUAD_ASC).unwrap(); + assert_eq!(config, 0); + let pce = pce.unwrap(); + let mut block = pce.as_ref(); + let asc = in_band_config(2, 48_000, 0, &mut block).unwrap(); + assert!(block.is_empty(), "the element is all that was framed"); + assert_eq!(asc, FFMPEG_QUAD_ASC[..FFMPEG_QUAD_ASC.len() - 3]); + } + + #[test] + fn in_band_channels_without_a_program_config_element() { + assert_eq!(in_band_channels(&[0x11, 0x90]).unwrap(), (2, None)); + assert_eq!(in_band_channels(&[0x11, 0xE0]).unwrap(), (12, None)); } #[test] fn in_band_config_refuses_a_block_without_a_leading_pce() { // ID_CPE first: any element past the channel data is out of reach. assert!(matches!( - in_band_config(2, 48_000, 0, &[0x20, 0x00]), + in_band_config(2, 48_000, 0, &mut [0x20, 0x00].as_slice()), Err(Error::ProgramConfigMissing) )); assert!(matches!( - in_band_config(2, 48_000, 0, &[]), + in_band_config(2, 48_000, 0, &mut [].as_slice()), Err(Error::ProgramConfigMissing) )); } @@ -621,7 +676,7 @@ mod tests { #[test] fn in_band_config_matches_encode() { // Without a PCE the block is unread and the config is the plain two-byte form. - let asc = in_band_config(2, 44_100, 2, &[]).unwrap(); + let asc = in_band_config(2, 44_100, 2, &mut [].as_slice()).unwrap(); let encoded = Config { profile: 2, sample_rate: 44_100, diff --git a/rs/moq-mux/src/container/ts/adts.rs b/rs/moq-mux/src/container/ts/adts.rs index 0166c05937..ac1aec6823 100644 --- a/rs/moq-mux/src/container/ts/adts.rs +++ b/rs/moq-mux/src/container/ts/adts.rs @@ -63,13 +63,17 @@ impl Header { pub(super) fn write_header( object_type: u8, sample_rate: u32, - channel_count: u32, + channel_config: u8, raw_len: usize, ) -> anyhow::Result<[u8; 7]> { // ADTS `profile` is the 2-bit audioObjectType - 1. let profile = object_type.saturating_sub(1) & 0x03; let freq_index = freq_index_from_rate(sample_rate)?; - let channel_config = channel_config_from_count(channel_count); + // ADTS has 3 bits for it; the higher configurations only fit an AudioSpecificConfig. + anyhow::ensure!( + channel_config < 8, + "channelConfiguration {channel_config} not representable in ADTS" + ); let frame_len = raw_len + 7; anyhow::ensure!(frame_len < (1 << 13), "AAC frame too large for ADTS framing"); @@ -98,7 +102,7 @@ fn freq_index_from_rate(sample_rate: u32) -> anyhow::Result { } /// Map a channel count to an AAC `channel_config` (ISO 14496-3 Table 1.19). -fn channel_config_from_count(channel_count: u32) -> u8 { +pub(super) fn channel_config_from_count(channel_count: u32) -> u8 { match channel_count { 1..=6 => channel_count as u8, 8 => 7, diff --git a/rs/moq-mux/src/container/ts/export.rs b/rs/moq-mux/src/container/ts/export.rs index e1380502ae..c53a7c141f 100644 --- a/rs/moq-mux/src/container/ts/export.rs +++ b/rs/moq-mux/src/container/ts/export.rs @@ -254,10 +254,14 @@ impl Track { enum Kind { /// Video carries its TS stream type (H.264 = 0x1B, H.265 = 0x24). Video(StreamType), + /// AAC, framed as ADTS. A `channel_config` of 0 defers the layout to a program config + /// element, which leads the next raw data block written and is then taken. A catalog update + /// rebuilds the kind and so repeats it once, which a decoder tuning in mid-stream welcomes. Aac { object_type: u8, sample_rate: u32, - channel_count: u32, + channel_config: u8, + program_config: Option, }, /// Opus (private stream_type 0x06). Each frame is one Opus packet, prefixed with /// the Opus-in-TS access-unit control header and announced with the 'Opus' @@ -1341,9 +1345,12 @@ impl Export { if self.span_counters.is_none() { self.span_counters = Some(self.counters.clone()); } - let track = self.tracks.get(name).context("missing track")?; + let track = self.tracks.get_mut(name).context("missing track")?; let pid = track.pid; let kind = track.kind.clone(); + if let Kind::Aac { program_config, .. } = &mut track.kind { + program_config.take(); + } let is_video = matches!(kind, Kind::Video(_)); let timestamp = frame.timestamp; let keyframe = frame.keyframe; @@ -1356,11 +1363,15 @@ impl Export { Kind::Aac { object_type, sample_rate, - channel_count, + channel_config, + program_config, } => { - let header = adts::write_header(*object_type, *sample_rate, *channel_count, frame.payload.len())?; - let mut framed = Vec::with_capacity(7 + frame.payload.len()); + let pce = program_config.as_deref().unwrap_or_default(); + let raw_len = pce.len() + frame.payload.len(); + let header = adts::write_header(*object_type, *sample_rate, *channel_config, raw_len)?; + let mut framed = Vec::with_capacity(header.len() + raw_len); framed.extend_from_slice(&header); + framed.extend_from_slice(pce); framed.extend_from_slice(&frame.payload); Some(framed) } @@ -2086,11 +2097,19 @@ fn video_es_payload(stream_type: StreamType, description: Option<&Bytes>, frame: fn audio_kind(config: &AudioConfig, name: &str) -> anyhow::Result { ensure_raw(&config.container, "audio", name)?; match &config.codec { - AudioCodec::AAC(aac) => Ok(Kind::Aac { - object_type: aac.profile, - sample_rate: config.sample_rate, - channel_count: config.channel_count, - }), + AudioCodec::AAC(aac) => { + // The description names the layout exactly; without one, the count is all there is. + let (channel_config, program_config) = match &config.description { + Some(asc) => crate::codec::aac::in_band_channels(asc)?, + None => (adts::channel_config_from_count(config.channel_count), None), + }; + Ok(Kind::Aac { + object_type: aac.profile, + sample_rate: config.sample_rate, + channel_config, + program_config, + }) + } AudioCodec::Mp2 => Ok(Kind::Mp2 { sample_rate: config.sample_rate, }), diff --git a/rs/moq-mux/src/container/ts/export_test.rs b/rs/moq-mux/src/container/ts/export_test.rs index 19dd6f9fae..00d9294308 100644 --- a/rs/moq-mux/src/container/ts/export_test.rs +++ b/rs/moq-mux/src/container/ts/export_test.rs @@ -1425,6 +1425,65 @@ async fn ac3_roundtrip_byte_exact() { assert_eq!(roundtripped, ingested, "AC-3 frames must survive byte-for-byte"); } +/// The first ADTS frame of the first AAC PES: its header and raw data block. +fn first_adts_frame(ts: &[u8]) -> (super::adts::Header, Vec) { + let mut pes = PesPacketReader::new(TsPacketReader::new(Cursor::new(ts))); + let packet = pes.read_pes_packet().unwrap().expect("an AAC PES"); + let header = super::adts::Header::parse(&packet.data).unwrap(); + (header, packet.data[header.header_len..header.frame_len].to_vec()) +} + +/// ffmpeg's quad AAC fixture has no channelConfiguration, so its layout rides in a program +/// config element. Import moves it into the description and export puts it back: channel_config +/// 0 in ADTS and the element leading the first raw data block, exactly as ffmpeg wrote it. +#[tokio::test(start_paused = true)] +async fn aac_program_config_roundtrip() { + let data = include_bytes!("test_data/aac_quad.ts"); + + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let consumer = broadcast.consume(); + let catalog = crate::catalog::Producer::new(&mut broadcast, crate::catalog::Config::default()).unwrap(); + let mut import = crate::container::ts::Import::new(broadcast, catalog.reserve()); + import.decode(&BytesMut::from(&data[..])).unwrap(); + import.finish().unwrap(); + + let snapshot = catalog.snapshot(); + let (name, audio) = snapshot.audio.renditions.iter().next().expect("an AAC track"); + assert_eq!(audio.channel_count, 4); + let ingested = read_frames(&consumer, name, Kind::Audio).await; + assert!(!ingested.is_empty(), "no AAC frames"); + + let ts = drain(consumer).await; + assert_packet_aligned(&ts); + + let (header, block) = first_adts_frame(&ts); + assert_eq!(header.channel_config, 0, "the layout is not a channelConfiguration"); + assert_eq!( + block, + first_adts_frame(data).1, + "the first raw data block, element and all" + ); + + let mut broadcast2 = moq_net::broadcast::Info::new().produce(); + let consumer2 = broadcast2.consume(); + let catalog2 = crate::catalog::Producer::new(&mut broadcast2, crate::catalog::Config::default()).unwrap(); + let mut import2 = crate::container::ts::Import::new(broadcast2, catalog2.reserve()); + import2.decode(&BytesMut::from(ts.as_ref())).unwrap(); + import2.finish().unwrap(); + + let snapshot2 = catalog2.snapshot(); + let (name2, audio2) = snapshot2 + .audio + .renditions + .iter() + .next() + .expect("round-trip lost the AAC track"); + assert_eq!(audio2.channel_count, 4); + assert_eq!(audio2.description, audio.description); + let roundtripped = read_frames(&consumer2, name2, Kind::Audio).await; + assert_eq!(roundtripped, ingested, "the element is written once, not per frame"); +} + /// The ffmpeg E-AC-3 fixture must survive TS -> MoQ -> TS byte-for-byte in an /// audio-only program; the PMT re-announces ATSC 0x87 with the 'EAC3' /// registration descriptor. diff --git a/rs/moq-mux/src/container/ts/import.rs b/rs/moq-mux/src/container/ts/import.rs index 643069c77d..057d998aa1 100644 --- a/rs/moq-mux/src/container/ts/import.rs +++ b/rs/moq-mux/src/container/ts/import.rs @@ -2132,17 +2132,19 @@ impl AacStream { } }; + let mut block = &data[offset + header.header_len..end]; let import = match &mut self.import { Some(import) => import, None => { // Synthesize the AudioSpecificConfig `description` so out-of-band consumers // (fMP4/MKV export, WebCodecs) can configure the decoder. A channel_config of 0 - // moves the program config element out of this first frame into it. + // moves the program config element out of this first frame into it, as + // ffmpeg's aac_adtstoasc does; the TS export puts it back. let asc = aac::in_band_config( header.object_type, header.sample_rate, header.channel_config, - &data[offset + header.header_len..end], + &mut block, )?; let mut config = aac::config(&asc)?; config.container = self.container.clone(); @@ -2157,7 +2159,7 @@ impl AacStream { } }; - import.decode(&data[offset + header.header_len..end], pts)?; + import.decode(block, pts)?; // Count only completed frames; input gaps and unfinished tails are not a media burst. burst += std::time::Duration::from_nanos((1024_u64 * 1_000_000_000).div_ceil(header.sample_rate as u64)); // The importer accumulates; cut each ADTS frame into its own group (one QUIC stream)