From f28a8b67339c5ba73081d8af3d47ce850739c33e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 09:26:06 -0700 Subject: [PATCH 1/6] chore: claim MSF captions quest Co-Authored-By: GPT-6 From 0d6205a1c5749293f72ae2983dcfb26e116a4045 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 10:04:42 -0700 Subject: [PATCH 2/6] feat(msf): preserve text tracks across catalog conversion Co-Authored-By: GPT-6 --- js/msf/src/catalog.test.ts | 13 ++++++ js/msf/src/catalog.ts | 4 ++ js/watch/src/msf.test.ts | 21 ++++++++++ js/watch/src/msf.ts | 23 +++++++++++ quest/m1/README.md | 1 - quest/m1/captions-import.md | 5 --- quest/m1/captions-msf.md | 37 ----------------- rs/moq-msf/src/lib.rs | 14 +++++++ rs/moq-mux/src/catalog/msf/consumer.rs | 56 ++++++++++++++++++++++++-- rs/moq-mux/src/catalog/msf/mod.rs | 6 +++ rs/moq-mux/src/catalog/producer.rs | 49 ++++++++++++++++++++++ 11 files changed, 182 insertions(+), 47 deletions(-) delete mode 100644 quest/m1/captions-msf.md diff --git a/js/msf/src/catalog.test.ts b/js/msf/src/catalog.test.ts index 7501ed5f5b..7ce0cd0651 100644 --- a/js/msf/src/catalog.test.ts +++ b/js/msf/src/catalog.test.ts @@ -250,3 +250,16 @@ test("round-trips a __proto__ extension member as data", () => { expect(Object.keys(wire)).toContain("__proto__"); expect(wire.ok).toBe(2); }); + +test("caption role, label, language and codec survive the wire", () => { + const track = { + name: "captions.en", + packaging: "loc", + isLive: true, + role: "caption", + codec: "ttml", + lang: "en", + label: "English SDH", + }; + expect(decode(encode({ tracks: [track] })).tracks).toEqual([track]); +}); diff --git a/js/msf/src/catalog.ts b/js/msf/src/catalog.ts index b3b3bb9224..ab67e98189 100644 --- a/js/msf/src/catalog.ts +++ b/js/msf/src/catalog.ts @@ -28,6 +28,10 @@ const trackShape = { // tracks, so accept its absence rather than reject the whole catalog. isLive: z.optional(z.boolean()), role: z.optional(RoleSchema), + /** A human-readable label for the track. */ + label: z.optional(z.string()), + /** The BCP-47 language tag of the track. */ + lang: z.optional(z.string()), codec: z.optional(z.string()), width: z.optional(z.number()), height: z.optional(z.number()), diff --git a/js/watch/src/msf.test.ts b/js/watch/src/msf.test.ts index 0b8ab50cda..83f3af5dfe 100644 --- a/js/watch/src/msf.test.ts +++ b/js/watch/src/msf.test.ts @@ -118,3 +118,24 @@ test("drops a rendition whose packaging is unknown", () => { expect(toHang(catalog).video?.renditions.video).toBeUndefined(); }); + +test("converts timed text without guessing its format", () => { + for (const codec of ["vtt", "ttml", "utf8", "future-text"]) { + const track = { name: "subs", packaging: "loc", role: "caption", codec, lang: "en", label: "SDH" }; + const text = toHang({ tracks: [track] }).text?.renditions.subs; + expect(text?.format).toBe(codec); + expect(text?.role).toBe("caption"); + expect(text?.lang).toBe("en"); + expect(text?.label).toBe("SDH"); + expect(text?.container).toEqual({ kind: "loc" }); + } +}); + +test("custom roles map only when the codec identifies text", () => { + for (const role of ["commentary", "audiodescription", "signlanguage"]) { + const catalog = toHang({ tracks: [{ name: "track", packaging: "legacy", role, codec: "opus" }] }); + expect(catalog.text).toBeUndefined(); + } + const text = toHang({ tracks: [{ name: "text", packaging: "legacy", role: "commentary", codec: "utf8" }] }); + expect(text.text?.renditions.text?.role).toBe("commentary"); +}); diff --git a/js/watch/src/msf.ts b/js/watch/src/msf.ts index d084293f3c..50dacf7421 100644 --- a/js/watch/src/msf.ts +++ b/js/watch/src/msf.ts @@ -100,6 +100,7 @@ function toAudioConfig(track: Msf.Track): Catalog.AudioConfig | undefined { export function toHang(msf: Msf.Catalog): Catalog.Root { const videoRenditions: Record = {}; const audioRenditions: Record = {}; + const textRenditions: Record = {}; for (const track of msf.tracks) { if (track.role === "video") { @@ -108,6 +109,26 @@ export function toHang(msf: Msf.Catalog): Catalog.Root { } else if (track.role === "audio") { const config = toAudioConfig(track); if (config) audioRenditions[track.name] = config; + } else if ( + track.role === "caption" || + track.role === "subtitle" || + (track.role && + track.role !== "audiodescription" && + track.role !== "signlanguage" && + ["vtt", "ttml", "utf8"].includes(track.codec ?? "")) + ) { + if (!track.codec) throw new Error(`MSF text track ${track.name} missing codec`); + const info = toContainer(track); + if (!info) throw new Error(`MSF text track ${track.name} has unsupported packaging`); + textRenditions[track.name] = { + format: track.codec, + role: track.role, + lang: track.lang, + label: track.label, + container: info.container, + jitter: track.jitter != null ? delayMillis(track.jitter) : undefined, + delay: track.delay != null ? delayMillis(track.delay) : undefined, + }; } } @@ -121,5 +142,7 @@ export function toHang(msf: Msf.Catalog): Catalog.Root { root.audio = { renditions: audioRenditions }; } + if (Object.keys(textRenditions).length > 0) root.text = { renditions: textRenditions }; + return root; } diff --git a/quest/m1/README.md b/quest/m1/README.md index 25c59174d5..dff3de6b27 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -163,7 +163,6 @@ ladders. - [Time stretch](/quest/m1/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent - [Native audio quality](/quest/m1/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device - [Caption import](/quest/m1/captions-import.md) - fMP4 and MKV subtitle tracks import as text renditions instead of erroring or being dropped -- [MSF caption roles](/quest/m1/captions-msf.md) - an MSF caption or subtitle track survives conversion to a hang catalog - [Fail loud on dropped timed metadata](/quest/m1/drop-loud.md) - FLV script tags and fMP4 emsg boxes are counted and warned about instead of silently dropped, until real carriage lands - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay` diff --git a/quest/m1/captions-import.md b/quest/m1/captions-import.md index 5000c051f9..87c1f72765 100644 --- a/quest/m1/captions-import.md +++ b/quest/m1/captions-import.md @@ -29,8 +29,3 @@ for a cue that may be minutes away. Cover a file with subtitles and no supported text format, a file with several subtitle languages, and a file whose subtitle track is empty. - -## Related - -- [MSF caption roles](/quest/m1/captions-msf.md) - the catalog half of the same - gap diff --git a/quest/m1/captions-msf.md b/quest/m1/captions-msf.md deleted file mode 100644 index 8cb2ae850c..0000000000 --- a/quest/m1/captions-msf.md +++ /dev/null @@ -1,37 +0,0 @@ -# [M] MSF caption roles - -## Goal - -An MSF catalog's caption and subtitle tracks survive conversion into a hang -catalog. Today they are dropped with a warning, so a -round trip through MSF silently loses every caption. - -## Plan - -`from_msf` skips any track "with no role, with an unsupported role (caption, -subtitle, sign language, audio description, custom roles)". Now that the hang -`text` section exists with a `TextRole` borrowing the MSF role names, two of -those map straight across and stop being unsupported: `TextRole` has only -`Subtitle`, `Caption`, and `Unknown(String)`, which preserves any other role -verbatim. - -Map the roles, and derive the rest of `TextConfig` from what MSF carries: -language onto `lang`, the display name onto `label`, and the packaging onto -`container` the way video and audio already do. Pick the cue `format` from the -track's codec rather than defaulting blindly, since an MSF caption track can -carry more than WebVTT and guessing wrong produces a track a player renders as -garbage. - -Prove the round trip in both directions, so an MSF catalog converted to hang -and back keeps its caption tracks, roles, and languages. Audio description is -deliberately left unmapped: it is an audio rendition with a role, not timed -text, and mapping it into `text` would be wrong. Sign language is likewise a -video rendition, so it stays unmapped too. A custom role maps to -`TextRole::Unknown` only when the track's codec is a text format. - -Per cross-package sync, mirror any schema movement in `js/msf`. - -## Related - -- [Caption import](/quest/m1/captions-import.md) - the container half of the - same gap diff --git a/rs/moq-msf/src/lib.rs b/rs/moq-msf/src/lib.rs index 9f7cf1fc31..4fc29eb2e6 100644 --- a/rs/moq-msf/src/lib.rs +++ b/rs/moq-msf/src/lib.rs @@ -123,6 +123,12 @@ pub struct Track { /// Content role. pub role: Option, + /// A human-readable label for the track. + pub label: Option, + + /// The BCP-47 language tag of the track. + pub lang: Option, + /// WebCodecs codec string. pub codec: Option, @@ -469,6 +475,8 @@ impl Track { packaging, is_live: false, role: None, + label: None, + lang: None, codec: None, width: None, height: None, @@ -620,6 +628,8 @@ mod test { packaging: Packaging::Legacy, is_live: true, role: Some(Role::Video), + label: None, + lang: None, codec: Some("avc3.64001f".to_string()), width: Some(1280), height: Some(720), @@ -644,6 +654,8 @@ mod test { packaging: Packaging::Legacy, is_live: true, role: Some(Role::Audio), + label: None, + lang: None, codec: Some("opus".to_string()), width: None, height: None, @@ -668,6 +680,8 @@ mod test { packaging: Packaging::Cmaf, is_live: true, role: Some(Role::Video), + label: None, + lang: None, codec: Some("avc1.640028".to_string()), width: Some(1920), height: Some(1080), diff --git a/rs/moq-mux/src/catalog/msf/consumer.rs b/rs/moq-mux/src/catalog/msf/consumer.rs index 511fc2bc5e..3ff9c6a97f 100644 --- a/rs/moq-mux/src/catalog/msf/consumer.rs +++ b/rs/moq-mux/src/catalog/msf/consumer.rs @@ -3,7 +3,7 @@ use std::str::FromStr; use std::task::{Poll, ready}; use base64::Engine; -use hang::catalog::{AudioCodec, AudioConfig, Container, VideoCodec, VideoConfig}; +use hang::catalog::{AudioCodec, AudioConfig, Container, TextConfig, TextFormat, TextRole, VideoCodec, VideoConfig}; use crate::Result; use crate::catalog::hang::{Catalog, CatalogExt}; @@ -94,8 +94,9 @@ impl From for Consumer { /// /// Each MSF track is mapped onto a `hang::Catalog` rendition based on its [`moq_msf::Role`]: /// video tracks become [`VideoConfig`] entries, audio tracks become [`AudioConfig`] entries. -/// Tracks with no role, with an unsupported role (caption, subtitle, sign language, audio -/// description, custom roles), or with packaging other than [`moq_msf::Packaging::Loc`], +/// Caption and subtitle tracks become text renditions; custom roles do too when their codec +/// identifies text. Tracks with no role, sign language, audio description, other custom roles, +/// or with packaging other than [`moq_msf::Packaging::Loc`], /// [`moq_msf::Packaging::Cmaf`], or [`moq_msf::Packaging::Legacy`] are skipped with a warning. /// /// [`moq_msf::Packaging::Loc`] and [`moq_msf::Packaging::Legacy`] map to their own containers: @@ -145,6 +146,16 @@ pub(crate) fn from_msf(msf: &moq_msf::Catalog) -> Result { + if let Some(config) = text_config_from_msf(track)? { + catalog.text.renditions.insert(track.name.clone(), config); + } + } + moq_msf::Role::Unknown(_) if matches!(track.codec.as_deref(), Some("vtt" | "ttml" | "utf8")) => { + if let Some(config) = text_config_from_msf(track)? { + catalog.text.renditions.insert(track.name.clone(), config); + } + } other => { tracing::warn!(track = %track.name, role = %other, "skipping MSF track with unsupported role"); } @@ -210,6 +221,25 @@ fn legacy_description(track: &moq_msf::Track) -> Result> { } } +fn text_config_from_msf(track: &moq_msf::Track) -> Result> { + let Some(container) = container_from_msf(track)? else { + return Ok(None); + }; + let codec = track + .codec + .as_deref() + .ok_or_else(|| Error::MissingTextCodec(track.name.clone()))?; + // Keep unfamiliar codecs unfamiliar: a guessed WebVTT format could make a player render garbage. + let mut config = TextConfig::new(TextFormat::from_str(codec)?); + config.role = TextRole::from_str(&track.role.as_ref().unwrap().to_string())?; + config.lang = track.lang.clone(); + config.label = track.label.clone(); + config.container = container; + config.jitter = track.jitter; + config.delay = track.delay; + Ok(Some(config)) +} + fn video_config_from_msf(track: &moq_msf::Track) -> Result> { // Unsupported packaging (e.g. MediaTimeline) bubbles up as Ok(None) so the caller can // skip the track with a warning rather than fail the whole catalog. @@ -424,6 +454,24 @@ mod test { track } + #[test] + fn a_text_codec_is_required_and_unknown_formats_are_preserved() { + let mut track = moq_msf::Track::new("text", moq_msf::Packaging::Loc); + track.role = Some(moq_msf::Role::Caption); + assert!( + from_msf(&moq_msf::Catalog::new(vec![track.clone()])) + .unwrap_err() + .to_string() + .contains("missing codec") + ); + track.codec = Some("future-text".into()); + let catalog = from_msf(&moq_msf::Catalog::new(vec![track])).unwrap(); + assert_eq!( + catalog.text.renditions["text"].format, + TextFormat::Unknown("future-text".into()) + ); + } + #[test] fn cmaf_video_yields_cmaf_container() { // "AAAYZ2Z0eXA=" decodes to a tiny ftyp-shaped stub; we just verify the bytes @@ -607,7 +655,7 @@ mod test { #[test] fn unsupported_role_is_skipped() { let mut track = audio_track("caption0", moq_msf::Packaging::Legacy); - track.role = Some(moq_msf::Role::Caption); + track.role = Some(moq_msf::Role::AudioDescription); let msf = moq_msf::Catalog::new(vec![track]); let catalog = from_msf::<()>(&msf).expect("unsupported role should be skipped, not error"); diff --git a/rs/moq-mux/src/catalog/msf/mod.rs b/rs/moq-mux/src/catalog/msf/mod.rs index 7e3c29892f..ffefd0b6ab 100644 --- a/rs/moq-mux/src/catalog/msf/mod.rs +++ b/rs/moq-mux/src/catalog/msf/mod.rs @@ -10,6 +10,9 @@ mod consumer; pub use consumer::Consumer; +#[cfg(test)] +pub(crate) use consumer::from_msf; + /// MSF catalog decoding errors. #[derive(Debug, Clone, thiserror::Error)] #[non_exhaustive] @@ -32,6 +35,9 @@ pub enum Error { #[error("MSF audio track {0:?} missing codec")] MissingAudioCodec(String), + #[error("MSF text track {0:?} missing codec")] + MissingTextCodec(String), + #[error("MSF video track {name:?} has invalid codec {codec:?}")] InvalidVideoCodec { name: String, codec: String }, diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index b3359c8d25..3f91958f75 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -1136,6 +1136,24 @@ fn to_msf_media(catalog: &hang::Catalog) -> moq_msf::Catalog { tracks.push(track); } + for (name, config) in &catalog.text.renditions { + let Some(packaging) = packaging_of(&config.container) else { + continue; + }; + let mut track = moq_msf::Track::new(name.clone(), packaging); + track.is_live = true; + track.role = Some(config.role.to_string().parse().unwrap()); + track.codec = Some(config.format.to_string()); + track.lang = config.lang.clone(); + track.label = config.label.clone(); + if let hang::catalog::Container::Cmaf { init, .. } = &config.container { + track.init_data = Some(base64::engine::general_purpose::STANDARD.encode(init)); + } + track.render_group = Some(1); + track.jitter = config.jitter; + track.delay = config.delay; + tracks.push(track); + } moq_msf::Catalog::with_ext(tracks, E::default()) } @@ -1176,6 +1194,37 @@ mod test { } } + #[test] + fn text_tracks_round_trip_through_msf() { + for (format, role) in [ + (hang::catalog::TextFormat::Vtt, hang::catalog::TextRole::Caption), + (hang::catalog::TextFormat::Ttml, hang::catalog::TextRole::Subtitle), + ( + hang::catalog::TextFormat::Utf8, + hang::catalog::TextRole::Unknown("commentary".into()), + ), + ] { + let mut config = hang::catalog::TextConfig::new(format); + config.role = role; + config.lang = Some("es-419".into()); + config.label = Some("Español".into()); + config.container = Container::Loc; + let mut catalog = hang::Catalog::<()>::default(); + catalog.text.insert("text", config.clone()).unwrap(); + let msf = to_msf_media::<()>(&catalog); + assert_eq!(msf.tracks.len(), 1); + let wire = msf.to_json().unwrap(); + assert!(wire.contains(r#""lang":"es-419""#)); + assert!(wire.contains(r#""label":"Español""#)); + let msf = moq_msf::Catalog::<()>::from_str(&wire).unwrap(); + let decoded = crate::catalog::msf::from_msf(&msf).unwrap(); + assert_eq!(decoded.text.renditions["text"], config); + let mut roundtrip = hang::Catalog::<()>::default(); + roundtrip.text = decoded.text; + assert_eq!(to_msf_media::<()>(&roundtrip).tracks, msf.tracks); + } + } + #[test] fn media_tracks_inherit_the_catalogs_declared_retention() { let mut broadcast = moq_net::broadcast::Info::new().produce(); From a0a268d5d42a359349c93509c01589ce2a706a2e Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 21:46:57 -0700 Subject: [PATCH 3/6] fix(msf): handle unsupported text packaging by consumer role Co-Authored-By: GPT-6 Astra --- drafts/draft-lcurley-moq-hang.md | 7 +++ js/watch/src/msf.test.ts | 17 ++++++++ js/watch/src/msf.ts | 3 +- rs/moq-mux/src/catalog/msf/consumer.rs | 59 ++++++++++++++++++-------- rs/moq-mux/src/catalog/msf/mod.rs | 4 ++ rs/moq-mux/src/catalog/producer.rs | 2 + 6 files changed, 74 insertions(+), 18 deletions(-) diff --git a/drafts/draft-lcurley-moq-hang.md b/drafts/draft-lcurley-moq-hang.md index b64d7e3523..e6c486b6b3 100644 --- a/drafts/draft-lcurley-moq-hang.md +++ b/drafts/draft-lcurley-moq-hang.md @@ -17,6 +17,7 @@ author: email: kixelated@gmail.com normative: + moqmsf: I-D.ietf-moq-msf moql: I-D.lcurley-moq-lite moqflate: title: "DEFLATE Compressed Tracks for MoQ" @@ -329,6 +330,10 @@ Unlike `format`, an unrecognized `role` never prevents rendering: it describes i The `lang` field is the BCP-47 {{!RFC5646}} language tag of the track, for example `en` or `es-419`. Two renditions sharing a `lang` are told apart by `label` ({{field-label}}), one of the common rendition fields ({{common}}). +A gateway translating an MSF {{moqmsf}} caption or subtitle track MUST preserve its `codec` as `format`, and its `role`, `lang`, and `label` when present. +It MUST preserve supported packaging and timing, and MUST refuse text packaging it cannot represent rather than silently delete the track. +A missing text codec MUST NOT be replaced with a guessed format. + Regardless of `format`, each frame's timestamp ({{container}}) is the authoritative cue start time on the media clock, so a relay and a consumer can order and schedule cues without parsing the payload. A text track has no delta frames: every frame is a self-contained cue, so a group MAY consist of multiple frames, following the same rule as a codec that lacks delta frames ({{container}}). @@ -1087,6 +1092,8 @@ This document has no IANA actions. ## moq-hang-04 {:numbered="false"} +- Specified MSF text metadata preservation and refusal of unrepresentable text packaging during gateway conversion. + - Added optional audio and video `warmup` durations, rounded up to whole milliseconds, and allowed video groups to start at convergent random access pictures. - Defined encoder `jitter` as flush lateness above the rendition's own recent minimum, replacing fixed frame-duration hints; container batches retain media-span estimates. - Timed CMAF samples from the frame timestamp, which is the fragment's earliest presentation time; `tfdt` is only relative within the fragment, and places the samples of an untimed frame. A timed CMAF track counts in its `mdhd` ticks. diff --git a/js/watch/src/msf.test.ts b/js/watch/src/msf.test.ts index 83f3af5dfe..06a01243ec 100644 --- a/js/watch/src/msf.test.ts +++ b/js/watch/src/msf.test.ts @@ -139,3 +139,20 @@ test("custom roles map only when the codec identifies text", () => { const text = toHang({ tracks: [{ name: "text", packaging: "legacy", role: "commentary", codec: "utf8" }] }); expect(text.text?.renditions.text?.role).toBe("commentary"); }); + +test("unsupported optional text packaging keeps playable audio and video", () => { + const tracks = [ + { name: "video", packaging: "loc", role: "video", codec: "vp09.00.10.08" }, + { name: "audio", packaging: "loc", role: "audio", codec: "opus", samplerate: 48000, channelConfig: "2" }, + ]; + for (const role of ["caption", "subtitle", "commentary"]) { + const catalog = toHang({ tracks: [...tracks, { name: "text", packaging: "future", role, codec: "vtt" }] }); + expect(catalog.video?.renditions.video?.codec).toBe("vp09.00.10.08"); + expect(catalog.audio?.renditions.audio?.codec).toBe("opus"); + expect(catalog.text).toBeUndefined(); + } +}); + +test("a text track with supported packaging still requires its codec", () => { + expect(() => toHang({ tracks: [{ name: "text", packaging: "loc", role: "caption" }] })).toThrow("missing codec"); +}); diff --git a/js/watch/src/msf.ts b/js/watch/src/msf.ts index 50dacf7421..b21f511c92 100644 --- a/js/watch/src/msf.ts +++ b/js/watch/src/msf.ts @@ -119,7 +119,8 @@ export function toHang(msf: Msf.Catalog): Catalog.Root { ) { if (!track.codec) throw new Error(`MSF text track ${track.name} missing codec`); const info = toContainer(track); - if (!info) throw new Error(`MSF text track ${track.name} has unsupported packaging`); + // A player selects usable renditions; an optional unsupported track cannot invalidate A/V. + if (!info) continue; textRenditions[track.name] = { format: track.codec, role: track.role, diff --git a/rs/moq-mux/src/catalog/msf/consumer.rs b/rs/moq-mux/src/catalog/msf/consumer.rs index 66f53511ad..fef36632b7 100644 --- a/rs/moq-mux/src/catalog/msf/consumer.rs +++ b/rs/moq-mux/src/catalog/msf/consumer.rs @@ -95,9 +95,10 @@ impl From for Consumer { /// Each MSF track is mapped onto a `hang::Catalog` rendition based on its [`moq_msf::Role`]: /// video tracks become [`VideoConfig`] entries, audio tracks become [`AudioConfig`] entries. /// Caption and subtitle tracks become text renditions; custom roles do too when their codec -/// identifies text. Tracks with no role, sign language, audio description, other custom roles, -/// or with packaging other than [`moq_msf::Packaging::Loc`], -/// [`moq_msf::Packaging::Cmaf`], or [`moq_msf::Packaging::Legacy`] are skipped with a warning. +/// identifies text. Unsupported text packaging is an error so a gateway cannot silently lose it. +/// Tracks with no role, sign language, audio description, other custom roles, and audio/video +/// with packaging other than [`moq_msf::Packaging::Loc`], [`moq_msf::Packaging::Cmaf`], or +/// [`moq_msf::Packaging::Legacy`] are skipped with a warning. /// /// [`moq_msf::Packaging::Loc`] and [`moq_msf::Packaging::Legacy`] map to their own containers: /// they carry the same codec payload but frame it differently. @@ -147,14 +148,16 @@ pub(crate) fn from_msf(msf: &moq_msf::Catalog) -> Result { - if let Some(config) = text_config_from_msf(track)? { - catalog.text.renditions.insert(track.name.clone(), config); - } + catalog + .text + .renditions + .insert(track.name.clone(), text_config_from_msf(track)?); } moq_msf::Role::Unknown(_) if matches!(track.codec.as_deref(), Some("vtt" | "ttml" | "utf8")) => { - if let Some(config) = text_config_from_msf(track)? { - catalog.text.renditions.insert(track.name.clone(), config); - } + catalog + .text + .renditions + .insert(track.name.clone(), text_config_from_msf(track)?); } other => { tracing::warn!(track = %track.name, role = %other, "skipping MSF track with unsupported role"); @@ -168,9 +171,8 @@ pub(crate) fn from_msf(msf: &moq_msf::Catalog) -> Result Result> { } } -fn text_config_from_msf(track: &moq_msf::Track) -> Result> { - let Some(container) = container_from_msf(track)? else { - return Ok(None); - }; +fn text_config_from_msf(track: &moq_msf::Track) -> Result { + // A gateway must not silently delete text that its output is expected to preserve. + let container = container_from_msf(track)?.ok_or_else(|| Error::UnsupportedPackaging { + name: track.name.clone(), + packaging: track.packaging.to_string(), + })?; let codec = track .codec .as_deref() @@ -237,7 +241,7 @@ fn text_config_from_msf(track: &moq_msf::Track) -> Result> { config.container = container; config.jitter = track.jitter; config.delay = track.delay; - Ok(Some(config)) + Ok(config) } fn video_config_from_msf(track: &moq_msf::Track) -> Result> { @@ -485,6 +489,27 @@ mod test { track } + #[test] + fn unsupported_text_packaging_is_refused_instead_of_lost() { + for packaging in [ + moq_msf::Packaging::MediaTimeline, + moq_msf::Packaging::EventTimeline, + moq_msf::Packaging::Unknown("future".into()), + ] { + for role in [ + moq_msf::Role::Caption, + moq_msf::Role::Subtitle, + moq_msf::Role::Unknown("commentary".into()), + ] { + let mut track = moq_msf::Track::new("text", packaging.clone()); + track.role = Some(role); + track.codec = Some("vtt".into()); + let error = from_msf(&moq_msf::Catalog::<()>::new(vec![track])).unwrap_err(); + assert!(matches!(error, crate::Error::Msf(Error::UnsupportedPackaging { .. }))); + } + } + } + #[test] fn a_text_codec_is_required_and_unknown_formats_are_preserved() { let mut track = moq_msf::Track::new("text", moq_msf::Packaging::Loc); diff --git a/rs/moq-mux/src/catalog/msf/mod.rs b/rs/moq-mux/src/catalog/msf/mod.rs index db9f5bf0cb..7c92c822c3 100644 --- a/rs/moq-mux/src/catalog/msf/mod.rs +++ b/rs/moq-mux/src/catalog/msf/mod.rs @@ -35,6 +35,10 @@ pub enum Error { #[error("MSF audio track {0:?} missing codec")] MissingAudioCodec(String), + /// The track's packaging cannot be converted without discarding it. + #[error("MSF track {name:?} has unsupported packaging {packaging:?}")] + UnsupportedPackaging { name: String, packaging: String }, + #[error("MSF text track {0:?} missing codec")] MissingTextCodec(String), diff --git a/rs/moq-mux/src/catalog/producer.rs b/rs/moq-mux/src/catalog/producer.rs index 3f91958f75..2c36439793 100644 --- a/rs/moq-mux/src/catalog/producer.rs +++ b/rs/moq-mux/src/catalog/producer.rs @@ -1208,6 +1208,8 @@ mod test { config.role = role; config.lang = Some("es-419".into()); config.label = Some("Español".into()); + config.jitter = Some(std::time::Duration::from_millis(7)); + config.delay = Some(std::time::Duration::from_millis(11)); config.container = Container::Loc; let mut catalog = hang::Catalog::<()>::default(); catalog.text.insert("text", config.clone()).unwrap(); From 226e4e86a17d476554bef7127886f24344cc6fce Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 10 Oct 2026 22:08:33 -0700 Subject: [PATCH 4/6] quest: retain caption interop validation blocker Co-Authored-By: GPT-6 Astra --- quest/m1/README.md | 1 + quest/m1/captions-import.md | 4 ++++ quest/m1/captions-msf.md | 41 +++++++++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 quest/m1/captions-msf.md diff --git a/quest/m1/README.md b/quest/m1/README.md index 962bba2402..b09bdbf0ac 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -161,6 +161,7 @@ bench all require its crate. - [Android capture](/quest/m1/mobile-capture-android.md) - Rust captures through NDK/JNI on Android, reusing the existing codecs and setting the catalog rotation - [Time stretch](/quest/m1/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent - [Native audio quality](/quest/m1/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device +- [MSF caption roles](/quest/m1/captions-msf.md) - conversion and review fixes pass; required interop has an unresolved native Bun restart failure - [Caption import](/quest/m1/captions-import.md) - fMP4 and MKV subtitle tracks import as text renditions instead of erroring or being dropped - [Fail loud on dropped timed metadata](/quest/m1/drop-loud.md) - FLV script tags and fMP4 emsg boxes are counted and warned about instead of silently dropped, until real carriage lands - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling diff --git a/quest/m1/captions-import.md b/quest/m1/captions-import.md index 87c1f72765..27b8969eff 100644 --- a/quest/m1/captions-import.md +++ b/quest/m1/captions-import.md @@ -29,3 +29,7 @@ for a cue that may be minutes away. Cover a file with subtitles and no supported text format, a file with several subtitle languages, and a file whose subtitle track is empty. + +## Related + +- [MSF caption roles](/quest/m1/captions-msf.md) - the catalog half of the same gap diff --git a/quest/m1/captions-msf.md b/quest/m1/captions-msf.md new file mode 100644 index 0000000000..c0911ad2e7 --- /dev/null +++ b/quest/m1/captions-msf.md @@ -0,0 +1,41 @@ +# [M] MSF caption roles + +## Goal + +An MSF catalog's caption and subtitle tracks survive conversion into a hang +catalog. Today they are dropped with a warning, so a +round trip through MSF silently loses every caption. + +## Plan + +`from_msf` skips any track "with no role, with an unsupported role (caption, +subtitle, sign language, audio description, custom roles)". Now that the hang +`text` section exists with a `TextRole` borrowing the MSF role names, two of +those map straight across and stop being unsupported: `TextRole` has only +`Subtitle`, `Caption`, and `Unknown(String)`, which preserves any other role +verbatim. + +Map the roles, and derive the rest of `TextConfig` from what MSF carries: +language onto `lang`, the display name onto `label`, and the packaging onto +`container` the way video and audio already do. Pick the cue `format` from the +track's codec rather than defaulting blindly, since an MSF caption track can +carry more than WebVTT and guessing wrong produces a track a player renders as +garbage. + +Prove the round trip in both directions, so an MSF catalog converted to hang +and back keeps its caption tracks, roles, and languages. Audio description is +deliberately left unmapped: it is an audio rendition with a role, not timed +text, and mapping it into `text` would be wrong. Sign language is likewise a +video rendition, so it stays unmapped too. A custom role maps to +`TextRole::Unknown` only when the track's codec is a text format. + +Per cross-package sync, mirror any schema movement in `js/msf`. + +### Validation remaining + +The conversion implementation and both review fixes are complete in [#5226](https://github.com/moq-dev/moq/pull/5226): full checks, draft validation, failing-before regressions and real browser playback pass. Keep this quest until required interop is green. The current all-language run fails only Python to native Bun: an accepted catalog subscription sees a restart announcement and is revoked, then reports `broadcast is closed`. All other matrix arms, refusals, exact browser close and finite tails pass. The cause is unproven; #5294 has no confirmed matching reproduction. Do not import it speculatively or retry away the failure. + +## Related + +- [Caption import](/quest/m1/captions-import.md) - the container half of the + same gap From 2d74fe9e4260c1cec47b95c7ccd8da6b6ea1430c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 15:48:15 -0700 Subject: [PATCH 5/6] quest: complete MSF caption roles Co-Authored-By: Claude Opus 5.5 --- quest/m1/README.md | 1 - quest/m1/captions-import.md | 4 ---- quest/m1/captions-msf.md | 41 ------------------------------------- 3 files changed, 46 deletions(-) delete mode 100644 quest/m1/captions-msf.md diff --git a/quest/m1/README.md b/quest/m1/README.md index e8a4b4322c..ef31cc0c01 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -156,7 +156,6 @@ session outcomes, and embedder refusals inside session outcomes. - [Android capture](/quest/m1/mobile-capture-android.md) - Rust captures through NDK/JNI on Android, reusing the existing codecs and setting the catalog rotation - [Time stretch](/quest/m1/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent - [Native audio quality](/quest/m1/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device -- [MSF caption roles](/quest/m1/captions-msf.md) - conversion and review fixes pass; required interop has an unresolved native Bun restart failure - [Caption import](/quest/m1/captions-import.md) - fMP4 and MKV subtitle tracks import as text renditions instead of erroring or being dropped - [Fail loud on dropped timed metadata](/quest/m1/drop-loud.md) - FLV script tags and fMP4 emsg boxes are counted and warned about instead of silently dropped, until real carriage lands - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling diff --git a/quest/m1/captions-import.md b/quest/m1/captions-import.md index 27b8969eff..87c1f72765 100644 --- a/quest/m1/captions-import.md +++ b/quest/m1/captions-import.md @@ -29,7 +29,3 @@ for a cue that may be minutes away. Cover a file with subtitles and no supported text format, a file with several subtitle languages, and a file whose subtitle track is empty. - -## Related - -- [MSF caption roles](/quest/m1/captions-msf.md) - the catalog half of the same gap diff --git a/quest/m1/captions-msf.md b/quest/m1/captions-msf.md deleted file mode 100644 index c0911ad2e7..0000000000 --- a/quest/m1/captions-msf.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] MSF caption roles - -## Goal - -An MSF catalog's caption and subtitle tracks survive conversion into a hang -catalog. Today they are dropped with a warning, so a -round trip through MSF silently loses every caption. - -## Plan - -`from_msf` skips any track "with no role, with an unsupported role (caption, -subtitle, sign language, audio description, custom roles)". Now that the hang -`text` section exists with a `TextRole` borrowing the MSF role names, two of -those map straight across and stop being unsupported: `TextRole` has only -`Subtitle`, `Caption`, and `Unknown(String)`, which preserves any other role -verbatim. - -Map the roles, and derive the rest of `TextConfig` from what MSF carries: -language onto `lang`, the display name onto `label`, and the packaging onto -`container` the way video and audio already do. Pick the cue `format` from the -track's codec rather than defaulting blindly, since an MSF caption track can -carry more than WebVTT and guessing wrong produces a track a player renders as -garbage. - -Prove the round trip in both directions, so an MSF catalog converted to hang -and back keeps its caption tracks, roles, and languages. Audio description is -deliberately left unmapped: it is an audio rendition with a role, not timed -text, and mapping it into `text` would be wrong. Sign language is likewise a -video rendition, so it stays unmapped too. A custom role maps to -`TextRole::Unknown` only when the track's codec is a text format. - -Per cross-package sync, mirror any schema movement in `js/msf`. - -### Validation remaining - -The conversion implementation and both review fixes are complete in [#5226](https://github.com/moq-dev/moq/pull/5226): full checks, draft validation, failing-before regressions and real browser playback pass. Keep this quest until required interop is green. The current all-language run fails only Python to native Bun: an accepted catalog subscription sees a restart announcement and is revoked, then reports `broadcast is closed`. All other matrix arms, refusals, exact browser close and finite tails pass. The cause is unproven; #5294 has no confirmed matching reproduction. Do not import it speculatively or retry away the failure. - -## Related - -- [Caption import](/quest/m1/captions-import.md) - the container half of the - same gap From 2485608c961b666d741b1c3c43a5c9a995b906fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sun, 11 Oct 2026 16:38:02 -0700 Subject: [PATCH 6/6] fix(watch): skip unsupported MSF text before requiring a codec Also key MSF renditions in null-prototype records so a track named __proto__ stays a rendition. Co-Authored-By: Claude Opus 5.5 --- js/watch/src/msf.test.ts | 10 ++++++++++ js/watch/src/msf.ts | 9 +++++---- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/js/watch/src/msf.test.ts b/js/watch/src/msf.test.ts index 06a01243ec..d62f3056a7 100644 --- a/js/watch/src/msf.test.ts +++ b/js/watch/src/msf.test.ts @@ -156,3 +156,13 @@ test("unsupported optional text packaging keeps playable audio and video", () => test("a text track with supported packaging still requires its codec", () => { expect(() => toHang({ tracks: [{ name: "text", packaging: "loc", role: "caption" }] })).toThrow("missing codec"); }); + +test("unsupported text packaging is skipped before its codec is required", () => { + const catalog = toHang({ tracks: [{ name: "text", packaging: "future", role: "caption" }] }); + expect(catalog.text).toBeUndefined(); +}); + +test("a text track named __proto__ is kept as a rendition", () => { + const catalog = toHang({ tracks: [{ name: "__proto__", packaging: "loc", role: "caption", codec: "vtt" }] }); + expect(Object.keys(catalog.text?.renditions ?? {})).toEqual(["__proto__"]); +}); diff --git a/js/watch/src/msf.ts b/js/watch/src/msf.ts index b21f511c92..2d5d9254da 100644 --- a/js/watch/src/msf.ts +++ b/js/watch/src/msf.ts @@ -98,9 +98,10 @@ function toAudioConfig(track: Msf.Track): Catalog.AudioConfig | undefined { /** Convert an MSF catalog to a hang catalog Root. */ export function toHang(msf: Msf.Catalog): Catalog.Root { - const videoRenditions: Record = {}; - const audioRenditions: Record = {}; - const textRenditions: Record = {}; + // Track names are untrusted keys; a null prototype keeps `__proto__` an ordinary rendition. + const videoRenditions: Record = Object.create(null); + const audioRenditions: Record = Object.create(null); + const textRenditions: Record = Object.create(null); for (const track of msf.tracks) { if (track.role === "video") { @@ -117,10 +118,10 @@ export function toHang(msf: Msf.Catalog): Catalog.Root { track.role !== "signlanguage" && ["vtt", "ttml", "utf8"].includes(track.codec ?? "")) ) { - if (!track.codec) throw new Error(`MSF text track ${track.name} missing codec`); const info = toContainer(track); // A player selects usable renditions; an optional unsupported track cannot invalidate A/V. if (!info) continue; + if (!track.codec) throw new Error(`MSF text track ${track.name} missing codec`); textRenditions[track.name] = { format: track.codec, role: track.role,