fix(msf): preserve caption and subtitle tracks across catalogs

drafts/draft-lcurley-moq-hang.md

@@ -17,6 +17,7 @@ author:
     email: kixelated@gmail.com
 
 normative:
+  moqmsf: I-D.ietf-moq-msf
   moql: I-D.lcurley-moq-lite
   moqflate:
     title: "DEFLATE Compressed Tracks for MoQ"
@@ -329,6 +330,10 @@ Unlike `format`, an unrecognized `role` never prevents rendering: it describes i
 The `lang` field is the BCP-47 {{!RFC5646}} language tag of the track, for example `en` or `es-419`.
 Two renditions sharing a `lang` are told apart by `label` ({{field-label}}), one of the common rendition fields ({{common}}).
 
+A gateway translating an MSF {{moqmsf}} caption or subtitle track MUST preserve its `codec` as `format`, and its `role`, `lang`, and `label` when present.
+It MUST preserve supported packaging and timing, and MUST refuse text packaging it cannot represent rather than silently delete the track.
+A missing text codec MUST NOT be replaced with a guessed format.
+
 Regardless of `format`, each frame's timestamp ({{container}}) is the authoritative cue start time on the media clock, so a relay and a consumer can order and schedule cues without parsing the payload.
 A text track has no delta frames: every frame is a self-contained cue, so a group MAY consist of multiple frames, following the same rule as a codec that lacks delta frames ({{container}}).
 
@@ -1087,6 +1092,8 @@ This document has no IANA actions.
 ## moq-hang-04
 {:numbered="false"}
 
+- Specified MSF text metadata preservation and refusal of unrepresentable text packaging during gateway conversion.
+
 - Added optional audio and video `warmup` durations, rounded up to whole milliseconds, and allowed video groups to start at convergent random access pictures.
 - Defined encoder `jitter` as flush lateness above the rendition's own recent minimum, replacing fixed frame-duration hints; container batches retain media-span estimates.
 - Timed CMAF samples from the frame timestamp, which is the fragment's earliest presentation time; `tfdt` is only relative within the fragment, and places the samples of an untimed frame. A timed CMAF track counts in its `mdhd` ticks.

js/msf/src/catalog.test.ts

@@ -250,3 +250,16 @@ test("round-trips a __proto__ extension member as data", () => {
 	expect(Object.keys(wire)).toContain("__proto__");
 	expect(wire.ok).toBe(2);
 });
+
+test("caption role, label, language and codec survive the wire", () => {
+	const track = {
+		name: "captions.en",
+		packaging: "loc",
+		isLive: true,
+		role: "caption",
+		codec: "ttml",
+		lang: "en",
+		label: "English SDH",
+	};
+	expect(decode(encode({ tracks: [track] })).tracks).toEqual([track]);
+});

js/msf/src/catalog.ts

@@ -28,6 +28,10 @@ const trackShape = {
 	// tracks, so accept its absence rather than reject the whole catalog.
 	isLive: z.optional(z.boolean()),
 	role: z.optional(RoleSchema),
+	/** A human-readable label for the track. */
+	label: z.optional(z.string()),
+	/** The BCP-47 language tag of the track. */
+	lang: z.optional(z.string()),
 	codec: z.optional(z.string()),
 	width: z.optional(z.number()),
 	height: z.optional(z.number()),

js/watch/src/msf.test.ts

@@ -118,3 +118,51 @@ test("drops a rendition whose packaging is unknown", () => {
 
 	expect(toHang(catalog).video?.renditions.video).toBeUndefined();
 });
+
+test("converts timed text without guessing its format", () => {
+	for (const codec of ["vtt", "ttml", "utf8", "future-text"]) {
+		const track = { name: "subs", packaging: "loc", role: "caption", codec, lang: "en", label: "SDH" };
+		const text = toHang({ tracks: [track] }).text?.renditions.subs;
+		expect(text?.format).toBe(codec);
+		expect(text?.role).toBe("caption");
+		expect(text?.lang).toBe("en");
+		expect(text?.label).toBe("SDH");
+		expect(text?.container).toEqual({ kind: "loc" });
+	}
+});
+
+test("custom roles map only when the codec identifies text", () => {
+	for (const role of ["commentary", "audiodescription", "signlanguage"]) {
+		const catalog = toHang({ tracks: [{ name: "track", packaging: "legacy", role, codec: "opus" }] });
+		expect(catalog.text).toBeUndefined();
+	}
+	const text = toHang({ tracks: [{ name: "text", packaging: "legacy", role: "commentary", codec: "utf8" }] });
+	expect(text.text?.renditions.text?.role).toBe("commentary");
+});
+
+test("unsupported optional text packaging keeps playable audio and video", () => {
+	const tracks = [
+		{ name: "video", packaging: "loc", role: "video", codec: "vp09.00.10.08" },
+		{ name: "audio", packaging: "loc", role: "audio", codec: "opus", samplerate: 48000, channelConfig: "2" },
+	];
+	for (const role of ["caption", "subtitle", "commentary"]) {
+		const catalog = toHang({ tracks: [...tracks, { name: "text", packaging: "future", role, codec: "vtt" }] });
+		expect(catalog.video?.renditions.video?.codec).toBe("vp09.00.10.08");
+		expect(catalog.audio?.renditions.audio?.codec).toBe("opus");
+		expect(catalog.text).toBeUndefined();
+	}
+});
+
+test("a text track with supported packaging still requires its codec", () => {
+	expect(() => toHang({ tracks: [{ name: "text", packaging: "loc", role: "caption" }] })).toThrow("missing codec");
+});
+
+test("unsupported text packaging is skipped before its codec is required", () => {
+	const catalog = toHang({ tracks: [{ name: "text", packaging: "future", role: "caption" }] });
+	expect(catalog.text).toBeUndefined();
+});
+
+test("a text track named __proto__ is kept as a rendition", () => {
+	const catalog = toHang({ tracks: [{ name: "__proto__", packaging: "loc", role: "caption", codec: "vtt" }] });
+	expect(Object.keys(catalog.text?.renditions ?? {})).toEqual(["__proto__"]);
+});

js/watch/src/msf.ts

@@ -98,8 +98,10 @@ function toAudioConfig(track: Msf.Track): Catalog.AudioConfig | undefined {
 
 /** Convert an MSF catalog to a hang catalog Root. */
 export function toHang(msf: Msf.Catalog): Catalog.Root {
-	const videoRenditions: Record<string, Catalog.VideoConfig> = {};
-	const audioRenditions: Record<string, Catalog.AudioConfig> = {};
+	// Track names are untrusted keys; a null prototype keeps `__proto__` an ordinary rendition.
+	const videoRenditions: Record<string, Catalog.VideoConfig> = Object.create(null);
+	const audioRenditions: Record<string, Catalog.AudioConfig> = Object.create(null);
+	const textRenditions: Record<string, Catalog.TextConfig> = Object.create(null);
 
 	for (const track of msf.tracks) {
 		if (track.role === "video") {
@@ -108,6 +110,27 @@ export function toHang(msf: Msf.Catalog): Catalog.Root {
 		} else if (track.role === "audio") {
 			const config = toAudioConfig(track);
 			if (config) audioRenditions[track.name] = config;
+		} else if (
+			track.role === "caption" ||
+			track.role === "subtitle" ||
+			(track.role &&
+				track.role !== "audiodescription" &&
+				track.role !== "signlanguage" &&
+				["vtt", "ttml", "utf8"].includes(track.codec ?? ""))
+		) {
+			const info = toContainer(track);
+			// A player selects usable renditions; an optional unsupported track cannot invalidate A/V.
+			if (!info) continue;
+			if (!track.codec) throw new Error(`MSF text track ${track.name} missing codec`);
+			textRenditions[track.name] = {
+				format: track.codec,
+				role: track.role,
+				lang: track.lang,
+				label: track.label,
+				container: info.container,
+				jitter: track.jitter != null ? delayMillis(track.jitter) : undefined,
+				delay: track.delay != null ? delayMillis(track.delay) : undefined,
+			};
 		}
 	}
 
@@ -121,5 +144,7 @@ export function toHang(msf: Msf.Catalog): Catalog.Root {
 		root.audio = { renditions: audioRenditions };
 	}
 
+	if (Object.keys(textRenditions).length > 0) root.text = { renditions: textRenditions };
+
 	return root;
 }

quest/m1/README.md

@@ -157,7 +157,6 @@ session outcomes, and embedder refusals inside session outcomes.
 - [Time stretch](/quest/m1/watch-audio-time-stretch.md) - js/watch: the audio ring converges by time-stretching instead of skipping or going silent
 - [Native audio quality](/quest/m1/audio-quality-native.md) - the browser lane's profiles, budgets, and metric schema run against `moq play` on a dummy device
 - [Caption import](/quest/m1/captions-import.md) - fMP4 and MKV subtitle tracks import as text renditions instead of erroring or being dropped
-- [MSF caption roles](/quest/m1/captions-msf.md) - an MSF caption or subtitle track survives conversion to a hang catalog
 - [Fail loud on dropped timed metadata](/quest/m1/drop-loud.md) - FLV script tags and fMP4 emsg boxes are counted and warned about instead of silently dropped, until real carriage lands
 - [Encoder colour](/quest/m1/color-model.md) - every moq-video encode path signals the colour its output actually has, or refuses instead of mislabelling
 - [T-STD TS export](/quest/m1/tstd/README.md) - `moq export ts` is a proper remux that passes the T-STD buffer model, starting with a fixed `--delay`

quest/m1/captions-import.md

@@ -29,8 +29,3 @@ for a cue that may be minutes away.
 
 Cover a file with subtitles and no supported text format, a file with several
 subtitle languages, and a file whose subtitle track is empty.
-
-## Related
-
-- [MSF caption roles](/quest/m1/captions-msf.md) - the catalog half of the same
-  gap

rs/moq-msf/src/lib.rs

@@ -123,6 +123,12 @@ pub struct Track {
 	/// Content role.
 	pub role: Option<Role>,
 
+	/// A human-readable label for the track.
+	pub label: Option<String>,
+
+	/// The BCP-47 language tag of the track.
+	pub lang: Option<String>,
+
 	/// WebCodecs codec string.
 	pub codec: Option<String>,
 
@@ -469,6 +475,8 @@ impl Track {
 			packaging,
 			is_live: false,
 			role: None,
+			label: None,
+			lang: None,
 			codec: None,
 			width: None,
 			height: None,
@@ -620,6 +628,8 @@ mod test {
 			packaging: Packaging::Legacy,
 			is_live: true,
 			role: Some(Role::Video),
+			label: None,
+			lang: None,
 			codec: Some("avc3.64001f".to_string()),
 			width: Some(1280),
 			height: Some(720),
@@ -644,6 +654,8 @@ mod test {
 			packaging: Packaging::Legacy,
 			is_live: true,
 			role: Some(Role::Audio),
+			label: None,
+			lang: None,
 			codec: Some("opus".to_string()),
 			width: None,
 			height: None,
@@ -668,6 +680,8 @@ mod test {
 			packaging: Packaging::Cmaf,
 			is_live: true,
 			role: Some(Role::Video),
+			label: None,
+			lang: None,
 			codec: Some("avc1.640028".to_string()),
 			width: Some(1920),
 			height: Some(1080),

rs/moq-mux/src/catalog/msf/consumer.rs

@@ -3,7 +3,7 @@ use std::str::FromStr;
 use std::task::{Poll, ready};
 
 use base64::Engine;
-use hang::catalog::{AudioCodec, AudioConfig, Container, VideoCodec, VideoConfig};
+use hang::catalog::{AudioCodec, AudioConfig, Container, TextConfig, TextFormat, TextRole, VideoCodec, VideoConfig};
 
 use crate::Result;
 use crate::catalog::hang::{Catalog, CatalogExt};
@@ -94,9 +94,11 @@ impl<E: CatalogExt> From<moq_net::track::Subscriber> for Consumer<E> {
 ///
 /// Each MSF track is mapped onto a `hang::Catalog` rendition based on its [`moq_msf::Role`]:
 /// video tracks become [`VideoConfig`] entries, audio tracks become [`AudioConfig`] entries.
-/// Tracks with no role, with an unsupported role (caption, subtitle, sign language, audio
-/// description, custom roles), or with packaging other than [`moq_msf::Packaging::Loc`],
-/// [`moq_msf::Packaging::Cmaf`], or [`moq_msf::Packaging::Legacy`] are skipped with a warning.
+/// Caption and subtitle tracks become text renditions; custom roles do too when their codec
+/// identifies text. Unsupported text packaging is an error so a gateway cannot silently lose it.
+/// Tracks with no role, sign language, audio description, other custom roles, and audio/video
+/// with packaging other than [`moq_msf::Packaging::Loc`], [`moq_msf::Packaging::Cmaf`], or
+/// [`moq_msf::Packaging::Legacy`] are skipped with a warning.
 ///
 /// [`moq_msf::Packaging::Loc`] and [`moq_msf::Packaging::Legacy`] map to their own containers:
 /// they carry the same codec payload but frame it differently.
@@ -145,6 +147,18 @@ pub(crate) fn from_msf<E: CatalogExt>(msf: &moq_msf::Catalog<E>) -> Result<Catal
 					);
 				}
 			},
+			moq_msf::Role::Caption | moq_msf::Role::Subtitle => {
+				catalog
+					.text
+					.renditions
+					.insert(track.name.clone(), text_config_from_msf(track)?);
+			}
+			moq_msf::Role::Unknown(_) if matches!(track.codec.as_deref(), Some("vtt" | "ttml" | "utf8")) => {
+				catalog
+					.text
+					.renditions
+					.insert(track.name.clone(), text_config_from_msf(track)?);
+			}
 			other => {
 				tracing::warn!(track = %track.name, role = %other, "skipping MSF track with unsupported role");
 			}
@@ -157,9 +171,8 @@ pub(crate) fn from_msf<E: CatalogExt>(msf: &moq_msf::Catalog<E>) -> Result<Catal
 /// Decode the [`Container`] for a track based on its packaging and `init_data`.
 ///
 /// Returns `Ok(None)` when the packaging is unsupported (e.g. `MediaTimeline`,
-/// `EventTimeline`, or an unknown variant). The caller skips these tracks with a warning
-/// rather than failing the whole catalog, since unsupported packaging is a downstream
-/// pipeline limitation, not a malformed catalog.
+/// `EventTimeline`, or an unknown variant). Text conversion refuses it to prevent silent
+/// loss; audio/video callers retain their existing skip policy.
 ///
 /// Returns `Err` when a CMAF track is missing or has malformed `init_data`. This is an
 /// intentional hard error: a CMAF rendition is unusable without its `ftyp+moov` init
@@ -210,6 +223,27 @@ fn legacy_description(track: &moq_msf::Track) -> Result<Option<bytes::Bytes>> {
 	}
 }
 
+fn text_config_from_msf(track: &moq_msf::Track) -> Result<TextConfig> {
+	// A gateway must not silently delete text that its output is expected to preserve.
+	let container = container_from_msf(track)?.ok_or_else(|| Error::UnsupportedPackaging {
+		name: track.name.clone(),
+		packaging: track.packaging.to_string(),
+	})?;
+	let codec = track
+		.codec
+		.as_deref()
+		.ok_or_else(|| Error::MissingTextCodec(track.name.clone()))?;
+	// Keep unfamiliar codecs unfamiliar: a guessed WebVTT format could make a player render garbage.
+	let mut config = TextConfig::new(TextFormat::from_str(codec)?);
+	config.role = TextRole::from_str(&track.role.as_ref().unwrap().to_string())?;
+	config.lang = track.lang.clone();
+	config.label = track.label.clone();
+	config.container = container;
+	config.jitter = track.jitter;
+	config.delay = track.delay;
+	Ok(config)
+}
+
 fn video_config_from_msf(track: &moq_msf::Track) -> Result<Option<VideoConfig>> {
 	// Unsupported packaging (e.g. MediaTimeline) bubbles up as Ok(None) so the caller can
 	// skip the track with a warning rather than fail the whole catalog.
@@ -455,6 +489,45 @@ mod test {
 		track
 	}
 
+	#[test]
+	fn unsupported_text_packaging_is_refused_instead_of_lost() {
+		for packaging in [
+			moq_msf::Packaging::MediaTimeline,
+			moq_msf::Packaging::EventTimeline,
+			moq_msf::Packaging::Unknown("future".into()),
+		] {
+			for role in [
+				moq_msf::Role::Caption,
+				moq_msf::Role::Subtitle,
+				moq_msf::Role::Unknown("commentary".into()),
+			] {
+				let mut track = moq_msf::Track::new("text", packaging.clone());
+				track.role = Some(role);
+				track.codec = Some("vtt".into());
+				let error = from_msf(&moq_msf::Catalog::<()>::new(vec![track])).unwrap_err();
+				assert!(matches!(error, crate::Error::Msf(Error::UnsupportedPackaging { .. })));
+			}
+		}
+	}
+
+	#[test]
+	fn a_text_codec_is_required_and_unknown_formats_are_preserved() {
+		let mut track = moq_msf::Track::new("text", moq_msf::Packaging::Loc);
+		track.role = Some(moq_msf::Role::Caption);
+		assert!(
+			from_msf(&moq_msf::Catalog::new(vec![track.clone()]))
+				.unwrap_err()
+				.to_string()
+				.contains("missing codec")
+		);
+		track.codec = Some("future-text".into());
+		let catalog = from_msf(&moq_msf::Catalog::new(vec![track])).unwrap();
+		assert_eq!(
+			catalog.text.renditions["text"].format,
+			TextFormat::Unknown("future-text".into())
+		);
+	}
+
 	#[test]
 	fn cmaf_video_yields_cmaf_container() {
 		// The complete init segment survives the MSF base64 conversion.
@@ -702,7 +775,7 @@ mod test {
 	#[test]
 	fn unsupported_role_is_skipped() {
 		let mut track = audio_track("caption0", moq_msf::Packaging::Legacy);
-		track.role = Some(moq_msf::Role::Caption);
+		track.role = Some(moq_msf::Role::AudioDescription);
 		let msf = moq_msf::Catalog::new(vec![track]);
 
 		let catalog = from_msf::<()>(&msf).expect("unsupported role should be skipped, not error");