diff --git a/Cargo.lock b/Cargo.lock index 08d5838edf..282876c569 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1471,6 +1471,16 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc-fast" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e75b2483e97a5a7da73ac68a05b629f9c53cff58d8ed1c77866079e18b00dba5" +dependencies = [ + "digest 0.10.7", + "spin 0.10.1", +] + [[package]] name = "crc32fast" version = "1.5.2" @@ -4050,6 +4060,16 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "md-5" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69b6441f590336821bb897fb28fc622898ccceb1d6cea3fde5ea86b090c4de98" +dependencies = [ + "cfg-if", + "digest 0.11.3", +] + [[package]] name = "mdns-sd" version = "0.21.4" @@ -4160,14 +4180,20 @@ dependencies = [ "async-trait", "bytes", "futures", + "hang", + "kio 0.6.0", + "moq-json", + "moq-mux", "moq-net", "object_store", "percent-encoding", + "quick_cache", "serde", "serde_json", "tempfile", "thiserror 2.0.21", "tokio", + "tracing", ] [[package]] @@ -4292,9 +4318,11 @@ dependencies = [ "bytes", "hang", "humantime", + "moq-archive", "moq-audio", "moq-auth", "moq-hls", + "moq-json", "moq-msf", "moq-mux", "moq-relay", @@ -4304,6 +4332,7 @@ dependencies = [ "moq-tokio", "moq-transcode", "moq-video", + "object_store", "pollster", "reqwest", "rustls", @@ -4395,12 +4424,16 @@ dependencies = [ name = "moq-hls" version = "0.5.4" dependencies = [ + "async-trait", "axum", "bytes", + "futures", "hang", "humantime", "kio 0.6.0", "m3u8-rs", + "moq-archive", + "moq-json", "moq-mux", "moq-net", "moq-tokio", @@ -6169,17 +6202,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1796bc93603f78c5760a69f2d58badc9618d22adade0a95385bb2adbae4eb94" dependencies = [ "async-trait", + "aws-lc-rs", + "base64 0.23.1", "bytes", "chrono", + "crc-fast", + "form_urlencoded", "futures-channel", "futures-core", "futures-util", "http", + "http-body-util", + "httparse", "humantime", + "hyper", "itertools 0.15.0", + "md-5", "nix", "parking_lot", "percent-encoding", + "quick-xml 0.41.0", + "rand 0.10.3", + "reqwest", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", "thiserror 2.0.21", "tokio", "tracing", @@ -6870,6 +6918,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e660451e55124f798a69a5af3f49ccfbefbd41910eefd25caf2393e1f3473ec1" dependencies = [ "memchr", + "serde", ] [[package]] @@ -6881,6 +6930,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "quick_cache" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c" +dependencies = [ + "equivalent", + "foldhash", + "hashbrown 0.17.1", + "parking_lot", +] + [[package]] name = "quinn" version = "0.11.12" @@ -7201,6 +7262,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", + "h2", "http", "http-body", "http-body-util", diff --git a/Cargo.toml b/Cargo.toml index e523d0943d..c532d53064 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -139,6 +139,7 @@ loom = { version = "0.7.2", features = ["futures"] } # DNS-SD advertisement and browsing for LAN peer discovery (moq-tokio's `mdns` feature). # `async` awaits the event channel instead of blocking a thread on it. mdns-sd = { version = "0.21", features = ["async"] } +moq-archive = { version = "0.0.4", path = "rs/moq-archive" } moq-audio = { version = "0.1.3", path = "rs/moq-audio", default-features = false } moq-auth = { version = "0.1.1", path = "rs/moq-auth" } moq-binary = { version = "0.1.3", path = "rs/moq-binary" } diff --git a/doc/bin/cli.md b/doc/bin/cli.md index 1376686f9d..7c8ecde6ca 100644 --- a/doc/bin/cli.md +++ b/doc/bin/cli.md @@ -18,9 +18,11 @@ or Docker; see [Install](/setup/install). | `import` | `capture` | Capture a camera, display, window, or app plus a microphone, and encode natively. | | `import` | `hls ` | Pull a remote HLS playlist. | | `import` | `rtmp`, `srt`, `rtc` | Accept pushes (`--listen`) or pull from a remote (`--connect`). | +| `import` | `archive ` | Replay a recording from an object store. | | `export` | `fmp4`, `mkv`, `ts`, `flv`, `h264`, `h265` | Write a container to stdout. | | `export` | `hls --listen` | Serve the broadcast as HLS over HTTP. | | `export` | `rtmp`, `srt`, `rtc` | Serve plays (`--listen`) or push to a remote (`--connect`). | +| `export` | `archive ` | Record the broadcast into an object store. | | `play` | | Decode and play in a native window with sound. | | `transcode` | | Publish a just-in-time rendition ladder next to a broadcast. | | `fetch` | `` | Write one group of a track to stdout. | @@ -178,6 +180,36 @@ refuses a listener or cluster flag. It gives up after 30 seconds, as `/fetch` does, and exits non-zero when the broadcast or group is not found, the relay refuses, or the deadline passes. +## Archive + +```bash +# Record a broadcast until it ends +moq --connect https://relay.example.com/anon --broadcast event.hang export archive s3://recordings/event + +# Replay it under another name +moq --connect https://relay.example.com/anon --broadcast event-replay.hang import archive s3://recordings/event +``` + +`export archive` records one broadcast with +[moq-archive](https://docs.rs/moq-archive), reading its catalog as it changes: +video and audio renditions pace the segments, and the catalog plus every text, +JSON, and binary track are recorded alongside. It refuses a rendition served +from another broadcast, and one that returns after the catalog dropped it. The +stage ends once the broadcast does, and it refuses a store URL that already +holds a recording. `--retention 1h` keeps only the last hour (a DVR), +deleting expired objects `--retention-grace` (default 30s) after the timeline +stops advertising them. + +`import archive` republishes a recording: the timeline replays as a live track +and every other track's groups are served on request, one object GET per group +range. By default it replays what is stored and ends the timeline there; +`--follow 2s` keeps checking for new segments of a recording still being made. + +Store URLs are `file:///absolute/path`, `s3://bucket/prefix`, +`gs://bucket/prefix`, or `az://container/prefix`. Cloud credentials come from +the usual `AWS_*`, `GOOGLE_*`, and `AZURE_*` environment variables. The `s3`, +`gcs`, and `azure` cargo features are on by default. + ## Multiple stages Separate stages with `--` to bridge several broadcasts, or both directions, @@ -186,7 +218,8 @@ over one connection: ```bash moq --connect https://relay.example.com/anon \ import --broadcast event.hang srt --listen 0.0.0.0:9000 \ - -- export --broadcast event.hang hls --listen 0.0.0.0:8080 + -- export --broadcast event.hang hls --listen 0.0.0.0:8080 \ + -- export --broadcast event.hang archive file:///recordings/event ``` ## Redundant publishers diff --git a/doc/bin/hls.md b/doc/bin/hls.md index d8b169ffdf..cc85030f74 100644 --- a/doc/bin/hls.md +++ b/doc/bin/hls.md @@ -38,13 +38,27 @@ broadcast by path: /{broadcast}/{video|audio}/{rendition}/seg/t{pts}.m4s ``` +A [`moq-archive`](https://docs.rs/moq-archive) recording replayed through its +`Reader` is served the same way, with no second stored copy. Playlists come +from the replayed timeline alone, and a segment GETs exactly one stored object +of its rendition, so switching renditions never downloads both. An +inline-parameter-set codec with no catalog `description` is the exception: +the first playlist render GETs one keyframe group to build the init segment, +then caches it. Out-of-band configs need no media GET. When the catalog's +`archive` entry names a `store` and no `replay` path, its ranges are durable on +this broadcast, so the playlists list the whole retained timeline and only the +recording's own retention trims them; DASH `timeShiftBufferDepth` is the listed +span. The playlist ends with `EXT-X-ENDLIST` only once the reader's caller +declares the recording finished; the store holds no completion marker. + The init URL carries a hash of its bytes, so a reconfigured rendition gets a new one. An embedder of the library can also label the publisher's run with `Broadcaster::set_generation`. Every segment URL then carries it (`seg/{generation}.{segment}.m4s`), since a restarted publisher reuses segment numbers for different media. -`--window` sets the playlist duration (default 16 s), +`--window` sets the live playlist duration (default 16 s) and caps segment +`Cache-Control: max-age` for every broadcast, `--listen-tls-cert`/`--listen-tls-key` or `--listen-tls-generate` serve HTTPS, and `--cors-origin` opens it to browsers. H.264/H.265 and AAC/Opus renditions are served. Import handles classic HLS; diff --git a/js/json/src/window/encoder.ts b/js/json/src/window/encoder.ts index e403ef919c..a5836a2507 100644 --- a/js/json/src/window/encoder.ts +++ b/js/json/src/window/encoder.ts @@ -129,6 +129,15 @@ export class Encoder { return this.#start + this.#window.length; } + /** + * Discard group-local state, so the next edit opens a new group with a header. + * + * Call this whenever the caller closes the current group behind the encoder's back. + */ + reset(): void { + this.#resyncGroup(); + } + /** Discard group-local state after an encoded frame did not reach the wire. */ #resyncGroup(): void { this.#flate = undefined; diff --git a/js/json/src/window/producer.ts b/js/json/src/window/producer.ts index b7051f60f7..20457f5e13 100644 --- a/js/json/src/window/producer.ts +++ b/js/json/src/window/producer.ts @@ -90,6 +90,24 @@ export class Producer { encoded.commit(); } + /** + * Finish the open group, leaving the next edit to open a replacement with a header. + * + * Idempotent: cutting when no group is open does nothing. A caller that stores complete groups + * cuts after its edits, so every edit so far sits in a group no later frame can extend. + */ + cut(): void { + if (!this.#group) return; + + // Reset first: the group closes either way below, and a throw must not leave the encoder + // appending ops to a group that is gone. + this.#encoder.reset(); + + const group = this.#group; + this.#group = undefined; + group.close(); + } + /** Finish the track, closing any open group. */ finish(): void { if (this.#finished) return; diff --git a/js/json/src/window/window.test.ts b/js/json/src/window/window.test.ts index 95d2f0bedd..f5bdb63cc5 100644 --- a/js/json/src/window/window.test.ts +++ b/js/json/src/window/window.test.ts @@ -75,6 +75,28 @@ test("push and pop round-trip", async () => { ]); }); +test("a cut is invisible to the consumer", async () => { + const live = new Live(); + await live.push(0); + live.producer.cut(); + live.producer.cut(); + await live.push(1); + + expect(await live.finish()).toEqual([ + { push: { index: 0, value: { n: 0 } } }, + { push: { index: 1, value: { n: 1 } } }, + ]); +}); + +test("reset makes the next edit a header", () => { + const encoder = new Encoder({}); + encoder.push({ n: 0 }).commit(); + expect(encoder.push({ n: 1 }).keyframe).toBe(false); + + encoder.reset(); + expect(encoder.push({ n: 2 }).keyframe).toBe(true); +}); + test("concurrent consumer reads are rejected", async () => { const track = new Track.Producer("test"); const consumer = new Consumer({ track: track.subscribe() }); diff --git a/quest/m1/archive/README.md b/quest/m1/archive/README.md index acff3c3023..1f3a9ff4d7 100644 --- a/quest/m1/archive/README.md +++ b/quest/m1/archive/README.md @@ -34,12 +34,41 @@ The segment engine is in `rs/moq-mux/src/timeline.rs`: - `moq-hls` renders live playlists from the timeline alone and FETCHes media per HTTP request (`rs/moq-hls/src/export/mod.rs:3-8`). A clean timeline finish ends every window with `EXT-X-ENDLIST` (:325-327). +- The same exporter serves a recording replayed through `moq_archive::Reader` + with no archive-specific code (`rs/moq-hls/src/export/archive_tests.rs`): + playlists read only the timeline (an inline parameter set also GETs one + keyframe group to build its init), and a segment GETs one object of its + rendition. The caller supplies the catalog. +- A catalog `archive` entry with a `store` and no `replay` path declares its + ranges durable on that broadcast, so the exporter lists the whole retained + timeline and only its pops trim it (`durable` in + `rs/moq-hls/src/export/mod.rs`). DASH `timeShiftBufferDepth` becomes the + listed span, and `--window` still bounds live playlists and caps segment + `max-age`. The catalog already states durability, so no per-broadcast + option or separate server is needed. `rs/moq-archive` stores the versioned objects on any `object_store::ObjectStore`: percent-encoded track names, `.info` JSON, the binary envelope, and put/get/list/delete. +`moq_archive::Writer` (`rs/moq-archive/src/writer.rs`) records enrolled tracks +through `Deferred`, omits failed tracks with `Pending::omit`, stores each +segment's timeline groups after `Producer::flush`, and expires DVR segments +with a deletion grace. On a prefix that already holds a recording, it replays the +retained timeline from a checkpoint through `timeline::Producer::resume`, refuses +groups at or below each track's largest stored group, and a DVR deletes +unreferenced group objects one grace period after recovery. +`moq_archive::Reader` (`rs/moq-archive/src/reader/mod.rs`) replays the timeline onto a +supplied `broadcast::Producer` and serves FETCH through `track::Dynamic` with a byte-bounded +object LRU. `Reader::refresh` follows by listing timeline keys after its cursor, so gaps and +DVR expiry recover from the next checkpoint; `Reader::finish` applies out-of-band finality. +`rs/moq-archive/src/proof.rs` records one multi-rendition broadcast end to end: its exact keys and +bytes match on memory, local disk, and an unordered listing, FETCH replays every group exactly, +and a rendition's playback GETs only that rendition's objects. ### Format +[Per-track timelines](/quest/m1/archive/track-timeline/README.md) replaces the +aligned segments below with one timeline per track. + The format is the draft's [Recording section](/drafts/draft-lcurley-moq-hang.md#recording). The application chooses the object prefix, selected tracks, retention, and credentials; `moq-archive` owns the @@ -103,12 +132,11 @@ owned by that prerequisite, not duplicated in archive storage. ## Quests -- [Recording writer](/quest/m1/archive/writer.md) - feed the segmenter from a `broadcast::Consumer`, store each segment, then commit its record -- [Recording reader](/quest/m1/archive/reader.md) - serve archived FETCH through a supplied `broadcast::Producer` +- [Per-track timelines](/quest/m1/archive/track-timeline/README.md) - every track segments and expires on its own timeline, and HLS is derived from group timestamps at the edge +- [Replay catalog](/quest/m1/archive/replay-catalog.md) - `moq import archive` publishes the recorded catalog live with `store` set, so stock `moq export hls` serves the whole replay - [Browser archive](/quest/m1/archive/browser.md) - the same contract for browser-published broadcasts -- [Offline archive HLS](/quest/m1/archive/hls.md) - render playlists from the archive timeline and fetch segment media lazily - [DVR rewind](/quest/m1/archive/dvr.md) - seek through a bounded archive and return to live playback -- [Archive proof](/quest/m1/archive/proof.md) - prove persistence ordering, selective reads, exact FETCH replay, and timeline-only HLS generation +- [DVR timeline pruning](/quest/m1/archive/pruning.md) - a DVR deletes timeline objects no retained checkpoint needs ## Related diff --git a/quest/m1/archive/browser.md b/quest/m1/archive/browser.md index 520a5cb09e..7d08f060cf 100644 --- a/quest/m1/archive/browser.md +++ b/quest/m1/archive/browser.md @@ -9,16 +9,16 @@ layout, and FETCH behavior as native `moq-archive` users. Port the archive contract to the JS packages with memory and OPFS storage. The application explicitly enrolls video, audio, catalog, or arbitrary data tracks; -the archive does not infer them from Hang. The JS timeline already publishes -through the Window (`js/hang/src/container/timeline.ts:134`, `:165`, backed by -`js/json/src/window/`) with pacing tracks and application-driven cuts; add the -deferred commit Rust has (`Producer::deferred`, -`rs/moq-mux/src/timeline.rs:917`). - -Persist one range-named object per track per segment after its groups complete, -then publish the archive timeline record. Match the 19-digit group-bound keys, -ascending delta-encoded IDs, and sequential timeline discovery used by Rust. A typical audio segment contains many -one-group-per-frame audio groups. Match the Rust binary envelope bytes and `.info` property values, per the [Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without inferring catalog-to-group applicability. +the archive does not infer them from Hang. Record against the per-track +timelines from [JS per-track timelines](/quest/m1/archive/track-timeline/js.md). + +Commit each track independently: persist each stored span, including a frame +range of a still-open group, then publish that track's timeline record. Match +the object keys, envelope bytes, `.info` property values, and timeline +discovery the Rust writer uses after +[Rust per-track timelines](/quest/m1/archive/track-timeline/core.md), per the +[Recording section](/drafts/draft-lcurley-moq-hang.md#recording), without +inferring catalog-to-group applicability. [Catalog track identity](/quest/m2/catalog-tracks.md) addresses that separately. Use [JavaScript FETCH](/quest/m1/js-fetch.md)'s on-demand group requests to @@ -34,5 +34,4 @@ remains downstream (moq.pro) work. ## Required - [JavaScript FETCH](/quest/m1/js-fetch.md) - generic on-demand group serving and IETF FETCH support - -- [Recording writer](/quest/m1/archive/writer.md) +- [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - the timeline this archive records against diff --git a/quest/m1/archive/dvr.md b/quest/m1/archive/dvr.md index 930571650f..48959cd37c 100644 --- a/quest/m1/archive/dvr.md +++ b/quest/m1/archive/dvr.md @@ -7,9 +7,9 @@ same timeline and group-range objects as an unbounded archive. ## Plan -The recording writer prerequisite owns retention, checkpoint recovery, deletion -grace, and restart cleanup. This quest consumes that contract and owns viewer -seek and return-to-live behavior, not a second writer implementation. +The recording writer owns retention, deletion grace, checkpoint recovery, and +restart cleanup. This quest consumes that contract and owns viewer seek and +return-to-live behavior, not a second writer implementation. The player reads the archive timeline, FETCHes old groups through the normal miss chain, and splices back to SUBSCRIBE at the live edge without opening a @@ -22,10 +22,18 @@ Test seeks within the retained window, expiry during a seek, missing groups, restart recovery, and return to live without duplicated or rewound playback. Use the writer/reader fixtures; a retention defect is fixed in its owning layer. +The reader evicts popped spans from its object cache, but a group it already +served stays in `moq_net`'s track cache until the pool reclaims it. Decide +whether expiry during a seek needs a group eviction API in `moq-net`. + +`moq-hls` reads a timeline from the catalog's own broadcast, and an +`archive.replay` path only marks it non-durable. Decide whether a live +broadcast's exporter follows `replay` to the recording for rewind, or whether +viewers address the replay broadcast directly. + ## Required -- [Recording writer](/quest/m1/archive/writer.md) -- [Recording reader](/quest/m1/archive/reader.md) +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - seeks through per-track timelines ## Closes diff --git a/quest/m1/archive/hls.md b/quest/m1/archive/hls.md deleted file mode 100644 index a967808de4..0000000000 --- a/quest/m1/archive/hls.md +++ /dev/null @@ -1,42 +0,0 @@ -# [L] Offline archive HLS - -## Goal - -`moq-hls` serves ordinary HLS from a growing or static archive without a second -stored media copy, and renders playlists without downloading media objects. - -## Plan - -Use the archive catalog for rendition and initialization metadata, then reuse -the live HLS renderer against the timeline. Segment number, PTS, duration, -track ranges, gaps, and keyframe state provide everything needed for a media -playlist. Playlist generation and reloads read the timeline only; a regression -test must fail if they GET a media segment. - -When a player requests media, use the recording reader to GET only the selected -range-named object and transmux its groups on demand. Switching between -360p and 1080p must not download both rendition objects. Emit media URIs with -the track and inclusive group bounds from the timeline record, so the handler -can derive `groups/.` directly without listing or a -segment-ID lookup. HLS sequence numbers do not appear in storage keys. Keep -one storage object per track per timeline segment; a MoQ group need not be an -HLS segment, especially for one-group-per-frame audio. No LL-HLS parts. - -Emit `EXT-X-ENDLIST` exactly when the timeline track the exporter reads -finishes cleanly, as the live export already does -(`rs/moq-hls/src/export/mod.rs:325-327`, `rendition.rs:243-246`). The store -holds no completion marker: the reader finishes the replayed timeline track -when its caller supplies finality out of band, so a standalone or BYOB archive -without such a caller stays a reloadable playlist. - -Use the catalog supplied to the exporter. This quest does not establish which -catalog update applies to a historic group; timestamps do not provide an -explicit binding. Track immutability and update correlation belong to -[Catalog track identity](/quest/m2/catalog-tracks.md), independently of DVR. - -Prove aligned audio/video switching, missing track segments, discontinuities, -caller-supplied finality, and bounded LRU reads. - -## Required - -- [Recording reader](/quest/m1/archive/reader.md) diff --git a/quest/m1/archive/proof.md b/quest/m1/archive/proof.md deleted file mode 100644 index 1f68bf7385..0000000000 --- a/quest/m1/archive/proof.md +++ /dev/null @@ -1,67 +0,0 @@ -# [M] Archive proof - -## Goal - -Prove deterministic segment storage, exact FETCH replay, selective rendition -reads, and timeline-only HLS generation from one multi-rendition broadcast. - -## Plan - -Record explicitly enrolled video, audio, catalog, and non-media tracks. Cut at -aligned keyframe boundaries, including multiple audio groups per segment, then -replay their original sequences, timestamps, and payloads through -`track::Dynamic`. - -Verify the exact object keys and bytes on memory, local, and S3-compatible -`object_store` implementations, including the percent-encoded track names. A -360p or audio-only FETCH must not GET the 1080p object, while adjacent group -requests should hit the segment LRU. - -Cover the persistence boundary: a crash after segment PUT but before timeline -commit leaves invisible orphan data that a listing bootstrap ignores; a failed -or mismatched `.info` exposes no ranges; a failed independent track PUT omits -only that track while the record's other tracks stay advertised; later -segments remain usable. Catalog-to-group applicability is outside this proof. Also -cover a stalled pacing track forced to a gap, sparse group ranges, malformed -table offsets, an unknown envelope or `.info` version treated as a missing -segment, segment create collisions under the single-writer rule, a missing -tail, and a clean end without a completion marker. Accept equivalent `.info` -JSON with reordered members or different whitespace without rewriting it; -reject differing properties and malformed or unsupported metadata. - -Exercise group and segment IDs 0 and 2^53 - 1; reject 2^53 and the largest -QUIC varint in keys and reconstructed group IDs. Cover consecutive zero -deltas, sparse deltas, overflow, stopping at ID exhaustion without wrapping -or inferring clean finality, empty objects, mismatched filename bounds, and overlapping -ranges across segments. Reject decreasing or duplicate group arrivals while -allowing accepted groups to complete out of order. Verify JSON-safe timescales -and timestamps, accepting timestamp 2^53 - 1 and rejecting 2^53. -Test ordered S3 lookup at both endpoints and between ranges, unordered paginated -results, incremental cursor recovery, DVR expiration while a reader is offline, -and stale media listings preceding a new timeline commit. Following N+1 must -not refresh all media listings. Wire these cases into CI for the store, writer, -and reader implementations; do not add an unconnected standalone proof script. - -The store's own tests also cover recording-prefix isolation (`rec` beside -`rec-other`), empty prefixes, track-prefix listings, continuation pages, and -every supported pagination option. A page must not lose directory entries -silently or fail because the backend matched a neighbouring recording. Every -publicly constructible key either serializes to a path its parser accepts or -fails before storage; direct `Key::Groups` construction must not bypass range -validation. These cases belong beside the store and codec code and need no -new public API or format change. - -Crash a DVR writer after its pop becomes durable but before media deletion. -On exclusive restart, prove that expired and uncommitted group objects are -removed after the grace period while retained media, `.info`, and checkpoint -objects survive. Failed or incomplete recovery/listing must delete nothing; -restart must finish this cleanup before accepting new groups. - -Finally render and reload HLS playlists while rejecting every media-object GET -until a segment URI is requested. The range-bearing URI must resolve one -object directly without any listing or separate index object. - -## Required - -- [Recording writer](/quest/m1/archive/writer.md) -- [Offline archive HLS](/quest/m1/archive/hls.md) diff --git a/quest/m1/archive/pruning.md b/quest/m1/archive/pruning.md new file mode 100644 index 0000000000..3e2dfcfbc6 --- /dev/null +++ b/quest/m1/archive/pruning.md @@ -0,0 +1,17 @@ +# [S] DVR timeline pruning + +## Goal + +A DVR writer deletes timeline objects that no retained checkpoint needs, so a +long-running DVR stores a bounded number of objects. + +## Plan + +`moq_archive::Writer` deletes expired group objects but never a +`segments/` timeline object, so a 24/7 DVR with 2s segments adds +about 43,000 objects a day, and every restart lists all of them. The draft +already allows it: the writer keeps the latest timeline object and enough +earlier groups to recover the retained window from a checkpoint. Delete the +oldest timeline objects no longer needed, one grace period after they stop +being needed, keeping the remaining keys contiguous, as recovery requires. +Extend the restart cleanup and its tests in `rs/moq-archive/src/writer.rs`. diff --git a/quest/m1/archive/reader.md b/quest/m1/archive/reader.md deleted file mode 100644 index f317970680..0000000000 --- a/quest/m1/archive/reader.md +++ /dev/null @@ -1,49 +0,0 @@ -# [L] Recording reader - -## Goal - -A reader takes an archive and a caller-provided `broadcast::Producer`, then -serves FETCH requests for every track and group the archive timeline -advertises. - -## Plan - -Bootstrap by listing the timeline track's `segments/` keys, sorting them, and -replaying from a retained Window checkpoint. Records derive each track's -`groups/.` key from its minimum and maximum advertised IDs. -Follow a growing archive with GET of timeline segment N+1; media listings do -not need refreshing. Not Found is not finality. A reader behind DVR retention -re-bootstraps from a retained checkpoint, and timeline pops evict cached ranges. -Alternatively enumerate timeline keys after the last replayed key with -`list_with_offset`, consuming and sorting the full result before replay. -Pagination tokens continue one enumeration, not future refreshes. - -Filename listings provide a cached group-range index without media GETs. A -backend with verified ordered offset listing may find a cold FETCH candidate -using `PaginatedListStore` with offset `groups/` (19 padded -digits, no dot) and `max_keys = 1`. Check the lower bound and committed timeline -membership; the table resolves internal gaps. Generic listings are unordered, -so never take their first entry as the nearest match. A stale listing cannot -veto a successful GET referenced by a newer durable timeline record. - -Use `track::Dynamic` (`rs/moq-net/src/model/track.rs:1652`, minted by -`Producer::dynamic` :1587) to accept requested tracks and groups on the -supplied producer; a cache-miss `Consumer::fetch_group` (:2352) parks on it. -Map `(track, group)` to its range-named object through the cached index, GET -that one object, validate the envelope version and every group/frame table -entry against the retrieved length, filename bounds, and exact committed -ranges, and place it in a byte-bounded LRU. -Adjacent group FETCHes reuse the same object. A request for one audio track or -rendition never downloads another track's object. - -Reproduce the original group sequence, frame timestamps, and payload bytes, -including a requested `frame_start`. A group absent from the timeline, a -missing object, an unknown envelope version, or a malformed table behaves -exactly like a group the source never delivered; siblings and later segments -remain usable. - -The reader does not interpret media, own routing, or expose storage paths. It -does not infer a terminal broadcast state: a caller such as a managed -recordings API may supply finality out of band, and the reader then finishes -the replayed timeline track. Otherwise an archive may be growing, crashed, or -missing its tail, and remains readable. diff --git a/quest/m1/archive/replay-catalog.md b/quest/m1/archive/replay-catalog.md new file mode 100644 index 0000000000..d6f5bb8ec5 --- /dev/null +++ b/quest/m1/archive/replay-catalog.md @@ -0,0 +1,48 @@ +# [S] Replay catalog + +## Goal + +A replayed recording plays as VOD HLS from the stock server: `moq export +archive`, then `moq import archive`, then `moq export hls` lists the whole +recording with no embedder supplying a catalog. The replay broadcast publishes +its recorded catalog live, stamped with the recording's `store` and `version`, +so the exporter treats its timeline as durable and lists past `--window`. A +`--follow` replay grows like an event playlist. + +Choosing which catalog applies to which media group stays with +[Catalog track identity](/quest/m2/catalog-tracks.md). + +## Plan + +Today the reader writes live groups only for timelines; the recorded catalog +is FETCH-only, so a SUBSCRIBE to it on the replay broadcast never sees a group +and `moq-hls` never finds the `archive` entry. The HLS archive tests work around +this by hand-building a catalog. + +- Republish each recorded catalog group live in its timeline's order, so the newest + one is at the live edge and a `--follow` replay picks up catalogs recorded + after it opened. +- Stamp `store` with the URL passed to `import archive`, and `version` with the + recording format. Refuse a URL carrying userinfo so credentials never land in + a catalog. `replay` stays unset: the timelines live on this broadcast. +- Keep the logic in `moq-cli`; `moq-archive` stays catalog-agnostic. Select the + catalog track with the CLI's catalog format as `export archive` does: hang + and hang.z are stamped, MSF is refused. +- The recorded catalog's `archive` entry describes the source's live + timelines, not the recording's, so replace it with the timelines the reader + replays rather than trusting the recorded ones. + +Add a CI test that records a broadcast longer than the default window, replays +it, and asserts the stock exporter lists segment 0 with a durable +`timeShiftBufferDepth`. Consider moving the HLS archive tests onto the +republished catalog instead of their hand-built one. + +Update `doc/bin/cli.md` and `doc/bin/hls.md` inline with an export, import, and +serve example. + +Cover a DVR recording whose catalog outlived its first video segment, and fail +loudly on a recording with no catalog. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - a DVR keeps its catalog's newest group diff --git a/quest/m1/archive/track-timeline/README.md b/quest/m1/archive/track-timeline/README.md new file mode 100644 index 0000000000..2b4190e23b --- /dev/null +++ b/quest/m1/archive/track-timeline/README.md @@ -0,0 +1,46 @@ +# Per-track timelines + +## Goal + +Every track carries its own timeline, so tracks segment, commit, and expire +independently. A DVR keeps each track's newest group, such as a catalog that +never changes, and an append-only group stays addressable as frames arrive. An +edge like `moq-hls` derives HLS and DASH from group timestamps, so a publisher +never needs to know about HLS. + +Nothing on `main` has users yet: break the timeline, catalog `archive` entry, +and recording format in place, with no compatibility path, even though hang +0.21 released the current shape. The children merge into this line's branch, +so Rust and JS reach `main` together. + +## Plan + +Today one timeline numbers aligned segments for every track. A DVR pops whole +segments, so a static catalog expires with the first video segment, and the +writer only records a non-pacing group once it completes, so a group that never +closes is never stored. `moq_json::window` pops from the front, so one shared +timeline cannot keep an old catalog record while dropping newer video. + +Decisions: + +- One timeline track per track, live and recorded. `moq-mux` publishes them for + every broadcast; an unsubscribed track costs nothing. +- The catalog's root `archive` entry maps each track to its timeline, including + the catalog track itself. `replay`, `store`, and `version` stay beside it. +- Each track cuts on its own: automatically at a group boundary between a + minimum and maximum duration (roughly 1s and 10s), splitting a long-lived + group by frame at the maximum. Manual cuts stay as an optimization, such as a + video keyframe cutting audio so derived segments need fewer objects. +- A stored object may hold a frame range of a group, not only whole groups. +- HLS and DASH segments are derived at the edge from group timestamps, not + from storage objects. Fetching extra objects is fine when they land in the + reader's cache for the next request. + +## Quests + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the draft, `hang`, `moq-mux`, `moq-archive`, and `moq-hls` move to per-track timelines in one change +- [JS per-track timelines](/quest/m1/archive/track-timeline/js.md) - `@moq/hang` publishes and reads the same per-track timelines as Rust + +## Related + +- [Catalog track identity](/quest/m2/catalog-tracks.md) - the catalog's own timeline gives it timestamps, but which catalog applies to a group stays there diff --git a/quest/m1/archive/track-timeline/core.md b/quest/m1/archive/track-timeline/core.md new file mode 100644 index 0000000000..4651a338f9 --- /dev/null +++ b/quest/m1/archive/track-timeline/core.md @@ -0,0 +1,46 @@ +# [XL] Rust per-track timelines + +## Goal + +The moq-hang draft, `hang`, `moq-mux`, `moq-archive`, and `moq-hls` use one +timeline per track, per the [line's decisions](/quest/m1/archive/track-timeline/README.md). +A DVR recording keeps every track's newest group, an append-only group is +stored as its frames arrive, and `moq-hls` serves aligned HLS and DASH from +live or recorded timelines without the publisher cutting for it. + +## Plan + +These crates compile together, so they change in one PR. Update the draft's +timeline and Recording sections in the same change, in a new draft revision; +most of their rules assume one aligned segment counter (one timeline per +broadcast, cross-track boundaries, whole-segment retention, reading segment N +of track T through record N). + +- **Timeline:** a record describes one stored span of its own track: sequence, + pts, duration, and the group and frame range. Drop cross-track pacing and the + completeness wait; keep manual cuts. Retention pops each track's window + independently and always keeps its newest record. +- **Catalog:** replace the root `archive.track` with a map from track to + timeline, covering the catalog track too. +- **Writer and reader:** commit each track independently, and store frame + ranges so a never-closing group is recorded. Bump the recording `version` and + refuse the old one. Recovery, grace deletion, and FETCH replay follow the + per-track index. +- **HLS and DASH:** take segment boundaries from a reference video rendition + (an audio one when the broadcast has no video), near a target duration, and + number them so every edge and every reload agrees, including after DVR pops. Every other video rendition snaps each + boundary to its nearest group start within a tolerance (around 1s); a + segment with no start in range becomes a gap (`EXT-X-GAP` in HLS), so a + player switching renditions lands on the next real segment. Gaps are a + best-effort fallback: the draft and HLS docs say a publisher wanting HLS + export SHOULD align video GOPs across renditions. Audio and other + renditions take the groups and frames whose timestamps fall in each span, + possibly from more than one object. The reader's cache absorbs the overlap. + +Carry over the existing tests (DVR trim, durable listing, archive replay) and +add a static catalog outliving its first video segment, an append-only group +spanning several objects, audio cut independently of video, renditions that +start at different times, video renditions whose group starts never +coincide, and an audio-only broadcast. + +Update `doc/concept/hang.md` and the HLS and CLI docs this makes stale. diff --git a/quest/m1/archive/track-timeline/js.md b/quest/m1/archive/track-timeline/js.md new file mode 100644 index 0000000000..461ff1cf75 --- /dev/null +++ b/quest/m1/archive/track-timeline/js.md @@ -0,0 +1,17 @@ +# [M] JS per-track timelines + +## Goal + +`@moq/hang` publishes one timeline per track with the same records, cuts, and +catalog `archive` map as Rust, and parses them identically. + +## Plan + +Port the landed Rust shape to `js/hang/src/timeline.ts` and its catalog schema, +replacing the aligned timeline and cross-track pacing rather than keeping both. +Cover the same cut rules and a static catalog outliving other tracks' records, +and check the records against Rust output in the interop suite. + +## Required + +- [Rust per-track timelines](/quest/m1/archive/track-timeline/core.md) - the format this mirrors diff --git a/quest/m1/archive/writer.md b/quest/m1/archive/writer.md deleted file mode 100644 index 16eb74108d..0000000000 --- a/quest/m1/archive/writer.md +++ /dev/null @@ -1,80 +0,0 @@ -# [L] Recording writer - -## Goal - -A generic `moq-archive` writer consumes explicitly selected tracks from a -`broadcast::Consumer`, feeds their complete groups to the existing segmenter, -stores one object per track per segment, and commits the timeline record only -after the objects it names are durable. - -## Plan - -The application supplies the broadcast, object prefix, and arbitrary track -registrations, each pacing or non-pacing; the writer does not parse a catalog -or know which tracks are media. - -Enrollment creates the track's `.info` before accepting any group. If that -create fails, enrollment fails and no range for the track can enter the -timeline. On `AlreadyExists`, validate the immutable `.info` and accept only matching -parsed `version`, `priority`, and `timescale` values; a priority or timescale mismatch fails enrollment. - -Feed the existing segmenter (`rs/moq-mux/src/timeline.rs`): take -`Producer::deferred` (:917), enroll each registration through -`Deferred::track` or `pacing_track` (:625-631), and report every complete -group read from the consumer through its `Recorder` (:1096). The application -calls `cut(pts)` (:635) when it knows an aligned keyframe boundary. A segment -may hold many groups per track, especially one-group-per-frame audio. A pacing -track that stops without closing blocks segment completion on purpose; the -application applies its own deadline and calls `cut(pts)` or removes the -track. Storage does not invent a timeout. - -Accept new group IDs in strictly increasing order per track; refuse duplicate -or decreasing arrivals. Already accepted groups may finish in any order, so -buffer completion independently and encode in sequence order. Require -nonoverlapping object ranges across segments, including forced cuts; never -backfill a closed segment or move its stalled groups into an overlapping range. - -For each `Pending` record from `Deferred::next` (:658), encode and PUT one -object per participating track at `groups/.`, buffering -groups independently of relay retention. The bounds come from the exact -recorded ranges; tracks with no complete groups have no object. After all PUTs settle, drop the ranges of every track whose PUT -failed and commit with `Producer::push` (:948). That needs a per-track -omission on `Pending` beside `Pending::gap()` (:712-719), which clears every -track; add it here. Never publish a range first and hope the relay still has -it. A later segment continues normally after any omission. - -Catalog update applicability and cross-track configuration dependencies belong -to [Catalog track identity](/quest/m2/catalog-tracks.md). Recording a catalog -as an ordinary track does not bind its updates to media groups; this writer -does not introduce catalog-specific commit or retention prerequisites. - -The archive timeline uses the same object envelope but is not enrolled in its -own records. After pushing segment N and applying retention pops, close the -recording-owned Window group and store the complete groups under the timeline -track's `segments/N` key (19-digit padded N). Commit IDs consecutively, -including all-gap segments. Make it durable before committing N+1; a timeline PUT -failure stops the recording at its preceding durable timeline object. Never -cut a source group. On a clean source end, drain with -`Deferred::finish` (:650), commit the final partial segment, and call -`Producer::finish` (:1015). Do not write a completion marker. - -Retention is writer policy. During the next segment commit, use -`Producer::pop` (:983) for expired records before closing that segment's -timeline group. Make it durable, wait the configured grace period, then delete -the corresponding segment objects. No trimming occurs after the final segment -is committed, and timeline objects are never rewritten. - -Before a DVR writer resumes input, recover its full retained timeline and -reconcile a complete media-key listing under exclusive prefix ownership. -Wait the deletion grace period from successful recovery, then delete unreferenced `groups/` objects -left by interrupted expiration or uploads. Failed or incomplete recovery must -prevent deletion. Preserve `.info` and timeline checkpoint objects. - -Keep archive policy out of protocol libraries. As the native application that -owns its storage and track choices, `moq-cli` attaches the writer to every -import path and enrolls the resulting `broadcast::Consumer` tracks. Downstream -(moq.pro) gateways attach the same writer once it ships in a release. - -## Closes - -- [#2281](https://github.com/moq-dev/moq/issues/2281) - close this issue when the quest finishes diff --git a/quest/m2/README.md b/quest/m2/README.md index f7cc1a225c..4cd3ed3c47 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -18,6 +18,7 @@ upstream release waits in [m4](/quest/m4/README.md). - [AV1 metadata separation](/quest/m2/av1-metadata.md) - retain metadata OBUs inline while evaluating separate delivery - [SEI separation](/quest/m2/sei/README.md) - retain inline SEI until measured savings or a metadata-only consumer justify a split - [Catalog track identity](/quest/m2/catalog-tracks.md) - compare immutable track definitions with explicit catalog-to-group binding +- [Archive S3 wire proof](/quest/m2/archive-s3.md) - the archive proof also runs through the S3 client against an in-process S3-compatible server - [Mobile ownership](/quest/m2/mobile-ownership.md) - decide whether Rust or platform code owns mobile capture, codecs, and rendering - [iOS capture](/quest/m2/mobile-capture-ios.md) - camera and screen capture if the mobile ownership decision selects Rust - [Android capture](/quest/m2/mobile-capture-android.md) - NDK/JNI capture using the existing codecs if mobile ownership selects Rust diff --git a/quest/m2/archive-s3.md b/quest/m2/archive-s3.md new file mode 100644 index 0000000000..d190490d8c --- /dev/null +++ b/quest/m2/archive-s3.md @@ -0,0 +1,22 @@ +# [S] Archive S3 wire proof + +## Goal + +The `moq-archive` proof (`rs/moq-archive/src/proof.rs`) also runs through +`object_store`'s S3 client against an in-process S3-compatible server, in CI +with no external network. + +## Plan + +The proof covers memory, local disk, and an S3-style listing fake +(`rs/moq-archive/src/mock.rs`), but not the S3 client: percent-encoded keys +such as `catalog%2Ejson` inside request URLs, `PutMode::Create` through +`If-None-Match`, `start-after` listing, and continuation tokens. Evaluate a +maintained in-process server (for example `s3s` with `s3s-fs`) on loopback; +refuse one that ignores conditional creates, since collisions would pass +silently. Gate it behind a dev-dependency feature if the build cost is large, +and wire it into at least the nightly workflow. + +## Related + +- [Timeline-indexed MoQ archives](/quest/m1/archive/README.md) diff --git a/rs/moq-archive/Cargo.toml b/rs/moq-archive/Cargo.toml index df94dc4d37..b5d0c96cd2 100644 --- a/rs/moq-archive/Cargo.toml +++ b/rs/moq-archive/Cargo.toml @@ -15,15 +15,22 @@ categories = ["multimedia", "filesystem"] [dependencies] bytes = { workspace = true } futures = { workspace = true } +hang = { workspace = true } +kio = { workspace = true } +moq-json = { workspace = true } +moq-mux = { workspace = true } moq-net = { workspace = true } object_store = { workspace = true } percent-encoding = { workspace = true } +quick_cache = "0.7" serde = { workspace = true } serde_json = { workspace = true } thiserror = { workspace = true } +tokio = { workspace = true, features = ["macros", "sync", "time"] } +tracing = { workspace = true } [dev-dependencies] async-trait = "0.1" object_store = { workspace = true, features = ["fs"] } tempfile = { workspace = true } -tokio = { workspace = true, features = ["macros", "rt"] } +tokio = { workspace = true, features = ["macros", "rt", "test-util"] } diff --git a/rs/moq-archive/src/error.rs b/rs/moq-archive/src/error.rs index ad372a3209..a8ef749d83 100644 --- a/rs/moq-archive/src/error.rs +++ b/rs/moq-archive/src/error.rs @@ -73,6 +73,26 @@ pub enum Error { /// `.info` JSON is malformed. #[error("json: {0}")] Json(String), + + /// Publishing the replayed archive failed. + #[error("moq: {0}")] + Moq(String), + + /// The track was already enrolled, or is the recording's own timeline. + #[error("track already enrolled: {0}")] + Enrolled(String), + + /// The source broadcast or one of its tracks failed. + #[error("source: {0}")] + Source(String), + + /// The recording's timeline could not be recovered, segmented, or published. + #[error("timeline: {0}")] + Timeline(String), + + /// The writer stopped accepting commands. + #[error("writer closed")] + Closed, } impl From for Error { @@ -84,6 +104,12 @@ impl From for Error { } } +impl From for Error { + fn from(err: moq_net::Error) -> Self { + Self::Moq(err.to_string()) + } +} + impl From for Error { fn from(err: serde_json::Error) -> Self { Self::Json(err.to_string()) diff --git a/rs/moq-archive/src/lib.rs b/rs/moq-archive/src/lib.rs index 603284461c..3edecbe87e 100644 --- a/rs/moq-archive/src/lib.rs +++ b/rs/moq-archive/src/lib.rs @@ -1,7 +1,9 @@ //! Versioned hang recording objects on any [`object_store::ObjectStore`]. //! //! The crate owns the portable layout and codecs: percent-encoded track names, `.info` JSON, -//! the binary segment envelope, and put/get/list/delete. Callers that need runtime dispatch +//! the binary segment envelope, and put/get/list/delete. A [`Writer`] records selected tracks of +//! a broadcast into those objects, and a [`Reader`] serves a recording back through a `moq_net` +//! broadcast. Callers that need runtime dispatch //! supply `Arc`; the archive API itself stays generic. //! //! Group bounds are finite inclusive ranges in first-to-last order: @@ -15,15 +17,24 @@ pub use object_store; mod error; pub mod info; +#[cfg(test)] +mod mock; mod path; +#[cfg(test)] +mod proof; +pub mod reader; +mod recover; pub mod segment; pub mod store; +pub mod writer; pub use error::{Error, Result}; pub use info::Info; pub use path::Key; +pub use reader::Reader; pub use segment::{Frame, Group, Object}; pub use store::Store; +pub use writer::Writer; /// Recording format version written into `.info` and the binary envelope. pub const VERSION: u64 = 1; diff --git a/rs/moq-archive/src/mock.rs b/rs/moq-archive/src/mock.rs new file mode 100644 index 0000000000..2d7275badf --- /dev/null +++ b/rs/moq-archive/src/mock.rs @@ -0,0 +1,272 @@ +//! A test object store: any inner store plus an operation log, injected failures, and +//! S3-style paginated listing. + +use std::sync::{Arc, Mutex}; + +use futures::stream::BoxStream; +use futures::{StreamExt, TryStreamExt}; +use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; +use object_store::memory::InMemory; +use object_store::path::Path; +use object_store::{ + CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, + PutOptions, PutPayload, PutResult, +}; + +/// S3's ListObjectsV2 page limit. +const MAX_KEYS: usize = 1000; + +/// One call made against a [`Mock`]. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Op { + Get(String), + Put(String), + Delete(String), + /// A streaming or paginated listing, with its prefix and exclusive offset. + List { + prefix: String, + offset: Option, + }, +} + +#[derive(Debug, Default)] +struct State { + ops: Vec, + /// PUTs whose path contains any of these fail. + fail_puts: Vec, + /// GETs whose path contains any of these return Not Found. + hide_gets: Vec, + /// Streaming listings end with an error after every entry. + fail_lists: bool, + /// Streaming listings yield entries in descending order, like a backend that promises none. + unordered: bool, +} + +#[derive(Debug, Clone)] +pub(crate) struct Mock { + inner: Arc, + state: Arc>, +} + +impl Mock { + pub fn new(inner: impl ObjectStore) -> Self { + Self { + inner: Arc::new(inner), + state: Default::default(), + } + } + + pub fn memory() -> Self { + Self::new(InMemory::new()) + } + + /// Share this store's objects, without its log or failures. + pub fn fork(&self) -> Self { + Self { + inner: self.inner.clone(), + state: Default::default(), + } + } + + /// Every call since the last take. + pub fn take(&self) -> Vec { + std::mem::take(&mut self.state().ops) + } + + /// Paths of every GET since the last take. + pub fn gets(&self) -> Vec { + self.take() + .into_iter() + .filter_map(|op| match op { + Op::Get(path) => Some(path), + _ => None, + }) + .collect() + } + + pub fn fail_puts(&self, pattern: &str) -> &Self { + self.state().fail_puts.push(pattern.to_string()); + self + } + + pub fn hide_gets(&self, pattern: &str) -> &Self { + self.state().hide_gets.push(pattern.to_string()); + self + } + + pub fn fail_lists(&self) -> &Self { + self.state().fail_lists = true; + self + } + + pub fn unordered(&self) -> &Self { + self.state().unordered = true; + self + } + + /// Clear every injected failure. + pub fn heal(&self) { + let mut state = self.state(); + state.fail_puts.clear(); + state.hide_gets.clear(); + state.fail_lists = false; + } + + fn state(&self) -> std::sync::MutexGuard<'_, State> { + self.state.lock().unwrap() + } + + fn log(&self, op: Op) { + self.state().ops.push(op); + } + + fn listed( + &self, + prefix: Option<&Path>, + offset: Option<&Path>, + ) -> BoxStream<'static, object_store::Result> { + self.log(Op::List { + prefix: prefix.map(ToString::to_string).unwrap_or_default(), + offset: offset.map(ToString::to_string), + }); + let listed = match offset { + Some(offset) => self.inner.list_with_offset(prefix, offset), + None => self.inner.list(prefix), + }; + let state = self.state(); + let listed = match state.unordered { + true => futures::stream::once(async move { + let mut metas: Vec = listed.try_collect().await?; + metas.sort_by(|a, b| b.location.cmp(&a.location)); + Ok::<_, object_store::Error>(futures::stream::iter(metas.into_iter().map(Ok))) + }) + .try_flatten() + .boxed(), + false => listed, + }; + match state.fail_lists { + true => listed + .chain(futures::stream::once(async { Err(unsupported("list")) })) + .boxed(), + false => listed, + } + } +} + +fn unsupported(operation: &str) -> object_store::Error { + object_store::Error::NotImplemented { + operation: operation.into(), + implementer: "Mock".into(), + } +} + +impl std::fmt::Display for Mock { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Mock({})", self.inner) + } +} + +#[async_trait::async_trait] +impl ObjectStore for Mock { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.log(Op::Put(location.to_string())); + if self.state().fail_puts.iter().any(|p| location.as_ref().contains(p)) { + return Err(unsupported("put")); + } + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.log(Op::Get(location.to_string())); + if self.state().hide_gets.iter().any(|p| location.as_ref().contains(p)) { + return Err(object_store::Error::NotFound { + path: location.to_string(), + source: "hidden".into(), + }); + } + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + let state = self.state.clone(); + let locations = locations + .inspect_ok(move |path| state.lock().unwrap().ops.push(Op::Delete(path.to_string()))) + .boxed(); + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.listed(prefix, None) + } + + fn list_with_offset( + &self, + prefix: Option<&Path>, + offset: &Path, + ) -> BoxStream<'static, object_store::Result> { + self.listed(prefix, Some(offset)) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// ListObjectsV2 semantics: a raw string prefix, lexical order, an exclusive `start-after` +/// offset, at most 1000 keys, and an opaque continuation token that resumes after the last key. +#[async_trait::async_trait] +impl PaginatedListStore for Mock { + async fn list_paginated( + &self, + prefix: Option<&str>, + opts: PaginatedListOptions, + ) -> object_store::Result { + if opts.delimiter.is_some() { + return Err(unsupported("delimiter")); + } + self.log(Op::List { + prefix: prefix.unwrap_or_default().to_string(), + offset: opts.offset.clone(), + }); + + let mut metas: Vec = self.inner.list(None).try_collect().await?; + metas.sort_by(|a, b| a.location.as_ref().cmp(b.location.as_ref())); + let after = opts.page_token.or(opts.offset); + metas.retain(|meta| { + let key = meta.location.as_ref(); + prefix.is_none_or(|prefix| key.starts_with(prefix)) && after.as_deref().is_none_or(|after| key > after) + }); + + let take = opts.max_keys.unwrap_or(MAX_KEYS).clamp(1, MAX_KEYS); + let page_token = (metas.len() > take).then(|| metas[take - 1].location.to_string()); + metas.truncate(take); + Ok(PaginatedListResult { + result: ListResult { + objects: metas, + common_prefixes: Vec::new(), + extensions: Default::default(), + }, + page_token, + }) + } +} diff --git a/rs/moq-archive/src/path.rs b/rs/moq-archive/src/path.rs index 86a8294e7b..4df4784b40 100644 --- a/rs/moq-archive/src/path.rs +++ b/rs/moq-archive/src/path.rs @@ -280,6 +280,10 @@ mod tests { parse_id("0009007199254740992"), Err(Error::Id(n)) if n == ID_MAX + 1 )); + // The largest QUIC varint still fits the 19-digit field. + let varint = (1u64 << 62) - 1; + assert!(matches!(format_id(varint), Err(Error::Id(n)) if n == varint)); + assert!(matches!(parse_id("4611686018427387903"), Err(Error::Id(n)) if n == varint)); assert!(matches!(parse_id("5"), Err(Error::Path(_)))); assert!(matches!(parse_id("000000000000000000X"), Err(Error::Path(_)))); } @@ -334,6 +338,52 @@ mod tests { assert!(matches!(Key::groups("v", ID_MAX + 1..=ID_MAX + 1), Err(Error::Id(_)))); } + #[test] + fn direct_construction_is_validated_when_serialized() { + let prefix = Path::from("rec"); + let empty = Key::Info { track: String::new() }; + assert!(matches!(empty.path(&prefix), Err(Error::Track))); + let segment = Key::Segments { + track: "t".to_string(), + segment: ID_MAX + 1, + }; + assert!(matches!(segment.path(&prefix), Err(Error::Id(_)))); + let varint = Key::Groups { + track: "v".to_string(), + range: 0..=(1 << 62) - 1, + }; + assert!(matches!(varint.path(&prefix), Err(Error::Id(_)))); + + // Everything that does serialize parses back to the same key. + for key in [ + Key::Info { + track: "a/b".to_string(), + }, + Key::Groups { + track: ".x".to_string(), + range: ID_MAX..=ID_MAX, + }, + Key::Segments { + track: "é".to_string(), + segment: 0, + }, + ] { + let path = key.path(&prefix).unwrap(); + assert_eq!(Key::parse(&prefix, &path).unwrap(), key); + } + } + + #[test] + fn keys_parse_under_an_empty_prefix() { + let key = Key::groups("catalog.json", 1..=2).unwrap(); + let path = key.path(&Path::ROOT).unwrap(); + assert_eq!( + path.as_ref(), + "catalog%2Ejson/groups/0000000000000000002.0000000000000000001" + ); + assert_eq!(Key::parse(&Path::ROOT, &path).unwrap(), key); + } + #[test] fn direct_inverted_range_is_rejected_when_serialized() { let (smallest, largest) = (2, 1); diff --git a/rs/moq-archive/src/proof.rs b/rs/moq-archive/src/proof.rs new file mode 100644 index 0000000000..4acd6238ab --- /dev/null +++ b/rs/moq-archive/src/proof.rs @@ -0,0 +1,395 @@ +//! End to end: one multi-rendition broadcast recorded by a [`Writer`], stored byte-identically on +//! every backend, and replayed exactly through a [`Reader`]. + +use std::collections::BTreeMap; +use std::time::Duration; + +use bytes::Bytes; +use futures::TryStreamExt; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::local::LocalFileSystem; +use object_store::path::Path; +use object_store::{ObjectStore, ObjectStoreExt}; + +use crate::mock::{Mock, Op}; +use crate::reader::Config as ReaderConfig; +use crate::writer::{Config, Retention}; +use crate::{Reader, Store, Writer}; + +const TIMELINE: &str = hang::timeline::DEFAULT_NAME; +const RENDITIONS: [&str; 2] = ["video/1080p", "video/360p"]; +const PACING: [&str; 3] = ["video/1080p", "video/360p", "audio"]; +const TRACKS: [&str; 5] = ["video/1080p", "video/360p", "audio", "catalog.json", "chat"]; + +/// Frames of one source group: (millisecond timestamp, payload). +type Frames = Vec<(u64, Bytes)>; + +/// Recorded segments: three 2s segments, then a one-frame tail. +const SEGMENTS: u64 = 4; + +/// The source groups of one segment: (track, sequence, frames). +/// +/// Video renditions share a 2s GOP and audio has four groups per segment. The catalog and a +/// non-media chat track publish sparsely, including a sequence skip inside one segment and an +/// empty payload. The tail keeps any single track from cutting the final segment once the others +/// end. +fn plan(segment: u64) -> Vec<(&'static str, u64, Frames)> { + let pts = segment * 2000; + let (frames, audio) = if segment + 1 < SEGMENTS { (4, 4) } else { (1, 1) }; + let mut groups = Vec::new(); + for track in RENDITIONS { + let frames = (0..frames) + .map(|i| (pts + i * 500, Bytes::from(format!("{track} {segment}.{i}")))) + .collect(); + groups.push((track, segment, frames)); + } + for sequence in segment * 4..segment * 4 + audio { + let frames = (0..2) + .map(|i| (sequence * 500 + i * 250, Bytes::from(format!("audio {sequence}.{i}")))) + .collect(); + groups.push(("audio", sequence, frames)); + } + let catalog = |sequence: u64| Bytes::from(format!(r#"{{"version":{sequence}}}"#)); + let chat = |text: &'static str| Bytes::from_static(text.as_bytes()); + match segment { + 0 => { + groups.push(("catalog.json", 0, vec![(0, catalog(0))])); + groups.push(("chat", 0, vec![(1200, chat("hello")), (1300, Bytes::new())])); + } + 2 => { + groups.push(("catalog.json", 1, vec![(4000, catalog(1))])); + groups.push(("chat", 3, vec![(4100, chat("skip"))])); + groups.push(("chat", 5, vec![(5100, chat("bye"))])); + } + _ => {} + } + groups +} + +fn ms(value: u64) -> Timestamp { + Timestamp::new(value, Timescale::MILLI).unwrap() +} + +fn write(track: &track::Producer, sequence: u64, frames: &Frames) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for (timestamp, payload) in frames { + group.write_frame(ms(*timestamp), payload.clone()).unwrap(); + } + group.finish().unwrap(); +} + +/// Let the writer read and report everything already published. Reads never touch the store. +async fn settle() { + for _ in 0..256 { + tokio::task::yield_now().await; + } +} + +/// Record every segment of [`plan`] into `store`. +async fn record(store: &Store) { + let source = broadcast::Info::new().produce(); + let tracks: BTreeMap<&str, track::Producer> = TRACKS + .iter() + .map(|&name| { + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + (name, source.create_track(name, info).unwrap()) + }) + .collect(); + + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + for name in TRACKS { + match PACING.contains(&name) { + true => control.pacing_track(name).await.unwrap(), + false => control.track(name).await.unwrap(), + } + } + let run = tokio::spawn(writer.run()); + + // A non-pacing track joins whichever segment is open when its group arrives, so publish each + // segment's pacing groups (closing the previous segment) before its sparse groups. + for segment in 0..SEGMENTS { + let (pacing, sparse): (Vec<_>, Vec<_>) = plan(segment) + .into_iter() + .partition(|(name, _, _)| PACING.contains(name)); + for (name, sequence, frames) in pacing { + write(&tracks[name], sequence, &frames); + } + settle().await; + for (name, sequence, frames) in sparse { + write(&tracks[name], sequence, &frames); + } + settle().await; + } + for track in tracks.values() { + track.finish().unwrap(); + } + source.finish(); + run.await.unwrap().unwrap(); +} + +/// Every object under the store's prefix, by path relative to the store root. +async fn objects(store: &Store) -> BTreeMap { + let metas: Vec<_> = store.inner().list(Some(store.prefix())).try_collect().await.unwrap(); + let mut objects = BTreeMap::new(); + for meta in metas { + let bytes = store.inner().get(&meta.location).await.unwrap().bytes().await.unwrap(); + objects.insert(meta.location.to_string(), bytes); + } + objects +} + +fn id(value: u64) -> String { + format!("{value:019}") +} + +/// The exact layout [`plan`] produces: no manifest, index, or completion marker. +fn layout() -> Vec { + let mut keys = Vec::new(); + let groups = |track: &str, ranges: &[(u64, u64)]| { + ranges + .iter() + .map(|&(smallest, largest)| format!("rec/{track}/groups/{}.{}", id(largest), id(smallest))) + .collect::>() + }; + keys.push("rec/audio/.info".to_string()); + keys.extend(groups("audio", &[(0, 3), (4, 7), (8, 11), (12, 12)])); + keys.push("rec/catalog%2Ejson/.info".to_string()); + keys.extend(groups("catalog%2Ejson", &[(0, 0), (1, 1)])); + keys.push("rec/chat/.info".to_string()); + keys.extend(groups("chat", &[(0, 0), (3, 5)])); + keys.push("rec/timeline%2Ez/.info".to_string()); + keys.extend((0..SEGMENTS).map(|segment| format!("rec/timeline%2Ez/segments/{}", id(segment)))); + for track in ["video%2F1080p", "video%2F360p"] { + keys.push(format!("rec/{track}/.info")); + keys.extend(groups(track, &[(0, 0), (1, 1), (2, 2), (3, 3)])); + } + keys.sort(); + keys +} + +/// FETCH one whole group, returning its frames. +async fn fetch(broadcast: &broadcast::Producer, track: &str, sequence: u64) -> moq_net::Result { + let track = broadcast.consume().track(track)?; + let mut group = track.fetch_group(sequence, group::Fetch::default()).await?; + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await? { + frames.push(( + frame.timestamp.convert(Timescale::MILLI).unwrap().value(), + frame.payload, + )); + } + Ok(frames) +} + +#[tokio::test] +async fn recordings_are_byte_identical_on_every_backend() { + let memory = Store::new(Mock::memory(), "rec"); + record(&memory).await; + let expected = objects(&memory).await; + assert_eq!(expected.keys().cloned().collect::>(), layout()); + assert_eq!( + &expected["rec/video%2F360p/.info"][..], + br#"{"version":1,"priority":0,"timescale":1000}"# + ); + + let dir = tempfile::tempdir().unwrap(); + let local = Store::new(Mock::new(LocalFileSystem::new_with_prefix(dir.path()).unwrap()), "rec"); + record(&local).await; + assert_eq!(objects(&local).await, expected, "local disk"); + + // A backend with no listing order, and a sibling recording sharing the prefix's stem. + let unordered = Mock::memory(); + unordered.unordered(); + let sibling = Store::new(unordered.clone(), "rec-other"); + record(&sibling).await; + let unordered = Store::new(unordered, "rec"); + record(&unordered).await; + assert_eq!(objects(&unordered).await, expected, "unordered listing"); + assert_eq!(objects(&sibling).await.len(), expected.len()); + + // Every group object holds exactly the source groups, in order, byte for byte. + let mut stored: BTreeMap<(String, u64), Frames> = BTreeMap::new(); + for (path, bytes) in &expected { + let key = crate::Key::parse(&Path::from("rec"), &Path::parse(path).unwrap()).unwrap(); + if let crate::Key::Groups { track, range } = key { + let object = crate::Object::decode_groups(bytes.clone(), range).unwrap(); + for group in object.groups { + let frames = group.frames.into_iter().map(|f| (f.timestamp, f.payload)).collect(); + stored.insert((track.clone(), group.sequence), frames); + } + } + } + let source: BTreeMap<(String, u64), Frames> = (0..SEGMENTS) + .flat_map(plan) + .map(|(track, sequence, frames)| ((track.to_string(), sequence), frames)) + .collect(); + assert_eq!(stored, source); +} + +#[tokio::test] +async fn fetch_replays_the_recording_and_reads_only_the_requested_rendition() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + record(&store).await; + + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(store, &broadcast, ReaderConfig::new(TIMELINE)) + .await + .unwrap(); + tokio::spawn(reader.serve()); + mock.take(); + + // Low-rendition playback never downloads the 1080p object. + for sequence in 0..SEGMENTS { + let expected = plan(sequence) + .into_iter() + .find(|(name, ..)| *name == "video/360p") + .unwrap() + .2; + assert_eq!(fetch(&broadcast, "video/360p", sequence).await.unwrap(), expected); + } + let object = + |track: &str, smallest: u64, largest: u64| format!("rec/{track}/groups/{}.{}", id(largest), id(smallest)); + // Each track request also GETs that track's `.info`, but nothing of another track. + let media = |gets: Vec, track: &str| { + let prefix = format!("rec/{track}/"); + assert!(gets.iter().all(|path| path.starts_with(&prefix)), "{gets:?}"); + gets.into_iter() + .filter(|path| path.contains("/groups/")) + .collect::>() + }; + assert_eq!( + media(mock.gets(), "video%2F360p"), + [ + object("video%2F360p", 0, 0), + object("video%2F360p", 1, 1), + object("video%2F360p", 2, 2), + object("video%2F360p", 3, 3), + ] + ); + + // Audio-only playback: four adjacent groups per GET, the rest from the cache. + for segment in 0..SEGMENTS { + for (_, sequence, frames) in plan(segment).into_iter().filter(|(name, ..)| *name == "audio") { + assert_eq!(fetch(&broadcast, "audio", sequence).await.unwrap(), frames); + } + } + assert_eq!( + media(mock.gets(), "audio"), + [ + object("audio", 0, 3), + object("audio", 4, 7), + object("audio", 8, 11), + object("audio", 12, 12), + ] + ); + + // Every other enrolled group replays its original sequence, timestamps, and payloads. + for segment in 0..SEGMENTS { + for (name, sequence, frames) in plan(segment) { + assert_eq!( + fetch(&broadcast, name, sequence).await.unwrap(), + frames, + "{name} {sequence}" + ); + } + } + for (track, sequence) in [("chat", 1), ("chat", 4), ("chat", 6), ("audio", 13), ("video/1080p", 4)] { + assert!( + matches!(fetch(&broadcast, track, sequence).await, Err(moq_net::Error::NotFound)), + "{track} {sequence} was never recorded" + ); + } + assert!( + !mock.take().iter().any(|op| matches!(op, Op::List { .. })), + "a group request resolves its object without listing" + ); +} + +#[tokio::test] +async fn an_offline_reader_follows_dvr_expiry() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let source = broadcast::Info::new().produce(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let video = source.create_track("video", info).unwrap(); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + + let frames = |sequence: u64| vec![(sequence * 1000, Bytes::from(format!("video {sequence}")))]; + let committed = |segment: u64| { + let store = store.clone(); + async move { + while store.get_segments(TIMELINE, segment).await.is_err() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + } + }; + + for sequence in 0..3 { + write(&video, sequence, &frames(sequence)); + } + committed(1).await; + + let broadcast = broadcast::Info::new().produce(); + let mut reader = Reader::open(store.clone(), &broadcast, ReaderConfig::new(TIMELINE)) + .await + .unwrap(); + tokio::spawn(reader.serve()); + assert_eq!(fetch(&broadcast, "video", 0).await.unwrap(), frames(0)); + + // The reader is offline while the DVR commits and expires several segments. + for sequence in 3..10 { + write(&video, sequence, &frames(sequence)); + } + committed(8).await; + // Segments 7 and 8 hold the 2s window. + while store.get_groups("video", 6..=6).await.is_ok() { + tokio::time::sleep(Duration::from_millis(5)).await; + } + + mock.take(); + reader.refresh().await.unwrap(); + let ops = mock.take(); + let (list, gets) = ops.split_first().unwrap(); + assert_eq!( + list, + &Op::List { + prefix: "rec/timeline%2Ez/segments".to_string(), + offset: Some(format!("rec/timeline%2Ez/segments/{}", id(1))), + } + ); + let segments: Vec<_> = (2..=8) + .map(|segment| Op::Get(format!("rec/timeline%2Ez/segments/{}", id(segment)))) + .collect(); + assert_eq!(gets, segments, "only the new timeline keys are read"); + + // Expired groups are gone from the index, so they cost no media GET. + for sequence in 0..7 { + assert!( + matches!( + fetch(&broadcast, "video", sequence).await, + Err(moq_net::Error::NotFound) + ), + "group {sequence} expired" + ); + } + let gets = mock.gets(); + assert!(!gets.iter().any(|path| path.contains("/groups/")), "{gets:?}"); + for sequence in 7..9 { + assert_eq!(fetch(&broadcast, "video", sequence).await.unwrap(), frames(sequence)); + } + + video.finish().unwrap(); + source.finish(); + run.await.unwrap().unwrap(); +} diff --git a/rs/moq-archive/src/reader/index.rs b/rs/moq-archive/src/reader/index.rs new file mode 100644 index 0000000000..bedbd7b9e2 --- /dev/null +++ b/rs/moq-archive/src/reader/index.rs @@ -0,0 +1,175 @@ +use std::collections::{BTreeMap, HashMap}; +use std::ops::{Range, RangeInclusive}; + +use hang::timeline::Record; + +use crate::path::check_range; + +/// One track's stored object for one record: its filename bounds and the exact runs it holds. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(super) struct Span { + /// Inclusive first-to-last group sequence, the object's filename bounds. + pub bounds: RangeInclusive, + /// The advertised runs, ascending and nonoverlapping; groups between them never existed. + pub runs: Vec>, +} + +impl Span { + /// Build from a record's ranges, refusing empty, reversed, out-of-range, or unordered runs. + fn new(ranges: &[hang::timeline::Range]) -> Option { + let mut runs: Vec> = Vec::with_capacity(ranges.len()); + for range in ranges { + let run = range.start..=range.end; + check_range(&run).ok()?; + if let Some(prev) = runs.last() + && run.start() <= prev.end() + { + return None; + } + runs.push(run); + } + let bounds = *runs.first()?.start()..=*runs.last()?.end(); + Some(Self { bounds, runs }) + } + + fn contains(&self, group: u64) -> bool { + self.runs.iter().any(|run| run.contains(&group)) + } +} + +/// The committed group ranges advertised by the replayed timeline window. +#[derive(Default)] +pub(super) struct Index { + /// Per track, each span keyed by its smallest group sequence. + tracks: HashMap>, + /// Per window index, the `(track, smallest)` spans that record added, so a pop can evict them. + records: BTreeMap>, +} + +impl Index { + /// Add a record's spans at window `index`. + /// + /// A track whose ranges are malformed or overlap another record's span is left out, so its + /// groups behave like ones the source never delivered; the record's other tracks stay usable. + pub fn push(&mut self, index: u64, record: &Record) { + let mut added = Vec::new(); + for (track, ranges) in &record.tracks { + let Some(span) = Span::new(ranges) else { + tracing::warn!(track, segment = record.segment, "ignoring malformed archive ranges"); + continue; + }; + + let spans = self.tracks.entry(track.clone()).or_default(); + let smallest = *span.bounds.start(); + let before = spans.range(..=span.bounds.end()).next_back(); + if before.is_some_and(|(_, prev)| prev.bounds.end() >= span.bounds.start()) { + tracing::warn!(track, segment = record.segment, "ignoring overlapping archive ranges"); + continue; + } + + spans.insert(smallest, span); + added.push((track.clone(), smallest)); + } + self.records.insert(index, added); + } + + /// Remove the records at these window indices, returning the evicted spans by track. + pub fn pop(&mut self, indices: Range) -> Vec<(String, Span)> { + let popped: Vec = self.records.range(indices).map(|(index, _)| *index).collect(); + let mut evicted = Vec::new(); + for index in popped { + for (track, smallest) in self.records.remove(&index).unwrap_or_default() { + let span = self.tracks.get_mut(&track).and_then(|spans| spans.remove(&smallest)); + if let Some(span) = span { + evicted.push((track, span)); + } + } + } + evicted + } + + /// Whether any replayed record named this track. + pub fn has_track(&self, track: &str) -> bool { + self.tracks.contains_key(track) + } + + /// The span advertising `group` on `track`, if the retained window commits it. + pub fn get(&self, track: &str, group: u64) -> Option { + let (_, span) = self.tracks.get(track)?.range(..=group).next_back()?; + span.contains(group).then(|| span.clone()) + } +} + +#[cfg(test)] +mod tests { + use hang::timeline::Range; + + use super::*; + use crate::ID_MAX; + + fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { + let mut record = Record::new(segment, segment * 1000, 1000); + for (name, ranges) in tracks { + let ranges = ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(); + record.tracks.insert(name.to_string(), ranges); + } + record + } + + #[test] + fn lookup_respects_bounds_and_internal_gaps() { + let mut index = Index::default(); + index.push(0, &record(0, &[("audio", &[(0, 2), (5, 6)]), ("video", &[(0, 0)])])); + index.push(1, &record(1, &[("audio", &[(7, 9)])])); + + assert_eq!(index.get("audio", 1).unwrap().bounds, 0..=6); + assert_eq!(index.get("audio", 6).unwrap().runs, vec![0..=2, 5..=6]); + assert!(index.get("audio", 3).is_none(), "internal gap"); + assert_eq!(index.get("audio", 8).unwrap().bounds, 7..=9); + assert!(index.get("audio", 10).is_none()); + assert!(index.get("video", 1).is_none()); + assert!(index.get("chat", 0).is_none()); + assert!(index.has_track("video")); + assert!(!index.has_track("chat")); + } + + #[test] + fn pop_evicts_only_the_popped_records() { + let mut index = Index::default(); + index.push(0, &record(0, &[("video", &[(0, 1)])])); + index.push(1, &record(1, &[("video", &[(2, 3)])])); + + let evicted = index.pop(0..1); + assert_eq!(evicted.len(), 1); + assert_eq!(evicted[0].1.bounds, 0..=1); + assert!(index.get("video", 0).is_none()); + assert!(index.get("video", 2).is_some()); + assert!(index.pop(0..1).is_empty(), "already popped"); + assert!(index.has_track("video"), "a popped track stays known"); + } + + #[test] + fn malformed_and_overlapping_tracks_are_ignored() { + let mut index = Index::default(); + index.push(0, &record(0, &[("video", &[(4, 6)]), ("audio", &[(3, 1)])])); + assert!(index.get("audio", 1).is_none(), "reversed run"); + + // Overlapping the earlier video span drops only the video entry. + index.push(1, &record(1, &[("video", &[(6, 8)]), ("audio", &[(10, 11)])])); + assert_eq!(index.get("video", 6).unwrap().bounds, 4..=6); + assert!(index.get("video", 7).is_none()); + assert!(index.get("audio", 10).is_some()); + + // Unordered runs and IDs past the recording range. + index.push( + 2, + &record(2, &[("chat", &[(5, 6), (1, 2)]), ("data", &[(ID_MAX, ID_MAX + 1)])]), + ); + assert!(index.get("chat", 5).is_none()); + assert!(index.get("data", ID_MAX).is_none()); + + // Popping a record whose track was ignored leaves the earlier span alone. + index.pop(1..2); + assert!(index.get("video", 4).is_some()); + } +} diff --git a/rs/moq-archive/src/reader/mod.rs b/rs/moq-archive/src/reader/mod.rs new file mode 100644 index 0000000000..5be88a433c --- /dev/null +++ b/rs/moq-archive/src/reader/mod.rs @@ -0,0 +1,401 @@ +//! Serve an archive's groups through a caller-supplied broadcast. +//! +//! A [`Reader`] replays the archive timeline onto its timeline track and answers FETCH for every +//! group that timeline commits. Each request maps to one range-named object, fetched with a single +//! GET, validated against the record that advertised it, and kept in a byte-bounded cache so +//! adjacent groups reuse it. A request for one track never downloads another track's object. +//! +//! The reader never infers that a recording ended: it keeps following until the caller, who may +//! know finality out of band, calls [`Reader::finish`]. +//! +//! ```no_run +//! # async fn example(store: moq_archive::Store) -> moq_archive::Result<()> { +//! use moq_archive::reader::{Config, Reader}; +//! +//! let broadcast = moq_net::broadcast::Info::new().produce(); +//! let mut reader = Reader::open(store, &broadcast, Config::new("timeline.z")).await?; +//! let serve = reader.serve(); +//! // Spawn `serve` on any executor, then call `reader.refresh()` to follow a growing archive. +//! # drop(serve); +//! # Ok(()) +//! # } +//! ``` + +mod index; + +use std::collections::HashMap; +use std::future::Future; +use std::ops::RangeInclusive; +use std::sync::{Arc, Mutex}; +use std::task::Poll; + +use futures::TryStreamExt; +use futures::future::BoxFuture; +use hang::timeline::Record; +use moq_json::window; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStore; + +use self::index::{Index, Span}; +use crate::store::list::Query; +use crate::{Error, Key, Object, Result, Store}; + +/// Configuration for [`Reader::open`]. +#[derive(Debug, Clone)] +#[non_exhaustive] +pub struct Config { + /// The timeline track name, from the catalog's `archive` entry. + pub timeline: String, + /// Upper bound on cached object bytes shared by every track. + pub cache: u64, +} + +impl Config { + /// Read the timeline named `timeline` with a 64 MiB object cache. + pub fn new(timeline: impl Into) -> Self { + Self { + timeline: timeline.into(), + cache: 64 * 1024 * 1024, + } + } + + /// Set [`cache`](Self::cache). + pub fn with_cache(mut self, bytes: u64) -> Self { + self.cache = bytes; + self + } +} + +/// Serves an archive through a [`broadcast::Producer`] and follows its timeline. +pub struct Reader { + shared: Arc>, + dynamic: broadcast::Dynamic, + timeline: track::Producer, + decoder: window::Decoder, + /// The last timeline segment replayed. + cursor: Option, +} + +struct Shared { + store: Store, + index: Mutex, + cache: quick_cache::sync::Cache, Weight>, + /// Each track's `.info`, which is immutable, so a re-requested track costs no GET. + infos: Mutex>, +} + +impl Reader { + /// Create the timeline track on `broadcast`, then replay every retained timeline object. + /// + /// Fails if the timeline's `.info` is missing or invalid, the timeline track already exists, + /// or the store cannot list the timeline. + pub async fn open(store: Store, broadcast: &broadcast::Producer, config: Config) -> Result { + let info = track_info(&store.get_info(&config.timeline).await?)?; + let timeline = broadcast.create_track(config.timeline, info)?; + + let items = usize::try_from(config.cache / (1024 * 1024)) + .unwrap_or(usize::MAX) + .max(16); + let shared = Arc::new(Shared { + store, + index: Mutex::default(), + cache: quick_cache::sync::Cache::with_weighter(items, config.cache, Weight), + infos: Mutex::default(), + }); + + let mut reader = Self { + shared, + dynamic: broadcast.dynamic(), + timeline, + decoder: window::Decoder::new(window::ConsumerConfig::default().with_compression(true)), + cursor: None, + }; + reader.refresh().await?; + Ok(reader) + } + + /// Replay timeline objects committed since the last refresh. + /// + /// Lists timeline keys after the last replayed segment, then GETs them in segment order. Each + /// stored group opens with a window checkpoint, so a segment that expired or is malformed is + /// skipped and the next one recovers the retained window, evicting whatever it popped. A + /// malformed newest object is retried on the next refresh. + pub async fn refresh(&mut self) -> Result<()> { + let timeline = self.timeline.name().to_string(); + let mut query = Query::segments(&timeline)?; + if let Some(cursor) = self.cursor { + query = query.after(&Key::segments(&timeline, cursor)?)?; + } + + let entries: Vec<_> = self.shared.store.list(&query).try_collect().await?; + let mut segments: Vec = entries + .into_iter() + .filter_map(|entry| match entry.key { + Key::Segments { segment, .. } => Some(segment), + _ => None, + }) + .filter(|segment| self.cursor.is_none_or(|cursor| *segment > cursor)) + .collect(); + segments.sort_unstable(); + segments.dedup(); + + for segment in segments { + match self.shared.store.get_segments(&timeline, segment).await { + Ok(object) => self.replay(&object)?, + Err(Error::Store(err)) => return Err(Error::Store(err)), + Err(err) => { + tracing::warn!(segment, %err, "skipping unreadable timeline segment"); + continue; + } + } + self.cursor = Some(segment); + } + + Ok(()) + } + + /// Finish the replayed timeline track once the caller knows the recording ended. + /// + /// Following stops; groups already committed stay servable through [`serve`](Self::serve). + pub fn finish(self) -> Result<()> { + Ok(self.timeline.finish()?) + } + + /// Answer track and group requests on the broadcast until it closes. + /// + /// The future is independent of this reader, so it keeps serving after + /// [`finish`](Self::finish); run it on any executor. + pub fn serve(&self) -> impl Future + Send + 'static + where + T: 'static, + { + serve(self.shared.clone(), self.dynamic.clone()) + } + + /// Index one timeline object's groups, then republish each on the timeline track. + fn replay(&mut self, object: &Object) -> Result<()> { + for stored in &object.groups { + let mut group = self.decoder.group(); + let decoded = stored.frames.iter().try_for_each(|frame| group.decode(&frame.payload)); + + // Apply whatever decoded, even from a group that failed partway: the next group's + // checkpoint restates the window regardless. + let mut index = self.shared.index.lock().unwrap(); + while let Some(event) = group.next_event() { + match event { + window::Event::Push { index: at, value } => index.push(at, &value), + window::Event::Pop(range) => { + for (track, span) in index.pop(range) { + if let Ok(key) = Key::groups(track, span.bounds) { + self.shared.cache.remove(&key); + } + } + } + _ => {} + } + } + drop(index); + + if let Err(err) = decoded { + tracing::warn!(group = stored.sequence, %err, "skipping malformed timeline group"); + continue; + } + + let mut producer = match self.timeline.create_group(stored.sequence.into()) { + Ok(producer) => producer, + Err(moq_net::Error::Duplicate) => continue, + Err(err) => return Err(err.into()), + }; + let timescale = producer.timescale(); + for frame in &stored.frames { + producer.write_frame(timestamp(frame.timestamp, timescale)?, frame.payload.clone())?; + } + producer.finish()?; + } + Ok(()) + } +} + +/// Serve requested tracks until the broadcast closes. +async fn serve(shared: Arc>, mut dynamic: broadcast::Dynamic) { + let mut tracks = kio::Tasks::new(); + kio::wait(|waiter| { + loop { + match dynamic.poll_requested_track(waiter) { + Poll::Ready(Ok(request)) => tracks.push(task(Box::pin(serve_track(shared.clone(), request)))), + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } + } + let _ = tracks.poll(waiter); + Poll::Pending + }) + .await +} + +/// Accept a track the timeline names, then serve its group requests until nobody uses it. +async fn serve_track(shared: Arc>, request: track::Request) { + let name = request.name().to_string(); + if !shared.index.lock().unwrap().has_track(&name) { + request.reject(moq_net::Error::NotFound); + return; + } + + let cached = shared.infos.lock().unwrap().get(&name).cloned(); + let info = match cached { + Some(info) => info, + None => match shared.store.get_info(&name).await.and_then(|info| track_info(&info)) { + Ok(info) => { + shared.infos.lock().unwrap().insert(name.clone(), info.clone()); + info + } + Err(err) => { + tracing::warn!(track = name, %err, "archived track has no usable .info"); + request.reject(moq_net::Error::NotFound); + return; + } + }, + }; + + let timescale = info.timescale; + let dynamic = request.dynamic(); + let _producer = request.accept(info); + let mut groups = kio::Tasks::new(); + + kio::wait(|waiter| { + loop { + match dynamic.poll_requested_group(waiter) { + Poll::Ready(Ok(request)) => { + let serve = serve_group(shared.clone(), name.clone(), timescale, request); + groups.push(task(Box::pin(serve))); + } + Poll::Ready(Err(_)) => return Poll::Ready(()), + Poll::Pending => break, + } + } + match groups.poll(waiter).is_ready() && dynamic.poll_unused(waiter).is_ready() { + true => Poll::Ready(()), + false => Poll::Pending, + } + }) + .await +} + +/// Answer one group request from its range-named object, or reject it as never delivered. +async fn serve_group( + shared: Arc>, + track: String, + timescale: Timescale, + request: group::Request, +) { + let sequence = request.sequence(); + let span = shared.index.lock().unwrap().get(&track, sequence); + let Some(span) = span else { + request.reject(moq_net::Error::NotFound); + return; + }; + + let object = match Key::groups(track.clone(), span.bounds.clone()) { + Ok(key) => { + shared + .cache + .get_or_insert_async(&key, load(&shared.store, &track, &span)) + .await + } + Err(err) => Err(err), + }; + let object = match object { + Ok(object) => object, + Err(err) => { + tracing::warn!(track, sequence, %err, "archived group is unavailable"); + request.reject(moq_net::Error::NotFound); + return; + } + }; + + // Load validated the table against the span, so every advertised group is present. + let Ok(position) = object.groups.binary_search_by_key(&sequence, |group| group.sequence) else { + request.reject(moq_net::Error::NotFound); + return; + }; + let frames = &object.groups[position].frames; + let Some(frames) = usize::try_from(request.frame_start()) + .ok() + .and_then(|start| frames.get(start..)) + else { + request.reject(moq_net::Error::NotFound); + return; + }; + + let start = request.frame_start(); + let Ok(mut producer) = request.accept(None) else { + return; + }; + let written = producer.start_at(start).and_then(|()| { + frames.iter().try_for_each(|frame| { + let timestamp = timestamp(frame.timestamp, timescale).map_err(|_| moq_net::Error::TimestampMismatch)?; + producer.write_frame(timestamp, frame.payload.clone()) + }) + }); + match written { + Ok(()) => { + let _ = producer.finish(); + } + Err(err) => { + let _ = producer.abort(err); + } + } +} + +/// GET a span's object and require its table to hold exactly the advertised groups. +async fn load(store: &Store, track: &str, span: &Span) -> Result> { + let object = store.get_groups(track, span.bounds.clone()).await?; + check_runs(&object, &span.runs)?; + Ok(Arc::new(object)) +} + +/// Refuse a table whose sequences differ from the advertised runs. +fn check_runs(object: &Object, runs: &[RangeInclusive]) -> Result<()> { + let mut groups = object.groups.iter(); + for run in runs { + for sequence in run.clone() { + if groups.next().map(|group| group.sequence) != Some(sequence) { + return Err(Error::Table); + } + } + } + match groups.next() { + Some(_) => Err(Error::Table), + None => Ok(()), + } +} + +fn track_info(info: &crate::Info) -> Result { + let timescale = Timescale::new(info.timescale).map_err(|_| Error::Timescale(info.timescale))?; + Ok(track::Info::default() + .with_timescale(timescale) + .with_priority(info.priority)) +} + +fn timestamp(value: u64, timescale: Timescale) -> Result { + Timestamp::new(value, timescale).map_err(|_| Error::Id(value)) +} + +/// Adapt a boxed future into a [`kio::Tasks`] task. +fn task(mut future: BoxFuture<'static, ()>) -> impl FnMut(&kio::Waiter) -> Poll<()> + Send { + move |waiter| waiter.poll_future(future.as_mut()) +} + +/// Weighs a cached object by its payload bytes plus a small per-frame overhead. +#[derive(Clone)] +struct Weight; + +impl quick_cache::Weighter> for Weight { + fn weight(&self, _key: &Key, object: &Arc) -> u64 { + let frames = object.groups.iter().flat_map(|group| &group.frames); + frames.map(|frame| frame.payload.len() as u64 + 32).sum::() + 64 + } +} + +#[cfg(test)] +mod tests; diff --git a/rs/moq-archive/src/reader/tests.rs b/rs/moq-archive/src/reader/tests.rs new file mode 100644 index 0000000000..c560788199 --- /dev/null +++ b/rs/moq-archive/src/reader/tests.rs @@ -0,0 +1,552 @@ +use bytes::Bytes; +use hang::timeline::{Range, Record}; +use moq_json::window; +use moq_net::{broadcast, group}; +use object_store::ObjectStoreExt; + +use super::*; +use crate::mock::{Mock, Op}; +use crate::segment::{Frame, Group}; +use crate::{ID_MAX, Info}; + +const TIMELINE: &str = "timeline.z"; + +/// Writes archive objects the way a recording writer lays them out. +struct Archive { + store: Store, + encoder: window::Encoder, + /// Next timeline group sequence. + sequence: u64, +} + +impl Archive { + async fn new() -> Self { + Self::with_op_ratio(0).await + } + + /// `op_ratio` 0 makes every window edit its own checkpoint group. + async fn with_op_ratio(op_ratio: u32) -> Self { + let store = Store::new(Mock::memory(), "rec"); + store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(op_ratio); + Self { + store, + encoder: window::Encoder::new(config), + sequence: 0, + } + } + + /// Store one track object holding `groups` of `(sequence, frame count)`. + async fn media(&self, track: &str, groups: &[(u64, usize)]) { + self.store.put_info(track, &Info::new(1, 1000).unwrap()).await.unwrap(); + let groups = groups + .iter() + .map(|&(sequence, count)| Group { + sequence, + frames: (0..count).map(|i| frame(track, sequence, i)).collect(), + }) + .collect(); + self.store.put_groups(track, &Object { groups }).await.unwrap(); + } + + /// Push `record`, pop `pop` records, and return the resulting timeline groups. + fn edit(&mut self, record: &Record, pop: u64) -> Vec { + let mut frames = Vec::new(); + let pending = self.encoder.push(record).unwrap(); + frames.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = self.encoder.pop(pop).unwrap() { + frames.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + + let mut groups: Vec = Vec::new(); + for (keyframe, payload) in frames { + if keyframe { + self.sequence += 1; + } + if groups.last().is_none_or(|group| group.sequence != self.sequence - 1) { + groups.push(Group { + sequence: self.sequence - 1, + frames: Vec::new(), + }); + } + let timestamp = record.pts; + groups.last_mut().unwrap().frames.push(Frame { timestamp, payload }); + } + groups + } + + async fn timeline(&self, segment: u64, groups: Vec) { + self.store + .put_segments(TIMELINE, segment, &Object { groups }) + .await + .unwrap(); + } + + /// Commit `record` as timeline segment `record.segment`, popping `pop` older records. + async fn commit(&mut self, record: &Record, pop: u64) { + let groups = self.edit(record, pop); + self.timeline(record.segment, groups).await; + } + + async fn raw(&self, key: &Key, bytes: &'static [u8]) { + let path = self.store.path(key).unwrap(); + self.store + .inner() + .put(&path, Bytes::from_static(bytes).into()) + .await + .unwrap(); + } +} + +fn frame(track: &str, sequence: u64, index: usize) -> Frame { + Frame { + timestamp: sequence * 100 + index as u64, + payload: Bytes::from(format!("{track}/{sequence}/{index}")), + } +} + +fn record(segment: u64, tracks: &[(&str, &[(u64, u64)])]) -> Record { + let mut record = Record::new(segment, segment * 2000, 2000); + for (name, ranges) in tracks { + let ranges = ranges.iter().map(|&(start, end)| Range::new(start, end)).collect(); + record.tracks.insert(name.to_string(), ranges); + } + record +} + +async fn open(archive: &Archive) -> (broadcast::Producer, Reader) { + let broadcast = broadcast::Info::new().produce(); + let reader = Reader::open(archive.store.clone(), &broadcast, Config::new(TIMELINE)) + .await + .unwrap(); + tokio::spawn(reader.serve()); + (broadcast, reader) +} + +/// FETCH one group starting at `frame_start`, returning each frame's timestamp and payload. +async fn fetch( + broadcast: &broadcast::Producer, + track: &str, + sequence: u64, + frame_start: u64, +) -> std::result::Result, moq_net::Error> { + let track = broadcast.consume().track(track)?; + let options = group::Fetch::default().with_frame_start(frame_start); + let mut group = track.fetch_group(sequence, options).await?; + let mut frames = Vec::new(); + while let Some(frame) = group.read_frame().await? { + frames.push((frame.timestamp.value(), frame.payload)); + } + Ok(frames) +} + +fn expected(track: &str, sequence: u64, frames: std::ops::Range) -> Vec<(u64, Bytes)> { + frames + .map(|i| { + let frame = frame(track, sequence, i); + (frame.timestamp, frame.payload) + }) + .collect() +} + +#[tokio::test] +async fn fetch_replays_original_groups() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 3), (1, 2)]).await; + archive.media("audio", &[(0, 1), (1, 1), (5, 2)]).await; + archive.media("video", &[(2, 2)]).await; + archive + .commit(&record(0, &[("video", &[(0, 1)]), ("audio", &[(0, 1), (5, 5)])]), 0) + .await; + archive.commit(&record(1, &[("video", &[(2, 2)])]), 0).await; + + let (broadcast, _reader) = open(&archive).await; + + assert_eq!( + fetch(&broadcast, "video", 0, 0).await.unwrap(), + expected("video", 0, 0..3) + ); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..2) + ); + assert_eq!( + fetch(&broadcast, "audio", 5, 0).await.unwrap(), + expected("audio", 5, 0..2) + ); + + // A requested frame_start keeps the original frame indices and timestamps. + let tail = fetch(&broadcast, "video", 1, 1).await.unwrap(); + assert_eq!(tail, expected("video", 1, 1..2)); + + let track = broadcast.consume().track("video").unwrap(); + let info = track.query().await.unwrap(); + assert_eq!(info.priority, 1); + assert_eq!(info.timescale.as_u64(), 1000); +} + +#[tokio::test] +async fn requests_download_only_their_object() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.media("audio", &[(0, 1), (1, 1), (2, 1)]).await; + archive + .commit(&record(0, &[("video", &[(0, 0)]), ("audio", &[(0, 2)])]), 0) + .await; + + let (broadcast, _reader) = open(&archive).await; + archive.store.inner().take(); + + for sequence in 0..3 { + assert_eq!( + fetch(&broadcast, "audio", sequence, 0).await.unwrap(), + expected("audio", sequence, 0..1) + ); + } + + let gets = archive.store.inner().gets(); + assert_eq!( + gets, + vec![ + "rec/audio/.info".to_string(), + "rec/audio/groups/0000000000000000002.0000000000000000000".to_string(), + ], + "adjacent groups reuse one GET and never touch video" + ); +} + +#[tokio::test] +async fn unadvertised_or_unreadable_groups_are_not_found() { + let mut archive = Archive::new().await; + archive.media("audio", &[(0, 1), (1, 1), (4, 1)]).await; + // The object holds group 3, which its record does not advertise. + archive.media("video", &[(0, 1), (3, 1)]).await; + archive.raw(&Key::groups("bad", 0..=0).unwrap(), b"\x02garbage").await; + archive + .store + .put_info("bad", &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + archive + .store + .put_info("missing", &Info::new(0, 1000).unwrap()) + .await + .unwrap(); + archive + .commit( + &record( + 0, + &[ + ("audio", &[(0, 1), (4, 4)]), + ("video", &[(0, 0), (2, 3)]), + ("bad", &[(0, 0)]), + ("missing", &[(0, 0)]), + ], + ), + 0, + ) + .await; + + let (broadcast, _reader) = open(&archive).await; + + let not_found = |result: std::result::Result, moq_net::Error>| { + assert!(matches!(result, Err(moq_net::Error::NotFound)), "{result:?}"); + }; + not_found(fetch(&broadcast, "audio", 2, 0).await); // internal gap + not_found(fetch(&broadcast, "audio", 5, 0).await); // past the last range + not_found(fetch(&broadcast, "video", 0, 0).await); // table disagrees with the record + not_found(fetch(&broadcast, "bad", 0, 0).await); // unknown envelope version + not_found(fetch(&broadcast, "missing", 0, 0).await); // no object + not_found(fetch(&broadcast, "chat", 0, 0).await); // track the timeline never named + + // Siblings remain usable. + assert_eq!( + fetch(&broadcast, "audio", 4, 0).await.unwrap(), + expected("audio", 4, 0..1) + ); +} + +#[tokio::test] +async fn refresh_follows_new_segments_and_pops() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1), (1, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 1)])]), 0).await; + + let (broadcast, mut reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 0, 0).await.is_ok()); + + archive.media("video", &[(2, 1)]).await; + // Refreshing before the commit finds nothing new; Not Found is not finality. + reader.refresh().await.unwrap(); + assert!(matches!( + fetch(&broadcast, "video", 2, 0).await, + Err(moq_net::Error::NotFound) + )); + + archive.commit(&record(1, &[("video", &[(2, 2)])]), 1).await; + reader.refresh().await.unwrap(); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..1) + ); + + // The popped record's groups are gone, although its object is still stored and was cached. + archive.store.inner().take(); + assert!(matches!( + fetch(&broadcast, "video", 1, 0).await, + Err(moq_net::Error::NotFound) + )); + assert_eq!(archive.store.inner().gets(), Vec::::new()); +} + +#[tokio::test] +async fn a_missing_timeline_segment_recovers_from_the_next_checkpoint() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.media("video", &[(1, 1)]).await; + archive.media("video", &[(2, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + // Segment 1 is committed to the window but its timeline object is unreadable. + archive.edit(&record(1, &[("video", &[(1, 1)])]), 0); + archive.raw(&Key::segments(TIMELINE, 1).unwrap(), b"\x01").await; + archive.commit(&record(2, &[("video", &[(2, 2)])]), 0).await; + + let (broadcast, _reader) = open(&archive).await; + + // Segment 2's checkpoint restates record 1, so its group is still served. + for sequence in 0..3 { + assert_eq!( + fetch(&broadcast, "video", sequence, 0).await.unwrap(), + expected("video", sequence, 0..1) + ); + } +} + +#[tokio::test] +async fn a_missing_tail_is_retried_on_the_next_refresh() { + let mut archive = Archive::new().await; + for segment in 0..3 { + archive.media("video", &[(segment, 1)]).await; + archive + .commit(&record(segment, &[("video", &[(segment, segment)])]), 0) + .await; + } + // Listed, but not yet readable. + archive.store.inner().hide_gets("segments/0000000000000000002"); + + let (broadcast, mut reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); + assert!(matches!( + fetch(&broadcast, "video", 2, 0).await, + Err(moq_net::Error::NotFound) + )); + + archive.store.inner().heal(); + archive.store.inner().take(); + reader.refresh().await.unwrap(); + assert_eq!( + archive.store.inner().take(), + [ + Op::List { + prefix: "rec/timeline%2Ez/segments".to_string(), + offset: Some("rec/timeline%2Ez/segments/0000000000000000001".to_string()), + }, + Op::Get("rec/timeline%2Ez/segments/0000000000000000002".to_string()), + ], + "the cursor stays before the missing tail" + ); + assert_eq!( + fetch(&broadcast, "video", 2, 0).await.unwrap(), + expected("video", 2, 0..1) + ); +} + +#[tokio::test] +async fn following_lists_only_new_timeline_keys() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + let (broadcast, mut reader) = open(&archive).await; + + for segment in 1..4 { + // A media object stored ahead of its commit is invisible until the timeline names it. + archive.media("video", &[(segment, 1)]).await; + reader.refresh().await.unwrap(); + assert!(matches!( + fetch(&broadcast, "video", segment, 0).await, + Err(moq_net::Error::NotFound) + )); + + archive + .commit(&record(segment, &[("video", &[(segment, segment)])]), 0) + .await; + archive.store.inner().take(); + reader.refresh().await.unwrap(); + let previous = format!("rec/timeline%2Ez/segments/{:019}", segment - 1); + assert_eq!( + archive.store.inner().take(), + [ + Op::List { + prefix: "rec/timeline%2Ez/segments".to_string(), + offset: Some(previous), + }, + Op::Get(format!("rec/timeline%2Ez/segments/{segment:019}")), + ], + "following segment {segment} touches no media listing" + ); + assert_eq!( + fetch(&broadcast, "video", segment, 0).await.unwrap(), + expected("video", segment, 0..1) + ); + } +} + +#[tokio::test] +async fn an_unordered_listing_replays_in_segment_order() { + let mut archive = Archive::new().await; + archive.store.inner().unordered(); + for segment in 0..4 { + archive.media("video", &[(segment, 1)]).await; + let pop = u64::from(segment >= 2); + archive + .commit(&record(segment, &[("video", &[(segment, segment)])]), pop) + .await; + } + + let (broadcast, mut reader) = open(&archive).await; + + // The cursor is the newest segment, not the last one listed. + archive.media("video", &[(4, 1)]).await; + archive.commit(&record(4, &[("video", &[(4, 4)])]), 1).await; + archive.store.inner().take(); + reader.refresh().await.unwrap(); + assert_eq!( + archive.store.inner().gets(), + ["rec/timeline%2Ez/segments/0000000000000000004"] + ); + + for segment in 0..3 { + assert!(matches!( + fetch(&broadcast, "video", segment, 0).await, + Err(moq_net::Error::NotFound) + )); + } + for segment in 3..5 { + assert_eq!( + fetch(&broadcast, "video", segment, 0).await.unwrap(), + expected("video", segment, 0..1) + ); + } +} + +#[tokio::test] +async fn a_track_without_usable_info_is_not_found() { + let mut archive = Archive::new().await; + archive.media("audio", &[(0, 1)]).await; + let object = |track| Object { + groups: vec![Group { + sequence: 0, + frames: vec![frame(track, 0, 0)], + }], + }; + archive.store.put_groups("bare", &object("bare")).await.unwrap(); + archive.store.put_groups("future", &object("future")).await.unwrap(); + archive + .raw( + &Key::info("future").unwrap(), + br#"{"version":2,"priority":0,"timescale":1000}"#, + ) + .await; + archive + .commit( + &record(0, &[("audio", &[(0, 0)]), ("bare", &[(0, 0)]), ("future", &[(0, 0)])]), + 0, + ) + .await; + + let (broadcast, _reader) = open(&archive).await; + for track in ["bare", "future"] { + assert!( + matches!(fetch(&broadcast, track, 0, 0).await, Err(moq_net::Error::NotFound)), + "{track}" + ); + } + assert_eq!( + fetch(&broadcast, "audio", 0, 0).await.unwrap(), + expected("audio", 0, 0..1) + ); +} + +#[tokio::test] +async fn multi_frame_timeline_groups_decode() { + let mut archive = Archive::with_op_ratio(64).await; + archive.media("video", &[(0, 1)]).await; + archive.media("video", &[(1, 1)]).await; + let mut groups = archive.edit(&record(0, &[("video", &[(0, 0)])]), 0); + let ops = archive.edit(&record(1, &[("video", &[(1, 1)])]), 0); + assert!( + ops[0].sequence == groups[0].sequence, + "the push stays in the open group" + ); + groups[0].frames.extend(ops.into_iter().flat_map(|group| group.frames)); + archive.timeline(0, groups).await; + + let (broadcast, _reader) = open(&archive).await; + assert!(fetch(&broadcast, "video", 1, 0).await.is_ok()); +} + +#[tokio::test] +async fn timeline_track_is_republished_and_finished_on_request() { + let mut archive = Archive::new().await; + archive.media("video", &[(0, 1)]).await; + archive.commit(&record(0, &[("video", &[(0, 0)])]), 0).await; + archive.commit(&record(1, &[]), 0).await; + + let (broadcast, reader) = open(&archive).await; + let track = broadcast.consume().track(TIMELINE).unwrap(); + let subscriber = track.subscribe(None).await.unwrap(); + let mut timeline = + window::Consumer::::new(subscriber, window::ConsumerConfig::default().with_compression(true)); + + reader.finish().unwrap(); + + let mut segments = Vec::new(); + while let Some(event) = timeline.next().await.unwrap() { + if let window::Event::Push { value, .. } = event { + segments.push(value.segment); + } + } + assert_eq!(segments, vec![0, 1]); +} + +#[tokio::test] +async fn open_requires_the_timeline_info() { + let store = Store::new(Mock::memory(), "rec"); + let broadcast = broadcast::Info::new().produce(); + let result = Reader::open(store, &broadcast, Config::new(TIMELINE)).await; + assert!(matches!(result, Err(Error::NotFound(_)))); +} + +#[test] +fn check_runs_requires_the_exact_sequences() { + let object = Object { + groups: [0, 1, 4, ID_MAX] + .into_iter() + .map(|sequence| Group { + sequence, + frames: Vec::new(), + }) + .collect(), + }; + check_runs(&object, &[0..=1, 4..=4, ID_MAX..=ID_MAX]).unwrap(); + assert!(check_runs(&object, &[0..=1, 4..=4]).is_err(), "extra group"); + assert!( + check_runs(&object, &[0..=4, ID_MAX..=ID_MAX]).is_err(), + "missing groups" + ); + assert!(check_runs(&object, &[0..=1, 3..=4, ID_MAX..=ID_MAX]).is_err()); +} diff --git a/rs/moq-archive/src/recover.rs b/rs/moq-archive/src/recover.rs new file mode 100644 index 0000000000..f7ccfd5ace --- /dev/null +++ b/rs/moq-archive/src/recover.rs @@ -0,0 +1,185 @@ +//! Recover a recording so a restarted [`Writer`](crate::Writer) continues it. + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::ops::RangeInclusive; + +use futures::TryStreamExt; +use hang::timeline::Record; +use moq_json::window::{self, Checkpoint}; +use object_store::ObjectStore; + +use crate::store::list::Query; +use crate::{Error, Key, Object, Result, Store}; + +/// What a restarted writer continues from. +pub(crate) struct Recovery { + /// The retained timeline window, or `None` when no timeline object exists. + pub checkpoint: Option>, + /// The next stored timeline group sequence, so the stored numbering keeps increasing. + pub sequence: u64, + /// Per track, the largest stored group; new groups must exceed it so no object overlaps. + pub floors: HashMap, + /// Stored group objects no retained record references. Only collected for a DVR. + pub orphans: Vec, +} + +/// List the whole recording, then replay its timeline from a retained checkpoint. +/// +/// A DVR (`complete`) replays far enough back to recover every retained record and reports the +/// unreferenced group objects; an archive only needs the newest checkpoint. Any listing, GET, or +/// decode failure fails recovery, so a caller never acts on a partial view. +pub(crate) async fn recover(store: &Store, timeline: &str, complete: bool) -> Result { + let entries: Vec<_> = store.list(&Query::new()).try_collect().await?; + let mut segments = Vec::new(); + let mut groups = Vec::new(); + let mut floors = HashMap::new(); + for entry in entries { + match entry.key { + Key::Segments { track, segment } if track == timeline => segments.push(segment), + Key::Groups { track, range } if track != timeline => { + raise(&mut floors, &track, *range.end()); + groups.push(Key::Groups { track, range }); + } + _ => {} + } + } + segments.sort_unstable(); + + let (checkpoint, sequence) = match (segments.first(), segments.last()) { + (Some(&first), Some(&last)) => { + if segments.len() as u64 != last - first + 1 { + return Err(Error::Timeline(format!( + "timeline segments {first}..={last} are not contiguous" + ))); + } + let (checkpoint, sequence) = replay(store, timeline, first..=last, complete).await?; + // The newest object is segment `last`, so the window must end on the next one. + // A shorter window would resume onto that segment and collide with it. + let next = last.checked_add(1).ok_or(Error::Overflow)?; + if checkpoint.range.end != next { + return Err(Error::Timeline(format!( + "recovered window ends at {}, not segment {next}", + checkpoint.range.end + ))); + } + (Some(checkpoint), sequence) + } + _ => (None, 0), + }; + + let mut referenced = HashSet::new(); + for record in checkpoint.iter().flat_map(|checkpoint| &checkpoint.records) { + for (track, ranges) in &record.tracks { + if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { + raise(&mut floors, track, last.end); + referenced.insert(Key::groups(track.clone(), first.start..=last.end)?); + } + } + } + let orphans = match complete { + true => groups.into_iter().filter(|key| !referenced.contains(key)).collect(), + false => Vec::new(), + }; + + Ok(Recovery { + checkpoint, + sequence, + floors, + orphans, + }) +} + +fn raise(floors: &mut HashMap, track: &str, group: u64) { + let floor = floors.entry(track.to_string()).or_insert(group); + *floor = (*floor).max(group); +} + +/// Replay `segments` from the newest checkpoint that restates every record `complete` needs. +/// +/// Returns the retained window and the next timeline group sequence. +async fn replay( + store: &Store, + timeline: &str, + segments: RangeInclusive, + complete: bool, +) -> Result<(Checkpoint, u64)> { + // Every object opens with a checkpoint. The newest one's offset bounds what is still retained, + // so walk back until a checkpoint restates from there. + let mut objects = VecDeque::new(); + let mut needed = None; + for segment in segments.rev() { + let object = store.get_segments(timeline, segment).await?; + let (offset, start) = checkpoint(&object)?; + let needed = *needed.get_or_insert(offset); + objects.push_front(object); + if !complete || start <= needed { + break; + } + } + + let mut decoder = decoder(); + let mut records = BTreeMap::new(); + for object in &objects { + for stored in &object.groups { + let mut group = decoder.group(); + for frame in &stored.frames { + group.decode(&frame.payload).map_err(json_error)?; + } + } + while let Some(event) = decoder.next_event() { + match event { + window::Event::Push { index, value } => { + records.insert(index, value); + } + window::Event::Pop(range) | window::Event::Skip(range) => { + let tail = records.split_off(&range.end); + records.retain(|index, _| *index < range.start); + records.extend(tail); + } + _ => {} + } + } + } + + // The records must be a contiguous suffix of the window, and all of it for a DVR. + let range = decoder.range(); + let start = records.keys().next().copied().unwrap_or(range.end); + if records.len() as u64 != range.end - start || (complete && start != range.start) { + return Err(Error::Timeline(format!( + "cannot recover timeline window {range:?} from the retained checkpoints" + ))); + } + + let last = objects.back().and_then(|object| object.groups.last()); + let sequence = last.map_or(Ok(0), |group| group.sequence.checked_add(1).ok_or(Error::Overflow))?; + let checkpoint = Checkpoint { + range, + records: records.into_values().collect(), + }; + Ok((checkpoint, sequence)) +} + +/// The retained offset and first restated index of the checkpoint opening `object`. +fn checkpoint(object: &Object) -> Result<(u64, u64)> { + let frame = object + .groups + .first() + .and_then(|group| group.frames.first()) + .ok_or_else(|| Error::Timeline("timeline object has no checkpoint".into()))?; + let mut decoder = decoder(); + decoder.group().decode(&frame.payload).map_err(json_error)?; + let offset = decoder.range().start; + let start = match decoder.next_event() { + Some(window::Event::Skip(skipped)) => skipped.end, + _ => offset, + }; + Ok((offset, start)) +} + +fn decoder() -> window::Decoder { + window::Decoder::new(window::ConsumerConfig::default().with_compression(true)) +} + +fn json_error(err: moq_json::Error) -> Error { + Error::Timeline(err.to_string()) +} diff --git a/rs/moq-archive/src/segment.rs b/rs/moq-archive/src/segment.rs index c2fd68c5aa..6b844dcdee 100644 --- a/rs/moq-archive/src/segment.rs +++ b/rs/moq-archive/src/segment.rs @@ -346,6 +346,39 @@ mod tests { assert!(matches!(Object::decode(bytes.freeze()), Err(Error::Id(_)))); } + /// A table with one frameless group per delta. + fn deltas(deltas: &[u64]) -> Bytes { + let mut bytes = BytesMut::new(); + write_varint(&mut bytes, 1).unwrap(); + write_varint(&mut bytes, deltas.len() as u64).unwrap(); + for &delta in deltas { + write_varint(&mut bytes, delta).unwrap(); + write_varint(&mut bytes, 0).unwrap(); + } + bytes.freeze() + } + + fn sequences(bytes: Bytes) -> Result> { + Ok(Object::decode(bytes)? + .groups + .iter() + .map(|group| group.sequence) + .collect()) + } + + #[test] + fn delta_extremes() { + let varint = (1u64 << 62) - 1; + assert_eq!(sequences(deltas(&[0, 0, 0])).unwrap(), vec![0, 1, 2]); + assert_eq!(sequences(deltas(&[0, 9, 0])).unwrap(), vec![0, 10, 11]); + assert_eq!(sequences(deltas(&[0, ID_MAX - 1])).unwrap(), vec![0, ID_MAX]); + assert_eq!(sequences(deltas(&[ID_MAX])).unwrap(), vec![ID_MAX]); + assert!(matches!(sequences(deltas(&[0, ID_MAX])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[ID_MAX + 1])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[varint])), Err(Error::Id(_)))); + assert!(matches!(sequences(deltas(&[ID_MAX, varint])), Err(Error::Id(_)))); + } + #[test] fn timestamp_bounds() { assert!( @@ -413,6 +446,34 @@ mod tests { assert!(matches!(Object::decode(&b"\x01"[..]), Err(Error::Table))); } + #[test] + fn frame_offsets_must_tile_the_payload() { + // One group of two frames at (offset, length), followed by `payload` bytes. + let table = |frames: [(u64, u64); 2], payload: usize| { + let mut bytes = BytesMut::new(); + for value in [1, 1, 0, 2] { + write_varint(&mut bytes, value).unwrap(); + } + for (offset, length) in frames { + for value in [0, offset, length] { + write_varint(&mut bytes, value).unwrap(); + } + } + bytes.extend_from_slice(&vec![b'x'; payload]); + Object::decode(bytes.freeze()) + }; + assert!(table([(0, 1), (1, 2)], 3).is_ok()); + assert!(matches!(table([(0, 1), (0, 2)], 3), Err(Error::Table)), "overlap"); + assert!(matches!(table([(0, 1), (2, 1)], 3), Err(Error::Table)), "gap"); + assert!(matches!(table([(1, 1), (2, 1)], 3), Err(Error::Table)), "late start"); + assert!(matches!(table([(0, 1), (1, 3)], 3), Err(Error::Table)), "past the end"); + let varint = (1u64 << 62) - 1; + assert!( + matches!(table([(0, 1), (1, varint)], 3), Err(Error::Table)), + "huge length" + ); + } + #[test] fn filename_bounds_must_match_the_table() { let object = object(vec![ diff --git a/rs/moq-archive/src/store.rs b/rs/moq-archive/src/store.rs index 3a9092703e..c79db85180 100644 --- a/rs/moq-archive/src/store.rs +++ b/rs/moq-archive/src/store.rs @@ -343,113 +343,16 @@ mod tests { use std::num::NonZeroUsize; use futures::TryStreamExt; - use object_store::list::{PaginatedListOptions, PaginatedListResult, PaginatedListStore}; + use object_store::ObjectStoreExt; use object_store::memory::InMemory; use object_store::path::Path; - use object_store::{ - CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, PutMultipartOptions, - PutOptions, PutPayload, PutResult, - }; use super::*; use crate::ID_MAX; + use crate::mock::Mock; use crate::path::encode_track; use crate::segment::{Frame, Group}; - /// In-memory store with a trivial offset-based paginated listing implementation. - /// Page tokens are decimal indexes into the filtered, sorted key list. - #[derive(Debug, Clone)] - struct PaginatedMemory { - inner: InMemory, - } - - impl std::fmt::Display for PaginatedMemory { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "PaginatedMemory") - } - } - - #[async_trait::async_trait] - impl ObjectStore for PaginatedMemory { - async fn put_opts( - &self, - location: &Path, - payload: PutPayload, - opts: PutOptions, - ) -> object_store::Result { - self.inner.put_opts(location, payload, opts).await - } - - async fn put_multipart_opts( - &self, - location: &Path, - opts: PutMultipartOptions, - ) -> object_store::Result> { - self.inner.put_multipart_opts(location, opts).await - } - - async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { - self.inner.get_opts(location, options).await - } - - fn delete_stream( - &self, - locations: BoxStream<'static, object_store::Result>, - ) -> BoxStream<'static, object_store::Result> { - self.inner.delete_stream(locations) - } - - fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { - self.inner.list(prefix) - } - - async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { - self.inner.list_with_delimiter(prefix).await - } - - async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { - self.inner.copy_opts(from, to, options).await - } - } - - #[async_trait::async_trait] - impl PaginatedListStore for PaginatedMemory { - async fn list_paginated( - &self, - prefix: Option<&str>, - opts: PaginatedListOptions, - ) -> object_store::Result { - let mut metas: Vec = self.inner.list(None).try_collect().await?; - metas.sort_by(|a, b| a.location.as_ref().cmp(b.location.as_ref())); - if let Some(prefix) = prefix { - metas.retain(|meta| meta.location.as_ref().starts_with(prefix)); - } - if let Some(offset) = opts.offset.as_deref() { - metas.retain(|meta| meta.location.as_ref() > offset); - } - let start: usize = match opts.page_token { - Some(token) => token.parse().map_err(|err| object_store::Error::Generic { - store: "PaginatedMemory", - source: Box::new(err), - })?, - None => 0, - }; - let remaining = &metas[start.min(metas.len())..]; - let take = opts.max_keys.unwrap_or(remaining.len()).min(remaining.len()); - let objects = remaining[..take].to_vec(); - let next = start.saturating_add(take); - let page_token = (next < metas.len()).then(|| next.to_string()); - Ok(PaginatedListResult { - result: ListResult { - objects, - common_prefixes: Vec::new(), - extensions: Default::default(), - }, - page_token, - }) - } - } - fn memory() -> Store { Store::new(InMemory::new(), "rec") } @@ -520,6 +423,30 @@ mod tests { assert_eq!(&kept[..], br#"{ "timescale": 1000, "priority": 1, "version": 1 }"#); } + #[tokio::test] + async fn malformed_or_unsupported_existing_info_is_refused_and_kept() { + let store = memory(); + let info = Info::new(0, 1_000).unwrap(); + for (track, existing, check) in [ + ( + "v2", + &br#"{"version":2,"priority":0,"timescale":1000}"#[..], + (|err| matches!(err, Error::Version(2))) as fn(&Error) -> bool, + ), + ("junk", b"not json", |err| matches!(err, Error::Json(_))), + ("zero", br#"{"version":1,"priority":0,"timescale":0}"#, |err| { + matches!(err, Error::Timescale(0)) + }), + ] { + let path = store.path(&Key::info(track).unwrap()).unwrap(); + store.inner().put(&path, existing.to_vec().into()).await.unwrap(); + let err = store.put_info(track, &info).await.unwrap_err(); + assert!(check(&err), "{track}: {err}"); + let kept = store.inner().get(&path).await.unwrap().bytes().await.unwrap(); + assert_eq!(&kept[..], existing, "{track} is not rewritten"); + } + } + #[tokio::test] async fn info_property_mismatch_is_a_hard_error() { let store = memory(); @@ -818,7 +745,7 @@ mod tests { #[tokio::test] async fn paginated_listing_walks_pages_and_excludes_siblings() { - let store = Store::new(PaginatedMemory { inner: InMemory::new() }, "rec"); + let store = Store::new(Mock::memory(), "rec"); for segment in 0..3 { store .put_segments("timeline.z", segment, &one_group(segment, b"t")) @@ -827,7 +754,6 @@ mod tests { } store .inner() - .inner .put( &Path::from("rec-other/video/.info"), Bytes::from_static(b"sibling").into(), @@ -896,6 +822,90 @@ mod tests { assert_eq!(paged, streamed); } + /// The first object listed after `groups_from(group)`, one S3 page of one key. + async fn lookup(store: &Store, group: u64) -> Option> { + let query = Query::groups_from("video", group) + .unwrap() + .page_size(NonZeroUsize::new(1).unwrap()); + let page = store.list_paginated(&query).await.unwrap(); + match page.entries.first().map(|entry| &entry.key) { + Some(Key::Groups { range, .. }) => Some(range.clone()), + Some(key) => panic!("unexpected {key:?}"), + None => None, + } + } + + #[tokio::test] + async fn ordered_lookup_finds_the_covering_object() { + let store = Store::new(Mock::memory(), "rec"); + for range in [0..=2, 5..=7, 10..=10, ID_MAX..=ID_MAX] { + let groups = range.clone().filter(|sequence| *sequence != 6); + let object = Object { + groups: groups + .map(|sequence| Group { + sequence, + frames: vec![frame(0, b"g")], + }) + .collect(), + }; + store.put_groups("video", &object).await.unwrap(); + } + // A sibling track sorts after `video/groups/` and must never be returned. + store.put_groups("video-alt", &one_group(3, b"a")).await.unwrap(); + + // Largest-first filenames make the first key at or past the group its only candidate. + for (group, found) in [ + (0, Some(0..=2)), + (1, Some(0..=2)), + (2, Some(0..=2)), + (3, Some(5..=7)), + (5, Some(5..=7)), + (6, Some(5..=7)), + (7, Some(5..=7)), + (8, Some(10..=10)), + (10, Some(10..=10)), + (11, Some(ID_MAX..=ID_MAX)), + (ID_MAX, Some(ID_MAX..=ID_MAX)), + ] { + assert_eq!(lookup(&store, group).await, found, "group {group}"); + } + store + .delete(&Key::groups("video", ID_MAX..=ID_MAX).unwrap()) + .await + .unwrap(); + assert_eq!(lookup(&store, 11).await, None); + assert!(Query::groups_from("video", ID_MAX + 1).is_err()); + } + + #[tokio::test] + async fn empty_prefix_lists_the_whole_store() { + let store = Store::new(Mock::memory(), ""); + store.put_info("catalog.json", &Info::new(0, 1).unwrap()).await.unwrap(); + store.put_groups("video", &one_group(4, b"a")).await.unwrap(); + assert_eq!(store.paginated_prefix(None), None); + + let expected = + std::collections::HashSet::from([Key::info("catalog.json").unwrap(), Key::groups("video", 4..=4).unwrap()]); + let streamed: std::collections::HashSet = store + .list(&Query::new()) + .map_ok(|entry| entry.key) + .try_collect() + .await + .unwrap(); + assert_eq!(streamed, expected); + let page = store.list_paginated(&Query::new()).await.unwrap(); + assert_eq!( + page.entries + .into_iter() + .map(|entry| entry.key) + .collect::>(), + expected + ); + assert!(page.next.is_none()); + let page = store.list_paginated(&Query::groups("video").unwrap()).await.unwrap(); + assert_eq!(page.entries.len(), 1); + } + #[test] fn directory_results_are_not_silently_dropped() { let store = memory(); diff --git a/rs/moq-archive/src/writer.rs b/rs/moq-archive/src/writer.rs new file mode 100644 index 0000000000..9ed25d86b7 --- /dev/null +++ b/rs/moq-archive/src/writer.rs @@ -0,0 +1,1473 @@ +//! Record explicitly selected tracks of a [`broadcast::Consumer`] into a [`Store`]. +//! +//! The application enrolls each track as pacing or non-pacing through a [`Control`]; the writer +//! never parses a catalog. Every complete group feeds the timeline segmenter. For each closed +//! segment the writer stores one object per participating track, omits any track whose object +//! failed, commits the record through its own timeline encoder, and stores that segment's timeline +//! groups before starting the next one. The timeline therefore only advertises durable objects. +//! +//! A writer started on a prefix that already holds a recording resumes it: see [`Writer::new`]. +//! +//! ```no_run +//! # async fn example(source: moq_net::broadcast::Consumer) -> moq_archive::Result<()> { +//! use moq_archive::object_store::memory::InMemory; +//! +//! let store = moq_archive::Store::new(InMemory::new(), "recordings/demo"); +//! let writer = moq_archive::Writer::new(store, source, Default::default()).await?; +//! let control = writer.control(); +//! control.pacing_track("video").await?; +//! control.pacing_track("audio").await?; +//! control.track("catalog.json").await?; +//! writer.run().await?; +//! # Ok(()) +//! # } +//! ``` + +use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; +use std::future::Future; +use std::sync::{Arc, Mutex}; +use std::task::Poll; +use std::time::Duration; + +use futures::future::BoxFuture; +use futures::stream::FuturesUnordered; +use futures::{FutureExt, StreamExt}; +use hang::timeline::Record; +use moq_mux::timeline::{self, Deferred, DeferredDrain, Pending, Recorder, Reserved}; +use moq_net::{Timescale, Timestamp, broadcast, group, track}; +use object_store::ObjectStore; +use tokio::sync::{mpsc, watch}; +use tokio::time::Instant; + +use crate::recover::recover; +use crate::segment::{Frame, Group, Object}; +use crate::{Error, Info, Key, Result, Store}; + +/// Subscribers ask for every cached group; the publisher clamps this to its own max age. +const REPLAY: Duration = Duration::from_secs(u32::MAX as u64); + +/// How a [`Writer`] segments and retains its recording. +#[derive(Clone, Debug, Default)] +#[non_exhaustive] +pub struct Config { + /// Segment pacing for the recording's timeline. + pub timeline: timeline::Config, + /// Expire old segments (a DVR), or keep everything when `None` (an archive). + pub retention: Option, +} + +impl Config { + /// Set [`timeline`](Self::timeline). + pub fn with_timeline(mut self, timeline: timeline::Config) -> Self { + self.timeline = timeline; + self + } + + /// Set [`retention`](Self::retention). + pub fn with_retention(mut self, retention: impl Into>) -> Self { + self.retention = retention.into(); + self + } +} + +/// A DVR retention policy. +#[derive(Clone, Debug)] +#[non_exhaustive] +pub struct Retention { + /// Keep at least this much content, measured from timeline records. + pub window: Duration, + /// How long expired objects outlive the timeline that stopped advertising them, so readers + /// holding an older timeline can finish their GETs. + pub grace: Duration, +} + +impl Retention { + /// Keep `window` of content and delete expired objects `grace` after they leave the timeline. + pub fn new(window: Duration, grace: Duration) -> Self { + Self { window, grace } + } +} + +/// Records enrolled tracks until the source broadcast closes. +/// +/// Enroll tracks through [`control`](Self::control), then drive the recording with +/// [`run`](Self::run). +pub struct Writer { + control: Control, + commands: mpsc::UnboundedReceiver, + committer: Committer, + grace: Option, + /// Deadlines for deleting expired or orphaned objects, oldest first. + deletions: VecDeque<(Instant, Vec)>, + // Owns the recording's timeline track. + _timeline: broadcast::Producer, +} + +/// Enrolls, cuts, and removes tracks on a [`Writer`]. +pub struct Control { + shared: Arc>, + deferred: Deferred, + commands: mpsc::UnboundedSender, +} + +impl Clone for Control { + fn clone(&self) -> Self { + Self { + shared: self.shared.clone(), + deferred: self.deferred.clone(), + commands: self.commands.clone(), + } + } +} + +/// Withholds segment commits until dropped. +pub struct Reservation { + _inner: Reserved, +} + +struct Shared { + store: Store, + source: broadcast::Consumer, + /// The recording-owned timeline track, which no source track may shadow. + timeline: String, + /// Every name ever enrolled. A name is never reused, so its object ranges stay increasing. + enrolled: Mutex>, + /// Per track, the largest group a resumed recording already stored. + floors: HashMap, +} + +enum Command { + Enroll { + name: String, + subscriber: Box, + recorder: Recorder, + timescale: Timescale, + }, + Remove(String), + /// A cut may have completed a segment. + Poke, +} + +impl Writer { + /// Start a recording under `store`'s prefix, reading tracks from `source`, or resume the one + /// already there. + /// + /// Resuming replays the retained timeline and continues at the next segment. A track refuses + /// any group at or below the largest one stored for it, so a source whose group sequences + /// restarted needs a new prefix. A DVR also deletes, one grace period after recovery, every + /// group object its retained records do not reference, such as interrupted expirations and + /// uploads. The writer must own the prefix exclusively. Fails, deleting nothing, when the + /// recording cannot be listed or its timeline cannot be replayed. + pub async fn new(store: Store, source: broadcast::Consumer, config: Config) -> Result { + let section = timeline::Segmenter::new(config.timeline.clone()).section(); + let recovery = recover(&store, §ion.track, config.retention.is_some()).await?; + + let broadcast = broadcast::Info::new().produce(); + let timeline = match &recovery.checkpoint { + Some(checkpoint) => { + timeline::Producer::resume(&broadcast, config.timeline, checkpoint).map_err(timeline_error)? + } + None => timeline::Producer::new(&broadcast, config.timeline), + }; + let deferred = timeline.deferred().map_err(timeline_error)?; + + let replay = track::Subscription::default().with_max_age(REPLAY); + let groups = broadcast + .consume() + .track(§ion.track) + .map_err(source_error)? + .subscribe(replay) + .await + .map_err(source_error)? + .ordered(); + let info = Info::new(groups.info().priority, groups.info().timescale.as_u64())?; + store.put_info(§ion.track, &info).await?; + + let shared = Arc::new(Shared { + store, + source, + timeline: section.track, + enrolled: Mutex::new(HashSet::new()), + floors: recovery.floors, + }); + let (commands, receiver) = mpsc::unbounded_channel(); + let retention = config.retention; + let mut deletions = VecDeque::new(); + if let Some(retention) = &retention + && !recovery.orphans.is_empty() + { + deletions.push_back((Instant::now() + retention.grace, recovery.orphans)); + } + Ok(Self { + control: Control { + shared: shared.clone(), + deferred, + commands, + }, + commands: receiver, + committer: Committer { + shared, + timeline, + groups, + timescale: section.timescale.into(), + retention: retention.as_ref().map(|r| r.window), + window: recovery.checkpoint.map(|c| c.records.into()).unwrap_or_default(), + sequence: recovery.sequence, + }, + grace: retention.map(|r| r.grace), + deletions, + _timeline: broadcast, + }) + } + + /// A handle that enrolls, cuts, and removes tracks, before or during [`run`](Self::run). + pub fn control(&self) -> Control { + self.control.clone() + } + + /// Record until the source broadcast closes and every enrolled track ends. + /// + /// Then flush the final segment and finish the timeline. Fails when the timeline cannot be + /// committed or stored, or an enrolled track delivers a group the recording cannot represent: + /// the recording stops at its last durable timeline object. Returns the source's error, after + /// finishing, when the broadcast aborted. + pub async fn run(self) -> Result<()> { + let Self { + control, + mut commands, + committer, + grace, + mut deletions, + _timeline, + } = self; + let shared = control.shared.clone(); + let source = shared.source.clone(); + let mut segments = Segments::Live(control.deferred.clone()); + drop(control); + + let mut tracks: HashMap = HashMap::new(); + // Reported groups waiting for the record that names them, kept past a track's removal. + let mut buffered: HashMap> = HashMap::new(); + let mut reads = FuturesUnordered::new(); + let mut committer = Some(committer); + let mut commit: Option> = None; + let mut closed = false; + // Cleared once the channel yields nothing more: every sender dropped, or it closed and drained. + let mut accepting = true; + + loop { + if commit.is_none() + && let Some(pending) = segments.next() + { + let objects = take_objects(&pending, &mut buffered); + let committer = committer.take().expect("idle committer"); + commit = Some(committer.commit(pending, objects).boxed()); + } + + if commit.is_none() && closed && tracks.is_empty() { + // Refuse late commands, so an enrollment racing the end fails instead of vanishing. + commands.close(); + if !accepting { + match segments { + Segments::Live(deferred) => { + segments = Segments::Drain(deferred.finish()); + continue; + } + Segments::Drain(_) => break, + } + } + } + + let deadline = deletions.front().map(|(deadline, _)| *deadline); + tokio::select! { + biased; + command = commands.recv(), if accepting => match command { + None => accepting = false, + Some(Command::Enroll { name, subscriber, recorder, timescale }) => { + let (cancel, cancelled) = watch::channel(()); + let largest = shared.floors.get(&name).copied(); + reads.push(guard(cancelled.clone(), recv(name.clone(), subscriber)).boxed()); + tracks.insert(name, TrackState { + recorder, + timescale, + largest, + reported: None, + accepted: BTreeMap::new(), + subscribed: true, + _cancel: cancel, + cancelled, + }); + } + Some(Command::Remove(name)) => { + tracks.remove(&name); + } + Some(Command::Poke) => {} + }, + Some(read) = reads.next(), if !reads.is_empty() => { + handle(read, &mut tracks, &mut buffered, &mut reads)?; + } + (done, result) = async { commit.as_mut().unwrap().await }, if commit.is_some() => { + commit = None; + committer = Some(done); + let expired = result?; + if let Some(grace) = grace && !expired.is_empty() { + deletions.push_back((Instant::now() + grace, expired)); + } + } + _ = source.closed(), if !closed => closed = true, + _ = tokio::time::sleep_until(deadline.unwrap_or_else(Instant::now)), if deadline.is_some() => { + let (_, keys) = deletions.pop_front().unwrap(); + delete(&shared.store, keys).await; + } + } + } + + let Segments::Drain(drain) = segments else { + unreachable!("the loop only ends after draining"); + }; + drain.result().map_err(timeline_error)?; + let mut committer = committer.expect("idle committer"); + committer.timeline.finish().map_err(timeline_error)?; + + // Complete expirations already committed to the timeline; nothing new expires after the + // final segment. + for (deadline, keys) in deletions { + tokio::time::sleep_until(deadline).await; + delete(&shared.store, keys).await; + } + + if source.is_finished() { + Ok(()) + } else { + Err(source_error(source.closed().await)) + } + } +} + +impl Control { + /// Enroll `name` without letting it influence segmentation, such as a catalog or metadata track. + /// + /// Subscribes to the track and creates its `.info` before accepting any group. Fails when the + /// name was already enrolled, is the recording's timeline, or `.info` conflicts. + pub async fn track(&self, name: &str) -> Result<()> { + self.enroll(name, false).await + } + + /// Enroll the continuously publishing track `name` as a pacing track. + /// + /// It votes on segment boundaries and holds every segment back until its groups are known. A + /// pacing track that stalls without closing stalls the recording by design; apply a deadline + /// with [`cut`](Self::cut) or [`remove`](Self::remove). + pub async fn pacing_track(&self, name: &str) -> Result<()> { + self.enroll(name, true).await + } + + /// Declare a segment boundary at `pts`, overriding the configured minimum-duration pacing. + /// + /// The first cut takes over boundaries for good, as with [`timeline::Producer::cut`]. + pub fn cut(&self, pts: Timestamp) -> Result<()> { + self.deferred.cut(pts).map_err(timeline_error)?; + self.send(Command::Poke) + } + + /// Hold segment commits back until this guard drops, so a batch of tracks can enroll first. + #[must_use = "dropping the reservation releases segment commits"] + pub fn reserve(&self) -> Reservation { + Reservation { + _inner: self.deferred.reserve(), + } + } + + /// Stop recording `name`, dropping its incomplete groups. The name cannot be enrolled again. + pub fn remove(&self, name: &str) -> Result<()> { + self.send(Command::Remove(name.to_string())) + } + + async fn enroll(&self, name: &str, pacing: bool) -> Result<()> { + if name == self.shared.timeline || !self.shared.enrolled.lock().unwrap().insert(name.to_string()) { + return Err(Error::Enrolled(name.to_string())); + } + let result = self.subscribe(name, pacing).await; + if result.is_err() { + self.shared.enrolled.lock().unwrap().remove(name); + } + result + } + + async fn subscribe(&self, name: &str, pacing: bool) -> Result<()> { + let replay = track::Subscription::default().with_max_age(REPLAY); + let subscriber = self + .shared + .source + .track(name) + .map_err(source_error)? + .subscribe(replay) + .await + .map_err(source_error)?; + let timescale = subscriber.info().timescale; + let info = Info::new(subscriber.info().priority, timescale.as_u64())?; + self.shared.store.put_info(name, &info).await?; + + let recorder = match pacing { + true => self.deferred.pacing_track(name), + false => self.deferred.track(name), + }; + self.send(Command::Enroll { + name: name.to_string(), + subscriber: Box::new(subscriber), + recorder, + timescale, + }) + } + + fn send(&self, command: Command) -> Result<()> { + self.commands.send(command).map_err(|_| Error::Closed) + } +} + +/// One enrolled track's accepted groups. +struct TrackState { + recorder: Recorder, + timescale: Timescale, + /// The newest group accepted or already stored; later arrivals must exceed it. + largest: Option, + /// The first-frame timestamp of the newest reported group; later groups must not precede it. + reported: Option, + /// Accepted groups in sequence order: `None` while reading, `Some` once complete. Reported in + /// order, so a complete group waits for every earlier accepted one. + accepted: BTreeMap>, + /// The subscription is still delivering groups. + subscribed: bool, + /// Dropping the sender cancels every read for this track. + _cancel: watch::Sender<()>, + cancelled: watch::Receiver<()>, +} + +impl TrackState { + /// Report every complete group at the front of the accepted queue. + /// + /// Fails on a group that starts before the previous one, which the timeline cannot place. + fn report(&mut self, name: &str, buffered: &mut HashMap>) -> Result<()> { + while let Some(entry) = self.accepted.first_entry() { + if entry.get().is_none() { + break; + } + let group = entry.remove().expect("checked above"); + let (Some(first), Some(last)) = (group.frames.first(), group.frames.last()) else { + continue; + }; + if let Some(reported) = self.reported + && first.timestamp < reported + { + return Err(malformed( + name, + group.sequence, + format!("timestamp {} precedes {reported}", first.timestamp), + )); + } + self.reported = Some(first.timestamp); + // Frame timestamps were validated while reading, so these conversions succeed. + let (Ok(first), Ok(last)) = ( + Timestamp::new(first.timestamp, self.timescale), + Timestamp::new(last.timestamp, self.timescale), + ) else { + continue; + }; + self.recorder.record(group.sequence, first, true); + self.recorder.end(last); + buffered + .entry(name.to_string()) + .or_default() + .insert(group.sequence, group); + } + Ok(()) + } +} + +// Each read is already boxed as a future, so the variant sizes cost nothing extra. +#[allow(clippy::large_enum_variant)] +enum Read { + Group { + name: String, + subscriber: Box, + result: moq_net::Result>, + }, + Frames { + name: String, + sequence: u64, + result: moq_net::Result>, + }, + Cancelled, +} + +async fn recv(name: String, mut subscriber: Box) -> Read { + let result = subscriber.recv_group().await; + Read::Group { + name, + subscriber, + result, + } +} + +async fn frames(name: String, mut group: group::Consumer) -> Read { + let sequence = group.sequence; + let mut frames = Vec::new(); + let result = loop { + match group.read_frame().await { + Ok(Some(frame)) => frames.push(frame), + Ok(None) => break Ok(frames), + Err(err) => break Err(err), + } + }; + Read::Frames { name, sequence, result } +} + +/// Resolve to [`Read::Cancelled`] once the track's state drops. +async fn guard(mut cancelled: watch::Receiver<()>, read: impl Future) -> Read { + tokio::select! { + read = read => read, + _ = cancelled.changed() => Read::Cancelled, + } +} + +/// Fails on malformed source input; a group the network aborted is dropped instead. +fn handle( + read: Read, + tracks: &mut HashMap, + buffered: &mut HashMap>, + reads: &mut FuturesUnordered>, +) -> Result<()> { + let name = match read { + Read::Cancelled => return Ok(()), + Read::Group { + name, + subscriber, + result, + } => { + let Some(track) = tracks.get_mut(&name) else { + return Ok(()); + }; + match result { + Ok(Some(group)) => { + let sequence = group.sequence; + if track.largest.is_some_and(|largest| sequence <= largest) { + tracing::debug!(track = %name, sequence, "refusing a duplicate or decreasing group"); + } else { + track.largest = Some(sequence); + track.accepted.insert(sequence, None); + reads.push(guard(track.cancelled.clone(), frames(name.clone(), group)).boxed()); + } + reads.push(guard(track.cancelled.clone(), recv(name.clone(), subscriber)).boxed()); + } + Ok(None) => track.subscribed = false, + Err(err) => { + tracing::warn!(track = %name, %err, "track ended"); + track.subscribed = false; + } + } + name + } + Read::Frames { name, sequence, result } => { + let Some(track) = tracks.get_mut(&name) else { + return Ok(()); + }; + match result { + Ok(frames) => { + let group = + convert(sequence, frames, track.timescale).map_err(|err| malformed(&name, sequence, err))?; + if group.frames.is_empty() { + track.accepted.remove(&sequence); + } else { + track.accepted.insert(sequence, Some(group)); + } + } + Err(err) => { + tracing::warn!(track = %name, sequence, %err, "dropping an incomplete group"); + track.accepted.remove(&sequence); + } + } + track.report(&name, buffered)?; + name + } + }; + + // A track whose subscription ended closes once its last accepted group settles. + if tracks + .get(&name) + .is_some_and(|track| !track.subscribed && track.accepted.is_empty()) + { + tracks.remove(&name); + } + Ok(()) +} + +/// Convert a complete group's frames into the track's timescale. +fn convert(sequence: u64, frames: Vec, timescale: Timescale) -> Result { + let frames = frames + .into_iter() + .map(|frame| { + let timestamp = frame.timestamp.convert(timescale).map_err(|_| Error::Overflow)?.value(); + crate::path::check_id(timestamp)?; + Ok(Frame { + timestamp, + payload: frame.payload, + }) + }) + .collect::>()?; + crate::path::check_id(sequence)?; + Ok(Group { sequence, frames }) +} + +/// Take the buffered groups a pending record names, one object per track. +/// +/// A track missing any advertised group cannot produce an object whose table matches its ranges, +/// so it gets `None` and is omitted. +fn take_objects( + pending: &Pending, + buffered: &mut HashMap>, +) -> Vec<(String, Option)> { + pending + .tracks + .iter() + .map(|(name, ranges)| { + let track = buffered.entry(name.clone()).or_default(); + let mut groups = Vec::new(); + let mut complete = true; + for range in ranges { + for sequence in range.start..=range.end { + match track.remove(&sequence) { + Some(group) => groups.push(group), + None => complete = false, + } + } + } + (name.clone(), complete.then_some(Object { groups })) + }) + .collect() +} + +/// An in-flight segment commit, returning the committer and the expired objects to delete. +type Commit = BoxFuture<'static, (Committer, Result>)>; + +/// Deferred records to commit: live, then the terminal drain. +enum Segments { + Live(Deferred), + Drain(DeferredDrain), +} + +impl Segments { + fn next(&self) -> Option { + match self { + Self::Live(deferred) => deferred.next(), + Self::Drain(drain) => drain.next(), + } + } +} + +/// Commits one segment at a time: media objects, the record, retention, then the timeline object. +struct Committer { + shared: Arc>, + timeline: timeline::Producer, + /// The recording's own timeline track, read back to store its complete groups. + groups: track::Ordered, + /// Units per second of record `pts` and `duration`. + timescale: u64, + retention: Option, + /// Committed records still in the timeline window, oldest first. + window: VecDeque, + /// Added to the timeline track's group sequences, continuing a resumed recording's numbering. + sequence: u64, +} + +impl Committer { + /// Commit `pending`, returning the committer and the expired objects to delete after the grace. + async fn commit(mut self, pending: Pending, objects: Vec<(String, Option)>) -> (Self, Result>) { + let result = self.commit_inner(pending, objects).await; + (self, result) + } + + async fn commit_inner(&mut self, mut pending: Pending, objects: Vec<(String, Option)>) -> Result> { + let shared = self.shared.clone(); + let store = &shared.store; + let puts = objects.iter().map(|(name, object)| async move { + let result = match object { + Some(object) => store.put_groups(name, object).await.map(|_| ()), + None => Err(Error::Timeline("advertised groups are not buffered".into())), + }; + (name, result) + }); + for (name, result) in futures::future::join_all(puts).await { + if let Err(err) = result { + tracing::warn!(track = %name, segment = pending.segment, %err, "omitting a track that was not stored"); + pending = pending.omit(name); + } + } + + let segment = pending.segment; + let record = (*pending).clone(); + let pts = Timestamp::new( + record.pts, + Timescale::new(self.timescale).map_err(|_| Error::Timescale(self.timescale))?, + ) + .map_err(|_| Error::Id(record.pts))?; + self.timeline.push(pending).map_err(timeline_error)?; + self.window.push_back(record); + + let expired = self.expire(); + if !expired.is_empty() { + self.timeline.pop(expired.len() as u64).map_err(timeline_error)?; + } + self.timeline.flush().map_err(timeline_error)?; + + let object = self.read_timeline(pts)?; + store.put_segments(&shared.timeline, segment, &object).await?; + + let mut keys = Vec::new(); + for record in expired { + for (name, ranges) in &record.tracks { + if let (Some(first), Some(last)) = (ranges.first(), ranges.last()) { + keys.push(Key::groups(name.clone(), first.start..=last.end)?); + } + } + } + Ok(keys) + } + + /// Pop the oldest records while the rest still cover the retention window. + fn expire(&mut self) -> Vec { + let Some(retention) = self.retention else { + return Vec::new(); + }; + let target = (retention.as_micros() * self.timescale as u128).div_ceil(1_000_000); + let mut total: u128 = self.window.iter().map(|record| record.duration as u128).sum(); + let mut expired = Vec::new(); + while self.window.len() > 1 { + let front = self.window.front().expect("non-empty").duration as u128; + if total - front < target { + break; + } + total -= front; + expired.push(self.window.pop_front().expect("non-empty")); + } + expired + } + + /// Collect the timeline groups completed since the last segment, stamped at `pts`. + /// + /// The live timeline track stamps frames with the wall clock; storing the segment's content + /// time instead keeps a recording's bytes a function of its content alone. + fn read_timeline(&mut self, pts: Timestamp) -> Result { + let waiter = kio::Waiter::noop(); + let timescale = self.groups.info().timescale; + let mut groups = Vec::new(); + while let Poll::Ready(result) = self.groups.poll_next_group(&waiter) { + let Some(mut group) = result.map_err(timeline_error_net)? else { + break; + }; + let mut frames = Vec::new(); + loop { + match group.poll_read_frame(&waiter) { + Poll::Ready(Ok(Some(mut frame))) => { + frame.timestamp = pts; + frames.push(frame); + } + Poll::Ready(Ok(None)) => break, + Poll::Ready(Err(err)) => return Err(timeline_error_net(err)), + // Flushing closed every group, so an open one is a bug. + Poll::Pending => return Err(Error::Timeline("timeline group is still open".into())), + } + } + let sequence = group.sequence.checked_add(self.sequence).ok_or(Error::Overflow)?; + groups.push(convert(sequence, frames, timescale)?); + } + Ok(Object { groups }) + } +} + +async fn delete(store: &Store, keys: Vec) { + let deletes = keys.iter().map(|key| async move { (key, store.delete(key).await) }); + for (key, result) in futures::future::join_all(deletes).await { + // An object that outlives its expiry advertises nothing; recovery cleans it up. + if let Err(err) = result { + tracing::warn!(?key, %err, "failed to delete an expired object"); + } + } +} + +fn timeline_error(err: moq_mux::Error) -> Error { + Error::Timeline(err.to_string()) +} + +fn timeline_error_net(err: moq_net::Error) -> Error { + Error::Timeline(err.to_string()) +} + +fn source_error(err: moq_net::Error) -> Error { + Error::Source(err.to_string()) +} + +fn malformed(track: &str, sequence: u64, err: impl std::fmt::Display) -> Error { + Error::Source(format!("track {track} group {sequence}: {err}")) +} + +#[cfg(test)] +mod tests { + + use futures::TryStreamExt; + use object_store::memory::InMemory; + + use super::*; + use crate::mock::Mock; + use crate::store::list::Query; + + const TIMELINE: &str = hang::timeline::DEFAULT_NAME; + + fn ms(v: u64) -> Timestamp { + Timestamp::from_millis(v).unwrap() + } + + /// A millisecond track that keeps an hour of history, so enrollment sees every group. + fn track(broadcast: &broadcast::Producer, name: &str) -> track::Producer { + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + broadcast.create_track(name, info).unwrap() + } + + /// Write one complete group with a frame at each timestamp. + fn group(track: &track::Producer, sequence: u64, timestamps: &[u64]) { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for ×tamp in timestamps { + group + .write_frame(ms(timestamp), format!("{sequence}@{timestamp}")) + .unwrap(); + } + group.finish().unwrap(); + } + + /// Replay every stored timeline object, returning the records still in the window. + async fn window(store: &Store) -> Vec { + let entries: Vec<_> = store + .list(&Query::segments(TIMELINE).unwrap()) + .try_collect() + .await + .unwrap(); + let mut segments: Vec = entries + .into_iter() + .map(|entry| match entry.key { + Key::Segments { segment, .. } => segment, + key => panic!("unexpected {key:?}"), + }) + .collect(); + segments.sort(); + assert_eq!( + segments, + (0..segments.len() as u64).collect::>(), + "timeline objects are consecutive" + ); + + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + let mut records = BTreeMap::new(); + for segment in segments { + let object = store.get_segments(TIMELINE, segment).await.unwrap(); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } + } + while let Some(event) = decoder.next_event() { + match event { + moq_json::window::Event::Push { index, value } => { + records.insert(index, value); + } + moq_json::window::Event::Pop(range) | moq_json::window::Event::Skip(range) => { + for index in range { + records.remove(&index); + } + } + _ => unreachable!(), + } + } + } + records.into_values().collect() + } + + /// The groups each track contributes, from the records. + fn ranges(records: &[Record], name: &str) -> Vec<(u64, u64)> { + records + .iter() + .flat_map(|record| record.tracks.get(name).into_iter().flatten()) + .map(|range| (range.start, range.end)) + .collect() + } + + /// Every range in the records resolves to one stored object with exactly those groups. + async fn check_objects(store: &Store, records: &[Record]) { + for record in records { + for (name, ranges) in &record.tracks { + let first = ranges.first().unwrap().start; + let last = ranges.last().unwrap().end; + let object = store.get_groups(name, first..=last).await.unwrap(); + let advertised: Vec = ranges.iter().flat_map(|range| range.start..=range.end).collect(); + let stored: Vec = object.groups.iter().map(|group| group.sequence).collect(); + assert_eq!(stored, advertised, "{name} segment {}", record.segment); + } + } + } + + #[tokio::test] + async fn records_selected_tracks() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let catalog = track(&source, "catalog.json"); + let _ignored = track(&source, "ignored"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + control.track("catalog.json").await.unwrap(); + + group(&catalog, 0, &[0]); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + catalog.finish().unwrap(); + source.finish(); + + tokio::spawn(writer.run()).await.unwrap().unwrap(); + + let records = window(&store).await; + assert_eq!(records.len(), 6); + assert_eq!(ranges(&records, "video"), (0..6).map(|s| (s, s)).collect::>()); + assert_eq!(ranges(&records, "catalog.json"), vec![(0, 0)]); + assert!(records.iter().all(|record| !record.tracks.contains_key("ignored"))); + check_objects(&store, &records).await; + + let object = store.get_groups("video", 2..=2).await.unwrap(); + assert_eq!(object.groups[0].frames[1].timestamp, 2500); + assert_eq!(object.groups[0].frames[1].payload, "2@2500"); + assert_eq!(store.get_info("video").await.unwrap(), Info::new(0, 1000).unwrap()); + store.get_info(TIMELINE).await.unwrap(); + assert!(store.get_info("ignored").await.is_err()); + } + + #[tokio::test] + async fn a_failed_put_omits_only_that_track() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let mock = Mock::memory(); + mock.fail_puts("/audio/groups/"); + let store = Store::new(mock, "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + control.pacing_track("audio").await.unwrap(); + + for sequence in 0..3 { + group(&video, sequence, &[sequence * 1000]); + group(&audio, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + audio.finish().unwrap(); + source.finish(); + + writer.run().await.unwrap(); + + let records = window(&store).await; + assert_eq!(records.len(), 3); + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2)]); + assert!(ranges(&records, "audio").is_empty()); + check_objects(&store, &records).await; + } + + #[tokio::test] + async fn retention_pops_then_deletes_expired_objects() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.finish(); + + writer.run().await.unwrap(); + + // Segments 3 and 4 hold two seconds; the partial final segment 5 is always kept. + let records = window(&store).await; + assert_eq!( + records.iter().map(|record| record.segment).collect::>(), + vec![3, 4, 5] + ); + check_objects(&store, &records).await; + + let stored: HashSet<_> = store + .list(&Query::groups("video").unwrap()) + .map_ok(|entry| entry.key) + .try_collect() + .await + .unwrap(); + let expected: HashSet<_> = (3..6).map(|s| Key::groups("video", s..=s).unwrap()).collect(); + assert_eq!(stored, expected); + } + + #[tokio::test] + async fn commands_are_refused_once_the_recording_ends() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + // A long grace keeps `run` waiting on deletions after the timeline finishes. + let config = + Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + source.finish(); + + let run = tokio::spawn(writer.run()); + while window(&store).await.last().map(|record| record.segment) != Some(5) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + assert!(!run.is_finished()); + assert_eq!(control.remove("video"), Err(Error::Closed)); + run.abort(); + } + + #[tokio::test] + async fn accepted_groups_may_complete_out_of_order() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + + let mut first = video.create_group(group::Info { sequence: 0 }).unwrap(); + first.write_frame(ms(0), "0@0").unwrap(); + group(&video, 1, &[1000]); + group(&video, 2, &[2000]); + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(window(&store).await.is_empty(), "group 1 waits for group 0"); + + first.write_frame(ms(500), "0@500").unwrap(); + first.finish().unwrap(); + video.finish().unwrap(); + source.finish(); + run.await.unwrap().unwrap(); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2)]); + check_objects(&store, &records).await; + let object = store.get_groups("video", 0..=0).await.unwrap(); + let frames: Vec<_> = object.groups[0].frames.iter().map(|f| f.timestamp).collect(); + assert_eq!(frames, vec![0, 500]); + } + + #[tokio::test] + async fn decreasing_arrivals_are_refused() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + group(&video, 0, &[0]); + group(&video, 2, &[1000]); + group(&video, 1, &[500]); + group(&video, 3, &[2000]); + video.finish().unwrap(); + source.finish(); + + writer.run().await.unwrap(); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (2, 2), (3, 3)]); + check_objects(&store, &records).await; + } + + #[tokio::test] + async fn an_unrepresentable_group_fails_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + group(&video, 0, &[0]); + // One past the recording's largest group ID. + group(&video, 1 << 53, &[1000]); + video.finish().unwrap(); + source.finish(); + + assert_eq!( + writer.run().await, + Err(Error::Source(format!( + "track video group {}: {}", + 1u64 << 53, + Error::Id(1 << 53) + ))) + ); + } + + #[tokio::test] + async fn a_decreasing_timestamp_fails_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + + group(&video, 0, &[1000]); + group(&video, 1, &[500]); + video.finish().unwrap(); + source.finish(); + + assert_eq!( + writer.run().await, + Err(Error::Source("track video group 1: timestamp 500 precedes 1000".into())) + ); + } + + #[tokio::test] + async fn removing_a_stalled_pacing_track_releases_the_recording() { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let audio = track(&source, "audio"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store.clone(), source.consume(), Config::default()) + .await + .unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + control.pacing_track("audio").await.unwrap(); + let run = tokio::spawn(writer.run()); + + group(&audio, 0, &[0]); + // The audio group never completes, so audio never reports past it. + let mut stalled = audio.create_group(group::Info { sequence: 1 }).unwrap(); + stalled.write_frame(ms(1000), "stalled").unwrap(); + for sequence in 0..4 { + group(&video, sequence, &[sequence * 1000]); + } + video.finish().unwrap(); + source.finish(); + + tokio::time::sleep(Duration::from_millis(50)).await; + assert!(!run.is_finished(), "a stalled pacing track holds the recording open"); + assert!(window(&store).await.is_empty(), "and holds every segment back"); + + control.remove("audio").unwrap(); + run.await.unwrap().unwrap(); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2), (3, 3)]); + assert_eq!( + ranges(&records, "audio"), + vec![(0, 0)], + "the incomplete group is dropped" + ); + check_objects(&store, &records).await; + drop(stalled); + } + + #[tokio::test] + async fn enrollment_is_refused_for_duplicates_and_the_timeline() { + let source = broadcast::Info::new().produce(); + let _video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + control.pacing_track("video").await.unwrap(); + assert_eq!(control.track("video").await, Err(Error::Enrolled("video".into()))); + assert_eq!(control.track(TIMELINE).await, Err(Error::Enrolled(TIMELINE.into()))); + } + + #[tokio::test] + async fn a_conflicting_info_fails_enrollment() { + let source = broadcast::Info::new().produce(); + let _video = track(&source, "video"); + + let store = Store::new(InMemory::new(), "rec"); + store.put_info("video", &Info::new(7, 1000).unwrap()).await.unwrap(); + let writer = Writer::new(store, source.consume(), Config::default()).await.unwrap(); + let control = writer.control(); + assert_eq!( + control.pacing_track("video").await, + Err(Error::Priority { + existing: 7, + intended: 0 + }) + ); + } + + /// Record `video` groups `sequences`, one per second, until the source ends. + async fn record(store: &Store, config: Config, sequences: std::ops::Range) { + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + for sequence in sequences { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.finish(); + writer.run().await.unwrap(); + } + + /// Every stored group object, across all tracks. + async fn stored_groups(store: &Store) -> HashSet { + store + .list(&Query::new()) + .try_filter_map(|entry| async move { Ok(matches!(entry.key, Key::Groups { .. }).then_some(entry.key)) }) + .try_collect() + .await + .unwrap() + } + + /// The group objects the retained records advertise. + fn referenced(records: &[Record]) -> HashSet { + records + .iter() + .flat_map(|record| &record.tracks) + .map(|(name, ranges)| Key::groups(name.clone(), ranges[0].start..=ranges.last().unwrap().end).unwrap()) + .collect() + } + + fn orphan(sequence: u64) -> Object { + Object { + groups: vec![Group { + sequence, + frames: vec![Frame { + timestamp: sequence * 1000, + payload: "orphan".into(), + }], + }], + } + } + + #[tokio::test] + async fn a_restarted_writer_resumes_the_recording() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..3).await; + // The source's cache replays groups the recording already holds. + record(&store, Config::default(), 0..6).await; + + let records = window(&store).await; + assert_eq!( + records.iter().map(|record| record.segment).collect::>(), + (0..6).collect::>() + ); + assert_eq!(ranges(&records, "video"), (0..6).map(|s| (s, s)).collect::>()); + check_objects(&store, &records).await; + + // The resumed timeline groups continue the stored numbering. + let mut sequences = Vec::new(); + for segment in 0..6 { + let object = store.get_segments(TIMELINE, segment).await.unwrap(); + sequences.extend(object.groups.iter().map(|group| group.sequence)); + } + assert!(sequences.windows(2).all(|pair| pair[0] < pair[1]), "{sequences:?}"); + } + + #[tokio::test] + async fn a_restarted_dvr_deletes_unreferenced_groups() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + record(&store, config.clone(), 0..6).await; + + // An interrupted expiration, an uncommitted upload, and a track no retained record names. + store.put_groups("video", &orphan(1)).await.unwrap(); + store.put_groups("video", &orphan(7)).await.unwrap(); + store.put_groups("audio", &orphan(0)).await.unwrap(); + + // Groups at or below the uncommitted upload are refused, so nothing overlaps it. + record(&store, config, 6..10).await; + + let records = window(&store).await; + assert_eq!( + records.iter().map(|record| record.segment).collect::>(), + vec![5, 6, 7] + ); + assert_eq!(ranges(&records, "video"), vec![(5, 5), (8, 8), (9, 9)]); + check_objects(&store, &records).await; + assert_eq!(stored_groups(&store).await, referenced(&records)); + store.get_info("video").await.unwrap(); + store.get_info(TIMELINE).await.unwrap(); + store.get_segments(TIMELINE, 0).await.unwrap(); + } + + #[tokio::test] + async fn a_dvr_crash_between_pop_and_delete_is_cleaned_on_restart() { + let store = Store::new(InMemory::new(), "rec"); + + // A grace longer than the test keeps every expired object past the crash. + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let config = + Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::from_secs(3600))); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + writer.control().pacing_track("video").await.unwrap(); + let run = tokio::spawn(writer.run()); + for sequence in 0..6 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + // Segment 5 stays open, so the newest durable timeline object is segment 4. + while window(&store).await.last().map(|record| record.segment) != Some(4) { + tokio::time::sleep(Duration::from_millis(10)).await; + } + run.abort(); + let _ = run.await; + + let retained = window(&store).await; + assert_eq!( + retained.iter().map(|record| record.segment).collect::>(), + vec![3, 4] + ); + let expired: HashSet<_> = (0..3).map(|s| Key::groups("video", s..=s).unwrap()).collect(); + assert_eq!(stored_groups(&store).await, &referenced(&retained) | &expired); + // An upload the crash left uncommitted. + store.put_groups("video", &orphan(5)).await.unwrap(); + + let grace = Duration::from_millis(200); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), grace)); + let source = broadcast::Info::new().produce(); + let video = track(&source, "video"); + let started = Instant::now(); + let writer = Writer::new(store.clone(), source.consume(), config).await.unwrap(); + assert!( + stored_groups(&store).await.is_superset(&expired), + "expired objects outlive the grace, for readers holding the old timeline" + ); + + writer.control().pacing_track("video").await.unwrap(); + // The source replays the uncommitted group; it is refused rather than overwritten. + for sequence in 5..9 { + group(&video, sequence, &[sequence * 1000, sequence * 1000 + 500]); + } + video.finish().unwrap(); + source.finish(); + writer.run().await.unwrap(); + assert!(started.elapsed() >= grace); + + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(6, 6), (7, 7), (8, 8)]); + check_objects(&store, &records).await; + assert_eq!(stored_groups(&store).await, referenced(&records)); + store.get_info("video").await.unwrap(); + store.get_info(TIMELINE).await.unwrap(); + for segment in 0..=7 { + store.get_segments(TIMELINE, segment).await.unwrap(); + } + } + + #[tokio::test] + async fn an_archive_restart_leaves_uncommitted_groups_unadvertised() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..3).await; + // Media stored after the last timeline commit: a crash before its record. + store.put_groups("video", &orphan(3)).await.unwrap(); + + record(&store, Config::default(), 3..6).await; + let records = window(&store).await; + assert_eq!(ranges(&records, "video"), vec![(0, 0), (1, 1), (2, 2), (4, 4), (5, 5)]); + check_objects(&store, &records).await; + // An archive deletes nothing; the orphan stays invisible. + assert!( + stored_groups(&store) + .await + .contains(&Key::groups("video", 3..=3).unwrap()) + ); + assert!(!referenced(&records).contains(&Key::groups("video", 3..=3).unwrap())); + } + + #[tokio::test] + async fn a_failed_recovery_deletes_nothing() { + let mock = Mock::memory(); + let store = Store::new(mock.clone(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(2), Duration::ZERO)); + record(&store, config.clone(), 0..6).await; + store.put_groups("video", &orphan(1)).await.unwrap(); + let before = stored_groups(&store).await; + + let failing = mock.fork(); + failing.fail_lists(); + let source = broadcast::Info::new().produce(); + let result = Writer::new(Store::new(failing, "rec"), source.consume(), config.clone()).await; + assert!(matches!(result, Err(Error::Store(_)))); + + // A missing timeline object leaves the retained window unrecoverable. + store.delete(&Key::segments(TIMELINE, 3).unwrap()).await.unwrap(); + let result = Writer::new(store.clone(), source.consume(), config).await; + assert!(matches!(result, Err(Error::Timeline(_)))); + + assert_eq!(stored_groups(&store).await, before); + } + + #[tokio::test] + async fn a_dvr_window_longer_than_one_checkpoint_is_recovered() { + let store = Store::new(InMemory::new(), "rec"); + let config = Config::default().with_retention(Retention::new(Duration::from_secs(280), Duration::ZERO)); + record(&store, config, 0..300).await; + + let recovery = recover(&store, TIMELINE, true).await.unwrap(); + let checkpoint = recovery.checkpoint.unwrap(); + let records = window(&store).await; + assert!(records.len() > 256, "the window outgrows one checkpoint"); + assert_eq!(checkpoint.records, records); + assert_eq!(checkpoint.range.end, 300); + assert!(recovery.orphans.is_empty()); + assert_eq!(recovery.floors["video"], 299); + } + + #[tokio::test] + async fn a_timeline_that_does_not_end_at_the_next_segment_fails_recovery() { + let store = Store::new(InMemory::new(), "rec"); + record(&store, Config::default(), 0..6).await; + let before = stored_groups(&store).await; + + // `segments/5` still decodes, but it restates an earlier window, so resuming + // would write the next segment on top of it. + let older = store.get_segments(TIMELINE, 0).await.unwrap(); + store.delete(&Key::segments(TIMELINE, 5).unwrap()).await.unwrap(); + store.put_segments(TIMELINE, 5, &older).await.unwrap(); + + let source = broadcast::Info::new().produce(); + match Writer::new(store.clone(), source.consume(), Config::default()).await { + Err(Error::Timeline(message)) => assert!(message.contains("not segment 6"), "{message}"), + Err(err) => panic!("expected a timeline error, got {err}"), + Ok(_) => panic!("expected recovery to fail"), + } + assert_eq!(stored_groups(&store).await, before); + } +} diff --git a/rs/moq-cli/Cargo.toml b/rs/moq-cli/Cargo.toml index 84846ca97e..bc610ac4ed 100644 --- a/rs/moq-cli/Cargo.toml +++ b/rs/moq-cli/Cargo.toml @@ -26,7 +26,7 @@ path = "src/main.rs" doc = false [features] -default = ["iroh", "cluster-lan", "noq", "websocket", "nvidia"] +default = ["iroh", "cluster-lan", "noq", "websocket", "nvidia", "s3", "gcs", "azure"] iroh = ["moq-tokio/iroh"] # LAN discovery and meshing (`--cluster-lan`). cluster-lan = ["moq-relay/cluster-lan"] @@ -37,6 +37,11 @@ noq = ["moq-tokio/noq"] # congestion-control testing. qlog = ["moq-tokio/qlog"] websocket = ["moq-tokio/websocket"] +# The cloud object stores `export archive` and `import archive` reach (`s3://`, +# `gs://`, `az://`). Local `file://` URLs need none of them. +s3 = ["object_store/aws"] +gcs = ["object_store/gcp"] +azure = ["object_store/azure"] # Device capture (camera + microphone) + encode/publish. Off by default because # it pulls in moq-video + moq-audio capture: on Linux the microphone needs # ALSA/libasound at build time through cpal (the camera needs nothing, moq-v4l @@ -93,10 +98,13 @@ base64 = { workspace = true } bytes = { workspace = true } hang = { workspace = true } humantime = { workspace = true } +moq-archive = { workspace = true } moq-audio = { workspace = true, optional = true, features = ["aac"] } moq-auth = { workspace = true, features = ["serve"] } # `server` enables the HTTP egress server for `moq export hls`; the importer is always available. moq-hls = { workspace = true, features = ["server"] } +# Names the MSF catalog track `export archive` records. +moq-msf = { workspace = true } moq-mux = { workspace = true } # Cluster, LAN mesh, and inbound `/.cluster` auth. default-features off so the # CLI's own feature set (iroh, noq, cluster-lan) is what selects them. @@ -108,6 +116,7 @@ moq-srt = { workspace = true } moq-tokio = { workspace = true, default-features = false, features = ["aws-lc-rs", "tcp", "uds"] } moq-transcode = { workspace = true, optional = true, features = ["openh264"] } moq-video = { workspace = true, optional = true, features = ["openh264"] } +object_store = { workspace = true, features = ["fs"] } pollster = { workspace = true, optional = true } reqwest = { workspace = true, features = ["rustls", "json"] } rustls = { version = "0.23", features = ["aws-lc-rs"], default-features = false } @@ -124,10 +133,8 @@ winit = { version = "0.30.13", optional = true } sd-notify = { workspace = true } [dev-dependencies] -# Authors an MSF-only catalog, which is the one shape that tells a completer reading -# the wrong catalog track apart from one reading the right one: `moq-mux`'s producer -# publishes hang and MSF from the same source, so both answer for an ordinary broadcast. -moq-msf = { path = "../moq-msf" } +# Decodes the timeline the archive round-trip test checks. +moq-json = { path = "../moq-json" } # `test_relay` stands up a real relay for the `fetch` tests to read through. moq-relay = { path = "../moq-relay", default-features = false, features = ["test-support"] } tempfile = { workspace = true } diff --git a/rs/moq-cli/README.md b/rs/moq-cli/README.md index de0241a778..95fd4ef6a7 100644 --- a/rs/moq-cli/README.md +++ b/rs/moq-cli/README.md @@ -19,7 +19,7 @@ Multi-arch images (`linux/amd64` and `linux/arm64`) are published to [Docker Hub ## Usage -`moq-cli` routes endpoints onto a shared MoQ Origin: `moq `. The MoQ side (before the verb) dials with `--connect `, self-hosts QUIC/WebTransport with `--listen `, or self-hosts raw qmux with `--listen-tcp-bind ` / `--listen-unix-bind ` (Unix only). `import` moves media into MoQ, `export` moves it out. The endpoint is a container format (`fmp4`, `ts`, `flv`, ... read from stdin / written to stdout), or a gateway (`hls`, `rtmp`, `srt`, `rtc`). A build with the `play` feature can also render a broadcast locally with `moq play`. +`moq-cli` routes endpoints onto a shared MoQ Origin: `moq `. The MoQ side (before the verb) dials with `--connect `, self-hosts QUIC/WebTransport with `--listen `, or self-hosts raw qmux with `--listen-tcp-bind ` / `--listen-unix-bind ` (Unix only). `import` moves media into MoQ, `export` moves it out. The endpoint is a container format (`fmp4`, `ts`, `flv`, ... read from stdin / written to stdout), or a gateway (`hls`, `rtmp`, `srt`, `rtc`, `archive`). A build with the `play` feature can also render a broadcast locally with `moq play`. Separate additional stages with `--` to bridge several broadcasts (or both directions) over one connection, each naming its own `--broadcast`: diff --git a/rs/moq-cli/src/archive.rs b/rs/moq-cli/src/archive.rs new file mode 100644 index 0000000000..2689096655 --- /dev/null +++ b/rs/moq-cli/src/archive.rs @@ -0,0 +1,474 @@ +//! Archive endpoints: record one broadcast into an object store (export), or +//! republish a recording from one (import), through `moq-archive`. + +use std::collections::HashSet; +use std::time::Duration; + +use anyhow::Context; +use hang::moq_net; +use moq_archive::writer::{Control, Retention}; +use moq_mux::catalog::{CatalogFormat, Stream as _}; +use object_store::ObjectStore; +use url::Url; + +use crate::moq::notify_ready; + +/// How long expired objects outlive the timeline that dropped them, unless +/// `--retention-grace` says otherwise. +const GRACE: Duration = Duration::from_secs(30); + +/// `export archive` args. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct ExportArgs { + /// Where to record: `file:///dir`, `s3://bucket/prefix`, `gs://bucket/prefix`, or `az://container/prefix`. + pub store: Url, + + /// Keep only this much recent content (a DVR), deleting older segments. Unset keeps everything. + #[usage(long)] + pub retention: Option, + + /// How long expired objects outlive the timeline that dropped them, so readers can finish + /// their downloads. Defaults to 30s; needs `--retention`. + #[usage(long)] + pub retention_grace: Option, +} + +/// `import archive` args. +#[derive(usage::Args, Clone)] +#[usage(unknown_flags = "error", args_override_self = false)] +pub struct ImportArgs { + /// The recording to replay, as `export archive` wrote it. + pub store: Url, + + /// Keep following a recording that is still growing, checking for new segments at this + /// interval. Unset replays what is stored now and ends the timeline there. + #[usage(long)] + pub follow: Option, +} + +/// Record the broadcast `name` into `args.store` until the broadcast ends. +/// +/// Reads the broadcast's own catalog: video and audio renditions pace the segments, while the +/// catalog and every other track are recorded alongside without pacing. +pub async fn export( + origin: moq_net::origin::Consumer, + name: String, + format: CatalogFormat, + args: ExportArgs, +) -> anyhow::Result<()> { + let retention = match (args.retention, args.retention_grace) { + (Some(window), grace) => Some(Retention::new( + window.into_std(), + grace.map_or(GRACE, crate::duration::Duration::into_std), + )), + (None, Some(_)) => anyhow::bail!("`--retention-grace` needs `--retention`"), + (None, None) => None, + }; + let catalog_track = catalog_track(format)?; + let store = open(&args.store)?; + + let broadcast = origin + .routed_broadcast(&name) + .await + .with_context(|| format!("broadcast `{name}` is unavailable"))?; + let config = moq_archive::writer::Config::default().with_retention(retention); + let writer = moq_archive::Writer::new(store, broadcast.clone(), config) + .await + .with_context(|| format!("failed to start recording into {}", args.store))?; + let control = writer.control(); + control.track(catalog_track).await?; + let catalog = moq_mux::catalog::Consumer::<()>::new(&broadcast, format).await?; + + tracing::info!(%name, store = %args.store, "recording"); + notify_ready(); + + // The writer decides when the recording is over; the catalog only feeds it tracks. + tokio::select! { + result = writer.run() => result.context("recording failed"), + Err(err) = enroll(control, catalog) => Err(err), + } +} + +/// Republish the recording at `args.store` as the broadcast `name`. +/// +/// The timeline replays as a live track and every other track's groups are served on request. +pub async fn import(origin: moq_net::origin::Producer, name: String, args: ImportArgs) -> anyhow::Result<()> { + let store = open(&args.store)?; + let broadcast = origin.create_broadcast(&name).context("failed to create broadcast")?; + let config = moq_archive::reader::Config::new(hang::timeline::DEFAULT_NAME); + let mut reader = moq_archive::Reader::open(store, &broadcast, config) + .await + .with_context(|| format!("no readable recording at {}", args.store))?; + let serve = reader.serve(); + broadcast + .announce(Default::default()) + .context("failed to announce broadcast")?; + + tracing::info!(%name, store = %args.store, "replaying"); + notify_ready(); + + // The store holds no end marker, so only the caller can say the recording is complete. + let Some(interval) = args.follow.map(crate::duration::Duration::into_std) else { + reader.finish()?; + serve.await; + return Ok(()); + }; + let refresh = async { + loop { + tokio::time::sleep(interval).await; + reader.refresh().await?; + } + }; + tokio::select! { + () = serve => Ok(()), + result = refresh => result, + } +} + +/// Open the object store `url` names, taking cloud credentials from the `AWS_*`, `GOOGLE_*`, +/// and `AZURE_*` environment variables. +fn open(url: &Url) -> anyhow::Result>> { + let credentials = std::env::vars_os() + .filter_map(|(key, value)| Some((key.into_string().ok()?, value.into_string().ok()?))) + .filter(|(key, _)| ["AWS_", "GOOGLE_", "AZURE_"].iter().any(|cloud| key.starts_with(cloud))); + let (store, prefix) = + object_store::parse_url_opts(url, credentials).with_context(|| format!("unsupported archive store {url}"))?; + Ok(moq_archive::Store::new(store, prefix)) +} + +/// The track carrying the catalog `format` reads. +fn catalog_track(format: CatalogFormat) -> anyhow::Result<&'static str> { + Ok(match format { + CatalogFormat::Hang => hang::Catalog::DEFAULT_NAME, + CatalogFormat::HangZ => hang::Catalog::COMPRESSED_NAME, + CatalogFormat::Msf => moq_msf::DEFAULT_NAME, + other => anyhow::bail!("`export archive` cannot record a {other:?} catalog"), + }) +} + +/// Enroll each track as the catalog lists it, until the catalog ends. +async fn enroll( + control: Control, + mut catalog: moq_mux::catalog::Consumer<()>, +) -> anyhow::Result<()> { + let mut tracks = Tracks::default(); + while let Some(snapshot) = catalog.next().await? { + // One rendition's subscription can outrun the rest of this snapshot. Without the + // hold, the writer closes a segment from that rendition alone and the others never + // enter the record. + let _hold = control.reserve(); + for change in tracks.update(&snapshot)? { + match change { + Change::Pacing(name) => control.pacing_track(&name).await?, + Change::Track(name) => control.track(&name).await?, + Change::Remove(name) => control.remove(&name)?, + } + } + } + Ok(()) +} + +/// A change to the recorded track set. +#[derive(Debug, PartialEq, Eq)] +enum Change { + /// Record a rendition that paces the segments. + Pacing(String), + /// Record a track without letting it pace the segments. + Track(String), + /// Stop recording a track the catalog dropped. + Remove(String), +} + +/// The catalog tracks being recorded, diffed against each new snapshot. +#[derive(Default)] +struct Tracks { + /// Every name ever enrolled. The writer never takes a name back. + enrolled: HashSet, + /// The names the latest snapshot listed. + listed: HashSet, +} + +impl Tracks { + /// The changes that bring the recording in line with `catalog`. + /// + /// Refuses a track served from another broadcast, since an archive holds exactly one, and a + /// track that returns after the catalog dropped it, since the writer cannot resume one. + fn update(&mut self, catalog: &hang::Catalog) -> anyhow::Result> { + let video = catalog + .video + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, true)); + let audio = catalog + .audio + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, true)); + let text = catalog + .text + .renditions + .iter() + .map(|(name, c)| (name, &c.broadcast, false)); + let json = catalog.json.tracks.iter().map(|(name, c)| (name, &c.broadcast, false)); + let binary = catalog + .binary + .tracks + .iter() + .map(|(name, c)| (name, &c.broadcast, false)); + + let mut listed = HashSet::new(); + let mut changes = Vec::new(); + for (name, broadcast, pacing) in video.chain(audio).chain(text).chain(json).chain(binary) { + if let Some(broadcast) = broadcast { + anyhow::bail!( + "track `{name}` is served from broadcast `{broadcast}`; an archive records one broadcast" + ); + } + if !listed.insert(name.clone()) || self.listed.contains(name) { + continue; + } + anyhow::ensure!( + self.enrolled.insert(name.clone()), + "track `{name}` returned to the catalog after it was dropped; the recording cannot resume it" + ); + changes.push(match pacing { + true => Change::Pacing(name.clone()), + false => Change::Track(name.clone()), + }); + } + + let mut dropped: Vec<_> = self.listed.difference(&listed).cloned().collect(); + dropped.sort(); + changes.extend(dropped.into_iter().map(Change::Remove)); + self.listed = listed; + Ok(changes) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + use hang::catalog::{AudioCodec, AudioConfig, JsonConfig, Mode}; + use moq_net::{Timescale, Timestamp, group, track}; + + fn audio() -> AudioConfig { + AudioConfig::new(AudioCodec::Opus, 48_000, 2) + } + + /// Renditions pace, data tracks don't, and a later snapshot only adds what is new. + #[test] + fn renditions_pace_and_the_rest_follow() { + let mut tracks = Tracks::default(); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), audio()); + catalog + .json + .tracks + .insert("chat".into(), JsonConfig::new(Mode::Snapshot)); + + assert_eq!( + tracks.update(&catalog).unwrap(), + [Change::Pacing("audio".into()), Change::Track("chat".into())] + ); + + catalog.audio.renditions.insert("audio2".into(), audio()); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Pacing("audio2".into())]); + assert_eq!(tracks.update(&catalog).unwrap(), []); + } + + /// A dropped rendition stops pacing, and coming back is refused rather than silently lost. + #[test] + fn a_dropped_track_is_removed_for_good() { + let mut tracks = Tracks::default(); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), audio()); + tracks.update(&catalog).unwrap(); + + let removed = catalog.audio.renditions.remove("audio").unwrap(); + assert_eq!(tracks.update(&catalog).unwrap(), [Change::Remove("audio".into())]); + + catalog.audio.renditions.insert("audio".into(), removed); + let err = tracks.update(&catalog).unwrap_err().to_string(); + assert!(err.contains("cannot resume"), "{err}"); + } + + /// A rendition living in a sibling broadcast would be missing from the recording. + #[test] + fn another_broadcast_is_refused() { + let mut rendition = audio(); + rendition.broadcast = Some(moq_net::path::RelativeOwned::new("./source")); + let mut catalog = hang::Catalog::default(); + catalog.audio.renditions.insert("audio".into(), rendition); + + let err = Tracks::default().update(&catalog).unwrap_err().to_string(); + assert!(err.contains("broadcast `source`"), "{err}"); + } + + /// What `export archive` records, `import archive` serves back group for group, and the + /// export ends cleanly with its broadcast. + #[tokio::test] + async fn a_recording_replays_its_groups() { + // `open` reads the credential variables. + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let mut broadcast = origin.create_broadcast("live.hang").unwrap(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let track = broadcast.create_track("audio", info).unwrap(); + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), audio()); + }) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let args = ExportArgs { + store: url.clone(), + retention: None, + retention_grace: None, + }; + let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + + for sequence in 0..3 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for offset in [0, 500] { + let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); + group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); + } + group.finish().unwrap(); + } + + // Finish only once the rendition is enrolled, which writes its `.info`. + tokio::time::timeout(Duration::from_secs(10), async { + while !dir.path().join("audio/.info").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("the rendition is enrolled"); + track.finish().unwrap(); + catalog.finish().unwrap(); + broadcast.finish(); + + tokio::time::timeout(Duration::from_secs(10), recording) + .await + .expect("the export ends with its broadcast") + .unwrap() + .expect("the recording succeeds"); + + let replay = moq_tokio::origin::spawn(); + let args = ImportArgs { + store: url, + follow: None, + }; + let serving = tokio::spawn(import(replay.clone(), "replay.hang".into(), args)); + + let consumer = replay.consume().routed_broadcast("replay.hang").await.unwrap(); + let audio = consumer.track("audio").unwrap(); + for sequence in 0..3 { + let mut group = audio.fetch_group(sequence, None).await.unwrap(); + for offset in [0, 500] { + let frame = group.read_frame().await.unwrap().expect("a frame"); + assert_eq!(frame.timestamp.as_millis(), u128::from(sequence * 1000 + offset)); + assert_eq!(frame.payload, format!("{sequence}+{offset}")); + } + assert!(group.read_frame().await.unwrap().is_none()); + } + + serving.abort(); + } + + /// Renditions already live in the opening catalog all land in its first segment. + /// + /// Groups are published before export starts, so the writer can drain the first rendition + /// while the next subscription is still in flight. + #[tokio::test] + async fn an_opening_snapshot_records_every_rendition() { + let _env = crate::test_env::EnvGuard::clear(&[]); + let dir = tempfile::tempdir().unwrap(); + let url = Url::from_directory_path(dir.path()).unwrap(); + + let origin = moq_tokio::origin::spawn(); + let mut broadcast = origin.create_broadcast("live.hang").unwrap(); + let mut catalog = moq_mux::catalog::Producer::new(&mut broadcast, Default::default()).unwrap(); + let info = track::Info::default() + .with_timescale(Timescale::MILLI) + .with_max_age(Duration::from_secs(3600)); + let first = broadcast.create_track("audio", info.clone()).unwrap(); + let second = broadcast.create_track("audio2", info).unwrap(); + for track in [&first, &second] { + for sequence in 0..3 { + let mut group = track.create_group(group::Info { sequence }).unwrap(); + for offset in [0, 500] { + let timestamp = Timestamp::from_millis(sequence * 1000 + offset).unwrap(); + group.write_frame(timestamp, format!("{sequence}+{offset}")).unwrap(); + } + group.finish().unwrap(); + } + } + catalog + .mutate(|catalog| { + catalog.audio.renditions.insert("audio".into(), audio()); + catalog.audio.renditions.insert("audio2".into(), audio()); + }) + .unwrap(); + broadcast.announce(Default::default()).unwrap(); + + let args = ExportArgs { + store: url.clone(), + retention: None, + retention_grace: None, + }; + let recording = tokio::spawn(export(origin.consume(), "live.hang".into(), CatalogFormat::Hang, args)); + + tokio::time::timeout(Duration::from_secs(10), async { + while !dir.path().join("audio/.info").exists() || !dir.path().join("audio2/.info").exists() { + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + .expect("both renditions are enrolled"); + first.finish().unwrap(); + second.finish().unwrap(); + catalog.finish().unwrap(); + broadcast.finish(); + + tokio::time::timeout(Duration::from_secs(10), recording) + .await + .expect("the export ends with its broadcast") + .unwrap() + .expect("the recording succeeds"); + + let store = super::open(&url).unwrap(); + let object = store + .get_segments(hang::timeline::DEFAULT_NAME, 0) + .await + .expect("segment 0"); + let config = moq_json::window::ConsumerConfig::default().with_compression(true); + let mut decoder = moq_json::window::Decoder::::new(config); + for stored in object.groups { + let mut group = decoder.group(); + for frame in stored.frames { + group.decode(&frame.payload).unwrap(); + } + } + let mut opening = None; + while let Some(event) = decoder.next_event() { + if let moq_json::window::Event::Push { value, .. } = event + && value.segment == 0 + { + opening = Some(value); + } + } + let opening = opening.expect("segment 0 has a record"); + assert!(opening.tracks.contains_key("audio"), "{opening:?}"); + assert!(opening.tracks.contains_key("audio2"), "{opening:?}"); + } +} diff --git a/rs/moq-cli/src/args.rs b/rs/moq-cli/src/args.rs index 2e5cff514d..27d87b0f6d 100644 --- a/rs/moq-cli/src/args.rs +++ b/rs/moq-cli/src/args.rs @@ -18,7 +18,7 @@ //! conditional on the subcommand. //! - The endpoint is one subcommand: a container format (`ts`, `fmp4`, ... read //! from stdin on import, written to stdout on export) or a gateway (`hls`, -//! `rtmp`, `srt`, `rtc`). Exactly one per stage, so "which endpoint" is +//! `rtmp`, `srt`, `rtc`, `archive`). Exactly one per stage, so "which endpoint" is //! unambiguous and there's no silently-ignored flag. //! - `--` starts another stage on the same Origin and the same MoQ attachment, so //! one process can bridge several broadcasts (or both directions at once). Usage @@ -761,6 +761,8 @@ pub enum ImportSource { Srt(crate::srt::Args), /// WebRTC: WHEP client pulling a remote (`--connect`) or WHIP server accepting publishes (`--listen`). Rtc(crate::rtc::Args), + /// Replay a recording from an object store, serving its groups on demand. + Archive(crate::archive::ImportArgs), /// Capture a local source (camera, display, window, app, microphone) and /// encode natively. Run `moq devices` to list them. #[cfg(feature = "capture")] @@ -854,6 +856,8 @@ pub enum ExportSink { Srt(crate::srt::Args), /// WebRTC: WHIP client pushing to a remote (`--connect`) or WHEP server serving plays (`--listen`). Rtc(crate::rtc::Args), + /// Record the broadcast into an object store until it ends. + Archive(crate::archive::ExportArgs), } impl ExportSink { diff --git a/rs/moq-cli/src/hls.rs b/rs/moq-cli/src/hls.rs index f09a193cde..6a50170c84 100644 --- a/rs/moq-cli/src/hls.rs +++ b/rs/moq-cli/src/hls.rs @@ -34,6 +34,7 @@ pub struct ExportArgs { /// Minimum media listed in each rendition's playlist window. Keep it within the /// relay's group-cache retention, since segments are fetched from there on request. + /// A timeline durable in a catalog-named store lists everything it retains instead. #[usage(long, default = "16s")] pub window: crate::duration::Duration, diff --git a/rs/moq-cli/src/main.rs b/rs/moq-cli/src/main.rs index 27ee260451..018c7bbb3a 100644 --- a/rs/moq-cli/src/main.rs +++ b/rs/moq-cli/src/main.rs @@ -4,6 +4,7 @@ //! grammar; this module orchestrates the shared Origin and spawns the MoQ side //! plus every stage's endpoint. +mod archive; mod args; mod auth; mod complete; @@ -638,6 +639,11 @@ fn spawn_import( tasks.spawn(rtc::connect_import(target(name), url)); } } + ImportSource::Archive(args) => { + // A replay serves the retention the recording was made with. + anyhow::ensure!(max_age.is_none(), "`--max-age` does not apply to `import archive`"); + tasks.spawn(archive::import(origin.clone(), name, args)); + } #[cfg(feature = "capture")] ImportSource::Capture(capture) => { warn_if_missing_format(&name); @@ -717,6 +723,18 @@ fn spawn_export( tasks.spawn(rtc::connect_export(origin.consume(), url, name)); } } + ExportSink::Archive(args) => { + anyhow::ensure!( + export.select.is_empty(), + "`export archive` records every rendition; drop the selection flags" + ); + let format = export + .catalog_format + .map(Into::into) + .or_else(|| moq_mux::catalog::CatalogFormat::detect(&name)) + .unwrap_or_default(); + tasks.spawn(archive::export(origin.consume(), name, format, args)); + } _ => unreachable!("container formats are handled by stdout_format above"), } } diff --git a/rs/moq-cli/src/subscribe.rs b/rs/moq-cli/src/subscribe.rs index 0680d83f8b..86cb0ae3a4 100644 --- a/rs/moq-cli/src/subscribe.rs +++ b/rs/moq-cli/src/subscribe.rs @@ -105,6 +105,14 @@ pub struct SelectArgs { } impl SelectArgs { + /// Whether no selection flag was given. + pub(crate) fn is_empty(&self) -> bool { + self.video_name.is_none() + && self.video_codec.is_none() + && self.audio_name.is_none() + && self.audio_codec.is_none() + } + /// Build the rendition selection shared by stdout exports and native playback. /// /// `force` takes the place of `--video-codec`, for a sink whose format implies diff --git a/rs/moq-hls/Cargo.toml b/rs/moq-hls/Cargo.toml index ece64dbde1..a201e05e2b 100644 --- a/rs/moq-hls/Cargo.toml +++ b/rs/moq-hls/Cargo.toml @@ -36,6 +36,10 @@ tracing = { workspace = true } url = { workspace = true } [dev-dependencies] +async-trait = "0.1" +futures = { workspace = true } +moq-archive = { path = "../moq-archive" } +moq-json = { path = "../moq-json" } moq-tokio = { path = "../moq-tokio", default-features = false, features = ["tcp", "noq", "aws-lc-rs"] } tokio = { workspace = true, features = ["test-util"] } tower = { version = "0.5", features = ["util"] } diff --git a/rs/moq-hls/src/export/archive_tests.rs b/rs/moq-hls/src/export/archive_tests.rs new file mode 100644 index 0000000000..40f327c8f6 --- /dev/null +++ b/rs/moq-hls/src/export/archive_tests.rs @@ -0,0 +1,562 @@ +//! Serving a `moq-archive` recording as HLS. +//! +//! A [`moq_archive::Reader`] replays the recording's timeline onto a broadcast and answers FETCH +//! from range-named objects, so the exporter serves it exactly like a live broadcast. These tests +//! pin the storage traffic that composition produces: playlists read only the timeline, and a +//! segment GETs exactly one object of the requested rendition. + +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; +use std::time::Duration; + +use bytes::{Bytes, BytesMut}; +use futures::stream::BoxStream; +use hang::timeline::{Range, Record}; +use moq_archive::object_store::memory::InMemory; +use moq_archive::object_store::path::Path; +use moq_archive::object_store::{ + self, CopyOptions, GetOptions, GetResult, ListResult, MultipartUpload, ObjectMeta, ObjectStore, + PutMultipartOptions, PutOptions, PutPayload, PutResult, +}; +use moq_archive::{Frame, Group, Info, Object, Store, reader}; +use moq_json::window; + +use super::*; + +const TIMELINE: &str = hang::timeline::DEFAULT_NAME; + +/// In-memory store that records every GET path and counts listings. +#[derive(Debug, Clone, Default)] +struct Counting { + inner: Arc, + gets: Arc>>, + lists: Arc, + /// Fail every media GET, so nothing can quietly depend on one. + reject_media: Arc, +} + +impl Counting { + /// Every GET since the last call. + fn take(&self) -> Vec { + std::mem::take(&mut *self.gets.lock().unwrap()) + } + + /// Listings since the store was created. + fn lists(&self) -> usize { + self.lists.load(Ordering::SeqCst) + } + + fn reject_media(&self, reject: bool) { + self.reject_media.store(reject, Ordering::SeqCst); + } +} + +impl std::fmt::Display for Counting { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "Counting") + } +} + +#[async_trait::async_trait] +impl ObjectStore for Counting { + async fn put_opts( + &self, + location: &Path, + payload: PutPayload, + opts: PutOptions, + ) -> object_store::Result { + self.inner.put_opts(location, payload, opts).await + } + + async fn put_multipart_opts( + &self, + location: &Path, + opts: PutMultipartOptions, + ) -> object_store::Result> { + self.inner.put_multipart_opts(location, opts).await + } + + async fn get_opts(&self, location: &Path, options: GetOptions) -> object_store::Result { + self.gets.lock().unwrap().push(location.to_string()); + if self.reject_media.load(Ordering::SeqCst) && is_media(location.as_ref()) { + return Err(object_store::Error::NotImplemented { + operation: "media GET".into(), + implementer: "Counting".into(), + }); + } + self.inner.get_opts(location, options).await + } + + fn delete_stream( + &self, + locations: BoxStream<'static, object_store::Result>, + ) -> BoxStream<'static, object_store::Result> { + self.inner.delete_stream(locations) + } + + fn list(&self, prefix: Option<&Path>) -> BoxStream<'static, object_store::Result> { + self.lists.fetch_add(1, Ordering::SeqCst); + self.inner.list(prefix) + } + + async fn list_with_delimiter(&self, prefix: Option<&Path>) -> object_store::Result { + self.inner.list_with_delimiter(prefix).await + } + + async fn copy_opts(&self, from: &Path, to: &Path, options: CopyOptions) -> object_store::Result<()> { + self.inner.copy_opts(from, to, options).await + } +} + +/// Writes recording objects the way `moq_archive::Writer` lays them out, with full control over +/// each record's timing and ranges. +struct Recording { + store: Store, + encoder: window::Encoder, + /// Next timeline group sequence. + sequence: u64, +} + +impl Recording { + async fn new(tracks: &[&str]) -> Self { + let store = Store::new(Counting::default(), "rec"); + store.put_info(TIMELINE, &Info::new(0, 1000).unwrap()).await.unwrap(); + for track in tracks { + // Legacy media is stamped in microseconds. + store.put_info(track, &Info::new(1, 1_000_000).unwrap()).await.unwrap(); + } + let config = window::ProducerConfig::default() + .with_compression(true) + .with_op_ratio(0); + Self { + store, + encoder: window::Encoder::new(config), + sequence: 0, + } + } + + /// Store one object for `track` holding `groups`, each a list of frame timestamps in micros. + async fn media(&self, track: &str, groups: &[(u64, &[u64])]) { + let groups = groups + .iter() + .map(|&(sequence, frames)| Group { + sequence, + frames: frames + .iter() + .enumerate() + .map(|(index, µs)| legacy(track, micros, index == 0)) + .collect(), + }) + .collect(); + self.store.put_groups(track, &Object { groups }).await.unwrap(); + } + + /// Commit `record` as timeline segment `record.segment`, popping `pop` older records. + async fn commit(&mut self, record: &Record, pop: u64) { + let mut payloads = Vec::new(); + let pending = self.encoder.push(record).unwrap(); + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + if let Some(pending) = self.encoder.pop(pop).unwrap() { + payloads.push((pending.keyframe, pending.payload.clone())); + pending.commit(); + } + + let mut groups: Vec = Vec::new(); + for (keyframe, payload) in payloads { + if keyframe || groups.is_empty() { + groups.push(Group { + sequence: self.sequence, + frames: Vec::new(), + }); + self.sequence += 1; + } + let frame = Frame { + timestamp: record.pts, + payload, + }; + groups.last_mut().unwrap().frames.push(frame); + } + self.store + .put_segments(TIMELINE, record.segment, &Object { groups }) + .await + .unwrap(); + } + + /// Every GET since the last call. + fn gets(&self) -> Vec { + self.store.inner().take() + } +} + +/// One Legacy frame: a VP8 keyframe for video (geometry the muxer can parse), filler otherwise. +fn legacy(track: &str, micros: u64, keyframe: bool) -> Frame { + let payload: &'static [u8] = match (track.starts_with("audio"), keyframe) { + (true, _) => &[0xFC, 0xFF, 0xFE], + (false, true) => &[0x10, 0x00, 0x00, 0x9d, 0x01, 0x2a, 0x40, 0x01, 0xf0, 0x00], + (false, false) => &[0x31, 0x00, 0x00], + }; + let frame = hang::container::Frame { + timestamp: moq_net::Timestamp::from_micros(micros).unwrap(), + payload: Bytes::from_static(payload), + }; + let mut encoded = BytesMut::new(); + frame.encode(&mut encoded).unwrap(); + Frame { + timestamp: micros, + payload: encoded.freeze(), + } +} + +fn record(segment: u64, pts: u64, duration: u64, tracks: &[(&str, u64, u64)]) -> Record { + let mut record = Record::new(segment, pts, duration); + for &(track, start, end) in tracks { + record.tracks.insert(track.to_string(), vec![Range::new(start, end)]); + } + record +} + +/// The archive entry a replay advertises: the recording's timeline, durable in its store. +fn durable() -> hang::catalog::Archive { + let mut archive = hang::catalog::Archive::new(TIMELINE); + archive.store = Some("memory:///rec/".parse().unwrap()); + archive.version = Some(hang::catalog::Archive::VERSION); + archive +} + +/// The catalog an exporter is handed: every rendition's config, plus `archive`. Out-of-band +/// configs, so no init needs media. +fn catalog(archive: hang::catalog::Archive) -> hang::Catalog { + let mut catalog = hang::Catalog::default(); + catalog.archive = Some(archive); + for (name, width, height) in [("360p", 640, 360), ("1080p", 1920, 1080)] { + let mut config = hang::catalog::VideoConfig::new(hang::catalog::VideoCodec::VP8); + config.coded_width = Some(width); + config.coded_height = Some(height); + catalog.video.renditions.insert(name.to_string(), config); + } + let audio = hang::catalog::AudioConfig::new(hang::catalog::AudioCodec::Opus, 48_000, 2); + catalog.audio.renditions.insert("audio".to_string(), audio); + catalog +} + +/// A recording served back through an origin: the reader's broadcast carries the supplied +/// catalog, and a broadcaster exports it. +struct Replay { + /// Taken to supply finality. + reader: Option>, + broadcaster: Arc, + _catalog: moq_json::snapshot::Producer, + _broadcast: moq_net::broadcast::Producer, + _origin: moq_net::origin::Producer, +} + +impl Replay { + async fn open(recording: &Recording, cache: u64, archive: hang::catalog::Archive) -> Self { + let (origin, driver) = moq_net::origin::Producer::new(moq_net::origin::Config::default()); + tokio::spawn(moq_net::time::run(driver)); + let broadcast = origin.create_broadcast("rec").unwrap(); + + let track = broadcast + .create_track(hang::Catalog::DEFAULT_NAME, hang::Catalog::default_track_info()) + .unwrap(); + let mut json = moq_json::snapshot::Config::default(); + json.delta_ratio = 0; + let mut catalog = moq_json::snapshot::Producer::new(track, json); + catalog.update(&self::catalog(archive)).unwrap(); + + let config = reader::Config::new(TIMELINE).with_cache(cache); + let reader = moq_archive::Reader::open(recording.store.clone(), &broadcast, config) + .await + .unwrap(); + tokio::spawn(reader.serve()); + broadcast.announce(Default::default()).unwrap(); + for _ in 0..10 { + tokio::task::yield_now().await; + } + + let source = moq_mux::Source::new(origin.consume(), "rec"); + let broadcaster = Broadcaster::new(source, Config::default()).await.unwrap(); + tokio::time::timeout(Duration::from_secs(5), broadcaster.ready()) + .await + .expect("a rendition becomes playable"); + + Self { + reader: Some(reader), + broadcaster, + _catalog: catalog, + _broadcast: broadcast, + _origin: origin, + } + } + + fn rendition(&self, kind: Kind, name: &str) -> Arc { + self.broadcaster + .rendition(kind, name) + .expect("rendition in the catalog") + } + + async fn playlist(&self, kind: Kind, name: &str) -> String { + let rendition = self.rendition(kind, name); + tokio::time::timeout(Duration::from_secs(5), rendition.playlist(None)) + .await + .expect("playlist renders") + .unwrap() + .expect("playlist is servable") + } + + /// Wait until `name`'s playlist satisfies `ready`, then return it. + async fn playlist_until(&self, kind: Kind, name: &str, ready: impl Fn(&str) -> bool) -> String { + for _ in 0..500 { + let playlist = self.playlist(kind, name).await; + if ready(&playlist) { + return playlist; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + panic!("{name} playlist never became ready"); + } +} + +fn is_media(path: &str) -> bool { + path.contains("/groups/") +} + +/// Three aligned 2s segments: one keyframe group per video rendition, four audio groups each. +async fn three_segments() -> Recording { + segments(3).await +} + +/// `count` aligned 2s segments, laid out like [`three_segments`]. +async fn segments(count: u64) -> Recording { + let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; + for segment in 0..count { + let pts = segment * 2_000_000; + for video in ["360p", "1080p"] { + recording.media(video, &[(segment, &[pts, pts + 1_000_000])]).await; + } + let audio: Vec<(u64, [u64; 1])> = (0..4).map(|i| (segment * 4 + i, [pts + i * 500_000])).collect(); + let audio: Vec<(u64, &[u64])> = audio + .iter() + .map(|(sequence, frames)| (*sequence, &frames[..])) + .collect(); + recording.media("audio", &audio).await; + + let tracks = [ + ("360p", segment, segment), + ("1080p", segment, segment), + ("audio", segment * 4, segment * 4 + 3), + ]; + recording + .commit(&record(segment, segment * 2000, 2000, &tracks), 0) + .await; + } + recording +} + +#[tokio::test] +async fn playlists_read_only_the_timeline_and_segments_one_object() { + let recording = three_segments().await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + + let master = replay.broadcaster.master_playlist(None); + assert!(master.contains("video/360p/media.m3u8") && master.contains("video/1080p/media.m3u8")); + + // Render and reload every playlist with media GETs refused: aligned numbering, and not one + // media GET. + recording.store.inner().reject_media(true); + for _ in 0..2 { + for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { + let playlist = replay.playlist(kind, name).await; + for segment in 0..3 { + assert!(playlist.contains(&format!("seg/{segment}.m4s\n")), "{name}: {playlist}"); + } + assert!(!playlist.contains("#EXT-X-ENDLIST"), "no finality was supplied"); + } + } + let gets = recording.gets(); + assert!(gets.iter().any(|path| path.contains("/segments/")), "{gets:?}"); + assert!( + !gets.iter().any(|path| is_media(path)), + "playlists must not GET media: {gets:?}" + ); + recording.store.inner().reject_media(false); + // A range-bearing segment URI resolves its object directly: no listing, no index object. + let lists = recording.store.inner().lists(); + + // Switching renditions downloads only the selected rendition's object. + let low = replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); + assert_eq!(&low[4..8], b"moof"); + assert_eq!( + recording.gets(), + [ + "rec/360p/.info", + "rec/360p/groups/0000000000000000001.0000000000000000001" + ] + ); + let high = replay + .rendition(Kind::Video, "1080p") + .segment(2) + .await + .unwrap() + .unwrap(); + assert_eq!(&high[4..8], b"moof"); + assert_eq!( + recording.gets(), + [ + "rec/1080p/.info", + "rec/1080p/groups/0000000000000000002.0000000000000000002" + ] + ); + + // An audio segment spans four groups but still costs one object GET. + let audio = replay + .rendition(Kind::Audio, "audio") + .segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + assert_eq!( + recording.gets(), + [ + "rec/audio/.info", + "rec/audio/groups/0000000000000000007.0000000000000000004" + ] + ); + + // A repeated request hits the reader's cache, including the immutable `.info`. + replay.rendition(Kind::Video, "360p").segment(1).await.unwrap().unwrap(); + assert_eq!(recording.gets(), Vec::::new()); + assert_eq!(recording.store.inner().lists(), lists, "segments never list"); +} + +#[tokio::test] +async fn a_bounded_cache_rereads_evicted_objects() { + let recording = three_segments().await; + // Too small for any object, so nothing stays cached. + let replay = Replay::open(&recording, 1, durable()).await; + replay.playlist(Kind::Audio, "audio").await; + recording.gets(); + + // Each of the segment's four groups misses the cache and GETs the same object again. + let audio = replay + .rendition(Kind::Audio, "audio") + .segment(1) + .await + .unwrap() + .unwrap(); + assert_eq!(&audio[4..8], b"moof"); + let gets: Vec = recording.gets().into_iter().filter(|path| is_media(path)).collect(); + assert_eq!( + gets, + vec!["rec/audio/groups/0000000000000000007.0000000000000000004"; 4] + ); +} + +#[tokio::test] +async fn missing_track_segments_are_gaps_and_time_jumps_are_discontinuities() { + let mut recording = Recording::new(&["360p", "1080p", "audio"]).await; + recording.media("360p", &[(0, &[0])]).await; + recording.media("1080p", &[(0, &[0])]).await; + recording + .commit(&record(0, 0, 2000, &[("360p", 0, 0), ("1080p", 0, 0)]), 0) + .await; + // 1080p stored nothing for segment 1. + recording.media("360p", &[(1, &[2_000_000])]).await; + recording.commit(&record(1, 2000, 2000, &[("360p", 1, 1)]), 0).await; + // Content time jumps from 4s to 10s. + recording.media("360p", &[(2, &[10_000_000])]).await; + recording.media("1080p", &[(1, &[10_000_000])]).await; + recording + .commit(&record(2, 10_000, 2000, &[("360p", 2, 2), ("1080p", 1, 1)]), 0) + .await; + + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + let high = replay.playlist(Kind::Video, "1080p").await; + let expected = concat!( + "#EXTINF:2.00000,\nseg/0.m4s\n", + "#EXT-X-GAP\n#EXTINF:2.00000,\nseg/1.m4s\n", + "#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n", + ); + assert!(high.contains(expected), "{high}"); + let low = replay.playlist(Kind::Video, "360p").await; + assert!(!low.contains("#EXT-X-GAP"), "{low}"); + assert!( + low.contains("#EXT-X-DISCONTINUITY\n#EXTINF:2.00000,\nseg/2.m4s\n"), + "{low}" + ); + + // A gap is never fetched. + recording.gets(); + let rendition = replay.rendition(Kind::Video, "1080p"); + assert!(rendition.segment(1).await.unwrap().is_none()); + assert!(!recording.gets().iter().any(|path| is_media(path))); + let after = rendition.segment(2).await.unwrap().unwrap(); + assert_eq!(&after[4..8], b"moof"); +} + +#[tokio::test] +async fn a_growing_recording_ends_only_on_caller_finality() { + let mut recording = three_segments().await; + let mut replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + replay.playlist(Kind::Video, "360p").await; + + // A DVR commit: segment 3 arrives and segment 0 expires. + recording.media("360p", &[(3, &[6_000_000])]).await; + recording.commit(&record(3, 6000, 2000, &[("360p", 3, 3)]), 1).await; + recording.gets(); + let mut reader = replay.reader.take().unwrap(); + reader.refresh().await.unwrap(); + let playlist = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/3.m4s\n")) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:1\n"), "{playlist}"); + assert!(!playlist.contains("seg/0.m4s"), "{playlist}"); + assert!( + !playlist.contains("#EXT-X-ENDLIST"), + "a recording without finality stays live" + ); + let gets = recording.gets(); + assert!( + !gets.iter().any(|path| is_media(path)), + "following reads only the timeline: {gets:?}" + ); + + // The store holds no completion marker; the caller supplies finality. + reader.finish().unwrap(); + let playlist = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("#EXT-X-ENDLIST")) + .await; + assert!(playlist.contains("seg/3.m4s\n#EXT-X-ENDLIST\n"), "{playlist}"); +} + +/// A durable timeline lists the whole recording past the default 16s window, and DASH offers +/// the whole listed span. A live-style entry, or a `replay` path that moves the durable ranges +/// to another broadcast, keeps the window. +#[tokio::test] +async fn a_durable_timeline_lists_past_the_window() { + let recording = segments(12).await; + let replay = Replay::open(&recording, 64 * 1024 * 1024, durable()).await; + let playlist = replay + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/11.m4s\n")) + .await; + assert!(playlist.contains("#EXT-X-MEDIA-SEQUENCE:0\n"), "{playlist}"); + assert!(playlist.contains("seg/0.m4s\n"), "{playlist}"); + + for (kind, name) in [(Kind::Video, "360p"), (Kind::Video, "1080p"), (Kind::Audio, "audio")] { + replay.rendition(kind, name).init().await.unwrap(); + } + let manifest = replay.broadcaster.manifest(None).expect("manifest renders"); + assert!(manifest.contains("timeShiftBufferDepth=\"PT24.000S\""), "{manifest}"); + + let mut elsewhere = durable(); + elsewhere.replay = Some(moq_net::path::RelativeOwned::new("./recording")); + for archive in [hang::catalog::Archive::new(TIMELINE), elsewhere] { + let live = Replay::open(&recording, 64 * 1024 * 1024, archive).await; + let playlist = live + .playlist_until(Kind::Video, "360p", |playlist| playlist.contains("seg/11.m4s\n")) + .await; + assert!(!playlist.contains("seg/0.m4s\n"), "{playlist}"); + } +} diff --git a/rs/moq-hls/src/export/mod.rs b/rs/moq-hls/src/export/mod.rs index 4b5770a10e..e32845e5e7 100644 --- a/rs/moq-hls/src/export/mod.rs +++ b/rs/moq-hls/src/export/mod.rs @@ -6,7 +6,10 @@ //! per-track group ranges), and media bytes move only when an HTTP client requests a segment, //! which FETCHes exactly the groups that segment covers from the relay cache and transmuxes //! them to CMAF. A broadcast whose catalog advertises no timeline can't be served this way and -//! is skipped. +//! is skipped. A `moq-archive` recording replayed onto a broadcast is served the same way: its +//! playlists read only the stored timeline, and each segment GETs one stored object. +//! An inline-parameter-set codec with no catalog `description` GETs one keyframe +//! group on the first playlist render to build its init, then caches it. //! //! The same machinery serves two kinds of consumer: //! @@ -18,6 +21,8 @@ //! yield every rendition and every finalized segment in order, for mirroring a broadcast to //! storage. +#[cfg(test)] +mod archive_tests; pub mod master; mod mpd; mod playlist; @@ -61,6 +66,10 @@ pub struct Config { /// Minimum duration of media listed in each rendition's playlist window. Older timeline /// records are evicted once the remaining segments still cover this span; keep it within /// the relay's group-cache retention, since segments are fetched from there on request. + /// + /// A durable timeline (its catalog `archive` entry names a `store` and no `replay` path) + /// lists everything it retains instead, since its own retention already bounds it. The + /// window still caps segment `Cache-Control: max-age` for every broadcast. pub window: Duration, } @@ -342,6 +351,9 @@ async fn watch_catalog( // records out to every rendition. if !timeline_started && let Some(archive) = catalog.archive.clone() { timeline_started = true; + if durable(&archive) { + renditions.fanout().unbound(); + } let watcher = tokio::spawn(watch_timeline(broadcast.clone(), archive, renditions.fanout())); *timeline_watcher.lock().unwrap() = Some(watcher); } @@ -358,6 +370,14 @@ async fn watch_catalog( renditions.close(); } +/// Whether every range `archive` advertises stays FETCHable from this broadcast until the +/// timeline pops it: a store makes the ranges durable, and no `replay` path means this +/// broadcast serves them. The catalog states this, so the playlists follow the timeline's own +/// retention rather than a window sized for relay caches. +fn durable(archive: &hang::catalog::Archive) -> bool { + archive.store.is_some() && archive.replay.is_none() +} + /// The broadcast's timeline watcher: read the single timeline track and fan each record out /// to every rendition's window. async fn watch_timeline( diff --git a/rs/moq-hls/src/export/mpd.rs b/rs/moq-hls/src/export/mpd.rs index bea274ff80..11a4c333ee 100644 --- a/rs/moq-hls/src/export/mpd.rs +++ b/rs/moq-hls/src/export/mpd.rs @@ -61,8 +61,9 @@ pub(crate) struct Manifest { pub availability_start: Option, /// When this render happened (`MPD@publishTime`, dynamic only). pub publish: SystemTime, - /// The playlist window (`MPD@timeShiftBufferDepth`, dynamic only). - pub window: Duration, + /// The playlist window (`MPD@timeShiftBufferDepth`, dynamic only), or `None` when the + /// timeline bounds itself and the depth is the span it lists. + pub window: Option, /// The broadcast ended: render a `static` presentation instead of a `dynamic` one. pub finished: bool, /// Video representations, in catalog order. @@ -105,6 +106,23 @@ fn frame_rate(rate: f64) -> Option { } } +/// `units` of `timescale` as a [`Duration`]. +fn duration(units: u64, timescale: u32) -> Duration { + Duration::from_nanos((u128::from(units) * 1_000_000_000 / u128::from(timescale.max(1))) as u64) +} + +/// The longest span any representation lists, oldest segment start to newest segment end. +fn listed_span<'a>(representations: impl Iterator) -> Duration { + representations + .filter_map(|rep| { + let (first, _) = rep.segments.first()?; + let (last, d) = rep.segments.last()?; + Some(duration(last + d - first, rep.timescale)) + }) + .max() + .unwrap_or_default() +} + /// The largest listed segment duration in whole seconds, for `MPD@maxSegmentDuration` (and the /// update cadence). Like HLS's target duration, derived from the segments when the publisher /// declared no bound. @@ -225,10 +243,7 @@ pub(crate) fn render_manifest(manifest: &Manifest, query: Option<&str>) -> Strin let duration = representations() .filter_map(|rep| { let (t, d) = rep.segments.last()?; - let timescale = rep.timescale.max(1) as u64; - Some(Duration::from_nanos( - ((t + d) as u128 * 1_000_000_000 / timescale as u128) as u64, - )) + Some(duration(t + d, rep.timescale)) }) .max() .unwrap_or_default(); @@ -242,14 +257,15 @@ pub(crate) fn render_manifest(manifest: &Manifest, query: Option<&str>) -> Strin // Reload cadence and live delay follow HLS conventions: players refresh about once // per segment and sit a few segments behind the live edge (bounded by the window). let update = Duration::from_secs(target); - let delay = Duration::from_secs(3 * target).min(manifest.window.max(update)); + let window = manifest.window.unwrap_or_else(|| listed_span(representations())); + let delay = Duration::from_secs(3 * target).min(window.max(update)); let _ = write!( out, " type=\"dynamic\" availabilityStartTime=\"{}\" publishTime=\"{}\" minimumUpdatePeriod=\"{}\" timeShiftBufferDepth=\"{}\" suggestedPresentationDelay=\"{}\"", humantime::format_rfc3339_millis(availability), humantime::format_rfc3339_millis(manifest.publish), xs_duration(update), - xs_duration(manifest.window), + xs_duration(window), xs_duration(delay), ); } @@ -315,7 +331,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH + Duration::from_millis(1_751_846_400_123)), publish: SystemTime::UNIX_EPOCH + Duration::from_millis(1_751_846_410_000), - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000), (2_000, 2_000)], false)], audio: vec![audio(vec![(0, 2_000), (2_000, 2_000)], false)], @@ -355,7 +371,7 @@ mod tests { let manifest = Manifest { availability_start: None, publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: true, // The window starts mid-broadcast: presentation time stays anchored at pts 0 (no // presentationTimeOffset), so the duration spans the lead-in and a live session @@ -379,7 +395,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000)], false)], audio: Vec::new(), @@ -397,7 +413,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![rep], audio: Vec::new(), @@ -420,7 +436,7 @@ mod tests { let manifest = Manifest { availability_start: Some(SystemTime::UNIX_EPOCH), publish: SystemTime::UNIX_EPOCH, - window: Duration::from_secs(16), + window: Some(Duration::from_secs(16)), finished: false, video: vec![video(vec![(0, 2_000)], false)], audio: vec![audio(Vec::new(), false)], diff --git a/rs/moq-hls/src/export/rendition.rs b/rs/moq-hls/src/export/rendition.rs index f1f1a03691..abb534c590 100644 --- a/rs/moq-hls/src/export/rendition.rs +++ b/rs/moq-hls/src/export/rendition.rs @@ -343,8 +343,9 @@ impl Rendition { } /// Feed one timeline record into this rendition's window: its own ranges (empty when the - /// record carries none for it, a gap), timed by the record. - pub(crate) fn push(&self, index: u64, entry: &Entry, discontinuity: u64, window: Duration) { + /// record carries none for it, a gap), timed by the record. With no `window`, only source + /// timeline pops trim it. + pub(crate) fn push(&self, index: u64, entry: &Entry, discontinuity: u64, window: Option) { if !self.media.admits(&self.live) { return; } diff --git a/rs/moq-hls/src/export/renditions.rs b/rs/moq-hls/src/export/renditions.rs index 63815fed7c..ab66923500 100644 --- a/rs/moq-hls/src/export/renditions.rs +++ b/rs/moq-hls/src/export/renditions.rs @@ -65,6 +65,10 @@ struct Feed { anchor: Option, /// The publisher run every segment URL carries; late-created renditions inherit it. generation: Option>, + /// The playlist window duration applied on every push (see + /// [`Config::window`](super::Config::window)), or `None` when the source timeline is + /// authoritative and only its pops trim the playlists. + window: Option, } /// The producing side of a broadcast's rendition set. @@ -87,8 +91,6 @@ pub(crate) struct Producer { #[derive(Clone)] pub(crate) struct Fanout { feed: Arc>, - /// The playlist window duration (see [`Config::window`](super::Config::window)), applied on every push. - window: Duration, } impl Producer { @@ -104,8 +106,8 @@ impl Producer { closed: false, anchor: None, generation: None, + window: Some(window), })), - window, }, } } @@ -115,9 +117,10 @@ impl Producer { self.fanout.clone() } - /// The playlist window duration every rendition's window is trimmed to. - pub fn window(&self) -> Duration { - self.fanout.window + /// The playlist window duration every rendition's window is trimmed to, or `None` when + /// only the source timeline trims them. + pub fn window(&self) -> Option { + self.fanout.feed.lock().unwrap().window } /// The estimated wall-clock time of timeline `pts` 0 (see [`Feed::anchor`]); `None` until @@ -184,6 +187,12 @@ impl Producer { } impl Fanout { + /// List every record the source timeline retains, trimming only on its pops, instead of + /// a window. Call before the first record. + pub fn unbound(&self) { + self.feed.lock().unwrap().window = None; + } + /// Fan one timeline record out to every living rendition, and into the replay history. pub fn push(&self, index: u64, entry: Entry) { let mut feed = self.feed.lock().unwrap(); @@ -205,17 +214,19 @@ impl Fanout { let pts = Duration::from(entry.pts); let discontinuity = feed.discontinuities.stamp(pts, pts + entry.duration); feed.history.push_back((index, entry.clone(), discontinuity)); - while feed.history.len() >= 2 { + let window = feed.window; + while let Some(window) = window + && feed.history.len() >= 2 + { let newest = &feed.history.back().unwrap().1; let span = (Duration::from(newest.pts) + newest.duration).saturating_sub(Duration::from(feed.history[1].1.pts)); - if span < self.window { + if span < window { break; } feed.history.pop_front(); } - let window = self.window; feed.targets.retain(|target| { let Some(rendition) = target.upgrade() else { return false; @@ -335,7 +346,7 @@ impl Producer { let mut feed = self.fanout.feed.lock().unwrap(); rendition.label(feed.generation.clone()); for (index, entry, discontinuity) in &feed.history { - rendition.push(*index, entry, *discontinuity, self.fanout.window); + rendition.push(*index, entry, *discontinuity, feed.window); } if feed.ended { rendition.end(); diff --git a/rs/moq-hls/src/export/segments.rs b/rs/moq-hls/src/export/segments.rs index 015f1f2509..6b8a92a0b6 100644 --- a/rs/moq-hls/src/export/segments.rs +++ b/rs/moq-hls/src/export/segments.rs @@ -166,8 +166,9 @@ impl Producer { } } - /// Append a row, evicting the front of the window past `window`. - pub fn push(&self, row: Row, window: Duration) { + /// Append a row, evicting the front of the window past `window`. With no `window`, only + /// source timeline pops remove rows. + pub fn push(&self, row: Row, window: Option) { let Ok(mut state) = self.state.write() else { return; }; @@ -187,7 +188,9 @@ impl Producer { state.rows.push_back(row); // Evict from the front while the remaining rows still cover the window. - while state.rows.len() >= 2 { + while let Some(window) = window + && state.rows.len() >= 2 + { let span = state.rows.back().unwrap().end.saturating_sub(state.rows[1].pts.into()); if span < window { break; @@ -439,8 +442,8 @@ mod tests { #[test] fn every_row_is_listed() { let live = Producer::new(); - live.push(row(0, 0, 0, 2_000), Duration::from_secs(30)); - live.push(row(1, 1, 2_000, 2_000), Duration::from_secs(30)); + live.push(row(0, 0, 0, 2_000), Some(Duration::from_secs(30))); + live.push(row(1, 1, 2_000, 2_000), Some(Duration::from_secs(30))); let window = live.window(); assert_eq!(window.sequence, 0); @@ -456,7 +459,7 @@ mod tests { #[test] fn window_evicts_and_advances_sequence() { let live = Producer::new(); - let window = Duration::from_secs(4); + let window = Some(Duration::from_secs(4)); for i in 0..6u64 { live.push(row(i, i, i * 2_000, 2_000), window); } @@ -473,7 +476,7 @@ mod tests { #[test] fn source_window_pop_removes_playlist_rows() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); for i in 0..4u64 { let mut row = row(i, i, i * 2_000, 2_000); row.index = i + 10; @@ -497,9 +500,9 @@ mod tests { #[test] fn a_skipped_source_range_clears_rows_before_the_next_segment() { let live = Producer::new(); - live.push(row(4, 4, 8_000, 2_000), Duration::from_secs(10)); + live.push(row(4, 4, 8_000, 2_000), Some(Duration::from_secs(10))); live.clear(); - live.push(row(10, 10, 20_000, 2_000), Duration::from_secs(10)); + live.push(row(10, 10, 20_000, 2_000), Some(Duration::from_secs(10))); let snapshot = live.window(); assert_eq!(snapshot.sequence, 10); @@ -567,7 +570,7 @@ mod tests { #[test] fn segment_ranges_and_gaps() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 1_000), window); // Segment 1 is a gap for this rendition: no ranges. live.push( @@ -595,7 +598,7 @@ mod tests { #[test] fn backwards_jump_resets_the_window() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 10_000, 2_000), window); live.push(row(1, 1, 12_000, 2_000), window); live.push(row(2, 2, 1_000, 2_000), window); // restart: pts rewound @@ -612,7 +615,7 @@ mod tests { #[test] fn next_after_walks_segments() { let live = Producer::new(); - let window = Duration::from_secs(30); + let window = Some(Duration::from_secs(30)); live.push(row(0, 0, 0, 2_000), window); live.push(row(1, 1, 2_000, 2_000), window); diff --git a/rs/moq-json/src/window/encoder.rs b/rs/moq-json/src/window/encoder.rs index 58abe657c1..f06d52b988 100644 --- a/rs/moq-json/src/window/encoder.rs +++ b/rs/moq-json/src/window/encoder.rs @@ -86,6 +86,15 @@ impl ProducerConfig { } } +/// A retained window to continue, such as one replayed from stored groups. +#[derive(Debug, Clone, PartialEq)] +pub struct Checkpoint { + /// Absolute index of the oldest retained record, and of the next to be pushed. + pub range: std::ops::Range, + /// The newest retained records, oldest first, ending just before `range.end`. + pub records: Vec, +} + /// One encoded frame, and the group boundary it implies. #[derive(Clone, Debug)] #[non_exhaustive] @@ -138,7 +147,7 @@ impl Pending<'_, T> { impl Drop for Pending<'_, T> { fn drop(&mut self) { if self.edit.is_some() { - self.encoder.resync(); + self.encoder.reset(); } } } @@ -216,8 +225,11 @@ impl Encoder { self.offset..self.start + self.window.len() as u64 } - /// Discard group-local state after an encoded frame did not reach the wire. - fn resync(&mut self) { + /// Discard group-local state, so the next edit opens a new group with a header. + /// + /// Call this whenever the caller closes the current group behind the encoder's back. It is also + /// how the encoder recovers after an encoded frame did not reach the wire. + pub fn reset(&mut self) { self.flate = None; self.op_bytes = 0; self.header_len = 0; @@ -288,7 +300,7 @@ impl Encoder { let encoded = self.frame(bytes)?; let group_bytes = self.header_len.saturating_add(self.op_bytes); if group_bytes > moq_net::group::MAX_CACHE_BYTES { - self.resync(); + self.reset(); Ok(None) } else { Ok(Some(encoded)) @@ -404,6 +416,30 @@ impl Encoder { } impl Encoder { + /// Create an encoder continuing `checkpoint`, so the first edit opens a group restating it. + /// + /// Fails when the records outnumber the range, or the range exceeds the safe integer range. + pub fn resume(config: ProducerConfig, checkpoint: &Checkpoint) -> Result { + let Checkpoint { range, records } = checkpoint; + if range.start > range.end || range.end > MAX_INDEX || records.len() as u64 > range.end - range.start { + return Err(Error::Json("invalid window checkpoint".into())); + } + + let mut encoder = Self::new(config); + let skip = encoder + .config + .checkpoint_records + .map(|limit| records.len().saturating_sub(limit)) + .unwrap_or_default(); + encoder.window = records[skip..] + .iter() + .map(serde_json::to_value) + .collect::>()?; + encoder.offset = range.start; + encoder.start = range.end - encoder.window.len() as u64; + Ok(encoder) + } + /// Append one record to the back of the window. /// /// Emits a push into the open group, or a header restating the window (the new record included) diff --git a/rs/moq-json/src/window/mod.rs b/rs/moq-json/src/window/mod.rs index d0c461b3b0..51d22b18c2 100644 --- a/rs/moq-json/src/window/mod.rs +++ b/rs/moq-json/src/window/mod.rs @@ -26,8 +26,9 @@ //! //! The publisher rolls a group when the ops in it outgrow //! [`ProducerConfig::op_ratio`](ProducerConfig::op_ratio) times the header that opened it, exactly as -//! [`snapshot`](crate::snapshot) rolls on its delta budget. That is purely a compression decision: -//! there is no caller-driven cut and no age bound, and a [`Consumer`] never surfaces it. A header +//! [`snapshot`](crate::snapshot) rolls on its delta budget. That is purely a compression decision +//! with no age bound, and a [`Consumer`] never surfaces it. A caller that stores complete groups +//! can also end one with [`Producer::cut`]; the next edit opens a new group with a header. A header //! restating records a reader already has yields nothing, so however often the publisher rolls, the //! reader sees one continuous stream of [`Event`]s. [`ProducerConfig::checkpoint_records`] bounds //! the suffix repeated on each roll for a long-lived window. @@ -54,7 +55,7 @@ mod producer; pub use consumer::Consumer; pub use decoder::{ConsumerConfig, Decoder, Event, Group}; -pub use encoder::{Encoded, Encoder, Pending, ProducerConfig}; +pub use encoder::{Checkpoint, Encoded, Encoder, Pending, ProducerConfig}; pub use producer::Producer; #[cfg(test)] @@ -290,6 +291,53 @@ mod test { ); } + #[test] + fn a_resumed_encoder_continues_the_window() { + let config = ProducerConfig::default().with_op_ratio(0).with_checkpoint_records(2); + let mut original = Encoder::::new(config.clone()); + let mut decoder = Decoder::::new(ConsumerConfig::default()); + for n in 0..4 { + let frame = original.push(&rec(n)).unwrap(); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + } + let frame = original.pop(1).unwrap().unwrap(); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + std::iter::from_fn(|| decoder.next_event()).for_each(drop); + + let checkpoint = Checkpoint { + range: 1..4, + records: vec![rec(1), rec(2), rec(3)], + }; + let mut resumed = Encoder::resume(config, &checkpoint).unwrap(); + assert_eq!(resumed.range(), 1..4); + assert_eq!( + resumed.window(), + vec![rec(2), rec(3)], + "trimmed to the checkpoint bound" + ); + + let frame = resumed.push(&rec(4)).unwrap(); + assert!(frame.keyframe, "the first edit restates the window"); + decoder.group().decode(&frame.payload).unwrap(); + frame.commit(); + assert_eq!( + std::iter::from_fn(|| decoder.next_event()).collect::>(), + vec![Event::Push { + index: 4, + value: rec(4) + }], + "a reader that kept up sees only the new record" + ); + + let invalid = Checkpoint { + range: 3..4, + records: vec![rec(2), rec(3)], + }; + assert!(Encoder::resume(ProducerConfig::default(), &invalid).is_err()); + } + #[test] fn pops_cross_the_omitted_checkpoint_prefix() { let config = ProducerConfig::default().with_checkpoint_records(2); @@ -604,6 +652,41 @@ mod test { assert!(group.decode(br#"{"offset":0,"records":[]}"#).is_err()); } + #[test] + fn cut_finishes_the_group_and_the_next_edit_restates() { + let track = moq_net::broadcast::Info::new() + .produce() + .create_track("test", None) + .unwrap(); + let replay = moq_net::track::Subscription::default().with_max_age(std::time::Duration::from_secs(30)); + let mut groups = track.subscribe(replay); + let mut producer = Producer::::new(track, ProducerConfig::default()); + + producer.cut().unwrap(); + producer.push(&rec(0)).unwrap(); + producer.push(&rec(1)).unwrap(); + producer.cut().unwrap(); + producer.cut().unwrap(); + producer.push(&rec(2)).unwrap(); + + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(Some(mut first))) = groups.poll_recv_group(&waiter) else { + panic!("the first group is published"); + }; + assert_eq!(first.frame_count(), 2); + assert!(first.poll_finished(&waiter).is_ready(), "cut finishes the open group"); + + let Poll::Ready(Ok(Some(mut second))) = groups.poll_recv_group(&waiter) else { + panic!("the next edit opens a group"); + }; + let Poll::Ready(Ok(Some(header))) = second.poll_read_frame(&waiter) else { + panic!("the group starts with a header"); + }; + let header: Value = serde_json::from_slice(&header.payload).unwrap(); + assert_eq!(header, json!({ "offset": 0, "records": [rec(0), rec(1), rec(2)] })); + assert!(groups.poll_recv_group(&waiter).is_pending()); + } + #[test] fn trailing_data_after_a_frame_is_rejected() { let mut decoder = Decoder::::new(ConsumerConfig::default()); diff --git a/rs/moq-json/src/window/producer.rs b/rs/moq-json/src/window/producer.rs index 1bdfba06e4..50d604bf9c 100644 --- a/rs/moq-json/src/window/producer.rs +++ b/rs/moq-json/src/window/producer.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use serde::Serialize; use serde_json::Value; -use super::{Encoded, Encoder, ProducerConfig}; +use super::{Checkpoint, Encoded, Encoder, ProducerConfig}; use crate::Result; /// Publishes a sliding window of JSON records over a track. @@ -32,13 +32,17 @@ impl Clone for Producer { impl Producer { /// Create a producer that publishes to the given track. pub fn new(track: moq_net::track::Producer, config: ProducerConfig) -> Self { + Self::with_encoder(track, Encoder::new(config)) + } + + fn with_encoder(track: moq_net::track::Producer, encoder: Encoder) -> Self { Self { inner: Arc::new(Mutex::new(Inner { track: Track { inner: track, group: None, }, - encoder: Encoder::new(config), + encoder, finished: false, })), _marker: PhantomData, @@ -70,6 +74,14 @@ impl Producer { self.inner.lock().unwrap().pop(count) } + /// Finish the open group, leaving the next edit to open a replacement with a header. + /// + /// Idempotent: cutting when no group is open does nothing. A caller that stores complete groups + /// cuts after its edits, so every edit so far sits in a group no later frame can extend. + pub fn cut(&mut self) -> Result<()> { + self.inner.lock().unwrap().cut() + } + /// Finish the track, closing any open group. /// /// Borrows rather than consumes, matching snapshot and stream, so the handle stays @@ -81,6 +93,14 @@ impl Producer { } impl Producer { + /// Create a producer continuing `checkpoint` on the given track. + /// + /// The first edit opens a group restating the checkpoint, so a reader that already holds those + /// records sees only what follows. Fails like [`Encoder::resume`]. + pub fn resume(track: moq_net::track::Producer, config: ProducerConfig, checkpoint: &Checkpoint) -> Result { + Ok(Self::with_encoder(track, Encoder::resume(config, checkpoint)?)) + } + /// Append one record to the back of the window. pub fn push(&mut self, value: &T) -> Result<()> { self.inner.lock().unwrap().push(value) @@ -115,6 +135,17 @@ impl Inner { Ok(()) } + fn cut(&mut self) -> Result<()> { + let Some(group) = self.track.group.take() else { + return Ok(()); + }; + // The group closes either way, so reset first: a `finish` error must not leave the encoder + // appending ops to a group that is gone. + self.encoder.reset(); + group.finish()?; + Ok(()) + } + fn finish(&mut self) -> Result<()> { if self.finished { return Ok(()); diff --git a/rs/moq-mux/src/error.rs b/rs/moq-mux/src/error.rs index a0a50989da..c43073dafc 100644 --- a/rs/moq-mux/src/error.rs +++ b/rs/moq-mux/src/error.rs @@ -204,6 +204,11 @@ pub enum Error { #[error("timeline segment {0} was not yielded for deferred publication")] TimelineDeferredRecord(u64), + /// [`timeline::Producer::resume`](crate::timeline::Producer::resume) received a checkpoint + /// whose record at this window index is a different segment. + #[error("timeline checkpoint record at index {0} is a different segment")] + TimelineCheckpoint(u64), + /// Error from a muxer/demuxer that reports via `anyhow` (currently MPEG-TS). /// Boxed in an `Arc` so the enum stays `Clone` (`anyhow::Error` is not). #[error("{0}")] diff --git a/rs/moq-mux/src/timeline.rs b/rs/moq-mux/src/timeline.rs index 4af4902d1e..669df15f49 100644 --- a/rs/moq-mux/src/timeline.rs +++ b/rs/moq-mux/src/timeline.rs @@ -57,6 +57,7 @@ use std::time::Duration; use hang::catalog::Archive; use hang::timeline::{DEFAULT_NAME, Range, Record, RecordExt}; +use moq_json::window::Checkpoint; use moq_net::{Timescale, Timestamp}; @@ -697,6 +698,19 @@ impl Pending { .clear(); self } + + /// Drop one track whose media could not be stored, keeping every other track's ranges. + /// + /// Like [`gap`](Self::gap) for a single track: the record stays committable without advertising + /// objects that are not durable. + pub fn omit(mut self, track: &str) -> Self { + self.record + .as_mut() + .expect("a pending record is present until commit") + .tracks + .remove(track); + self + } } impl Deref for Pending { @@ -726,16 +740,30 @@ struct Output { impl Output { fn prepare(&mut self) -> crate::Result<()> { if self.sink.is_none() && !self.closed { - let info = moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog); - let net = self.broadcast.create_track(DEFAULT_NAME, info)?; - let config = moq_json::window::ProducerConfig::default() - .with_compression(true) - .with_checkpoint_records(CHECKPOINT_RECORDS); - self.sink = Some(moq_json::window::Producer::new(net, config)); + let net = self.create_track()?; + self.sink = Some(moq_json::window::Producer::new(net, Self::config())); } Ok(()) } + /// Create the timeline track now, continuing `checkpoint`. + fn resume(&mut self, checkpoint: &Checkpoint) -> crate::Result<()> { + let net = self.create_track()?; + self.sink = Some(moq_json::window::Producer::resume(net, Self::config(), checkpoint)?); + Ok(()) + } + + fn create_track(&self) -> crate::Result { + let info = moq_net::track::Info::default().with_priority(hang::catalog::PRIORITY.catalog); + Ok(self.broadcast.create_track(DEFAULT_NAME, info)?) + } + + fn config() -> moq_json::window::ProducerConfig { + moq_json::window::ProducerConfig::default() + .with_compression(true) + .with_checkpoint_records(CHECKPOINT_RECORDS) + } + fn push(&mut self, record: &Record) -> crate::Result<()> { self.prepare()?; let Some(sink) = self.sink.as_mut() else { @@ -759,6 +787,17 @@ impl Output { } } + fn flush(&mut self) -> crate::Result<()> { + let Some(sink) = self.sink.as_mut() else { + return Ok(()); + }; + match sink.cut() { + Ok(()) => Ok(()), + Err(moq_json::Error::Net(err)) => Err(err.into()), + Err(err) => Err(err.into()), + } + } + fn finish(&mut self) -> crate::Result<()> { self.closed = true; let Some(sink) = self.sink.as_mut() else { @@ -800,6 +839,31 @@ impl Producer { } } + /// A timeline for `broadcast` continuing `checkpoint`, such as a window recovered from storage. + /// + /// A timeline's window index is its segment number, so the next record is segment + /// `checkpoint.range.end`. Creates the timeline track immediately; its first group restates the + /// checkpoint. Fails when a checkpoint record is not the segment at its index, the checkpoint + /// is malformed, or the track cannot be created. + pub fn resume( + broadcast: &moq_net::broadcast::Producer, + config: Config, + checkpoint: &Checkpoint, + ) -> crate::Result { + let producer = Self::new(broadcast, config); + producer.output.lock().unwrap().resume(checkpoint)?; + + // The encoder accepted the checkpoint, so the records fit before `range.end`. + let start = checkpoint.range.end - checkpoint.records.len() as u64; + for (index, record) in (start..).zip(&checkpoint.records) { + if record.segment != index { + return Err(crate::Error::TimelineCheckpoint(index)); + } + } + producer.segmenter.state.lock().unwrap().next_segment = checkpoint.range.end; + Ok(producer) + } + /// Enroll `name` without letting it influence segmentation, returning its [`Recorder`]. /// /// Its groups are recorded into whichever segment is open when they arrive, but the track @@ -964,6 +1028,14 @@ impl Producer { self.output.lock().unwrap().pop(count) } + /// Close the timeline track's open group, so every push and pop so far sits in a complete group. + /// + /// A recorder stores those groups before committing the next segment. The next edit opens a + /// new group restating the window. + pub fn flush(&self) -> crate::Result<()> { + self.output.lock().unwrap().flush() + } + fn publish_ready(&self) { // Serialize the dequeue and publish steps together. Recorders can report from different // tasks, and taking one record before locking the output would let a later record win the @@ -1290,6 +1362,46 @@ mod test { (broadcast, timeline) } + #[tokio::test] + async fn a_resumed_timeline_continues_the_checkpoint() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let checkpoint = Checkpoint { + range: 1..3, + records: vec![Record::new(1, 1_000, 1_000), Record::new(2, 2_000, 1_000)], + }; + let mut timeline = Producer::resume(&broadcast, Config::default(), &checkpoint).unwrap(); + + let mut video = timeline.pacing_track("video0").unwrap(); + video.record(10, ms(3_000), true); + video.record(11, ms(4_000), true); + video.end(ms(5_000)); + drop(video); + timeline.finish().unwrap(); + + assert_eq!( + drain(&broadcast, &timeline).await, + vec![ + entry(1, 1_000, 1_000, &[]), + entry(2, 2_000, 1_000, &[]), + entry(3, 3_000, 1_000, &[("video0", &[(10, 10)])]), + entry(4, 4_000, 1_000, &[("video0", &[(11, 11)])]), + ] + ); + } + + #[test] + fn a_checkpoint_record_must_be_the_segment_at_its_index() { + let broadcast = moq_net::broadcast::Info::new().produce(); + let checkpoint = Checkpoint { + range: 0..1, + records: vec![Record::new(5, 0, 1_000)], + }; + assert!(matches!( + Producer::resume(&broadcast, Config::default(), &checkpoint), + Err(crate::Error::TimelineCheckpoint(0)) + )); + } + #[tokio::test] async fn deferred_records_are_invisible_until_committed() { let (broadcast, timeline) = setup(); @@ -1345,6 +1457,61 @@ mod test { )); } + #[tokio::test] + async fn omitting_a_track_keeps_the_others() { + let (broadcast, mut timeline) = setup(); + let segmenter = timeline.deferred().unwrap(); + let mut video = segmenter.pacing_track("video0"); + let mut audio = segmenter.pacing_track("audio0"); + video.record(0, ms(0), true); + audio.record(0, ms(0), true); + video.record(1, ms(2_000), true); + audio.record(1, ms(2_000), true); + + let pending = segmenter.next().expect("the complete segment is ready for storage"); + timeline.push(pending.omit("audio0")).unwrap(); + drop((video, audio)); + let tail = segmenter.finish(); + while let Some(pending) = tail.next() { + timeline.push(pending).unwrap(); + } + timeline.finish().unwrap(); + + assert_eq!( + drain(&broadcast, &timeline).await, + vec![ + entry(0, 0, 2_000, &[("video0", &[(0, 0)])]), + entry(1, 2_000, 0, &[("audio0", &[(1, 1)]), ("video0", &[(1, 1)])]), + ] + ); + } + + #[test] + fn flush_closes_the_timeline_group() { + let (broadcast, timeline) = setup(); + let segmenter = timeline.deferred().unwrap(); + let replay = moq_net::track::Subscription::default().with_max_age(Duration::from_secs(30)); + let groups = broadcast.consume().track(DEFAULT_NAME).unwrap().subscribe(replay); + let waiter = kio::Waiter::noop(); + let Poll::Ready(Ok(mut groups)) = groups.poll_ok(&waiter) else { + panic!("the deferred timeline track exists"); + }; + + let mut video = segmenter.pacing_track("video0"); + video.record(0, ms(0), true); + video.record(1, ms(2_000), true); + timeline.push(segmenter.next().unwrap()).unwrap(); + timeline.pop(1).unwrap(); + timeline.flush().unwrap(); + + let Poll::Ready(Ok(Some(mut group))) = groups.poll_recv_group(&waiter) else { + panic!("the pushed record opened a group"); + }; + assert_eq!(group.frame_count(), 2, "the push header and the pop share one group"); + assert!(group.poll_finished(&waiter).is_ready(), "flush closes it"); + assert!(groups.poll_recv_group(&waiter).is_pending()); + } + #[test] fn dropping_a_pending_record_requeues_it() { let (_broadcast, timeline) = setup();