From bc4a428f803cd9a55b8c637420ec112cc14dd6fc Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 23:42:23 -0700 Subject: [PATCH 1/2] chore(quest): claim quest/m1/audio-codecs/layout Co-Authored-By: Claude Opus 5.5 From f8a1cc00d8b9695b8e91ed49291ed8b137ff8810 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 00:14:01 -0700 Subject: [PATCH 2/2] feat(audio): carry surround layouts through decode, remix, and playback Co-Authored-By: Claude Opus 5.5 --- doc/lib/c/index.md | 1 + doc/lib/go/index.md | 6 + doc/lib/kt/index.md | 6 + doc/lib/py/index.md | 6 + doc/lib/rs/moq-audio.md | 10 +- doc/lib/swift/index.md | 6 + js/hang/src/util/aac.test.ts | 8 +- js/hang/src/util/aac.ts | 10 +- quest/m1/audio-codecs/README.md | 1 - quest/m1/audio-codecs/decode-audiotoolbox.md | 1 - quest/m1/audio-codecs/decode-backend.md | 4 - quest/m1/audio-codecs/encode-audiotoolbox.md | 1 - quest/m1/audio-codecs/layout.md | 54 ---- quest/m1/audio-codecs/opus-surround.md | 4 - quest/m2/aac-encode-refusal.md | 1 - quest/m2/audio-decode-mediacodec.md | 1 - quest/m2/audio-decode-mediafoundation.md | 1 - quest/m2/audio-encode-mediacodec.md | 1 - quest/m2/audio-encode-mediafoundation.md | 1 - rs/libmoq/src/audio.rs | 7 +- rs/moq-audio/src/aec.rs | 28 +- rs/moq-audio/src/decode/consumer.rs | 16 +- rs/moq-audio/src/encode/capture.rs | 35 +-- rs/moq-audio/src/encode/encoder.rs | 6 +- rs/moq-audio/src/encode/producer.rs | 16 +- rs/moq-audio/src/layout.rs | 132 ++++++++- rs/moq-audio/src/playback.rs | 7 +- rs/moq-audio/src/playback/device.rs | 120 +++++--- rs/moq-audio/src/playback/driver.rs | 30 +- rs/moq-audio/src/playback/mixer.rs | 205 ++++++++------ rs/moq-audio/src/playback/sink.rs | 170 ++++++++---- rs/moq-audio/src/resample.rs | 271 ++++++++++++++++--- rs/moq-audio/tests/roundtrip.rs | 68 +++++ rs/moq-ffi/src/audio.rs | 7 +- 34 files changed, 901 insertions(+), 340 deletions(-) delete mode 100644 quest/m1/audio-codecs/layout.md diff --git a/doc/lib/c/index.md b/doc/lib/c/index.md index 15de234894..69fec60fbd 100644 --- a/doc/lib/c/index.md +++ b/doc/lib/c/index.md @@ -36,6 +36,7 @@ and `target/include/moq.h`. - **Connection health.** `moq_session_stats()` reports available metrics with per-field validity flags. `moq_session_snapshot()` samples those metrics and the negotiated draft name together from the same connection. Its protocol string is backed by static storage. Both return an offline error between reconnects and leave the destination untouched. `moq_session_bandwidth()` mints an allocator over the send estimate; `moq_bandwidth_reserve` claims a share for an app-owned track, and `moq_encode_video` / `moq_encode_audio` take the same handle so the built-in video encoder follows the grant. - **Raw playback.** Raw audio and video consumers start at the newest cached group when opened, so rebuilding a live decoder skips the retained backlog. - **Raw audio encode.** `moq_audio_encoder_output.codec` names the codec (only `"opus"` today), and `frame_duration_us` sets the Opus frame length: 2500, 5000, 10000, 20000, 40000, or 60000, with 0 meaning the 20 ms default. +- **Audio channel layouts.** A `channels` count also names the speaker layout, by the WAVE convention: 1 is mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, and 8 7.1, interleaved front left, front right, center, LFE, back, then side. `moq_decode_audio` remixes to the count you ask for; past 8 channels the samples pass through but can't be remixed. - **Raw decode output.** `moq_video_decoder_output` selects the decoded CPU pixel format (`MOQ_VIDEO_PIXEL_FORMAT_I420` or `_RGBA`) and target size (`width`/`height`, both zero for native; otherwise even and non-zero). Unknown formats and invalid sizes fail `moq_decode_video` before subscribing; accepted requests deliver exactly that layout or fail on the terminal callback. - **Encoded video metadata.** `moq_video_init.hint` is a zero-initialized `moq_video_hint` with `has_*` flags for coded dimensions, bitrate (bits per second), frame rate, and latency preference. Hints seed a video codec track's catalog; detected dimensions take precedence. - **Client config.** A zeroed `moq_client_config` means the defaults for every knob, which is what lets a new one be appended without disturbing callers. Fields cover protocol (`versions`), TLS (`tls_fingerprints`, `tls_roots`, `tls_cert`/`_key`, `tls_host_name`), transport (`bind`, `connect_timeout_us`, the Happy Eyeballs delays, `websocket_enabled`/`_delay_us`), and tuning (reconnect backoff, `quic_*`). Every duration is in microseconds. A knob whose default isn't zero carries a `has_*` flag, so setting `backoff_timeout_us = 0` needs `has_backoff_timeout = true` to mean "retry forever" rather than "use the default". `moq_client_defaults()` reports what a NULL config dials with. diff --git a/doc/lib/go/index.md b/doc/lib/go/index.md index 001f5a50e4..0540cafb90 100644 --- a/doc/lib/go/index.md +++ b/doc/lib/go/index.md @@ -115,6 +115,12 @@ one: `FetchGroup`/`FetchMediaGroup`, `Dynamic()` with `Requests(ctx)`, `AppendDatagram`/`Datagrams(ctx)`, `SetCatalogSection`, `Demand()` for `Used`/`Unused`, `Session().Stats()`. `moq.IsAuthError` and `moq.IsShutdown` classify errors. `moq.ProtocolError(err)` is the structured protocol failure (scope, verbatim code, kind) when the peer sent one. +Audio `Channels` also names the speaker layout, by the WAVE convention: 1 is +mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, and 8 7.1, interleaved +front left, front right, center, LFE, back, then side. Decoding remixes to the +count you ask for; past 8 channels the samples pass through but can't be +remixed. + `DecodeVideo` picks the decoded CPU pixel layout: `VideoDecoderOutput.Format` is I420 when nil, or `VideoPixelFormatRgba` for four bytes a pixel, and every `VideoDecodedFrame` repeats the layout it was decoded to. `Resize` is best diff --git a/doc/lib/kt/index.md b/doc/lib/kt/index.md index a241ad6b0c..8f863b942b 100644 --- a/doc/lib/kt/index.md +++ b/doc/lib/kt/index.md @@ -94,6 +94,12 @@ native side. `AudioCodec.opus()`, and `AudioEncoderOutput.frameDurationUs` sets the Opus frame length: 2500, 5000, 10000, 20000 (the default), 40000, or 60000. +Audio `channels` also names the speaker layout, by the WAVE convention: 1 is +mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, and 8 7.1, interleaved +front left, front right, center, LFE, back, then side. Decoding remixes to the +count you ask for; past 8 channels the samples pass through but can't be +remixed. + `decodeVideo` picks the decoded CPU pixel layout: `VideoDecoderOutput.format` is `VideoPixelFormat.I420` when null, or `VideoPixelFormat.RGBA` for four bytes a pixel, and every frame repeats the layout it was decoded to. `resize` is best diff --git a/doc/lib/py/index.md b/doc/lib/py/index.md index a41cb7255b..5fe4e12de3 100644 --- a/doc/lib/py/index.md +++ b/doc/lib/py/index.md @@ -110,6 +110,12 @@ WebSocket, TCP, and Unix sockets. `moq.AudioCodec.opus()`, and `AudioEncoderOutput.frame_duration_us` sets the Opus frame length: 2500, 5000, 10000, 20000 (the default), 40000, or 60000. +Audio `channels` also names the speaker layout, by the WAVE convention: 1 is +mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, and 8 7.1, interleaved +front left, front right, center, LFE, back, then side. Decoding remixes to the +count you ask for; past 8 channels the samples pass through but can't be +remixed. + `decode_video` picks the decoded CPU pixel layout: `VideoDecoderOutput.format` is `VideoPixelFormat.I420` when unset, or `VideoPixelFormat.RGBA` for four bytes a pixel, and every frame repeats the layout it was decoded to. `resize` diff --git a/doc/lib/rs/moq-audio.md b/doc/lib/rs/moq-audio.md index 5fee2602e7..2d57b4a5bc 100644 --- a/doc/lib/rs/moq-audio.md +++ b/doc/lib/rs/moq-audio.md @@ -13,8 +13,14 @@ the far end. Everything is Rust, so there is no C toolchain, CMake step, or codec to install. `Layout` names speaker meaning separately from a channel count. `Mono` is center, -`Stereo` is left then right, and `Discrete(n)` preserves unnamed channels without -inventing speaker positions. Encoding keeps source PCM in `encode::Input` and +`Stereo` is left then right, and the surround layouts up to `SevenPointOne` +interleave in the SMPTE/WAVE order (front left, front right, center, LFE, back, +side). A catalog carries only a count, which reads as that count's WAVE default +(`Layout::from_channels`: 6 is 5.1, 8 is 7.1). `Discrete(n)` preserves unnamed +channels without inventing speaker positions, so it passes through but never +remixes. Decoding and playback downmix with the ITU-R BS.775 coefficients and +upmix by leaving the extra speakers silent; the playback mix runs in whatever +layout the output device opened. Encoding keeps source PCM in `encode::Input` and codec requirements in `encode::Settings`; `encode::Options` adds publication policy. Decoding likewise separates low-level `decode::Config`, PCM `decode::Output`, and subscription `decode::Options`. diff --git a/doc/lib/swift/index.md b/doc/lib/swift/index.md index a9aa8b3c44..dc02120fe7 100644 --- a/doc/lib/swift/index.md +++ b/doc/lib/swift/index.md @@ -98,6 +98,12 @@ divides the connection's send estimate; pass it to `encodeVideo` / `AudioCodec.opus()`, and `AudioEncoderOutput.frameDurationUs` sets the Opus frame length: 2500, 5000, 10000, 20000 (the default), 40000, or 60000. +Audio `channels` also names the speaker layout, by the WAVE convention: 1 is +mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, and 8 7.1, interleaved +front left, front right, center, LFE, back, then side. Decoding remixes to the +count you ask for; past 8 channels the samples pass through but can't be +remixed. + `decodeVideo` picks the decoded CPU pixel layout: `VideoDecoderOutput.format` is `.i420` when unset, or `.rgba` for four bytes a pixel, and every frame repeats the layout it was decoded to. `resize` is best effort: only NVDEC has a diff --git a/js/hang/src/util/aac.test.ts b/js/hang/src/util/aac.test.ts index c31059ec80..50dfe903ad 100644 --- a/js/hang/src/util/aac.test.ts +++ b/js/hang/src/util/aac.test.ts @@ -48,9 +48,11 @@ describe("audioSpecificConfig", () => { expect(audioSpecificConfig(48000, 8)).toEqual(new Uint8Array([0x11, 0xb8])); }); - // Unsupported channel counts fall back to stereo (config 2). - it("unsupported channel count falls back to stereo", () => { - expect(audioSpecificConfig(48000, 7)).toEqual(audioSpecificConfig(48000, 2)); + // A count with no channel configuration is refused rather than described as stereo. + it("refuses a channel count with no configuration", () => { + for (const count of [0, 7, 9]) { + expect(() => audioSpecificConfig(48000, count)).toThrow(); + } }); // Non-table sample rates use the 5-byte explicit-frequency form (freqIndex 0xF). diff --git a/js/hang/src/util/aac.ts b/js/hang/src/util/aac.ts index f417e4c0bd..ca0cc2fd45 100644 --- a/js/hang/src/util/aac.ts +++ b/js/hang/src/util/aac.ts @@ -45,19 +45,19 @@ export function pickRate(rate: number): number { const AAC_LC = 2; // audioObjectType for AAC-LC // Map a channel count to its AAC channelConfiguration (ISO 14496-3 Table 1.19). Configs 1..=6 are -// identity (5.1 is config 6 / 6 channels); 8 channels is config 7 (7.1). Anything else has no valid -// config, so fall back to stereo (matching the Rust muxer in rs/moq-mux/src/codec/aac). +// identity (5.1 is config 6 / 6 channels); 8 channels is config 7 (7.1). Anything else has no +// config, and describing it as stereo would mislabel every channel past the first two. function channelConfig(channelCount: number): number { - if (channelCount >= 1 && channelCount <= 6) return channelCount; + if (Number.isInteger(channelCount) && channelCount >= 1 && channelCount <= 6) return channelCount; if (channelCount === 8) return 7; - return 2; + throw new Error(`no AAC channel configuration for ${channelCount} channels`); } /** * Build the AAC-LC AudioSpecificConfig that decoders need when frames are raw (no ADTS header). * * Standard sample rates produce the 2-byte form; non-table rates fall back to the 5-byte form - * with an explicit 24-bit frequency. Mirrors the Rust muxer so JS and Rust agree on the bytes. + * with an explicit 24-bit frequency. Throws for a channel count no configuration names. */ export function audioSpecificConfig(sampleRate: number, channelCount: number): Uint8Array { const config = channelConfig(channelCount); diff --git a/quest/m1/audio-codecs/README.md b/quest/m1/audio-codecs/README.md index 6682cbc9c2..78912f392c 100644 --- a/quest/m1/audio-codecs/README.md +++ b/quest/m1/audio-codecs/README.md @@ -44,7 +44,6 @@ its own decode and encode quest so verification stays per host. ## Quests - [TS export PCE](/quest/m1/audio-codecs/ts-export-pce.md) - a TS export of a PCE-described AAC track writes channel_config 0 and the PCE instead of a count-derived config -- [Layout](/quest/m1/audio-codecs/layout.md) - the settled `Layout` carries up to 7.1 through decode, resample, playback, and the FFI - [Decode seam](/quest/m1/audio-codecs/decode-backend.md) - `decode::backend` selects a platform decoder before symphonia, mirroring moq-video - [AudioToolbox decode](/quest/m1/audio-codecs/decode-audiotoolbox.md) - macOS and iOS decode HE-AAC, multichannel AAC, and what else the framework offers - [Opus surround](/quest/m1/audio-codecs/opus-surround.md) - mapping family 1 decodes on every host through the multistream decoder diff --git a/quest/m1/audio-codecs/decode-audiotoolbox.md b/quest/m1/audio-codecs/decode-audiotoolbox.md index effbebb809..e81db0d554 100644 --- a/quest/m1/audio-codecs/decode-audiotoolbox.md +++ b/quest/m1/audio-codecs/decode-audiotoolbox.md @@ -32,7 +32,6 @@ binding, alongside the `objc2-core-audio-types` the crate already carries. ## Required - [Decode seam](/quest/m1/audio-codecs/decode-backend.md) - the candidate order this backend joins -- [Layout](/quest/m1/audio-codecs/layout.md) - what a multichannel frame is delivered as ## Related diff --git a/quest/m1/audio-codecs/decode-backend.md b/quest/m1/audio-codecs/decode-backend.md index 490f5eac84..271ef84b52 100644 --- a/quest/m1/audio-codecs/decode-backend.md +++ b/quest/m1/audio-codecs/decode-backend.md @@ -39,7 +39,3 @@ stats and `moq play` surface. backend like the video seam's `probe`. The FFI does not expose `Kind` until a consumer asks. - -## Related - -- [Layout](/quest/m1/audio-codecs/layout.md) - independent; the platform backends need both diff --git a/quest/m1/audio-codecs/encode-audiotoolbox.md b/quest/m1/audio-codecs/encode-audiotoolbox.md index 0f86953835..2ceda88280 100644 --- a/quest/m1/audio-codecs/encode-audiotoolbox.md +++ b/quest/m1/audio-codecs/encode-audiotoolbox.md @@ -22,5 +22,4 @@ the encode seam as the platform candidate on macOS and iOS. ## Required - [Encode seam](/quest/m1/audio-codecs/encode-backend.md) - the candidate order this backend joins -- [Layout](/quest/m1/audio-codecs/layout.md) - the input layout the encoder accepts - [AudioToolbox decode](/quest/m1/audio-codecs/decode-audiotoolbox.md) - the round-trip regression decodes through it diff --git a/quest/m1/audio-codecs/layout.md b/quest/m1/audio-codecs/layout.md deleted file mode 100644 index e92dfbbe38..0000000000 --- a/quest/m1/audio-codecs/layout.md +++ /dev/null @@ -1,54 +0,0 @@ -# [M] Channel layouts through decode, playback, and the FFI - -## Goal - -`moq-audio` carries up to 7.1 end to end. A decoded frame says which layout -it is in, the playback mixer downmixes it to whatever the device opened, a raw -PCM consumer can ask for a layout, and the FFI and libmoq expose the same. -Proven with multichannel PCM, the one codec that needs no new decoder. - -## Plan - -Extend the Layout contract settled in main with supported surround layouts -(2.1, quad, 5.0, 5.1, 6.1, 7.1, and the other AAC channelConfiguration and -Opus mapping family 1 entries) in one canonical order, the SMPTE/WAVE order. -Keep the representation extensible. Each codec module maps its native -order into it: AAC's `C L R Ls Rs LFE` and Opus's Vorbis `L C R Ls Rs LFE` -both become `L R C LFE Ls Rs`. An unspecified discrete PCM layout remains -valid for passthrough but is refused for spatial remixing; never invent speaker -positions from an arbitrary channel count. An AAC stream with -channelConfiguration 0 takes its layout from the program config element's -front, side, back, and LFE elements, which `moq_mux::codec::aac` already parses -for the channel count. - -- Reuse the settled PCM descriptors and codec/consumer settings. This quest - adds supported layout behavior rather than replacing public field types. - Frame stays layout-free since the consumer fixed it at construction. -- `resample::remix` becomes a generic remix over layouts: ITU-R BS.775 - coefficients for downmix, silence in the extra speakers for upmix, and the - existing mono/stereo paths as the two-channel special cases. The resampler is - already channel-generic. -- Playback: the mix bus takes the device's layout instead of fixed stereo, the - device chooser prefers the widest well-known layout the device offers (a - 5.1 HDMI sink opens at six channels, a headset at two), and each `Sink` - remixes into the bus. Today's "silence past the front pair" fan-out goes. -- moq-ffi and libmoq keep `channels` as a count, and the count means the - default layout for that count (the WAVE convention: 3 is 2.1, 4 is quad, 6 - is 5.1, 8 is 7.1), delivered or accepted in the canonical order. No record, - `repr(C)` struct, or binding changes, so this stays on `main`; only the - Rust API names the layout, and a binding that needs 5.0 rather than 5.1 is - a later additive field. Document the mapping in every binding's audio doc. -- The catalog does not change: `channel_count` already carries what the - description implies, and `js/hang`'s `aac.ts` stops falling back to stereo - for a count it cannot map. -- Regressions: a 5.1 PCM broadcast published through `encode::Producer` and - read back through `decode::Consumer` at 5.1, at stereo, and at mono; the - mixer fed a 5.1 sink into a stereo bus and a stereo sink into a 5.1 bus; the - device chooser picking six channels when offered. - -Capture stays mono/stereo, and Opus encode stays mapping family 0. - -## Related - -- [Opus surround](/quest/m1/audio-codecs/opus-surround.md) - the first coded multichannel consumer of the layout -- [AudioToolbox decode](/quest/m1/audio-codecs/decode-audiotoolbox.md) - the first platform decoder producing more than stereo diff --git a/quest/m1/audio-codecs/opus-surround.md b/quest/m1/audio-codecs/opus-surround.md index de6247ae0c..059552ddde 100644 --- a/quest/m1/audio-codecs/opus-surround.md +++ b/quest/m1/audio-codecs/opus-surround.md @@ -20,7 +20,3 @@ so this is the one multichannel path Linux gets. channels. - Regression: a family-1 5.1 fixture decodes to six channels in canonical order; family 255 is refused at construction. - -## Required - -- [Layout](/quest/m1/audio-codecs/layout.md) - the type the mapping resolves to diff --git a/quest/m2/aac-encode-refusal.md b/quest/m2/aac-encode-refusal.md index 3008bf0702..316e7f72e2 100644 --- a/quest/m2/aac-encode-refusal.md +++ b/quest/m2/aac-encode-refusal.md @@ -19,4 +19,3 @@ every count from 1 to 8 and one beyond. ## Related - [AAC PCE](https://github.com/moq-dev/moq/pull/4093) - the parse half -- [Layout](/quest/m1/audio-codecs/layout.md) - the layout a PCE would be derived from diff --git a/quest/m2/audio-decode-mediacodec.md b/quest/m2/audio-decode-mediacodec.md index 936bc13a32..5b7df87c71 100644 --- a/quest/m2/audio-decode-mediacodec.md +++ b/quest/m2/audio-decode-mediacodec.md @@ -26,7 +26,6 @@ behind a new optional audio `mediacodec` feature and the decode seam, on `target ## Required - [Decode seam](/quest/m1/audio-codecs/decode-backend.md) - the candidate order this backend joins -- [Layout](/quest/m1/audio-codecs/layout.md) - what a multichannel frame is delivered as ## Related diff --git a/quest/m2/audio-decode-mediafoundation.md b/quest/m2/audio-decode-mediafoundation.md index 20a2518458..1da6a8337c 100644 --- a/quest/m2/audio-decode-mediafoundation.md +++ b/quest/m2/audio-decode-mediafoundation.md @@ -25,7 +25,6 @@ ones. Behind the decode seam as the first candidate on `target_os = ## Required - [Decode seam](/quest/m1/audio-codecs/decode-backend.md) - the candidate order this backend joins -- [Layout](/quest/m1/audio-codecs/layout.md) - what a multichannel frame is delivered as ## Related diff --git a/quest/m2/audio-encode-mediacodec.md b/quest/m2/audio-encode-mediacodec.md index 7bb26674ea..d488ec184b 100644 --- a/quest/m2/audio-encode-mediacodec.md +++ b/quest/m2/audio-encode-mediacodec.md @@ -21,5 +21,4 @@ behind the `mediacodec` feature and the encode seam. ## Required - [Encode seam](/quest/m1/audio-codecs/encode-backend.md) - the candidate order this backend joins -- [Layout](/quest/m1/audio-codecs/layout.md) - the input layout the encoder accepts - [MediaCodec decode](/quest/m2/audio-decode-mediacodec.md) - the round-trip regression decodes through it diff --git a/quest/m2/audio-encode-mediafoundation.md b/quest/m2/audio-encode-mediafoundation.md index b1c3440f6b..8347dd1854 100644 --- a/quest/m2/audio-encode-mediafoundation.md +++ b/quest/m2/audio-encode-mediafoundation.md @@ -21,7 +21,6 @@ behind the encode seam on Windows. ## Required - [Encode seam](/quest/m1/audio-codecs/encode-backend.md) - the candidate order this backend joins -- [Layout](/quest/m1/audio-codecs/layout.md) - the input layout the encoder accepts - [Media Foundation decode](/quest/m2/audio-decode-mediafoundation.md) - the round-trip regression decodes through it ## Related diff --git a/rs/libmoq/src/audio.rs b/rs/libmoq/src/audio.rs index b23a57c18e..d389abc0fe 100644 --- a/rs/libmoq/src/audio.rs +++ b/rs/libmoq/src/audio.rs @@ -67,6 +67,9 @@ pub struct moq_audio_encoder_input { /// `moq_audio_sample_format` discriminant. pub format: u32, pub sample_rate: u32, + /// Interleaved channel count, which also names the speaker layout by the + /// WAVE convention: 1 mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, + /// 8 7.1, in front left, front right, center, LFE, back, side order. pub channels: u32, } @@ -98,7 +101,9 @@ pub struct moq_audio_decoder_output { pub format: u32, /// 0 = deliver at the codec's native sample rate. pub sample_rate: u32, - /// 0 = deliver at the codec's native channel count. + /// 0 = deliver at the codec's native channel count. A count names its + /// layout as `moq_audio_encoder_input.channels` describes, and the decoder + /// remixes to it. pub channels: u32, /// Upper bound on buffering before skipping a stalled group, in /// microseconds. Same congestion-control knob as diff --git a/rs/moq-audio/src/aec.rs b/rs/moq-audio/src/aec.rs index 607d0111ca..da24dda60a 100644 --- a/rs/moq-audio/src/aec.rs +++ b/rs/moq-audio/src/aec.rs @@ -40,7 +40,7 @@ use sonora::config::{EchoCanceller, GainController2, NoiseSuppression, NoiseSupp use sonora::{AudioProcessing, StreamConfig}; use crate::Error; -use crate::playback::{self, BUS_CHANNELS}; +use crate::playback::{self, REFERENCE_CHANNELS}; /// Sample rate the echo reference is resampled to on its way out of the mixer. /// @@ -187,7 +187,7 @@ impl Control { "echo cancellation needs a microphone between 8 and 384 kHz (got {sample_rate})" ))); } - if channels == 0 || channels > BUS_CHANNELS as u32 { + if channels == 0 || channels > REFERENCE_CHANNELS as u32 { return Err(Error::Unsupported(format!( "echo cancellation accepts a mono or stereo microphone (got {channels} channels)" ))); @@ -381,9 +381,9 @@ impl State { self.pending = Vec::with_capacity(headroom); self.processed = Vec::with_capacity(headroom); - self.reference_frame = vec![0.0; REFERENCE_FRAME * BUS_CHANNELS]; - self.render_in = vec![0.0; REFERENCE_FRAME * BUS_CHANNELS]; - self.render_out = vec![0.0; REFERENCE_FRAME * BUS_CHANNELS]; + self.reference_frame = vec![0.0; REFERENCE_FRAME * REFERENCE_CHANNELS]; + self.render_in = vec![0.0; REFERENCE_FRAME * REFERENCE_CHANNELS]; + self.render_out = vec![0.0; REFERENCE_FRAME * REFERENCE_CHANNELS]; self.capture_in = vec![0.0; frame * channels]; self.capture_out = vec![0.0; frame * channels]; @@ -453,7 +453,7 @@ impl State { if let Some(reference) = reference { while reference.available_frames() >= REFERENCE_FRAME { reference.read_interleaved(reference_frame, false); - deinterleave(reference_frame, render_in, BUS_CHANNELS); + deinterleave(reference_frame, render_in, REFERENCE_CHANNELS); let _ = process_render(processor, render_in, render_out); } } @@ -484,7 +484,7 @@ impl State { /// The format the mixer's reference tap arrives in. fn reference_config() -> StreamConfig { - StreamConfig::new(REFERENCE_RATE, BUS_CHANNELS as u16) + StreamConfig::new(REFERENCE_RATE, REFERENCE_CHANNELS as u16) } /// Feed one reference frame to the echo model. @@ -602,7 +602,7 @@ impl Reference { /// canceller reads it back at [`REFERENCE_RATE`]. fn channel(rate: u32) -> (ResamplingProd, ResamplingCons) { resampling_channel::( - BUS_CHANNELS, + REFERENCE_CHANNELS, rate, REFERENCE_RATE, true, @@ -652,7 +652,7 @@ mod tests { /// Push one 10 ms stereo frame of `value` into the tap. fn play(prod: &mut ResamplingProd, value: f32) { - prod.push_interleaved(&vec![value; REFERENCE_FRAME * BUS_CHANNELS]); + prod.push_interleaved(&vec![value; REFERENCE_FRAME * REFERENCE_CHANNELS]); } #[test] @@ -976,10 +976,10 @@ mod tests { fn round(&mut self) -> (f64, f64) { let played: Vec = (0..self.frame).map(|_| self.noise.next()).collect(); - let mut reference = vec![0.0f32; self.frame * BUS_CHANNELS]; + let mut reference = vec![0.0f32; self.frame * REFERENCE_CHANNELS]; for (i, sample) in played.iter().enumerate() { - reference[i * BUS_CHANNELS] = *sample; - reference[i * BUS_CHANNELS + 1] = *sample; + reference[i * REFERENCE_CHANNELS] = *sample; + reference[i * REFERENCE_CHANNELS + 1] = *sample; } self.prod.push_interleaved(&reference); @@ -1112,13 +1112,13 @@ mod tests { let mut tone = Vec::with_capacity(frames * 2 * 4); for frame in 0..frames { let value = (std::f32::consts::TAU * 440.0 * frame as f32 / 48_000.0).sin() * 0.5; - for _ in 0..BUS_CHANNELS { + for _ in 0..REFERENCE_CHANNELS { tone.extend_from_slice(&value.to_le_bytes()); } } let mut energy = 0.0f64; - let mut buf = vec![0.0f32; REFERENCE_FRAME * BUS_CHANNELS]; + let mut buf = vec![0.0f32; REFERENCE_FRAME * REFERENCE_CHANNELS]; for _ in 0..20 { let _ = sink.write(&tone).expect("write"); diff --git a/rs/moq-audio/src/decode/consumer.rs b/rs/moq-audio/src/decode/consumer.rs index 0f7ee37fb6..5110db70e5 100644 --- a/rs/moq-audio/src/decode/consumer.rs +++ b/rs/moq-audio/src/decode/consumer.rs @@ -5,7 +5,7 @@ use std::collections::VecDeque; use bytes::Bytes; use super::decoder::{Config, Decoder}; -use crate::resample::{Resampler, remix, validate_remix}; +use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Format, Frame, Layout}; /// Where a consumer starts on a track that already holds groups. @@ -63,6 +63,8 @@ pub struct Consumer { decoder: Decoder, track: moq_mux::container::Consumer, resampler: Option, + /// Converts the decoded layout to the output's, when they differ. + remix: Option, options: Options, max_age: std::time::Duration, resolved_sample_rate: u32, @@ -111,7 +113,9 @@ impl Consumer { let decoder = Decoder::new(catalog, &options.decoder)?; let sample_rate = options.output.sample_rate.unwrap_or_else(|| decoder.sample_rate()); let layout = options.output.layout.unwrap_or_else(|| decoder.layout()); - validate_remix(decoder.layout(), layout)?; + let remix = (decoder.layout() != layout) + .then(|| Remix::new(decoder.layout(), layout)) + .transpose()?; let resampler = if sample_rate == decoder.sample_rate() { None @@ -165,6 +169,7 @@ impl Consumer { decoder, track, resampler, + remix, options, max_age, resolved_sample_rate: sample_rate, @@ -438,10 +443,9 @@ impl Consumer { /// Remix and pack decoded PCM into an output frame. fn frame(&self, pcm: Vec, timestamp: moq_net::Timestamp, activity: Activity) -> Result { - let pcm = if self.decoder.layout() == self.resolved_layout { - pcm - } else { - remix(&pcm, self.decoder.layout(), self.resolved_layout)? + let pcm = match &self.remix { + Some(remix) => remix.process(&pcm), + None => pcm, }; let bytes = self diff --git a/rs/moq-audio/src/encode/capture.rs b/rs/moq-audio/src/encode/capture.rs index e166e19f1a..32d89909be 100644 --- a/rs/moq-audio/src/encode/capture.rs +++ b/rs/moq-audio/src/encode/capture.rs @@ -12,7 +12,7 @@ use moq_mux::catalog::hang::CatalogExt; use super::producer::Reserved; use super::{Input, Options, Producer}; use crate::capture; -use crate::resample::{Resampler, remix, validate_remix}; +use crate::resample::{Remix, Resampler}; use crate::{Error, Format, Frame, Layout as PcmLayout}; /// Backoff bounds for reopening a capture source. The quick first retry covers @@ -445,7 +445,7 @@ impl Driver { None => continue, }, }; - let pcm_layout = match PcmLayout::from_channels(layout.channels) { + let pcm_layout = match device_layout(layout.channels) { Ok(layout) => layout, Err(err) => match self.failed(err, track, desired.revision).await { Some(result) => return Some(result), @@ -891,23 +891,29 @@ impl EncoderOutput<'_, E> { } } +/// The layout of a capture device, which reports only a channel count. That names +/// speakers for mono and stereo alone: a six-channel microphone array is not a +/// 5.1 speaker layout, so it stays discrete rather than being remixed as one. +fn device_layout(channels: u32) -> Result { + match channels { + 0..=2 => PcmLayout::from_channels(channels), + channels => Ok(PcmLayout::Discrete(channels)), + } +} + /// Converts one opened stream's native layout into the producer's fixed input /// layout. A new instance per open keeps filter state out of recovery gaps. struct Converter { - input: capture::Layout, - output: capture::Layout, + remix: Option, resampler: Option, anchor_us: Option, } impl Converter { fn new(input: capture::Layout, output: capture::Layout) -> Result { - if input.channels != output.channels { - validate_remix( - PcmLayout::from_channels(input.channels)?, - PcmLayout::from_channels(output.channels)?, - )?; - } + let remix = (input.channels != output.channels) + .then(|| Remix::new(device_layout(input.channels)?, device_layout(output.channels)?)) + .transpose()?; let resampler = if input.sample_rate == output.sample_rate { None @@ -924,8 +930,7 @@ impl Converter { }; Ok(Self { - input, - output, + remix, resampler, anchor_us: None, }) @@ -956,10 +961,8 @@ impl Converter { let data = resampler.process(&samples.data, moq_net::Timestamp::from_micros(timestamp_us)?)?; samples.replace(data); } - if self.input.channels != self.output.channels { - let input = PcmLayout::from_channels(self.input.channels)?; - let output = PcmLayout::from_channels(self.output.channels)?; - let data = remix(&samples.data, input, output)?; + if let Some(remix) = &self.remix { + let data = remix.process(&samples.data); samples.replace(data); } if samples.data.is_empty() { diff --git a/rs/moq-audio/src/encode/encoder.rs b/rs/moq-audio/src/encode/encoder.rs index ca2045a633..90479471db 100644 --- a/rs/moq-audio/src/encode/encoder.rs +++ b/rs/moq-audio/src/encode/encoder.rs @@ -883,8 +883,10 @@ mod tests { assert!(matches!(Encoder::new(&settings), Err(Error::Unsupported(_)))); } + /// The catalog carries only a count, so a discrete layout comes back as the + /// count's default one, with the samples untouched. #[test] - fn pcm_preserves_discrete_multichannel_layout() { + fn pcm_passes_discrete_multichannel_samples_through() { let settings = Settings { codec: Codec::Pcm, ..Settings::new(48_000, Layout::Discrete(3)) @@ -895,7 +897,7 @@ mod tests { let input = [0.1, 0.2, 0.3].repeat(encoder.frame_size()); let output = decoder.decode(&encoder.encode(&input).unwrap().payload).unwrap(); - assert_eq!(decoder.layout(), Layout::Discrete(3)); + assert_eq!(decoder.layout(), Layout::TwoPointOne); assert_eq!(output.samples, input); } diff --git a/rs/moq-audio/src/encode/producer.rs b/rs/moq-audio/src/encode/producer.rs index dcc3d1c956..1b61545141 100644 --- a/rs/moq-audio/src/encode/producer.rs +++ b/rs/moq-audio/src/encode/producer.rs @@ -10,7 +10,7 @@ use moq_net::Timestamp; use super::encoded::Encoded; use super::encoder::{Encoder, Input, Settings}; -use crate::resample::{Resampler, remix, validate_remix}; +use crate::resample::{Remix, Resampler}; use crate::{Activity, Error, Frame}; /// Encode and publication policy for [`Producer`]. @@ -64,6 +64,8 @@ impl Default for Options { pub struct Producer { encoder: Encoder, input: Input, + /// Converts the input layout to the codec's, when they differ. + remix: Option, resampler: Option, track: moq_mux::container::Producer, _ext: std::marker::PhantomData E>, @@ -140,7 +142,9 @@ impl Reserved { /// Separate from [`encode`](Self::encode), which cannot fail, so a layout the /// codec rejects leaves the reservation intact for another input. pub(crate) fn register(&mut self, input: Input, options: &Options) -> Result { - validate_remix(input.layout, options.settings.layout)?; + let remix = (input.layout != options.settings.layout) + .then(|| Remix::new(input.layout, options.settings.layout)) + .transpose()?; let encoder = Encoder::new(&options.settings)?; let resampler = if input.sample_rate == encoder.codec_rate() { @@ -163,6 +167,7 @@ impl Reserved { Ok(Registered { encoder, input, + remix, resampler, }) } @@ -172,6 +177,7 @@ impl Reserved { Producer { encoder: registered.encoder, input: registered.input, + remix: registered.remix, resampler: registered.resampler, track: self.track, _ext: self._ext, @@ -193,6 +199,7 @@ impl Reserved { pub(crate) struct Registered { encoder: Encoder, input: Input, + remix: Option, resampler: Option, } @@ -337,7 +344,10 @@ impl Producer { let input = &self.input; let (format, channels) = (input.format, input.layout.channels()); let pcm = format.as_interleaved_f32(frame.data.as_ref(), channels)?; - let pcm = remix(&pcm, input.layout, self.encoder.settings().layout)?; + let pcm = match &self.remix { + Some(remix) => remix.process(&pcm), + None => pcm.into_owned(), + }; let pcm: Vec = match self.resampler.as_mut() { Some(r) => r.process(&pcm, frame.timestamp)?, None => pcm, diff --git a/rs/moq-audio/src/layout.rs b/rs/moq-audio/src/layout.rs index 0ff4d0d09b..90373ed715 100644 --- a/rs/moq-audio/src/layout.rs +++ b/rs/moq-audio/src/layout.rs @@ -1,6 +1,11 @@ use crate::Error; /// Speaker meaning and interleaving order for PCM channels. +/// +/// Named layouts interleave in the SMPTE/WAVE order (front left, front right, +/// center, LFE, back left, back right, back center, side left, side right), +/// keeping only the speakers the layout has. Every decoder reorders its codec's +/// native order into this one. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] #[non_exhaustive] pub enum Layout { @@ -10,11 +15,47 @@ pub enum Layout { #[default] Stereo, /// Channels with no declared speaker positions, in source order. + /// + /// Passes through unchanged but can't be remixed into another layout. Discrete(u32), + /// 2.1: left, right, LFE. + TwoPointOne, + /// 3.0: left, right, center. + ThreePointZero, + /// Quad: left, right, back left, back right. + Quad, + /// 4.0: left, right, center, back center. + FourPointZero, + /// 5.0: left, right, center, side left, side right. + FivePointZero, + /// 5.1: left, right, center, LFE, side left, side right. + FivePointOne, + /// 6.1: left, right, center, LFE, back center, side left, side right. + SixPointOne, + /// 7.1: left, right, center, LFE, back left, back right, side left, side right. + SevenPointOne, +} + +/// One speaker position, declared in the canonical interleaving order. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(crate) enum Speaker { + FrontLeft, + FrontRight, + FrontCenter, + Lfe, + BackLeft, + BackRight, + BackCenter, + SideLeft, + SideRight, } impl Layout { - /// Infer today's conventional layout from a channel count. + /// The default layout for a channel count, by the WAVE convention: 1 is mono, + /// 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, and 8 7.1. + /// + /// Any other count is [`Discrete`](Self::Discrete), since there is no + /// convention to take speaker positions from. pub fn from_channels(channels: u32) -> Result { match channels { 0 => Err(Error::Unsupported( @@ -22,6 +63,12 @@ impl Layout { )), 1 => Ok(Self::Mono), 2 => Ok(Self::Stereo), + 3 => Ok(Self::TwoPointOne), + 4 => Ok(Self::Quad), + 5 => Ok(Self::FivePointZero), + 6 => Ok(Self::FivePointOne), + 7 => Ok(Self::SixPointOne), + 8 => Ok(Self::SevenPointOne), channels => Ok(Self::Discrete(channels)), } } @@ -29,12 +76,40 @@ impl Layout { /// Number of interleaved channels in this layout. pub fn channels(self) -> u32 { match self { - Self::Mono => 1, - Self::Stereo => 2, Self::Discrete(channels) => channels, + layout => layout.speakers().map_or(0, <[_]>::len) as u32, } } + /// The speaker each channel feeds, in interleaved order, or `None` for a + /// discrete layout. + pub(crate) fn speakers(self) -> Option<&'static [Speaker]> { + use Speaker::*; + + Some(match self { + Self::Mono => &[FrontCenter], + Self::Stereo => &[FrontLeft, FrontRight], + Self::Discrete(_) => return None, + Self::TwoPointOne => &[FrontLeft, FrontRight, Lfe], + Self::ThreePointZero => &[FrontLeft, FrontRight, FrontCenter], + Self::Quad => &[FrontLeft, FrontRight, BackLeft, BackRight], + Self::FourPointZero => &[FrontLeft, FrontRight, FrontCenter, BackCenter], + Self::FivePointZero => &[FrontLeft, FrontRight, FrontCenter, SideLeft, SideRight], + Self::FivePointOne => &[FrontLeft, FrontRight, FrontCenter, Lfe, SideLeft, SideRight], + Self::SixPointOne => &[FrontLeft, FrontRight, FrontCenter, Lfe, BackCenter, SideLeft, SideRight], + Self::SevenPointOne => &[ + FrontLeft, + FrontRight, + FrontCenter, + Lfe, + BackLeft, + BackRight, + SideLeft, + SideRight, + ], + }) + } + pub(crate) fn validate(self) -> Result<(), Error> { if self.channels() == 0 { return Err(Error::Unsupported( @@ -44,3 +119,54 @@ impl Layout { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn counts_map_to_the_wave_defaults() { + let expected = [ + Layout::Mono, + Layout::Stereo, + Layout::TwoPointOne, + Layout::Quad, + Layout::FivePointZero, + Layout::FivePointOne, + Layout::SixPointOne, + Layout::SevenPointOne, + ]; + for (count, layout) in (1..).zip(expected) { + assert_eq!(Layout::from_channels(count).unwrap(), layout); + assert_eq!(layout.channels(), count); + } + + assert_eq!(Layout::from_channels(9).unwrap(), Layout::Discrete(9)); + assert!(Layout::from_channels(0).is_err()); + } + + /// Canonical order is the WAVE bit order, so every layout's speakers must + /// ascend through [`Speaker`]'s declaration order without repeating. + #[test] + fn speakers_follow_the_canonical_order() { + let layouts = [ + Layout::Mono, + Layout::Stereo, + Layout::TwoPointOne, + Layout::ThreePointZero, + Layout::Quad, + Layout::FourPointZero, + Layout::FivePointZero, + Layout::FivePointOne, + Layout::SixPointOne, + Layout::SevenPointOne, + ]; + for layout in layouts { + let speakers = layout.speakers().unwrap(); + assert!( + speakers.windows(2).all(|pair| (pair[0] as u8) < (pair[1] as u8)), + "{layout:?} is out of order" + ); + } + } +} diff --git a/rs/moq-audio/src/playback.rs b/rs/moq-audio/src/playback.rs index 49e81de354..1f5e514a68 100644 --- a/rs/moq-audio/src/playback.rs +++ b/rs/moq-audio/src/playback.rs @@ -50,7 +50,7 @@ pub use sink::{Control, Input, Sink, Write}; #[cfg(feature = "aec")] pub(crate) use driver::Shared; #[cfg(feature = "aec")] -pub(crate) use mixer::BUS_CHANNELS; +pub(crate) use mixer::REFERENCE_CHANNELS; use crate::Error; @@ -137,13 +137,14 @@ impl Engine { /// Add a stream to the mix, taking PCM in the layout `input` describes. /// /// Independent of the device: several sinks can play at different rates and - /// channel counts, and each is resampled on its way to the mix. One device + /// layouts, and each is resampled and remixed to the device's on its way to + /// the mix. One device /// mixes up to 64 of them, past which this returns an error rather than /// handing back a sink that plays nothing. pub fn sink(&self, input: Input) -> Result { let sink = self .shared - .add(|id, rate| sink::new(id, rate, input, self.shared.clone(), self.handle.clone()))?; + .add(|id, rate, bus| sink::new(id, rate, bus, input, self.shared.clone(), self.handle.clone()))?; // Covers the case where the mixer's command queue was momentarily full, // so a sink is never left silently unmixed. diff --git a/rs/moq-audio/src/playback/device.rs b/rs/moq-audio/src/playback/device.rs index 8d55a28a42..680e6a80f7 100644 --- a/rs/moq-audio/src/playback/device.rs +++ b/rs/moq-audio/src/playback/device.rs @@ -11,10 +11,9 @@ use crate::Error; /// second choice. const RATES: &[u32] = &[48_000, 44_100]; -/// Channel counts to try, best first. The mixer produces stereo, and mono is -/// the only other count worth naming: anything else is a surround layout we -/// would be guessing the speaker order of. -const CHANNELS: &[u16] = &[2, 1]; +/// The widest channel count with a well-known layout. Wider counts name no +/// speaker positions to mix into, so they are never opened. +const MAX_CHANNELS: u16 = 8; /// Sample formats we can write, best first: `f32` is what the mixer produces, /// and the rest are conversions on the way out. @@ -129,16 +128,24 @@ pub(super) fn open(selector: Option<&str>) -> Result { /// Pick the stream format to open `device` with. /// -/// Only considers formats in [`FORMATS`], and prefers in that order: a channel -/// count in [`CHANNELS`], a rate the pipeline already runs at, then a format we -/// write without converting. Failing all of those it takes the highest rate the -/// device supports, since resampling down is kinder than resampling up. +/// Only considers formats in [`FORMATS`] and well-known layouts, and prefers in +/// that order: the device's own channel count, the widest layout, a rate the +/// pipeline already runs at, then a format we write without converting. Failing +/// the rates it takes the highest the device supports, since resampling down is +/// kinder than resampling up. pub(super) fn negotiate(device: &cpal::Device) -> Result { let supported = device .supported_output_configs() .map_err(|err| Error::Playback(format!("cannot enumerate output configs: {err}")))?; - choose(supported).ok_or_else(|| Error::Unsupported("output device offers no sample format we can write".into())) + // What the device is set up for: a 5.1 HDMI sink reports six channels and a + // headset two. Taken first because a sound server's plugin also accepts + // every count up to its limit, where the widest would upmix a headset. + let native = device.default_output_config().ok().map(|config| config.channels()); + + choose(supported, native).ok_or_else(|| { + Error::Unsupported("output device offers no sample format and speaker layout we can write".into()) + }) } /// Pick the best of the stream configurations a device reports. @@ -146,10 +153,14 @@ pub(super) fn negotiate(device: &cpal::Device) -> Result) -> Option { +fn choose( + supported: impl Iterator, + native: Option, +) -> Option { supported .filter(|config| FORMATS.contains(&config.sample_format())) - .min_by_key(preference) + .filter(|config| (1..=MAX_CHANNELS).contains(&config.channels())) + .min_by_key(|config| preference(config, native)) .map(|config| match preferred_rate(&config) { Some(rate) => config.try_with_sample_rate(rate).expect("a rate the range covers"), None => config.with_max_sample_rate(), @@ -164,11 +175,11 @@ fn choose(supported: impl Iterator) -> /// and a pass per channel count that gave up when neither preferred rate matched /// would hand that to the mono device and downmix. The channel count leads /// because a downmix is audible where a resample is not. -fn preference(config: &cpal::SupportedStreamConfigRange) -> (usize, usize, std::cmp::Reverse, usize) { - let channels = CHANNELS - .iter() - .position(|count| *count == config.channels()) - .unwrap_or(CHANNELS.len()); +fn preference( + config: &cpal::SupportedStreamConfigRange, + native: Option, +) -> ((bool, std::cmp::Reverse), usize, std::cmp::Reverse, usize) { + let channels = (native != Some(config.channels()), std::cmp::Reverse(config.channels())); let rate = match preferred_rate(config) { Some(rate) => (RATES.iter().position(|r| *r == rate).expect("from RATES"), rate), // Nothing we asked for, so take the most the device offers: resampling @@ -223,33 +234,72 @@ mod tests { /// the way to a stereo sink. #[test] fn stereo_wins_even_when_the_device_lists_mono_first() { - let chosen = choose([range(1, 48_000, SampleFormat::F32), range(2, 48_000, SampleFormat::F32)].into_iter()) - .expect("a config"); + let chosen = choose( + [range(1, 48_000, SampleFormat::F32), range(2, 48_000, SampleFormat::F32)].into_iter(), + None, + ) + .expect("a config"); assert_eq!(chosen.channels(), 2); } /// Preferring stereo must not refuse a device that has no stereo to offer. #[test] fn mono_is_taken_when_that_is_all_there_is() { - let chosen = choose([range(1, 48_000, SampleFormat::F32)].into_iter()).expect("a config"); + let chosen = choose([range(1, 48_000, SampleFormat::F32)].into_iter(), None).expect("a config"); assert_eq!(chosen.channels(), 1); } - /// Neither preferred count is offered, so the pass that accepts any count - /// has to catch it. Without it a surround-only sink would not open at all. + /// A surround-only sink opens at its own width rather than being refused. #[test] - fn a_count_we_do_not_prefer_still_opens() { - let chosen = choose([range(6, 48_000, SampleFormat::F32)].into_iter()).expect("a config"); + fn a_surround_only_device_opens() { + let chosen = choose([range(6, 48_000, SampleFormat::F32)].into_iter(), None).expect("a config"); assert_eq!(chosen.channels(), 6); } + /// A 5.1 HDMI sink opens at six channels rather than being folded to stereo. + #[test] + fn six_channels_are_chosen_when_offered() { + let offered = || [range(2, 48_000, SampleFormat::F32), range(6, 48_000, SampleFormat::F32)].into_iter(); + assert_eq!(choose(offered(), None).expect("a config").channels(), 6); + assert_eq!(choose(offered(), Some(6)).expect("a config").channels(), 6); + } + + /// A sound server's plugin accepts every count up to its limit, so the + /// device's own count beats the widest: a headset stays at two rather than + /// being upmixed to 7.1 and folded back down by the server. + #[test] + fn the_native_count_beats_the_widest() { + let offered = (1..=8).map(|channels| range(channels, 48_000, SampleFormat::F32)); + assert_eq!(choose(offered, Some(2)).expect("a config").channels(), 2); + } + + /// Past 7.1 there is no convention naming the speakers, so there is nothing + /// to remix into. + #[test] + fn counts_without_a_layout_are_refused() { + assert!(choose([range(12, 48_000, SampleFormat::F32)].into_iter(), Some(12)).is_none()); + let chosen = choose( + [ + range(12, 48_000, SampleFormat::F32), + range(2, 48_000, SampleFormat::F32), + ] + .into_iter(), + Some(12), + ) + .expect("a config"); + assert_eq!(chosen.channels(), 2); + } + /// Rate is preferred within a channel count, not across one: a stereo config /// at an awkward rate beats a mono config at the pipeline's own rate, /// because resampling is inaudible and a downmix is not. #[test] fn channels_outrank_the_sample_rate() { - let chosen = choose([range(1, 48_000, SampleFormat::F32), range(2, 44_100, SampleFormat::F32)].into_iter()) - .expect("a config"); + let chosen = choose( + [range(1, 48_000, SampleFormat::F32), range(2, 44_100, SampleFormat::F32)].into_iter(), + None, + ) + .expect("a config"); assert_eq!((chosen.channels(), chosen.sample_rate()), (2, 44_100)); } @@ -257,8 +307,11 @@ mod tests { /// produces rather than one that costs a conversion. #[test] fn f32_is_preferred_over_a_format_we_convert_to() { - let chosen = choose([range(2, 48_000, SampleFormat::I16), range(2, 48_000, SampleFormat::F32)].into_iter()) - .expect("a config"); + let chosen = choose( + [range(2, 48_000, SampleFormat::I16), range(2, 48_000, SampleFormat::F32)].into_iter(), + None, + ) + .expect("a config"); assert_eq!(chosen.sample_format(), SampleFormat::F32); } @@ -266,8 +319,8 @@ mod tests { /// stream that plays noise. #[test] fn a_device_we_cannot_write_to_is_rejected() { - assert!(choose([range(2, 48_000, SampleFormat::I8)].into_iter()).is_none()); - assert!(choose(std::iter::empty()).is_none()); + assert!(choose([range(2, 48_000, SampleFormat::I8)].into_iter(), None).is_none()); + assert!(choose(std::iter::empty(), None).is_none()); } /// Channels lead even when neither preferred rate is on offer. @@ -278,8 +331,11 @@ mod tests { /// downmix the whole preference exists to avoid. #[test] fn stereo_at_an_awkward_rate_beats_mono_at_a_preferred_one() { - let chosen = choose([range(1, 48_000, SampleFormat::F32), range(2, 96_000, SampleFormat::F32)].into_iter()) - .expect("a config"); + let chosen = choose( + [range(1, 48_000, SampleFormat::F32), range(2, 96_000, SampleFormat::F32)].into_iter(), + None, + ) + .expect("a config"); assert_eq!((chosen.channels(), chosen.sample_rate()), (2, 96_000)); } @@ -293,6 +349,7 @@ mod tests { range(2, 96_000, SampleFormat::F32), ] .into_iter(), + None, ) .expect("a config"); assert_eq!((chosen.channels(), chosen.sample_rate()), (2, 96_000)); @@ -309,6 +366,7 @@ mod tests { range(2, 32_000, SampleFormat::F32), ] .into_iter(), + None, ) .expect("a config"); assert_eq!( diff --git a/rs/moq-audio/src/playback/driver.rs b/rs/moq-audio/src/playback/driver.rs index ce30e3688d..12bbd70fa5 100644 --- a/rs/moq-audio/src/playback/driver.rs +++ b/rs/moq-audio/src/playback/driver.rs @@ -19,7 +19,7 @@ use rand::RngExt; use super::mixer::{self, Mixer}; use super::sink::{Registration, Sink}; -use crate::Error; +use crate::{Error, Layout}; /// Backoff bounds for reopening a device that failed. The first retry is quick because the common /// case is a device that came right back (a USB re-enumerate, a sample-rate change); the ceiling @@ -78,6 +78,9 @@ struct State { /// Rate the device is running at, which is what sinks resample to. Zero /// until the first stream opens. rate: u32, + /// Layout the device is running in, which is what sinks remix to. Stereo + /// stands in until the first stream opens. + layout: Layout, /// Registration channel to the live mixer, replaced every time the stream is /// rebuilt. `None` while no stream is running. mixer: Option>, @@ -108,13 +111,14 @@ struct State { impl Shared { /// Build a sink, register it, and start mixing it. /// - /// `build` is handed the sink's id and the rate its channel should target. + /// `build` is handed the sink's id and the rate and layout its channel should + /// target. /// It runs with no device open too: the registration waits for the next /// restart, so a device that is briefly missing doesn't become an error the /// caller has to retry. pub(super) fn add(&self, build: F) -> Result where - F: FnOnce(u64, u32) -> Result<(Sink, Registration), Error>, + F: FnOnce(u64, u32, Layout) -> Result<(Sink, Registration), Error>, { let mut state = self.state.lock().unwrap(); @@ -131,7 +135,7 @@ impl Shared { // 48 kHz stands in until a device opens and the channel is rebuilt at // the real rate. let rate = if state.rate == 0 { 48_000 } else { state.rate }; - let (sink, mut registration) = build(state.next_id, rate)?; + let (sink, mut registration) = build(state.next_id, rate, state.layout)?; state.next_id += 1; if let Some(mixer) = &state.mixer { @@ -270,11 +274,11 @@ impl Shared { } /// Point every sink at a freshly opened stream: rebuild each channel at - /// `rate` and hand the new consumers to `mixer`. - fn rebind(&self, rate: u32, mixer: SyncSender) { + /// `rate` and `layout` and hand the new consumers to `mixer`. + fn rebind(&self, rate: u32, layout: Layout, mixer: SyncSender) { let mut state = self.state.lock().unwrap(); for sink in &mut state.sinks { - sink.rebuild(rate); + sink.rebuild(rate, layout); sink.attach(&mixer); } @@ -289,6 +293,7 @@ impl Shared { } state.rate = rate; + state.layout = layout; state.mixer = Some(mixer); // The old mixer is gone, and with it every sink it was told about. state.detaching.clear(); @@ -787,7 +792,8 @@ impl Driver { // a full retirement channel is the one case where the mixer has to free // on the audio thread after all. let (retired_tx, retired_rx) = sync_channel(COMMAND_QUEUE); - let mixer = Mixer::new(rx, retired_tx, rate, channels); + let layout = Layout::from_channels(channels as u32)?; + let mixer = Mixer::new(rx, retired_tx, rate, layout)?; let failures = Arc::new(Failures::default()); let reporter = FailureReporter { @@ -799,7 +805,7 @@ impl Driver { .play() .map_err(|err| Error::Playback(format!("cannot start output stream: {err}")))?; - self.shared.rebind(rate, tx); + self.shared.rebind(rate, layout, tx); self.stream = Some(stream); self.failures = Some(failures); // Replaces the previous receiver, dropping anything the old stream @@ -807,7 +813,7 @@ impl Driver { self.retired = Some(retired_rx); self.retry = RETRY_MIN; - tracing::info!(rate, channels, ?format, "opened audio output"); + tracing::info!(rate, ?layout, ?format, "opened audio output"); Ok(()) } @@ -1005,7 +1011,7 @@ mod tests { let handle = Arc::new(super::super::Handle { commands }); let (tx, mixer) = sync_channel(depth); - shared.rebind(48_000, tx); + shared.rebind(48_000, Layout::Stereo, tx); Wired { shared, @@ -1016,7 +1022,7 @@ mod tests { } fn add(shared: &Arc, handle: &Arc) -> Result { - shared.add(|id, rate| sink::new(id, rate, Input::default(), shared.clone(), handle.clone())) + shared.add(|id, rate, bus| sink::new(id, rate, bus, Input::default(), shared.clone(), handle.clone())) } /// Long enough that only a lost wake, rather than a loaded machine, trips a diff --git a/rs/moq-audio/src/playback/mixer.rs b/rs/moq-audio/src/playback/mixer.rs index d3f23124e3..6187f2ca9a 100644 --- a/rs/moq-audio/src/playback/mixer.rs +++ b/rs/moq-audio/src/playback/mixer.rs @@ -19,6 +19,10 @@ use fixed_resample::ResamplingCons; #[cfg(feature = "aec")] use fixed_resample::ResamplingProd; +#[cfg(feature = "aec")] +use crate::resample::Remix; +use crate::{Error, Layout}; + /// Frames mixed per pass. The callback buffer is chunked to this so the scratch /// buffers stay a fixed size no matter what period the device asks for. const CHUNK: usize = 1024; @@ -27,9 +31,10 @@ const CHUNK: usize = 1024; /// inaudible as a click, short enough to feel instant. const RAMP: f32 = 0.003; -/// The mix bus is stereo: sinks resample into it and it fans out to however many -/// channels the device wants. -pub(crate) const BUS_CHANNELS: usize = 2; +/// The echo reference is the mix downmixed to stereo, whatever layout the +/// device plays, so the canceller's model survives a device switch. +#[cfg(feature = "aec")] +pub(crate) const REFERENCE_CHANNELS: usize = 2; /// Sinks one device will mix. The entry list is allocated to this up front and /// never grows, which is what keeps registration off the allocator. @@ -138,36 +143,61 @@ pub(super) struct Mixer { /// Where anything the mixer is done with goes to be dropped, since dropping /// it here would free on the audio thread. retired: SyncSender, - /// Channels the device takes. + /// Channels the device takes, which is also the bus layout's: every sink + /// remixes into it on its way here. channels: usize, /// Per-frame gain step, so any change spans [`RAMP`] regardless of rate. step: f32, - /// Stereo accumulator for one chunk. + /// Accumulator for one chunk, in the device's layout. bus: Vec, - /// Stereo scratch for the sink being read. + /// Scratch for the sink being read, in the device's layout. scratch: Vec, - /// Where echo cancellation reads what was played. Fed the mix after - /// clipping but before it fans out, since that is the signal the speaker - /// gets and therefore the one the microphone hears back. + /// Where echo cancellation reads what was played. Fed the clipped mix, since + /// that is the signal the speaker gets and therefore the one the microphone + /// hears back. #[cfg(feature = "aec")] reference: Option>, + /// Downmixes the bus to the reference's stereo. + #[cfg(feature = "aec")] + downmix: Remix, + /// Stereo scratch for the downmixed reference. + #[cfg(feature = "aec")] + stereo: Vec, } impl Mixer { - /// `rate` and `channels` describe the device, not the sinks: each sink - /// resamples into the bus on its way here. - pub(super) fn new(commands: Receiver, retired: SyncSender, rate: u32, channels: usize) -> Self { - Self { + /// `rate` and `layout` describe the device, not the sinks: each sink + /// resamples and remixes into the bus on its way here. + pub(super) fn new( + commands: Receiver, + retired: SyncSender, + rate: u32, + layout: Layout, + ) -> Result { + // Sinks remix by speaker position, so a bus without any has nothing to + // remix into. + if layout.speakers().is_none() { + return Err(Error::Unsupported(format!( + "cannot mix into an output without speaker positions ({layout:?})" + ))); + } + + let channels = layout.channels() as usize; + Ok(Self { entries: Vec::with_capacity(MAX_SINKS), commands, retired, channels, step: 1.0 / (rate as f32 * RAMP), - bus: vec![0.0; CHUNK * BUS_CHANNELS], - scratch: vec![0.0; CHUNK * BUS_CHANNELS], + bus: vec![0.0; CHUNK * channels], + scratch: vec![0.0; CHUNK * channels], #[cfg(feature = "aec")] reference: None, - } + #[cfg(feature = "aec")] + downmix: Remix::new(layout, Layout::Stereo)?, + #[cfg(feature = "aec")] + stereo: vec![0.0; CHUNK * REFERENCE_CHANNELS], + }) } /// Hand something the mixer is done with back to the driver to drop. @@ -234,6 +264,10 @@ impl Mixer { scratch, #[cfg(feature = "aec")] reference, + #[cfg(feature = "aec")] + downmix, + #[cfg(feature = "aec")] + stereo, .. } = self; let channels = *channels; @@ -242,7 +276,7 @@ impl Mixer { let mut done = 0; while done < total { let frames = (total - done).min(CHUNK); - let samples = frames * BUS_CHANNELS; + let samples = frames * channels; bus[..samples].fill(0.0); @@ -257,13 +291,16 @@ impl Mixer { let mut applied = entry.applied; let mut peak = 0.0f32; - for frame in 0..frames { + for (input, mixed) in scratch[..samples] + .chunks_exact(channels) + .zip(bus[..samples].chunks_exact_mut(channels)) + { applied += (target - applied).clamp(-*step, *step); - let left = scratch[frame * 2] * applied; - let right = scratch[frame * 2 + 1] * applied; - peak = peak.max(left.abs()).max(right.abs()); - bus[frame * 2] += left; - bus[frame * 2 + 1] += right; + for (input, mixed) in input.iter().zip(mixed) { + let sample = input * applied; + peak = peak.max(sample.abs()); + *mixed += sample; + } } entry.applied = applied; @@ -280,26 +317,12 @@ impl Mixer { // useful to do about it from here and nothing may be logged. #[cfg(feature = "aec")] if let Some(reference) = reference.as_mut() { - reference.push_interleaved(&bus[..samples]); + let stereo = &mut stereo[..frames * REFERENCE_CHANNELS]; + downmix.apply(&bus[..samples], stereo); + reference.push_interleaved(stereo); } - let out = &mut out[done * channels..(done + frames) * channels]; - match channels { - 1 => { - for (frame, out) in out.iter_mut().enumerate() { - *out = (bus[frame * 2] + bus[frame * 2 + 1]) * 0.5; - } - } - _ => { - for (frame, out) in out.chunks_exact_mut(channels).enumerate() { - out[0] = bus[frame * 2]; - out[1] = bus[frame * 2 + 1]; - // Surround devices get silence past the front pair, which - // is better than duplicating stereo into the rears. - out[2..].fill(0.0); - } - } - } + out[done * channels..(done + frames) * channels].copy_from_slice(&bus[..samples]); done += frames; } @@ -316,6 +339,9 @@ mod tests { const RATE: u32 = 48_000; + /// The bus most tests mix into. + const STEREO: usize = 2; + /// Frames pushed per test: a tenth of a second, far more than any `fill` /// below drains, so a short read never starves by accident. const FRAMES: usize = RATE as usize / 10; @@ -328,21 +354,23 @@ mod tests { /// audio thread never drops one. retired: Receiver, next: u64, + channels: usize, } impl Harness { - fn new(channels: usize) -> Self { - Self::with_depth(channels, 8) + fn new(layout: Layout) -> Self { + Self::with_depth(layout, 8) } - fn with_depth(channels: usize, depth: usize) -> Self { + fn with_depth(layout: Layout, depth: usize) -> Self { let (commands, rx) = sync_channel(depth); let (retired_tx, retired) = sync_channel(MAX_SINKS); Self { - mixer: Mixer::new(rx, retired_tx, RATE, channels), + mixer: Mixer::new(rx, retired_tx, RATE, layout).unwrap(), commands, retired, next: 0, + channels: layout.channels() as usize, } } @@ -352,7 +380,7 @@ mod tests { /// pushed samples and the assertions. fn add(&mut self, gain: Arc) -> (u64, ResamplingProd) { let (prod, cons) = resampling_channel::( - BUS_CHANNELS, + self.channels, RATE, RATE, true, @@ -389,12 +417,12 @@ mod tests { /// Push `frames` stereo frames of a constant sample. fn push(prod: &mut ResamplingProd, value: f32, frames: usize) { - prod.push_interleaved(&vec![value; frames * BUS_CHANNELS]); + prod.push_interleaved(&vec![value; frames * STEREO]); } #[test] fn discards_writes_until_the_device_reads() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let (_, mut prod) = harness.add(Arc::new(Gain::new())); // Nothing has read yet, so these samples are dropped rather than queued @@ -411,7 +439,7 @@ mod tests { #[test] fn sums_sinks_and_clips() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 2048]; // Three sinks at 0.5 sum to 1.5, which must clip to 1.0. @@ -433,7 +461,7 @@ mod tests { #[test] fn volume_ramps_instead_of_stepping() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 2048]; let gain = Arc::new(Gain::new()); @@ -453,8 +481,8 @@ mod tests { // The whole ramp is monotonic, not just its endpoints. let ramp = (RATE as f32 * RAMP) as usize; assert!(ramp < out.len() / 2, "test buffer is shorter than the ramp"); - for frame in out[..ramp * BUS_CHANNELS] - .as_chunks::() + for frame in out[..ramp * STEREO] + .as_chunks::() .0 .iter() .collect::>() @@ -484,7 +512,7 @@ mod tests { /// device buffer into NaN. #[test] fn output_stays_finite_after_a_non_finite_volume() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 2048]; let gain = Arc::new(Gain::new()); @@ -500,7 +528,7 @@ mod tests { #[test] fn peak_reports_the_loudest_sample_then_resets() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 2048]; let gain = Arc::new(Gain::new()); @@ -517,7 +545,7 @@ mod tests { #[test] fn removed_sinks_stop_mixing() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 2048]; let (id, mut prod) = harness.add(Arc::new(Gain::new())); @@ -536,7 +564,7 @@ mod tests { /// callback would free on the audio thread, so it goes back to the driver. #[test] fn removed_sinks_are_handed_back_rather_than_dropped() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 512]; let (id, _prod) = harness.add(Arc::new(Gain::new())); @@ -561,7 +589,7 @@ mod tests { #[cfg(feature = "aec")] #[test] fn the_echo_reference_gets_the_mix() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 2048]; let (reference, mut tap) = reference_channel(); @@ -574,7 +602,7 @@ mod tests { // does, so that nothing queues up while nobody is listening. That first // read also primes the channel with its configured latency in silence, // which is why the buffer below is read past it rather than at it. - let mut heard = vec![0.0f32; 2048 * BUS_CHANNELS]; + let mut heard = vec![0.0f32; 2048 * REFERENCE_CHANNELS]; tap.read_interleaved(&mut heard, false); for prod in &mut prods { @@ -596,7 +624,7 @@ mod tests { #[cfg(feature = "aec")] #[test] fn a_replaced_echo_reference_is_handed_back() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 512]; harness @@ -629,7 +657,7 @@ mod tests { #[cfg(feature = "aec")] fn reference_channel() -> (ResamplingProd, ResamplingCons) { resampling_channel::( - BUS_CHANNELS, + REFERENCE_CHANNELS, RATE, RATE, true, @@ -645,7 +673,7 @@ mod tests { /// thread, which is why the driver caps registrations at MAX_SINKS. #[test] fn the_entry_list_never_grows() { - let mut harness = Harness::with_depth(2, MAX_SINKS); + let mut harness = Harness::with_depth(Layout::Stereo, MAX_SINKS); let mut out = vec![0.0f32; 512]; let capacity = harness.mixer.entries.capacity(); @@ -660,50 +688,65 @@ mod tests { #[test] fn silence_when_no_sink_is_registered() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![1.0f32; 512]; harness.fill(&mut out); assert!(out.iter().all(|s| *s == 0.0), "callback buffer was not overwritten"); } + /// The bus is the device's own layout, so every speaker of a surround device + /// carries what the sinks put there rather than silence past the front pair. #[test] - fn mono_device_gets_the_stereo_average() { - let mut harness = Harness::new(1); - let mut out = vec![0.0f32; 1024]; + fn a_surround_bus_mixes_every_speaker() { + let mut harness = Harness::new(Layout::FivePointOne); + let mut out = vec![0.0f32; 6 * 512]; let (_, mut prod) = harness.add(Arc::new(Gain::new())); harness.fill(&mut out); - - // Hard left, so a mono device should hear half of it. - let mut samples = vec![0.0f32; FRAMES * BUS_CHANNELS]; - for frame in samples.as_chunks_mut::().0.iter_mut() { - frame[0] = 1.0; - } - prod.push_interleaved(&samples); + let frame = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]; + prod.push_interleaved(&frame.repeat(FRAMES)); harness.settle(&mut out); - assert!((out[out.len() - 1] - 0.5).abs() < 1e-5, "got {}", out[out.len() - 1]); + + let last = &out[out.len() - 6..]; + for (got, want) in last.iter().zip(frame) { + assert!((got - want).abs() < 1e-5, "got {last:?}"); + } } + /// The canceller models a stereo reference whatever the device plays, so a + /// surround bus reaches it downmixed rather than cut to its front pair. + #[cfg(feature = "aec")] #[test] - fn surround_devices_get_silence_past_the_front_pair() { - let mut harness = Harness::new(6); - let mut out = vec![0.0f32; 6 * 512]; + fn the_echo_reference_downmixes_a_surround_bus() { + let mut harness = Harness::new(Layout::FivePointOne); + let mut out = vec![0.0f32; 6 * 2048]; + let (reference, mut tap) = reference_channel(); + harness.commands.send(Command::Reference(Some(reference))).unwrap(); let (_, mut prod) = harness.add(Arc::new(Gain::new())); harness.fill(&mut out); - push(&mut prod, 1.0, FRAMES); + let mut heard = vec![0.0f32; 2048 * REFERENCE_CHANNELS]; + tap.read_interleaved(&mut heard, false); + + // Center only, which a stereo downmix splits evenly at -3 dB. + prod.push_interleaved(&[0.0, 0.0, 0.5, 0.0, 0.0, 0.0].repeat(FRAMES)); harness.settle(&mut out); + tap.read_interleaved(&mut heard, false); - let last = &out[out.len() - 6..]; - assert!(last[0] > 0.9 && last[1] > 0.9, "front pair was silent"); - assert!(last[2..].iter().all(|s| *s == 0.0), "rear channels were not silent"); + let want = 0.5 * std::f32::consts::FRAC_1_SQRT_2; + let tail = &heard[heard.len() - 64..]; + assert!( + tail.iter().all(|s| (*s - want).abs() < 1e-5), + "the tap saw {:?}, not the downmix", + &tail[..4] + ); } #[test] fn underflow_reads_as_silence_rather_than_stale_samples() { - let mut harness = Harness::new(2); + let mut harness = Harness::new(Layout::Stereo); let mut out = vec![0.0f32; 8192]; let (_, mut prod) = harness.add(Arc::new(Gain::new())); diff --git a/rs/moq-audio/src/playback/sink.rs b/rs/moq-audio/src/playback/sink.rs index 41b8d0f79d..74bed20c19 100644 --- a/rs/moq-audio/src/playback/sink.rs +++ b/rs/moq-audio/src/playback/sink.rs @@ -1,6 +1,5 @@ //! [`Sink`]: one stream of PCM on its way to the speaker. -use std::borrow::Cow; use std::sync::mpsc::{SyncSender, TrySendError}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -8,8 +7,8 @@ use std::time::Duration; use fixed_resample::{PushStatus, ResamplingChannelConfig, ResamplingCons, ResamplingProd, resampling_channel}; use super::driver::Shared; -use super::mixer::{self, BUS_CHANNELS, Gain}; -use crate::resample::remix; +use super::mixer::{self, Gain}; +use crate::resample::Remix; use crate::{Error, Format, Layout}; /// Default for [`Input::latency`]: audio buffered between [`Sink::write`] and @@ -28,7 +27,7 @@ const HEADROOM: f64 = 3.0; /// /// The playback counterpart to [`encode::Input`](crate::encode::Input): it /// describes the buffers you hand in, not the device, which is free to run at -/// its own rate and channel count. +/// its own rate and layout. /// /// `#[non_exhaustive]`: construct via [`Input::default`] and set fields, so new /// options can be added without breaking callers. @@ -40,7 +39,8 @@ pub struct Input { /// Samples per second per channel. Resampled to the device rate if they /// differ. pub sample_rate: u32, - /// Speaker meaning and channel order. + /// Speaker meaning and channel order. Remixed to the device's layout, so it + /// must name speaker positions rather than be [`Layout::Discrete`]. pub layout: Layout, /// How much audio to hold between [`Sink::write`] and the speaker (default: @@ -77,9 +77,9 @@ impl Input { if self.sample_rate == 0 { return Err(Error::Unsupported("sample rate must be > 0".into())); } - if !matches!(self.layout, Layout::Mono | Layout::Stereo) { + if self.layout.speakers().is_none() { return Err(Error::Unsupported(format!( - "playback accepts named mono or stereo input (got {:?})", + "playback needs speaker positions to remix (got {:?})", self.layout ))); } @@ -120,13 +120,13 @@ impl Write { /// One stream of PCM being played, mixed with every other sink on the device. /// /// Write decoded samples with [`write`](Self::write) and drop the sink to stop. -/// Writes are cheap and never block on the device: they hand samples to a ring -/// buffer that the audio thread drains on its own clock, resampling to the -/// device rate on the way. +/// Writes are cheap and never block on the device: they remix to the device's +/// layout and hand samples to a ring buffer that the audio thread drains on its +/// own clock, resampling to the device rate on the way. pub struct Sink { id: u64, input: Input, - prod: Arc>>, + channel: Arc>, control: Control, /// Whether the last write overflowed, so a writer that stays ahead of the /// device logs once rather than on every write. @@ -151,17 +151,21 @@ impl Sink { /// The returned [`Write`] counts input sample frames accepted and dropped; /// dropped live audio should be observed for telemetry, not retried. pub fn write(&mut self, samples: &[u8]) -> Result { - let pcm = self - .input - .format - .as_interleaved_f32(samples, self.input.layout.channels())?; - let pcm = match self.input.layout.channels() as usize { - BUS_CHANNELS => pcm, - _ => Cow::Owned(remix(&pcm, self.input.layout, Layout::Stereo)?), + let channels = self.input.layout.channels(); + let pcm = self.input.format.as_interleaved_f32(samples, channels)?; + let requested_sample_frames = pcm.len() / channels as usize; + + let mut channel = self.channel.lock().unwrap(); + let mixed; + let pcm = match &channel.remix { + Some(remix) => { + mixed = remix.process(&pcm); + &mixed + } + None => pcm.as_ref(), }; - let requested_sample_frames = pcm.len() / BUS_CHANNELS; - let accepted_sample_frames = match self.prod.lock().unwrap().push_interleaved(&pcm) { + let accepted_sample_frames = match channel.prod.push_interleaved(pcm) { // OutputNotReady means the device has not read yet, so these samples // are dropped rather than queued to play late. PushStatus::Ok => { @@ -201,7 +205,7 @@ impl Sink { /// climbs when the writer runs ahead, and falls toward zero when it falls /// behind. pub fn buffered(&self) -> Duration { - Duration::from_secs_f64(self.prod.lock().unwrap().occupied_seconds().max(0.0)) + Duration::from_secs_f64(self.channel.lock().unwrap().prod.occupied_seconds().max(0.0)) } /// The PCM layout this sink was built with. @@ -278,15 +282,21 @@ impl Control { } } +/// The ring into the mixer and the remix that fills it, swapped together when +/// the device changes rate or layout. +struct Channel { + prod: ResamplingProd, + /// Converts the sink's layout to the device's, when they differ. + remix: Option, +} + /// A sink as the driver sees it: enough to rebuild its channel when the device /// changes underneath it. pub(super) struct Registration { pub(super) id: u64, - /// The caller's rate, which is the input side of the channel. - rate: u32, - /// The depth the rebuilt channel has to keep, from the caller's [`Input`]. - latency: Duration, - prod: Arc>>, + /// The caller's rate, layout, and latency: the input side of the channel. + input: Input, + channel: Arc>, gain: Arc, /// The consumer waiting to be handed to a mixer. Taken once it is attached, /// and refilled by [`rebuild`](Self::rebuild). @@ -319,34 +329,36 @@ impl Registration { } } - /// Re-create the channel for a device now running at `rate`, swapping the - /// producer the caller's [`Sink`] writes into. - pub(super) fn rebuild(&mut self, rate: u32) { - let (prod, cons) = channel(self.rate, rate, self.latency); - *self.prod.lock().unwrap() = prod; + /// Re-create the channel for a device now running at `rate` in `bus`, + /// swapping the producer the caller's [`Sink`] writes into. + pub(super) fn rebuild(&mut self, rate: u32, bus: Layout) { + let (channel, cons) = channel(&self.input, rate, bus); + *self.channel.lock().unwrap() = channel; self.pending = Some(cons); } } /// Build a sink and its registration. The device may not be open yet, in which -/// case `rate` is a placeholder the driver replaces on the next rebuild. +/// case `rate` and `bus` are placeholders the driver replaces on the next +/// rebuild. pub(super) fn new( id: u64, rate: u32, + bus: Layout, input: Input, shared: Arc, engine: Arc, ) -> Result<(Sink, Registration), Error> { input.validate()?; - let (prod, cons) = channel(input.sample_rate, rate, input.latency); - let prod = Arc::new(Mutex::new(prod)); + let (channel, cons) = self::channel(&input, rate, bus); + let channel = Arc::new(Mutex::new(channel)); let gain = Arc::new(Gain::new()); let sink = Sink { id, input, - prod: prod.clone(), + channel: channel.clone(), control: Control { gain: gain.clone() }, overflowing: false, shared, @@ -355,9 +367,8 @@ pub(super) fn new( let registration = Registration { id, - rate: sink.input.sample_rate, - latency: sink.input.latency, - prod, + input: sink.input.clone(), + channel, gain, pending: Some(cons), }; @@ -365,14 +376,17 @@ pub(super) fn new( Ok((sink, registration)) } -/// The ring buffer between a writer and the audio thread, resampling the -/// caller's rate to the device's. -fn channel(from: u32, to: u32, latency: Duration) -> (ResamplingProd, ResamplingCons) { - let latency = latency.as_secs_f64(); - resampling_channel::( - BUS_CHANNELS, - from, - to, +/// The ring buffer between a writer and the audio thread, remixing the caller's +/// layout to the device's and resampling its rate to the device's. +fn channel(input: &Input, rate: u32, bus: Layout) -> (Channel, ResamplingCons) { + let remix = + (input.layout != bus).then(|| Remix::new(input.layout, bus).expect("sink and bus layouts name their speakers")); + + let latency = input.latency.as_secs_f64(); + let (prod, cons) = resampling_channel::( + bus.channels() as usize, + input.sample_rate, + rate, // We only ever push interleaved, which lets the channel skip its planar // staging buffer. true, @@ -385,7 +399,9 @@ fn channel(from: u32, to: u32, latency: Duration) -> (ResamplingProd, Resam overflow_autocorrect_percent_threshold: Some(75.0), ..Default::default() }, - ) + ); + + (Channel { prod, remix }, cons) } #[cfg(test)] @@ -393,20 +409,65 @@ mod tests { use super::*; fn sink(input: Input, output_rate: u32) -> (Sink, ResamplingCons) { + sink_into(input, output_rate, Layout::Stereo) + } + + fn sink_into(input: Input, output_rate: u32, bus: Layout) -> (Sink, ResamplingCons) { let shared = Arc::new(Shared::default()); let engine = Arc::new(super::super::Handle { commands: super::super::driver::Commands::default(), }); - let (sink, mut registration) = new(0, output_rate, input, shared, engine).unwrap(); + let (sink, mut registration) = new(0, output_rate, bus, input, shared, engine).unwrap(); (sink, registration.pending.take().unwrap()) } + /// Write `frame` repeated as `input` and read back what the bus got. + fn mix(input: Layout, bus: Layout, frame: &[f32]) -> Vec { + let input = Input { + layout: input, + ..Default::default() + }; + let (mut sink, mut cons) = sink_into(input, 48_000, bus); + let channels = bus.channels() as usize; + cons.read_interleaved(&mut vec![0.0; channels], false); + + let pcm: Vec = frame.repeat(4800).iter().flat_map(|s| s.to_le_bytes()).collect(); + assert_eq!(sink.write(&pcm).unwrap().dropped_sample_frames, 0); + + let mut out = vec![0.0; 4800 * channels]; + cons.read_interleaved(&mut out, false); + out[out.len() - channels..].to_vec() + } + + fn close(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len(), "{got:?} vs {want:?}"); + for (g, w) in got.iter().zip(want) { + assert!((g - w).abs() < 1e-4, "{got:?} vs {want:?}"); + } + } + + /// A 5.1 track on a stereo device plays its downmix, center and surrounds + /// folded into the front pair at -3 dB. + #[test] + fn a_surround_sink_downmixes_into_a_stereo_bus() { + let h = std::f32::consts::FRAC_1_SQRT_2; + let got = mix(Layout::FivePointOne, Layout::Stereo, &[0.1, 0.2, 0.3, 0.4, 0.05, 0.06]); + close(&got, &[0.1 + h * 0.3 + h * 0.05, 0.2 + h * 0.3 + h * 0.06]); + } + + /// A stereo track on a 5.1 device plays from the front pair alone. + #[test] + fn a_stereo_sink_fills_the_front_of_a_surround_bus() { + let got = mix(Layout::Stereo, Layout::FivePointOne, &[0.25, 0.75]); + close(&got, &[0.25, 0.75, 0.0, 0.0, 0.0, 0.0]); + } + fn s16(frames: usize, channels: usize) -> Vec { vec![0; frames * channels * 2] } fn ready(cons: &mut ResamplingCons) { - cons.read_interleaved(&mut [0.0; BUS_CHANNELS], false); + cons.read_interleaved(&mut [0.0; 2], false); } #[test] @@ -495,7 +556,7 @@ mod tests { #[test] fn rejects_layouts_it_cannot_mix() { - for layout in [Layout::Discrete(0), Layout::Discrete(6)] { + for layout in [Layout::Discrete(0), Layout::Discrete(2), Layout::Discrete(6)] { let input = Input { layout, ..Default::default() @@ -511,8 +572,13 @@ mod tests { } #[test] - fn accepts_mono_and_stereo() { - for layout in [Layout::Mono, Layout::Stereo] { + fn accepts_named_layouts() { + for layout in [ + Layout::Mono, + Layout::Stereo, + Layout::FivePointOne, + Layout::SevenPointOne, + ] { let input = Input { layout, ..Default::default() diff --git a/rs/moq-audio/src/resample.rs b/rs/moq-audio/src/resample.rs index 173a3b1ba2..402f0b8529 100644 --- a/rs/moq-audio/src/resample.rs +++ b/rs/moq-audio/src/resample.rs @@ -2,14 +2,17 @@ //! //! Wraps [`rubato`] with a small interleaved-`f32` interface so the //! producer/consumer doesn't have to convert to planar on every call. -//! The resampler keeps the channel layout unchanged; [`remix`] converts mono -//! and stereo after sample-rate conversion. +//! The resampler keeps the channel layout unchanged; [`Remix`] converts between +//! layouts after sample-rate conversion. + +use std::f32::consts::FRAC_1_SQRT_2; use rubato::audioadapter_buffers::direct::SequentialSliceOfVecs; use rubato::{ Async, FixedAsync, Resampler as RubatoTrait, SincInterpolationParameters, SincInterpolationType, WindowFunction, }; +use crate::layout::Speaker; use crate::{Error, Layout}; #[derive(Debug, thiserror::Error)] @@ -301,50 +304,147 @@ impl Resampler { } } -/// Convert between known layouts without assigning positions to discrete channels. -pub(crate) fn remix(samples: &[f32], input: Layout, output: Layout) -> Result, Error> { - validate_remix(input, output)?; - match (input, output) { - (input, output) if input == output => Ok(samples.to_vec()), - (Layout::Mono, Layout::Stereo) => { - let mut output = Vec::with_capacity(samples.len() * 2); - for &sample in samples { - output.extend_from_slice(&[sample, sample]); +/// A channel mix from one layout to another, each output channel a weighted sum +/// of the input channels. +/// +/// Refuses to give discrete channels speaker positions, though it will drop +/// them from named channels. +/// +/// Downmixing uses the ITU-R BS.775 coefficients: center and surrounds fold into +/// the front pair at -3 dB and the LFE is dropped. Upmixing leaves the speakers +/// the input lacks silent. Mono is the exception both ways: it plays at full +/// level from both front speakers when there is no center, and a mono output +/// averages the stereo downmix. +pub(crate) struct Remix { + inputs: usize, + outputs: usize, + /// One row of `inputs` weights per output channel. + weights: Vec, +} + +impl Remix { + pub(crate) fn new(input: Layout, output: Layout) -> Result { + input.validate()?; + output.validate()?; + + let (inputs, outputs) = (input.channels() as usize, output.channels() as usize); + // Dropping speaker positions is always safe; inventing them is not. + let unchanged = input == output || output == Layout::Discrete(inputs as u32); + let weights = match (input.speakers(), output.speakers()) { + _ if unchanged => (0..outputs) + .flat_map(|o| (0..inputs).map(move |i| if i == o { 1.0 } else { 0.0 })) + .collect(), + (Some(from), Some(to)) => weights(from, to), + _ => { + return Err(Error::Unsupported(format!( + "cannot convert audio layout {input:?} to {output:?} without speaker positions" + ))); + } + }; + + Ok(Self { + inputs, + outputs, + weights, + }) + } + + /// Mix whole interleaved input frames into `output`, which holds as many + /// frames. Never allocates, so the audio thread can call it. + pub(crate) fn apply(&self, input: &[f32], output: &mut [f32]) { + for (frame, out) in input + .chunks_exact(self.inputs) + .zip(output.chunks_exact_mut(self.outputs)) + { + for (sample, row) in out.iter_mut().zip(self.weights.chunks_exact(self.inputs)) { + *sample = row.iter().zip(frame).map(|(weight, input)| weight * input).sum(); } - Ok(output) } - (Layout::Stereo, Layout::Mono) => Ok(samples - .as_chunks::<2>() - .0 - .iter() - .map(|pair| (pair[0] + pair[1]) * 0.5) - .collect()), - _ => Err(Error::Unsupported(format!( - "cannot convert audio layout {input:?} to {output:?} without speaker positions" - ))), + } + + /// Mix whole interleaved input frames into a new buffer. + pub(crate) fn process(&self, input: &[f32]) -> Vec { + let mut output = vec![0.0; input.len() / self.inputs * self.outputs]; + self.apply(input, &mut output); + output } } -/// Check that [`remix`] can convert between two layouts. -pub(crate) fn validate_remix(input: Layout, output: Layout) -> Result<(), Error> { - input.validate()?; - output.validate()?; - if input == output - || matches!( - (input, output), - (Layout::Mono, Layout::Stereo) | (Layout::Stereo, Layout::Mono) - ) { - return Ok(()); - } - Err(Error::Unsupported(format!( - "cannot convert audio layout {input:?} to {output:?} without speaker positions" - ))) +/// The weights mixing `input` speakers into `output` speakers, one row per output. +fn weights(input: &[Speaker], output: &[Speaker]) -> Vec { + use Speaker::*; + + // The only layout without a front pair. Average the stereo downmix rather + // than invent a center weight for every speaker. + if output == [FrontCenter] && input != [FrontCenter] { + let stereo = weights(input, &[FrontLeft, FrontRight]); + let (left, right) = stereo.split_at(input.len()); + return left.iter().zip(right).map(|(l, r)| (l + r) * 0.5).collect(); + } + + let mut weights = vec![0.0; output.len() * input.len()]; + let has = |speaker| output.contains(&speaker); + + for (i, &speaker) in input.iter().enumerate() { + let mut feed = |to: Speaker, weight: f32| { + if let Some(o) = output.iter().position(|s| *s == to) { + weights[o * input.len() + i] += weight; + } + }; + + if has(speaker) { + feed(speaker, 1.0); + continue; + } + + match speaker { + FrontCenter => { + let weight = if input == [FrontCenter] { 1.0 } else { FRAC_1_SQRT_2 }; + feed(FrontLeft, weight); + feed(FrontRight, weight); + } + Lfe => {} + SideLeft | BackLeft | SideRight | BackRight => { + let (front, side, back) = match speaker { + SideLeft | BackLeft => (FrontLeft, SideLeft, BackLeft), + _ => (FrontRight, SideRight, BackRight), + }; + // Side and back are the same surround to a layout with only one of them. + let other = if speaker == side { back } else { side }; + if has(other) { + feed(other, 1.0); + } else { + feed(front, FRAC_1_SQRT_2); + } + } + BackCenter => { + if has(BackLeft) { + feed(BackLeft, FRAC_1_SQRT_2); + feed(BackRight, FRAC_1_SQRT_2); + } else if has(SideLeft) { + feed(SideLeft, FRAC_1_SQRT_2); + feed(SideRight, FRAC_1_SQRT_2); + } else { + feed(FrontLeft, 0.5); + feed(FrontRight, 0.5); + } + } + // Every output but mono, handled above, has a front pair. + FrontLeft | FrontRight => unreachable!("{output:?} has no front pair"), + } + } + + weights } #[cfg(test)] mod tests { use super::*; + fn remix(samples: &[f32], input: Layout, output: Layout) -> Result, Error> { + Ok(Remix::new(input, output)?.process(samples)) + } + /// `frames` into a stream at `rate`, as a timestamp in the source's own scale. fn at(frames: u64, rate: u64) -> moq_net::Timestamp { moq_net::Timestamp::from_scale(frames, rate).unwrap() @@ -548,4 +648,105 @@ mod tests { [2.0, 3.0] ); } + + /// One frame of 5.1 with a distinct level per speaker, in canonical order. + const FIVE_ONE: [f32; 6] = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6]; + + fn close(got: &[f32], want: &[f32]) { + assert_eq!(got.len(), want.len(), "{got:?} vs {want:?}"); + for (g, w) in got.iter().zip(want) { + assert!((g - w).abs() < 1e-6, "{got:?} vs {want:?}"); + } + } + + #[test] + fn remix_downmixes_five_one_to_stereo_by_bs775() { + let h = FRAC_1_SQRT_2; + let [l, r, c, _lfe, ls, rs] = FIVE_ONE; + close( + &remix(&FIVE_ONE, Layout::FivePointOne, Layout::Stereo).unwrap(), + &[l + h * c + h * ls, r + h * c + h * rs], + ); + } + + #[test] + fn remix_downmixes_five_one_to_mono_through_stereo() { + let stereo = remix(&FIVE_ONE, Layout::FivePointOne, Layout::Stereo).unwrap(); + close( + &remix(&FIVE_ONE, Layout::FivePointOne, Layout::Mono).unwrap(), + &[(stereo[0] + stereo[1]) * 0.5], + ); + } + + #[test] + fn remix_upmixes_stereo_into_the_front_pair() { + close( + &remix(&[0.25, 0.75], Layout::Stereo, Layout::FivePointOne).unwrap(), + &[0.25, 0.75, 0.0, 0.0, 0.0, 0.0], + ); + } + + #[test] + fn remix_upmixes_mono_into_the_center() { + close( + &remix(&[0.5], Layout::Mono, Layout::FivePointOne).unwrap(), + &[0.0, 0.0, 0.5, 0.0, 0.0, 0.0], + ); + // Quad has no center, so mono plays from both fronts as it does in stereo. + close( + &remix(&[0.5], Layout::Mono, Layout::Quad).unwrap(), + &[0.5, 0.5, 0.0, 0.0], + ); + } + + /// 7.1 to 5.1 folds the back pair into the sides at full level, since a 5.1 + /// surround pair is the only surround it has. + #[test] + fn remix_folds_back_into_side_surrounds() { + let seven = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8]; + close( + &remix(&seven, Layout::SevenPointOne, Layout::FivePointOne).unwrap(), + &[0.1, 0.2, 0.3, 0.4, 0.5 + 0.7, 0.6 + 0.8], + ); + } + + #[test] + fn remix_refuses_positions_it_would_invent() { + for (input, output) in [ + (Layout::Discrete(6), Layout::Stereo), + (Layout::Mono, Layout::Discrete(2)), + (Layout::Discrete(0), Layout::Discrete(0)), + ] { + assert!( + matches!(Remix::new(input, output), Err(Error::Unsupported(_))), + "{input:?} -> {output:?}" + ); + } + + assert_eq!( + remix(&[1.0, 2.0, 3.0], Layout::Discrete(3), Layout::Discrete(3)).unwrap(), + [1.0, 2.0, 3.0] + ); + assert_eq!( + remix(&[1.0, 2.0, 3.0], Layout::TwoPointOne, Layout::Discrete(3)).unwrap(), + [1.0, 2.0, 3.0] + ); + assert!(Remix::new(Layout::Discrete(3), Layout::TwoPointOne).is_err()); + } + + /// Every pair of named layouts converts, and the mix is sized to the output. + #[test] + fn remix_converts_between_every_named_layout() { + let layouts: Vec = (1..=8) + .map(|n| Layout::from_channels(n).unwrap()) + .chain([Layout::ThreePointZero, Layout::FourPointZero]) + .collect(); + for &input in &layouts { + for &output in &layouts { + let frame = vec![0.5; input.channels() as usize * 2]; + let mixed = remix(&frame, input, output).unwrap(); + assert_eq!(mixed.len(), output.channels() as usize * 2, "{input:?} -> {output:?}"); + } + } + } } diff --git a/rs/moq-audio/tests/roundtrip.rs b/rs/moq-audio/tests/roundtrip.rs index 570dacac45..55e09bfab3 100644 --- a/rs/moq-audio/tests/roundtrip.rs +++ b/rs/moq-audio/tests/roundtrip.rs @@ -203,3 +203,71 @@ async fn pcm_round_trip_is_lossless() { assert_eq!(decoded, samples); assert!(consumer.read().await.unwrap().is_none()); } + +/// A 5.1 broadcast carries only its channel count in the catalog, which reads +/// back as 5.1 in canonical order and remixes to whatever the subscriber asks +/// for. +#[tokio::test] +async fn pcm_five_one_reads_back_at_any_layout() { + let mut broadcast = moq_net::broadcast::Info::new().produce(); + let catalog = moq_mux::catalog::Producer::new(&mut broadcast, moq_mux::catalog::Config::default()).unwrap(); + let mut catalog_consumer = catalog.consume().unwrap(); + let broadcast_consumer = broadcast.consume(); + + let input = encode::Input::new(48_000, Layout::FivePointOne); + let mut options = encode::Options::default(); + options.track = Some("pcm".to_string()); + options.settings = encode::Settings::new(48_000, Layout::FivePointOne); + options.settings.codec = encode::Codec::Pcm; + + // A level per speaker: left, right, center, LFE, side left, side right. + let speakers = [0.1f32, 0.2, 0.3, 0.4, 0.05, 0.06]; + let mut producer = encode::Producer::new(&mut broadcast, catalog.clone(), input, &options).unwrap(); + producer + .write(&Frame::new( + f32_bytes(&speakers.repeat(960)), + Timestamp::from_micros(0).unwrap(), + )) + .unwrap(); + + let snapshot = catalog_consumer.next().await.unwrap().unwrap(); + let rendition = snapshot.audio.renditions.get("pcm").unwrap(); + assert_eq!(rendition.channel_count, 6); + + let h = std::f32::consts::FRAC_1_SQRT_2; + let left = 0.1 + h * 0.3 + h * 0.05; + let right = 0.2 + h * 0.3 + h * 0.06; + let cases = [ + (None, Layout::FivePointOne, speakers.to_vec()), + (Some(Layout::Stereo), Layout::Stereo, vec![left, right]), + (Some(Layout::Mono), Layout::Mono, vec![(left + right) * 0.5]), + ]; + + let mut consumers = Vec::new(); + for (output, _, _) in &cases { + let mut options = decode::Options::default(); + options.output.layout = *output; + consumers.push( + decode::Consumer::new(&broadcast_consumer, rendition, "pcm", options) + .await + .unwrap(), + ); + } + producer.finish().unwrap(); + + for (mut consumer, (_, layout, frame)) in consumers.into_iter().zip(cases) { + assert_eq!(consumer.layout(), layout); + let decoded = consumer.read().await.unwrap().unwrap(); + let samples: Vec = decoded + .data + .as_chunks::<4>() + .0 + .iter() + .map(|sample| f32::from_le_bytes([sample[0], sample[1], sample[2], sample[3]])) + .collect(); + assert_eq!(samples.len(), 960 * frame.len(), "{layout:?}"); + for (got, want) in samples.iter().zip(frame.iter().cycle()) { + assert!((got - want).abs() < 1e-6, "{layout:?}: got {got}, want {want}"); + } + } +} diff --git a/rs/moq-ffi/src/audio.rs b/rs/moq-ffi/src/audio.rs index 16750c1375..5e8bea78ce 100644 --- a/rs/moq-ffi/src/audio.rs +++ b/rs/moq-ffi/src/audio.rs @@ -76,6 +76,9 @@ impl MoqAudioCodec { pub struct MoqAudioEncoderInput { pub format: MoqAudioSampleFormat, pub sample_rate: u32, + /// Interleaved channel count, which also names the speaker layout by the + /// WAVE convention: 1 mono, 2 stereo, 3 2.1, 4 quad, 5 5.0, 6 5.1, 7 6.1, + /// 8 7.1, in front left, front right, center, LFE, back, side order. pub channels: u32, } @@ -105,7 +108,9 @@ pub struct MoqAudioDecoderOutput { /// `None` delivers samples at the codec's native rate. #[uniffi(default = None)] pub sample_rate: Option, - /// `None` delivers samples at the codec's native channel count. + /// `None` delivers samples at the codec's native channel count. A count + /// names its layout as [`MoqAudioEncoderInput::channels`] describes, and + /// the decoder remixes to it. #[uniffi(default = None)] pub channels: Option, /// Upper bound on buffering before skipping a stalled group, in