@remotion/media: Bound paused audio decoding with a keep-alive context

packages/media/src/audio-iterator-manager.ts

@@ -76,6 +76,7 @@ export const audioIteratorManager = ({
 	getSequenceDurationInSeconds,
 	getMediaEndTimestamp,
 	getStartTime,
+	getIsPlaying,
 	initialMuted,
 	initialVolume,
 	toneFrequency,
@@ -89,6 +90,7 @@ export const audioIteratorManager = ({
 	getSequenceDurationInSeconds: () => number;
 	getMediaEndTimestamp: () => number;
 	getStartTime: () => number;
+	getIsPlaying: () => boolean;
 	initialMuted: boolean;
 	initialVolume: number;
 	toneFrequency: number;
@@ -280,6 +282,7 @@ export const audioIteratorManager = ({
 		onDone,
 		logLevel,
 		currentTime,
+		startFromSecond,
 		getAudioContextCurrentTimeMockedInTest,
 	}: {
 		iterator: AudioIterator;
@@ -298,6 +301,7 @@ export const audioIteratorManager = ({
 		onDestroyed: () => void;
 		logLevel: LogLevel;
 		currentTime: number;
+		startFromSecond: number;
 		getAudioContextCurrentTimeMockedInTest: () => number;
 	}) => {
 		waitForTurn({
@@ -321,7 +325,16 @@ export const audioIteratorManager = ({
 					sequenceStartTime: getStartTime(),
 				});
 
-				return scheduledTime - getAudioContextCurrentTimeMockedInTest();
+				// Always finish the initial prebuffer, even for future or very slow clips.
+				// Bound further paused decoding by prepared media, not a future start time.
+				const preparedDuration = guessedNextTimestamp - startFromSecond;
+				if (getIsPlaying()) {
+					return scheduledTime - getAudioContextCurrentTimeMockedInTest();
+				}
+
+				return hasEnoughAudioToStartPlayback(preparedDuration)
+					? preparedDuration / playbackRate
+					: 0;
 			},
 			fn: () => iterator.getNextFn(),
 			onDone: (result, next) => {
@@ -364,6 +377,7 @@ export const audioIteratorManager = ({
 					onDone,
 					logLevel,
 					currentTime,
+					startFromSecond,
 					getAudioContextCurrentTimeMockedInTest,
 				});
 				next();
@@ -512,6 +526,7 @@ export const audioIteratorManager = ({
 			},
 			logLevel,
 			currentTime: sharedAudioContext.audioContext.currentTime,
+			startFromSecond,
 			getAudioContextCurrentTimeMockedInTest,
 		});
 	};

packages/media/src/media-player.ts

@@ -453,6 +453,7 @@ export class MediaPlayer {
 						getMediaEndTimestamp: () => this.getMediaEndTimestamp(),
 						getSequenceEndTimestamp: () => this.getSequenceEndTimestamp(),
 						getStartTime: () => this.getStartTime(),
+						getIsPlaying: () => this.playing,
 						initialMuted,
 						initialVolume,
 						toneFrequency: this.toneFrequency,

packages/media/src/test/audio-anchor-reschedule.test.tsx

@@ -1,8 +1,9 @@
 import {Player, type PlayerRef} from '@remotion/player';
 import React from 'react';
 import {createRoot} from 'react-dom/client';
-import {Internals} from 'remotion';
+import {Internals, Sequence} from 'remotion';
 import {expect, test, vi} from 'vitest';
+import {page} from 'vitest/browser';
 import {Audio} from '../audio/audio';
 
 test('replaces queued audio after a suspension re-anchors a stationary Player', async () => {
@@ -172,3 +173,104 @@ test('paused seeks unschedule unstarted nodes while playing cleanup still stops
 		vi.restoreAllMocks();
 	}
 });
+
+test.each([
+	{from: 0, playbackRate: 1},
+	{from: 90, playbackRate: 1},
+	{from: 90, playbackRate: 0.01},
+])(
+	'keep-alive bounds paused audio from frame $from at $playbackRate× without blocking playback',
+	async ({from, playbackRate}) => {
+		const nodes: AudioBufferSourceNode[] = [];
+		const originalCreateBufferSource =
+			AudioContext.prototype.createBufferSource;
+		const createBufferSourceSpy = vi
+			.spyOn(AudioContext.prototype, 'createBufferSource')
+			.mockImplementation(function (this: AudioContext) {
+				const node = originalCreateBufferSource.call(this);
+				nodes.push(node);
+				return node;
+			});
+		let audioContext: AudioContext | null = null;
+		const Composition: React.FC = () => {
+			audioContext =
+				React.useContext(Internals.SharedAudioContext)?.audioContext ?? null;
+			return (
+				<Sequence from={from} premountFor={from}>
+					<Audio
+						src="/voice-note.m4a"
+						playbackRate={playbackRate}
+						disallowFallbackToHtml5Audio
+					/>
+				</Sequence>
+			);
+		};
+
+		const container = document.createElement('div');
+		document.body.appendChild(container);
+		const root = createRoot(container);
+		const playerRef = React.createRef<PlayerRef>();
+		root.render(
+			<Player
+				ref={playerRef}
+				acknowledgeRemotionLicense
+				component={Composition}
+				compositionWidth={100}
+				compositionHeight={100}
+				durationInFrames={4000}
+				fps={30}
+				inputProps={{}}
+				_experimentalKeepAudioContextAlive
+			/>,
+		);
+		try {
+			await vi.waitFor(() => {
+				expect(
+					nodes.reduce((sum, node) => sum + (node.buffer?.duration ?? 0), 0),
+				).toBeGreaterThanOrEqual(0.1);
+			});
+			if (!audioContext) throw new Error('Audio context did not initialize');
+			const context: AudioContext = audioContext;
+			const resumeButton = document.createElement('button');
+			resumeButton.textContent = 'Resume audio context';
+			resumeButton.onclick = () => context.resume();
+			container.appendChild(resumeButton);
+			await page.getByRole('button', {name: 'Resume audio context'}).click();
+			await vi.waitFor(() => expect(context.state).toBe('running'));
+			const startedAt = context.currentTime;
+			// Let the real audio clock move beyond the two-second decode horizon.
+			await vi.waitFor(
+				() => {
+					expect(context.currentTime - startedAt).toBeGreaterThan(3);
+				},
+				{timeout: 5000},
+			);
+			expect(playerRef.current?.getCurrentFrame()).toBe(0);
+			expect(playerRef.current?.isPlaying()).toBe(false);
+			const queuedDuration = nodes.reduce(
+				(sum, node) => sum + (node.buffer?.duration ?? 0),
+				0,
+			);
+			// The final decoded chunk can straddle the two-second boundary.
+			expect(queuedDuration).toBeGreaterThanOrEqual(
+				Math.max(1.9 * playbackRate, 0.1),
+			);
+			expect(queuedDuration).toBeLessThan(Math.max(2.2 * playbackRate, 0.2));
+			playerRef.current?.play();
+			await vi.waitFor(
+				() => {
+					expect(playerRef.current?.getCurrentFrame()).toBeGreaterThan(
+						from + 5,
+					);
+				},
+				{timeout: 5000},
+			);
+		} finally {
+			root.unmount();
+			container.remove();
+			createBufferSourceSpy.mockRestore();
+			if (audioContext) await (audioContext as AudioContext).close();
+		}
+	},
+	15000,
+);

packages/media/src/test/audio-iterator.test.ts

@@ -57,6 +57,7 @@ const prepare = async (options?: {
 	let playbackUnblocks = 0;
 
 	const manager = audioIteratorManager({
+		getIsPlaying: () => true,
 		audioTrack,
 		delayPlaybackHandleIfNotPremounting: () => ({
 			unblock: () => {

packages/media/src/test/audio-seek-tolerance.test.ts

@@ -79,6 +79,7 @@ test.each([
 			scheduledTime: 0,
 		});
 		const manager = audioIteratorManager({
+			getIsPlaying: () => true,
 			audioTrack,
 			delayPlaybackHandleIfNotPremounting: () => ({
 				unblock: () => undefined,

packages/media/src/test/video-with-no-frames-in-beginning.test.ts

@@ -108,6 +108,7 @@ test('same goes for audio', async () => {
 	}
 
 	const manager = audioIteratorManager({
+		getIsPlaying: () => true,
 		audioTrack,
 		delayPlaybackHandleIfNotPremounting: () => ({
 			unblock: () => {},