@remotion/media: Bound unshifted audio decoding to the sequence end

.gitignore

@@ -43,3 +43,13 @@ packages/cloudrun/container/ensure-browser.mjs
 packages/skills-evals/.runs
 packages/skills-evals/.site
 packages/skills-evals/.playground
+
+# Downloaded example media served by browser tests
+packages/example-videos/videos/bigbuckbunny.mp4
+packages/example-videos/videos/music-visualization-demo-track.mp3
+packages/example-videos/videos/corrupted.mp4
+packages/example-videos/videos/variablefps.webm
+packages/example-videos/videos/iphonevideo.mov
+packages/example-videos/videos/music.mp3
+packages/example-videos/videos/iphone-hevc.mov
+packages/example-videos/videos/stretched-vp8.webm

packages/example-videos/src/cached-example-media.ts

@@ -45,9 +45,9 @@ const downloadFile = async (url: string, location: string) => {
 
 export const downloadCachedExampleMedia = async () => {
 	for (const filename of Object.values(cachedExampleMedia)) {
-		await downloadFile(
-			`${REMOTE_BASE}/${filename}`,
-			path.join(cachedDir, filename),
-		);
+		const location = path.join(cachedDir, filename);
+		await downloadFile(`${REMOTE_BASE}/${filename}`, location);
+		// Browser suites serve this directory; keep their existing fixture URLs.
+		fs.copyFileSync(location, path.join(rootDir, 'videos', filename));
 	}
 };

packages/media/src/audio-iterator-manager.ts

@@ -428,7 +428,11 @@ export const audioIteratorManager = ({
 			return;
 		}
 
-		const maximumTimestamp = getMediaEndTimestamp();
+		// Pitch shifting needs future samples; truncating its input changes the tail.
+		const maximumTimestamp =
+			loop || currentToneFrequency !== 1
+				? getMediaEndTimestamp()
+				: Math.min(getMediaEndTimestamp(), getSequenceEndTimestamp());
 		if (startFromSecond >= maximumTimestamp) {
 			return;
 		}

packages/media/src/media-player.ts

@@ -799,6 +799,7 @@ export class MediaPlayer {
 		const previousDuration = this.sequenceDurationInFrames;
 		if (previousDuration !== sequenceDurationInFrames) {
 			this.sequenceDurationInFrames = sequenceDurationInFrames;
+			this.audioIteratorManager?.destroyIterator();
 			await this.seekTo(unloopedTimeInSeconds, 'discontinuity');
 		}
 	}

packages/media/src/test/media-player.test.ts

@@ -55,7 +55,14 @@ const makeAudioPlayer = (
 				Partial<
 					Pick<
 						ConstructorParameters<typeof MediaPlayer>[0],
-						'getEffects' | 'getEffectChainState' | 'bufferState'
+						| 'getEffects'
+						| 'getEffectChainState'
+						| 'bufferState'
+						| 'durationInFrames'
+						| 'playbackRate'
+						| 'trimBefore'
+						| 'trimAfter'
+						| 'loop'
 					>
 				>)
 		| null = null,
@@ -67,10 +74,10 @@ const makeAudioPlayer = (
 			(tagType === 'audio' ? '/voice-note.m4a' : '/bigbuckbunny.mp4'),
 		logLevel: 'error',
 		sharedAudioContext,
-		loop: false,
-		trimBefore: undefined,
-		trimAfter: undefined,
-		playbackRate: 1,
+		loop: options?.loop ?? false,
+		trimBefore: options?.trimBefore,
+		trimAfter: options?.trimAfter,
+		playbackRate: options?.playbackRate ?? 1,
 		toneFrequency: 1,
 		globalPlaybackRate: 1,
 		audioStreamIndex: null,
@@ -79,7 +86,7 @@ const makeAudioPlayer = (
 		bufferState: options?.bufferState ?? makeBufferState(),
 		isPremounting: false,
 		isPostmounting: false,
-		durationInFrames: 300,
+		durationInFrames: options?.durationInFrames ?? 300,
 		onVideoFrameCallback: options?.onVideoFrameCallback ?? null,
 		playing: false,
 		sequenceOffset: 0,
@@ -624,3 +631,121 @@ test.each([
 		}
 	},
 );
+
+// Regression for kino-ai/kino#4930: short clips decoded unused source audio,
+// and retaining an exhausted iterator could leave live extensions silent.
+test.each([
+	{
+		tagType: 'audio' as const,
+		playbackRate: 1,
+		durationInFrames: 6,
+		loop: false,
+		trimAfter: undefined,
+	},
+	{
+		tagType: 'video' as const,
+		playbackRate: 2,
+		durationInFrames: 6,
+		loop: false,
+		trimAfter: undefined,
+	},
+	{
+		tagType: 'audio' as const,
+		playbackRate: 0.5,
+		durationInFrames: 1,
+		loop: false,
+		trimAfter: undefined,
+	},
+	{
+		tagType: 'audio' as const,
+		playbackRate: 1,
+		durationInFrames: 6,
+		loop: false,
+		trimAfter: 33,
+	},
+	{
+		tagType: 'audio' as const,
+		playbackRate: 1,
+		durationInFrames: 6,
+		loop: true,
+		trimAfter: undefined,
+	},
+])(
+	'bounds $tagType audio decoding at $playbackRate×, $durationInFrames frames, loop=$loop, trimAfter=$trimAfter',
+	async (options) => {
+		const shared = makeSharedAudioContext();
+		const player = makeAudioPlayer(shared, null, options.tagType, {
+			...options,
+			canvas: null,
+			onVideoFrameCallback: null,
+			src:
+				options.tagType === 'audio' ? '/voice-note.m4a' : '/bigbuckbunny.mp4',
+			trimBefore: 30,
+		});
+		const buffers = vi.spyOn(AudioBufferSink.prototype, 'buffers');
+		try {
+			expect((await player.initialize(0, false, 1)).type).toBe('success');
+			await vi.waitFor(() => expect(buffers).toHaveBeenCalled());
+			const end = buffers.mock.calls[0][1]!;
+			const sequenceEnd =
+				1 + (options.durationInFrames / 30) * options.playbackRate;
+			if (options.loop) {
+				expect(end).toBeGreaterThan(sequenceEnd);
+			} else {
+				expect(end).toBeCloseTo(
+					Math.min(sequenceEnd, (options.trimAfter ?? Infinity) / 30),
+				);
+			}
+
+			await vi.waitFor(() => {
+				const queued = player
+					.audioIteratorManager!.getAudioBufferIterator()!
+					.getQueuedPeriod();
+				expect(queued?.until).toBeCloseTo(Math.min(sequenceEnd, end));
+			});
+		} finally {
+			await player.dispose();
+			await shared.audioContext.close();
+			buffers.mockRestore();
+		}
+	},
+);
+
+test.each(['audio', 'video'] as const)(
+	'reopens capped %s audio after extension, shrink and re-extension',
+	async (tagType) => {
+		const shared = makeSharedAudioContext();
+		const player = makeAudioPlayer(shared, null, tagType, {
+			canvas: null,
+			onVideoFrameCallback: null,
+			src: tagType === 'audio' ? '/voice-note.m4a' : '/bigbuckbunny.mp4',
+			durationInFrames: 6,
+		});
+		try {
+			expect((await player.initialize(0, false, 1)).type).toBe('success');
+			await vi.waitFor(() =>
+				expect(
+					player
+						.audioIteratorManager!.getAudioBufferIterator()!
+						.getQueuedPeriod()?.until,
+				).toBeCloseTo(0.2),
+			);
+			player.play();
+			for (const duration of [15, 6, 15]) {
+				// The playhead is still inside the queued old range. A seek alone
+				// would retain the old iterator and its original decode endpoint.
+				await player.setSequenceDurationInFrames(duration, 0.1);
+				await vi.waitFor(() => {
+					const queued = player
+						.audioIteratorManager!.getAudioBufferIterator()!
+						.getQueuedPeriod();
+					expect(queued?.from).toBeLessThanOrEqual(0.1);
+					expect(queued?.until).toBeCloseTo(duration / 30);
+				});
+			}
+		} finally {
+			await player.dispose();
+			await shared.audioContext.close();
+		}
+	},
+);

turbo.json

@@ -88,7 +88,18 @@
 		},
 		"@remotion/example-videos#make": {
 			"dependsOn": ["^make"],
-			"outputs": ["dist", "node_modules/.videos/**"],
+			"outputs": [
+				"dist",
+				"node_modules/.videos/**",
+				"videos/bigbuckbunny.mp4",
+				"videos/music-visualization-demo-track.mp3",
+				"videos/corrupted.mp4",
+				"videos/variablefps.webm",
+				"videos/iphonevideo.mov",
+				"videos/music.mp3",
+				"videos/iphone-hevc.mov",
+				"videos/stretched-vp8.webm"
+			],
 			"outputLogs": "new-only"
 		},
 		"@remotion/example#bundle": {