diff --git a/.gitignore b/.gitignore index a1eb571a3c4..a4533c34c76 100644 --- a/.gitignore +++ b/.gitignore @@ -43,3 +43,13 @@ packages/cloudrun/container/ensure-browser.mjs packages/skills-evals/.runs packages/skills-evals/.site packages/skills-evals/.playground + +# Downloaded example media served by browser tests +packages/example-videos/videos/bigbuckbunny.mp4 +packages/example-videos/videos/music-visualization-demo-track.mp3 +packages/example-videos/videos/corrupted.mp4 +packages/example-videos/videos/variablefps.webm +packages/example-videos/videos/iphonevideo.mov +packages/example-videos/videos/music.mp3 +packages/example-videos/videos/iphone-hevc.mov +packages/example-videos/videos/stretched-vp8.webm diff --git a/packages/example-videos/src/cached-example-media.ts b/packages/example-videos/src/cached-example-media.ts index 5d35f6b346b..b27c94943cd 100644 --- a/packages/example-videos/src/cached-example-media.ts +++ b/packages/example-videos/src/cached-example-media.ts @@ -45,9 +45,9 @@ const downloadFile = async (url: string, location: string) => { export const downloadCachedExampleMedia = async () => { for (const filename of Object.values(cachedExampleMedia)) { - await downloadFile( - `${REMOTE_BASE}/${filename}`, - path.join(cachedDir, filename), - ); + const location = path.join(cachedDir, filename); + await downloadFile(`${REMOTE_BASE}/${filename}`, location); + // Browser suites serve this directory; keep their existing fixture URLs. + fs.copyFileSync(location, path.join(rootDir, 'videos', filename)); } }; diff --git a/packages/media/src/audio-iterator-manager.ts b/packages/media/src/audio-iterator-manager.ts index 98d16ee50f9..7b55a97c637 100644 --- a/packages/media/src/audio-iterator-manager.ts +++ b/packages/media/src/audio-iterator-manager.ts @@ -428,7 +428,11 @@ export const audioIteratorManager = ({ return; } - const maximumTimestamp = getMediaEndTimestamp(); + // Pitch shifting needs future samples; truncating its input changes the tail. + const maximumTimestamp = + loop || currentToneFrequency !== 1 + ? getMediaEndTimestamp() + : Math.min(getMediaEndTimestamp(), getSequenceEndTimestamp()); if (startFromSecond >= maximumTimestamp) { return; } diff --git a/packages/media/src/media-player.ts b/packages/media/src/media-player.ts index 2169ffc81bd..dd76003fcb1 100644 --- a/packages/media/src/media-player.ts +++ b/packages/media/src/media-player.ts @@ -799,6 +799,7 @@ export class MediaPlayer { const previousDuration = this.sequenceDurationInFrames; if (previousDuration !== sequenceDurationInFrames) { this.sequenceDurationInFrames = sequenceDurationInFrames; + this.audioIteratorManager?.destroyIterator(); await this.seekTo(unloopedTimeInSeconds, 'discontinuity'); } } diff --git a/packages/media/src/test/media-player.test.ts b/packages/media/src/test/media-player.test.ts index 93d170cd15a..26489b11f6d 100644 --- a/packages/media/src/test/media-player.test.ts +++ b/packages/media/src/test/media-player.test.ts @@ -55,7 +55,14 @@ const makeAudioPlayer = ( Partial< Pick< ConstructorParameters[0], - 'getEffects' | 'getEffectChainState' | 'bufferState' + | 'getEffects' + | 'getEffectChainState' + | 'bufferState' + | 'durationInFrames' + | 'playbackRate' + | 'trimBefore' + | 'trimAfter' + | 'loop' > >) | null = null, @@ -67,10 +74,10 @@ const makeAudioPlayer = ( (tagType === 'audio' ? '/voice-note.m4a' : '/bigbuckbunny.mp4'), logLevel: 'error', sharedAudioContext, - loop: false, - trimBefore: undefined, - trimAfter: undefined, - playbackRate: 1, + loop: options?.loop ?? false, + trimBefore: options?.trimBefore, + trimAfter: options?.trimAfter, + playbackRate: options?.playbackRate ?? 1, toneFrequency: 1, globalPlaybackRate: 1, audioStreamIndex: null, @@ -79,7 +86,7 @@ const makeAudioPlayer = ( bufferState: options?.bufferState ?? makeBufferState(), isPremounting: false, isPostmounting: false, - durationInFrames: 300, + durationInFrames: options?.durationInFrames ?? 300, onVideoFrameCallback: options?.onVideoFrameCallback ?? null, playing: false, sequenceOffset: 0, @@ -624,3 +631,121 @@ test.each([ } }, ); + +// Regression for kino-ai/kino#4930: short clips decoded unused source audio, +// and retaining an exhausted iterator could leave live extensions silent. +test.each([ + { + tagType: 'audio' as const, + playbackRate: 1, + durationInFrames: 6, + loop: false, + trimAfter: undefined, + }, + { + tagType: 'video' as const, + playbackRate: 2, + durationInFrames: 6, + loop: false, + trimAfter: undefined, + }, + { + tagType: 'audio' as const, + playbackRate: 0.5, + durationInFrames: 1, + loop: false, + trimAfter: undefined, + }, + { + tagType: 'audio' as const, + playbackRate: 1, + durationInFrames: 6, + loop: false, + trimAfter: 33, + }, + { + tagType: 'audio' as const, + playbackRate: 1, + durationInFrames: 6, + loop: true, + trimAfter: undefined, + }, +])( + 'bounds $tagType audio decoding at $playbackRate×, $durationInFrames frames, loop=$loop, trimAfter=$trimAfter', + async (options) => { + const shared = makeSharedAudioContext(); + const player = makeAudioPlayer(shared, null, options.tagType, { + ...options, + canvas: null, + onVideoFrameCallback: null, + src: + options.tagType === 'audio' ? '/voice-note.m4a' : '/bigbuckbunny.mp4', + trimBefore: 30, + }); + const buffers = vi.spyOn(AudioBufferSink.prototype, 'buffers'); + try { + expect((await player.initialize(0, false, 1)).type).toBe('success'); + await vi.waitFor(() => expect(buffers).toHaveBeenCalled()); + const end = buffers.mock.calls[0][1]!; + const sequenceEnd = + 1 + (options.durationInFrames / 30) * options.playbackRate; + if (options.loop) { + expect(end).toBeGreaterThan(sequenceEnd); + } else { + expect(end).toBeCloseTo( + Math.min(sequenceEnd, (options.trimAfter ?? Infinity) / 30), + ); + } + + await vi.waitFor(() => { + const queued = player + .audioIteratorManager!.getAudioBufferIterator()! + .getQueuedPeriod(); + expect(queued?.until).toBeCloseTo(Math.min(sequenceEnd, end)); + }); + } finally { + await player.dispose(); + await shared.audioContext.close(); + buffers.mockRestore(); + } + }, +); + +test.each(['audio', 'video'] as const)( + 'reopens capped %s audio after extension, shrink and re-extension', + async (tagType) => { + const shared = makeSharedAudioContext(); + const player = makeAudioPlayer(shared, null, tagType, { + canvas: null, + onVideoFrameCallback: null, + src: tagType === 'audio' ? '/voice-note.m4a' : '/bigbuckbunny.mp4', + durationInFrames: 6, + }); + try { + expect((await player.initialize(0, false, 1)).type).toBe('success'); + await vi.waitFor(() => + expect( + player + .audioIteratorManager!.getAudioBufferIterator()! + .getQueuedPeriod()?.until, + ).toBeCloseTo(0.2), + ); + player.play(); + for (const duration of [15, 6, 15]) { + // The playhead is still inside the queued old range. A seek alone + // would retain the old iterator and its original decode endpoint. + await player.setSequenceDurationInFrames(duration, 0.1); + await vi.waitFor(() => { + const queued = player + .audioIteratorManager!.getAudioBufferIterator()! + .getQueuedPeriod(); + expect(queued?.from).toBeLessThanOrEqual(0.1); + expect(queued?.until).toBeCloseTo(duration / 30); + }); + } + } finally { + await player.dispose(); + await shared.audioContext.close(); + } + }, +); diff --git a/turbo.json b/turbo.json index 80672ad62c4..a8048c11eb6 100644 --- a/turbo.json +++ b/turbo.json @@ -88,7 +88,18 @@ }, "@remotion/example-videos#make": { "dependsOn": ["^make"], - "outputs": ["dist", "node_modules/.videos/**"], + "outputs": [ + "dist", + "node_modules/.videos/**", + "videos/bigbuckbunny.mp4", + "videos/music-visualization-demo-track.mp3", + "videos/corrupted.mp4", + "videos/variablefps.webm", + "videos/iphonevideo.mov", + "videos/music.mp3", + "videos/iphone-hevc.mov", + "videos/stretched-vp8.webm" + ], "outputLogs": "new-only" }, "@remotion/example#bundle": {