fix(qta): follow-up to the #762 review (track choice in clips, codec query caching)

docs/ENGINES.md

@@ -797,7 +797,9 @@ in the preset library only). Video and audio need FFmpeg.
   is encoded to AAC.
 - **Merge videos** copies the clips with the concat demuxer only when all share the codec setup (ffprobe's extradata
   hash), size, pixel format, frame rate, turn and kind of sound; otherwise every clip is fitted into the first clip's
-  frame and encoded, a clip without sound silent for its length. **Merge audio** always decodes and joins the sound
+  frame and encoded, a clip without sound silent for its length. Sound tracks FFmpeg can't decode are skipped: each
+  clip gives its first track FFmpeg can decode, and the clips are copied only when that track has the same number in
+  every clip. **Merge audio** always decodes and joins the sound
   into one M4A, so no encoder padding or gap is left between files. Both are in `Media/MediaJoining.cs`.
 - **Video from images**: each picture is fitted into 1920 × 1080 as a JPEG, then shown for the preset's seconds with
   crossfades (`xfade`) of 1 s, at most half a picture's time, or cuts. At most `MediaOptions.MaxSlides` (300)
@@ -1578,7 +1580,9 @@ format, and FFmpeg reads what the file holds, so a `.dat` with no video in it fa
   subtitle engine (ASS and WebVTT extraction cost 11 so SRT is the hub).
 - The first video stream (cover pictures excluded) is converted. Video to video keeps every audio track (languages,
   commentary), each copied where the target takes its codec or encoded on its own, with its language tag; FLV and DV
-  hold one and keep the first. Video to audio takes the first. Files without sound still convert to video. Title, artist
+  hold one and keep the first. Video to audio takes the first. Tracks FFmpeg can't decode (no decoder, like the spatial
+  APAC track of iPhone recordings, or a codec ffprobe can't name) are skipped, unless the file has no other sound; the
+  clip window's waveform, *Remove silence* and clip music take the same track. Files without sound still convert to video. Title, artist
   and other tags and chapters are kept, album art too for MP3, M4A, M4B and FLAC. Portrait phone videos are turned
   upright. Odd sizes are evened out, frame rates that MPEG-2/MXF/MPEG-4 part 2 cannot store are converted, and channels
   WMA or AMR cannot hold are mixed down.

src/Filee.Engines/Media/ClipPreview.cs

@@ -77,7 +77,7 @@ public static async Task PlayAsync(string path, ClipEdit edit, int height, Actio
         {
             if ((info.Audio is not null || clip.Music is not null) && Player() is { } ffplay)
             {
-                decoder = Start(ffmpeg, MediaEncoding.ClipPreviewAudio(input, clip, video, info.Audio is not null, info.Audio?.Ordinal ?? 0), stdin: false);
+                decoder = Start(ffmpeg, await PreviewSoundArgumentsAsync(ffprobe, input, clip, video, info.Audio, stop.Token).ConfigureAwait(false), stdin: false);
                 player = Start(ffplay, ["-hide_banner", "-loglevel", "error", "-nodisp", "-autoexit",
                     .. PlayerVolume is { } volume ? new[] { "-volume", volume.ToString(CultureInfo.InvariantCulture) } : [], "-i", "-"], stdin: true);
                 pump = Pump(decoder, player, stop.Token);
@@ -131,22 +131,41 @@ public static async Task PlayAsync(string path, ClipEdit edit, int height, Actio
     }
 
     /// <summary>
-    /// How loud the sound of the video at <paramref name="path"/> is over its <paramref name="duration"/> seconds, in
+    /// The arguments of the preview's sound: the music's track probed like the video's.
+    /// </summary>
+    internal static async Task<List<string>> PreviewSoundArgumentsAsync(string ffprobe, string input, ClipEdit clip,
+        VideoStreamInfo video, AudioStreamInfo? audio, CancellationToken cancellationToken)
+    {
+        var musicTrack = clip.Music is { } song
+            ? await MediaInfo.AudioTrackAsync(ffprobe, Path.GetFullPath(song.Path), cancellationToken).ConfigureAwait(false) : 0;
+        return MediaEncoding.ClipPreviewAudio(input, clip, video, audio is not null, audio?.Ordinal ?? 0, musicTrack);
+    }
+
+    /// <summary>
+    /// How loud the sound (the track conversions use) of the video at <paramref name="path"/> is over its <paramref name="duration"/> seconds, in
     /// <paramref name="count"/> steps from 0 (silent) to 1 (its loudest step); empty when it has no sound.
     /// </summary>
     /// <param name="workFolder">Where the samples are written for a moment.</param>
     public static Task<float[]> ReadLoudnessAsync(string path, double duration, int count, string workFolder, CancellationToken cancellationToken) =>
         ReadLoudnessAsync(path, 0, duration, count, workFolder, cancellationToken);
 
     /// <summary>
-    /// How loud the sound of the file at <paramref name="path"/> is from <paramref name="start"/> to
+    /// How loud the sound (the track conversions use) of the file at <paramref name="path"/> is from <paramref name="start"/> to
     /// <paramref name="end"/> seconds, in <paramref name="count"/> steps from 0 to 1 (its loudest step); empty
     /// without sound.
     /// </summary>
     public static async Task<float[]> ReadLoudnessAsync(string path, double start, double end, int count, string workFolder,
         CancellationToken cancellationToken)
     {
-        var (ffmpeg, _) = FfmpegConverter.Locate() ?? throw new InvalidOperationException("FFmpeg was not found.");
+        return await ReadLoudnessAsync(FfmpegConverter.Locate() ?? throw new InvalidOperationException("FFmpeg was not found."),
+            path, start, end, count, workFolder, cancellationToken).ConfigureAwait(false);
+    }
+
+    internal static async Task<float[]> ReadLoudnessAsync((string Ffmpeg, string Ffprobe) tools, string path, double start, double end,
+        int count, string workFolder, CancellationToken cancellationToken)
+    {
+        var input = Path.GetFullPath(path);
+        var track = await MediaInfo.AudioTrackAsync(tools.Ffprobe, input, cancellationToken).ConfigureAwait(false);
         var samples = Path.Combine(workFolder, Guid.NewGuid().ToString("N") + ".f32");
         try
         {
@@ -155,9 +174,9 @@ public static async Task<float[]> ReadLoudnessAsync(string path, double start, d
                 args.AddRange(["-ss", ClipRange.Ffmpeg(start)]);
             if (end > start)
                 args.AddRange(["-to", ClipRange.Ffmpeg(end)]);
-            args.AddRange(["-i", Path.GetFullPath(path), "-map", "0:a:0", "-ac", "1", "-ar", PeakRate.ToString(CultureInfo.InvariantCulture),
+            args.AddRange(["-i", input, "-map", $"0:a:{track}", "-ac", "1", "-ar", PeakRate.ToString(CultureInfo.InvariantCulture),
                 "-c:a", "pcm_f32le", "-f", "f32le", samples]);
-            var result = await FfmpegRunner.RunAsync(ffmpeg, args, null, cancellationToken);
+            var result = await FfmpegRunner.RunAsync(tools.Ffmpeg, args, null, cancellationToken);
             if (result.ExitCode != 0 || !File.Exists(samples))
                 return []; // no sound
             var bytes = await File.ReadAllBytesAsync(samples, cancellationToken);
@@ -186,18 +205,27 @@ public static async Task<double> ReadSoundLengthAsync(string path, CancellationT
     /// <summary>The silent stretches of the video at <paramref name="path"/> between <paramref name="start"/> and <paramref name="end"/>.</summary>
     public static async Task<IReadOnlyList<ClipRange>> FindSilencesAsync(string path, double start, double end, CancellationToken cancellationToken)
     {
-        var (ffmpeg, _) = FfmpegConverter.Locate() ?? throw new InvalidOperationException("FFmpeg was not found.");
-        var result = await ProcessRunner.RunAsync(ffmpeg, SilenceArguments(Path.GetFullPath(path), start, end), TimeSpan.FromMinutes(10), cancellationToken);
+        return await FindSilencesAsync(FfmpegConverter.Locate() ?? throw new InvalidOperationException("FFmpeg was not found."),
+            path, start, end, cancellationToken).ConfigureAwait(false);
+    }
+
+    internal static async Task<IReadOnlyList<ClipRange>> FindSilencesAsync((string Ffmpeg, string Ffprobe) tools, string path,
+        double start, double end, CancellationToken cancellationToken)
+    {
+        var input = Path.GetFullPath(path);
+        var track = await MediaInfo.AudioTrackAsync(tools.Ffprobe, input, cancellationToken).ConfigureAwait(false);
+        var result = await ProcessRunner.RunAsync(tools.Ffmpeg, SilenceArguments(input, start, end, track), TimeSpan.FromMinutes(10), cancellationToken);
         if (result.ExitCode != 0)
             throw new InvalidOperationException("FFmpeg failed: " + FfmpegRunner.Summarize(result.StandardError));
         return ParseSilences(result.StandardError, start, end);
     }
 
     /// <summary>FFmpeg's silencedetect over <paramref name="start"/> to <paramref name="end"/> of the video, reported on stderr.</summary>
-    internal static List<string> SilenceArguments(string input, double start, double end) =>
+    /// <param name="audioTrack">The selected audio track's ordinal in the input file.</param>
+    internal static List<string> SilenceArguments(string input, double start, double end, int audioTrack = 0) =>
     [
         "-hide_banner", "-nostdin", "-nostats", "-v", "info", "-ss", ClipRange.Ffmpeg(start), "-to", ClipRange.Ffmpeg(end), "-i", input,
-        "-map", "0:a:0", "-af", $"silencedetect=noise={SilenceThreshold.ToString(CultureInfo.InvariantCulture)}dB:d={ClipCutter.MinSilence.ToString(CultureInfo.InvariantCulture)}",
+        "-map", $"0:a:{audioTrack}", "-af", $"silencedetect=noise={SilenceThreshold.ToString(CultureInfo.InvariantCulture)}dB:d={ClipCutter.MinSilence.ToString(CultureInfo.InvariantCulture)}",
         "-f", "null", "-",
     ];
 

src/Filee.Engines/Media/FfmpegConverter.cs

@@ -124,7 +124,7 @@ public async Task<IReadOnlyList<string>> ConvertAsync(ConversionStep step, IProg
         if (step.MotionPhoto)
             return await ExtractMotionAsync(step, progress, cancellationToken);
         if (step.LivePhoto)
-            return await LivePhotoWriter.ConvertAsync(ffmpeg, ffprobe, step, EncodersOf(ffmpeg), filter => HasFilter(ffmpeg, filter),
+            return await LivePhotoWriter.ConvertAsync(ffmpeg, ffprobe, step, await EncodersOfAsync(ffmpeg), filter => HasFilter(ffmpeg, filter),
                 progress, cancellationToken);
         if (step.To == "heic")
             return await HeicWriter.ConvertAsync(ffmpeg, step, progress, cancellationToken);
@@ -135,7 +135,7 @@ public async Task<IReadOnlyList<string>> ConvertAsync(ConversionStep step, IProg
         if (step.FrameSequence)
             return await FrameSequenceWriter.ConvertAsync(ffmpeg, step, input, info, progress, cancellationToken);
         if (step.Preset.CutsClipOf(step.From) && MediaEncoding.VideoTargets.ContainsKey(step.To))
-            return await CutClipAsync(ffmpeg, step, input, info, progress, cancellationToken);
+            return await CutClipAsync(ffmpeg, ffprobe, step, input, info, progress, cancellationToken);
 
         var output = step.Output.Allocate(FormatRegistry.Get(step.To).PrimaryExtension);
         if (output is null)
@@ -179,7 +179,7 @@ await RunPassesAsync(ffmpeg, [new FfmpegPass(MediaEncoding.WithoutMetadata(input
                 throw new ConversionRefusedException("error.over_size_limit", $"{info.Duration.Value.TotalSeconds:0} s in {media.MaxSizeMb:0.##} MB");
         }
 
-        var encoders = still ? null : EncodersOf(ffmpeg);
+        var encoders = still ? null : await EncodersOfAsync(ffmpeg);
         // Hardware encoding (preset option): the first GPU encoder that passes its test encode, if any.
         HardwareEncoder? hardware = null;
         if (!still && MediaEncoding.HardwareCodec(step.To, media, info) is { } codec)
@@ -260,7 +260,7 @@ public async Task JoinAsync(IReadOnlyList<string> inputPaths, string outputPath,
         if (media.Slideshow)
         {
             var slides = await PrepareSlidesAsync(ffmpeg, inputPaths, scratch, progress, cancellationToken);
-            var encoder = MediaEncoding.VideoEncoder("mp4", "libx264", media.VideoCodec, EncodersOf(ffmpeg));
+            var encoder = MediaEncoding.VideoEncoder("mp4", "libx264", media.VideoCodec, await EncodersOfAsync(ffmpeg));
             args = MediaJoining.Slideshow(slides, outputPath, media, encoder, scratch);
             duration = MediaJoining.SlideshowDuration(slides.Count, media);
             start = 0.2;
@@ -291,7 +291,7 @@ public async Task JoinAsync(IReadOnlyList<string> inputPaths, string outputPath,
             }
             else
             {
-                var encoder = MediaEncoding.VideoEncoder("mp4", "libx264", media.VideoCodec, EncodersOf(ffmpeg));
+                var encoder = MediaEncoding.VideoEncoder("mp4", "libx264", media.VideoCodec, await EncodersOfAsync(ffmpeg));
                 args = MediaJoining.EncodeVideos(clips, outputPath, media, encoder, scratch);
             }
         }
@@ -343,7 +343,7 @@ private static async Task RunJoinPassAsync(string ffmpeg, IReadOnlyList<string>
     /// the target can't take (a codec its container doesn't hold) is encoded exactly instead, like the preset's other
     /// videos. Several parts, speeds, volumes, pauses or fades are encoded through one filter graph.
     /// </summary>
-    private async Task<IReadOnlyList<string>> CutClipAsync(string ffmpeg, ConversionStep step, string input, MediaInfo info,
+    private async Task<IReadOnlyList<string>> CutClipAsync(string ffmpeg, string ffprobe, ConversionStep step, string input, MediaInfo info,
         IProgress<double>? progress, CancellationToken cancellationToken)
     {
         if (info.Video is null)
@@ -359,7 +359,10 @@ private async Task<IReadOnlyList<string>> CutClipAsync(string ffmpeg, Conversion
             // Parts, speeds, volumes, pauses or fades: one filter graph, always encoded.
             try
             {
-                var pass = new FfmpegPass(MediaEncoding.ClipEditArguments(input, output, step.To, edit, media, info, EncodersOf(ffmpeg)), 1);
+                var musicTrack = edit.Music is { } song && !media.RemoveAudio
+                    ? await MediaInfo.AudioTrackAsync(ffprobe, Path.GetFullPath(song.Path), cancellationToken) : 0;
+                var pass = new FfmpegPass(MediaEncoding.ClipEditArguments(input, output, step.To, edit, media, info,
+                    await EncodersOfAsync(ffmpeg), musicTrack), 1);
                 await RunPassesAsync(ffmpeg, [pass], output, duration, progress, cancellationToken);
             }
             catch
@@ -390,7 +393,7 @@ private async Task<IReadOnlyList<string>> CutClipAsync(string ffmpeg, Conversion
                 }
             }
 
-            var encoders = EncodersOf(ffmpeg);
+            var encoders = await EncodersOfAsync(ffmpeg);
             HardwareEncoder? hardware = null;
             if (MediaEncoding.HardwareCodec(step.To, media, info, encode: true) is { } codec)
             {
@@ -557,45 +560,58 @@ internal static string VersionOf(string ffmpeg, TimeSpan? timeout = null) =>
     [GeneratedRegex(@"version\s+n?(\d+(?:\.\d+)+|\S+)")]
     private static partial Regex VersionPattern();
 
-    private static readonly ConcurrentDictionary<string, IReadOnlySet<string>?> Encoders = new();
+    private static readonly ConcurrentDictionary<string, Lazy<Task<IReadOnlySet<string>?>>> Encoders = new();
 
     /// <summary>
     /// The encoders of <paramref name="ffmpeg"/> (<c>ffmpeg -encoders</c>), cached per executable: a copy the user
     /// picked may lack x265 or SVT-AV1. Null (assume Filee's build) when the list can't be read.
+    /// Queried once at a time and kept when read; asked again after a failure.
     /// </summary>
-    internal static IReadOnlySet<string>? EncodersOf(string ffmpeg, TimeSpan? timeout = null) =>
-        Encoders.GetOrAdd($"{ffmpeg}|{File.GetLastWriteTimeUtc(ffmpeg).Ticks}", _ =>
-        {
-            try
-            {
-                var result = FfmpegRunner.RunBriefly(ffmpeg, ["-hide_banner", "-encoders"], timeout ?? QueryTimeout);
-                return result.ExitCode == 0 ? ParseEncoders(result.StandardOutput) : null;
-            }
-            catch (Exception ex) when (ex is TimeoutException or InvalidOperationException or System.ComponentModel.Win32Exception or IOException)
-            {
-                return null;
-            }
-        });
+    internal static Task<IReadOnlySet<string>?> EncodersOfAsync(string ffmpeg, TimeSpan? timeout = null) =>
+        QueryOnceAsync(Encoders, ffmpeg, "-encoders", timeout, ParseEncoders);
 
-    private static readonly ConcurrentDictionary<string, IReadOnlySet<string>?> Decoders = new();
+    private static readonly ConcurrentDictionary<string, Lazy<Task<IReadOnlySet<string>?>>> Decoders = new();
 
     /// <summary>
     /// The decodable codec names of this ffmpeg, cached per executable. Null assumes every codec can be decoded
     /// when the table cannot be read.
+    /// Queried once at a time and kept when read; asked again after a failure.
     /// </summary>
-    internal static IReadOnlySet<string>? DecodableCodecsOf(string ffmpeg, TimeSpan? timeout = null) =>
-        Decoders.GetOrAdd($"{ffmpeg}|{File.GetLastWriteTimeUtc(ffmpeg).Ticks}", _ =>
+    internal static Task<IReadOnlySet<string>?> DecodableCodecsOfAsync(string ffmpeg, TimeSpan? timeout = null) =>
+        QueryOnceAsync(Decoders, ffmpeg, "-codecs", timeout, ParseDecodableCodecs);
+
+    private static async Task<IReadOnlySet<string>?> QueryOnceAsync(
+        ConcurrentDictionary<string, Lazy<Task<IReadOnlySet<string>?>>> cache, string ffmpeg, string option,
+        TimeSpan? timeout, Func<string, IReadOnlySet<string>> parse)
+    {
+        var key = $"{ffmpeg}|{File.GetLastWriteTimeUtc(ffmpeg).Ticks}";
+        var entry = cache.GetOrAdd(key, _ => new Lazy<Task<IReadOnlySet<string>?>>(() => Task.Run(Query)));
+        try
+        {
+            var result = await entry.Value.ConfigureAwait(false);
+            if (result is null)
+                cache.TryRemove(KeyValuePair.Create(key, entry));
+            return result;
+        }
+        catch
+        {
+            cache.TryRemove(KeyValuePair.Create(key, entry));
+            throw;
+        }
+
+        IReadOnlySet<string>? Query()
         {
             try
             {
-                var result = FfmpegRunner.RunBriefly(ffmpeg, ["-hide_banner", "-codecs"], timeout ?? QueryTimeout);
-                return result.ExitCode == 0 ? ParseDecodableCodecs(result.StandardOutput) : null;
+                var result = FfmpegRunner.RunBriefly(ffmpeg, ["-hide_banner", option], timeout ?? QueryTimeout);
+                return result.ExitCode == 0 ? parse(result.StandardOutput) : null;
             }
             catch (Exception ex) when (ex is TimeoutException or InvalidOperationException or System.ComponentModel.Win32Exception or IOException)
             {
                 return null;
             }
-        });
+        }
+    }
 
     private static readonly ConcurrentDictionary<string, IReadOnlySet<string>?> Filters = new();