Export: stream frames through a shared CPU/GPU work queue, convert to YUV 4:2:0 on the GPU, and tune to the hardware

apps/effectcraft-cli/Cargo.toml

@@ -13,12 +13,14 @@ name = "effectcraft-cli"
 path = "src/main.rs"
 
 [dependencies]
+rayon = { workspace = true }
 effectcraft-engine = { workspace = true }
 effectcraft-host = { workspace = true }
 effectcraft-automation = { workspace = true }
 effectcraft-ui-egui = { workspace = true }
 effectcraft-render = { workspace = true }
 effectcraft-gpu = { workspace = true }
+effectcraft-export = { workspace = true }
 effectcraft-time = { workspace = true }
 serde_json = { workspace = true }
 image = { workspace = true }

apps/effectcraft-cli/src/export_host.rs

@@ -0,0 +1,109 @@
+//! Host facts for the export tuning (`effectcraft_export::tuning::Host`): the export library
+//! probes nothing itself, so the CLI measures RAM, core kinds and VRAM here (process spawns
+//! and system files) and passes them in.
+
+use effectcraft_export::tuning::Host;
+
+/// RAM, core kinds and dedicated VRAM of this machine; unknown values stay `None`.
+pub fn probe() -> Host {
+    let (ram_bytes, avail_ram_bytes) = memory();
+    Host { ram_bytes, avail_ram_bytes, core_kinds: core_kinds(), nvidia_vram_bytes: nvidia_smi_vram_mb().map(|mb| mb << 20), other_vram_bytes: sysfs_vram() }
+}
+
+/// `sysctl -n NAME` (macOS), parsed.
+#[cfg(target_os = "macos")]
+fn sysctl(name: &str) -> Option<u64> {
+    let out = std::process::Command::new("/usr/sbin/sysctl").args(["-n", name]).output().ok()?;
+    String::from_utf8_lossy(&out.stdout).trim().parse().ok()
+}
+
+/// Physical memory and memory available now (free + inactive + speculative + purgeable pages on
+/// macOS, `MemAvailable` on Linux); `None` where not queried.
+fn memory() -> (Option<u64>, Option<u64>) {
+    #[cfg(target_os = "macos")]
+    {
+        let total = sysctl("hw.memsize");
+        let avail = std::process::Command::new("/usr/bin/vm_stat").output().ok().and_then(|o| {
+            let text = String::from_utf8_lossy(&o.stdout).into_owned();
+            let page = text.split("page size of ").nth(1)?.split_whitespace().next()?.parse::<u64>().ok()?;
+            let pages = |key: &str| {
+                text.lines()
+                    .find(|l| l.starts_with(key))
+                    .and_then(|l| l.rsplit(':').next())
+                    .and_then(|v| v.trim().trim_end_matches('.').parse::<u64>().ok())
+                    .unwrap_or(0)
+            };
+            let n = pages("Pages free") + pages("Pages inactive") + pages("Pages speculative") + pages("Pages purgeable");
+            Some(n.saturating_mul(page))
+        });
+        (total, avail)
+    }
+    #[cfg(target_os = "linux")]
+    {
+        let Ok(m) = std::fs::read_to_string("/proc/meminfo") else { return (None, None) };
+        let kb =
+            |key: &str| m.lines().find(|l| l.starts_with(key)).and_then(|l| l.split_whitespace().nth(1)).and_then(|v| v.parse::<u64>().ok()).map(|v| v * 1024);
+        (kb("MemTotal:"), kb("MemAvailable:"))
+    }
+    #[cfg(windows)]
+    {
+        // No platform calls (the workspace has no `unsafe`): ask CIM through PowerShell, in KB.
+        let out = std::process::Command::new("powershell.exe")
+            .args([
+                "-NoProfile",
+                "-NonInteractive",
+                "-Command",
+                "$o = Get-CimInstance Win32_OperatingSystem; \"$($o.TotalVisibleMemorySize) $($o.FreePhysicalMemory)\"",
+            ])
+            .output();
+        let Ok(out) = out else { return (None, None) };
+        let text = String::from_utf8_lossy(&out.stdout).into_owned();
+        let mut kb = text.split_whitespace().map(|v| v.parse::<u64>().ok().map(|v| v * 1024));
+        (kb.next().flatten(), kb.next().flatten())
+    }
+    #[cfg(not(any(target_os = "macos", target_os = "linux", windows)))]
+    {
+        (None, None)
+    }
+}
+
+/// Performance and efficiency cores (Apple silicon's `hw.perflevel0/1`); `None` elsewhere.
+fn core_kinds() -> Option<(usize, usize)> {
+    #[cfg(target_os = "macos")]
+    {
+        let p = sysctl("hw.perflevel0.logicalcpu")? as usize;
+        let e = sysctl("hw.perflevel1.logicalcpu").unwrap_or(0) as usize;
+        Some((p, e))
+    }
+    #[cfg(not(target_os = "macos"))]
+    {
+        None
+    }
+}
+
+/// Dedicated VRAM from Linux sysfs (amdgpu reports it; NVIDIA's proprietary driver and Intel do
+/// not); `None` elsewhere. NVIDIA (Windows and Linux): [`nvidia_smi_vram_mb`], which ships with
+/// the driver. No platform calls: the workspace has no `unsafe`.
+fn sysfs_vram() -> Option<u64> {
+    #[cfg(target_os = "linux")]
+    {
+        let dir = std::fs::read_dir("/sys/class/drm").ok()?;
+        dir.filter_map(|e| e.ok())
+            .filter_map(|e| std::fs::read_to_string(e.path().join("device/mem_info_vram_total")).ok())
+            .filter_map(|t| t.trim().parse::<u64>().ok())
+            .max()
+    }
+    #[cfg(not(target_os = "linux"))]
+    {
+        None
+    }
+}
+
+/// Total memory of the largest NVIDIA GPU in MB, from `nvidia-smi`; `None` without it.
+fn nvidia_smi_vram_mb() -> Option<u64> {
+    let out = std::process::Command::new("nvidia-smi").args(["--query-gpu=memory.total", "--format=csv,noheader,nounits"]).output().ok()?;
+    if !out.status.success() {
+        return None;
+    }
+    String::from_utf8_lossy(&out.stdout).lines().filter_map(|l| l.trim().parse::<u64>().ok()).max()
+}

apps/effectcraft-cli/src/main.rs

@@ -41,6 +41,7 @@
 
 use std::io::Write;
 
+mod export_host;
 mod template;
 
 use effectcraft_automation::tools::{self, Reply};
@@ -103,6 +104,10 @@ const USAGE: &str = "usage: effectcraft-cli <info|commands|exec|run|props|get|se
          [--jpeg-quality N] [--bitrate KBPS] [--prores PROFILE] [--audio auto|on|off]
          [--profile main|main10] [--level auto|4.1] [--rate-control bitrate|quality] [--video-quality N]
          [--keyint FRAMES] [--webm-codec vp9|av1] [--audio-bitrate KBPS] [--opus-app audio|voice] | --queue
+         [--gpu] [--print-tuning] [--threads N] [--tuning-config FILE] [--<export option> VALUE ...]
+         (export options: backend, gpu-frames, gpu-inflight, cpu-frames, cpu-pool, cache-mb,
+          precomp-resident, gpu-yuv, async-readback, vram-budget-mb, ...; see TUNING.md; each also
+          EFFECTCRAFT_EXPORT_<OPTION>; CLI > env > config > auto)
                                            (formats h264|hevc|av1|prores|webm|png|jpeg|tiff|exr|gif|wav|aiff;
                                            --profile..--keyint: HEVC / AV1, --audio-bitrate/--opus-app: WebM Opus)
                                            a relative --out is relative to the working directory
@@ -169,11 +174,29 @@ const VALUED: &[&str] = &[
     // mcp
     "--automation-read-root",
     "--automation-write-root",
+    "--backend",
+    "--gpu-frames",
+    "--gpu-inflight",
+    "--cpu-frames",
+    "--cpu-pool",
+    "--gpu-pool",
+    "--cache-mb",
+    "--precomp-resident",
+    "--gpu-yuv",
+    "--async-readback",
+    "--vram-budget-mb",
+    "--ram-budget-mb",
+    "--cpu-batch",
+    "--upload-budget-mb",
+    "--pool-budget-mb",
+    "--tuning-config",
+    "--threads",
 ];
 
 /// Options without a value. Any other `--option` is a usage error.
 const FLAGS: &[&str] = &[
     "--json",
+    "--print-tuning",
     "--save",
     "--demo",
     "--empty",
@@ -361,6 +384,40 @@ fn session(args: &Args) -> Result<effectcraft_engine::Session, Failure> {
     Ok(s)
 }
 
+/// `render` export options (`effectcraft_export::tuning`): every `--<option> VALUE` of
+/// `tuning::OPTIONS`, `--tuning-config FILE`, `--print-tuning` (resolve and print, render
+/// nothing) and `--threads N` (rayon threads; simulates a smaller CPU). `RUST_LOG=info` logs
+/// the resolved table and which compositor rendered each frame.
+fn export_tuning(args: &Args) -> Result<(), Failure> {
+    use effectcraft_export::tuning::{OPTIONS, Overrides, Request, set_request};
+    if std::env::var_os("RUST_LOG").is_some() {
+        effectcraft_engine::logging::install();
+    }
+    let threads = match args.opt("--threads") {
+        Some(v) => Some(v.parse::<usize>().map_err(|_| Failure::Usage(format!("--threads {v}: expected a number")))?),
+        None => std::env::var("EFFECTCRAFT_EXPORT_THREADS").ok().and_then(|v| v.trim().parse().ok()),
+    };
+    if let Some(n) = threads.filter(|n| *n > 0) {
+        rayon::ThreadPoolBuilder::new().num_threads(n).build_global().map_err(|e| Failure::Error(format!("--threads: {e}")))?;
+    }
+    let mut cli = Overrides::default();
+    for (key, _) in OPTIONS {
+        if let Some(v) = args.opt(&format!("--{key}")) {
+            cli.set(key, v).map_err(Failure::Usage)?;
+        }
+    }
+    let read = |p: &str| std::fs::read_to_string(p).map_err(|e| e.to_string()).and_then(|t| Overrides::from_config(&t));
+    let config = match args.opt("--tuning-config") {
+        Some(p) => Some(read(p).map_err(|e| Failure::Usage(format!("--tuning-config {p}: {e}")))?),
+        None => std::env::var("EFFECTCRAFT_EXPORT_CONFIG")
+            .ok()
+            .and_then(|p| read(&p).map_err(|e| eprintln!("warning: EFFECTCRAFT_EXPORT_CONFIG {p}: {e} (ignored)")).ok()),
+    };
+    let env = Overrides::from_lookup(|k| std::env::var(k).ok());
+    set_request(Request { cli, env, config, host: export_host::probe(), print_only: args.flag("--print-tuning") });
+    Ok(())
+}
+
 /// `--save` / `--save-as` after a mutating command.
 fn maybe_save(b: &mut Backend, args: &Args) -> Result<Option<String>, Failure> {
     let r = if let Some(p) = args.opt("--save-as") {
@@ -657,6 +714,7 @@ fn render(args: &Args, json_out: bool) -> Result<(), Failure> {
     if args.opt("--bridge").is_some() {
         return usage_err("render runs headless; use `exec renderQueue.add` / `renderQueue.render` with --bridge");
     }
+    export_tuning(args)?;
     let mut s = session(args)?;
     if args.opt("--template").is_some() {
         template::prepare(&mut s, args)?;
@@ -763,6 +821,10 @@ fn render(args: &Args, json_out: bool) -> Result<(), Failure> {
     if tty {
         note!();
     }
+    if args.flag("--print-tuning") {
+        // The export printed its tuning and stopped before rendering.
+        return Ok(());
+    }
     let mut results = vec![];
     let mut failed = vec![];
     for it in s.project.render_queue.iter().filter(|i| i.render && i.started.is_some()) {

crates/export/TUNING.md

@@ -0,0 +1,91 @@
+# Export tuning
+
+Every pipeline improvement of the GPU export is an independent option. Each has three
+spellings, resolved per export with precedence **CLI > environment > config file > auto**:
+
+| CLI (`render`)        | environment                          | config file          |
+|-----------------------|--------------------------------------|----------------------|
+| `--gpu-inflight 3`    | `EFFECTCRAFT_EXPORT_GPU_INFLIGHT=3`  | `gpu-inflight = 3`   |
+
+Config file: `--tuning-config FILE` or `EFFECTCRAFT_EXPORT_CONFIG=FILE` (`key = value` lines,
+`#` comments). The CLI reads the environment variables and the config file and passes them to the
+export library (`tuning::Request`); other callers pass their own values or none. The value `auto` (or leaving an option out) lets the auto rules decide.
+`--print-tuning` prints every resolved value, where it came from and why, and renders nothing.
+`--threads N` / `EFFECTCRAFT_EXPORT_THREADS` sizes the rayon pool (simulates a smaller CPU).
+`RUST_LOG=effectcraft_export=info` logs the same table at the start of every export.
+
+## Options
+
+| option | values | auto |
+|---|---|---|
+| `backend` | auto, gpu, cpu | `auto` with a GPU (GPU workers + adaptive CPU workers); stock path without one; `cpu` if one frame does not fit the VRAM budget (warning logged) |
+| `gpu-frames` | N (0 = stock batches) | 1 GPU recording worker |
+| `gpu-inflight` | N ≥ 1 | 1 (below: room for up to 3 is reported, not used) |
+| `cpu-frames` | N, auto | adaptive: start GPU-only, add CPU workers while measured frames/s improves, max min(threads/2, 8); 0 with `backend gpu` |
+| `cpu-pool` | N (0 = global pool) | threads/2 when there are CPU workers |
+| `gpu-pool` | N (0 = global pool) | 0 |
+| `cache-mb` | MB (0 = off) | 1024: the job-wide layer cache at the stock 1 GiB; a larger cache is opt-in |
+| `precomp-resident` | on, off | on with a GPU: precomp layers' nested comps render on the GPU (2D precomp layers without masks/styles/proxies/overrides/frame blending; others take the CPU path) |
+| `gpu-yuv` | on, off | on with a native GPU (H.264 only; byte-identical to the CPU conversion) |
+| `async-readback` | on, off | on when `gpu-yuv` is on (off = wait for each frame) |
+| `cpu-batch` | N | one CPU-compositor frame per thread, capped so ~8x the frame buffer each fits 60% of the RAM budget after the layer cache |
+| `ram-budget-mb` | MB | detected RAM: unified-memory GPUs plan from it; the layer cache is capped at 15% of it |
+| `vram-budget-mb` | MB | dedicated VRAM measured by the CLI (`nvidia-smi`; amdgpu sysfs on Linux); unified memory: from RAM; else 2048 |
+| `upload-budget-mb` | MB | 25% of the VRAM budget, clamped to 256..1536 (a warning is logged once when it overflows) |
+| `pool-budget-mb` | MB | 5% of the VRAM budget, clamped to 128..768 |
+
+## Auto rules
+
+* **No GPU attached** (or `backend cpu`): the stock CPU path: parallel batches, the job layer cache,
+  software encoder, the project's renderer. The output is byte-identical to the stock export.
+* **VRAM plan**: the first frame in flight needs about 1,332 B per output pixel (≈ 10.3 GB at 4K,
+  2.6 GB at 1080p), calibrated from measured peaks without working-texture reuse (the GPU composites in
+  RGBA f32 whatever the project's bit depth); each further frame adds two RGBA f32 frames.
+  `free = 90% of budget − upload cache − texture pool − 384 MB`, and room for `clamp(frames that fit / gpu-frames, 1, 3)` frames in flight (`--print-tuning` shows it).
+  The default `gpu-inflight` is **1** all the same: with more, a frame that runs out of GPU memory is
+  retried, and the retry (plus a few frames after it) does not render exactly as it would have without
+  the failure, so the same export can differ between runs (Singularity at 1080p: frames 55-59 at ~30 dB).
+  Until that is fixed, set `gpu-inflight` above 1 only where a run-to-run difference is acceptable.
+  One frame in flight also measured no slower there (46.8 s vs 43-65 s with 3, which kept running out of memory).
+  Room at 4K: 24 GB → 3; 16 GB → 3; 8 GB and less → 0, so the backend becomes `cpu` up front and one warning says why
+  (GPU memory savings such as #688's texture reuse would let it render on the GPU). 1080p: 8 GB → 3, 4 GB and less → `cpu`.
+  A forced `backend gpu` that does not fit is kept but warned about.
+* **Out of memory while rendering**: a failed GPU allocation (reported to the device's error handler from
+  whichever thread allocated) invalidates only that frame: the device stays in use. The GPU worker waits for its frames in flight, keeps one fewer in flight from then on (at least 1, logged once)
+  and retries the frame on the GPU; only a frame that fails again is rendered on the CPU.
+* **CPU workers** stay adaptive by measured throughput. Before the first one is tried, a CPU frame is
+  estimated (frame rendered at quarter size, ×16). No worker is tried when a CPU frame would take more than
+  half the reorder buffer's worth of GPU frames (it would stall the in-order encoder) or add < 3%. Every frame the GPU declines is rendered on
+  the CPU and logged as a warning with the reason (device failure / out of memory, unsupported
+  content, frame larger than the texture limit), and the end summary counts these frames. So there
+  is no silent fallback.
+* **Encoder:** H.264 always uses the pure-Rust software encoder (no hardware encoder).
+
+## Unified memory (Apple silicon, integrated GPUs)
+
+* Detected from the adapter (integrated, or an Apple GPU). There is no separate VRAM: the GPU budget
+  (`vram-budget-mb`) is min(40% of RAM, 60% of the RAM available when the export starts); RAM and
+  availability come from `sysctl hw.memsize` + `vm_stat` (macOS) or `/proc/meminfo` (Linux), measured by
+  the CLI and passed in (`tuning::Host`); the export library itself spawns no processes and reads no system files.
+* The upload cache gets 256 MB plus what the frame slots leave of that budget (at most 1024 MB): an
+  upload is a copy within shared memory, so frame slots come first.
+* The CPU layer cache stays at the stock 1 GiB unless `cache-mb` asks for more.
+* Performance/efficiency cores (`hw.perflevel0/1`): adaptive CPU workers and their pool use at most
+  the efficiency cores; the performance cores stay with the GPU worker's CPU side and the encoder.
+  With `--threads N` the split is scaled to N.
+* If a frame does not fit the budget (little free memory, small `ram-budget-mb`), the export renders
+  on the CPU and logs a warning saying so.
+
+## Which GPUs get the tuned pipeline automatically
+
+`auto` applies the tuned GPU pipeline (scheduler, resident precomps,
+GPU YUV, budgets) only on GPUs it was verified on: **NVIDIA discrete** (RTX 3090, Windows) and
+**Apple silicon** (M4, macOS). On AMD, Intel, NVIDIA integrated and unknown adapters, `auto` keeps
+the stock path and `--print-tuning` / the export log say why. Users opt in with
+`--backend gpu` (the whole tuned pipeline) or with individual options (e.g. `--precomp-resident on`).
+
+## Memory limits on the CPU path
+
+With a RAM budget (detected, or `ram-budget-mb`), the CPU compositor renders at most `cpu-batch`
+frames at once, so a constrained machine (or a GPU-less fallback) does not run out of memory.
+The layer cache is capped at 15% of the budget.