diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6c49815..ed9eee7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,10 +3,12 @@ # # macOS: Wisp__aarch64.dmg (Apple Silicon only) # Windows: Wisp__x64-setup.exe (NSIS, smaller) + Wisp__x64_en-US.msi +# Linux: Wisp__amd64.deb + Wisp__amd64.AppImage # # Intel macOS is intentionally unsupported (Metal/screencapture need the macOS 15 SDK no Intel runner -# provides; WebRTC AEC won't cross-compile from Apple Silicon). The Windows build is mic + sherpa-onnx -# (system audio / AEC / Metal are macOS-only and degrade gracefully). +# provides; WebRTC AEC won't cross-compile from Apple Silicon). The Windows and Linux builds are the +# universal GPU installers (whisper.cpp on Vulkan, with ggml's CPU fallback when there's no Vulkan +# device; sherpa stays CPU). # # Trigger: push a `v*` tag (e.g. `git tag v0.2.0 && git push --tags`), or run manually with a tag. # @@ -57,6 +59,11 @@ jobs: - os: windows-latest label: Windows x64 args: "--features whisper-vulkan --config src-tauri/tauri.vulkan.conf.json" + # Same universal GPU build on Linux (Vulkan whisper.cpp + PulseAudio monitor system audio + + # WebRTC AEC). 22.04 keeps the .deb's glibc floor usable on the oldest supported Ubuntu LTS. + - os: ubuntu-22.04 + label: Linux x64 + args: "--features whisper-vulkan" runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -80,11 +87,26 @@ jobs: with: node-version: 20 - # The WebRTC AEC crate's bundled build compiles vendored WebRTC with meson + ninja (macOS only). + # The WebRTC AEC crate's bundled build compiles vendored WebRTC with meson + ninja. - name: Install meson + ninja if: runner.os == 'macOS' run: brew install meson ninja + # Linux system deps: Tauri's webkit/gtk stack, ALSA for cpal mic capture, libpulse for the + # monitor system-audio source, meson+ninja for the WebRTC AEC build, and the Vulkan headers + + # glslc compiler the whisper.cpp Vulkan backend needs (the runtime loader comes from the + # user's GPU driver, so nothing Vulkan-related is bundled). + - name: Install Linux system deps + if: runner.os == 'Linux' + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends \ + build-essential pkg-config curl wget file \ + libwebkit2gtk-4.1-dev libappindicator3-dev librsvg2-dev patchelf \ + libasound2-dev libpulse-dev meson ninja-build libclang-dev \ + libvulkan-dev glslc spirv-headers spirv-tools glslang-tools \ + libayatana-appindicator3-dev + # Windows ships the universal GPU installer (whisper.cpp + Vulkan). ggml-vulkan's vulkan-shaders-gen # sub-build nests deeply and overruns Windows' 260-char MAX_PATH from the default target dir, so # shorten it; the Ninja generator needs cl.exe on PATH; and the SDK provides the headers + diff --git a/.gitignore b/.gitignore index 02bc3ba..8658adb 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,9 @@ /target **/target +# Build-time staging dir for the sherpa/onnxruntime .so copies the Linux bundle ships +/app/src-tauri/linux-runtime/ + # Node / frontend node_modules/ dist/ diff --git a/Cargo.toml b/Cargo.toml index 1b44caa..a88af8e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -3,14 +3,16 @@ resolver = "2" members = ["crates/*"] # Kept out of the library CI build: the Tauri app, and the per-OS native crates (heavy vendored # libs built at compile time — sherpa/onnxruntime, whisper.cpp/Metal — or OS frameworks: -# ScreenCaptureKit, WebRTC AEC3, WASAPI loopback). The Windows one is covered by the app build on -# the Windows CI runner; the macOS ones by running the app on a Mac. +# ScreenCaptureKit, WebRTC AEC3, WASAPI loopback, PulseAudio monitor). The Windows one is covered +# by the app build on the Windows CI runner; the macOS ones by running the app on a Mac; the Linux +# one by the app build on the Linux release runner. exclude = [ "app/src-tauri", "crates/wisp-engine-sherpa", "crates/wisp-engine-whisper-cpp", "crates/wisp-screencapture", "crates/wisp-loopback", + "crates/wisp-pulseaudio", "crates/wisp-aec", "crates/wisp-embed", ] diff --git a/app/src-tauri/Cargo.lock b/app/src-tauri/Cargo.lock index 6ed6c5d..e7ef37b 100644 --- a/app/src-tauri/Cargo.lock +++ b/app/src-tauri/Cargo.lock @@ -116,6 +116,7 @@ dependencies = [ "wisp-loopback", "wisp-models", "wisp-pipeline", + "wisp-pulseaudio", "wisp-screencapture", "wisp-textinject", ] @@ -2650,6 +2651,33 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "libpulse-binding" +version = "2.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909eb3049e16e373680fe65afe6e2a722ace06b671250cc4849557bc57d6a397" +dependencies = [ + "bitflags 2.11.1", + "libc", + "libpulse-sys", + "num-derive", + "num-traits", + "winapi", +] + +[[package]] +name = "libpulse-sys" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d74371848b22e989f829cc1621d2ebd74960711557d8b45cfe740f60d0a05e61" +dependencies = [ + "libc", + "num-derive", + "num-traits", + "pkg-config", + "winapi", +] + [[package]] name = "libredox" version = "0.1.17" @@ -6823,6 +6851,14 @@ dependencies = [ "wisp-core", ] +[[package]] +name = "wisp-pulseaudio" +version = "0.0.0" +dependencies = [ + "libpulse-binding", + "wisp-core", +] + [[package]] name = "wisp-screencapture" version = "0.0.0" diff --git a/app/src-tauri/Cargo.toml b/app/src-tauri/Cargo.toml index 6c1b067..063ce9d 100644 --- a/app/src-tauri/Cargo.toml +++ b/app/src-tauri/Cargo.toml @@ -39,8 +39,9 @@ sherpa-cpu = ["wisp-engine-sherpa/download-binaries"] # weekly smoke workflow (cargo build --no-default-features --features sherpa-gpu) — a ~2h compile, # too slow to gate every PR. Mutually exclusive with sherpa-cpu (sherpa-rs-sys enforces this). sherpa-gpu = ["wisp-engine-sherpa/directml"] -# whisper.cpp with the Vulkan GPU backend on Windows (generic GPU — AMD/Intel/NVIDIA — with a CPU -# fallback). Off by default; the Windows GPU release turns it on. No effect on macOS (always Metal). +# whisper.cpp with the Vulkan GPU backend on Windows and Linux (generic GPU — AMD/Intel/NVIDIA — +# with a CPU fallback). Off by default; the Windows and Linux GPU releases turn it on. No effect on +# macOS (always Metal). whisper-vulkan = ["wisp-engine-whisper-cpp/vulkan", "wisp-models/whisper-vulkan"] [build-dependencies] @@ -100,3 +101,13 @@ wisp-engine-whisper-cpp = { path = "../../crates/wisp-engine-whisper-cpp" } # Total physical memory (GlobalMemoryStatusEx) to recommend a machine-appropriate model. Same # `windows` major as wisp-loopback, so no extra version in the tree. windows = { version = "0.61", features = ["Win32_System_SystemInformation"] } + +# Linux-only native deps. System-audio capture via the PulseAudio/PipeWire monitor — the counterpart +# to macOS ScreenCaptureKit and Windows WASAPI loopback, the same one-click "capture the meeting" +# experience — plus WebRTC AEC3 echo cancellation (also used on macOS). +[target.'cfg(target_os = "linux")'.dependencies] +wisp-pulseaudio = { path = "../../crates/wisp-pulseaudio" } +wisp-aec = { path = "../../crates/wisp-aec" } +# whisper.cpp ASR engine — built with the Vulkan GPU backend only under the `whisper-vulkan` feature +# (generic GPU + CPU fallback); an empty shell in the default CPU build. +wisp-engine-whisper-cpp = { path = "../../crates/wisp-engine-whisper-cpp" } diff --git a/app/src-tauri/build.rs b/app/src-tauri/build.rs index cb0b0e0..55b20d4 100644 --- a/app/src-tauri/build.rs +++ b/app/src-tauri/build.rs @@ -24,9 +24,89 @@ fn main() { #[cfg(target_os = "windows")] stage_windows_runtime_libs(); + #[cfg(target_os = "linux")] + { + // The sherpa-onnx + onnxruntime shared libs the binary links against land in the cargo + // target dir at build time. `tauri dev` finds them via `$ORIGIN`; the .deb/AppImage ship + // them under /lib/Wisp (the Tauri resource dir). Emit RPATH (not the modern + // RUNPATH, which does not apply transitively) so libsherpa-onnx-c-api.so also resolves + // its own deps — libonnxruntime.so.1 and libcargs.so — from the same directory. + println!("cargo:rustc-link-arg=-Wl,--disable-new-dtags"); + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN"); + println!("cargo:rustc-link-arg=-Wl,-rpath,$ORIGIN/../lib/Wisp"); + stage_linux_runtime_libs(); + } + tauri_build::build() } +/// Copies the sherpa-onnx + onnxruntime runtime libs from the cargo target dir into +/// `linux-runtime/` (next to this build script) so `tauri.linux.conf.json` can bundle them into +/// the installer's resource dir. Every `lib*.so*` the binary needs is staged flat, so the +/// `$ORIGIN/../lib/Wisp` rpath finds each DT_NEEDED name — including the versioned sonames +/// (`libonnxruntime.so.1`) the prebuilt libs require. +#[cfg(target_os = "linux")] +fn stage_linux_runtime_libs() { + use std::path::Path; + + let target_dir = cargo_target_dir_linux(); + let staged = Path::new(env!("CARGO_MANIFEST_DIR")).join("linux-runtime"); + std::fs::create_dir_all(&staged).expect("create linux-runtime dir"); + + let mut staged_any = false; + for entry in std::fs::read_dir(&target_dir) + .unwrap_or_else(|e| panic!("read {}: {e}", target_dir.display())) + { + let entry = entry.expect("read target dir entry"); + let name = entry.file_name().to_string_lossy().into_owned(); + let is_runtime_lib = (name.starts_with("libsherpa-onnx") + || name.starts_with("libcargs") + || name.starts_with("libonnxruntime")) + && name.contains(".so"); + if !is_runtime_lib { + continue; + } + let src = entry.path(); + if !src.is_file() { + continue; // skip soname symlinks — the real lib is staged under its own name + } + let dst = staged.join(&name); + let stale = dst + .metadata() + .map(|m| m.len() != src.metadata().map(|s| s.len()).unwrap_or(0)) + .unwrap_or(true); + if stale { + std::fs::copy(&src, &dst).unwrap_or_else(|e| panic!("stage {}: {e}", src.display())); + } + staged_any = true; + } + + assert!( + staged_any, + "no sherpa/onnxruntime shared libs found in {} — build wisp-engine-sherpa first", + target_dir.display() + ); +} + +/// The cargo `target/` dir, found by walking up from `OUT_DIR` — the same place +/// sherpa-rs-sys copies its shared libs to. Same logic as the Windows `cargo_target_dir` below. +#[cfg(target_os = "linux")] +fn cargo_target_dir_linux() -> std::path::PathBuf { + let out_dir = std::path::PathBuf::from(std::env::var("OUT_DIR").unwrap()); + let profile = std::env::var("PROFILE").unwrap(); + let mut dir = out_dir.as_path(); + while let Some(parent) = dir.parent() { + if parent.ends_with(&profile) { + return parent.to_path_buf(); + } + dir = parent; + } + panic!( + "could not find target/{profile} above OUT_DIR {}", + out_dir.display() + ); +} + /// Copies the sherpa-onnx + onnxruntime runtime DLLs from the cargo target dir into /// `windows-runtime/` (next to this build script) so they can be bundled as Tauri resources. A /// size check skips the copy when already current, so the build script doesn't perpetually re-fire. diff --git a/app/src-tauri/src/lib.rs b/app/src-tauri/src/lib.rs index 17440f1..0cf08d9 100644 --- a/app/src-tauri/src/lib.rs +++ b/app/src-tauri/src/lib.rs @@ -14,7 +14,7 @@ use std::time::{Duration, Instant}; use serde::{Deserialize, Serialize}; use tauri::{path::BaseDirectory, AppHandle, Emitter, Manager, State}; -#[cfg(target_os = "macos")] +#[cfg(any(target_os = "macos", target_os = "linux"))] use wisp_aec::WebrtcEchoCanceller; use wisp_audio::{ normalize_for_asr_in_place, tee, to_mono_16k, ChannelSource, EchoCancellingSource, MediaSource, @@ -57,6 +57,8 @@ use wisp_pipeline::{ remap_to_original, transcribe_in_windows, EnergySegmenter, EnergyVad, GatedClip, LiveStream, Segmenter, Session, Transcriber, Vad, DEFAULT_SILENCE_HANGOVER, }; +#[cfg(target_os = "linux")] +use wisp_pulseaudio::PulseMonitorSource; #[cfg(target_os = "macos")] use wisp_screencapture::ScreenCaptureSource; @@ -611,11 +613,15 @@ fn coerce_param(raw: &serde_json::Value, kind: &ParamKind) -> Option } } -/// Builds the GPU whisper.cpp engine from a downloaded GGUF model — Metal on macOS, Vulkan on Windows -/// (under the `whisper-vulkan` feature). Where the engine isn't built, the stub below reports it. +/// Builds the GPU whisper.cpp engine from a downloaded GGUF model — Metal on macOS, Vulkan on +/// Windows and Linux (under the `whisper-vulkan` feature). Where the engine isn't built, the stub +/// below reports it. #[cfg(any( target_os = "macos", - all(target_os = "windows", feature = "whisper-vulkan") + all( + any(target_os = "windows", target_os = "linux"), + feature = "whisper-vulkan" + ) ))] fn build_whisper_cpp_engine( descriptor: &ModelDescriptor, @@ -634,7 +640,10 @@ fn build_whisper_cpp_engine( #[cfg(not(any( target_os = "macos", - all(target_os = "windows", feature = "whisper-vulkan") + all( + any(target_os = "windows", target_os = "linux"), + feature = "whisper-vulkan" + ) )))] fn build_whisper_cpp_engine( _descriptor: &ModelDescriptor, @@ -642,7 +651,7 @@ fn build_whisper_cpp_engine( _language: &str, ) -> WispResult> { Err(WispError::Engine( - "the whisper.cpp GPU engine is only available on macOS, and on Windows GPU builds" + "the whisper.cpp GPU engine is only available on macOS, and on Windows/Linux GPU builds" .to_owned(), )) } @@ -3178,7 +3187,16 @@ fn open_system_capture() -> Result, String> { .map_err(|e| e.to_string()) } -#[cfg(not(any(target_os = "macos", target_os = "windows")))] +/// Linux captures the default output's monitor through the PulseAudio/PipeWire server — one +/// click, no virtual device, like ScreenCaptureKit and WASAPI loopback. +#[cfg(target_os = "linux")] +fn open_system_capture() -> Result, String> { + PulseMonitorSource::new() + .map(|s| Box::new(s) as Box) + .map_err(|e| e.to_string()) +} + +#[cfg(not(any(target_os = "macos", target_os = "windows", target_os = "linux")))] fn open_system_capture() -> Result, String> { Err("system-audio capture isn't available on this platform yet".to_owned()) } @@ -3210,11 +3228,11 @@ fn open_mic_within(device: Option) -> Result, Strin }) } -/// The echo canceller for this platform: WebRTC AEC on macOS (falling back to passthrough if it -/// won't init), passthrough elsewhere. Keeping it a `Box` lets the dual-stream -/// path stay identical on every platform — the cross-stream dedup handles residual echo where there -/// is no real AEC. -#[cfg(target_os = "macos")] +/// The echo canceller for this platform: WebRTC AEC on macOS and Linux (falling back to +/// passthrough if it won't init), passthrough elsewhere. Keeping it a `Box` +/// lets the dual-stream path stay identical on every platform — the cross-stream dedup handles +/// residual echo where there is no real AEC. +#[cfg(any(target_os = "macos", target_os = "linux"))] fn echo_canceller() -> Box { match WebrtcEchoCanceller::new() { Ok(c) => Box::new(c), @@ -3225,7 +3243,7 @@ fn echo_canceller() -> Box { } } -#[cfg(not(target_os = "macos"))] +#[cfg(not(any(target_os = "macos", target_os = "linux")))] fn echo_canceller() -> Box { Box::new(PassthroughEchoCanceller) } diff --git a/app/src-tauri/tauri.conf.json b/app/src-tauri/tauri.conf.json index bf8ecf8..78d553a 100644 --- a/app/src-tauri/tauri.conf.json +++ b/app/src-tauri/tauri.conf.json @@ -26,7 +26,7 @@ }, "bundle": { "active": true, - "targets": ["nsis", "dmg"], + "targets": ["nsis", "dmg", "deb", "appimage"], "windows": { "minimumWebview2Version": "120.0.2210.61" }, diff --git a/app/src-tauri/tauri.linux.conf.json b/app/src-tauri/tauri.linux.conf.json new file mode 100644 index 0000000..40df7f5 --- /dev/null +++ b/app/src-tauri/tauri.linux.conf.json @@ -0,0 +1,9 @@ +{ + "$schema": "https://schema.tauri.app/config/2", + "bundle": { + "resources": { + "resources/silero_vad.onnx": "resources/silero_vad.onnx", + "linux-runtime/*.so*": "" + } + } +} diff --git a/crates/wisp-engine-whisper-cpp/Cargo.toml b/crates/wisp-engine-whisper-cpp/Cargo.toml index 1926d6a..cf25180 100644 --- a/crates/wisp-engine-whisper-cpp/Cargo.toml +++ b/crates/wisp-engine-whisper-cpp/Cargo.toml @@ -11,8 +11,8 @@ wisp-core = { path = "../wisp-core" } num_cpus = "1" [features] -# Build whisper.cpp with the Vulkan backend on Windows (generic GPU — AMD/Intel/NVIDIA — with a CPU -# fallback). Off by default so the standard Windows build stays a no-op shell and needs no Vulkan SDK. +# Build whisper.cpp with the Vulkan backend on Windows/Linux (generic GPU — AMD/Intel/NVIDIA — with +# a CPU fallback). Off by default so the standard build stays a no-op shell and needs no Vulkan SDK. # No effect on macOS, which always builds the Metal + Core ML backend. vulkan = [] diff --git a/crates/wisp-engine-whisper-cpp/build.rs b/crates/wisp-engine-whisper-cpp/build.rs index 1f3c38b..7dcf6ad 100644 --- a/crates/wisp-engine-whisper-cpp/build.rs +++ b/crates/wisp-engine-whisper-cpp/build.rs @@ -1,9 +1,9 @@ //! Builds the vendored whisper.cpp and generates Rust bindings for its C API. //! //! - **macOS**: Metal + Core ML backend (always). -//! - **Windows**: Vulkan backend — generic GPU across AMD/Intel/NVIDIA, with ggml's built-in CPU -//! fallback — but only when the `vulkan` feature is on, so the default Windows build stays a no-op -//! shell and needs no Vulkan SDK. +//! - **Windows / Linux**: Vulkan backend — generic GPU across AMD/Intel/NVIDIA, with ggml's +//! built-in CPU fallback — but only when the `vulkan` feature is on, so the default build stays +//! a no-op shell and needs no Vulkan SDK. //! - **Other targets**: a no-op, so the crate is an empty shell. use std::env; @@ -11,9 +11,10 @@ use std::path::{Path, PathBuf}; fn main() { let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap_or_default(); - let windows_vulkan = target_os == "windows" && env::var("CARGO_FEATURE_VULKAN").is_ok(); + let unix_vulkan = matches!(target_os.as_str(), "windows" | "linux") + && env::var("CARGO_FEATURE_VULKAN").is_ok(); - if target_os != "macos" && !windows_vulkan { + if target_os != "macos" && !unix_vulkan { return; } @@ -26,8 +27,10 @@ fn main() { if target_os == "macos" { build_macos(&src); - } else { + } else if target_os == "windows" { build_windows_vulkan(&src); + } else { + build_linux_vulkan(&src); } generate_bindings(&src); @@ -160,6 +163,43 @@ fn build_windows_vulkan(src: &Path) { println!("cargo:rustc-link-lib=vulkan-1"); } +/// Builds whisper.cpp + ggml as static libs with the Vulkan backend on Linux (generic GPU — +/// AMD/Intel/NVIDIA — with ggml's built-in CPU fallback). Requires `libvulkan-dev` (headers + +/// loader) and `glslc` (shaderc, compiles ggml's compute shaders) at build time; the GPU driver +/// ships `libvulkan.so.1` at runtime, so nothing extra is bundled. +fn build_linux_vulkan(src: &Path) { + let dst = cmake::Config::new(src) + .profile("Release") + .define("BUILD_SHARED_LIBS", "OFF") + .define("WHISPER_BUILD_EXAMPLES", "OFF") + .define("WHISPER_BUILD_TESTS", "OFF") + .define("WHISPER_BUILD_SERVER", "OFF") + .define("GGML_VULKAN", "ON") + .define("GGML_OPENMP", "OFF") + .build(); + + // The static libs land in the install prefix and/or the build tree (single-config generator — + // no per-config subdirs like MSVC's Release). + let build = dst.join("build"); + for dir in [ + dst.join("lib"), + build.join("src"), + build.join("ggml/src"), + build.join("ggml/src/ggml-vulkan"), + ] { + println!("cargo:rustc-link-search=native={}", dir.display()); + } + + for lib in ["whisper", "ggml", "ggml-cpu", "ggml-vulkan", "ggml-base"] { + println!("cargo:rustc-link-lib=static={lib}"); + } + + // The system Vulkan loader (`libvulkan.so` — libvulkan-dev at build time, GPU driver at + // runtime) and the C++ standard library the vendored C++ code needs. + println!("cargo:rustc-link-lib=vulkan"); + println!("cargo:rustc-link-lib=stdc++"); +} + /// Generates the Rust FFI bindings for whisper.cpp's C API. Platform-independent — it only parses the /// public headers, so the same bindings serve every backend. fn generate_bindings(src: &Path) { diff --git a/crates/wisp-engine-whisper-cpp/src/lib.rs b/crates/wisp-engine-whisper-cpp/src/lib.rs index 9e8f00e..b27a25b 100644 --- a/crates/wisp-engine-whisper-cpp/src/lib.rs +++ b/crates/wisp-engine-whisper-cpp/src/lib.rs @@ -1,12 +1,15 @@ -//! whisper.cpp ASR engine with GPU acceleration — Metal on macOS, Vulkan on Windows. +//! whisper.cpp ASR engine with GPU acceleration — Metal on macOS, Vulkan on Windows/Linux. //! //! Wraps the vendored whisper.cpp behind [`wisp_core::AsrEngine`] so it drops into the pipeline //! like the sherpa engines — but runs on the GPU (Apple Metal, or Vulkan across AMD/Intel/NVIDIA) //! instead of CPU-only ONNX, which makes large models (e.g. large-v3-turbo) usable in real time. -// macOS always builds the Metal + Core ML backend; Windows builds the Vulkan backend only under the -// `vulkan` feature (see build.rs). Elsewhere this crate is an empty shell. -#![cfg(any(target_os = "macos", all(target_os = "windows", feature = "vulkan")))] +// macOS always builds the Metal + Core ML backend; Windows and Linux build the Vulkan backend only +// under the `vulkan` feature (see build.rs). Elsewhere this crate is an empty shell. +#![cfg(any( + target_os = "macos", + all(any(target_os = "windows", target_os = "linux"), feature = "vulkan") +))] mod sys { #![allow( diff --git a/crates/wisp-models/src/machine.rs b/crates/wisp-models/src/machine.rs index a41bbd7..572d557 100644 --- a/crates/wisp-models/src/machine.rs +++ b/crates/wisp-models/src/machine.rs @@ -169,12 +169,13 @@ fn resolve(ideal: &str, catalog: &[ModelDescriptor]) -> ModelId { /// scattered `cfg` checks. pub fn family_runnable(family: ModelFamily, accelerator: Accelerator) -> bool { match family { - // whisper.cpp runs on the Apple GPU (Metal), or — in a Windows GPU build — on Vulkan with a - // built-in CPU fallback, so it's offered on Windows too (no GPU required to run, just to - // accelerate). Apple on-device speech stays macOS/Metal-only. + // whisper.cpp runs on the Apple GPU (Metal), or — in a Windows/Linux GPU build — on Vulkan + // with a built-in CPU fallback, so it's offered on those platforms too (no GPU required to + // run, just to accelerate). Apple on-device speech stays macOS/Metal-only. ModelFamily::WhisperCpp => { accelerator == Accelerator::Metal - || (cfg!(target_os = "windows") && cfg!(feature = "whisper-vulkan")) + || (cfg!(any(target_os = "windows", target_os = "linux")) + && cfg!(feature = "whisper-vulkan")) } ModelFamily::AppleSpeech => accelerator == Accelerator::Metal, _ => true, @@ -220,7 +221,9 @@ pub fn model_fit(descriptor: &ModelDescriptor, profile: &MachineProfile) -> Mode /// Why a family can't run on a host lacking its accelerator — the text shown on the greyed entry. fn platform_block_reason(family: ModelFamily) -> String { match family { - ModelFamily::WhisperCpp => "Needs a macOS Metal GPU".to_owned(), + ModelFamily::WhisperCpp => { + "Needs a macOS Metal GPU, or a Windows/Linux Vulkan GPU build".to_owned() + } ModelFamily::AppleSpeech => "Needs macOS".to_owned(), _ => "Not supported on this machine".to_owned(), } @@ -292,9 +295,30 @@ mod tests { } #[test] - fn whisper_cpp_runs_only_on_metal() { - // The GPU whisper.cpp engine is macOS/Metal only — every other accelerator must hide it. + fn whisper_cpp_runs_only_on_metal_or_vulkan_builds() { + // whisper.cpp's GPU engine is Metal on macOS; Windows/Linux get it under the + // `whisper-vulkan` feature, where ggml Vulkan accelerates on any GPU and falls back to + // CPU — there it's runnable regardless of the reported accelerator. assert!(family_runnable(ModelFamily::WhisperCpp, Accelerator::Metal)); + #[cfg(all( + any(target_os = "windows", target_os = "linux"), + feature = "whisper-vulkan" + ))] + for accel in [ + Accelerator::Cpu, + Accelerator::Cuda, + Accelerator::Vulkan, + Accelerator::DirectMl, + ] { + assert!( + family_runnable(ModelFamily::WhisperCpp, accel), + "{accel:?} on a Vulkan build should still run whisper.cpp (CPU fallback)" + ); + } + #[cfg(not(all( + any(target_os = "windows", target_os = "linux"), + feature = "whisper-vulkan" + )))] for accel in [ Accelerator::Cpu, Accelerator::Cuda, @@ -303,7 +327,7 @@ mod tests { ] { assert!( !family_runnable(ModelFamily::WhisperCpp, accel), - "{accel:?} has no Metal whisper.cpp engine" + "{accel:?} has no whisper.cpp GPU engine" ); } } @@ -333,16 +357,25 @@ mod tests { fn model_fit_blocks_metal_only_families_off_metal() { let catalog = builtin_catalog(); let cpu = MachineProfile::new(Accelerator::Cpu, 16 * GIB); - for d in catalog - .iter() - .filter(|d| d.family == ModelFamily::WhisperCpp || d.family == ModelFamily::AppleSpeech) - { + let mut checked_any = false; + for d in catalog.iter().filter(|d| { + let gpu_engine_here = d.family == ModelFamily::WhisperCpp + && cfg!(all( + any(target_os = "windows", target_os = "linux"), + feature = "whisper-vulkan" + )); + // WhisperCpp is only Metal-blocked when this build has no Vulkan whisper.cpp. + (d.family == ModelFamily::WhisperCpp && !gpu_engine_here) + || d.family == ModelFamily::AppleSpeech + }) { + checked_any = true; assert!( matches!(model_fit(d, &cpu), ModelFit::Blocked(_)), "{:?} should be blocked off Metal", d.id ); } + assert!(checked_any, "catalog must contain a Metal-only family"); } #[test] diff --git a/crates/wisp-pulseaudio/Cargo.lock b/crates/wisp-pulseaudio/Cargo.lock new file mode 100644 index 0000000..93bfc05 --- /dev/null +++ b/crates/wisp-pulseaudio/Cargo.lock @@ -0,0 +1,177 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + +[[package]] +name = "bitflags" +version = "2.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libpulse-binding" +version = "2.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "909eb3049e16e373680fe65afe6e2a722ace06b671250cc4849557bc57d6a397" +dependencies = [ + "bitflags", + "libc", + "libpulse-sys", + "num-derive", + "num-traits", + "winapi", +] + +[[package]] +name = "libpulse-sys" +version = "1.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d74371848b22e989f829cc1621d2ebd74960711557d8b45cfe740f60d0a05e61" +dependencies = [ + "libc", + "num-derive", + "num-traits", + "pkg-config", + "winapi", +] + +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + +[[package]] +name = "pkg-config" +version = "0.3.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "3.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "thiserror" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "2.0.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" +dependencies = [ + "proc-macro2", + "quote", + "syn 3.0.6", +] + +[[package]] +name = "unicode-ident" +version = "1.0.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d245f478577f809a851594d02313b640fb437e0bb33866753cff937863096954" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "wisp-core" +version = "0.0.0" +dependencies = [ + "thiserror", +] + +[[package]] +name = "wisp-pulseaudio" +version = "0.0.0" +dependencies = [ + "libpulse-binding", + "wisp-core", +] diff --git a/crates/wisp-pulseaudio/Cargo.toml b/crates/wisp-pulseaudio/Cargo.toml new file mode 100644 index 0000000..0407b08 --- /dev/null +++ b/crates/wisp-pulseaudio/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "wisp-pulseaudio" +version = "0.0.0" +description = "Linux system-audio capture for Wisp via the PulseAudio/PipeWire monitor source (no virtual device)" +edition = "2021" +license = "MIT" + +[dependencies] +wisp-core = { path = "../wisp-core" } + +[target.'cfg(target_os = "linux")'.dependencies] +# Async PulseAudio client API — reaches the sound server (PulseAudio itself, or PipeWire's +# pipewire-pulse compat) to record the default sink's `.monitor` source. +libpulse-binding = "2.30" diff --git a/crates/wisp-pulseaudio/src/lib.rs b/crates/wisp-pulseaudio/src/lib.rs new file mode 100644 index 0000000..91c6b05 --- /dev/null +++ b/crates/wisp-pulseaudio/src/lib.rs @@ -0,0 +1,347 @@ +//! Linux system-audio capture via the PulseAudio monitor source. +//! +//! Records whatever is playing on the default output — the sink's `.monitor` source — and exposes +//! it as a [`wisp_core::AudioSource`]: the Linux counterpart of the macOS ScreenCaptureKit source +//! and the Windows WASAPI loopback. One click, no virtual device, no setup — on PulseAudio and on +//! PipeWire's `pipewire-pulse` server (the default on modern desktops). +//! +//! The PulseAudio objects live entirely on a dedicated capture thread that feeds frames to a +//! channel; the public handle holds only the receiver + a stop flag and stays `Send` (the same +//! split the Windows WASAPI source uses for its non-`Send` COM objects). + +#![cfg(target_os = "linux")] + +use std::cell::RefCell; +use std::rc::Rc; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::mpsc::{self, Sender}; +use std::sync::{Arc, Mutex}; +use std::thread::{self, JoinHandle}; +use std::time::{Duration, Instant}; + +use libpulse_binding::context::{Context, FlagSet as ContextFlags, State as ContextState}; +use libpulse_binding::def::BufferAttr; +use libpulse_binding::mainloop::threaded::Mainloop; +use libpulse_binding::operation::State as OperationState; +use libpulse_binding::sample::{Format, Spec}; +use libpulse_binding::stream::{FlagSet as StreamFlags, PeekResult, State as StreamState, Stream}; + +use wisp_core::audio::{AudioFrame, AudioSource, AudioSourceInfo}; +use wisp_core::channel::{frame_channel, FrameReceiver, FrameSender}; +use wisp_core::error::{Result, WispError}; +use wisp_core::transcript::AudioSourceKind; + +/// Bounded capacity of the capture→pipeline frame channel. Drop-oldest on overflow keeps capture +/// real-time if the consumer briefly stalls (same policy as the mic and loopback sources). +const FRAME_CHANNEL_CAPACITY: usize = 1024; + +/// The stream's sample spec — the pipeline's native input (16 kHz mono f32), so the server does +/// any resample/downmix once, server-side, and capture hands the engines their exact format. +const SAMPLE_SPEC: Spec = Spec { + format: Format::F32le, + rate: 16_000, + channels: 1, +}; +const BYTES_PER_SAMPLE: usize = 4; +const CHANNELS: u16 = SAMPLE_SPEC.channels as u16; + +/// How often the capture thread drains the record buffer. Well under the server fragment size, so +/// monitor audio never overflows while keeping latency low. +const POLL: Duration = Duration::from_millis(10); + +/// How long `new` waits for the context + stream to come up before reporting unavailable — covers +/// a slow `pipewire-pulse` autospawn without hanging Start forever. +const CONNECT_TIMEOUT: Duration = Duration::from_secs(5); + +/// An [`AudioSource`] capturing the default output's monitor through PulseAudio/PipeWire. +pub struct PulseMonitorSource { + rx: FrameReceiver, + stop: Arc, + handle: Option>, +} + +impl PulseMonitorSource { + /// Starts capturing the default sink's monitor. Errors if the sound server can't be reached or + /// has no output device, so the caller can degrade to mic-only. + pub fn new() -> Result { + let (tx, rx) = frame_channel(FRAME_CHANNEL_CAPACITY); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let stop = Arc::new(AtomicBool::new(false)); + let stop_for_thread = Arc::clone(&stop); + + let handle = thread::spawn(move || capture_thread(&tx, &stop_for_thread, &ready_tx)); + + match ready_rx.recv() { + Ok(Ok(())) => Ok(Self { + rx, + stop, + handle: Some(handle), + }), + // The thread already returned after signalling failure — reap it so the error path + // doesn't leave a zombie. + Ok(Err(e)) => { + let _ = handle.join(); + Err(WispError::Audio(e)) + } + Err(_) => Err(WispError::Audio( + "PulseAudio capture thread exited before signalling readiness".to_owned(), + )), + } + } +} + +impl AudioSource for PulseMonitorSource { + fn info(&self) -> AudioSourceInfo { + AudioSourceInfo { + kind: AudioSourceKind::System, + name: "System audio".to_owned(), + } + } + + fn next_frame(&mut self) -> Result> { + Ok(self.rx.recv()) + } +} + +impl Drop for PulseMonitorSource { + fn drop(&mut self) { + self.stop.store(true, Ordering::Relaxed); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// Runs the capture, signalling readiness to `new` once the stream is up, then closes the channel +/// so a blocked receiver ends cleanly when capture stops or fails. +fn capture_thread( + tx: &FrameSender, + stop: &AtomicBool, + ready: &Sender>, +) { + if let Err(e) = run_capture(tx, stop, ready) { + // Only reached for a failure *before* readiness was signalled; tell `new` to degrade. + let _ = ready.send(Err(e)); + } + tx.close(); +} + +/// Releases the PulseAudio objects in order: stream, context, mainloop. The context is +/// disconnected under the mainloop lock and the loop stopped unlocked, per libpulse's contract. +struct ServerGuard { + mainloop: Rc>, + context: Rc>, + stream: Option>>, +} + +impl Drop for ServerGuard { + fn drop(&mut self) { + self.mainloop.borrow_mut().lock(); + if let Some(stream) = &self.stream { + let _ = stream.borrow_mut().disconnect(); + } + self.context.borrow_mut().disconnect(); + self.mainloop.borrow_mut().unlock(); + self.mainloop.borrow_mut().stop(); + } +} + +/// Builds the PulseAudio connection and record stream on this thread, then drains the monitor +/// into `tx` until `stop`. Returns `Err` only for a setup failure before `ready` is signalled. +fn run_capture( + tx: &FrameSender, + stop: &AtomicBool, + ready: &Sender>, +) -> std::result::Result<(), String> { + let mainloop = Rc::new(RefCell::new( + Mainloop::new().ok_or("create PulseAudio mainloop")?, + )); + let context = Rc::new(RefCell::new( + Context::new(&*mainloop.borrow(), "Wisp").ok_or("create PulseAudio context")?, + )); + context + .borrow_mut() + .connect(None, ContextFlags::NOFLAGS, None) + .map_err(|e| format!("connect to sound server: {e}"))?; + mainloop + .borrow_mut() + .start() + .map_err(|e| format!("start PulseAudio mainloop: {e}"))?; + + let mut guard = ServerGuard { + mainloop: Rc::clone(&mainloop), + context: Rc::clone(&context), + stream: None, + }; + + wait_for("context", CONNECT_TIMEOUT, || { + mainloop.borrow_mut().lock(); + let state = context.borrow().get_state(); + mainloop.borrow_mut().unlock(); + match state { + ContextState::Ready => PollState::Ready, + ContextState::Failed | ContextState::Terminated => PollState::Failed, + _ => PollState::Pending, + } + })?; + + let monitor = default_sink_monitor(&mainloop, &context)?; + + // pa_stream_new touches the context's object list — lock while the mainloop thread is live. + mainloop.borrow_mut().lock(); + let stream = Stream::new( + &mut context.borrow_mut(), + "meeting audio", + &SAMPLE_SPEC, + None, + ); + mainloop.borrow_mut().unlock(); + let stream = Rc::new(RefCell::new(stream.ok_or("create record stream")?)); + guard.stream = Some(Rc::clone(&stream)); + + let attr = buffer_attr(); + mainloop.borrow_mut().lock(); + let connected = stream.borrow_mut().connect_record( + Some(&monitor), + Some(&attr), + StreamFlags::ADJUST_LATENCY, + ); + mainloop.borrow_mut().unlock(); + connected.map_err(|e| format!("record the monitor source '{monitor}': {e}"))?; + + wait_for("record stream", CONNECT_TIMEOUT, || { + mainloop.borrow_mut().lock(); + let state = stream.borrow().get_state(); + mainloop.borrow_mut().unlock(); + match state { + StreamState::Ready => PollState::Ready, + StreamState::Failed | StreamState::Terminated => PollState::Failed, + _ => PollState::Pending, + } + })?; + + let _ = ready.send(Ok(())); + let start = Instant::now(); + + while !stop.load(Ordering::Relaxed) { + let mut produced: Vec = Vec::new(); + + mainloop.borrow_mut().lock(); + let mut stream_mut = stream.borrow_mut(); + loop { + match stream_mut.readable_size() { + Some(0) | None => break, + Some(_) => match stream_mut.peek() { + Ok(PeekResult::Data(bytes)) => { + produced.extend( + bytes + .as_chunks::() + .0 + .iter() + .map(|c| f32::from_le_bytes(*c)), + ); + let _ = stream_mut.discard(); + } + // A hole is a reported gap in the stream — skip it; the pipeline handles gaps. + Ok(PeekResult::Hole(_)) => { + let _ = stream_mut.discard(); + } + Ok(PeekResult::Empty) | Err(_) => break, + }, + } + } + drop(stream_mut); + mainloop.borrow_mut().unlock(); + + if !produced.is_empty() { + tx.send(AudioFrame::new( + produced, + SAMPLE_SPEC.rate, + CHANNELS, + start.elapsed(), + )); + } + + thread::sleep(POLL); + } + + Ok(()) +} + +/// Outcome of one poll of an object's state. +enum PollState { + Ready, + Pending, + Failed, +} + +/// Polls `state` until it reports ready or failed, or `timeout` elapses. Polling (rather than a +/// state callback + `mainloop.wait`) keeps the wait logic identical for the context, the stream, +/// and the introspection operation, and can't deadlock on a missed signal. +fn wait_for( + what: &str, + timeout: Duration, + mut state: impl FnMut() -> PollState, +) -> std::result::Result<(), String> { + let deadline = Instant::now() + timeout; + loop { + match state() { + PollState::Ready => return Ok(()), + PollState::Failed => return Err(format!("{what} failed to start")), + PollState::Pending if Instant::now() >= deadline => { + return Err(format!("{what} took over {timeout:?} to start")) + } + PollState::Pending => thread::sleep(POLL), + } + } +} + +/// The record stream's buffering: ~40 ms fragments inside a ~1 s ceiling — small enough to keep +/// transcription latency invisible, large enough that a momentary engine stall never overflows. +fn buffer_attr() -> BufferAttr { + let bytes_per_sec = + SAMPLE_SPEC.rate as usize * SAMPLE_SPEC.channels as usize * BYTES_PER_SAMPLE; + BufferAttr { + maxlength: bytes_per_sec as u32, // ~1 s + tlength: u32::MAX, // playback-only field — let the server pick + prebuf: u32::MAX, // playback-only field + minreq: u32::MAX, // playback-only field + fragsize: (bytes_per_sec / 25) as u32, // 40 ms fragments + } +} + +/// The name of the source monitoring the default output — `.monitor` — resolved via +/// the server-info introspection op. Errors when the server has no output device to monitor. +fn default_sink_monitor( + mainloop: &Rc>, + context: &Rc>, +) -> std::result::Result { + let sink_name: Arc>> = Arc::new(Mutex::new(None)); + + mainloop.borrow_mut().lock(); + let op = context.borrow().introspect().get_server_info({ + let sink_name = Arc::clone(&sink_name); + move |info| { + if let Some(name) = info.default_sink_name.as_deref() { + *sink_name.lock().expect("sink name poisoned") = Some(name.to_owned()); + } + } + }); + mainloop.borrow_mut().unlock(); + + wait_for("server-info query", CONNECT_TIMEOUT, || { + mainloop.borrow_mut().lock(); + let state = op.get_state(); + mainloop.borrow_mut().unlock(); + match state { + OperationState::Done => PollState::Ready, + OperationState::Cancelled => PollState::Failed, + _ => PollState::Pending, + } + })?; + + let sink_name = sink_name.lock().expect("sink name poisoned").clone(); + sink_name + .map(|name| format!("{name}.monitor")) + .ok_or_else(|| "sound server reports no default output device".to_owned()) +}