diff --git a/build/app.zig b/build/app.zig index 09c7010d1..85f0429cc 100644 --- a/build/app.zig +++ b/build/app.zig @@ -934,6 +934,7 @@ fn linkPlatform(b: *std.Build, dep: *std.Build.Dependency, target: std.Build.Res const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = dep.path("src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = dep.path("src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -963,6 +964,9 @@ fn linkPlatform(b: *std.Build, dep: *std.Build.Dependency, target: std.Build.Res app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the video player's // AVPlayerItemVideoOutput frames). CoreMedia's CMTime use stays // header-only, but the pixel-buffer calls are real symbols. diff --git a/changelog.d/macos-audio-capture.md b/changelog.d/macos-audio-capture.md new file mode 100644 index 000000000..e1d027ff1 --- /dev/null +++ b/changelog.d/macos-audio-capture.md @@ -0,0 +1 @@ +feature: **macOS audio capture**: add macOS 15+ system-audio and selectable-microphone PCM WAV capture, microphone enumeration, explicit permission flows, and Zig/TypeScript effects APIs. diff --git a/docs/src/app/docs/audio-capture/layout.tsx b/docs/src/app/docs/audio-capture/layout.tsx new file mode 100644 index 000000000..b6e5f5035 --- /dev/null +++ b/docs/src/app/docs/audio-capture/layout.tsx @@ -0,0 +1,7 @@ +import { pageMetadata } from "@/lib/page-metadata"; + +export const metadata = pageMetadata("audio-capture"); + +export default function Layout({ children }: { children: React.ReactNode }) { + return children; +} diff --git a/docs/src/app/docs/audio-capture/page.mdx b/docs/src/app/docs/audio-capture/page.mdx new file mode 100644 index 000000000..04ee6c81a --- /dev/null +++ b/docs/src/app/docs/audio-capture/page.mdx @@ -0,0 +1,176 @@ +# Audio Capture + +Native SDK can record system audio, a microphone, or both into one signed 16-bit PCM WAV file on macOS 15 and later. The implementation lives in the Objective-C/AppKit system-engine host: ScreenCaptureKit supplies system and combined capture, while AVFoundation handles microphone-only capture without asking for Screen Recording access. + +Use the three feature flags before presenting capture UI: + +```zig +const can_system = runtime.supports(.system_audio_capture); +const can_microphone = runtime.supports(.microphone_capture); +const can_list_microphones = runtime.supports(.microphone_device_enumeration); +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
HostSystem audioMicrophoneDevice enumeration
macOS 15+ system engineSupportedSupportedSupported
macOS 11–14 system engineUnsupportedUnsupportedUnsupported
macOS ChromiumUnsupportedUnsupportedUnsupported
Linux, Windows, iOS, AndroidUnsupportedUnsupportedUnsupported
+ +## Manifest and privacy strings + +Declare filesystem access and exactly the source permissions the app uses. Nonempty privacy strings are mandatory when their matching permission is present: + +```zig:app.zon +.{ + .permissions = .{ "filesystem", "microphone", "system_audio" }, + .privacy = .{ + .microphone_usage = "Record the microphone selected for a meeting.", + .system_audio_usage = "Record meeting audio played by other apps.", + }, +} +``` + +Packaging maps `microphone_usage` to `NSMicrophoneUsageDescription`. It maps `system_audio_usage` to both `NSScreenCaptureUsageDescription` and `NSAudioCaptureUsageDescription`, covering ScreenCaptureKit and the system-audio purpose key. + +App Sandbox remains opt-in. A sandboxed app that records a microphone must add `com.apple.security.device.audio-input` to its own entitlement file. Native SDK does not rewrite custom entitlements. + +## Permission flow + +Starting a capture never opens a system prompt. Query or request access explicitly first with `fx.audioCaptureAccess` or `Cmd.audioCaptureAccess`: + +```ts +Cmd.audioCaptureAccess("mic-access", "microphone", "status", { event: "capture_access" }); +Cmd.audioCaptureAccess("system-access", "system_audio", "request", { event: "capture_access" }); +``` + +System audio reports the coarse ScreenCaptureKit truth: `authorized`, `not_authorized`, or `unavailable`. Microphone access additionally reports `not_determined`, `denied`, and `restricted`. A system-audio request can return `restartRequired: true`; show that result instead of pretending the running process can already capture. + +The manifest is an independent gate. Missing `filesystem`, `system_audio`, or `microphone` permission produces a rejected operation even when macOS has granted the corresponding TCC access. + +## Zig effects + +Every request carries a journal key and a fixed Msg constructor. This example records the current default microphone together with system audio at 48 kHz stereo: + +```zig +pub const Msg = union(enum) { + start_capture, + stop_capture, + list_microphones, + capture: native_sdk.EffectAudioCapture, + microphone: native_sdk.EffectMicrophoneDevice, + access: native_sdk.EffectAudioCaptureAccess, + microphones_changed, +}; + +const App = native_sdk.UiApp(Model, Msg); +const Fx = App.Effects; + +pub fn update(model: *Model, msg: Msg, fx: *Fx) void { + switch (msg) { + .start_capture => fx.startAudioCapture(.{ + .key = 1, + .path = "/tmp/meeting.wav", + .system_audio = true, + .microphone = .default, + .sample_rate_hz = 48_000, + .channel_count = 2, + .exclude_current_process_audio = true, + .on_event = Fx.audioCaptureMsg(.capture), + }), + .stop_capture => fx.stopAudioCapture(), + .list_microphones => fx.listMicrophoneDevices(.{ + .key = 2, + .on_event = Fx.microphoneDeviceMsg(.microphone), + }), + .capture => |event| model.recordCapture(event), + .microphone => |event| model.recordMicrophone(event), + .access => |event| model.recordAccess(event), + .microphones_changed => fx.listMicrophoneDevices(.{ + .key = 3, + .on_event = Fx.microphoneDeviceMsg(.microphone), + }), + } +} + +pub fn boot(_: *Model, fx: *Fx) void { + fx.observeMicrophoneDevices(Fx.microphoneDevicesChangedMsg(.microphones_changed)); + fx.audioCaptureAccess(.{ + .key = 4, + .source = .microphone, + .action = .status, + .on_event = Fx.audioCaptureAccessMsg(.access), + }); +} +``` + +`stopAudioCapture` stops the one active capture. Calling it while idle is a no-op. A second start is rejected with `already_recording` and does not disturb the current session. + +## TypeScript commands + +The TypeScript bytecode ABI uses fixed records. A selected device ID is a `Uint8Array`; `"default"` and `"none"` are the other microphone selections. + +```ts +import { Cmd, Sub, asciiBytes } from "@native-sdk/core"; + +const start = Cmd.audioCaptureStart( + "meeting", + { + path: asciiBytes("/tmp/meeting.wav"), + systemAudio: true, + microphone: "default", + sampleRate: 48000, + channels: 2, + excludeCurrentProcessAudio: true, + }, + { event: "capture_event" }, +); + +const stop = Cmd.audioCaptureStop("meeting"); +const devices = Cmd.microphoneDevices("microphones", { event: "microphone_device" }); +const changes = Sub.microphoneDevicesChanged("microphones_changed"); +``` + +The `capture_event` Msg arm must contain `key`, `state`, `reason`, `durationMs`, `bytesWritten`, and `outputCommitted`. States are `started`, `stopped`, `failed`, or `rejected`. A terminal failure can still carry `outputCommitted: true` when a usable partial WAV was finalized. + +The `microphone_device` arm receives zero or more `device` records with `id`, `name`, `isDefault`, `index`, and `total`, followed by exactly one `completed`, `failed`, or `rejected` record. `microphones_changed` is only invalidation: issue `Cmd.microphoneDevices` again to obtain a new snapshot. + +## Device and file guarantees + +`"default"` resolves when recording starts and stays pinned for that session. Passing an enumerated `AVCaptureDevice.uniqueID` selects that exact device. A missing or disconnected explicit device fails instead of switching silently; disconnecting the retained default device follows the same rule. + +Sample rates are 16, 24, 44.1, or 48 kHz; 48 kHz is the default. Channels are mono or stereo; stereo is the default. Combined capture timestamp-aligns both inputs, inserts silence for gaps, resamples into the requested format, and mixes with equal gain and clipping headroom. + +The recorder writes a sibling temporary file, finalizes its WAV header, and atomically publishes the destination. It never overwrites an existing file. Stream or device failure preserves a usable partial WAV when possible, and graceful shutdown makes a bounded finalization attempt. + +Raw PCM streaming, level meters, source gains, video capture, and per-application audio filtering are not part of this API. diff --git a/docs/src/app/docs/capabilities/page.mdx b/docs/src/app/docs/capabilities/page.mdx index f87f28f2c..5c82e5c20 100644 --- a/docs/src/app/docs/capabilities/page.mdx +++ b/docs/src/app/docs/capabilities/page.mdx @@ -73,6 +73,13 @@ Web content itself is declare-to-use: an app ships the embedded web layer only w None. No bridge surface. macOS system-engine host (an MTAudioProcessingTap on the app's single AVPlayer feeding a vDSP FFT — local files, cache entries, and streams alike), Windows system WebView (process-scoped WASAPI loopback capture of THIS app's audio session only — never other apps' audio — with an in-box FFT), and Linux system WebView (GStreamer's spectrum element as the playbin's audio-filter; requires gst-plugins-good, probed live); hosts that cannot analyze — macOS Chromium, pre-2004 Windows, spectrum-less GStreamer setups, iOS, Android — report audio_spectrum unsupported and simply never send the events: honest absence, never fabricated bands + + Audio capture + fx.startAudioCapture(options) / fx.stopAudioCapture() / fx.listMicrophoneDevices(options) / fx.audioCaptureAccess(options) + Cmd.audioCaptureStart / Cmd.audioCaptureStop / Cmd.microphoneDevices / Cmd.audioCaptureAccess / Sub.microphoneDevicesChanged + filesystem plus system_audio and/or microphone, matching the requested sources + macOS 15+ system-engine host: microphone-only through AVFoundation, system-audio-only or combined capture through ScreenCaptureKit. Older macOS, macOS Chromium, Linux, Windows, iOS, and Android report the three capture features unsupported. + File drops Event.files_dropped diff --git a/docs/src/app/docs/platform-support/page.mdx b/docs/src/app/docs/platform-support/page.mdx index 8d0b802a5..c6bed5209 100644 --- a/docs/src/app/docs/platform-support/page.mdx +++ b/docs/src/app/docs/platform-support/page.mdx @@ -298,6 +298,14 @@ macOS, Linux, and Windows run full desktop apps through their own platform hosts Unsupported Supported on Windows 10 2004+ (process-scoped WASAPI loopback of this app only, probed live) + + System audio / microphone capture + Supported on macOS 15+ (ScreenCaptureKit + AVFoundation) + Unsupported + Unsupported + Unsupported + Unsupported + @@ -305,6 +313,8 @@ macOS, Linux, and Windows run full desktop apps through their own platform hosts Audio spectrum analysis reads the app's OWN playback and nothing else: macOS taps the app's single player pre-effects, Windows captures the app's own audio session through process-scoped loopback (never system-wide — other apps' audio can never appear in the bands), and Linux analyzes inside the app's own playbin. Hosts that cannot analyze report `audio_spectrum` unsupported and simply never deliver `.spectrum` events; see the capabilities page for the band shape and cadence contract. +Audio capture is intentionally narrower than playback: only the macOS 15+ system-engine host exposes `system_audio_capture`, `microphone_capture`, and `microphone_device_enumeration`. The feature flags stay false on older macOS and every Chromium, Linux, Windows, iOS, and Android host. See [Audio Capture](/docs/audio-capture) for permissions, source selection, device identity, and output guarantees. + ## How Support Is Verified macOS is the primary development platform and carries the deepest support. The Linux and Windows columns are not aspirational: the repository carries reproducible live-verification loops that build every showcase app and drive it for real. `tools/linux-truth` runs the apps in real windows against the platform toolkit under Xvfb — clicks, keys, wheel input, multi-window flows, resize clamps, window-close paths, and both engine and X-server screenshots. `tools/windows-truth` drives the same scenarios on a real Windows desktop, plus clipboard round-trips, effect-stream cancellation, record/replay, and launching the packaged artifact. CI additionally runs headless Linux and Windows canvas smokes on every change. iOS is exercised on the simulator through the toolkit host and the embed library — the `native dev --target ios` loop launches real apps, and the input and layout verification scripts inject hardware-true touches and keyboard events; a fresh `native package --target ios` output is archived with `xcodebuild` as part of verification. Android is exercised on the emulator the same way: the `native dev --target android` loop assembles, installs, and launches real apps, input is injected over adb (taps, keys, soft-keyboard text), and a fresh `native package --target android` output assembles the debug APK; the embed ABI additionally cross-compiles for both Android architectures in CI. diff --git a/docs/src/app/docs/typescript/page.mdx b/docs/src/app/docs/typescript/page.mdx index 6c960e2cf..b9ba48b14 100644 --- a/docs/src/app/docs/typescript/page.mdx +++ b/docs/src/app/docs/typescript/page.mdx @@ -294,6 +294,10 @@ The runtime interprets the command after the model commits and dispatches any re Cmd.audioPlay(key, source, { event }) + audioPause/audioResume/audioStop/audioSeek/audioSetVolume The audio player: one event stream (loaded, position, completed, failed, spectrum, ...) until audioStop closes it + + Cmd.audioCaptureStart(key, options, { event }) / audioCaptureStop(key) / microphoneDevices(key, { event }) / audioCaptureAccess(key, source, action, { event }) + macOS 15+ PCM WAV capture, microphone enumeration, and explicit permission status/request operations; see Audio Capture + Cmd.showWindow(label) / Cmd.quitApp() The menu-bar lifecycle verbs: un-hide + activate the labeled window (the tray "Open" consequence, the counterpart to close_policy = "hide"), and the real graceful terminate diff --git a/docs/src/lib/docs-navigation.ts b/docs/src/lib/docs-navigation.ts index 134a10d0d..e736b5c8a 100644 --- a/docs/src/lib/docs-navigation.ts +++ b/docs/src/lib/docs-navigation.ts @@ -51,6 +51,7 @@ const unprefixedNavSections: NavSection[] = [ title: "Native Platform", items: [ { name: "Windows", href: "/windows" }, + { name: "Audio Capture", href: "/audio-capture" }, { name: "Native Surfaces", href: "/native-surfaces" }, { name: "Menus", href: "/menus" }, { name: "Dialogs", href: "/dialogs" }, diff --git a/docs/src/lib/page-titles.ts b/docs/src/lib/page-titles.ts index 522b14a02..aff7c3864 100644 --- a/docs/src/lib/page-titles.ts +++ b/docs/src/lib/page-titles.ts @@ -22,6 +22,7 @@ export const PAGE_TITLES: Record = { "native-surfaces": "Native Surfaces", "media-producers": "Media Producers", windows: "Windows", + "audio-capture": "Audio Capture", webviews: "Multiple WebViews", "keyboard-shortcuts": "Keyboard Shortcuts", commands: "Commands", diff --git a/examples/README.md b/examples/README.md index c80969a84..f53da82ac 100644 --- a/examples/README.md +++ b/examples/README.md @@ -29,6 +29,8 @@ native build # produce a ReleaseFast binary in zig-out/bin/ | `gpu-components` | The retained GPU widget controls in one native-first component lab. | | `canvas-preview` | Canvas + WebView in one window, panes snapped to canvas anchors, a status item. | | `effects-probe` | The effect system live: spawn/fetch/file effects, cancellation, worker wakes. | +| `audio-capture` | macOS 15+ system audio and selectable-microphone recording through Zig effects. | +| `audio-capture-ts` | The same capture, permission, and device-enumeration flow from a TypeScript core. | | `menu-bar` | The menu-bar app lifecycle: `close_policy = "hide"`, a status item whose Open/Quit rows drive `fx.showWindow`/`fx.quitApp`, Dock reopen. | ## Examples that own their build diff --git a/examples/audio-capture-ts/README.md b/examples/audio-capture-ts/README.md new file mode 100644 index 000000000..b8ba51af7 --- /dev/null +++ b/examples/audio-capture-ts/README.md @@ -0,0 +1,9 @@ +# Audio Capture (TypeScript) + +The TypeScript counterpart to `examples/audio-capture`. It demonstrates fixed capture, device, and access Msg records plus `Sub.microphoneDevicesChanged` invalidation. + +```sh +native dev +``` + +The combined recording is atomically published as `/tmp/native-sdk-combined-ts.wav`; an existing destination is rejected instead of overwritten. diff --git a/examples/audio-capture-ts/app.zon b/examples/audio-capture-ts/app.zon new file mode 100644 index 000000000..116e02552 --- /dev/null +++ b/examples/audio-capture-ts/app.zon @@ -0,0 +1,36 @@ +.{ + .id = "dev.native_sdk.audio_capture_ts", + .name = "audio-capture-ts", + .display_name = "Audio Capture TS", + .description = "Record macOS system audio and the default microphone from a TypeScript core.", + .version = "0.1.0", + .platforms = .{"macos"}, + .permissions = .{ "view", "filesystem", "microphone", "system_audio" }, + .privacy = .{ + .microphone_usage = "Record the microphone selected in the TypeScript audio capture example.", + .system_audio_usage = "Record system audio in the TypeScript audio capture example.", + }, + .capabilities = .{ "native_views", "gpu_surfaces" }, + .shell = .{ + .windows = .{ + .{ + .label = "main", + .title = "Native SDK Audio Capture TS", + .width = 660, + .height = 390, + .restore_state = false, + .restore_policy = "center_on_primary", + .views = .{ + .{ .label = "capture-canvas", .kind = "gpu_surface", .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture TypeScript example", .gpu_backend = "metal", .gpu_pixel_format = "bgra8_unorm", .gpu_present_mode = "timer", .gpu_alpha_mode = "opaque", .gpu_color_space = "srgb", .gpu_vsync = true }, + }, + }, + }, + }, + .security = .{ + .navigation = .{ + .allowed_origins = .{ "zero://app", "zero://inline" }, + .external_links = .{ .action = "deny" }, + }, + }, + .web_engine = "system", +} diff --git a/examples/audio-capture-ts/package.json b/examples/audio-capture-ts/package.json new file mode 100644 index 000000000..a4b980856 --- /dev/null +++ b/examples/audio-capture-ts/package.json @@ -0,0 +1,7 @@ +{ + "name": "audio-capture-ts", + "private": true, + "dependencies": { + "@native-sdk/core": "0.7.1" + } +} diff --git a/examples/audio-capture-ts/src/app.native b/examples/audio-capture-ts/src/app.native new file mode 100644 index 000000000..b62328b2c --- /dev/null +++ b/examples/audio-capture-ts/src/app.native @@ -0,0 +1,26 @@ + + Audio capture + macOS 15+ · TypeScript command and subscription API + + + + + + Microphone: {microphoneAccess} + System audio: {systemAccess} + {deviceCount} connected microphone(s) + + + + + + + + Status: {captureState} + Reason: {captureReason} + {durationMs} ms · {bytesWritten} bytes + + + + Output: /tmp/native-sdk-combined-ts.wav + diff --git a/examples/audio-capture-ts/src/core.ts b/examples/audio-capture-ts/src/core.ts new file mode 100644 index 000000000..28d68376c --- /dev/null +++ b/examples/audio-capture-ts/src/core.ts @@ -0,0 +1,96 @@ +import { Cmd, Sub, asciiBytes, type Cmd as Command, type Sub as Subscription } from "@native-sdk/core"; + +export type CaptureState = "started" | "stopped" | "failed" | "rejected"; +export type CaptureReason = + | "none" | "invalid_options" | "permission_missing" | "permission_required" + | "already_recording" | "device_not_found" | "device_disconnected" + | "output_exists" | "io_failed" | "capture_failed" | "no_audio" | "unsupported"; +export type AccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; +export type AccessSource = "system_audio" | "microphone"; +export type DeviceState = "device" | "completed" | "failed" | "rejected"; + +export interface Model { + readonly captureState: CaptureState; + readonly captureReason: CaptureReason; + readonly durationMs: number; + readonly bytesWritten: number; + readonly outputCommitted: boolean; + readonly deviceCount: number; + readonly microphoneAccess: AccessStatus; + readonly systemAccess: AccessStatus; + readonly restartRequired: boolean; +} + +export type Msg = + | { readonly kind: "start_capture" } + | { readonly kind: "stop_capture" } + | { readonly kind: "list_microphones" } + | { readonly kind: "request_microphone" } + | { readonly kind: "request_system_audio" } + | { readonly kind: "microphones_changed" } + | { readonly kind: "capture_event"; readonly key: string; readonly state: CaptureState; readonly reason: CaptureReason; readonly durationMs: number; readonly bytesWritten: number; readonly outputCommitted: boolean } + | { readonly kind: "microphone_device"; readonly key: string; readonly state: DeviceState; readonly id: Uint8Array; readonly name: Uint8Array; readonly isDefault: boolean; readonly index: number; readonly total: number } + | { readonly kind: "capture_access"; readonly key: string; readonly source: AccessSource; readonly status: AccessStatus; readonly restartRequired: boolean }; + +export const viewUnbound = ["microphones_changed", "capture_event", "microphone_device", "capture_access", "outputCommitted", "restartRequired"] as const; + +export function initialModel(): Model { + return { + captureState: "stopped", + captureReason: "none", + durationMs: 0, + bytesWritten: 0, + outputCommitted: false, + deviceCount: 0, + microphoneAccess: "not_determined", + systemAccess: "not_authorized", + restartRequired: false, + }; +} + +export function update(model: Model, msg: Msg): [Model, Command] { + switch (msg.kind) { + case "start_capture": + return [ + { ...model, captureReason: "none", durationMs: 0, bytesWritten: 0, outputCommitted: false }, + Cmd.audioCaptureStart("meeting", { + path: asciiBytes("/tmp/native-sdk-combined-ts.wav"), + systemAudio: true, + microphone: "default", + sampleRate: 48000, + channels: 2, + excludeCurrentProcessAudio: true, + }, { event: "capture_event" }), + ]; + case "stop_capture": + return [model, Cmd.audioCaptureStop("meeting")]; + case "list_microphones": + case "microphones_changed": + return [{ ...model, deviceCount: 0 }, Cmd.microphoneDevices("microphones", { event: "microphone_device" })]; + case "request_microphone": + return [model, Cmd.audioCaptureAccess("mic-access", "microphone", "request", { event: "capture_access" })]; + case "request_system_audio": + return [model, Cmd.audioCaptureAccess("system-access", "system_audio", "request", { event: "capture_access" })]; + case "capture_event": + return [{ + ...model, + captureState: msg.state, + captureReason: msg.reason, + durationMs: msg.durationMs, + bytesWritten: msg.bytesWritten, + outputCommitted: msg.outputCommitted, + }, Cmd.none]; + case "microphone_device": + if (msg.state === "device") return [{ ...model, deviceCount: model.deviceCount + 1 }, Cmd.none]; + return [model, Cmd.none]; + case "capture_access": + if (msg.source === "microphone") { + return [{ ...model, microphoneAccess: msg.status, restartRequired: msg.restartRequired }, Cmd.none]; + } + return [{ ...model, systemAccess: msg.status, restartRequired: msg.restartRequired }, Cmd.none]; + } +} + +export function subscriptions(_model: Model): Subscription { + return Sub.microphoneDevicesChanged("microphones_changed"); +} diff --git a/examples/audio-capture-ts/tsconfig.json b/examples/audio-capture-ts/tsconfig.json new file mode 100644 index 000000000..27b52cd12 --- /dev/null +++ b/examples/audio-capture-ts/tsconfig.json @@ -0,0 +1,10 @@ +{ + "compilerOptions": { + "strict": true, + "target": "ES2022", + "module": "ESNext", + "moduleResolution": "Bundler", + "noEmit": true + }, + "include": ["src/**/*.ts"] +} diff --git a/examples/audio-capture/README.md b/examples/audio-capture/README.md new file mode 100644 index 000000000..0b6cdf5a9 --- /dev/null +++ b/examples/audio-capture/README.md @@ -0,0 +1,11 @@ +# Audio Capture (Zig) + +A small macOS 15+ app showing the complete Zig effects flow: explicit permission requests, microphone enumeration and invalidation, system-only, microphone-only, and combined capture, plus terminal event handling. + +```sh +native dev +``` + +Recordings are atomically published as `/tmp/native-sdk-system.wav`, `/tmp/native-sdk-microphone.wav`, or `/tmp/native-sdk-combined.wav`. The recorder never overwrites an existing file, so remove or move an earlier output before repeating that source. + +Starting capture never prompts. Use the permission buttons first and follow any macOS restart instruction reported by the app. diff --git a/examples/audio-capture/app.zon b/examples/audio-capture/app.zon new file mode 100644 index 000000000..fafe8b7fa --- /dev/null +++ b/examples/audio-capture/app.zon @@ -0,0 +1,36 @@ +.{ + .id = "dev.native_sdk.audio_capture", + .name = "audio-capture", + .display_name = "Audio Capture", + .description = "Record macOS system audio and a selected microphone to PCM WAV.", + .version = "0.1.0", + .platforms = .{"macos"}, + .permissions = .{ "view", "filesystem", "microphone", "system_audio" }, + .privacy = .{ + .microphone_usage = "Record the microphone selected in the audio capture example.", + .system_audio_usage = "Record system audio in the audio capture example.", + }, + .capabilities = .{ "native_views", "gpu_surfaces" }, + .shell = .{ + .windows = .{ + .{ + .label = "main", + .title = "Native SDK Audio Capture", + .width = 680, + .height = 430, + .restore_state = false, + .restore_policy = "center_on_primary", + .views = .{ + .{ .label = "capture-canvas", .kind = "gpu_surface", .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture", .gpu_backend = "metal", .gpu_pixel_format = "bgra8_unorm", .gpu_present_mode = "timer", .gpu_alpha_mode = "opaque", .gpu_color_space = "srgb", .gpu_vsync = true }, + }, + }, + }, + }, + .security = .{ + .navigation = .{ + .allowed_origins = .{ "zero://app", "zero://inline" }, + .external_links = .{ .action = "deny" }, + }, + }, + .web_engine = "system", +} diff --git a/examples/audio-capture/src/main.zig b/examples/audio-capture/src/main.zig new file mode 100644 index 000000000..04b45bd0a --- /dev/null +++ b/examples/audio-capture/src/main.zig @@ -0,0 +1,190 @@ +//! macOS 15+ system-audio and microphone capture through Zig effects. + +const std = @import("std"); +const runner = @import("runner"); +const native_sdk = @import("native_sdk"); + +pub const panic = std.debug.FullPanic(native_sdk.debug.capturePanic); + +const canvas = native_sdk.canvas; +const geometry = native_sdk.geometry; +const canvas_label = "capture-canvas"; +const window_width: f32 = 680; +const window_height: f32 = 430; + +const shell_views = [_]native_sdk.ShellView{ + .{ .label = canvas_label, .kind = .gpu_surface, .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture", .gpu_backend = .metal, .gpu_pixel_format = .bgra8_unorm, .gpu_present_mode = .timer, .gpu_alpha_mode = .@"opaque", .gpu_color_space = .srgb, .gpu_vsync = true }, +}; +const shell_windows = [_]native_sdk.ShellWindow{.{ + .label = "main", + .title = "Native SDK Audio Capture", + .width = window_width, + .height = window_height, + .restore_state = false, + .views = &shell_views, +}}; +const shell_scene: native_sdk.ShellConfig = .{ .windows = &shell_windows }; + +const CaptureStatus = enum { idle, starting, recording, stopped, failed, rejected }; + +pub const Model = struct { + status: CaptureStatus = .idle, + reason: native_sdk.EffectAudioCaptureReason = .none, + duration_ms: u64 = 0, + bytes_written: u64 = 0, + output_committed: bool = false, + microphone_count: u32 = 0, + microphone_access: native_sdk.EffectAudioCaptureAccessStatus = .not_determined, + system_access: native_sdk.EffectAudioCaptureAccessStatus = .not_authorized, + restart_required: bool = false, + + pub fn statusText(model: *const Model, arena: std.mem.Allocator) []const u8 { + return std.fmt.allocPrint(arena, "{s} · {s} · {d} ms · {d} bytes · committed={any}", .{ + @tagName(model.status), + @tagName(model.reason), + model.duration_ms, + model.bytes_written, + model.output_committed, + }) catch "audio capture"; + } + + pub fn accessText(model: *const Model, arena: std.mem.Allocator) []const u8 { + return std.fmt.allocPrint(arena, "microphone={s} · system={s} · restart required={any}", .{ + @tagName(model.microphone_access), + @tagName(model.system_access), + model.restart_required, + }) catch "access status unavailable"; + } +}; + +pub const Msg = union(enum) { + request_microphone, + request_system_audio, + list_microphones, + start_system, + start_microphone, + start_combined, + stop, + capture: native_sdk.EffectAudioCapture, + microphone: native_sdk.EffectMicrophoneDevice, + access: native_sdk.EffectAudioCaptureAccess, + microphones_changed, +}; + +const CaptureApp = native_sdk.UiApp(Model, Msg); +pub const Effects = CaptureApp.Effects; + +pub fn boot(_: *Model, fx: *Effects) void { + fx.observeMicrophoneDevices(Effects.microphoneDevicesChangedMsg(.microphones_changed)); + fx.audioCaptureAccess(.{ .key = 1, .source = .microphone, .on_event = Effects.audioCaptureAccessMsg(.access) }); + fx.audioCaptureAccess(.{ .key = 2, .source = .system_audio, .on_event = Effects.audioCaptureAccessMsg(.access) }); +} + +pub fn update(model: *Model, msg: Msg, fx: *Effects) void { + switch (msg) { + .request_microphone => fx.audioCaptureAccess(.{ .key = 3, .source = .microphone, .action = .request, .on_event = Effects.audioCaptureAccessMsg(.access) }), + .request_system_audio => fx.audioCaptureAccess(.{ .key = 4, .source = .system_audio, .action = .request, .on_event = Effects.audioCaptureAccessMsg(.access) }), + .list_microphones, .microphones_changed => { + model.microphone_count = 0; + fx.listMicrophoneDevices(.{ .key = 5, .on_event = Effects.microphoneDeviceMsg(.microphone) }); + }, + .start_system => start(model, fx, .system), + .start_microphone => start(model, fx, .microphone), + .start_combined => start(model, fx, .combined), + .stop => fx.stopAudioCapture(), + .capture => |event| { + model.status = switch (event.state) { + .started => .recording, + .stopped => .stopped, + .failed => .failed, + .rejected => .rejected, + }; + model.reason = event.reason; + model.duration_ms = event.duration_ms; + model.bytes_written = event.bytes_written; + model.output_committed = event.output_committed; + }, + .microphone => |event| if (event.state == .device) { + model.microphone_count += 1; + }, + .access => |event| { + switch (event.source) { + .microphone => model.microphone_access = event.status, + .system_audio => model.system_access = event.status, + } + model.restart_required = event.restart_required; + }, + } +} + +const Source = enum { system, microphone, combined }; + +fn start(model: *Model, fx: *Effects, source: Source) void { + model.status = .starting; + model.reason = .none; + model.duration_ms = 0; + model.bytes_written = 0; + model.output_committed = false; + fx.startAudioCapture(.{ + .key = 10, + .path = switch (source) { + .system => "/tmp/native-sdk-system.wav", + .microphone => "/tmp/native-sdk-microphone.wav", + .combined => "/tmp/native-sdk-combined.wav", + }, + .system_audio = source != .microphone, + .microphone = if (source == .system) .none else .default, + .on_event = Effects.audioCaptureMsg(.capture), + }); +} + +pub const CaptureUi = canvas.Ui(Msg); + +pub fn view(ui: *CaptureUi, model: *const Model) CaptureUi.Node { + return ui.column(.{ .padding = 20, .gap = 14, .style_tokens = .{ .background = .background } }, .{ + ui.text(.{ .size = .lg }, "Audio capture"), + ui.text(.{ .style_tokens = .{ .foreground = .text_muted } }, "macOS 15+ · signed 16-bit PCM WAV · one active capture"), + ui.row(.{ .gap = 8 }, .{ + ui.button(.{ .on_press = .request_microphone }, "Request microphone"), + ui.button(.{ .on_press = .request_system_audio }, "Request system audio"), + ui.button(.{ .on_press = .list_microphones }, "List microphones"), + }), + ui.text(.{}, model.accessText(ui.arena)), + ui.text(.{}, ui.fmt("{d} connected microphone(s)", .{model.microphone_count})), + ui.separator(.{}), + ui.row(.{ .gap = 8 }, .{ + ui.button(.{ .variant = .primary, .on_press = .start_combined }, "System + default mic"), + ui.button(.{ .on_press = .start_system }, "System only"), + ui.button(.{ .on_press = .start_microphone }, "Mic only"), + ui.button(.{ .variant = .destructive, .on_press = .stop }, "Stop"), + }), + ui.panel(.{ .padding = 14, .style_tokens = .{ .background = .surface, .radius = .md } }, .{ + ui.text(.{}, model.statusText(ui.arena)), + }), + ui.spacer(1), + ui.statusBar(.{}, "Outputs are written under /tmp; existing files are never overwritten."), + }); +} + +pub fn main(init: std.process.Init) !void { + const app_state = try std.heap.page_allocator.create(CaptureApp); + defer std.heap.page_allocator.destroy(app_state); + app_state.* = CaptureApp.init(std.heap.page_allocator, .{}, .{ + .name = "audio-capture", + .scene = shell_scene, + .canvas_label = canvas_label, + .init_fx = boot, + .update_fx = update, + .view = view, + }); + defer app_state.deinit(); + try runner.runWithOptions(app_state.app(), .{ + .app_name = "audio-capture", + .window_title = "Native SDK Audio Capture", + .bundle_id = "dev.native_sdk.audio_capture", + .default_frame = geometry.RectF.init(0, 0, window_width, window_height), + .restore_state = false, + .js_window_api = false, + .security = .{ .navigation = .{ .allowed_origins = &.{ "zero://inline", "zero://app" } } }, + }, init); +} diff --git a/examples/browser/build.zig b/examples/browser/build.zig index fa794db1c..02924f151 100644 --- a/examples/browser/build.zig +++ b/examples/browser/build.zig @@ -180,6 +180,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -207,6 +208,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/capabilities/build.zig b/examples/capabilities/build.zig index 42dd10f6b..f5c4173c0 100644 --- a/examples/capabilities/build.zig +++ b/examples/capabilities/build.zig @@ -198,6 +198,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -227,6 +228,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/capabilities/src/main.zig b/examples/capabilities/src/main.zig index 9cf50d226..410b9ac99 100644 --- a/examples/capabilities/src/main.zig +++ b/examples/capabilities/src/main.zig @@ -110,7 +110,7 @@ const CapabilitiesApp = struct { }, else => {}, }, - .appearance_changed, .command, .shortcut, .timer, .effects_wake, .audio, .video, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance => {}, + .appearance_changed, .command, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance => {}, } } }; diff --git a/examples/command-app/build.zig b/examples/command-app/build.zig index 78d86f5f2..770affef6 100644 --- a/examples/command-app/build.zig +++ b/examples/command-app/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/command-app/src/main.zig b/examples/command-app/src/main.zig index 2b670e674..cfaf41450 100644 --- a/examples/command-app/src/main.zig +++ b/examples/command-app/src/main.zig @@ -179,7 +179,7 @@ const CommandApp = struct { try self.handleCommand(runtime, command); } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/gpu-components/src/app.zig b/examples/gpu-components/src/app.zig index 7d4ad2283..dc412f798 100644 --- a/examples/gpu-components/src/app.zig +++ b/examples/gpu-components/src/app.zig @@ -183,7 +183,7 @@ pub const GpuComponentsApp = struct { .canvas_widget_keyboard => |keyboard_event| try self.handleWidgetKeyboard(runtime, keyboard_event), .canvas_widget_dismiss => |dismiss_event| try self.handleWidgetDismiss(runtime, dismiss_event), .appearance_changed => |appearance| try self.applySystemAppearance(runtime, appearance), - .gpu_surface_resized, .gpu_surface_input, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .gpu_surface_resized, .gpu_surface_input, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/gpu-surface/src/main.zig b/examples/gpu-surface/src/main.zig index 90d892b88..59d0a8bbe 100644 --- a/examples/gpu-surface/src/main.zig +++ b/examples/gpu-surface/src/main.zig @@ -150,7 +150,7 @@ const GpuSurfaceApp = struct { self.gpu_input_count += 1; } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/hello/build.zig b/examples/hello/build.zig index eb616fa01..932352f32 100644 --- a/examples/hello/build.zig +++ b/examples/hello/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/native-panels/build.zig b/examples/native-panels/build.zig index 9ecaff009..356dc1517 100644 --- a/examples/native-panels/build.zig +++ b/examples/native-panels/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/native-panels/src/main.zig b/examples/native-panels/src/main.zig index bd7a5bac6..7ee0c1c8f 100644 --- a/examples/native-panels/src/main.zig +++ b/examples/native-panels/src/main.zig @@ -172,7 +172,7 @@ const NativePanelsApp = struct { try self.apply(runtime, command); } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/native-shell/build.zig b/examples/native-shell/build.zig index 35d71ae98..3564a9095 100644 --- a/examples/native-shell/build.zig +++ b/examples/native-shell/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/native-shell/src/main.zig b/examples/native-shell/src/main.zig index cd29a16fe..ddc6727aa 100644 --- a/examples/native-shell/src/main.zig +++ b/examples/native-shell/src/main.zig @@ -200,7 +200,7 @@ const NativeShellApp = struct { try self.closePreview(runtime); } }, - .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, + .appearance_changed, .shortcut, .timer, .effects_wake, .audio, .audio_capture, .microphone_device, .microphone_devices_changed, .audio_capture_access, .video, .files_dropped, .gpu_surface_frame, .gpu_surface_resized, .gpu_surface_input, .canvas_widget_pointer, .canvas_widget_keyboard, .canvas_widget_scroll, .canvas_widget_file_drop, .canvas_widget_drag, .canvas_widget_context_menu, .canvas_widget_context_menu_shown, .canvas_widget_context_menu_dismissed, .canvas_widget_context_menu_request, .canvas_widget_dismiss, .canvas_widget_context_press, .canvas_widget_resize, .canvas_widget_change, .window_closed, .automation_provenance, .lifecycle => {}, } } diff --git a/examples/next/build.zig b/examples/next/build.zig index e67e91068..83af42b8e 100644 --- a/examples/next/build.zig +++ b/examples/next/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/react/build.zig b/examples/react/build.zig index 089a02d67..87a06a334 100644 --- a/examples/react/build.zig +++ b/examples/react/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/svelte/build.zig b/examples/svelte/build.zig index 5afd1d7fd..cdbbdb879 100644 --- a/examples/svelte/build.zig +++ b/examples/svelte/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/vue/build.zig b/examples/vue/build.zig index e60f08db8..2d63418ff 100644 --- a/examples/vue/build.zig +++ b/examples/vue/build.zig @@ -243,6 +243,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -272,6 +273,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/examples/webview/build.zig b/examples/webview/build.zig index eabef87eb..e1ad4f14a 100644 --- a/examples/webview/build.zig +++ b/examples/webview/build.zig @@ -195,6 +195,7 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else ""; const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" }; app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/appkit_host.m"), .flags = flags }); + app_mod.addCSourceFile(.{ .file = nativeSdkPath(b, native_sdk_path, "src/platform/macos/audio_capture.m"), .flags = flags }); app_mod.linkFramework("WebKit", .{}); }, .chromium => { @@ -224,6 +225,9 @@ fn linkPlatform(b: *std.Build, target: std.Build.ResolvedTarget, app_mod: *std.B app_mod.linkFramework("AppKit", .{}); // The audio playback service (the AppKit host's single AVPlayer). app_mod.linkFramework("AVFoundation", .{}); + app_mod.linkFramework("ScreenCaptureKit", .{}); + app_mod.linkFramework("AudioToolbox", .{}); + app_mod.linkFramework("CoreMedia", .{}); // CVPixelBuffer for the video frame path (the AppKit host's // AVPlayerItemVideoOutput frames are real CoreVideo symbols). app_mod.linkFramework("CoreVideo", .{}); diff --git a/packages/core/rt/rt.zig b/packages/core/rt/rt.zig index e121fa251..665c74a24 100644 --- a/packages/core/rt/rt.zig +++ b/packages/core/rt/rt.zig @@ -1410,7 +1410,7 @@ pub fn Kernel(comptime opts: Options) type { // copies) them after the model commit and BEFORE frameReset — the same // boundary the committed model crosses. - pub const cmd_format_version: u32 = 3; + pub const cmd_format_version: u32 = 4; /// An encoded command value: op records per the layout above. pub const Cmd = []const u8; @@ -1444,6 +1444,10 @@ pub fn Kernel(comptime opts: Options) type { pty_write = 0x1A, pty_resize = 0x1B, pty_kill = 0x1C, + audio_capture_start = 0x1D, + audio_capture_stop = 0x1E, + microphone_devices = 0x1F, + audio_capture_access = 0x20, }; /// The spawn record's "no line routing" sentinel: a `line_tag` of @@ -1736,6 +1740,57 @@ pub fn Kernel(comptime opts: Options) type { return out; } + pub fn cmdAudioCaptureStart(key: []const u8, event_tag: u8, path: []const u8, system_audio: bool, microphone_kind: u8, microphone_id: []const u8, sample_rate: u32, channels: u8, exclude_current_process_audio: bool) Cmd { + std.debug.assert(key.len <= 255); + std.debug.assert(path.len <= std.math.maxInt(u32)); + std.debug.assert(microphone_id.len <= std.math.maxInt(u32)); + const out = frameAlloc(u8, 2 + key.len + 1 + 1 + 1 + 4 + 1 + 4 + path.len + 4 + microphone_id.len); + out[0] = @intFromEnum(CmdOp.audio_capture_start); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + var off: usize = 2 + key.len; + out[off] = event_tag; + out[off + 1] = @as(u8, @intFromBool(system_audio)) | (@as(u8, @intFromBool(exclude_current_process_audio)) << 1); + out[off + 2] = microphone_kind; + std.mem.writeInt(u32, out[off + 3 ..][0..4], sample_rate, .little); + out[off + 7] = channels; + off += 8; + off = writeLongBytes(out, off, path); + _ = writeLongBytes(out, off, microphone_id); + return out; + } + + pub fn cmdAudioCaptureStop(key: []const u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 2 + key.len); + out[0] = @intFromEnum(CmdOp.audio_capture_stop); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + return out; + } + + pub fn cmdMicrophoneDevices(key: []const u8, event_tag: u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 3 + key.len); + out[0] = @intFromEnum(CmdOp.microphone_devices); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + return out; + } + + pub fn cmdAudioCaptureAccess(key: []const u8, event_tag: u8, source: u8, action: u8) Cmd { + std.debug.assert(key.len <= 255); + const out = frameAlloc(u8, 5 + key.len); + out[0] = @intFromEnum(CmdOp.audio_capture_access); + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + out[3 + key.len] = source; + out[4 + key.len] = action; + return out; + } + pub fn cmdWindowShow(label: []const u8) Cmd { // The emitter's byte gate on the literal label is the // build-time teaching; this is the loud runtime backstop. @@ -1954,7 +2009,7 @@ pub fn Kernel(comptime opts: Options) type { /// An encoded subscription set: records per the layout above. pub const Sub = []const u8; - pub const SubOp = enum(u8) { timer = 0x01 }; + pub const SubOp = enum(u8) { timer = 0x01, microphone_devices_changed = 0x02 }; pub const sub_none: Sub = &.{}; @@ -1969,6 +2024,13 @@ pub fn Kernel(comptime opts: Options) type { return out; } + pub fn subMicrophoneDevicesChanged(msg_tag: u8) Sub { + const out = frameAlloc(u8, 2); + out[0] = @intFromEnum(SubOp.microphone_devices_changed); + out[1] = msg_tag; + return out; + } + pub fn subBatch(subs: []const Sub) Sub { return cmdBatch(subs); } diff --git a/packages/core/sdk/core.ts b/packages/core/sdk/core.ts index 52bbd3342..4aecb0d36 100644 --- a/packages/core/sdk/core.ts +++ b/packages/core/sdk/core.ts @@ -387,6 +387,72 @@ export type AudioEventKind = M extends Msgish : never : never; +export type AudioCaptureState = "started" | "stopped" | "failed" | "rejected"; +export type AudioCaptureReason = + | "none" | "invalid_options" | "permission_missing" | "permission_required" + | "already_recording" | "device_not_found" | "device_disconnected" + | "output_exists" | "io_failed" | "capture_failed" | "no_audio" | "unsupported"; +export type AudioCaptureEventArm = { + readonly key: string; + readonly state: AudioCaptureState; + readonly reason: AudioCaptureReason; + readonly durationMs: number; + readonly bytesWritten: number; + readonly outputCommitted: boolean; +}; +export type AudioCaptureEventKind = M extends Msgish + ? [Exclude] extends [keyof AudioCaptureEventArm] + ? [keyof AudioCaptureEventArm] extends [Exclude] + ? M extends Msgish & AudioCaptureEventArm + ? [AudioCaptureState] extends [M["state"]] + ? [AudioCaptureReason] extends [M["reason"]] ? M["kind"] : never + : never + : never + : never + : never + : never; + +export type MicrophoneDeviceState = "device" | "completed" | "failed" | "rejected"; +export type MicrophoneDeviceEventArm = { + readonly key: string; + readonly state: MicrophoneDeviceState; + readonly id: Uint8Array; + readonly name: Uint8Array; + readonly isDefault: boolean; + readonly index: number; + readonly total: number; +}; +export type MicrophoneDeviceEventKind = M extends Msgish + ? [Exclude] extends [keyof MicrophoneDeviceEventArm] + ? [keyof MicrophoneDeviceEventArm] extends [Exclude] + ? M extends Msgish & MicrophoneDeviceEventArm + ? [MicrophoneDeviceState] extends [M["state"]] ? M["kind"] : never + : never + : never + : never + : never; + +export type AudioCaptureAccessSource = "system_audio" | "microphone"; +export type AudioCaptureAccessAction = "status" | "request"; +export type AudioCaptureAccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; +export type AudioCaptureAccessEventArm = { + readonly key: string; + readonly source: AudioCaptureAccessSource; + readonly status: AudioCaptureAccessStatus; + readonly restartRequired: boolean; +}; +export type AudioCaptureAccessEventKind = M extends Msgish + ? [Exclude] extends [keyof AudioCaptureAccessEventArm] + ? [keyof AudioCaptureAccessEventArm] extends [Exclude] + ? M extends Msgish & AudioCaptureAccessEventArm + ? [AudioCaptureAccessSource] extends [M["source"]] + ? [AudioCaptureAccessStatus] extends [M["status"]] ? M["kind"] : never + : never + : never + : never + : never + : never; + /// The video event states, the audio vocabulary without spectrum: `loaded` /// acknowledges a successful load with the player's duration estimate and /// the stream's decoded pixel dimensions; `position` ticks at the @@ -704,6 +770,19 @@ export interface AudioRoute { readonly event: AudioEventKind; } +export type MicrophoneSelection = "none" | "default" | Uint8Array; +export interface AudioCaptureOptions { + readonly path: Uint8Array; + readonly systemAudio?: boolean; + readonly microphone?: MicrophoneSelection; + readonly sampleRate?: 16000 | 24000 | 44100 | 48000; + readonly channels?: 1 | 2; + readonly excludeCurrentProcessAudio?: boolean; +} +export interface AudioCaptureRoute { readonly event: AudioCaptureEventKind; } +export interface MicrophoneDevicesRoute { readonly event: MicrophoneDeviceEventKind; } +export interface AudioCaptureAccessRoute { readonly event: AudioCaptureAccessEventKind; } + /// A `Cmd.videoLoad` source. `surface` is the model-owned media-surface id /// the markup binds — the texture channel the decoded frames feed. The /// local `path` is tried first; a missing file falls through to `url` @@ -844,6 +923,21 @@ export type Cmd = /// Seek position (ms) / volume (0..1); 0 for the value-less verbs. readonly value: number; } + | { + readonly op: "audio_capture_start"; + readonly key: string; + readonly eventKind: string; + readonly options: AudioCaptureOptions; + } + | { readonly op: "audio_capture_stop"; readonly key: string } + | { readonly op: "microphone_devices"; readonly key: string; readonly eventKind: string } + | { + readonly op: "audio_capture_access"; + readonly key: string; + readonly eventKind: string; + readonly source: AudioCaptureAccessSource; + readonly action: AudioCaptureAccessAction; + } | { readonly op: "video_load"; readonly key: string; @@ -1119,6 +1213,22 @@ export const Cmd = { return { op: "audio_ctl", key, verb: "volume", value: volume }; }, + audioCaptureStart(key: string, options: AudioCaptureOptions, route: AudioCaptureRoute): Cmd { + return { op: "audio_capture_start", key, eventKind: route.event, options }; + }, + + audioCaptureStop(key: string): Cmd { + return { op: "audio_capture_stop", key }; + }, + + microphoneDevices(key: string, route: MicrophoneDevicesRoute): Cmd { + return { op: "microphone_devices", key, eventKind: route.event }; + }, + + audioCaptureAccess(key: string, source: AudioCaptureAccessSource, action: AudioCaptureAccessAction, route: AudioCaptureAccessRoute): Cmd { + return { op: "audio_capture_access", key, eventKind: route.event, source, action }; + }, + /// Open (or replace — one player is the whole surface) the keyed video /// event stream: claim the media-surface the source names, resolve the /// source cascade (local path, then url) and start playback (autoplay, @@ -1375,6 +1485,7 @@ export const Cmd = { export type Sub = | { readonly op: "none" } | { readonly op: "timer"; readonly key: string; readonly everyMs: number; readonly msgKind: string } + | { readonly op: "microphone_devices_changed"; readonly msgKind: string } | { readonly op: "batch"; readonly subs: readonly Sub[] }; export const Sub = { @@ -1388,6 +1499,10 @@ export const Sub = { return { op: "timer", key, everyMs, msgKind }; }, + microphoneDevicesChanged(msgKind: EmptyKind): Sub { + return { op: "microphone_devices_changed", msgKind }; + }, + /// Several subscriptions at once. batch(subs: readonly Sub[]): Sub { return { op: "batch", subs }; diff --git a/packages/core/sdk/events.ts b/packages/core/sdk/events.ts index 078240897..2deaea999 100644 --- a/packages/core/sdk/events.ts +++ b/packages/core/sdk/events.ts @@ -181,3 +181,34 @@ export interface AudioEvent { readonly buffering: boolean; readonly bands: Uint8Array; } + +export type AudioCaptureState = "started" | "stopped" | "failed" | "rejected"; +export type AudioCaptureReason = "none" | "invalid_options" | "permission_missing" | "permission_required" | "already_recording" | "device_not_found" | "device_disconnected" | "output_exists" | "io_failed" | "capture_failed" | "no_audio" | "unsupported"; +export interface AudioCaptureEvent { + readonly key: string; + readonly state: AudioCaptureState; + readonly reason: AudioCaptureReason; + readonly durationMs: number; + readonly bytesWritten: number; + readonly outputCommitted: boolean; +} + +export type MicrophoneDeviceState = "device" | "completed" | "failed" | "rejected"; +export interface MicrophoneDeviceEvent { + readonly key: string; + readonly state: MicrophoneDeviceState; + readonly id: Uint8Array; + readonly name: Uint8Array; + readonly isDefault: boolean; + readonly index: number; + readonly total: number; +} + +export type AudioCaptureAccessSource = "system_audio" | "microphone"; +export type AudioCaptureAccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; +export interface AudioCaptureAccessEvent { + readonly key: string; + readonly source: AudioCaptureAccessSource; + readonly status: AudioCaptureAccessStatus; + readonly restartRequired: boolean; +} diff --git a/packages/core/src/emitter.ts b/packages/core/src/emitter.ts index fce7f3210..11af7f6e1 100644 --- a/packages/core/src/emitter.ts +++ b/packages/core/src/emitter.ts @@ -14,7 +14,7 @@ import path from "node:path"; import { ts, TypedAst, hasExportModifier, exportListBindings, sdkCoreModulePath } from "./typed_ast.ts"; -import { TypeTable, snakeCase, zigDeclName, zigLocalName, isZigPrimitiveName, isStaticMember, mangleZType, type ZType, type ZField, type UnionInfo, type StructInfo, type ClassInfo } from "./types.ts"; +import { TypeTable, snakeCase, zigDeclName, zigLocalName, isZigPrimitiveName, isStaticMember, mangleZType, type ZType, type ZField, type UnionArm, type UnionInfo, type StructInfo, type ClassInfo } from "./types.ts"; import { IntInference, returnExpressionsOf } from "./infer.ts"; import { thrownShapeOf, thrownArmsOfShape, THROWN_UNION_NAME, type CheckResult } from "./checker.ts"; import type { RuleId } from "./diagnostics.ts"; @@ -232,6 +232,11 @@ const FETCH_METHODS = ["GET", "POST", "PUT", "DELETE", "PATCH", "HEAD"]; /// engine's event vocabulary, matched by member NAME (declaration order is /// the app's own). const AUDIO_STATES = ["loaded", "position", "completed", "failed", "rejected", "spectrum"]; +const AUDIO_CAPTURE_STATES = ["started", "stopped", "failed", "rejected"]; +const AUDIO_CAPTURE_REASONS = ["none", "invalid_options", "permission_missing", "permission_required", "already_recording", "device_not_found", "device_disconnected", "output_exists", "io_failed", "capture_failed", "no_audio", "unsupported"]; +const MICROPHONE_DEVICE_STATES = ["device", "completed", "failed", "rejected"]; +const AUDIO_CAPTURE_ACCESS_SOURCES = ["system_audio", "microphone"]; +const AUDIO_CAPTURE_ACCESS_STATUSES = ["authorized", "not_authorized", "not_determined", "denied", "restricted", "unavailable"]; /// The video event states an event arm's `state` union must carry — the /// engine's event vocabulary, matched by member NAME (declaration order is @@ -2456,6 +2461,13 @@ export class Emitter { if (method === "audioPlay") { return this.emitAudioPlayCmd(e, ctx); } + if (method === "audioCaptureStart") return this.emitAudioCaptureStartCmd(e, ctx); + if (method === "audioCaptureStop") { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureStop"); + return `rt.cmdAudioCaptureStop("${escapeZigString(key)}")`; + } + if (method === "microphoneDevices") return this.emitMicrophoneDevicesCmd(e, ctx); + if (method === "audioCaptureAccess") return this.emitAudioCaptureAccessCmd(e, ctx); if (method === "imageLoad") { return this.emitImageLoadCmd(e, ctx); } @@ -2570,7 +2582,7 @@ export class Emitter { } this.fail( e, - `Cmd.${method} (the v3 command set is none, persist, now, host, request, cancel, readFile, writeFile, fetch, clipboardWrite, clipboardRead, delay, spawn, audioPlay, audioPause, audioResume, audioStop, audioSeek, audioSetVolume, videoLoad, videoPlay, videoPause, videoStop, videoSeek, videoSetVolume, videoSetMuted, videoSetLoop, showWindow, quitApp, imageLoad, imageCancel, imageUnregister, channelOpen, channelClose, ptySpawn, ptyWrite, ptyResize, ptyKill, batch)`, + `Cmd.${method} (the v4 command set is none, persist, now, host, request, cancel, readFile, writeFile, fetch, clipboardWrite, clipboardRead, delay, spawn, audioPlay, audioPause, audioResume, audioStop, audioSeek, audioSetVolume, videoLoad, videoPlay, videoPause, videoStop, videoSeek, videoSetVolume, videoSetMuted, videoSetLoop, showWindow, quitApp, imageLoad, imageCancel, imageUnregister, channelOpen, channelClose, ptySpawn, ptyWrite, ptyResize, ptyKill, audioCaptureStart, audioCaptureStop, microphoneDevices, audioCaptureAccess, batch)`, ); } this.fail(expr, "command expression (Cmd values are built inline from the Cmd.* factories)"); @@ -2847,6 +2859,135 @@ export class Emitter { return `rt.cmdAudioPlay("${escapeZigString(keyArg.text)}", ${tag}, ${audio_path}, ${url}, ${cache_path}, ${expected})`; } + private literalEffectKey(call: ts.CallExpression, arg: ts.Expression | undefined, factory: string): string { + if (!arg || !ts.isStringLiteral(arg)) this.fail(call, `\`${factory}\` takes its key as a string literal`, "NS1027"); + if (utf8ByteLength(arg.text) > 255) this.fail(arg, `${factory} key over 255 bytes`); + return arg.text; + } + + private eventRouteArm(call: ts.CallExpression, arg: ts.Expression | undefined, factory: string): ts.StringLiteral { + let route = arg; + while (route && (ts.isParenthesizedExpression(route) || ts.isAsExpression(route) || ts.isSatisfiesExpression(route))) route = route.expression; + if (!route || !ts.isObjectLiteralExpression(route)) this.fail(arg ?? call, `\`${factory}\` routing is an inline \`{ event }\` object`, "NS1027"); + let event: ts.StringLiteral | null = null; + for (const p of route.properties) { + if (!ts.isPropertyAssignment(p) || !ts.isIdentifier(p.name) || p.name.text !== "event") this.fail(p, `\`${factory}\` routing accepts only \`event\``, "NS1027"); + let value: ts.Expression = p.initializer; + while (ts.isParenthesizedExpression(value) || ts.isAsExpression(value) || ts.isSatisfiesExpression(value)) value = value.expression; + if (!ts.isStringLiteral(value)) this.fail(value, `\`${factory}\` event arm is not a string literal`, "NS1027"); + event = value; + } + if (!event) this.fail(route, `\`${factory}\` routing without an \`event\` arm`, "NS1027"); + return event; + } + + private emitAudioCaptureStartCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureStart"); + let options = e.arguments[1]; + while (options && (ts.isParenthesizedExpression(options) || ts.isAsExpression(options) || ts.isSatisfiesExpression(options))) options = options.expression; + if (!options || !ts.isObjectLiteralExpression(options)) this.fail(e.arguments[1] ?? e, `\`Cmd.audioCaptureStart\` options must be an inline object`, "NS1029"); + let path: string | null = null; + let systemAudio = false; + let microphoneKind = 0; + let microphoneID = '""'; + let sampleRate = 48000; + let channels = 2; + let exclude = true; + for (const p of options.properties) { + if (!ts.isPropertyAssignment(p) || !ts.isIdentifier(p.name)) this.fail(p, `invalid Cmd.audioCaptureStart option`, "NS1029"); + const name = p.name.text; + let value: ts.Expression = p.initializer; + while (ts.isParenthesizedExpression(value) || ts.isAsExpression(value) || ts.isSatisfiesExpression(value)) value = value.expression; + if (name === "path") path = this.effectBytesArg(e, p.initializer, "Cmd.audioCaptureStart path", MAX_AUDIO_PATH_BYTES, ctx); + else if (name === "systemAudio" || name === "excludeCurrentProcessAudio") { + if (value.kind !== ts.SyntaxKind.TrueKeyword && value.kind !== ts.SyntaxKind.FalseKeyword) this.fail(value, `Cmd.audioCaptureStart ${name} must be a boolean literal`, "NS1030"); + const on = value.kind === ts.SyntaxKind.TrueKeyword; + if (name === "systemAudio") systemAudio = on; else exclude = on; + } else if (name === "microphone") { + if (ts.isStringLiteral(value)) { + if (value.text === "none") microphoneKind = 0; + else if (value.text === "default") microphoneKind = 1; + else this.fail(value, `Cmd.audioCaptureStart microphone string must be "none" or "default"`, "NS1030"); + } else { + microphoneKind = 2; + microphoneID = this.effectBytesArg(e, p.initializer, "Cmd.audioCaptureStart microphone device id", 512, ctx); + } + } else if (name === "sampleRate") { + const literal = this.numberLiteralValue(value); + if (literal === null || ![16000, 24000, 44100, 48000].includes(literal)) this.fail(value, `Cmd.audioCaptureStart sampleRate must be 16000, 24000, 44100, or 48000`, "NS1030"); + sampleRate = literal; + } else if (name === "channels") { + const literal = this.numberLiteralValue(value); + if (literal !== 1 && literal !== 2) this.fail(value, `Cmd.audioCaptureStart channels must be 1 or 2`, "NS1030"); + channels = literal; + } else this.fail(p, `unknown Cmd.audioCaptureStart option \`${name}\``, "NS1029"); + } + if (path === null) this.fail(options, `Cmd.audioCaptureStart requires path`, "NS1029"); + if (!systemAudio && microphoneKind === 0) this.fail(options, `Cmd.audioCaptureStart must enable systemAudio and/or microphone`, "NS1030"); + const event = this.eventRouteArm(e, e.arguments[2], "Cmd.audioCaptureStart"); + const tag = this.audioCaptureEventArmTag(event, ctx); + return `rt.cmdAudioCaptureStart("${escapeZigString(key)}", ${tag}, ${path}, ${systemAudio}, ${microphoneKind}, ${microphoneID}, ${sampleRate}, ${channels}, ${exclude})`; + } + + private emitMicrophoneDevicesCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.microphoneDevices"); + const event = this.eventRouteArm(e, e.arguments[1], "Cmd.microphoneDevices"); + return `rt.cmdMicrophoneDevices("${escapeZigString(key)}", ${this.microphoneDeviceEventArmTag(event, ctx)})`; + } + + private emitAudioCaptureAccessCmd(e: ts.CallExpression, ctx: Ctx): string { + const key = this.literalEffectKey(e, e.arguments[0], "Cmd.audioCaptureAccess"); + const source = e.arguments[1]; + const action = e.arguments[2]; + if (!source || !ts.isStringLiteral(source) || !AUDIO_CAPTURE_ACCESS_SOURCES.includes(source.text)) this.fail(source ?? e, `Cmd.audioCaptureAccess source must be "system_audio" or "microphone"`, "NS1030"); + if (!action || !ts.isStringLiteral(action) || (action.text !== "status" && action.text !== "request")) this.fail(action ?? e, `Cmd.audioCaptureAccess action must be "status" or "request"`, "NS1030"); + const event = this.eventRouteArm(e, e.arguments[3], "Cmd.audioCaptureAccess"); + return `rt.cmdAudioCaptureAccess("${escapeZigString(key)}", ${this.audioCaptureAccessEventArmTag(event, ctx)}, ${source.text === "microphone" ? 1 : 0}, ${action.text === "request" ? 1 : 0})`; + } + + private fixedEventArm(arg: ts.StringLiteral, ctx: Ctx): { arm: UnionArm; unionName: string } { + const unionName = ctx.cmdReturn!.msgUnion; + const info = this.table.unions.get(unionName); + if (!info) this.fail(arg, `unknown union ${unionName}`); + const arm = info.arms.find((candidate) => candidate.tag === arg.text); + if (!arm) this.fail(arg, `routing target \`${arg.text}\` is not an arm of ${unionName}`, "NS1027"); + return { arm, unionName }; + } + + private enumFieldMatches(field: ZField | undefined, members: readonly string[]): boolean { + return field !== undefined && field.type.k === "enum" && field.type.members.length === members.length && members.every((member) => field.type.k === "enum" && field.type.members.includes(member)); + } + + private audioCaptureEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const number = (name: string) => ["number", "i64", "f64"].includes(fields.get(name)?.type.k ?? ""); + const ok = arm.fields.length === 6 && fields.get("key")?.type.k === "string" && + this.enumFieldMatches(fields.get("state"), AUDIO_CAPTURE_STATES) && this.enumFieldMatches(fields.get("reason"), AUDIO_CAPTURE_REASONS) && + number("durationMs") && number("bytesWritten") && fields.get("outputCommitted")?.type.k === "bool"; + if (!ok) this.fail(arg, `audio capture event arm must carry key, state, reason, durationMs, bytesWritten, and outputCommitted`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + + private microphoneDeviceEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const number = (name: string) => ["number", "i64", "f64"].includes(fields.get(name)?.type.k ?? ""); + const ok = arm.fields.length === 7 && fields.get("key")?.type.k === "string" && this.enumFieldMatches(fields.get("state"), MICROPHONE_DEVICE_STATES) && + fields.get("id")?.type.k === "bytes" && fields.get("name")?.type.k === "bytes" && fields.get("isDefault")?.type.k === "bool" && number("index") && number("total"); + if (!ok) this.fail(arg, `microphone device event arm must carry key, state, id, name, isDefault, index, and total`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + + private audioCaptureAccessEventArmTag(arg: ts.StringLiteral, ctx: Ctx): string { + const { arm, unionName } = this.fixedEventArm(arg, ctx); + const fields = new Map(arm.fields.map((f) => [f.tsName, f])); + const ok = arm.fields.length === 4 && fields.get("key")?.type.k === "string" && this.enumFieldMatches(fields.get("source"), AUDIO_CAPTURE_ACCESS_SOURCES) && + this.enumFieldMatches(fields.get("status"), AUDIO_CAPTURE_ACCESS_STATUSES) && fields.get("restartRequired")?.type.k === "bool"; + if (!ok) this.fail(arg, `audio capture access event arm must carry key, source, status, and restartRequired`, "NS1027"); + return `@intFromEnum(std.meta.Tag(${unionName}).${zigId(arg.text)})`; + } + /// `Cmd.imageLoad(id, source, route)`: the app's numeric ImageId (any /// number expression — ids are model data), an inline /// `{ path?, url?, cachePath?, expectedBytes? }` source (at least one of @@ -3864,6 +4005,14 @@ export class Emitter { } return `rt.subTimer("${escapeZigString(keyArg.text)}", ${every}, @intFromEnum(std.meta.Tag(${unionName}).${zigId(kindArg.text)}))`; } + if (method === "microphoneDevicesChanged") { + const kindArg = e.arguments[0]; + if (!kindArg || !ts.isStringLiteral(kindArg)) this.fail(e, `Sub.microphoneDevicesChanged takes its target Msg kind as a string literal`, "NS1027"); + const unionName = ctx.subReturn!.msgUnion; + const arm = this.table.unions.get(unionName)?.arms.find((candidate) => candidate.tag === kindArg.text); + if (!arm || arm.fields.length !== 0) this.fail(kindArg, `microphone device change target must carry no payload fields`, "NS1027"); + return `rt.subMicrophoneDevicesChanged(@intFromEnum(std.meta.Tag(${unionName}).${zigId(kindArg.text)}))`; + } if (method === "batch") { const arr = e.arguments[0]; if (!arr || !ts.isArrayLiteralExpression(arr)) this.fail(e, "Sub.batch argument (an array literal)"); @@ -3871,7 +4020,7 @@ export class Emitter { if (parts.length === 0) return "rt.sub_none"; return `rt.subBatch(&.{ ${parts.join(", ")} })`; } - this.fail(e, `Sub.${method} (the subscription set is none, timer, batch)`); + this.fail(e, `Sub.${method} (the subscription set is none, timer, microphoneDevicesChanged, batch)`); } this.fail(expr, "subscription expression (Sub values are built inline from the Sub.* factories)"); } diff --git a/packages/core/test/effects.test.ts b/packages/core/test/effects.test.ts index aff6b8561..cde524ddb 100644 --- a/packages/core/test/effects.test.ts +++ b/packages/core/test/effects.test.ts @@ -72,9 +72,9 @@ fn dispatch(msg: core.Msg) []const u8 { } test "cmd bytes flow through update -> commit -> effect log" { - // v3 is additive: the v1/v2 op records asserted below are + // v4 is additive: the v1/v2/v3 op records asserted below are // byte-identical under the bumped version. - try std.testing.expectEqual(@as(u32, 3), rt.cmd_format_version); + try std.testing.expectEqual(@as(u32, 4), rt.cmd_format_version); rt.resetAll(); g_model = core.commitModelRoot(core.initialModel()); @@ -237,7 +237,7 @@ fn expectRecord(bytes: []const u8, expected: []const u8) !void { } test "v2 wire: init command, payloads, routing, cancel, subscriptions" { - try std.testing.expectEqual(@as(u32, 3), rt.cmd_format_version); + try std.testing.expectEqual(@as(u32, 4), rt.cmd_format_version); var log: [512]u8 = undefined; @@ -409,7 +409,7 @@ fn appendLong(list: *std.ArrayList(u8), a: std.mem.Allocator, bytes: []const u8) } test "named-op wire records match the documented additive layout" { - try std.testing.expectEqual(@as(u32, 3), rt.cmd_format_version); + try std.testing.expectEqual(@as(u32, 4), rt.cmd_format_version); const a = std.testing.allocator; var log: [512]u8 = undefined; @@ -1217,3 +1217,120 @@ test("channels: wire bytes through the real dispatch cycle", { skip: !hasZig, ti fs.rmSync(work, { recursive: true, force: true }); } }); + +// ---------------------------------------------------------------- audio capture v4 + +const coreAudioCapture = ` +import { Cmd, Sub, asciiBytes } from "@native-sdk/core"; + +type CaptureState = "started" | "stopped" | "failed" | "rejected"; +type CaptureReason = "none" | "invalid_options" | "permission_missing" | "permission_required" | "already_recording" | "device_not_found" | "device_disconnected" | "output_exists" | "io_failed" | "capture_failed" | "no_audio" | "unsupported"; +type DeviceState = "device" | "completed" | "failed" | "rejected"; +type AccessSource = "system_audio" | "microphone"; +type AccessStatus = "authorized" | "not_authorized" | "not_determined" | "denied" | "restricted" | "unavailable"; + +export interface Model { readonly events: number; } +export type Msg = + | { readonly kind: "start" } + | { readonly kind: "stop" } + | { readonly kind: "list" } + | { readonly kind: "access" } + | { readonly kind: "capture_event"; readonly key: string; readonly state: CaptureState; readonly reason: CaptureReason; readonly durationMs: number; readonly bytesWritten: number; readonly outputCommitted: boolean } + | { readonly kind: "device_event"; readonly key: string; readonly state: DeviceState; readonly id: Uint8Array; readonly name: Uint8Array; readonly isDefault: boolean; readonly index: number; readonly total: number } + | { readonly kind: "access_event"; readonly key: string; readonly source: AccessSource; readonly status: AccessStatus; readonly restartRequired: boolean } + | { readonly kind: "devices_changed" }; + +export function initialModel(): Model { return { events: 0 }; } + +export function update(model: Model, msg: Msg): Model | [Model, Cmd] { + switch (msg.kind) { + case "start": return [model, Cmd.audioCaptureStart("capture", { path: asciiBytes("out.wav"), systemAudio: true, microphone: asciiBytes("usb"), sampleRate: 44100, channels: 1 }, { event: "capture_event" })]; + case "stop": return [model, Cmd.audioCaptureStop("capture")]; + case "list": return [model, Cmd.microphoneDevices("list", { event: "device_event" })]; + case "access": return [model, Cmd.audioCaptureAccess("access", "microphone", "request", { event: "access_event" })]; + case "capture_event": case "device_event": case "access_event": case "devices_changed": return { events: model.events + 1 }; + } +} + +export function subscriptions(model: Model): Sub { + return Sub.microphoneDevicesChanged("devices_changed"); +} +`; + +const harnessAudioCapture = ` +const std = @import("std"); +const core = @import("core.zig"); +const rt = core.rt; + +var g_model: *const core.Model = undefined; + +fn dispatch(msg: core.Msg, log: []u8) []const u8 { + const r = core.update(g_model, msg); + g_model = core.commitModelRoot(r.model); + @memcpy(log[0..r.cmd.len], r.cmd); + const out = log[0..r.cmd.len]; + rt.frameReset(); + return out; +} + +fn expectLong(bytes: []const u8, at: *usize, expected: []const u8) !void { + const len = std.mem.readInt(u32, bytes[at.*..][0..4], .little); + try std.testing.expectEqual(@as(u32, @intCast(expected.len)), len); + try std.testing.expectEqualStrings(expected, bytes[at.* + 4 ..][0..expected.len]); + at.* += 4 + expected.len; +} + +test "audio capture command and subscription wire records" { + try std.testing.expectEqual(@as(u32, 4), rt.cmd_format_version); + var log: [512]u8 = undefined; + rt.resetAll(); + g_model = core.commitModelRoot(core.initialModel()); + rt.frameReset(); + + const start = dispatch(.start, &log); + try std.testing.expectEqual(@as(u8, 0x1D), start[0]); + try std.testing.expectEqual(@as(u8, 7), start[1]); + try std.testing.expectEqualStrings("capture", start[2..9]); + try std.testing.expectEqual(@as(u8, @intFromEnum(std.meta.Tag(core.Msg).capture_event)), start[9]); + try std.testing.expectEqual(@as(u8, 3), start[10]); // system audio + exclude current process + try std.testing.expectEqual(@as(u8, 2), start[11]); // explicit microphone id + try std.testing.expectEqual(@as(u32, 44100), std.mem.readInt(u32, start[12..16], .little)); + try std.testing.expectEqual(@as(u8, 1), start[16]); + var at: usize = 17; + try expectLong(start, &at, "out.wav"); + try expectLong(start, &at, "usb"); + try std.testing.expectEqual(start.len, at); + + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x1E, 7 } ++ "capture", dispatch(.stop, &log)); + + const devices = dispatch(.list, &log); + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x1F, 4 } ++ "list" ++ .{@intFromEnum(std.meta.Tag(core.Msg).device_event)}, devices); + + const access = dispatch(.access, &log); + try std.testing.expectEqualSlices(u8, &[_]u8{ 0x20, 6 } ++ "access" ++ .{ @intFromEnum(std.meta.Tag(core.Msg).access_event), 1, 1 }, access); + + const subs = core.subscriptions(g_model); + try std.testing.expectEqualSlices(u8, &.{ 0x02, @intFromEnum(std.meta.Tag(core.Msg).devices_changed) }, subs); + rt.frameReset(); +} +`; + +test("audio capture effects: wire bytes through the real dispatch cycle", { skip: !hasZig, timeout: 300_000 }, () => { + const result = transpile(coreAudioCapture); + const details = result.diagnostics.map((d) => `${d.id} ${d.message}`).join("\n"); + assert.equal(result.ok, true, `transpile failed\n${result.typeErrors.join("\n")}\n${details}`); + const work = fs.mkdtempSync(path.join(os.tmpdir(), "native-core-effects-audio-capture-")); + try { + fs.copyFileSync(path.join(pkg, "rt", "rt.zig"), path.join(work, "rt.zig")); + fs.writeFileSync(path.join(work, "core.zig"), result.zig!); + fs.writeFileSync(path.join(work, "harness.zig"), harnessAudioCapture); + try { + execFileSync("zig", ["test", "harness.zig"], { cwd: work, encoding: "utf8", stdio: "pipe" }); + } catch (e) { + const err = e as { stderr?: string; stdout?: string }; + assert.fail(`audio capture harness failed:\n${err.stderr ?? ""}${err.stdout ?? ""}`); + } + } finally { + fs.rmSync(work, { recursive: true, force: true }); + } +}); diff --git a/packages/native-sdk/native-sdk.d.ts b/packages/native-sdk/native-sdk.d.ts index 2338a6bb4..94b1a7143 100644 --- a/packages/native-sdk/native-sdk.d.ts +++ b/packages/native-sdk/native-sdk.d.ts @@ -450,6 +450,12 @@ export type NativeSdkPlatformFeature = | "audioStreaming" | "audio_spectrum" | "audioSpectrum" + | "system_audio_capture" + | "systemAudioCapture" + | "microphone_capture" + | "microphoneCapture" + | "microphone_device_enumeration" + | "microphoneDeviceEnumeration" | "window_hide_on_close" | "windowHideOnClose" | "video_playback" diff --git a/src/app_runner/root.zig b/src/app_runner/root.zig index 261461974..79dee3e98 100644 --- a/src/app_runner/root.zig +++ b/src/app_runner/root.zig @@ -65,6 +65,7 @@ pub const RunOptions = struct { .description = manifestStringField("description"), .has_web_content = manifestHasWebContent(), .declares_tray = manifestDeclaresTrayCapability(), + .permissions = self.security.permissions, .window_title = self.window_title, .bundle_id = self.bundle_id, .icon_path = self.icon_path, diff --git a/src/platform/linux/root.zig b/src/platform/linux/root.zig index fd77d3910..a273a7c83 100644 --- a/src/platform/linux/root.zig +++ b/src/platform/linux/root.zig @@ -456,6 +456,7 @@ pub const LinuxPlatform = struct { // a host whose plugin set lacks it answers false and the // deck's glass rests honestly instead of dancing on fakes. .audio_spectrum => self.web_engine == .system and audioSpectrumAvailable(self.host), + .system_audio_capture, .microphone_capture, .microphone_device_enumeration => false, .tray => false, // No tray means no affordance to bring a policy-hidden // window back — reporting support would strand windows, so diff --git a/src/platform/macos/appkit_host.h b/src/platform/macos/appkit_host.h index af9ddfc2b..ded9163fb 100644 --- a/src/platform/macos/appkit_host.h +++ b/src/platform/macos/appkit_host.h @@ -34,6 +34,10 @@ typedef enum { NATIVE_SDK_APPKIT_EVENT_AUDIO = 20, NATIVE_SDK_APPKIT_EVENT_VIDEO = 21, NATIVE_SDK_APPKIT_EVENT_VIEW_FOCUSED = 22, + NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE = 23, + NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICE = 24, + NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICES_CHANGED = 25, + NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE_ACCESS = 26, } native_sdk_appkit_event_kind_t; /* Audio player reports (EVENT_AUDIO payloads). LOADED acknowledges a @@ -356,6 +360,22 @@ typedef struct { * event kind. */ uint64_t video_width; uint64_t video_height; + int audio_capture_state; + int audio_capture_reason; + uint64_t audio_capture_duration_ms; + uint64_t audio_capture_bytes_written; + int audio_capture_output_committed; + int microphone_device_state; + const char *microphone_device_id; + size_t microphone_device_id_len; + const char *microphone_device_name; + size_t microphone_device_name_len; + int microphone_device_is_default; + uint32_t microphone_device_index; + uint32_t microphone_device_total; + int audio_capture_access_source; + int audio_capture_access_status; + int audio_capture_restart_required; } native_sdk_appkit_event_t; typedef void (*native_sdk_appkit_event_callback_t)(void *context, const native_sdk_appkit_event_t *event); @@ -530,6 +550,11 @@ int native_sdk_appkit_audio_pause(native_sdk_appkit_host_t *host); int native_sdk_appkit_audio_stop(native_sdk_appkit_host_t *host); int native_sdk_appkit_audio_seek(native_sdk_appkit_host_t *host, uint64_t position_ms); int native_sdk_appkit_audio_set_volume(native_sdk_appkit_host_t *host, double volume); +int native_sdk_appkit_audio_capture_start(native_sdk_appkit_host_t *host, const char *path, size_t path_len, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio); +void native_sdk_appkit_audio_capture_stop(native_sdk_appkit_host_t *host); +void native_sdk_appkit_microphone_devices(native_sdk_appkit_host_t *host); +void native_sdk_appkit_audio_capture_access(native_sdk_appkit_host_t *host, int source, int action); +void native_sdk_appkit_observe_microphone_devices(native_sdk_appkit_host_t *host, int enabled); /* Where the video player delivers decoded frames: one tightly packed, * row-major, straight-alpha RGBA8 frame per call (len = width * height diff --git a/src/platform/macos/appkit_host.m b/src/platform/macos/appkit_host.m index 2db878e9f..705bcbc4a 100644 --- a/src/platform/macos/appkit_host.m +++ b/src/platform/macos/appkit_host.m @@ -1,4 +1,5 @@ #import "appkit_host.h" +#import "audio_capture.h" #import #import @@ -25,6 +26,7 @@ #include @class NativeSdkAppKitHost; +static void NativeSdkAudioCaptureCallback(void *context, const native_sdk_audio_capture_event_t *captureEvent); static const NSUInteger NativeSdkMaxChildWebViews = 16; static const NSUInteger NativeSdkMaxNativeViews = 32; @@ -831,6 +833,7 @@ @interface NativeSdkAppKitHost : NSObject * installed by the load entries. Every timer callback runs on the main * run loop, so no locking guards the pair. */ @property(nonatomic, assign) native_sdk_appkit_video_sink_push_t videoSinkPush; +@property(nonatomic, assign) native_sdk_audio_capture_t *audioCapture; @property(nonatomic, assign) void *videoSinkContext; /* The reusable RGBA conversion target: malloc'd once per load to the * output's max frame size, freed on stop/replace. Frames are converted @@ -1077,6 +1080,43 @@ - (BOOL)handleShortcutEvent:(NSEvent *)event; - (void)emitShortcutWithId:(NSString *)identifier key:(NSString *)key modifiers:(uint32_t)modifiers event:(NSEvent *)event; @end +static void NativeSdkAudioCaptureCallback(void *context, const native_sdk_audio_capture_event_t *captureEvent) { + NativeSdkAppKitHost *host = (__bridge NativeSdkAppKitHost *)context; + if (!host || !captureEvent) return; + native_sdk_appkit_event_t event = { .timestamp_ns = NativeSdkTimestampNanoseconds() }; + switch (captureEvent->kind) { + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_CAPTURE: + event.kind = NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE; + event.audio_capture_state = captureEvent->state; + event.audio_capture_reason = captureEvent->reason; + event.audio_capture_duration_ms = captureEvent->duration_ms; + event.audio_capture_bytes_written = captureEvent->bytes_written; + event.audio_capture_output_committed = captureEvent->output_committed; + break; + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE: + event.kind = NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICE; + event.microphone_device_state = captureEvent->state; + event.microphone_device_id = captureEvent->device_id; + event.microphone_device_id_len = captureEvent->device_id_len; + event.microphone_device_name = captureEvent->device_name; + event.microphone_device_name_len = captureEvent->device_name_len; + event.microphone_device_is_default = captureEvent->device_is_default; + event.microphone_device_index = captureEvent->device_index; + event.microphone_device_total = captureEvent->device_total; + break; + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED: + event.kind = NATIVE_SDK_APPKIT_EVENT_MICROPHONE_DEVICES_CHANGED; + break; + case NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS: + event.kind = NATIVE_SDK_APPKIT_EVENT_AUDIO_CAPTURE_ACCESS; + event.audio_capture_access_source = captureEvent->access_source; + event.audio_capture_access_status = captureEvent->access_status; + event.audio_capture_restart_required = captureEvent->restart_required; + break; + } + [host emitEvent:event]; +} + // Recursively re-emit the gpu-surface resize event for every metal // surface under `view` (the tall-titlebar chrome re-query path). static void NativeSdkEmitGpuSurfaceResizes(NSView *view) { @@ -7254,6 +7294,7 @@ - (instancetype)initWithAppName:(NSString *)appName displayName:(NSString *)disp self.allowedExternalURLs = @[]; self.externalLinkAction = 0; self.shortcuts = @[]; + self.audioCapture = native_sdk_audio_capture_create(NativeSdkAudioCaptureCallback, (__bridge void *)self); [self configureApplication]; NativeSdkLaunchLap("app_configured"); @@ -11576,6 +11617,11 @@ void native_sdk_appkit_destroy(native_sdk_appkit_host_t *host) { if (!host) { return; } + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + if (object.audioCapture) { + native_sdk_audio_capture_destroy(object.audioCapture); + object.audioCapture = NULL; + } CFBridgingRelease(host); } @@ -11689,6 +11735,32 @@ int native_sdk_appkit_audio_set_volume(native_sdk_appkit_host_t *host, double vo return [object audioSetVolume:volume]; } +int native_sdk_appkit_audio_capture_start(native_sdk_appkit_host_t *host, const char *path, size_t path_len, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + if (!object.audioCapture) return 6; + return native_sdk_audio_capture_start(object.audioCapture, path, path_len, system_audio, microphone_kind, microphone_id, microphone_id_len, sample_rate_hz, channel_count, exclude_current_process_audio); +} + +void native_sdk_appkit_audio_capture_stop(native_sdk_appkit_host_t *host) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_stop(object.audioCapture); +} + +void native_sdk_appkit_microphone_devices(native_sdk_appkit_host_t *host) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_list_microphones(object.audioCapture); +} + +void native_sdk_appkit_audio_capture_access(native_sdk_appkit_host_t *host, int source, int action) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_access(object.audioCapture, source, action); +} + +void native_sdk_appkit_observe_microphone_devices(native_sdk_appkit_host_t *host, int enabled) { + NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; + native_sdk_audio_capture_observe_microphones(object.audioCapture, enabled); +} + int native_sdk_appkit_video_load(native_sdk_appkit_host_t *host, const char *path, size_t path_len, uint64_t token, native_sdk_appkit_video_sink_push_t push_fn, void *push_context) { NativeSdkAppKitHost *object = (__bridge NativeSdkAppKitHost *)host; NSString *path_string = [[NSString alloc] initWithBytes:path length:path_len encoding:NSUTF8StringEncoding]; diff --git a/src/platform/macos/audio_capture.h b/src/platform/macos/audio_capture.h new file mode 100644 index 000000000..fa739b513 --- /dev/null +++ b/src/platform/macos/audio_capture.h @@ -0,0 +1,54 @@ +#ifndef NATIVE_SDK_AUDIO_CAPTURE_H +#define NATIVE_SDK_AUDIO_CAPTURE_H + +#include +#include + +typedef struct native_sdk_audio_capture native_sdk_audio_capture_t; + +typedef enum { + NATIVE_SDK_AUDIO_CAPTURE_EVENT_CAPTURE = 0, + NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE = 1, + NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED = 2, + NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS = 3, +} native_sdk_audio_capture_event_kind_t; + +typedef struct { + native_sdk_audio_capture_event_kind_t kind; + int state; + int reason; + uint64_t duration_ms; + uint64_t bytes_written; + int output_committed; + const char *device_id; + size_t device_id_len; + const char *device_name; + size_t device_name_len; + int device_is_default; + uint32_t device_index; + uint32_t device_total; + int access_source; + int access_status; + int restart_required; +} native_sdk_audio_capture_event_t; + +typedef void (*native_sdk_audio_capture_callback_t)(void *context, const native_sdk_audio_capture_event_t *event); + +native_sdk_audio_capture_t *native_sdk_audio_capture_create(native_sdk_audio_capture_callback_t callback, void *context); +void native_sdk_audio_capture_destroy(native_sdk_audio_capture_t *capture); + +/* Start results: 0 accepted; 1 invalid options; 2 already active; + * 3 destination exists; 4 permission required; 5 device missing; + * 6 unavailable; 7 I/O failure. */ +int native_sdk_audio_capture_start(native_sdk_audio_capture_t *capture, + const char *path, size_t path_len, + int system_audio, int microphone_kind, + const char *microphone_id, size_t microphone_id_len, + uint32_t sample_rate_hz, uint8_t channel_count, + int exclude_current_process_audio); +void native_sdk_audio_capture_stop(native_sdk_audio_capture_t *capture); +void native_sdk_audio_capture_list_microphones(native_sdk_audio_capture_t *capture); +void native_sdk_audio_capture_access(native_sdk_audio_capture_t *capture, int source, int action); +void native_sdk_audio_capture_observe_microphones(native_sdk_audio_capture_t *capture, int enabled); + +#endif diff --git a/src/platform/macos/audio_capture.m b/src/platform/macos/audio_capture.m new file mode 100644 index 000000000..b0c15d9ec --- /dev/null +++ b/src/platform/macos/audio_capture.m @@ -0,0 +1,553 @@ +#import "audio_capture.h" + +/* Zig 0.16 diagnoses inconsistencies in older Apple SDK umbrella headers as + * errors. Keep those SDK-owned diagnostics isolated from this translation + * unit without suppressing warnings in the implementation below. */ +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Weverything" +#import +#import +#import +#import +#import +#pragma clang diagnostic pop +#include +#include +#include +#include +#include +#include +#include + +/* These ordinals intentionally mirror platform/types.zig. */ +enum { NS_CAPTURE_STARTED = 0, NS_CAPTURE_STOPPED = 1, NS_CAPTURE_FAILED = 2, NS_CAPTURE_REJECTED = 3 }; +enum { + NS_REASON_NONE = 0, NS_REASON_INVALID_OPTIONS = 1, NS_REASON_PERMISSION_MISSING = 2, + NS_REASON_PERMISSION_REQUIRED = 3, NS_REASON_ALREADY_RECORDING = 4, + NS_REASON_DEVICE_NOT_FOUND = 5, NS_REASON_DEVICE_DISCONNECTED = 6, + NS_REASON_OUTPUT_EXISTS = 7, NS_REASON_IO_FAILED = 8, NS_REASON_CAPTURE_FAILED = 9, + NS_REASON_NO_AUDIO = 10, NS_REASON_UNSUPPORTED = 11, +}; +enum { NS_DEVICE = 0, NS_DEVICES_COMPLETED = 1, NS_DEVICES_FAILED = 2, NS_DEVICES_REJECTED = 3 }; +enum { + NS_ACCESS_AUTHORIZED = 0, NS_ACCESS_NOT_AUTHORIZED = 1, NS_ACCESS_NOT_DETERMINED = 2, + NS_ACCESS_DENIED = 3, NS_ACCESS_RESTRICTED = 4, NS_ACCESS_UNAVAILABLE = 5, +}; + +API_AVAILABLE(macos(15.0)) +@interface NativeSdkAudioCapture : NSObject +@property(nonatomic, assign) native_sdk_audio_capture_callback_t callback; +@property(nonatomic, assign) void *callbackContext; +@property(nonatomic, strong) dispatch_queue_t sampleQueue; +@property(nonatomic, strong) SCStream *screenStream; +@property(nonatomic, strong) AVCaptureSession *microphoneSession; +@property(nonatomic, strong) AVCaptureAudioDataOutput *microphoneOutput; +@property(nonatomic, strong) NSString *selectedMicrophoneID; +@property(nonatomic, strong) NSString *destinationPath; +@property(nonatomic, strong) NSString *wavTemporaryPath; +@property(nonatomic, strong) NSString *mixTemporaryPath; +@property(nonatomic, assign) int mixFD; +@property(nonatomic, assign) uint32_t sampleRate; +@property(nonatomic, assign) uint8_t channelCount; +@property(nonatomic, assign) BOOL combined; +@property(nonatomic, assign) BOOL active; +@property(nonatomic, assign) BOOL terminalEmitted; +@property(nonatomic, assign) BOOL observingDevices; +@property(nonatomic, assign) BOOL hasBasePTS; +@property(nonatomic, assign) CMTime basePTS; +@property(nonatomic, assign) uint64_t maxFrame; +@property(nonatomic, assign) uint64_t sampleBufferCount; +@property(nonatomic, assign) int publishFailureReason; +@property(nonatomic, strong) id connectedObserver; +@property(nonatomic, strong) id disconnectedObserver; +@property(nonatomic, strong) dispatch_semaphore_t finalizationSemaphore; +- (int)startPath:(NSString *)path systemAudio:(BOOL)systemAudio microphoneKind:(int)microphoneKind microphoneID:(NSString *)microphoneID sampleRate:(uint32_t)sampleRate channels:(uint8_t)channels excludeCurrentProcessAudio:(BOOL)exclude; +- (void)stopCapture; +- (void)finishWithState:(int)state reason:(int)reason; +@end + +static void NativeSdkWriteLE16(uint8_t *out, uint16_t value) { + out[0] = (uint8_t)(value & 0xff); out[1] = (uint8_t)(value >> 8); +} +static void NativeSdkWriteLE32(uint8_t *out, uint32_t value) { + out[0] = (uint8_t)(value & 0xff); out[1] = (uint8_t)((value >> 8) & 0xff); + out[2] = (uint8_t)((value >> 16) & 0xff); out[3] = (uint8_t)(value >> 24); +} + +static NSArray *NativeSdkMicrophones(void) API_AVAILABLE(macos(15.0)); +static NSArray *NativeSdkMicrophones(void) { + AVCaptureDeviceDiscoverySession *session = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:@[ AVCaptureDeviceTypeMicrophone ] + mediaType:AVMediaTypeAudio position:AVCaptureDevicePositionUnspecified]; + return session.devices; +} + +static AVCaptureDevice *NativeSdkMicrophone(NSString *identifier, BOOL useDefault) API_AVAILABLE(macos(15.0)); +static AVCaptureDevice *NativeSdkMicrophone(NSString *identifier, BOOL useDefault) { + if (useDefault) return [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + for (AVCaptureDevice *device in NativeSdkMicrophones()) { + if ([device.uniqueID isEqualToString:identifier]) return device; + } + return nil; +} + +/* Xcode 15's macOS 14 SDK does not declare ScreenCaptureKit's macOS 15 + * microphone additions. Resolve the setters dynamically so applications can + * still build with that SDK while using the capability when running on 15+. */ +static BOOL NativeSdkConfigureScreenCaptureMicrophone(SCStreamConfiguration *configuration, NSString *deviceID) API_AVAILABLE(macos(15.0)); +static BOOL NativeSdkConfigureScreenCaptureMicrophone(SCStreamConfiguration *configuration, NSString *deviceID) { + SEL captureSelector = NSSelectorFromString(@"setCaptureMicrophone:"); + SEL deviceSelector = NSSelectorFromString(@"setMicrophoneCaptureDeviceID:"); + if (![configuration respondsToSelector:captureSelector] || ![configuration respondsToSelector:deviceSelector]) return NO; + [configuration setValue:@YES forKey:@"captureMicrophone"]; + [configuration setValue:deviceID forKey:@"microphoneCaptureDeviceID"]; + return YES; +} + +/* SCStreamOutputType is an ABI-stable NS_ENUM: screen=0, audio=1, and the + * microphone case added in macOS 15 is 2. */ +static SCStreamOutputType NativeSdkMicrophoneOutputType(void) API_AVAILABLE(macos(15.0)); +static SCStreamOutputType NativeSdkMicrophoneOutputType(void) { return (SCStreamOutputType)2; } + +@implementation NativeSdkAudioCapture + +- (instancetype)initWithCallback:(native_sdk_audio_capture_callback_t)callback context:(void *)context { + self = [super init]; + if (!self) return nil; + _callback = callback; + _callbackContext = context; + _sampleQueue = dispatch_queue_create("dev.native-sdk.audio-capture", DISPATCH_QUEUE_SERIAL); + _mixFD = -1; + return self; +} + +- (void)dealloc { + [self stopDeviceObservers]; + if (_screenStream) [_screenStream stopCaptureWithCompletionHandler:nil]; + if (_microphoneSession.running) [_microphoneSession stopRunning]; + if (_mixFD >= 0) close(_mixFD); +} + +- (void)emit:(native_sdk_audio_capture_event_t)event { + if ([NSThread isMainThread]) { + native_sdk_audio_capture_callback_t callback = self.callback; + if (callback) callback(self.callbackContext, &event); + return; + } + dispatch_async(dispatch_get_main_queue(), ^{ + native_sdk_audio_capture_callback_t callback = self.callback; + if (callback) callback(self.callbackContext, &event); + }); +} + +- (void)emitCaptureState:(int)state reason:(int)reason duration:(uint64_t)duration bytes:(uint64_t)bytes committed:(BOOL)committed { + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_CAPTURE, + .state = state, .reason = reason, .duration_ms = duration, + .bytes_written = bytes, .output_committed = committed ? 1 : 0 }; + [self emit:event]; +} + +- (void)startDeviceObservers { + if (self.connectedObserver || self.disconnectedObserver) return; + __weak NativeSdkAudioCapture *weakSelf = self; + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + self.connectedObserver = [center addObserverForName:AVCaptureDeviceWasConnectedNotification object:nil queue:[NSOperationQueue mainQueue] usingBlock:^(NSNotification *note) { + (void)note; NativeSdkAudioCapture *strongSelf = weakSelf; if (!strongSelf) return; + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED }; + [strongSelf emit:event]; + }]; + self.disconnectedObserver = [center addObserverForName:AVCaptureDeviceWasDisconnectedNotification object:nil queue:[NSOperationQueue mainQueue] usingBlock:^(NSNotification *note) { + NativeSdkAudioCapture *strongSelf = weakSelf; if (!strongSelf) return; + AVCaptureDevice *device = note.object; + if (strongSelf.active && strongSelf.selectedMicrophoneID.length > 0 && [device.uniqueID isEqualToString:strongSelf.selectedMicrophoneID]) { + [strongSelf finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_DEVICE_DISCONNECTED]; + } + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICES_CHANGED }; + [strongSelf emit:event]; + }]; +} + +- (void)stopDeviceObservers { + NSNotificationCenter *center = [NSNotificationCenter defaultCenter]; + if (self.connectedObserver) [center removeObserver:self.connectedObserver]; + if (self.disconnectedObserver) [center removeObserver:self.disconnectedObserver]; + self.connectedObserver = nil; self.disconnectedObserver = nil; +} + +- (int)prepareFilesAtPath:(NSString *)path { + if ([[NSFileManager defaultManager] fileExistsAtPath:path]) return 3; + NSString *directory = [path stringByDeletingLastPathComponent]; + if (directory.length == 0) directory = @"."; + BOOL isDirectory = NO; + if (![[NSFileManager defaultManager] fileExistsAtPath:directory isDirectory:&isDirectory] || !isDirectory) return 7; + NSString *nonce = NSUUID.UUID.UUIDString; + self.wavTemporaryPath = [directory stringByAppendingPathComponent:[NSString stringWithFormat:@".%@.%@.tmp", path.lastPathComponent, nonce]]; + self.mixTemporaryPath = [directory stringByAppendingPathComponent:[NSString stringWithFormat:@".%@.%@.mix", path.lastPathComponent, nonce]]; + self.mixFD = open(self.mixTemporaryPath.fileSystemRepresentation, O_CREAT | O_EXCL | O_RDWR, 0600); + if (self.mixFD < 0) return 7; + return 0; +} + +- (int)startPath:(NSString *)path systemAudio:(BOOL)systemAudio microphoneKind:(int)microphoneKind microphoneID:(NSString *)microphoneID sampleRate:(uint32_t)sampleRate channels:(uint8_t)channels excludeCurrentProcessAudio:(BOOL)exclude { + if (self.active) return 2; + if (@available(macOS 15.0, *)) {} else { return 6; } + if (path.length == 0 || (!systemAudio && microphoneKind == 0) || (channels != 1 && channels != 2)) return 1; + if (sampleRate != 16000 && sampleRate != 24000 && sampleRate != 44100 && sampleRate != 48000) return 1; + if (systemAudio && !CGPreflightScreenCaptureAccess()) return 4; + if (microphoneKind != 0 && [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio] != AVAuthorizationStatusAuthorized) return 4; + AVCaptureDevice *device = nil; + if (microphoneKind != 0) { + device = NativeSdkMicrophone(microphoneID, microphoneKind == 1); + if (!device) return 5; + } + int fileResult = [self prepareFilesAtPath:path]; + if (fileResult != 0) return fileResult; + self.destinationPath = path; + self.selectedMicrophoneID = device.uniqueID; + self.sampleRate = sampleRate; + self.channelCount = channels; + self.combined = systemAudio && microphoneKind != 0; + self.active = YES; self.terminalEmitted = NO; self.hasBasePTS = NO; + self.maxFrame = 0; self.sampleBufferCount = 0; + self.finalizationSemaphore = dispatch_semaphore_create(0); + [self startDeviceObservers]; + + if (systemAudio) { + [SCShareableContent getShareableContentExcludingDesktopWindows:YES onScreenWindowsOnly:YES completionHandler:^(SCShareableContent *content, NSError *error) { + if (!self.active) return; + if (error || content.displays.count == 0) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; return; } + NSArray *excluded = @[]; + if (exclude) { + NSString *bundleID = NSBundle.mainBundle.bundleIdentifier; + if (bundleID.length > 0) { + NSPredicate *predicate = [NSPredicate predicateWithBlock:^BOOL(SCRunningApplication *application, NSDictionary *bindings) { + (void)bindings; return [application.bundleIdentifier isEqualToString:bundleID]; + }]; + excluded = [content.applications filteredArrayUsingPredicate:predicate]; + } + } + SCContentFilter *filter = [[SCContentFilter alloc] initWithDisplay:content.displays.firstObject excludingApplications:excluded exceptingWindows:@[]]; + SCStreamConfiguration *configuration = [SCStreamConfiguration new]; + configuration.width = 2; configuration.height = 2; configuration.minimumFrameInterval = CMTimeMake(1, 1); + configuration.showsCursor = NO; configuration.capturesAudio = YES; + configuration.sampleRate = sampleRate; configuration.channelCount = channels; + configuration.excludesCurrentProcessAudio = exclude; + if (microphoneKind != 0 && !NativeSdkConfigureScreenCaptureMicrophone(configuration, device.uniqueID)) { + [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_UNSUPPORTED]; return; + } + SCStream *stream = [[SCStream alloc] initWithFilter:filter configuration:configuration delegate:self]; + NSError *addError = nil; + if (![stream addStreamOutput:self type:SCStreamOutputTypeAudio sampleHandlerQueue:self.sampleQueue error:&addError] || + (microphoneKind != 0 && ![stream addStreamOutput:self type:NativeSdkMicrophoneOutputType() sampleHandlerQueue:self.sampleQueue error:&addError])) { + [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; return; + } + self.screenStream = stream; + [stream startCaptureWithCompletionHandler:^(NSError *startError) { + if (startError) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; + else if (self.active && !self.terminalEmitted) [self emitCaptureState:NS_CAPTURE_STARTED reason:NS_REASON_NONE duration:0 bytes:0 committed:NO]; + }]; + }]; + return 0; + } + + dispatch_async(self.sampleQueue, ^{ + NSError *inputError = nil; + AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&inputError]; + if (!input) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_DEVICE_NOT_FOUND]; return; } + AVCaptureSession *session = [AVCaptureSession new]; + AVCaptureAudioDataOutput *output = [AVCaptureAudioDataOutput new]; + [output setSampleBufferDelegate:self queue:self.sampleQueue]; + [session beginConfiguration]; + if (![session canAddInput:input] || ![session canAddOutput:output]) { + [session commitConfiguration]; [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; return; + } + [session addInput:input]; [session addOutput:output]; [session commitConfiguration]; + self.microphoneSession = session; self.microphoneOutput = output; + [session startRunning]; + if (!session.running) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; + else [self emitCaptureState:NS_CAPTURE_STARTED reason:NS_REASON_NONE duration:0 bytes:0 committed:NO]; + }); + return 0; +} + +- (void)stream:(SCStream *)stream didStopWithError:(NSError *)error { + (void)stream; (void)error; + if (self.active && !self.terminalEmitted) [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_CAPTURE_FAILED]; +} + +- (void)stream:(SCStream *)stream didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer ofType:(SCStreamOutputType)type { + (void)stream; (void)type; [self consumeSampleBuffer:sampleBuffer]; +} + +- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection { + (void)output; (void)connection; [self consumeSampleBuffer:sampleBuffer]; +} + +- (void)consumeSampleBuffer:(CMSampleBufferRef)sampleBuffer { + if (!self.active || !CMSampleBufferDataIsReady(sampleBuffer)) return; + CMAudioFormatDescriptionRef description = CMSampleBufferGetFormatDescription(sampleBuffer); + const AudioStreamBasicDescription *asbd = description ? CMAudioFormatDescriptionGetStreamBasicDescription(description) : NULL; + if (!asbd) return; + size_t listSize = 0; + OSStatus sizeStatus = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, &listSize, NULL, 0, NULL, NULL, 0, NULL); + if (sizeStatus != noErr || listSize == 0) return; + AudioBufferList *list = malloc(listSize); + if (!list) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_IO_FAILED]; return; } + CMBlockBufferRef block = NULL; + OSStatus listStatus = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, NULL, list, listSize, NULL, NULL, 0, &block); + if (listStatus != noErr) { free(list); return; } + AVAudioFormat *inputFormat = [[AVAudioFormat alloc] initWithStreamDescription:asbd]; + AVAudioPCMBuffer *input = [[AVAudioPCMBuffer alloc] initWithPCMFormat:inputFormat bufferListNoCopy:list deallocator:^(const AudioBufferList *bufferList) { + (void)bufferList; if (block) CFRelease(block); free(list); + }]; + if (!input) { if (block) CFRelease(block); free(list); return; } + input.frameLength = (AVAudioFrameCount)CMSampleBufferGetNumSamples(sampleBuffer); + AVAudioFormat *outputFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:self.sampleRate channels:self.channelCount interleaved:NO]; + AVAudioConverter *converter = [[AVAudioConverter alloc] initFromFormat:inputFormat toFormat:outputFormat]; + if (!converter) return; + AVAudioFrameCount capacity = (AVAudioFrameCount)ceil((double)input.frameLength * self.sampleRate / inputFormat.sampleRate) + 32; + AVAudioPCMBuffer *converted = [[AVAudioPCMBuffer alloc] initWithPCMFormat:outputFormat frameCapacity:capacity]; + __block BOOL supplied = NO; + NSError *conversionError = nil; + AVAudioConverterOutputStatus status = [converter convertToBuffer:converted error:&conversionError withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount requested, AVAudioConverterInputStatus *inputStatus) { + (void)requested; + if (supplied) { *inputStatus = AVAudioConverterInputStatus_EndOfStream; return nil; } + supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input; + }]; + if (status == AVAudioConverterOutputStatus_Error || converted.frameLength == 0) return; + CMTime pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + if (!self.hasBasePTS) { self.basePTS = pts; self.hasBasePTS = YES; } + double offsetSeconds = CMTimeGetSeconds(CMTimeSubtract(pts, self.basePTS)); + uint64_t startFrame = offsetSeconds > 0 ? (uint64_t)llround(offsetSeconds * self.sampleRate) : 0; + [self mixFloatChannels:converted.floatChannelData frames:converted.frameLength atFrame:startFrame]; +} + +- (void)mixFloatChannels:(float *const *)channels frames:(AVAudioFrameCount)frames atFrame:(uint64_t)startFrame { + if (self.mixFD < 0 || frames == 0) return; + const size_t samples = (size_t)frames * self.channelCount; + int32_t *mixed = calloc(samples, sizeof(int32_t)); + if (!mixed) { [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_IO_FAILED]; return; } + const off_t offset = (off_t)(startFrame * self.channelCount * sizeof(int32_t)); + ssize_t readCount = pread(self.mixFD, mixed, samples * sizeof(int32_t), offset); + if (readCount < 0 && errno != 0) { free(mixed); [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_IO_FAILED]; return; } + const float gain = self.combined ? 16384.0f : 32767.0f; + for (AVAudioFrameCount frame = 0; frame < frames; frame++) { + for (uint8_t channel = 0; channel < self.channelCount; channel++) { + float sample = channels[channel][frame]; + if (!isfinite(sample)) sample = 0; + int64_t value = (int64_t)mixed[(size_t)frame * self.channelCount + channel] + (int64_t)lrintf(fmaxf(-1.0f, fminf(1.0f, sample)) * gain); + mixed[(size_t)frame * self.channelCount + channel] = (int32_t)MAX(INT32_MIN, MIN(INT32_MAX, value)); + } + } + if (pwrite(self.mixFD, mixed, samples * sizeof(int32_t), offset) != (ssize_t)(samples * sizeof(int32_t))) { + free(mixed); [self finishWithState:NS_CAPTURE_FAILED reason:NS_REASON_IO_FAILED]; return; + } + free(mixed); + self.maxFrame = MAX(self.maxFrame, startFrame + frames); + self.sampleBufferCount += 1; +} + +- (void)stopCapture { + if (!self.active || self.terminalEmitted) return; + self.active = NO; + SCStream *stream = self.screenStream; + self.screenStream = nil; + if (stream) { + [stream stopCaptureWithCompletionHandler:^(NSError *error) { + (void)error; [self finishWithState:NS_CAPTURE_STOPPED reason:NS_REASON_NONE]; + }]; + return; + } + AVCaptureSession *session = self.microphoneSession; + self.microphoneSession = nil; self.microphoneOutput = nil; + dispatch_async(self.sampleQueue, ^{ if (session.running) [session stopRunning]; [self finishWithState:NS_CAPTURE_STOPPED reason:NS_REASON_NONE]; }); +} + +- (BOOL)publishWavBytes:(uint64_t *)bytes duration:(uint64_t *)duration { + self.publishFailureReason = NS_REASON_IO_FAILED; + if (self.mixFD < 0 || self.sampleBufferCount == 0 || self.maxFrame == 0) return NO; + int wavFD = open(self.wavTemporaryPath.fileSystemRepresentation, O_CREAT | O_EXCL | O_WRONLY, 0600); + if (wavFD < 0) return NO; + uint64_t dataBytes64 = self.maxFrame * self.channelCount * sizeof(int16_t); + if (dataBytes64 > UINT32_MAX - 36) { close(wavFD); return NO; } + uint8_t header[44] = {0}; + memcpy(header, "RIFF", 4); NativeSdkWriteLE32(header + 4, (uint32_t)dataBytes64 + 36); memcpy(header + 8, "WAVEfmt ", 8); + NativeSdkWriteLE32(header + 16, 16); NativeSdkWriteLE16(header + 20, 1); NativeSdkWriteLE16(header + 22, self.channelCount); + NativeSdkWriteLE32(header + 24, self.sampleRate); NativeSdkWriteLE32(header + 28, self.sampleRate * self.channelCount * 2); + NativeSdkWriteLE16(header + 32, self.channelCount * 2); NativeSdkWriteLE16(header + 34, 16); memcpy(header + 36, "data", 4); NativeSdkWriteLE32(header + 40, (uint32_t)dataBytes64); + if (write(wavFD, header, sizeof(header)) != sizeof(header)) { close(wavFD); return NO; } + const size_t chunkSamples = 16384; + int32_t *source = calloc(chunkSamples, sizeof(int32_t)); + int16_t *target = malloc(chunkSamples * sizeof(int16_t)); + if (!source || !target) { free(source); free(target); close(wavFD); return NO; } + uint64_t remaining = self.maxFrame * self.channelCount; + off_t inputOffset = 0; + while (remaining > 0) { + size_t count = (size_t)MIN((uint64_t)chunkSamples, remaining); + memset(source, 0, count * sizeof(int32_t)); + ssize_t got = pread(self.mixFD, source, count * sizeof(int32_t), inputOffset); + if (got < 0) { free(source); free(target); close(wavFD); return NO; } + for (size_t index = 0; index < count; index++) target[index] = (int16_t)MAX(INT16_MIN, MIN(INT16_MAX, source[index])); + if (write(wavFD, target, count * sizeof(int16_t)) != (ssize_t)(count * sizeof(int16_t))) { free(source); free(target); close(wavFD); return NO; } + inputOffset += (off_t)(count * sizeof(int32_t)); remaining -= count; + } + free(source); free(target); + if (fsync(wavFD) != 0) { close(wavFD); return NO; } + close(wavFD); + if ([[NSFileManager defaultManager] fileExistsAtPath:self.destinationPath]) { + self.publishFailureReason = NS_REASON_OUTPUT_EXISTS; + return NO; + } + if (renamex_np(self.wavTemporaryPath.fileSystemRepresentation, self.destinationPath.fileSystemRepresentation, RENAME_EXCL) != 0) { + if (errno == EEXIST) self.publishFailureReason = NS_REASON_OUTPUT_EXISTS; + return NO; + } + *bytes = 44 + dataBytes64; + *duration = (self.maxFrame * 1000) / self.sampleRate; + return YES; +} + +- (void)finishWithState:(int)state reason:(int)reason { + @synchronized (self) { + if (self.terminalEmitted) return; + self.terminalEmitted = YES; self.active = NO; + } + SCStream *stream = self.screenStream; self.screenStream = nil; + if (stream) [stream stopCaptureWithCompletionHandler:nil]; + AVCaptureSession *session = self.microphoneSession; self.microphoneSession = nil; self.microphoneOutput = nil; + dispatch_async(self.sampleQueue, ^{ + if (session.running) [session stopRunning]; + uint64_t bytes = 0, duration = 0; + BOOL committed = [self publishWavBytes:&bytes duration:&duration]; + if (self.mixFD >= 0) { close(self.mixFD); self.mixFD = -1; } + if (self.mixTemporaryPath) unlink(self.mixTemporaryPath.fileSystemRepresentation); + if (!committed && self.wavTemporaryPath) unlink(self.wavTemporaryPath.fileSystemRepresentation); + int terminalReason = reason; + if (self.sampleBufferCount == 0 && reason == NS_REASON_NONE) terminalReason = NS_REASON_NO_AUDIO; + else if (!committed && self.sampleBufferCount > 0 && reason == NS_REASON_NONE) terminalReason = self.publishFailureReason; + int terminalState = state; + if (terminalReason != NS_REASON_NONE && state == NS_CAPTURE_STOPPED) terminalState = NS_CAPTURE_FAILED; + [self emitCaptureState:terminalState reason:terminalReason duration:duration bytes:bytes committed:committed]; + self.destinationPath = nil; self.wavTemporaryPath = nil; self.mixTemporaryPath = nil; self.selectedMicrophoneID = nil; + if (!self.observingDevices) [self stopDeviceObservers]; + dispatch_semaphore_t semaphore = self.finalizationSemaphore; + self.finalizationSemaphore = nil; + if (semaphore) dispatch_semaphore_signal(semaphore); + }); +} + +@end + +struct native_sdk_audio_capture { void *object; }; + +native_sdk_audio_capture_t *native_sdk_audio_capture_create(native_sdk_audio_capture_callback_t callback, void *context) { + native_sdk_audio_capture_t *handle = calloc(1, sizeof(*handle)); + if (!handle) return NULL; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = [[NativeSdkAudioCapture alloc] initWithCallback:callback context:context]; + if (!object) { free(handle); return NULL; } + handle->object = (__bridge_retained void *)object; + return handle; + } + free(handle); + return NULL; +} + +void native_sdk_audio_capture_destroy(native_sdk_audio_capture_t *capture) { + if (!capture) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge_transfer NativeSdkAudioCapture *)capture->object; + object.callback = NULL; + object.callbackContext = NULL; + dispatch_semaphore_t semaphore = object.finalizationSemaphore; + [object stopCapture]; + if (semaphore) { + (void)dispatch_semaphore_wait(semaphore, dispatch_time(DISPATCH_TIME_NOW, 2 * NSEC_PER_SEC)); + } + } + free(capture); +} + +int native_sdk_audio_capture_start(native_sdk_audio_capture_t *capture, const char *path, size_t path_len, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio) { + if (!capture || !capture->object) return 6; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + NSString *pathString = [[NSString alloc] initWithBytes:path length:path_len encoding:NSUTF8StringEncoding]; + NSString *deviceID = [[NSString alloc] initWithBytes:microphone_id length:microphone_id_len encoding:NSUTF8StringEncoding] ?: @""; + if (!pathString) return 1; + return [object startPath:pathString systemAudio:(system_audio != 0) microphoneKind:microphone_kind microphoneID:deviceID sampleRate:sample_rate_hz channels:channel_count excludeCurrentProcessAudio:(exclude_current_process_audio != 0)]; + } + return 6; +} + +void native_sdk_audio_capture_stop(native_sdk_audio_capture_t *capture) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) [(__bridge NativeSdkAudioCapture *)capture->object stopCapture]; +} + +void native_sdk_audio_capture_list_microphones(native_sdk_audio_capture_t *capture) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + dispatch_async(dispatch_get_main_queue(), ^{ + NSArray *devices = NativeSdkMicrophones(); + AVCaptureDevice *defaultDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; + uint32_t total = (uint32_t)MIN((NSUInteger)UINT32_MAX, devices.count); + [devices enumerateObjectsUsingBlock:^(AVCaptureDevice *device, NSUInteger index, BOOL *stop) { + (void)stop; + const char *identifier = device.uniqueID.UTF8String ?: ""; + const char *name = device.localizedName.UTF8String ?: ""; + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE, .state = NS_DEVICE, + .device_id = identifier, .device_id_len = strlen(identifier), .device_name = name, .device_name_len = strlen(name), + .device_is_default = [device.uniqueID isEqualToString:defaultDevice.uniqueID] ? 1 : 0, .device_index = (uint32_t)index, .device_total = total }; + [object emit:event]; + }]; + native_sdk_audio_capture_event_t completed = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_DEVICE, .state = NS_DEVICES_COMPLETED, .device_index = total, .device_total = total }; + [object emit:completed]; + }); + } +} + +void native_sdk_audio_capture_access(native_sdk_audio_capture_t *capture, int source, int action) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + dispatch_async(dispatch_get_main_queue(), ^{ + if (source == 0) { + BOOL before = CGPreflightScreenCaptureAccess(); + BOOL granted = before; + if (action == 1 && !before) granted = CGRequestScreenCaptureAccess(); + BOOL after = CGPreflightScreenCaptureAccess(); + int status = (before || after || granted) ? NS_ACCESS_AUTHORIZED : NS_ACCESS_NOT_AUTHORIZED; + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS, .access_source = source, + .access_status = status, .restart_required = (granted && !after) ? 1 : 0 }; + [object emit:event]; + return; + } + AVAuthorizationStatus auth = [AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]; + void (^emitStatus)(AVAuthorizationStatus) = ^(AVAuthorizationStatus value) { + int status = NS_ACCESS_DENIED; + switch (value) { case AVAuthorizationStatusAuthorized: status = NS_ACCESS_AUTHORIZED; break; + case AVAuthorizationStatusNotDetermined: status = NS_ACCESS_NOT_DETERMINED; break; + case AVAuthorizationStatusRestricted: status = NS_ACCESS_RESTRICTED; break; + case AVAuthorizationStatusDenied: default: status = NS_ACCESS_DENIED; break; } + native_sdk_audio_capture_event_t event = { .kind = NATIVE_SDK_AUDIO_CAPTURE_EVENT_ACCESS, .access_source = source, .access_status = status }; + [object emit:event]; + }; + if (action == 1 && auth == AVAuthorizationStatusNotDetermined) { + [AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) { + (void)granted; emitStatus([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]); + }]; + } else emitStatus(auth); + }); + } +} + +void native_sdk_audio_capture_observe_microphones(native_sdk_audio_capture_t *capture, int enabled) { + if (!capture || !capture->object) return; + if (@available(macOS 15.0, *)) { + NativeSdkAudioCapture *object = (__bridge NativeSdkAudioCapture *)capture->object; + object.observingDevices = enabled != 0; + if (enabled) [object startDeviceObservers]; else if (!object.active) [object stopDeviceObservers]; + } +} diff --git a/src/platform/macos/cef_host.mm b/src/platform/macos/cef_host.mm index 1d5a2c86e..a37e1630d 100644 --- a/src/platform/macos/cef_host.mm +++ b/src/platform/macos/cef_host.mm @@ -2651,6 +2651,18 @@ int native_sdk_appkit_audio_set_volume(native_sdk_appkit_host_t *host, double vo return 0; } +int native_sdk_appkit_audio_capture_start(native_sdk_appkit_host_t *host, const char *path, size_t path_len, int system_audio, int microphone_kind, const char *microphone_id, size_t microphone_id_len, uint32_t sample_rate_hz, uint8_t channel_count, int exclude_current_process_audio) { + (void)host; (void)path; (void)path_len; (void)system_audio; (void)microphone_kind; + (void)microphone_id; (void)microphone_id_len; (void)sample_rate_hz; (void)channel_count; + (void)exclude_current_process_audio; + return 6; +} + +void native_sdk_appkit_audio_capture_stop(native_sdk_appkit_host_t *host) { (void)host; } +void native_sdk_appkit_microphone_devices(native_sdk_appkit_host_t *host) { (void)host; } +void native_sdk_appkit_audio_capture_access(native_sdk_appkit_host_t *host, int source, int action) { (void)host; (void)source; (void)action; } +void native_sdk_appkit_observe_microphone_devices(native_sdk_appkit_host_t *host, int enabled) { (void)host; (void)enabled; } + /* Video playback lives in the system-engine AppKit host (AVFoundation). * The Chromium host reports the feature unsupported and the Zig side * refuses before calling, so these exist only to satisfy the shared C diff --git a/src/platform/macos/root.zig b/src/platform/macos/root.zig index eb3815f70..d1279e08b 100644 --- a/src/platform/macos/root.zig +++ b/src/platform/macos/root.zig @@ -2,6 +2,19 @@ const std = @import("std"); const builtin = @import("builtin"); const geometry = @import("geometry"); const platform_mod = @import("../root.zig"); + +fn isMacOS15OrNewer() bool { + // This backend is imported into every desktop target's platform tests. + // Keep the Darwin-only sysctl out of Linux and Windows compilation. + if (comptime builtin.os.tag != .macos) return false; + var version_buf: [64]u8 = undefined; + var version_len: usize = version_buf.len; + if (std.posix.system.sysctlbyname("kern.osproductversion", &version_buf, &version_len, null, 0) != 0) return false; + const version = std.mem.sliceTo(version_buf[0..@min(version_len, version_buf.len)], 0); + const dot = std.mem.indexOfScalar(u8, version, '.') orelse version.len; + const major = std.fmt.parseUnsigned(u16, version[0..dot], 10) catch return false; + return major >= 15; +} const policy_values = @import("../policy_values.zig"); const security = @import("../../security/root.zig"); // The packaging pipeline's one-image icon machinery: dev runs borrow its @@ -42,6 +55,10 @@ const AppKitEventKind = enum(c_int) { audio = 20, video = 21, view_focused = 22, + audio_capture = 23, + microphone_device = 24, + microphone_devices_changed = 25, + audio_capture_access = 26, }; const AppKitEvent = extern struct { @@ -139,6 +156,22 @@ const AppKitEvent = extern struct { /// sink's pixel budget. Zeros on every other kind. video_width: u64, video_height: u64, + audio_capture_state: c_int, + audio_capture_reason: c_int, + audio_capture_duration_ms: u64, + audio_capture_bytes_written: u64, + audio_capture_output_committed: c_int, + microphone_device_state: c_int, + microphone_device_id: [*]const u8, + microphone_device_id_len: usize, + microphone_device_name: [*]const u8, + microphone_device_name_len: usize, + microphone_device_is_default: c_int, + microphone_device_index: u32, + microphone_device_total: u32, + audio_capture_access_source: c_int, + audio_capture_access_status: c_int, + audio_capture_restart_required: c_int, }; const AppKitCallback = *const fn (context: ?*anyopaque, event: *const AppKitEvent) callconv(.c) void; @@ -205,6 +238,11 @@ extern fn native_sdk_appkit_audio_pause(host: *AppKitHost) c_int; extern fn native_sdk_appkit_audio_stop(host: *AppKitHost) c_int; extern fn native_sdk_appkit_audio_seek(host: *AppKitHost, position_ms: u64) c_int; extern fn native_sdk_appkit_audio_set_volume(host: *AppKitHost, volume: f64) c_int; +extern fn native_sdk_appkit_audio_capture_start(host: *AppKitHost, path: [*]const u8, path_len: usize, system_audio: c_int, microphone_kind: c_int, microphone_id: [*]const u8, microphone_id_len: usize, sample_rate_hz: u32, channel_count: u8, exclude_current_process_audio: c_int) c_int; +extern fn native_sdk_appkit_audio_capture_stop(host: *AppKitHost) void; +extern fn native_sdk_appkit_microphone_devices(host: *AppKitHost) void; +extern fn native_sdk_appkit_audio_capture_access(host: *AppKitHost, source: c_int, action: c_int) void; +extern fn native_sdk_appkit_observe_microphone_devices(host: *AppKitHost, enabled: c_int) void; extern fn native_sdk_appkit_video_load(host: *AppKitHost, path: [*]const u8, path_len: usize, token: u64, push_fn: AppKitVideoSinkPush, push_context: ?*anyopaque) c_int; extern fn native_sdk_appkit_video_load_url(host: *AppKitHost, url: [*]const u8, url_len: usize, token: u64, push_fn: AppKitVideoSinkPush, push_context: ?*anyopaque) c_int; extern fn native_sdk_appkit_video_play(host: *AppKitHost) c_int; @@ -750,6 +788,11 @@ pub const MacPlatform = struct { .audio_stop_fn = audioStop, .audio_seek_fn = audioSeek, .audio_set_volume_fn = audioSetVolume, + .audio_capture_start_fn = audioCaptureStart, + .audio_capture_stop_fn = audioCaptureStop, + .microphone_devices_fn = microphoneDevices, + .audio_capture_access_fn = audioCaptureAccess, + .microphone_devices_observe_fn = observeMicrophoneDevices, .video_load_fn = videoLoad, .video_load_url_fn = videoLoadUrl, .video_play_fn = videoPlay, @@ -822,6 +865,10 @@ pub const MacPlatform = struct { .audio_streaming, .audio_spectrum, => self.web_engine == .system, + .system_audio_capture, + .microphone_capture, + .microphone_device_enumeration, + => self.web_engine == .system and isMacOS15OrNewer(), // AVFoundation video ships in the AppKit host only (one // AVPlayer whose AVPlayerItemVideoOutput frames feed the // media-surface sink); the CEF host stubs the C ABI and @@ -1007,6 +1054,27 @@ fn appkitCallback(context: ?*anyopaque, event: *const AppKitEvent) callconv(.c) .width = event.video_width, .height = event.video_height, } }), + .audio_capture => state.emit(.{ .audio_capture = .{ + .state = audioCaptureStateFromInt(event.audio_capture_state), + .reason = audioCaptureReasonFromInt(event.audio_capture_reason), + .duration_ms = event.audio_capture_duration_ms, + .bytes_written = event.audio_capture_bytes_written, + .output_committed = event.audio_capture_output_committed != 0, + } }), + .microphone_device => state.emit(.{ .microphone_device = .{ + .state = microphoneDeviceStateFromInt(event.microphone_device_state), + .id = appKitEventBytes(event.microphone_device_id, event.microphone_device_id_len), + .name = appKitEventBytes(event.microphone_device_name, event.microphone_device_name_len), + .is_default = event.microphone_device_is_default != 0, + .index = event.microphone_device_index, + .total = event.microphone_device_total, + } }), + .microphone_devices_changed => state.emit(.microphone_devices_changed), + .audio_capture_access => state.emit(.{ .audio_capture_access = .{ + .source = if (event.audio_capture_access_source == 1) .microphone else .system_audio, + .status = audioCaptureAccessStatusFromInt(event.audio_capture_access_status), + .restart_required = event.audio_capture_restart_required != 0, + } }), .widget_accessibility_action => if (widgetAccessibilityActionFromInt(event.widget_action)) |action| { state.emit(.{ .widget_accessibility_action = .{ .window_id = event.window_id, @@ -1062,6 +1130,52 @@ fn videoEventKindFromInt(value: c_int) platform_mod.VideoEventKind { }; } +fn audioCaptureStateFromInt(value: c_int) platform_mod.AudioCaptureEventState { + return switch (value) { + 0 => .started, + 1 => .stopped, + 2 => .failed, + else => .rejected, + }; +} + +fn audioCaptureReasonFromInt(value: c_int) platform_mod.AudioCaptureEventReason { + return switch (value) { + 0 => .none, + 1 => .invalid_options, + 2 => .permission_missing, + 3 => .permission_required, + 4 => .already_recording, + 5 => .device_not_found, + 6 => .device_disconnected, + 7 => .output_exists, + 8 => .io_failed, + 9 => .capture_failed, + 10 => .no_audio, + else => .unsupported, + }; +} + +fn microphoneDeviceStateFromInt(value: c_int) platform_mod.MicrophoneDeviceEventState { + return switch (value) { + 0 => .device, + 1 => .completed, + 2 => .failed, + else => .rejected, + }; +} + +fn audioCaptureAccessStatusFromInt(value: c_int) platform_mod.AudioCaptureAccessStatus { + return switch (value) { + 0 => .authorized, + 1 => .not_authorized, + 2 => .not_determined, + 3 => .denied, + 4 => .restricted, + else => .unavailable, + }; +} + fn gpuSurfaceInputEventFromAppKitEvent(event: *const AppKitEvent) platform_mod.GpuSurfaceInputEvent { return .{ .window_id = event.window_id, @@ -1531,6 +1645,63 @@ fn audioSetVolume(context: ?*anyopaque, volume: f32) anyerror!void { _ = native_sdk_appkit_audio_set_volume(self.host, volume); } +fn audioCaptureStart(context: ?*anyopaque, config: platform_mod.AudioCaptureConfig) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + if (!security.hasPermission(self.app_info.permissions, security.permission_filesystem)) return error.PermissionMissing; + if (config.system_audio and !security.hasPermission(self.app_info.permissions, security.permission_system_audio)) return error.PermissionMissing; + if (config.microphone != .none and !security.hasPermission(self.app_info.permissions, security.permission_microphone)) return error.PermissionMissing; + return switch (native_sdk_appkit_audio_capture_start( + self.host, + config.path.ptr, + config.path.len, + @intFromBool(config.system_audio), + @intFromEnum(config.microphone), + config.microphone_device_id.ptr, + config.microphone_device_id.len, + config.sample_rate_hz, + config.channel_count, + @intFromBool(config.exclude_current_process_audio), + )) { + 0 => {}, + 1 => error.InvalidAudioCaptureOptions, + 2 => error.AudioCaptureAlreadyActive, + 3 => error.AudioCaptureOutputExists, + 4 => error.AudioCapturePermissionRequired, + 5 => error.MicrophoneDeviceNotFound, + 7 => error.AudioCaptureIoFailed, + else => error.UnsupportedService, + }; +} + +fn audioCaptureStop(context: ?*anyopaque) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system) return error.UnsupportedService; + native_sdk_appkit_audio_capture_stop(self.host); +} + +fn microphoneDevices(context: ?*anyopaque) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + if (!security.hasPermission(self.app_info.permissions, security.permission_microphone)) return error.PermissionMissing; + native_sdk_appkit_microphone_devices(self.host); +} + +fn audioCaptureAccess(context: ?*anyopaque, source: platform_mod.AudioCaptureAccessSource, action: platform_mod.AudioCaptureAccessAction) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + const permission = if (source == .system_audio) security.permission_system_audio else security.permission_microphone; + if (!security.hasPermission(self.app_info.permissions, permission)) return error.PermissionMissing; + native_sdk_appkit_audio_capture_access(self.host, @intFromEnum(source), @intFromEnum(action)); +} + +fn observeMicrophoneDevices(context: ?*anyopaque, enabled: bool) anyerror!void { + const self: *MacPlatform = @ptrCast(@alignCast(context.?)); + if (self.web_engine != .system or !isMacOS15OrNewer()) return error.UnsupportedService; + if (!security.hasPermission(self.app_info.permissions, security.permission_microphone)) return error.PermissionMissing; + native_sdk_appkit_observe_microphone_devices(self.host, @intFromBool(enabled)); +} + /// The C-callable bridge for `VideoFrameSink.push`: the sink's `push_fn` /// is a Zig-calling-convention error-union fn the host cannot invoke, so /// the host is handed this trampoline (context = the `MacPlatform`) and diff --git a/src/platform/null_platform.zig b/src/platform/null_platform.zig index 68cfd814b..240e1c1a6 100644 --- a/src/platform/null_platform.zig +++ b/src/platform/null_platform.zig @@ -282,6 +282,28 @@ pub const NullAudio = struct { } }; +pub const NullAudioCapture = struct { + active: bool = false, + started_pending: bool = false, + path_storage: [types.max_audio_capture_path_bytes]u8 = undefined, + path_len: usize = 0, + system_audio: bool = false, + microphone: types.MicrophoneSelectionKind = .none, + microphone_device_id_storage: [types.max_microphone_device_id_bytes]u8 = undefined, + microphone_device_id_len: usize = 0, + sample_rate_hz: u32 = 48_000, + channel_count: u8 = 2, + exclude_current_process_audio: bool = true, + + pub fn path(self: *const NullAudioCapture) []const u8 { + return self.path_storage[0..self.path_len]; + } + + pub fn microphoneDeviceId(self: *const NullAudioCapture) []const u8 { + return self.microphone_device_id_storage[0..self.microphone_device_id_len]; + } +}; + pub const NullPlatform = struct { surface_value: Surface = .{}, web_engine: WebEngine = .system, @@ -625,6 +647,20 @@ pub const NullPlatform = struct { audio_stop_count: usize = 0, audio_seek_count: usize = 0, audio_volume_count: usize = 0, + audio_capture: bool = true, + microphone_device_enumeration: bool = true, + capture: NullAudioCapture = .{}, + audio_capture_start_count: usize = 0, + audio_capture_stop_count: usize = 0, + microphone_devices_count: usize = 0, + microphone_devices_observing: bool = false, + default_microphone_index: u8 = 0, + microphone_connected: [2]bool = .{ true, true }, + audio_capture_output_exists: bool = false, + audio_capture_io_failure: bool = false, + system_audio_access: types.AudioCaptureAccessStatus = .authorized, + microphone_access: types.AudioCaptureAccessStatus = .authorized, + access_pending: ?types.AudioCaptureAccessEvent = null, /// Whether this modeled host has a video decoder. On by default (the /// fake below stands in for AVFoundation); tests modelling a staged /// host (Windows/Linux today) set it false BEFORE `platform()` so @@ -851,6 +887,11 @@ pub const NullPlatform = struct { .audio_stop_fn = if (self.audio_playback) audioStop else null, .audio_seek_fn = if (self.audio_playback) audioSeek else null, .audio_set_volume_fn = if (self.audio_playback) audioSetVolume else null, + .audio_capture_start_fn = if (self.audio_capture) audioCaptureStart else null, + .audio_capture_stop_fn = if (self.audio_capture) audioCaptureStop else null, + .microphone_devices_fn = if (self.microphone_device_enumeration) microphoneDevices else null, + .audio_capture_access_fn = if (self.audio_capture) audioCaptureAccess else null, + .microphone_devices_observe_fn = if (self.microphone_device_enumeration) observeMicrophoneDevices else null, .video_load_fn = if (self.video_playback) videoLoad else null, .video_load_url_fn = if (self.video_playback) videoLoadUrl else null, .video_play_fn = if (self.video_playback) videoPlay else null, @@ -915,6 +956,8 @@ pub const NullPlatform = struct { .audio_playback => self.audio_playback, .audio_streaming => self.audio_playback and self.audio_streaming, .audio_spectrum => self.audio_playback and self.audio_spectrum, + .system_audio_capture, .microphone_capture => self.audio_capture, + .microphone_device_enumeration => self.microphone_device_enumeration, .video_playback => self.video_playback, }; } @@ -1667,6 +1710,134 @@ pub const NullPlatform = struct { self.audio.volume = volume; } + fn audioCaptureStart(context: ?*anyopaque, config: types.AudioCaptureConfig) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + if (self.capture.active) return error.AudioCaptureAlreadyActive; + if ((!config.system_audio and config.microphone == .none) or + config.path.len == 0 or config.path.len > self.capture.path_storage.len or + config.microphone_device_id.len > self.capture.microphone_device_id_storage.len) + return error.InvalidAudioCaptureOptions; + if (config.system_audio and self.system_audio_access != .authorized) return error.AudioCapturePermissionRequired; + if (config.microphone != .none and self.microphone_access != .authorized) return error.AudioCapturePermissionRequired; + if (self.audio_capture_output_exists) return error.AudioCaptureOutputExists; + if (self.audio_capture_io_failure) return error.AudioCaptureIoFailed; + var resolved_microphone_id = config.microphone_device_id; + if (config.microphone == .default) { + if (self.default_microphone_index >= self.microphone_connected.len or !self.microphone_connected[self.default_microphone_index]) return error.MicrophoneDeviceNotFound; + resolved_microphone_id = if (self.default_microphone_index == 0) "default-mic" else "usb-mic"; + } else if (config.microphone == .device_id) { + const index: ?usize = if (std.mem.eql(u8, config.microphone_device_id, "default-mic")) 0 else if (std.mem.eql(u8, config.microphone_device_id, "usb-mic")) 1 else null; + if (index == null or !self.microphone_connected[index.?]) return error.MicrophoneDeviceNotFound; + } + self.audio_capture_start_count += 1; + self.capture = .{ + .active = true, + .started_pending = true, + .system_audio = config.system_audio, + .microphone = config.microphone, + .sample_rate_hz = config.sample_rate_hz, + .channel_count = config.channel_count, + .exclude_current_process_audio = config.exclude_current_process_audio, + }; + @memcpy(self.capture.path_storage[0..config.path.len], config.path); + self.capture.path_len = config.path.len; + @memcpy(self.capture.microphone_device_id_storage[0..resolved_microphone_id.len], resolved_microphone_id); + self.capture.microphone_device_id_len = resolved_microphone_id.len; + } + + fn audioCaptureStop(context: ?*anyopaque) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + if (!self.capture.active) return; + self.audio_capture_stop_count += 1; + self.capture.active = false; + } + + fn microphoneDevices(context: ?*anyopaque) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + self.microphone_devices_count += 1; + } + + fn audioCaptureAccess(context: ?*anyopaque, source: types.AudioCaptureAccessSource, action: types.AudioCaptureAccessAction) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + _ = action; + self.access_pending = .{ + .source = source, + .status = switch (source) { + .system_audio => self.system_audio_access, + .microphone => self.microphone_access, + }, + }; + } + + fn observeMicrophoneDevices(context: ?*anyopaque, enabled: bool) anyerror!void { + const self: *NullPlatform = @ptrCast(@alignCast(context.?)); + self.microphone_devices_observing = enabled; + } + + pub fn takeAudioCaptureStarted(self: *NullPlatform) ?Event { + if (!self.capture.started_pending) return null; + self.capture.started_pending = false; + return .{ .audio_capture = .{ .state = .started } }; + } + + pub fn completeAudioCapture(self: *NullPlatform, duration_ms: u64, bytes_written: u64) ?Event { + if (self.capture.active) self.capture.active = false; + return .{ .audio_capture = .{ + .state = .stopped, + .duration_ms = duration_ms, + .bytes_written = bytes_written, + .output_committed = true, + } }; + } + + pub fn microphoneDeviceEvent(self: *const NullPlatform, index: u32) Event { + const total: u32 = @as(u32, @intFromBool(self.microphone_connected[0])) + @as(u32, @intFromBool(self.microphone_connected[1])); + var emitted: u32 = 0; + for (self.microphone_connected, 0..) |connected, device_index| { + if (!connected) continue; + if (emitted == index) return .{ .microphone_device = .{ + .state = .device, + .id = if (device_index == 0) "default-mic" else "usb-mic", + .name = if (device_index == 0) "Default Microphone" else "USB Microphone", + .is_default = device_index == self.default_microphone_index, + .index = emitted, + .total = total, + } }; + emitted += 1; + } + return .{ .microphone_device = .{ .state = .completed, .index = total, .total = total } }; + } + + pub fn setDefaultMicrophone(self: *NullPlatform, index: u8) !void { + if (index >= self.microphone_connected.len or !self.microphone_connected[index]) return error.MicrophoneDeviceNotFound; + self.default_microphone_index = index; + } + + pub fn disconnectMicrophone(self: *NullPlatform, id: []const u8, duration_ms: u64, bytes_written: u64) ?Event { + const index: usize = if (std.mem.eql(u8, id, "default-mic")) 0 else if (std.mem.eql(u8, id, "usb-mic")) 1 else return null; + self.microphone_connected[index] = false; + if (!self.capture.active or !std.mem.eql(u8, self.capture.microphoneDeviceId(), id)) return self.microphoneDevicesChanged(); + self.capture.active = false; + return .{ .audio_capture = .{ + .state = .failed, + .reason = .device_disconnected, + .duration_ms = duration_ms, + .bytes_written = bytes_written, + .output_committed = bytes_written > 44, + } }; + } + + pub fn takeAudioCaptureAccess(self: *NullPlatform) ?Event { + const access = self.access_pending orelse return null; + self.access_pending = null; + return .{ .audio_capture_access = access }; + } + + pub fn microphoneDevicesChanged(self: *NullPlatform) ?Event { + if (!self.microphone_devices_observing) return null; + return .microphone_devices_changed; + } + fn audioUrlHash(url: []const u8) u64 { return std.hash.Wyhash.hash(0, url); } diff --git a/src/platform/root.zig b/src/platform/root.zig index f88976698..ae953fb2d 100644 --- a/src/platform/root.zig +++ b/src/platform/root.zig @@ -134,6 +134,20 @@ pub const AudioEvent = types.AudioEvent; pub const AudioEventKind = types.AudioEventKind; pub const AudioLoadResolution = types.AudioLoadResolution; pub const max_audio_path_bytes = types.max_audio_path_bytes; +pub const max_audio_capture_path_bytes = types.max_audio_capture_path_bytes; +pub const max_microphone_device_id_bytes = types.max_microphone_device_id_bytes; +pub const max_microphone_device_name_bytes = types.max_microphone_device_name_bytes; +pub const MicrophoneSelectionKind = types.MicrophoneSelectionKind; +pub const AudioCaptureConfig = types.AudioCaptureConfig; +pub const AudioCaptureEventState = types.AudioCaptureEventState; +pub const AudioCaptureEventReason = types.AudioCaptureEventReason; +pub const AudioCaptureEvent = types.AudioCaptureEvent; +pub const MicrophoneDeviceEventState = types.MicrophoneDeviceEventState; +pub const MicrophoneDeviceEvent = types.MicrophoneDeviceEvent; +pub const AudioCaptureAccessSource = types.AudioCaptureAccessSource; +pub const AudioCaptureAccessAction = types.AudioCaptureAccessAction; +pub const AudioCaptureAccessStatus = types.AudioCaptureAccessStatus; +pub const AudioCaptureAccessEvent = types.AudioCaptureAccessEvent; pub const audio_spectrum_band_count = types.audio_spectrum_band_count; pub const audio_spectrum_floor_db = types.audio_spectrum_floor_db; pub const VideoEvent = types.VideoEvent; diff --git a/src/platform/types.zig b/src/platform/types.zig index 9fc185f2e..c60d03dfb 100644 --- a/src/platform/types.zig +++ b/src/platform/types.zig @@ -158,6 +158,12 @@ pub const PlatformFeature = enum { /// keeps emitting; the null platform models the rule through its /// windows' modeled occlusion so the suites can pin it. audio_spectrum, + /// Capture all system output audio into an app-selected PCM WAV file. + system_audio_capture, + /// Capture a default or explicitly selected microphone into PCM WAV. + microphone_capture, + /// Enumerate connected microphones and observe list invalidations. + microphone_device_enumeration, /// The `close_policy = .hide` window shape: the host can intercept /// the user's close affordance, keep the window alive off the /// glass, and re-show it later (`show_window_fn`, tray actions, the @@ -1208,6 +1214,9 @@ pub const AppInfo = struct { /// folds this into its `window_hide_on_close` answer; macOS ignores /// it (the Dock reopen path always exists). declares_tray: bool = false, + /// Manifest permission grants, threaded to native services so APIs + /// that bypass the web bridge still enforce the same declaration. + permissions: []const []const u8 = &.{}, window_title: []const u8 = "", bundle_id: []const u8 = "dev.native_sdk.app", icon_path: []const u8 = "", @@ -1452,6 +1461,98 @@ pub const AudioLoadResolution = enum(u8) { stream, }; +pub const max_audio_capture_path_bytes: usize = 1024; +pub const max_microphone_device_id_bytes: usize = 512; +pub const max_microphone_device_name_bytes: usize = 512; + +pub const MicrophoneSelectionKind = enum(u8) { + none, + default, + device_id, +}; + +pub const AudioCaptureConfig = struct { + path: []const u8, + system_audio: bool = false, + microphone: MicrophoneSelectionKind = .none, + microphone_device_id: []const u8 = &.{}, + sample_rate_hz: u32 = 48_000, + channel_count: u8 = 2, + exclude_current_process_audio: bool = true, +}; + +pub const AudioCaptureEventState = enum(u8) { + started, + stopped, + failed, + rejected, +}; + +pub const AudioCaptureEventReason = enum(u8) { + none, + invalid_options, + permission_missing, + permission_required, + already_recording, + device_not_found, + device_disconnected, + output_exists, + io_failed, + capture_failed, + no_audio, + unsupported, +}; + +pub const AudioCaptureEvent = struct { + state: AudioCaptureEventState, + reason: AudioCaptureEventReason = .none, + duration_ms: u64 = 0, + bytes_written: u64 = 0, + output_committed: bool = false, +}; + +pub const MicrophoneDeviceEventState = enum(u8) { + device, + completed, + failed, + rejected, +}; + +/// Device strings are borrowed for the duration of EventHandler dispatch. +pub const MicrophoneDeviceEvent = struct { + state: MicrophoneDeviceEventState, + id: []const u8 = &.{}, + name: []const u8 = &.{}, + is_default: bool = false, + index: u32 = 0, + total: u32 = 0, +}; + +pub const AudioCaptureAccessSource = enum(u8) { + system_audio, + microphone, +}; + +pub const AudioCaptureAccessAction = enum(u8) { + status, + request, +}; + +pub const AudioCaptureAccessStatus = enum(u8) { + authorized, + not_authorized, + not_determined, + denied, + restricted, + unavailable, +}; + +pub const AudioCaptureAccessEvent = struct { + source: AudioCaptureAccessSource, + status: AudioCaptureAccessStatus, + restart_required: bool = false, +}; + /// Longest video source string (local path or URL) `videoLoad`/ /// `videoLoadUrl` accepts; longer strings are rejected with /// `error.VideoPathTooLarge` before the platform is asked. @@ -2255,6 +2356,10 @@ pub const Event = union(enum) { /// Audio player reports: load acknowledgment, coarse position ticks /// while playing, one completion at natural end, async failures. audio: AudioEvent, + audio_capture: AudioCaptureEvent, + microphone_device: MicrophoneDeviceEvent, + microphone_devices_changed, + audio_capture_access: AudioCaptureAccessEvent, /// Video player reports — the same shape, plus the stream's decoded /// dimensions on `.loaded`. Pixels never ride here. video: VideoEvent, @@ -2286,6 +2391,10 @@ pub const Event = union(enum) { .context_menu_action => "context_menu_action", .widget_accessibility_action => "widget_accessibility_action", .audio => "audio", + .audio_capture => "audio_capture", + .microphone_device => "microphone_device", + .microphone_devices_changed => "microphone_devices_changed", + .audio_capture_access => "audio_capture_access", .video => "video", }; } @@ -2461,6 +2570,11 @@ pub const PlatformServices = struct { audio_seek_fn: ?*const fn (context: ?*anyopaque, position_ms: u64) anyerror!void = null, /// Set the player volume, `0.0` (silent) through `1.0` (full). audio_set_volume_fn: ?*const fn (context: ?*anyopaque, volume: f32) anyerror!void = null, + audio_capture_start_fn: ?*const fn (context: ?*anyopaque, config: AudioCaptureConfig) anyerror!void = null, + audio_capture_stop_fn: ?*const fn (context: ?*anyopaque) anyerror!void = null, + microphone_devices_fn: ?*const fn (context: ?*anyopaque) anyerror!void = null, + audio_capture_access_fn: ?*const fn (context: ?*anyopaque, source: AudioCaptureAccessSource, action: AudioCaptureAccessAction) anyerror!void = null, + microphone_devices_observe_fn: ?*const fn (context: ?*anyopaque, enabled: bool) anyerror!void = null, /// Load a local video file into THE app's single video player, /// leaving it PAUSED at position zero (transport is a separate /// verb, exactly like audio). Loading replaces whatever was loaded @@ -3064,6 +3178,31 @@ pub const PlatformServices = struct { return volume_fn(self.context, volume); } + pub fn audioCaptureStart(self: PlatformServices, config: AudioCaptureConfig) anyerror!void { + const start_fn = self.audio_capture_start_fn orelse return error.UnsupportedService; + return start_fn(self.context, config); + } + + pub fn audioCaptureStop(self: PlatformServices) anyerror!void { + const stop_fn = self.audio_capture_stop_fn orelse return error.UnsupportedService; + return stop_fn(self.context); + } + + pub fn microphoneDevices(self: PlatformServices) anyerror!void { + const list_fn = self.microphone_devices_fn orelse return error.UnsupportedService; + return list_fn(self.context); + } + + pub fn audioCaptureAccess(self: PlatformServices, source: AudioCaptureAccessSource, action: AudioCaptureAccessAction) anyerror!void { + const access_fn = self.audio_capture_access_fn orelse return error.UnsupportedService; + return access_fn(self.context, source, action); + } + + pub fn observeMicrophoneDevices(self: PlatformServices, enabled: bool) anyerror!void { + const observe_fn = self.microphone_devices_observe_fn orelse return error.UnsupportedService; + return observe_fn(self.context, enabled); + } + /// Load a local video file into the app's single video player (see /// `video_load_fn`). Platforms without video playback answer /// `error.UnsupportedService`; bad arguments are rejected here @@ -3303,6 +3442,9 @@ fn defaultSupportsFeature(services: PlatformServices, feature: PlatformFeature) // cannot see it; platforms that analyze answer through their own // `supports_fn` (like file_drops and gpu_surfaces above). .audio_spectrum => false, + .system_audio_capture => services.audio_capture_start_fn != null, + .microphone_capture => services.audio_capture_start_fn != null, + .microphone_device_enumeration => services.microphone_devices_fn != null, // Hide-on-close is host close-delegate behavior, not a service // verb: hosts that implement it answer through their own // `supports_fn`. The generic floor is honest refusal. diff --git a/src/platform/windows/root.zig b/src/platform/windows/root.zig index 34455422d..99c36b9c6 100644 --- a/src/platform/windows/root.zig +++ b/src/platform/windows/root.zig @@ -486,6 +486,7 @@ pub const WindowsPlatform = struct { .audio_playback, .audio_streaming, => self.web_engine == .system, + .system_audio_capture, .microphone_capture, .microphone_device_enumeration => false, // close_policy .hide: WM_CLOSE hides (ShowWindow SW_HIDE), // the window stays in the host map, and the tray is the // ONLY re-show affordance — SW_HIDE removes the taskbar diff --git a/src/primitives/app_manifest/root.zig b/src/primitives/app_manifest/root.zig index edd7cbd45..81184e6f2 100644 --- a/src/primitives/app_manifest/root.zig +++ b/src/primitives/app_manifest/root.zig @@ -28,6 +28,7 @@ pub const max_file_associations = types.max_file_associations; pub const max_file_association_extensions = types.max_file_association_extensions; pub const max_file_association_mime_types = types.max_file_association_mime_types; pub const max_url_schemes = types.max_url_schemes; +pub const max_privacy_usage_bytes = types.max_privacy_usage_bytes; pub const Platform = types.Platform; pub const PackageKind = types.PackageKind; pub const WebEngine = types.WebEngine; @@ -36,6 +37,7 @@ pub const CefConfig = types.CefConfig; pub const IconPurpose = types.IconPurpose; pub const PermissionKind = types.PermissionKind; pub const Permission = types.Permission; +pub const PrivacyUsage = types.PrivacyUsage; pub const CapabilityKind = types.CapabilityKind; pub const Capability = types.Capability; pub const AppIdentity = types.AppIdentity; @@ -101,6 +103,7 @@ pub const validateName = validation.validateName; pub const validateUrl = validation.validateUrl; pub const validateIcons = validation.validateIcons; pub const validatePermissions = validation.validatePermissions; +pub const validatePrivacy = validation.validatePrivacy; pub const validateCapabilities = validation.validateCapabilities; pub const validateBridge = validation.validateBridge; pub const validateFrontend = validation.validateFrontend; diff --git a/src/primitives/app_manifest/tests.zig b/src/primitives/app_manifest/tests.zig index 3a2c6a4d9..b2d1b3673 100644 --- a/src/primitives/app_manifest/tests.zig +++ b/src/primitives/app_manifest/tests.zig @@ -30,6 +30,7 @@ const CefConfig = types.CefConfig; const IconPurpose = types.IconPurpose; const PermissionKind = types.PermissionKind; const Permission = types.Permission; +const PrivacyUsage = types.PrivacyUsage; const CapabilityKind = types.CapabilityKind; const Capability = types.Capability; const AppIdentity = types.AppIdentity; @@ -87,6 +88,7 @@ const validateDescription = validation.validateDescription; const validateUrl = validation.validateUrl; const validateIcons = validation.validateIcons; const validatePermissions = validation.validatePermissions; +const validatePrivacy = validation.validatePrivacy; const validateCapabilities = validation.validateCapabilities; const validateBridge = validation.validateBridge; const validateFrontend = validation.validateFrontend; @@ -690,6 +692,29 @@ test "permission validation catches duplicates" { try std.testing.expectError(error.InvalidName, validatePermissions(&.{.{ .custom = "bad/name" }})); } +test "audio permissions require valid privacy purpose strings" { + try validatePrivacy(.{ + .microphone_usage = "Record your voice in meeting notes.", + .system_audio_usage = "Record meeting audio for transcription.", + }, &.{ .filesystem, .microphone, .system_audio }); + + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{}, &.{.microphone})); + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{}, &.{.system_audio})); + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{ .microphone_usage = " " }, &.{.microphone})); + try std.testing.expectError(error.InvalidPrivacyUsage, validatePrivacy(.{ .system_audio_usage = "bad\nvalue" }, &.{.system_audio})); + + const manifest: Manifest = .{ + .identity = .{ .id = "com.example.recorder", .name = "recorder" }, + .version = .{ .major = 1, .minor = 0, .patch = 0 }, + .permissions = &.{ .filesystem, .microphone, .system_audio }, + .privacy = PrivacyUsage{ + .microphone_usage = "Record your voice in meeting notes.", + .system_audio_usage = "Record meeting audio for transcription.", + }, + }; + try validateManifest(manifest); +} + test "platform validation catches duplicates and invalid overrides" { try validatePlatforms(&.{ .{ .platform = .macos, .id_override = "com.example.app.macos" }, .{ .platform = .linux } }); diff --git a/src/primitives/app_manifest/types.zig b/src/primitives/app_manifest/types.zig index 21bc79bcc..5d8f57bfa 100644 --- a/src/primitives/app_manifest/types.zig +++ b/src/primitives/app_manifest/types.zig @@ -8,6 +8,7 @@ pub const ValidationError = error{ InvalidDimension, DuplicateIcon, DuplicatePermission, + InvalidPrivacyUsage, DuplicateCapability, DuplicateBridgeCommand, DuplicateCommand, @@ -61,6 +62,7 @@ pub const max_file_association_mime_types: usize = 32; pub const max_url_schemes: usize = 32; /// Cap for the identity `description` — one sentence, not a README. pub const max_description_bytes: usize = 256; +pub const max_privacy_usage_bytes: usize = 1024; pub const Platform = enum { macos, @@ -113,6 +115,7 @@ pub const PermissionKind = enum { filesystem, camera, microphone, + system_audio, location, notifications, clipboard, @@ -129,6 +132,7 @@ pub const Permission = union(PermissionKind) { filesystem: void, camera: void, microphone: void, + system_audio: void, location: void, notifications: void, clipboard: void, @@ -209,6 +213,15 @@ pub const AppIdentity = struct { homepage: ?[]const u8 = null, }; +/// Human-facing purpose strings copied into platform privacy metadata. +/// They are separate from permission declarations: permissions say what an +/// app may use, while these strings explain that use to the person granting +/// operating-system access. +pub const PrivacyUsage = struct { + microphone_usage: ?[]const u8 = null, + system_audio_usage: ?[]const u8 = null, +}; + pub const Version = struct { major: u32, minor: u32, @@ -630,6 +643,7 @@ pub const Manifest = struct { version: Version, icons: []const Icon = &.{}, permissions: []const Permission = &.{}, + privacy: PrivacyUsage = .{}, capabilities: []const Capability = &.{}, bridge: BridgeConfig = .{}, frontend: ?FrontendConfig = null, diff --git a/src/primitives/app_manifest/validation.zig b/src/primitives/app_manifest/validation.zig index a218f1348..71915fef4 100644 --- a/src/primitives/app_manifest/validation.zig +++ b/src/primitives/app_manifest/validation.zig @@ -23,6 +23,7 @@ const max_file_associations = types.max_file_associations; const max_file_association_extensions = types.max_file_association_extensions; const max_file_association_mime_types = types.max_file_association_mime_types; const max_url_schemes = types.max_url_schemes; +const max_privacy_usage_bytes = types.max_privacy_usage_bytes; const Platform = types.Platform; const PackageKind = types.PackageKind; const WebEngine = types.WebEngine; @@ -30,6 +31,7 @@ const CefConfig = types.CefConfig; const IconPurpose = types.IconPurpose; const PermissionKind = types.PermissionKind; const Permission = types.Permission; +const PrivacyUsage = types.PrivacyUsage; const CapabilityKind = types.CapabilityKind; const Capability = types.Capability; const AppIdentity = types.AppIdentity; @@ -73,6 +75,7 @@ pub fn validateManifest(manifest: Manifest) ValidationError!void { try validateVersion(manifest.version); try validateIcons(manifest.icons); try validatePermissions(manifest.permissions); + try validatePrivacy(manifest.privacy, manifest.permissions); try validateCapabilities(manifest.capabilities); try validateBridge(manifest.bridge); if (manifest.frontend) |frontend| try validateFrontend(frontend); @@ -532,6 +535,24 @@ pub fn validatePermissions(permissions: []const Permission) ValidationError!void } } +pub fn validatePrivacy(privacy: PrivacyUsage, permissions: []const Permission) ValidationError!void { + if (privacy.microphone_usage) |usage| try validatePrivacyUsage(usage); + if (privacy.system_audio_usage) |usage| try validatePrivacyUsage(usage); + + for (permissions) |permission| switch (permission) { + .microphone => if (privacy.microphone_usage == null) return error.InvalidPrivacyUsage, + .system_audio => if (privacy.system_audio_usage == null) return error.InvalidPrivacyUsage, + else => {}, + }; +} + +fn validatePrivacyUsage(usage: []const u8) ValidationError!void { + if (usage.len == 0 or usage.len > max_privacy_usage_bytes or std.mem.trim(u8, usage, " ").len == 0) return error.InvalidPrivacyUsage; + for (usage) |ch| { + if (ch < 0x20 or ch == 0x7f) return error.InvalidPrivacyUsage; + } +} + pub fn validateCapabilities(capabilities: []const Capability) ValidationError!void { for (capabilities, 0..) |capability, i| { if (capability == .custom) try validateName(capability.custom); diff --git a/src/root.zig b/src/root.zig index 8b057ea75..239c1f46f 100644 --- a/src/root.zig +++ b/src/root.zig @@ -75,6 +75,16 @@ pub const max_effect_timers = runtime.max_effect_timers; pub const EffectAudio = runtime.EffectAudio; pub const EffectAudioEventKind = runtime.EffectAudioEventKind; pub const EffectAudioSource = runtime.EffectAudioSource; +pub const EffectAudioCapture = runtime.EffectAudioCapture; +pub const EffectAudioCaptureState = runtime.EffectAudioCaptureState; +pub const EffectAudioCaptureReason = runtime.EffectAudioCaptureReason; +pub const EffectMicrophoneDevice = runtime.EffectMicrophoneDevice; +pub const EffectMicrophoneDeviceState = runtime.EffectMicrophoneDeviceState; +pub const EffectAudioCaptureAccess = runtime.EffectAudioCaptureAccess; +pub const EffectAudioCaptureAccessSource = runtime.EffectAudioCaptureAccessSource; +pub const EffectAudioCaptureAccessAction = runtime.EffectAudioCaptureAccessAction; +pub const EffectAudioCaptureAccessStatus = runtime.EffectAudioCaptureAccessStatus; +pub const MicrophoneSelection = runtime.MicrophoneSelection; pub const audioCachePath = runtime.audioCachePath; pub const max_effect_audio_path_bytes = runtime.max_effect_audio_path_bytes; pub const EffectVideo = runtime.EffectVideo; diff --git a/src/runtime/api.zig b/src/runtime/api.zig index 770098ef1..29a9a4ddd 100644 --- a/src/runtime/api.zig +++ b/src/runtime/api.zig @@ -331,6 +331,10 @@ pub const Event = union(enum) { /// tick, completion, failure): the ui-app layer routes it back /// through `Effects.takeAudioMsg` into the app's `on_event` Msg. audio: platform.AudioEvent, + audio_capture: platform.AudioCaptureEvent, + microphone_device: platform.MicrophoneDeviceEvent, + microphone_devices_changed, + audio_capture_access: platform.AudioCaptureAccessEvent, /// A platform video player report (load acknowledgment with /// dimensions, position tick, completion, failure): routed back /// through `Effects.takeVideoMsg` into the app's `on_event` Msg. @@ -366,6 +370,10 @@ pub const Event = union(enum) { .timer => "timer", .effects_wake => "effects_wake", .audio => "audio", + .audio_capture => "audio_capture", + .microphone_device => "microphone_device", + .microphone_devices_changed => "microphone_devices_changed", + .audio_capture_access => "audio_capture_access", .video => "video", .files_dropped => "files_dropped", .gpu_surface_frame => "gpu_surface_frame", diff --git a/src/runtime/bridge_payload.zig b/src/runtime/bridge_payload.zig index 0ac11d31d..e9a797f40 100644 --- a/src/runtime/bridge_payload.zig +++ b/src/runtime/bridge_payload.zig @@ -125,6 +125,9 @@ pub fn platformFeatureFromString(value: []const u8) ?platform.PlatformFeature { if (std.mem.eql(u8, value, "audioPlayback")) return .audio_playback; if (std.mem.eql(u8, value, "audioStreaming")) return .audio_streaming; if (std.mem.eql(u8, value, "audioSpectrum")) return .audio_spectrum; + if (std.mem.eql(u8, value, "systemAudioCapture")) return .system_audio_capture; + if (std.mem.eql(u8, value, "microphoneCapture")) return .microphone_capture; + if (std.mem.eql(u8, value, "microphoneDeviceEnumeration")) return .microphone_device_enumeration; if (std.mem.eql(u8, value, "windowHideOnClose")) return .window_hide_on_close; if (std.mem.eql(u8, value, "videoPlayback")) return .video_playback; return null; diff --git a/src/runtime/effects.zig b/src/runtime/effects.zig index 2ab31881f..4ea5fe02d 100644 --- a/src/runtime/effects.zig +++ b/src/runtime/effects.zig @@ -651,6 +651,47 @@ pub const EffectAudioSource = enum(u8) { stream, }; +pub const EffectAudioCaptureState = platform.AudioCaptureEventState; +pub const EffectAudioCaptureReason = platform.AudioCaptureEventReason; +pub const EffectMicrophoneDeviceState = platform.MicrophoneDeviceEventState; +pub const EffectAudioCaptureAccessSource = platform.AudioCaptureAccessSource; +pub const EffectAudioCaptureAccessAction = platform.AudioCaptureAccessAction; +pub const EffectAudioCaptureAccessStatus = platform.AudioCaptureAccessStatus; + +pub const EffectAudioCapture = struct { + key: u64, + state: EffectAudioCaptureState, + reason: EffectAudioCaptureReason = .none, + duration_ms: u64 = 0, + bytes_written: u64 = 0, + output_committed: bool = false, +}; + +/// Strings borrow platform event storage and must be copied before the +/// update callback returns when an app wants to retain them in its model. +pub const EffectMicrophoneDevice = struct { + key: u64, + state: EffectMicrophoneDeviceState, + id: []const u8 = &.{}, + name: []const u8 = &.{}, + is_default: bool = false, + index: u32 = 0, + total: u32 = 0, +}; + +pub const EffectAudioCaptureAccess = struct { + key: u64, + source: EffectAudioCaptureAccessSource, + status: EffectAudioCaptureAccessStatus, + restart_required: bool = false, +}; + +pub const MicrophoneSelection = union(enum) { + none, + default, + device_id: []const u8, +}; + /// Longest video source string (path or url) `loadVideo` accepts, /// mirroring the platform bound. Longer strings deliver exactly one /// `.rejected` video event Msg. @@ -2403,6 +2444,10 @@ pub fn Effects(comptime Msg: type) type { pub const ClipboardMsgFn = *const fn (result: EffectClipboardResult) Msg; pub const TimerMsgFn = *const fn (timer: EffectTimer) Msg; pub const AudioMsgFn = *const fn (event: EffectAudio) Msg; + pub const AudioCaptureMsgFn = *const fn (event: EffectAudioCapture) Msg; + pub const MicrophoneDeviceMsgFn = *const fn (event: EffectMicrophoneDevice) Msg; + pub const AudioCaptureAccessMsgFn = *const fn (event: EffectAudioCaptureAccess) Msg; + pub const MicrophoneDevicesChangedMsgFn = *const fn () Msg; pub const VideoMsgFn = *const fn (event: EffectVideo) Msg; pub const HostMsgFn = *const fn (result: EffectHostResult) Msg; pub const ImageMsgFn = *const fn (result: EffectImageResult) Msg; @@ -2492,6 +2537,38 @@ pub fn Effects(comptime Msg: type) type { }.make; } + pub fn audioCaptureMsg(comptime tag: std.meta.Tag(Msg)) AudioCaptureMsgFn { + return struct { + fn make(event: EffectAudioCapture) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn microphoneDeviceMsg(comptime tag: std.meta.Tag(Msg)) MicrophoneDeviceMsgFn { + return struct { + fn make(event: EffectMicrophoneDevice) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn audioCaptureAccessMsg(comptime tag: std.meta.Tag(Msg)) AudioCaptureAccessMsgFn { + return struct { + fn make(event: EffectAudioCaptureAccess) Msg { + return @unionInit(Msg, @tagName(tag), event); + } + }.make; + } + + pub fn microphoneDevicesChangedMsg(comptime tag: std.meta.Tag(Msg)) MicrophoneDevicesChangedMsgFn { + return struct { + fn make() Msg { + return @unionInit(Msg, @tagName(tag), {}); + } + }.make; + } + /// Comptime Msg constructor for `on_event` of video playback: /// `videoMsg(.video_event)` builds /// `Msg{ .video_event = event }` — the variant's payload type @@ -2794,6 +2871,29 @@ pub fn Effects(comptime Msg: type) type { on_event: ?AudioMsgFn = null, }; + pub const StartAudioCaptureOptions = struct { + key: u64, + path: []const u8, + system_audio: bool = false, + microphone: MicrophoneSelection = .none, + sample_rate_hz: u32 = 48_000, + channel_count: u8 = 2, + exclude_current_process_audio: bool = true, + on_event: ?AudioCaptureMsgFn = null, + }; + + pub const ListMicrophoneDevicesOptions = struct { + key: u64, + on_event: ?MicrophoneDeviceMsgFn = null, + }; + + pub const AudioCaptureAccessOptions = struct { + key: u64, + source: EffectAudioCaptureAccessSource, + action: EffectAudioCaptureAccessAction = .status, + on_event: ?AudioCaptureAccessMsgFn = null, + }; + pub const LoadImageOptions = struct { /// The ImageId the decoded pixels register under — model- /// owned, chosen by the app, exactly the id `image`/`avatar` @@ -3132,6 +3232,25 @@ pub fn Effects(comptime Msg: type) type { } }; + const AudioCaptureChannel = struct { + active: bool = false, + fake: bool = false, + key: u64 = 0, + on_event: ?AudioCaptureMsgFn = null, + }; + + const MicrophoneDeviceQuery = struct { + active: bool = false, + key: u64 = 0, + on_event: ?MicrophoneDeviceMsgFn = null, + }; + + const AudioCaptureAccessQuery = struct { + active: bool = false, + key: u64 = 0, + on_event: ?AudioCaptureAccessMsgFn = null, + }; + /// Playback state the automation snapshot exposes: honest — it /// reports what the platform has told us, not what the UI wishes. pub const AudioSnapshot = struct { @@ -3436,6 +3555,10 @@ pub fn Effects(comptime Msg: type) type { /// `takeAudioMsg`. Non-resolving entries (rejections and /// synchronous failures) are fully formed at enqueue. audio: struct { event: EffectAudio, audio_fn: ?AudioMsgFn, resolve: bool }, + audio_capture: struct { event: EffectAudioCapture, capture_fn: ?AudioCaptureMsgFn }, + microphone_device: struct { event: EffectMicrophoneDevice, device_fn: ?MicrophoneDeviceMsgFn }, + audio_capture_access: struct { event: EffectAudioCaptureAccess, access_fn: ?AudioCaptureAccessMsgFn }, + microphone_devices_changed: struct { changed_fn: ?MicrophoneDevicesChangedMsgFn }, /// The audio entry's shape for the video channel, staged /// in the non-lossy `pending_videos` (see `PendingVideo`) /// and taking this union shape only at drain time. @@ -3493,6 +3616,7 @@ pub fn Effects(comptime Msg: type) type { // EffectAudio carries no drop counter either; the // next position tick supersedes a lost one. .audio => {}, + .audio_capture, .microphone_device, .audio_capture_access, .microphone_devices_changed => {}, // Video events never enter the ring (they stage in // the non-lossy `pending_videos`): a loop-side // `.rejected`/`.failed` is its load's only @@ -3532,6 +3656,7 @@ pub fn Effects(comptime Msg: type) type { .clipboard => |entry| entry.result.dropped_before, .timer => 0, .audio => 0, + .audio_capture, .microphone_device, .audio_capture_access, .microphone_devices_changed => 0, .pty => 0, .host => 0, // Never in the ring; see `addDropped`. @@ -4254,6 +4379,10 @@ pub fn Effects(comptime Msg: type) type { /// The single audio playback channel (see `AudioChannel`). /// Loop-thread only, like the timer table. audio: AudioChannel = .{}, + audio_capture: AudioCaptureChannel = .{}, + microphone_device_query: MicrophoneDeviceQuery = .{}, + audio_capture_access_query: AudioCaptureAccessQuery = .{}, + microphone_devices_changed_fn: ?MicrophoneDevicesChangedMsgFn = null, /// The single video playback channel (see `VideoChannel`). video: VideoChannel = .{}, /// Monotonic per-load video token mint (see @@ -4485,6 +4614,16 @@ pub fn Effects(comptime Msg: type) type { if (self.services) |services| services.audioStop() catch {}; } self.audio = .{}; + if (self.audio_capture.active and !self.audio_capture.fake) { + if (self.services) |services| services.audioCaptureStop() catch {}; + } + if (self.microphone_devices_changed_fn != null) { + if (self.services) |services| services.observeMicrophoneDevices(false) catch {}; + } + self.audio_capture = .{}; + self.microphone_device_query = .{}; + self.audio_capture_access_query = .{}; + self.microphone_devices_changed_fn = null; // Stop the platform video player (best effort), release the // media-surface claim, and clear the channel. if (self.video.active and !self.video.fake) { @@ -7824,6 +7963,194 @@ pub fn Effects(comptime Msg: type) type { services.audioSetVolume(clamped) catch {}; } + pub fn startAudioCapture(self: *Self, options: StartAudioCaptureOptions) void { + var microphone_kind: platform.MicrophoneSelectionKind = .none; + var microphone_id: []const u8 = &.{}; + switch (options.microphone) { + .none => {}, + .default => microphone_kind = .default, + .device_id => |id| { + microphone_kind = .device_id; + microphone_id = id; + }, + } + const valid_rate = options.sample_rate_hz == 16_000 or options.sample_rate_hz == 24_000 or + options.sample_rate_hz == 44_100 or options.sample_rate_hz == 48_000; + const rejected = options.path.len == 0 or options.path.len > platform.max_audio_capture_path_bytes or + (!options.system_audio and microphone_kind == .none) or + (microphone_kind == .device_id and (microphone_id.len == 0 or microphone_id.len > platform.max_microphone_device_id_bytes)) or + !valid_rate or (options.channel_count != 1 and options.channel_count != 2); + if (rejected) { + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .invalid_options, + }, .capture_fn = options.on_event } }); + return; + } + if (self.audio_capture.active) { + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = options.key, + .state = .rejected, + .reason = .already_recording, + }, .capture_fn = options.on_event } }); + return; + } + self.audio_capture = .{ + .active = true, + .fake = self.executor == .fake, + .key = options.key, + .on_event = options.on_event, + }; + if (self.audio_capture.fake) { + // Session replay parks the request until the recorded + // platform events arrive. Ordinary fake-executor tests + // keep their deterministic synthetic lifecycle. + if (self.replay) return; + self.deliverPending(.{ .audio_capture = .{ .event = .{ .key = options.key, .state = .started }, .capture_fn = options.on_event } }); + return; + } + const services = self.services orelse return self.rejectAudioCapture(.unsupported); + services.audioCaptureStart(.{ + .path = options.path, + .system_audio = options.system_audio, + .microphone = microphone_kind, + .microphone_device_id = microphone_id, + .sample_rate_hz = options.sample_rate_hz, + .channel_count = options.channel_count, + .exclude_current_process_audio = options.exclude_current_process_audio, + }) catch |err| return self.rejectAudioCapture(switch (err) { + error.UnsupportedService => .unsupported, + error.PermissionMissing => .permission_missing, + error.AudioCapturePermissionRequired => .permission_required, + error.AudioCaptureAlreadyActive => .already_recording, + error.MicrophoneDeviceNotFound => .device_not_found, + error.AudioCaptureOutputExists => .output_exists, + error.AudioCaptureIoFailed => .io_failed, + error.InvalidAudioCaptureOptions => .invalid_options, + else => .capture_failed, + }); + } + + pub fn stopAudioCapture(self: *Self) void { + if (!self.audio_capture.active) return; + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + if (self.audio_capture.fake) { + // The recorded terminal event is authoritative during + // replay; keep the parked route alive until it arrives. + if (self.replay) return; + self.audio_capture = .{}; + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .stopped, + .output_committed = true, + }, .capture_fn = on_event } }); + return; + } + const services = self.services orelse return self.failAudioCapture(.unsupported); + services.audioCaptureStop() catch return self.failAudioCapture(.capture_failed); + } + + pub fn listMicrophoneDevices(self: *Self, options: ListMicrophoneDevicesOptions) void { + if (self.microphone_device_query.active) { + self.deliverPending(.{ .microphone_device = .{ .event = .{ + .key = options.key, + .state = .rejected, + }, .device_fn = options.on_event } }); + return; + } + self.microphone_device_query = .{ .active = true, .key = options.key, .on_event = options.on_event }; + if (self.executor == .fake) { + if (self.replay) return; + self.deliverPending(.{ .microphone_device = .{ .event = .{ .key = options.key, .state = .device, .id = "default-mic", .name = "Default Microphone", .is_default = true, .index = 0, .total = 2 }, .device_fn = options.on_event } }); + self.deliverPending(.{ .microphone_device = .{ .event = .{ .key = options.key, .state = .device, .id = "usb-mic", .name = "USB Microphone", .index = 1, .total = 2 }, .device_fn = options.on_event } }); + self.deliverPending(.{ .microphone_device = .{ .event = .{ .key = options.key, .state = .completed, .index = 2, .total = 2 }, .device_fn = options.on_event } }); + self.microphone_device_query = .{}; + return; + } + const services = self.services orelse return self.finishMicrophoneDevices(.rejected); + services.microphoneDevices() catch |err| return self.finishMicrophoneDevices(if (err == error.UnsupportedService or err == error.PermissionMissing) .rejected else .failed); + } + + pub fn audioCaptureAccess(self: *Self, options: AudioCaptureAccessOptions) void { + if (self.audio_capture_access_query.active) { + self.deliverPending(.{ .audio_capture_access = .{ .event = .{ + .key = options.key, + .source = options.source, + .status = .unavailable, + }, .access_fn = options.on_event } }); + return; + } + self.audio_capture_access_query = .{ .active = true, .key = options.key, .on_event = options.on_event }; + if (self.executor == .fake) { + if (self.replay) return; + self.deliverPending(.{ .audio_capture_access = .{ .event = .{ + .key = options.key, + .source = options.source, + .status = .authorized, + }, .access_fn = options.on_event } }); + self.audio_capture_access_query = .{}; + return; + } + const services = self.services orelse return self.failAudioCaptureAccess(options.source); + services.audioCaptureAccess(options.source, options.action) catch return self.failAudioCaptureAccess(options.source); + } + + pub fn observeMicrophoneDevices(self: *Self, on_change: ?MicrophoneDevicesChangedMsgFn) void { + self.microphone_devices_changed_fn = on_change; + if (self.executor == .fake) return; + const services = self.services orelse return; + services.observeMicrophoneDevices(on_change != null) catch {}; + } + + pub fn takeAudioCaptureMsg(self: *Self, event: platform.AudioCaptureEvent) ?Msg { + if (!self.audio_capture.active) return null; + const key = self.audio_capture.key; + const event_fn = self.audio_capture.on_event; + if (event.state != .started) self.audio_capture = .{}; + const map = event_fn orelse return null; + return map(.{ + .key = key, + .state = event.state, + .reason = event.reason, + .duration_ms = event.duration_ms, + .bytes_written = event.bytes_written, + .output_committed = event.output_committed, + }); + } + + pub fn takeMicrophoneDeviceMsg(self: *Self, event: platform.MicrophoneDeviceEvent) ?Msg { + if (!self.microphone_device_query.active) return null; + const key = self.microphone_device_query.key; + const event_fn = self.microphone_device_query.on_event; + if (event.state != .device) self.microphone_device_query = .{}; + const map = event_fn orelse return null; + return map(.{ + .key = key, + .state = event.state, + .id = event.id, + .name = event.name, + .is_default = event.is_default, + .index = event.index, + .total = event.total, + }); + } + + pub fn takeAudioCaptureAccessMsg(self: *Self, event: platform.AudioCaptureAccessEvent) ?Msg { + if (!self.audio_capture_access_query.active) return null; + const key = self.audio_capture_access_query.key; + const event_fn = self.audio_capture_access_query.on_event; + self.audio_capture_access_query = .{}; + const map = event_fn orelse return null; + return map(.{ .key = key, .source = event.source, .status = event.status, .restart_required = event.restart_required }); + } + + pub fn takeMicrophoneDevicesChangedMsg(self: *Self) ?Msg { + const map = self.microphone_devices_changed_fn orelse return null; + return map(); + } + /// Route a platform audio event back into an `on_event` Msg for /// the active channel, updating the playback mirrors on the way. /// Null when the channel is idle (a straggler after `stopAudio`) @@ -8974,6 +9301,22 @@ pub fn Effects(comptime Msg: type) type { }); return event_fn(event); }, + .audio_capture => |entry| { + const event_fn = entry.capture_fn orelse continue; + return event_fn(entry.event); + }, + .microphone_device => |entry| { + const event_fn = entry.device_fn orelse continue; + return event_fn(entry.event); + }, + .audio_capture_access => |entry| { + const event_fn = entry.access_fn orelse continue; + return event_fn(entry.event); + }, + .microphone_devices_changed => |entry| { + const event_fn = entry.changed_fn orelse continue; + return event_fn(); + }, .video => |entry| { var event = entry.event; const video_fn = entry.video_fn; @@ -11250,6 +11593,49 @@ pub fn Effects(comptime Msg: type) type { self.deliverLoopAudio(.{ .key = key, .kind = .failed }, on_event); } + fn failAudioCapture(self: *Self, reason: EffectAudioCaptureReason) void { + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + self.audio_capture = .{}; + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .failed, + .reason = reason, + }, .capture_fn = on_event } }); + } + + fn rejectAudioCapture(self: *Self, reason: EffectAudioCaptureReason) void { + const key = self.audio_capture.key; + const on_event = self.audio_capture.on_event; + self.audio_capture = .{}; + self.deliverPending(.{ .audio_capture = .{ .event = .{ + .key = key, + .state = .rejected, + .reason = reason, + }, .capture_fn = on_event } }); + } + + fn finishMicrophoneDevices(self: *Self, state: EffectMicrophoneDeviceState) void { + const key = self.microphone_device_query.key; + const on_event = self.microphone_device_query.on_event; + self.microphone_device_query = .{}; + self.deliverPending(.{ .microphone_device = .{ .event = .{ + .key = key, + .state = state, + }, .device_fn = on_event } }); + } + + fn failAudioCaptureAccess(self: *Self, source: EffectAudioCaptureAccessSource) void { + const key = self.audio_capture_access_query.key; + const on_event = self.audio_capture_access_query.on_event; + self.audio_capture_access_query = .{}; + self.deliverPending(.{ .audio_capture_access = .{ .event = .{ + .key = key, + .source = source, + .status = .unavailable, + }, .access_fn = on_event } }); + } + /// Queue an audio event Msg produced on the loop thread /// (rejections and synchronous failures) for the next drain. fn deliverLoopAudio(self: *Self, event: EffectAudio, audio_fn: ?AudioMsgFn) void { diff --git a/src/runtime/effects_audio_tests.zig b/src/runtime/effects_audio_tests.zig index 87a3f5b0d..fc631d20c 100644 --- a/src/runtime/effects_audio_tests.zig +++ b/src/runtime/effects_audio_tests.zig @@ -621,6 +621,136 @@ fn effectsCachePathNoExt(buffer: []u8) ![]const u8 { return effects_mod.audioCachePath(buffer, "/tmp/caches/app", "https://music.example.test/stream?id=42"); } +const CaptureMsg = union(enum) { + capture: effects_mod.EffectAudioCapture, + device: effects_mod.EffectMicrophoneDevice, + access: effects_mod.EffectAudioCaptureAccess, + devices_changed, +}; +const CaptureFx = effects_mod.Effects(CaptureMsg); + +test "audio capture fake covers source combinations validation and duplicate starts" { + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.executor = .fake; + + fx.startAudioCapture(.{ .key = 1, .path = "system.wav", .system_audio = true, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeMsg().?.capture.state); + fx.startAudioCapture(.{ .key = 2, .path = "duplicate.wav", .microphone = .default, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const duplicate = fx.takeMsg().?.capture; + try std.testing.expectEqual(@as(u64, 2), duplicate.key); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, duplicate.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.already_recording, duplicate.reason); + fx.stopAudioCapture(); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.stopped, fx.takeMsg().?.capture.state); + + fx.startAudioCapture(.{ .key = 3, .path = "microphone.wav", .microphone = .default, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeMsg().?.capture.state); + fx.stopAudioCapture(); + _ = fx.takeMsg(); + + fx.startAudioCapture(.{ .key = 4, .path = "combined.wav", .system_audio = true, .microphone = .{ .device_id = "usb-mic" }, .sample_rate_hz = 44_100, .channel_count = 1, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeMsg().?.capture.state); + fx.stopAudioCapture(); + _ = fx.takeMsg(); + + fx.startAudioCapture(.{ .key = 5, .path = "invalid.wav", .sample_rate_hz = 12_345, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const invalid = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, invalid.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.invalid_options, invalid.reason); +} + +test "audio capture fake lists fixed device records and reports access" { + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.executor = .fake; + + fx.listMicrophoneDevices(.{ .key = 7, .on_event = CaptureFx.microphoneDeviceMsg(.device) }); + const first = fx.takeMsg().?.device; + const second = fx.takeMsg().?.device; + const completed = fx.takeMsg().?.device; + try std.testing.expectEqual(effects_mod.EffectMicrophoneDeviceState.device, first.state); + try std.testing.expectEqualStrings("default-mic", first.id); + try std.testing.expect(first.is_default); + try std.testing.expectEqualStrings("usb-mic", second.id); + try std.testing.expectEqual(effects_mod.EffectMicrophoneDeviceState.completed, completed.state); + try std.testing.expectEqual(@as(u32, 2), completed.total); + + fx.audioCaptureAccess(.{ .key = 8, .source = .microphone, .action = .status, .on_event = CaptureFx.audioCaptureAccessMsg(.access) }); + const access = fx.takeMsg().?.access; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureAccessStatus.authorized, access.status); + + fx.observeMicrophoneDevices(CaptureFx.microphoneDevicesChangedMsg(.devices_changed)); + try std.testing.expect(fx.takeMicrophoneDevicesChangedMsg().? == .devices_changed); +} + +test "null audio capture resolves defaults rejects missing devices and preserves partial disconnects" { + var null_platform: platform.NullPlatform = .{}; + var platform_value = null_platform.platform(); + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.bindServices(&platform_value.services); + + try null_platform.setDefaultMicrophone(1); + fx.startAudioCapture(.{ .key = 10, .path = "default.wav", .microphone = .default, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + try std.testing.expectEqualStrings("usb-mic", null_platform.capture.microphoneDeviceId()); + const started_event = null_platform.takeAudioCaptureStarted().?.audio_capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.started, fx.takeAudioCaptureMsg(started_event).?.capture.state); + const disconnected_event = null_platform.disconnectMicrophone("usb-mic", 1250, 4096).?.audio_capture; + const disconnected = fx.takeAudioCaptureMsg(disconnected_event).?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.failed, disconnected.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.device_disconnected, disconnected.reason); + try std.testing.expect(disconnected.output_committed); + try std.testing.expectEqual(@as(u64, 4096), disconnected.bytes_written); + + fx.startAudioCapture(.{ .key = 11, .path = "missing.wav", .microphone = .{ .device_id = "missing" }, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const missing = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, missing.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.device_not_found, missing.reason); + + null_platform.audio_capture_output_exists = true; + fx.startAudioCapture(.{ .key = 12, .path = "exists.wav", .system_audio = true, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const collision = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, collision.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.output_exists, collision.reason); + null_platform.audio_capture_output_exists = false; + + null_platform.system_audio_access = .not_authorized; + fx.startAudioCapture(.{ .key = 13, .path = "permission.wav", .system_audio = true, .on_event = CaptureFx.audioCaptureMsg(.capture) }); + const permission = fx.takeMsg().?.capture; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureState.rejected, permission.state); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureReason.permission_required, permission.reason); +} + +test "null microphone listing access and device-change observation are deterministic" { + var null_platform: platform.NullPlatform = .{}; + var platform_value = null_platform.platform(); + var fx = CaptureFx.init(std.testing.allocator); + defer fx.deinit(); + fx.bindServices(&platform_value.services); + + try null_platform.setDefaultMicrophone(1); + fx.listMicrophoneDevices(.{ .key = 20, .on_event = CaptureFx.microphoneDeviceMsg(.device) }); + try std.testing.expectEqual(@as(usize, 1), null_platform.microphone_devices_count); + const first = fx.takeMicrophoneDeviceMsg(null_platform.microphoneDeviceEvent(0).microphone_device).?.device; + const second = fx.takeMicrophoneDeviceMsg(null_platform.microphoneDeviceEvent(1).microphone_device).?.device; + const completed = fx.takeMicrophoneDeviceMsg(null_platform.microphoneDeviceEvent(2).microphone_device).?.device; + try std.testing.expect(!first.is_default); + try std.testing.expect(second.is_default); + try std.testing.expectEqual(effects_mod.EffectMicrophoneDeviceState.completed, completed.state); + + null_platform.microphone_access = .denied; + fx.audioCaptureAccess(.{ .key = 21, .source = .microphone, .action = .status, .on_event = CaptureFx.audioCaptureAccessMsg(.access) }); + const access_event = null_platform.takeAudioCaptureAccess().?.audio_capture_access; + try std.testing.expectEqual(effects_mod.EffectAudioCaptureAccessStatus.denied, fx.takeAudioCaptureAccessMsg(access_event).?.access.status); + + fx.observeMicrophoneDevices(CaptureFx.microphoneDevicesChangedMsg(.devices_changed)); + try std.testing.expect(null_platform.microphone_devices_observing); + try std.testing.expect(fx.takeMicrophoneDevicesChangedMsg().? == .devices_changed); + fx.observeMicrophoneDevices(null); + try std.testing.expect(!null_platform.microphone_devices_observing); +} + test "quit while playing: the stop hook silences audio on the live platform, and app deinit after platform teardown answers inert" { // The desktop runner's exit ordering, replayed exactly: main defers // app deinit FIRST and calls the runner, whose own defers destroy diff --git a/src/runtime/flow.zig b/src/runtime/flow.zig index 68ba0741a..4dc3e40bb 100644 --- a/src/runtime/flow.zig +++ b/src/runtime/flow.zig @@ -450,6 +450,10 @@ pub fn RuntimeFlow(comptime Runtime: type) type { .audio => |audio_event| { try dispatchEvent(self, app, .{ .audio = audio_event }); }, + .audio_capture => |capture_event| try dispatchEvent(self, app, .{ .audio_capture = capture_event }), + .microphone_device => |device_event| try dispatchEvent(self, app, .{ .microphone_device = device_event }), + .microphone_devices_changed => try dispatchEvent(self, app, .microphone_devices_changed), + .audio_capture_access => |access_event| try dispatchEvent(self, app, .{ .audio_capture_access = access_event }), .video => |video_event| { try dispatchEvent(self, app, .{ .video = video_event }); }, @@ -518,6 +522,10 @@ pub fn RuntimeFlow(comptime Runtime: type) type { .timer => {}, .effects_wake => {}, .audio => {}, + .audio_capture => {}, + .microphone_device => {}, + .microphone_devices_changed => {}, + .audio_capture_access => {}, .video => {}, .files_dropped => {}, .gpu_surface_frame => {}, diff --git a/src/runtime/root.zig b/src/runtime/root.zig index e5c0a4d57..77acd0ff4 100644 --- a/src/runtime/root.zig +++ b/src/runtime/root.zig @@ -96,6 +96,16 @@ pub const effect_timer_platform_id_base = runtime_effects.effect_timer_platform_ pub const EffectAudio = runtime_effects.EffectAudio; pub const EffectAudioEventKind = runtime_effects.EffectAudioEventKind; pub const EffectAudioSource = runtime_effects.EffectAudioSource; +pub const EffectAudioCapture = runtime_effects.EffectAudioCapture; +pub const EffectAudioCaptureState = runtime_effects.EffectAudioCaptureState; +pub const EffectAudioCaptureReason = runtime_effects.EffectAudioCaptureReason; +pub const EffectMicrophoneDevice = runtime_effects.EffectMicrophoneDevice; +pub const EffectMicrophoneDeviceState = runtime_effects.EffectMicrophoneDeviceState; +pub const EffectAudioCaptureAccess = runtime_effects.EffectAudioCaptureAccess; +pub const EffectAudioCaptureAccessSource = runtime_effects.EffectAudioCaptureAccessSource; +pub const EffectAudioCaptureAccessAction = runtime_effects.EffectAudioCaptureAccessAction; +pub const EffectAudioCaptureAccessStatus = runtime_effects.EffectAudioCaptureAccessStatus; +pub const MicrophoneSelection = runtime_effects.MicrophoneSelection; pub const audioCachePath = runtime_effects.audioCachePath; pub const max_effect_audio_path_bytes = runtime_effects.max_effect_audio_path_bytes; pub const EffectVideo = runtime_effects.EffectVideo; diff --git a/src/runtime/session_journal.zig b/src/runtime/session_journal.zig index fdb2e9cf6..9fdc09ee8 100644 --- a/src/runtime/session_journal.zig +++ b/src/runtime/session_journal.zig @@ -186,6 +186,9 @@ fn formatLayoutDescription(comptime epoch: u32) []const u8 { "menu_command=" ++ layout_fingerprint.describe(platform.MenuCommandEvent) ++ "\n" ++ "timer=" ++ layout_fingerprint.describe(platform.TimerEvent) ++ "\n" ++ "audio=" ++ layout_fingerprint.describe(platform.AudioEvent) ++ "\n" ++ + "audio_capture=" ++ layout_fingerprint.describe(platform.AudioCaptureEvent) ++ "\n" ++ + "microphone_device=" ++ layout_fingerprint.describe(platform.MicrophoneDeviceEvent) ++ "\n" ++ + "audio_capture_access=" ++ layout_fingerprint.describe(platform.AudioCaptureAccessEvent) ++ "\n" ++ "video=" ++ layout_fingerprint.describe(platform.VideoEvent) ++ "\n" ++ "files_dropped=" ++ layout_fingerprint.describe(platform.FileDropEvent) ++ "\n" ++ // gpu_surface_frame journals a deliberate SUBSET of a @@ -464,6 +467,10 @@ const EventTag = enum(u8) { audio = 24, video = 25, view_focused = 26, + audio_capture = 27, + microphone_device = 28, + microphone_devices_changed = 29, + audio_capture_access = 30, }; // The bit assignments below are hand-written wire layout: they are @@ -622,6 +629,30 @@ pub fn encodeEvent(event: platform.Event, buffer: []u8) JournalError![]const u8 try cursor.writeBool(audio.buffering); try cursor.writeBytes(&audio.bands); }, + .audio_capture => |capture| { + try cursor.writeEnum(EventTag.audio_capture); + try cursor.writeEnum(capture.state); + try cursor.writeEnum(capture.reason); + try cursor.writeInt(u64, capture.duration_ms); + try cursor.writeInt(u64, capture.bytes_written); + try cursor.writeBool(capture.output_committed); + }, + .microphone_device => |device| { + try cursor.writeEnum(EventTag.microphone_device); + try cursor.writeEnum(device.state); + try cursor.writeStr(device.id); + try cursor.writeStr(device.name); + try cursor.writeBool(device.is_default); + try cursor.writeInt(u32, device.index); + try cursor.writeInt(u32, device.total); + }, + .microphone_devices_changed => try cursor.writeEnum(EventTag.microphone_devices_changed), + .audio_capture_access => |access| { + try cursor.writeEnum(EventTag.audio_capture_access); + try cursor.writeEnum(access.source); + try cursor.writeEnum(access.status); + try cursor.writeBool(access.restart_required); + }, // Recorded for stream fidelity like `.audio`. On replay the // journaled video EFFECT records are the Msg source; the // platform events steer only the channel MIRRORS (the house @@ -849,6 +880,27 @@ pub fn decodeEvent(bytes: []const u8, storage: *EventDecodeStorage) JournalError @memcpy(&decoded.bands, try cursor.readBytes(decoded.bands.len)); break :blk .{ .audio = decoded }; }, + .audio_capture => .{ .audio_capture = .{ + .state = try cursor.readEnum(platform.AudioCaptureEventState), + .reason = try cursor.readEnum(platform.AudioCaptureEventReason), + .duration_ms = try cursor.readInt(u64), + .bytes_written = try cursor.readInt(u64), + .output_committed = try cursor.readBool(), + } }, + .microphone_device => .{ .microphone_device = .{ + .state = try cursor.readEnum(platform.MicrophoneDeviceEventState), + .id = try cursor.readStr(), + .name = try cursor.readStr(), + .is_default = try cursor.readBool(), + .index = try cursor.readInt(u32), + .total = try cursor.readInt(u32), + } }, + .microphone_devices_changed => .microphone_devices_changed, + .audio_capture_access => .{ .audio_capture_access = .{ + .source = try cursor.readEnum(platform.AudioCaptureAccessSource), + .status = try cursor.readEnum(platform.AudioCaptureAccessStatus), + .restart_required = try cursor.readBool(), + } }, .video => blk: { const kind = try cursor.readEnum(platform.VideoEventKind); break :blk .{ .video = .{ @@ -1483,6 +1535,47 @@ test "event codec round-trips every payload variant" { try testing.expectEqual(@as(u64, 1280), decoded.video.width); try testing.expectEqual(@as(u64, 720), decoded.video.height); } + { + const decoded = try roundTripEvent(.{ .audio_capture = .{ + .state = .failed, + .reason = .device_disconnected, + .duration_ms = 1250, + .bytes_written = 4096, + .output_committed = true, + } }); + try testing.expectEqual(platform.AudioCaptureEventState.failed, decoded.audio_capture.state); + try testing.expectEqual(platform.AudioCaptureEventReason.device_disconnected, decoded.audio_capture.reason); + try testing.expectEqual(@as(u64, 1250), decoded.audio_capture.duration_ms); + try testing.expectEqual(@as(u64, 4096), decoded.audio_capture.bytes_written); + try testing.expect(decoded.audio_capture.output_committed); + } + { + const decoded = try roundTripEvent(.{ .microphone_device = .{ + .state = .device, + .id = "usb-mic", + .name = "USB Microphone", + .is_default = true, + .index = 1, + .total = 2, + } }); + try testing.expectEqual(platform.MicrophoneDeviceEventState.device, decoded.microphone_device.state); + try testing.expectEqualStrings("usb-mic", decoded.microphone_device.id); + try testing.expectEqualStrings("USB Microphone", decoded.microphone_device.name); + try testing.expect(decoded.microphone_device.is_default); + try testing.expectEqual(@as(u32, 2), decoded.microphone_device.total); + } + { + const changed = try roundTripEvent(.microphone_devices_changed); + try testing.expect(changed == .microphone_devices_changed); + const decoded = try roundTripEvent(.{ .audio_capture_access = .{ + .source = .system_audio, + .status = .authorized, + .restart_required = true, + } }); + try testing.expectEqual(platform.AudioCaptureAccessSource.system_audio, decoded.audio_capture_access.source); + try testing.expectEqual(platform.AudioCaptureAccessStatus.authorized, decoded.audio_capture_access.status); + try testing.expect(decoded.audio_capture_access.restart_required); + } { const paths = [_][]const u8{ "/tmp/a.txt", "/tmp/b.txt" }; const decoded = try roundTripEvent(.{ .files_dropped = .{ diff --git a/src/runtime/session_tests.zig b/src/runtime/session_tests.zig index a5e94e238..f4f204d5e 100644 --- a/src/runtime/session_tests.zig +++ b/src/runtime/session_tests.zig @@ -633,6 +633,181 @@ test "a native-only session records and replays like a web-layer one" { try std.testing.expectEqual(recorded.fingerprint, replayed.fingerprint); } +const AudioCaptureSessionModel = struct { + capture_events: u32 = 0, + capture_state: u8 = 0, + capture_reason: u8 = 0, + duration_ms: u64 = 0, + bytes_written: u64 = 0, + output_committed: bool = false, + microphone_records: u32 = 0, + microphone_listing_completed: bool = false, + microphone_access: effects_mod.EffectAudioCaptureAccessStatus = .unavailable, + restart_required: bool = false, + device_changes: u32 = 0, +}; + +const AudioCaptureSessionMsg = union(enum) { + start, + stop, + list_microphones, + check_access, + capture: effects_mod.EffectAudioCapture, + microphone: effects_mod.EffectMicrophoneDevice, + access: effects_mod.EffectAudioCaptureAccess, + microphones_changed, +}; + +const AudioCaptureSessionApp = ui_app_mod.UiApp(AudioCaptureSessionModel, AudioCaptureSessionMsg); + +fn audioCaptureSessionBoot(_: *AudioCaptureSessionModel, fx: *AudioCaptureSessionApp.Effects) void { + fx.observeMicrophoneDevices(AudioCaptureSessionApp.Effects.microphoneDevicesChangedMsg(.microphones_changed)); +} + +fn audioCaptureSessionUpdate(model: *AudioCaptureSessionModel, msg: AudioCaptureSessionMsg, fx: *AudioCaptureSessionApp.Effects) void { + switch (msg) { + .start => fx.startAudioCapture(.{ + .key = 41, + .path = "session-capture.wav", + .system_audio = true, + .microphone = .default, + .on_event = AudioCaptureSessionApp.Effects.audioCaptureMsg(.capture), + }), + .stop => fx.stopAudioCapture(), + .list_microphones => fx.listMicrophoneDevices(.{ + .key = 42, + .on_event = AudioCaptureSessionApp.Effects.microphoneDeviceMsg(.microphone), + }), + .check_access => fx.audioCaptureAccess(.{ + .key = 43, + .source = .microphone, + .on_event = AudioCaptureSessionApp.Effects.audioCaptureAccessMsg(.access), + }), + .capture => |event| { + model.capture_events += 1; + model.capture_state = @intFromEnum(event.state) + 1; + model.capture_reason = @intFromEnum(event.reason); + model.duration_ms = event.duration_ms; + model.bytes_written = event.bytes_written; + model.output_committed = event.output_committed; + }, + .microphone => |event| switch (event.state) { + .device => model.microphone_records += 1, + .completed => model.microphone_listing_completed = true, + .failed, .rejected => {}, + }, + .access => |event| { + model.microphone_access = event.status; + model.restart_required = event.restart_required; + }, + .microphones_changed => model.device_changes += 1, + } +} + +fn audioCaptureSessionView(ui: *AudioCaptureSessionApp.Ui, model: *const AudioCaptureSessionModel) AudioCaptureSessionApp.Ui.Node { + return ui.text(.{}, ui.fmt("capture {d} · microphones {d}", .{ model.capture_events, model.microphone_records })); +} + +fn audioCaptureSessionCommand(name: []const u8) ?AudioCaptureSessionMsg { + if (std.mem.eql(u8, name, "capture.start")) return .start; + if (std.mem.eql(u8, name, "capture.stop")) return .stop; + if (std.mem.eql(u8, name, "capture.microphones")) return .list_microphones; + if (std.mem.eql(u8, name, "capture.access")) return .check_access; + return null; +} + +fn audioCaptureSessionOptions() AudioCaptureSessionApp.Options { + return .{ + .name = "audio-capture-session", + .scene = session_scene, + .canvas_label = canvas_label, + .init_fx = audioCaptureSessionBoot, + .update_fx = audioCaptureSessionUpdate, + .view = audioCaptureSessionView, + .on_command = audioCaptureSessionCommand, + }; +} + +test "audio capture device and access platform events replay without hardware side effects" { + const gpa = std.testing.allocator; + const buffer = try std.heap.page_allocator.create(JournalBuffer); + defer std.heap.page_allocator.destroy(buffer); + buffer.len = 0; + + const recorder = try std.heap.page_allocator.create(session_record.SessionRecorder); + defer std.heap.page_allocator.destroy(recorder); + recorder.* = session_record.SessionRecorder.init(buffer.sink()); + recorder.begin(.{ .platform_name = "test", .app_name = "audio-capture-session", .window_width = 400, .window_height = 300 }); + + const record_harness = try core.TestHarness().create(gpa, .{ .size = geometry.SizeF.init(400, 300) }); + defer record_harness.destroy(gpa); + record_harness.null_platform.gpu_surfaces = true; + record_harness.runtime.options.session_recorder = recorder; + + const recorded_app = try gpa.create(AudioCaptureSessionApp); + defer gpa.destroy(recorded_app); + recorded_app.* = AudioCaptureSessionApp.init(std.heap.page_allocator, .{}, audioCaptureSessionOptions()); + defer recorded_app.deinit(); + const app = recorded_app.app(); + + try record_harness.start(app); + try record_harness.runtime.dispatchPlatformEvent(app, .{ .gpu_surface_frame = .{ + .label = canvas_label, + .size = geometry.SizeF.init(400, 300), + .scale_factor = 2, + .frame_index = 1, + .timestamp_ns = 1_000_000, + } }); + try record_harness.runtime.dispatchPlatformEvent(app, .frame_requested); + + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.start", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.takeAudioCaptureStarted().?); + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.stop", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.completeAudioCapture(1_250, 4_096).?); + + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.microphones", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDeviceEvent(0)); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDeviceEvent(1)); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDeviceEvent(2)); + + try record_harness.runtime.dispatchPlatformEvent(app, .{ .menu_command = .{ .name = "capture.access", .window_id = 1 } }); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.takeAudioCaptureAccess().?); + try record_harness.runtime.dispatchPlatformEvent(app, record_harness.null_platform.microphoneDevicesChanged().?); + try record_harness.runtime.dispatchPlatformEvent(app, .frame_requested); + + recorder.finish(); + try std.testing.expect(!recorder.failed); + const recorded_model = recorded_app.model; + const recorded_fingerprint = record_harness.runtime.sessionStateFingerprint(); + try std.testing.expectEqual(@as(u32, 2), recorded_model.capture_events); + try std.testing.expectEqual(@as(u64, 1_250), recorded_model.duration_ms); + try std.testing.expectEqual(@as(u64, 4_096), recorded_model.bytes_written); + try std.testing.expect(recorded_model.output_committed); + try std.testing.expectEqual(@as(u32, 2), recorded_model.microphone_records); + try std.testing.expect(recorded_model.microphone_listing_completed); + try std.testing.expectEqual(effects_mod.EffectAudioCaptureAccessStatus.authorized, recorded_model.microphone_access); + try std.testing.expectEqual(@as(u32, 1), recorded_model.device_changes); + + const replay_harness = try core.TestHarness().create(gpa, .{ .size = geometry.SizeF.init(400, 300) }); + defer replay_harness.destroy(gpa); + replay_harness.null_platform.gpu_surfaces = true; + const replayed_app = try gpa.create(AudioCaptureSessionApp); + defer gpa.destroy(replayed_app); + replayed_app.* = AudioCaptureSessionApp.init(std.heap.page_allocator, .{}, audioCaptureSessionOptions()); + defer replayed_app.deinit(); + + const report = try session_replay.replaySession(&replay_harness.runtime, replayed_app.app(), buffer.journalBytes(), .{ + .verify = true, + .require_same_platform = false, + }); + try std.testing.expect(report.ok()); + try std.testing.expectEqual(@as(usize, 0), replay_harness.null_platform.audio_capture_start_count); + try std.testing.expectEqual(@as(usize, 0), replay_harness.null_platform.audio_capture_stop_count); + try std.testing.expectEqual(@as(usize, 0), replay_harness.null_platform.microphone_devices_count); + try std.testing.expectEqualDeep(recorded_model, replayed_app.model); + try std.testing.expectEqual(recorded_fingerprint, replay_harness.runtime.sessionStateFingerprint()); +} + test "accessibility actions journal once and replay without double-dispatch" { // A journaled `widget_accessibility_action` re-runs its verb on // replay, and the verb synthesizes REAL platform events (press its diff --git a/src/runtime/ts_core_host.zig b/src/runtime/ts_core_host.zig index 33562b992..e3187226b 100644 --- a/src/runtime/ts_core_host.zig +++ b/src/runtime/ts_core_host.zig @@ -1,6 +1,6 @@ //! The native host consumer for transpiled app cores: bridges the //! versioned command/subscription wire format a transpiled core emits -//! (`packages/core/rt/rt.zig`, `cmd_format_version` 3) onto the +//! (`packages/core/rt/rt.zig`, `cmd_format_version` 4) onto the //! real effect engine (`effects.zig`). The transpiler's output is a //! pure Model/Msg/update core whose effects are INERT BYTES — this //! module is the one place those bytes become engine calls, so the @@ -390,6 +390,9 @@ pub const spawn_key_base: u64 = 0x5453_5350_0000_0000; /// ("TSAU"). Audio keys are their own engine namespace and one player /// is the whole surface, so one constant key is the honest shape. pub const audio_key_base: u64 = 0x5453_4155_0000_0000; +pub const audio_capture_key_base: u64 = 0x5453_4143_0000_0000; +pub const microphone_devices_key_base: u64 = 0x5453_4D44_0000_0000; +pub const audio_capture_access_key_base: u64 = 0x5453_4141_0000_0000; /// The engine-key namespace of the bridge's single video playback /// channel ("TSVI") — the audio key's twin, except the low byte @@ -544,6 +547,8 @@ pub fn TsCoreHost(comptime core: type) type { } }; + const RoutedStreamEntry = AudioEntry; + /// The single video stream entry — the audio entry's exact /// shape (one player is the whole engine surface). Non-retiring: /// video_ctl `stop` closes it, a new video_load re-keys and @@ -621,6 +626,10 @@ pub fn TsCoreHost(comptime core: type) type { var delays: [runtime_effects.max_effect_timers]DelayEntry = @splat(.{}); var streams: [runtime_effects.max_effects]StreamEntry = @splat(.{}); var audio_entry: AudioEntry = .{}; + var audio_capture_entries: [runtime_effects.max_effects]RoutedStreamEntry = @splat(.{}); + var microphone_device_entries: [runtime_effects.max_effects]RoutedStreamEntry = @splat(.{}); + var audio_capture_access_entries: [runtime_effects.max_effects]RoutedStreamEntry = @splat(.{}); + var microphone_devices_changed_tag: ?u8 = null; var video_entry: VideoEntry = .{}; var images: [runtime_effects.max_effects]ImageEntry = @splat(.{}); var channels: [runtime_effects.max_effect_channels]ChannelEntry = @splat(.{}); @@ -695,6 +704,10 @@ pub fn TsCoreHost(comptime core: type) type { delays = @splat(.{}); streams = @splat(.{}); audio_entry = .{}; + audio_capture_entries = @splat(.{}); + microphone_device_entries = @splat(.{}); + audio_capture_access_entries = @splat(.{}); + microphone_devices_changed_tag = null; video_entry = .{}; images = @splat(.{}); channels = @splat(.{}); @@ -1217,6 +1230,70 @@ pub fn TsCoreHost(comptime core: type) type { // retiring the entry in ptyEventMsg. if (findPty(key)) |index| fx.ptyKill(pty_key_base + index); }, + // audio_capture_start [op][key][event][flags][mic kind] + // [sample rate u32][channels][path long][device id long] + 0x1D => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const flags = takeByte(cmd, &at); + const microphone_kind = takeByte(cmd, &at); + const rate_bytes = takeBytes(cmd, &at, 4); + const sample_rate = std.mem.readInt(u32, rate_bytes[0..4], .little); + const channels_count = takeByte(cmd, &at); + const path = takeLongBytes(cmd, &at); + const device_id = takeLongBytes(cmd, &at); + const index = allocRoutedStreamEntry(&audio_capture_entries, key, event_tag) orelse + @panic("ts core host: more pending audio capture starts than the effects table can route"); + const microphone: runtime_effects.MicrophoneSelection = switch (microphone_kind) { + 0 => .none, + 1 => .default, + 2 => .{ .device_id = device_id }, + else => @panic("ts core host: invalid microphone selection wire value"), + }; + fx.startAudioCapture(.{ + .key = audio_capture_key_base + index, + .path = path, + .system_audio = (flags & 1) != 0, + .microphone = microphone, + .sample_rate_hz = sample_rate, + .channel_count = channels_count, + .exclude_current_process_audio = (flags & 2) != 0, + .on_event = audioCaptureEventMsg, + }); + }, + // audio_capture_stop [op][key] + 0x1E => { + const key = takeShortBytes(cmd, &at); + for (&audio_capture_entries) |*entry| { + if (entry.used and std.mem.eql(u8, entry.wireKey(), key)) { + fx.stopAudioCapture(); + break; + } + } + }, + // microphone_devices [op][key][event] + 0x1F => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const index = allocRoutedStreamEntry(µphone_device_entries, key, event_tag) orelse + @panic("ts core host: more pending microphone device queries than the effects table can route"); + fx.listMicrophoneDevices(.{ .key = microphone_devices_key_base + index, .on_event = microphoneDeviceEventMsg }); + }, + // audio_capture_access [op][key][event][source][action] + 0x20 => { + const key = takeShortBytes(cmd, &at); + const event_tag = takeByte(cmd, &at); + const source_byte = takeByte(cmd, &at); + const action_byte = takeByte(cmd, &at); + const index = allocRoutedStreamEntry(&audio_capture_access_entries, key, event_tag) orelse + @panic("ts core host: more pending audio access queries than the effects table can route"); + fx.audioCaptureAccess(.{ + .key = audio_capture_access_key_base + index, + .source = if (source_byte == 1) .microphone else .system_audio, + .action = if (action_byte == 1) .request else .status, + .on_event = audioCaptureAccessEventMsg, + }); + }, else => @panic("ts core host: unknown command wire record - the core and this runtime disagree on cmd_format_version"), } } @@ -1225,6 +1302,18 @@ pub fn TsCoreHost(comptime core: type) type { /// The shared routed-op head: [key_len][key][ok_tag][err_tag]. const RoutedHead = struct { key: []const u8, ok_tag: u8, err_tag: u8 }; + fn allocRoutedStreamEntry(entries: []RoutedStreamEntry, key: []const u8, event_tag: u8) ?usize { + for (entries, 0..) |*entry, index| { + if (entry.used) continue; + entry.used = true; + entry.key_len = key.len; + @memcpy(entry.key[0..key.len], key); + entry.event_tag = event_tag; + return index; + } + return null; + } + fn takeRoutedHead(cmd: []const u8, at: *usize) RoutedHead { const key = takeShortBytes(cmd, at); const ok_tag = takeByte(cmd, at); @@ -1473,6 +1562,45 @@ pub fn TsCoreHost(comptime core: type) type { return msgFromTagAudio(audio_entry.event_tag, event); } + fn audioCaptureEventMsg(event: runtime_effects.EffectAudioCapture) Msg { + if (event.key < audio_capture_key_base) @panic("ts core host: audio capture event outside bridge namespace"); + const index = event.key - audio_capture_key_base; + if (index >= audio_capture_entries.len or !audio_capture_entries[index].used) @panic("ts core host: audio capture event has no routed command"); + const entry = &audio_capture_entries[index]; + const tag = entry.event_tag; + const key = entry.wireKey(); + const msg = msgFromTagAudioCapture(tag, key, event); + if (event.state != .started) entry.used = false; + return msg; + } + + fn microphoneDeviceEventMsg(event: runtime_effects.EffectMicrophoneDevice) Msg { + if (event.key < microphone_devices_key_base) @panic("ts core host: microphone event outside bridge namespace"); + const index = event.key - microphone_devices_key_base; + if (index >= microphone_device_entries.len or !microphone_device_entries[index].used) @panic("ts core host: microphone event has no routed command"); + const entry = µphone_device_entries[index]; + const tag = entry.event_tag; + const key = entry.wireKey(); + const msg = msgFromTagMicrophoneDevice(tag, key, event); + if (event.state != .device) entry.used = false; + return msg; + } + + fn audioCaptureAccessEventMsg(event: runtime_effects.EffectAudioCaptureAccess) Msg { + if (event.key < audio_capture_access_key_base) @panic("ts core host: audio access event outside bridge namespace"); + const index = event.key - audio_capture_access_key_base; + if (index >= audio_capture_access_entries.len or !audio_capture_access_entries[index].used) @panic("ts core host: audio access event has no routed command"); + const entry = &audio_capture_access_entries[index]; + const msg = msgFromTagAudioCaptureAccess(entry.event_tag, entry.wireKey(), event); + entry.used = false; + return msg; + } + + fn microphoneDevicesChangedMsg() Msg { + return msgFromTagVoid(microphone_devices_changed_tag orelse + @panic("ts core host: microphone device invalidation arrived without a subscription")); + } + // ------------------------------------------------- video stream /// The video_ctl record: drive the single playback channel, @@ -2119,9 +2247,17 @@ pub fn TsCoreHost(comptime core: type) type { if (comptime !has_subscriptions) return; const subs = core.subscriptions(model_root); var seen = [_]bool{false} ** timers.len; + var microphone_devices_seen = false; + var next_microphone_devices_tag: u8 = 0; var at: usize = 0; while (at < subs.len) { const op = takeByte(subs, &at); + if (op == 0x02) { + if (microphone_devices_seen) @panic("ts core host: duplicate Sub.microphoneDevicesChanged descriptor"); + microphone_devices_seen = true; + next_microphone_devices_tag = takeByte(subs, &at); + continue; + } if (op != 0x01) { @panic("ts core host: unknown subscription wire record - the core and this runtime disagree on cmd_format_version"); } @@ -2180,6 +2316,14 @@ pub fn TsCoreHost(comptime core: type) type { fx.cancelTimer(timer_key_base + index); } } + const microphone_devices_was_active = microphone_devices_changed_tag != null; + if (microphone_devices_seen) { + microphone_devices_changed_tag = next_microphone_devices_tag; + if (!microphone_devices_was_active) fx.observeMicrophoneDevices(microphoneDevicesChangedMsg); + } else if (microphone_devices_was_active) { + microphone_devices_changed_tag = null; + fx.observeMicrophoneDevices(null); + } } fn freeTimerIndex() ?usize { @@ -2397,6 +2541,148 @@ pub fn TsCoreHost(comptime core: type) type { @panic("ts core host: an audio event names a Msg tag outside the union"); } + fn enumValueNamed(comptime E: type, name: []const u8) E { + inline for (@typeInfo(E).@"enum".fields) |field| { + if (std.mem.eql(u8, field.name, name)) return @enumFromInt(field.value); + } + @panic("ts core host: event enum member missing from routed Msg arm"); + } + + fn copyFrameBytes(bytes: []const u8) []const u8 { + if (bytes.len == 0) return ""; + const copy = core.rt.frameAlloc(u8, bytes.len); + @memcpy(copy, bytes); + return copy; + } + + fn audioCaptureArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 6) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "state") or std.mem.eql(u8, field.name, "reason")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "durationMs") or std.mem.eql(u8, field.name, "bytesWritten")) { + if (field.type != i64 and field.type != u64 and field.type != f64) ok = false; + } else if (std.mem.eql(u8, field.name, "outputCommitted")) { + if (field.type != bool) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagAudioCapture(tag: u8, wire_key: []const u8, event: runtime_effects.EffectAudioCapture) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime audioCaptureArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "state")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.state)); + } else if (comptime std.mem.eql(u8, field.name, "reason")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.reason)); + } else if (comptime std.mem.eql(u8, field.name, "durationMs")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.duration_ms) else @intCast(event.duration_ms); + } else if (comptime std.mem.eql(u8, field.name, "bytesWritten")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.bytes_written) else @intCast(event.bytes_written); + } else { + @field(payload, field.name) = event.output_committed; + } + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid audio capture event arm shape"); + }; + @panic("ts core host: audio capture event tag outside Msg union"); + } + + fn microphoneDeviceArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 7) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key") or std.mem.eql(u8, field.name, "id") or std.mem.eql(u8, field.name, "name")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "state")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "isDefault")) { + if (field.type != bool) ok = false; + } else if (std.mem.eql(u8, field.name, "index") or std.mem.eql(u8, field.name, "total")) { + if (field.type != i64 and field.type != u64 and field.type != f64) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagMicrophoneDevice(tag: u8, wire_key: []const u8, event: runtime_effects.EffectMicrophoneDevice) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime microphoneDeviceArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "state")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.state)); + } else if (comptime std.mem.eql(u8, field.name, "id")) { + @field(payload, field.name) = copyFrameBytes(event.id); + } else if (comptime std.mem.eql(u8, field.name, "name")) { + @field(payload, field.name) = copyFrameBytes(event.name); + } else if (comptime std.mem.eql(u8, field.name, "isDefault")) { + @field(payload, field.name) = event.is_default; + } else if (comptime std.mem.eql(u8, field.name, "index")) { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.index) else @intCast(event.index); + } else { + @field(payload, field.name) = if (comptime field.type == f64) @floatFromInt(event.total) else @intCast(event.total); + } + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid microphone device event arm shape"); + }; + @panic("ts core host: microphone device event tag outside Msg union"); + } + + fn audioCaptureAccessArmShape(comptime T: type) bool { + const info = @typeInfo(T); + if (info != .@"struct" or info.@"struct".fields.len != 4) return false; + var ok = true; + for (info.@"struct".fields) |field| { + if (std.mem.eql(u8, field.name, "key")) { + if (field.type != []const u8) ok = false; + } else if (std.mem.eql(u8, field.name, "source") or std.mem.eql(u8, field.name, "status")) { + if (@typeInfo(field.type) != .@"enum") ok = false; + } else if (std.mem.eql(u8, field.name, "restartRequired")) { + if (field.type != bool) ok = false; + } else ok = false; + } + return ok; + } + + fn msgFromTagAudioCaptureAccess(tag: u8, wire_key: []const u8, event: runtime_effects.EffectAudioCaptureAccess) Msg { + inline for (msg_arms, 0..) |arm, index| if (tag == index) { + if (comptime audioCaptureAccessArmShape(arm.type)) { + var payload: arm.type = undefined; + inline for (@typeInfo(arm.type).@"struct".fields) |field| { + if (comptime std.mem.eql(u8, field.name, "key")) { + @field(payload, field.name) = copyFrameBytes(wire_key); + } else if (comptime std.mem.eql(u8, field.name, "source")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.source)); + } else if (comptime std.mem.eql(u8, field.name, "status")) { + @field(payload, field.name) = enumValueNamed(field.type, @tagName(event.status)); + } else { + @field(payload, field.name) = event.restart_required; + } + } + return @unionInit(Msg, arm.name, payload); + } + @panic("ts core host: invalid audio capture access event arm shape"); + }; + @panic("ts core host: audio capture access event tag outside Msg union"); + } + /// Whether an arm payload struct is the video event record: the /// seven SDK-fixed fields, matched by NAME — `state` (any enum; /// its members are matched by member name at delivery), diff --git a/src/runtime/ts_core_host_tests.zig b/src/runtime/ts_core_host_tests.zig index bf55b77b5..58e0650a4 100644 --- a/src/runtime/ts_core_host_tests.zig +++ b/src/runtime/ts_core_host_tests.zig @@ -83,6 +83,11 @@ const mini_core = struct { /// carries TWO name-matched unions). pub const PtyState = enum { exit, output }; pub const PtyReason = enum { cancelled, exited, rejected, spawn_failed, signaled }; + pub const CaptureState = enum { rejected, failed, stopped, started }; + pub const CaptureReason = enum { unsupported, no_audio, capture_failed, io_failed, output_exists, device_disconnected, device_not_found, already_recording, permission_required, permission_missing, invalid_options, none }; + pub const DeviceState = enum { rejected, failed, completed, device }; + pub const AccessSource = enum { microphone, system_audio }; + pub const AccessStatus = enum { unavailable, restricted, denied, not_determined, not_authorized, authorized }; pub const Model = struct { polling: bool, @@ -145,6 +150,22 @@ const mini_core = struct { // Unsigned-class mirrors (u64-classed arm routing). ustamp_ms: u64, ucode: u64, + mic_watch: bool, + capture_state: CaptureState, + capture_reason: CaptureReason, + capture_duration: f64, + capture_bytes: f64, + capture_committed: bool, + capture_events: i64, + device_state: DeviceState, + device_total: f64, + device_default: bool, + device_events: i64, + access_source: AccessSource, + access_status: AccessStatus, + access_restart: bool, + access_events: i64, + device_change_events: i64, }; pub const Msg = union(enum) { @@ -289,6 +310,35 @@ const mini_core = struct { uget, // 81: fetch "uget" -> ufetched/failed ufetched: struct { status: u64, body: []const u8 }, // 82: fetch ok // record with a u64-classed number field + start_capture, // 83: combined capture -> capture_evt + stop_capture, // 84: stop the active capture key + list_mics, // 85: enumerate microphone records -> device_evt + capture_evt: struct { // 86 + key: []const u8, + state: CaptureState, + reason: CaptureReason, + durationMs: f64, + bytesWritten: f64, + outputCommitted: bool, + }, + device_evt: struct { // 87 + key: []const u8, + state: DeviceState, + id: []const u8, + name: []const u8, + isDefault: bool, + index: f64, + total: f64, + }, + capture_access, // 88: microphone permission request -> access_evt + access_evt: struct { // 89 + key: []const u8, + source: AccessSource, + status: AccessStatus, + restartRequired: bool, + }, + toggle_mic_watch, // 90: subscription on/off + devices_changed, // 91: no-payload subscription event }; pub const InitResult = struct { model: *const Model, cmd: []const u8 }; @@ -353,6 +403,22 @@ const mini_core = struct { .video2_events = 0, .ustamp_ms = 0, .ucode = 0, + .mic_watch = false, + .capture_state = .rejected, + .capture_reason = .none, + .capture_duration = 0, + .capture_bytes = 0, + .capture_committed = false, + .capture_events = 0, + .device_state = .completed, + .device_total = 0, + .device_default = false, + .device_events = 0, + .access_source = .microphone, + .access_status = .unavailable, + .access_restart = false, + .access_events = 0, + .device_change_events = 0, }), .cmd = cmdRequest("status.read", "status", 7, 8, "boot"), }; @@ -640,6 +706,46 @@ const mini_core = struct { @memcpy(out[first.len..], second); return .{ .model = model, .cmd = out }; }, + .start_capture => return .{ .model = model, .cmd = cmdAudioCaptureStart("meeting", 86, "meeting.wav", true, 2, "usb-mic", 44_100, 1, true) }, + .stop_capture => return .{ .model = model, .cmd = cmdKeyOnly(0x1E, "meeting") }, + .list_mics => return .{ .model = model, .cmd = cmdRoutedKey(0x1F, "mics", 87) }, + .capture_evt => |event| { + const out = frameCreate(model.*); + out.capture_state = event.state; + out.capture_reason = event.reason; + out.capture_duration = event.durationMs; + out.capture_bytes = event.bytesWritten; + out.capture_committed = event.outputCommitted; + out.capture_events = model.capture_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .device_evt => |event| { + const out = frameCreate(model.*); + out.device_state = event.state; + out.device_total = event.total; + out.device_default = event.isDefault; + out.device_events = model.device_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .capture_access => return .{ .model = model, .cmd = cmdAudioCaptureAccess("access", 89, 1, 1) }, + .access_evt => |event| { + const out = frameCreate(model.*); + out.access_source = event.source; + out.access_status = event.status; + out.access_restart = event.restartRequired; + out.access_events = model.access_events + 1; + return .{ .model = out, .cmd = "" }; + }, + .toggle_mic_watch => { + const out = frameCreate(model.*); + out.mic_watch = !model.mic_watch; + return .{ .model = out, .cmd = "" }; + }, + .devices_changed => { + const out = frameCreate(model.*); + out.device_change_events = model.device_change_events + 1; + return .{ .model = out, .cmd = "" }; + }, } } @@ -651,6 +757,7 @@ const mini_core = struct { } pub fn subscriptions(model: *const Model) []const u8 { + if (model.mic_watch) return subMicrophoneDevicesChanged(91); if (!model.polling) return ""; return subTimer("tick", if (model.fast) 40 else 100, 9); } @@ -852,6 +959,51 @@ const mini_core = struct { return out; } + fn cmdAudioCaptureStart(key: []const u8, event_tag: u8, path: []const u8, system_audio: bool, microphone_kind: u8, microphone_id: []const u8, sample_rate: u32, channels: u8, exclude_current_process_audio: bool) []const u8 { + const out = rt.frameAlloc(u8, 2 + key.len + 8 + 4 + path.len + 4 + microphone_id.len); + out[0] = 0x1D; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + var off: usize = 2 + key.len; + out[off] = event_tag; + out[off + 1] = @as(u8, @intFromBool(system_audio)) | (@as(u8, @intFromBool(exclude_current_process_audio)) << 1); + out[off + 2] = microphone_kind; + std.mem.writeInt(u32, out[off + 3 ..][0..4], sample_rate, .little); + out[off + 7] = channels; + off += 8; + off = writeLongBytes(out, off, path); + _ = writeLongBytes(out, off, microphone_id); + return out; + } + + fn cmdKeyOnly(op: u8, key: []const u8) []const u8 { + const out = rt.frameAlloc(u8, 2 + key.len); + out[0] = op; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + return out; + } + + fn cmdRoutedKey(op: u8, key: []const u8, event_tag: u8) []const u8 { + const out = rt.frameAlloc(u8, 3 + key.len); + out[0] = op; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + return out; + } + + fn cmdAudioCaptureAccess(key: []const u8, event_tag: u8, source: u8, action: u8) []const u8 { + const out = rt.frameAlloc(u8, 5 + key.len); + out[0] = 0x20; + out[1] = @intCast(key.len); + @memcpy(out[2..][0..key.len], key); + out[2 + key.len] = event_tag; + out[3 + key.len] = source; + out[4 + key.len] = action; + return out; + } + fn cmdVideoLoad(key: []const u8, event_tag: u8, surface: f64, video_path: []const u8, url: []const u8, flags: u8) []const u8 { const out = rt.frameAlloc(u8, 2 + key.len + 1 + 8 + 4 + video_path.len + 4 + url.len + 1); out[0] = 0x17; @@ -976,6 +1128,13 @@ const mini_core = struct { out[2 + key.len + 8] = msg_tag; return out; } + + fn subMicrophoneDevicesChanged(msg_tag: u8) []const u8 { + const out = rt.frameAlloc(u8, 2); + out[0] = 0x02; + out[1] = msg_tag; + return out; + } }; const Host = ts_core_host.TsCoreHost(mini_core); @@ -1815,6 +1974,49 @@ test "audio_ctl verbs drive the engine channel, gated by the wire key" { try std.testing.expectError(error.EffectNotFound, fx.feedAudioEvent(.position, 50_000, 183_000, true)); } +test "audio capture commands route fixed records and retire on stop" { + const fx = freshChannel(); + defer fx.deinit(); + Host.init(fx); + + Host.dispatch(fx, .start_capture); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureState.started, Host.model().capture_state); + try std.testing.expectEqual(mini_core.CaptureReason.none, Host.model().capture_reason); + try std.testing.expectEqual(@as(i64, 1), Host.model().capture_events); + + Host.dispatch(fx, .stop_capture); + Host.drain(fx); + try std.testing.expectEqual(mini_core.CaptureState.stopped, Host.model().capture_state); + try std.testing.expect(Host.model().capture_committed); + try std.testing.expectEqual(@as(i64, 2), Host.model().capture_events); +} + +test "microphone listing access and changed subscription route through the TS host" { + const fx = freshChannel(); + defer fx.deinit(); + Host.init(fx); + + Host.dispatch(fx, .list_mics); + Host.drain(fx); + try std.testing.expectEqual(@as(i64, 3), Host.model().device_events); + try std.testing.expectEqual(mini_core.DeviceState.completed, Host.model().device_state); + try std.testing.expectEqual(@as(f64, 2), Host.model().device_total); + + Host.dispatch(fx, .capture_access); + Host.drain(fx); + try std.testing.expectEqual(@as(i64, 1), Host.model().access_events); + try std.testing.expectEqual(mini_core.AccessSource.microphone, Host.model().access_source); + try std.testing.expectEqual(mini_core.AccessStatus.authorized, Host.model().access_status); + + Host.dispatch(fx, .toggle_mic_watch); + const changed = fx.takeMicrophoneDevicesChangedMsg() orelse return error.TestExpectedMsg; + Host.dispatch(fx, changed); + try std.testing.expectEqual(@as(i64, 1), Host.model().device_change_events); + Host.dispatch(fx, .toggle_mic_watch); + try std.testing.expect(fx.takeMicrophoneDevicesChangedMsg() == null); +} + test "a replacing audio_play re-keys the stream and the url source decodes whole" { const fx = freshChannel(); defer fx.deinit(); diff --git a/src/runtime/ui_app.zig b/src/runtime/ui_app.zig index ebe93644c..7fd0b4061 100644 --- a/src/runtime/ui_app.zig +++ b/src/runtime/ui_app.zig @@ -3968,6 +3968,18 @@ pub fn UiAppWithFeatures(comptime ModelT: type, comptime MsgT: type, comptime fe .audio => |audio_event| if (self.effects.takeAudioMsg(audio_event)) |msg| { try self.dispatch(runtime, self.canvas_window_id, msg); }, + .audio_capture => |capture_event| if (self.effects.takeAudioCaptureMsg(capture_event)) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, + .microphone_device => |device_event| if (self.effects.takeMicrophoneDeviceMsg(device_event)) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, + .audio_capture_access => |access_event| if (self.effects.takeAudioCaptureAccessMsg(access_event)) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, + .microphone_devices_changed => if (self.effects.takeMicrophoneDevicesChangedMsg()) |msg| { + try self.dispatch(runtime, self.canvas_window_id, msg); + }, // Platform video reports route the same way: through // the effects channel into the app's `on_event` Msg, // journaled at the delivery boundary. Without an app diff --git a/src/security/root.zig b/src/security/root.zig index 29f73df50..9d1574126 100644 --- a/src/security/root.zig +++ b/src/security/root.zig @@ -9,6 +9,8 @@ pub const permission_clipboard = "clipboard"; pub const permission_network = "network"; pub const permission_notifications = "notifications"; pub const permission_credentials = "credentials"; +pub const permission_microphone = "microphone"; +pub const permission_system_audio = "system_audio"; pub const ExternalLinkAction = enum(c_int) { deny = 0, diff --git a/src/tooling/manifest.zig b/src/tooling/manifest.zig index e668d5de7..d80583256 100644 --- a/src/tooling/manifest.zig +++ b/src/tooling/manifest.zig @@ -21,6 +21,7 @@ pub const Metadata = struct { icons: []const []const u8 = &.{}, platforms: []const []const u8 = &.{}, permissions: []const []const u8 = &.{}, + privacy: PrivacyMetadata = .{}, capabilities: []const []const u8 = &.{}, bridge_commands: []const BridgeCommandMetadata = &.{}, web_engine: []const u8 = "system", @@ -71,6 +72,8 @@ pub const Metadata = struct { if (self.platforms.len > 0) allocator.free(self.platforms); for (self.permissions) |value| allocator.free(value); if (self.permissions.len > 0) allocator.free(self.permissions); + if (self.privacy.microphone_usage) |value| allocator.free(value); + if (self.privacy.system_audio_usage) |value| allocator.free(value); for (self.capabilities) |value| allocator.free(value); if (self.capabilities.len > 0) allocator.free(self.capabilities); for (self.bridge_commands) |command| { @@ -184,6 +187,11 @@ pub const Metadata = struct { } }; +pub const PrivacyMetadata = struct { + microphone_usage: ?[]const u8 = null, + system_audio_usage: ?[]const u8 = null, +}; + pub const BridgeCommandMetadata = struct { name: []const u8, permissions: []const []const u8 = &.{}, @@ -437,6 +445,10 @@ pub fn validateFile(allocator: std.mem.Allocator, io: std.Io, path: []const u8) .identity = .{ .id = metadata.id, .name = metadata.name, .display_name = metadata.display_name, .description = metadata.description }, .version = parseVersion(metadata.version) catch return .{ .ok = false, .message = "app.zon version is invalid" }, .permissions = permissions, + .privacy = .{ + .microphone_usage = metadata.privacy.microphone_usage, + .system_audio_usage = metadata.privacy.system_audio_usage, + }, .capabilities = capabilities, .bridge = .{ .commands = bridge_commands }, .frontend = frontend, @@ -510,6 +522,10 @@ pub fn parseText(allocator: std.mem.Allocator, source: []const u8) !Metadata { .icons = try duplicateStringList(allocator, raw.icons), .platforms = try duplicateStringList(allocator, raw.platforms), .permissions = try duplicateStringList(allocator, raw.permissions), + .privacy = .{ + .microphone_usage = try duplicateOptionalString(allocator, raw.privacy.microphone_usage), + .system_audio_usage = try duplicateOptionalString(allocator, raw.privacy.system_audio_usage), + }, .capabilities = try duplicateStringList(allocator, raw.capabilities), .bridge_commands = try convertRawBridgeCommands(allocator, raw.bridge.commands), .web_engine = try allocator.dupe(u8, raw.web_engine), @@ -1095,6 +1111,7 @@ fn parsePermission(value: []const u8) app_manifest.Permission { if (std.mem.eql(u8, value, "filesystem")) return .filesystem; if (std.mem.eql(u8, value, "camera")) return .camera; if (std.mem.eql(u8, value, "microphone")) return .microphone; + if (std.mem.eql(u8, value, "system_audio")) return .system_audio; if (std.mem.eql(u8, value, "location")) return .location; if (std.mem.eql(u8, value, "notifications")) return .notifications; if (std.mem.eql(u8, value, "clipboard")) return .clipboard; @@ -1641,6 +1658,31 @@ test "manifest metadata parser reads identity version and lists" { }); } +test "manifest metadata parser reads audio privacy and permissions" { + const metadata = try parseText(std.testing.allocator, + \\.{ + \\ .id = "com.example.recorder", + \\ .name = "recorder", + \\ .version = "1.0.0", + \\ .permissions = .{ "filesystem", "microphone", "system_audio" }, + \\ .privacy = .{ + \\ .microphone_usage = "Record your voice.", + \\ .system_audio_usage = "Record meeting audio.", + \\ }, + \\} + ); + defer metadata.deinit(std.testing.allocator); + + try std.testing.expectEqualStrings("microphone", metadata.permissions[1]); + try std.testing.expectEqualStrings("system_audio", metadata.permissions[2]); + try std.testing.expectEqualStrings("Record your voice.", metadata.privacy.microphone_usage.?); + try std.testing.expectEqualStrings("Record meeting audio.", metadata.privacy.system_audio_usage.?); + const permissions = try parsePermissions(std.testing.allocator, metadata.permissions); + defer std.testing.allocator.free(permissions); + try std.testing.expectEqual(app_manifest.PermissionKind.microphone, permissions[1].kind()); + try std.testing.expectEqual(app_manifest.PermissionKind.system_audio, permissions[2].kind()); +} + test "manifest metadata parser reads structured security policy" { const metadata = try parseText(std.testing.allocator, \\.{ diff --git a/src/tooling/package.zig b/src/tooling/package.zig index 02f78969f..c804ab818 100644 --- a/src/tooling/package.zig +++ b/src/tooling/package.zig @@ -672,6 +672,8 @@ fn macosInfoPlist(allocator: std.mem.Allocator, metadata: manifest_tool.Metadata // dev runs pass to the panel directly. const about_line = try macosAboutLine(allocator, metadata); defer allocator.free(about_line); + const privacy_entries = try macosPrivacyEntries(allocator, metadata); + defer allocator.free(privacy_entries); // CFBundleName is the SHORT user-visible name — the application // menu's title next to the Apple menu reads it — while // CFBundleDisplayName serves the Finder and longer surfaces. Both @@ -702,11 +704,31 @@ fn macosInfoPlist(allocator: std.mem.Allocator, metadata: manifest_tool.Metadata \\ {s} \\ CFBundleVersion \\ {s} - \\{s}{s}{s} + \\{s}{s}{s}{s} \\ \\ \\ - , .{ bundle_id, display_name, display_name, executable, icon, version, version, about_line, document_types, url_types }); + , .{ bundle_id, display_name, display_name, executable, icon, version, version, about_line, privacy_entries, document_types, url_types }); +} + +fn macosPrivacyEntries(allocator: std.mem.Allocator, metadata: manifest_tool.Metadata) ![]const u8 { + var out: std.ArrayList(u8) = .empty; + defer out.deinit(allocator); + if (metadata.privacy.microphone_usage) |usage| { + const escaped = try xmlEscapeAlloc(allocator, usage); + defer allocator.free(escaped); + const entry = try std.fmt.allocPrint(allocator, " NSMicrophoneUsageDescription\n {s}\n", .{escaped}); + defer allocator.free(entry); + try out.appendSlice(allocator, entry); + } + if (metadata.privacy.system_audio_usage) |usage| { + const escaped = try xmlEscapeAlloc(allocator, usage); + defer allocator.free(escaped); + const entry = try std.fmt.allocPrint(allocator, " NSScreenCaptureUsageDescription\n {s}\n NSAudioCaptureUsageDescription\n {s}\n", .{ escaped, escaped }); + defer allocator.free(entry); + try out.appendSlice(allocator, entry); + } + return out.toOwnedSlice(allocator); } /// The optional NSHumanReadableCopyright entry (with trailing newline) @@ -2297,6 +2319,25 @@ test "plist template includes identity executable and version" { try std.testing.expect(std.mem.indexOf(u8, bare_plist, "NSHumanReadableCopyright") == null); } +test "plist template emits audio capture purpose keys" { + const metadata: manifest_tool.Metadata = .{ + .id = "dev.example.recorder", + .name = "recorder", + .version = "1.2.3", + .privacy = .{ + .microphone_usage = "Record voice & commentary.", + .system_audio_usage = "Record meeting