Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions build/app.zig
Original file line number Diff line number Diff line change
Expand Up @@ -934,6 +934,7 @@ fn linkPlatform(b: *std.Build, dep: *std.Build.Dependency, target: std.Build.Res
const sdk_include = if (b.sysroot) |sysroot| b.fmt("-I{s}/usr/include", .{sysroot}) else "";
const flags: []const []const u8 = if (b.sysroot) |sysroot| &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0", "-isysroot", sysroot, sdk_include } else &.{ "-fobjc-arc", "-fno-sanitize=builtin", "-ObjC", "-mmacosx-version-min=11.0" };
app_mod.addCSourceFile(.{ .file = dep.path("src/platform/macos/appkit_host.m"), .flags = flags });
app_mod.addCSourceFile(.{ .file = dep.path("src/platform/macos/audio_capture.m"), .flags = flags });
app_mod.linkFramework("WebKit", .{});
},
.chromium => {
Expand Down Expand Up @@ -963,6 +964,9 @@ fn linkPlatform(b: *std.Build, dep: *std.Build.Dependency, target: std.Build.Res
app_mod.linkFramework("AppKit", .{});
// The audio playback service (the AppKit host's single AVPlayer).
app_mod.linkFramework("AVFoundation", .{});
app_mod.linkFramework("ScreenCaptureKit", .{});
app_mod.linkFramework("AudioToolbox", .{});
app_mod.linkFramework("CoreMedia", .{});
// CVPixelBuffer for the video frame path (the video player's
// AVPlayerItemVideoOutput frames). CoreMedia's CMTime use stays
// header-only, but the pixel-buffer calls are real symbols.
Expand Down
1 change: 1 addition & 0 deletions changelog.d/macos-audio-capture.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
feature: **macOS audio capture**: add macOS 15+ system-audio and selectable-microphone PCM WAV capture, microphone enumeration, explicit permission flows, and Zig/TypeScript effects APIs.
7 changes: 7 additions & 0 deletions docs/src/app/docs/audio-capture/layout.tsx
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
import { pageMetadata } from "@/lib/page-metadata";

export const metadata = pageMetadata("audio-capture");

export default function Layout({ children }: { children: React.ReactNode }) {
return children;
}
176 changes: 176 additions & 0 deletions docs/src/app/docs/audio-capture/page.mdx
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# Audio Capture

Native SDK can record system audio, a microphone, or both into one signed 16-bit PCM WAV file on macOS 15 and later. The implementation lives in the Objective-C/AppKit system-engine host: ScreenCaptureKit supplies system and combined capture, while AVFoundation handles microphone-only capture without asking for Screen Recording access.

Use the three feature flags before presenting capture UI:

```zig
const can_system = runtime.supports(.system_audio_capture);
const can_microphone = runtime.supports(.microphone_capture);
const can_list_microphones = runtime.supports(.microphone_device_enumeration);
```

<table>
<thead>
<tr>
<th>Host</th>
<th>System audio</th>
<th>Microphone</th>
<th>Device enumeration</th>
</tr>
</thead>
<tbody>
<tr>
<td>macOS 15+ system engine</td>
<td>Supported</td>
<td>Supported</td>
<td>Supported</td>
</tr>
<tr>
<td>macOS 11–14 system engine</td>
<td>Unsupported</td>
<td>Unsupported</td>
<td>Unsupported</td>
</tr>
<tr>
<td>macOS Chromium</td>
<td>Unsupported</td>
<td>Unsupported</td>
<td>Unsupported</td>
</tr>
<tr>
<td>Linux, Windows, iOS, Android</td>
<td>Unsupported</td>
<td>Unsupported</td>
<td>Unsupported</td>
</tr>
</tbody>
</table>

## Manifest and privacy strings

Declare filesystem access and exactly the source permissions the app uses. Nonempty privacy strings are mandatory when their matching permission is present:

```zig:app.zon
.{
.permissions = .{ "filesystem", "microphone", "system_audio" },
.privacy = .{
.microphone_usage = "Record the microphone selected for a meeting.",
.system_audio_usage = "Record meeting audio played by other apps.",
},
}
```

Packaging maps `microphone_usage` to `NSMicrophoneUsageDescription`. It maps `system_audio_usage` to both `NSScreenCaptureUsageDescription` and `NSAudioCaptureUsageDescription`, covering ScreenCaptureKit and the system-audio purpose key.

App Sandbox remains opt-in. A sandboxed app that records a microphone must add `com.apple.security.device.audio-input` to its own entitlement file. Native SDK does not rewrite custom entitlements.

## Permission flow

Starting a capture never opens a system prompt. Query or request access explicitly first with `fx.audioCaptureAccess` or `Cmd.audioCaptureAccess`:

```ts
Cmd.audioCaptureAccess("mic-access", "microphone", "status", { event: "capture_access" });
Cmd.audioCaptureAccess("system-access", "system_audio", "request", { event: "capture_access" });
```

System audio reports the coarse ScreenCaptureKit truth: `authorized`, `not_authorized`, or `unavailable`. Microphone access additionally reports `not_determined`, `denied`, and `restricted`. A system-audio request can return `restartRequired: true`; show that result instead of pretending the running process can already capture.

The manifest is an independent gate. Missing `filesystem`, `system_audio`, or `microphone` permission produces a rejected operation even when macOS has granted the corresponding TCC access.

## Zig effects

Every request carries a journal key and a fixed Msg constructor. This example records the current default microphone together with system audio at 48 kHz stereo:

```zig
pub const Msg = union(enum) {
start_capture,
stop_capture,
list_microphones,
capture: native_sdk.EffectAudioCapture,
microphone: native_sdk.EffectMicrophoneDevice,
access: native_sdk.EffectAudioCaptureAccess,
microphones_changed,
};

const App = native_sdk.UiApp(Model, Msg);
const Fx = App.Effects;

pub fn update(model: *Model, msg: Msg, fx: *Fx) void {
switch (msg) {
.start_capture => fx.startAudioCapture(.{
.key = 1,
.path = "/tmp/meeting.wav",
.system_audio = true,
.microphone = .default,
.sample_rate_hz = 48_000,
.channel_count = 2,
.exclude_current_process_audio = true,
.on_event = Fx.audioCaptureMsg(.capture),
}),
.stop_capture => fx.stopAudioCapture(),
.list_microphones => fx.listMicrophoneDevices(.{
.key = 2,
.on_event = Fx.microphoneDeviceMsg(.microphone),
}),
.capture => |event| model.recordCapture(event),
.microphone => |event| model.recordMicrophone(event),
.access => |event| model.recordAccess(event),
.microphones_changed => fx.listMicrophoneDevices(.{
.key = 3,
.on_event = Fx.microphoneDeviceMsg(.microphone),
}),
}
}

pub fn boot(_: *Model, fx: *Fx) void {
fx.observeMicrophoneDevices(Fx.microphoneDevicesChangedMsg(.microphones_changed));
fx.audioCaptureAccess(.{
.key = 4,
.source = .microphone,
.action = .status,
.on_event = Fx.audioCaptureAccessMsg(.access),
});
}
```

`stopAudioCapture` stops the one active capture. Calling it while idle is a no-op. A second start is rejected with `already_recording` and does not disturb the current session.

## TypeScript commands

The TypeScript bytecode ABI uses fixed records. A selected device ID is a `Uint8Array`; `"default"` and `"none"` are the other microphone selections.

```ts
import { Cmd, Sub, asciiBytes } from "@native-sdk/core";

const start = Cmd.audioCaptureStart(
"meeting",
{
path: asciiBytes("/tmp/meeting.wav"),
systemAudio: true,
microphone: "default",
sampleRate: 48000,
channels: 2,
excludeCurrentProcessAudio: true,
},
{ event: "capture_event" },
);

const stop = Cmd.audioCaptureStop("meeting");
const devices = Cmd.microphoneDevices("microphones", { event: "microphone_device" });
const changes = Sub.microphoneDevicesChanged("microphones_changed");
```

The `capture_event` Msg arm must contain `key`, `state`, `reason`, `durationMs`, `bytesWritten`, and `outputCommitted`. States are `started`, `stopped`, `failed`, or `rejected`. A terminal failure can still carry `outputCommitted: true` when a usable partial WAV was finalized.

The `microphone_device` arm receives zero or more `device` records with `id`, `name`, `isDefault`, `index`, and `total`, followed by exactly one `completed`, `failed`, or `rejected` record. `microphones_changed` is only invalidation: issue `Cmd.microphoneDevices` again to obtain a new snapshot.

## Device and file guarantees

`"default"` resolves when recording starts and stays pinned for that session. Passing an enumerated `AVCaptureDevice.uniqueID` selects that exact device. A missing or disconnected explicit device fails instead of switching silently; disconnecting the retained default device follows the same rule.

Sample rates are 16, 24, 44.1, or 48 kHz; 48 kHz is the default. Channels are mono or stereo; stereo is the default. Combined capture timestamp-aligns both inputs, inserts silence for gaps, resamples into the requested format, and mixes with equal gain and clipping headroom.

The recorder writes a sibling temporary file, finalizes its WAV header, and atomically publishes the destination. It never overwrites an existing file. Stream or device failure preserves a usable partial WAV when possible, and graceful shutdown makes a bounded finalization attempt.

Raw PCM streaming, level meters, source gains, video capture, and per-application audio filtering are not part of this API.
7 changes: 7 additions & 0 deletions docs/src/app/docs/capabilities/page.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,13 @@ Web content itself is declare-to-use: an app ships the embedded web layer only w
<td>None. No bridge surface.</td>
<td>macOS system-engine host (an MTAudioProcessingTap on the app's single AVPlayer feeding a vDSP FFT — local files, cache entries, and streams alike), Windows system WebView (process-scoped WASAPI loopback capture of THIS app's audio session only — never other apps' audio — with an in-box FFT), and Linux system WebView (GStreamer's <code>spectrum</code> element as the playbin's audio-filter; requires gst-plugins-good, probed live); hosts that cannot analyze — macOS Chromium, pre-2004 Windows, spectrum-less GStreamer setups, iOS, Android — report <code>audio_spectrum</code> unsupported and simply never send the events: honest absence, never fabricated bands</td>
</tr>
<tr>
<td><a href="/docs/audio-capture">Audio capture</a></td>
<td><code>fx.startAudioCapture(options)</code> / <code>fx.stopAudioCapture()</code> / <code>fx.listMicrophoneDevices(options)</code> / <code>fx.audioCaptureAccess(options)</code></td>
<td><code>Cmd.audioCaptureStart</code> / <code>Cmd.audioCaptureStop</code> / <code>Cmd.microphoneDevices</code> / <code>Cmd.audioCaptureAccess</code> / <code>Sub.microphoneDevicesChanged</code></td>
<td><code>filesystem</code> plus <code>system_audio</code> and/or <code>microphone</code>, matching the requested sources</td>
<td>macOS 15+ system-engine host: microphone-only through AVFoundation, system-audio-only or combined capture through ScreenCaptureKit. Older macOS, macOS Chromium, Linux, Windows, iOS, and Android report the three capture features unsupported.</td>
</tr>
<tr>
<td>File drops</td>
<td><code>Event.files_dropped</code></td>
Expand Down
10 changes: 10 additions & 0 deletions docs/src/app/docs/platform-support/page.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -298,13 +298,23 @@ macOS, Linux, and Windows run full desktop apps through their own platform hosts
<td>Unsupported</td>
<td>Supported on Windows 10 2004+ (process-scoped WASAPI loopback of this app only, probed live)</td>
</tr>
<tr>
<td>System audio / microphone capture</td>
<td>Supported on macOS 15+ (ScreenCaptureKit + AVFoundation)</td>
<td>Unsupported</td>
<td>Unsupported</td>
<td>Unsupported</td>
<td>Unsupported</td>
</tr>
</tbody>
</table>

"Anchored fallback" for native context menus means the declared menu still presents — as an anchored canvas surface at the click point — because that host has no native menu presenter. All three desktop system engines present natively (`NSMenu` on macOS, `TrackPopupMenu` on Windows, `GtkPopoverMenu` on Linux). Authors declare one menu either way; the platform decides presentation.

Audio spectrum analysis reads the app's OWN playback and nothing else: macOS taps the app's single player pre-effects, Windows captures the app's own audio session through process-scoped loopback (never system-wide — other apps' audio can never appear in the bands), and Linux analyzes inside the app's own playbin. Hosts that cannot analyze report `audio_spectrum` unsupported and simply never deliver `.spectrum` events; see the capabilities page for the band shape and cadence contract.

Audio capture is intentionally narrower than playback: only the macOS 15+ system-engine host exposes `system_audio_capture`, `microphone_capture`, and `microphone_device_enumeration`. The feature flags stay false on older macOS and every Chromium, Linux, Windows, iOS, and Android host. See [Audio Capture](/docs/audio-capture) for permissions, source selection, device identity, and output guarantees.

## How Support Is Verified

macOS is the primary development platform and carries the deepest support. The Linux and Windows columns are not aspirational: the repository carries reproducible live-verification loops that build every showcase app and drive it for real. `tools/linux-truth` runs the apps in real windows against the platform toolkit under Xvfb — clicks, keys, wheel input, multi-window flows, resize clamps, window-close paths, and both engine and X-server screenshots. `tools/windows-truth` drives the same scenarios on a real Windows desktop, plus clipboard round-trips, effect-stream cancellation, record/replay, and launching the packaged artifact. CI additionally runs headless Linux and Windows canvas smokes on every change. iOS is exercised on the simulator through the toolkit host and the embed library — the `native dev --target ios` loop launches real apps, and the input and layout verification scripts inject hardware-true touches and keyboard events; a fresh `native package --target ios` output is archived with `xcodebuild` as part of verification. Android is exercised on the emulator the same way: the `native dev --target android` loop assembles, installs, and launches real apps, input is injected over adb (taps, keys, soft-keyboard text), and a fresh `native package --target android` output assembles the debug APK; the embed ABI additionally cross-compiles for both Android architectures in CI.
Expand Down
4 changes: 4 additions & 0 deletions docs/src/app/docs/typescript/page.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -294,6 +294,10 @@ The runtime interprets the command after the model commits and dispatches any re
<td><code>Cmd.audioPlay(key, source, &#123; event &#125;)</code> + <code>audioPause</code>/<code>audioResume</code>/<code>audioStop</code>/<code>audioSeek</code>/<code>audioSetVolume</code></td>
<td>The audio player: one event stream (<code>loaded</code>, <code>position</code>, <code>completed</code>, <code>failed</code>, <code>spectrum</code>, ...) until <code>audioStop</code> closes it</td>
</tr>
<tr>
<td><code>Cmd.audioCaptureStart(key, options, &#123; event &#125;)</code> / <code>audioCaptureStop(key)</code> / <code>microphoneDevices(key, &#123; event &#125;)</code> / <code>audioCaptureAccess(key, source, action, &#123; event &#125;)</code></td>
<td>macOS 15+ PCM WAV capture, microphone enumeration, and explicit permission status/request operations; see <a href="/docs/audio-capture">Audio Capture</a></td>
</tr>
<tr>
<td><code>Cmd.showWindow(label)</code> / <code>Cmd.quitApp()</code></td>
<td>The menu-bar lifecycle verbs: un-hide + activate the labeled window (the tray "Open" consequence, the counterpart to <code>close_policy = "hide"</code>), and the real graceful terminate</td>
Expand Down
1 change: 1 addition & 0 deletions docs/src/lib/docs-navigation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ const unprefixedNavSections: NavSection[] = [
title: "Native Platform",
items: [
{ name: "Windows", href: "/windows" },
{ name: "Audio Capture", href: "/audio-capture" },
{ name: "Native Surfaces", href: "/native-surfaces" },
{ name: "Menus", href: "/menus" },
{ name: "Dialogs", href: "/dialogs" },
Expand Down
1 change: 1 addition & 0 deletions docs/src/lib/page-titles.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export const PAGE_TITLES: Record<string, string> = {
"native-surfaces": "Native Surfaces",
"media-producers": "Media Producers",
windows: "Windows",
"audio-capture": "Audio Capture",
webviews: "Multiple WebViews",
"keyboard-shortcuts": "Keyboard Shortcuts",
commands: "Commands",
Expand Down
2 changes: 2 additions & 0 deletions examples/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ native build # produce a ReleaseFast binary in zig-out/bin/
| `gpu-components` | The retained GPU widget controls in one native-first component lab. |
| `canvas-preview` | Canvas + WebView in one window, panes snapped to canvas anchors, a status item. |
| `effects-probe` | The effect system live: spawn/fetch/file effects, cancellation, worker wakes. |
| `audio-capture` | macOS 15+ system audio and selectable-microphone recording through Zig effects. |
| `audio-capture-ts` | The same capture, permission, and device-enumeration flow from a TypeScript core. |
| `menu-bar` | The menu-bar app lifecycle: `close_policy = "hide"`, a status item whose Open/Quit rows drive `fx.showWindow`/`fx.quitApp`, Dock reopen. |

## Examples that own their build
Expand Down
9 changes: 9 additions & 0 deletions examples/audio-capture-ts/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
# Audio Capture (TypeScript)

The TypeScript counterpart to `examples/audio-capture`. It demonstrates fixed capture, device, and access Msg records plus `Sub.microphoneDevicesChanged` invalidation.

```sh
native dev
```

The combined recording is atomically published as `/tmp/native-sdk-combined-ts.wav`; an existing destination is rejected instead of overwritten.
36 changes: 36 additions & 0 deletions examples/audio-capture-ts/app.zon
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
.{
.id = "dev.native_sdk.audio_capture_ts",
.name = "audio-capture-ts",
.display_name = "Audio Capture TS",
.description = "Record macOS system audio and the default microphone from a TypeScript core.",
.version = "0.1.0",
.platforms = .{"macos"},
.permissions = .{ "view", "filesystem", "microphone", "system_audio" },
.privacy = .{
.microphone_usage = "Record the microphone selected in the TypeScript audio capture example.",
.system_audio_usage = "Record system audio in the TypeScript audio capture example.",
},
.capabilities = .{ "native_views", "gpu_surfaces" },
.shell = .{
.windows = .{
.{
.label = "main",
.title = "Native SDK Audio Capture TS",
.width = 660,
.height = 390,
.restore_state = false,
.restore_policy = "center_on_primary",
.views = .{
.{ .label = "capture-canvas", .kind = "gpu_surface", .fill = true, .role = "Audio capture canvas", .accessibility_label = "Audio capture TypeScript example", .gpu_backend = "metal", .gpu_pixel_format = "bgra8_unorm", .gpu_present_mode = "timer", .gpu_alpha_mode = "opaque", .gpu_color_space = "srgb", .gpu_vsync = true },
},
},
},
},
.security = .{
.navigation = .{
.allowed_origins = .{ "zero://app", "zero://inline" },
.external_links = .{ .action = "deny" },
},
},
.web_engine = "system",
}
7 changes: 7 additions & 0 deletions examples/audio-capture-ts/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"name": "audio-capture-ts",
"private": true,
"dependencies": {
"@native-sdk/core": "0.7.1"
}
}
Loading
Loading