@@ -429,6 +470,10 @@ export function CreatorScreen({
{settings.width}×{settings.height} · {formatMegapixels(settings.width, settings.height)} · {settings.steps} steps
+ ) : mode === "audio" ? (
+
+ {settings.durationSeconds}s · WAV · {settings.steps} steps
+
) : null}
No prompt, reference, or output is uploaded by Tapioca.
@@ -458,6 +503,43 @@ function FilePicker(props: { title: string; description: string; file?: LocalFil
return
{props.file?.previewUrl && props.accept === "image" ?

:
{props.accept === "image" ? "▧" : "≋"}}
{props.file?.name ?? props.title}{props.file ? "Selected from this computer" : props.description}
{props.file && }
;
}
+function AudioSetup({
+ model,
+ settings,
+ disabled,
+ onChange,
+}: {
+ model?: CreatorModel;
+ settings: CreatorAdvancedSettings;
+ disabled: boolean;
+ onChange(value: CreatorAdvancedSettings): void;
+}) {
+ const maxDuration = model?.limits?.maxDurationSeconds ?? 120;
+ return (
+
+
+
Audio setup
Choose the length and generation quality of the WAV output.
+
{settings.durationSeconds}s
+
+
+
+ );
+}
+
function LoraEditor(props: {
loras: CreatorLora[];
options: CreatorLoraOption[];
@@ -906,7 +988,7 @@ function formatMegapixels(width: number, height: number): string {
function AdvancedSettings({ mode, settings, disabled, onChange }: { mode: CreatorMode; settings: CreatorAdvancedSettings; disabled: boolean; onChange(value: CreatorAdvancedSettings): void }) {
const number = (key: keyof CreatorAdvancedSettings, value: string) => onChange({ ...settings, [key]: value === "" && key === "seed" ? undefined : Number(value) });
const dimensionStep = mode === "video" ? 32 : 8;
- return
{mode === "video" ? "Expert overrides" : "Advanced settings"} {mode === "video" ? "Exact dimensions, frames, playback, and seed" : "Only runtime-supported controls are shown"}
{(mode === "image" || mode === "video") && <>>}{mode === "video" && <>>}
{mode === "video" ? Changing frames or FPS updates the duration summary. MiniMax-H3 uses 17n+5 frames; other models may require 4n+1 or 8n+1.
: null};
+ return
{mode === "video" ? "Expert overrides" : "Advanced settings"} {mode === "video" ? "Exact dimensions, frames, playback, and seed" : "Only runtime-supported controls are shown"}
{(mode === "image" || mode === "video") && <>>}{(mode === "image" || mode === "video" || mode === "audio") && }{mode === "video" && <>>}
{mode === "video" ? Changing frames or FPS updates the duration summary. MiniMax-H3 uses 17n+5 frames; other models may require 4n+1 or 8n+1.
: null};
}
function OutputCard({ output, onReveal, onMetadata }: { output: CreatorOutput; onReveal(): Promise
; onMetadata(): Promise }) {
@@ -914,7 +996,7 @@ function OutputCard({ output, onReveal, onMetadata }: { output: CreatorOutput; o
}
function ModeGlyph({ mode }: { mode: CreatorMode }) {
- return {{ image: "◇", video: "▷", speech: "≋", "voice-clone": "◉" }[mode]};
+ return {{ image: "◇", video: "▷", audio: "♫", speech: "≋", "voice-clone": "◉" }[mode]};
}
function formatElapsed(seconds: number): string {
@@ -938,6 +1020,7 @@ function readDurationEstimate(request: CreatorRequest): number {
if (Number.isFinite(saved) && saved >= 1_000) return saved;
}
if (request.mode === "speech" || request.mode === "voice-clone") return 45_000;
+ if (request.mode === "audio") return Math.max(60_000, request.settings.durationSeconds * 8_000);
const pixels = request.settings.width * request.settings.height;
if (request.mode === "image") {
return Math.max(20_000, 90_000 * (pixels / (512 * 512)) * (request.settings.steps / 4));
diff --git a/desktop/src/features/create/state.ts b/desktop/src/features/create/state.ts
index a446244..d07add5 100644
--- a/desktop/src/features/create/state.ts
+++ b/desktop/src/features/create/state.ts
@@ -11,6 +11,7 @@ export const defaultCreatorSettings = (): CreatorAdvancedSettings => ({
steps: 4,
frames: 49,
fps: 24,
+ durationSeconds: 30,
});
export function videoDurationSeconds(frames: number, fps: number): number {
@@ -117,7 +118,16 @@ export function validateCreatorRequest(request: CreatorRequest): string[] {
if (request.mode === "voice-clone" && !request.voiceReference) {
errors.push("Choose a voice reference recording.");
}
- const { width, height, steps, frames, fps, seed } = request.settings;
+ if (request.modelId.includes("custom-voice") && !request.speaker?.trim()) {
+ errors.push("Choose a speaker for this CustomVoice model.");
+ }
+ if (request.modelId.includes("voice-design") && !request.instruct?.trim()) {
+ errors.push("Describe the voice to design.");
+ }
+ if (request.modelId.startsWith("audio8-tts:") && request.voiceReference && !request.transcript?.trim()) {
+ errors.push("Enter the transcript of the Audio8 reference recording.");
+ }
+ const { width, height, steps, frames, fps, seed, durationSeconds } = request.settings;
const dimensionStep = request.mode === "video" ? 32 : 8;
if (width < 256 || width > 2048 || width % dimensionStep !== 0) {
errors.push(`Width must be 256–2048 and divisible by ${dimensionStep}.`);
@@ -141,6 +151,9 @@ export function validateCreatorRequest(request: CreatorRequest): string[] {
if (request.mode === "video" && (fps < 1 || fps > 60)) {
errors.push("FPS must be between 1 and 60.");
}
+ if (request.mode === "audio" && (durationSeconds < 1 || durationSeconds > 120)) {
+ errors.push("Audio duration must be between 1 and 120 seconds.");
+ }
if (seed !== undefined && (!Number.isInteger(seed) || seed < 0)) {
errors.push("Seed must be a positive whole number.");
}
@@ -154,6 +167,7 @@ export function modeLabel(mode: CreatorMode): string {
return {
image: "Image",
video: "Video",
+ audio: "Audio / Music",
speech: "Speech / TTS",
"voice-clone": "Voice Clone",
}[mode];
diff --git a/desktop/src/features/create/types.ts b/desktop/src/features/create/types.ts
index b6b72d5..b65e30e 100644
--- a/desktop/src/features/create/types.ts
+++ b/desktop/src/features/create/types.ts
@@ -1,4 +1,4 @@
-export const creatorModes = ["image", "video", "speech", "voice-clone"] as const;
+export const creatorModes = ["image", "video", "audio", "speech", "voice-clone"] as const;
export type CreatorMode = (typeof creatorModes)[number];
export interface CreatorModel {
@@ -11,6 +11,7 @@ export interface CreatorModel {
maxWidth?: number;
maxHeight?: number;
maxFrames?: number;
+ maxDurationSeconds?: number;
};
supportsInputImage?: boolean;
supportsLoRA?: boolean;
@@ -60,6 +61,7 @@ export interface CreatorAdvancedSettings {
seed?: number;
frames: number;
fps: number;
+ durationSeconds: number;
}
export interface CreatorRequest {
@@ -67,10 +69,13 @@ export interface CreatorRequest {
modelId: string;
prompt: string;
text?: string;
+ transcript?: string;
+ language?: string;
+ speaker?: string;
+ instruct?: string;
inputImage?: LocalFileSelection;
voiceReference?: LocalFileSelection;
voiceConsent?: boolean;
- transcript?: string;
loras: CreatorLora[];
settings: CreatorAdvancedSettings;
}
diff --git a/desktop/src/features/models/ModelsScreen.tsx b/desktop/src/features/models/ModelsScreen.tsx
index 6c439d0..b2bb706 100644
--- a/desktop/src/features/models/ModelsScreen.tsx
+++ b/desktop/src/features/models/ModelsScreen.tsx
@@ -37,6 +37,7 @@ const kinds: Array<{ value: "all" | ModelKind; label: string }> = [
{ value: "image", label: "Images" },
{ value: "video", label: "Video" },
{ value: "speech", label: "Voice" },
+ { value: "audio", label: "Audio / Music" },
];
export function ModelsScreen({ adapter, machine }: ModelsScreenProps) {
@@ -120,7 +121,7 @@ export function ModelsScreen({ adapter, machine }: ModelsScreenProps) {
}, [compatibleOnly, installedOnly, kind, machine, models, platform, query]);
const kindCounts = useMemo(() => {
- const counts: Record<"all" | ModelKind, number> = { all: 0, chat: 0, image: 0, video: 0, speech: 0 };
+ const counts: Record<"all" | ModelKind, number> = { all: 0, chat: 0, image: 0, video: 0, speech: 0, audio: 0 };
for (const model of models ?? []) {
counts.all += 1;
counts[model.kind] += 1;
diff --git a/desktop/src/features/models/types.ts b/desktop/src/features/models/types.ts
index 17229ef..02866ab 100644
--- a/desktop/src/features/models/types.ts
+++ b/desktop/src/features/models/types.ts
@@ -1,4 +1,4 @@
-export type ModelKind = "chat" | "image" | "video" | "speech";
+export type ModelKind = "chat" | "image" | "video" | "speech" | "audio";
export type ModelPlatform = "macos" | "windows" | "linux";
export type Accelerator = "apple" | "nvidia" | "amd" | "intel" | "cpu";
diff --git a/desktop/src/shared/ipc.ts b/desktop/src/shared/ipc.ts
index 8446784..fd69eb7 100644
--- a/desktop/src/shared/ipc.ts
+++ b/desktop/src/shared/ipc.ts
@@ -4,7 +4,7 @@ import {
controlEventSchema,
} from "./sidecar";
-export const modelKindSchema = z.enum(["chat", "image", "video", "speech"]);
+export const modelKindSchema = z.enum(["chat", "image", "video", "speech", "audio"]);
export const desktopModelSchema = z
.object({
id: z.string().min(1),
@@ -236,6 +236,7 @@ export const creatorCapabilitiesSchema = z
image: creatorCapabilitySchema,
video: creatorCapabilitySchema,
speech: creatorCapabilitySchema,
+ audio: creatorCapabilitySchema,
voice_clone: creatorCapabilitySchema,
outputs: z
.object({
@@ -295,14 +296,17 @@ const creatorLoraSchema = z.discriminatedUnion("type", [
export const creatorGenerateInputSchema = z
.object({
jobId: z.string().min(1).max(128),
- mode: z.enum(["image", "video", "speech", "voice-clone"]),
+ mode: z.enum(["image", "video", "audio", "speech", "voice-clone"]),
model: z.string().min(1).max(256),
prompt: z.string().max(20_000),
text: z.string().max(20_000).optional(),
+ transcript: z.string().max(20_000).optional(),
+ language: z.string().max(64).optional(),
+ speaker: z.string().max(128).optional(),
+ instruct: z.string().max(2_000).optional(),
inputImageToken: z.string().uuid().optional(),
voiceReferenceToken: z.string().uuid().optional(),
voiceConsent: z.boolean().optional(),
- transcript: z.string().max(20_000).optional(),
loras: z.array(creatorLoraSchema).max(8),
settings: z.object({
width: z.number().int().min(64).max(4096),
@@ -311,12 +315,13 @@ export const creatorGenerateInputSchema = z
seed: z.number().int().nonnegative().optional(),
frames: z.number().int().min(1).max(513),
fps: z.number().int().min(1).max(60),
+ durationSeconds: z.number().int().min(1).max(120),
}).strict(),
})
.strict();
export const creatorOutputSchema = z.object({
id: z.string().uuid(),
- mode: z.enum(["image", "video", "speech", "voice-clone"]),
+ mode: z.enum(["image", "video", "audio", "speech", "voice-clone"]),
mediaType: z.enum(["image", "video", "audio"]),
url: z.string().url(),
createdAt: z.string().datetime(),
diff --git a/desktop/src/shared/sidecar.ts b/desktop/src/shared/sidecar.ts
index 1c18739..314fffb 100644
--- a/desktop/src/shared/sidecar.ts
+++ b/desktop/src/shared/sidecar.ts
@@ -26,6 +26,7 @@ export const controlMethodSchema = z.enum([
"image.generate",
"video.generate",
"speech.generate",
+ "audio.generate",
"voice.clone",
"lora.list",
"lora.inspect",
@@ -169,6 +170,9 @@ export const catalogModelSchema = z
steps: z.number().int().positive().optional(),
frames: z.number().int().positive().optional(),
fps: z.number().int().positive().optional(),
+ duration_seconds: z.number().int().positive().optional(),
+ max_duration_seconds: z.number().int().positive().optional(),
+ sample_rate: z.number().int().positive().optional(),
gated: z.boolean().optional(),
license: z.string().optional(),
license_url: z.string().url().optional(),
diff --git a/docs/README.md b/docs/README.md
index d3fe464..b444e57 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -14,6 +14,7 @@ Then choose what you want to do:
- [Install the Codex plugin or Claude Code skill](agents/install-integrations.md)
- [Generate an image](guides/image-generation.md)
- [Generate a video](guides/video-generation.md)
+- [Generate music and sound effects](guides/audio-generation.md)
- [Generate speech and clone a voice](guides/speech-and-voices.md)
- [Run Tapioca on Linux](guides/linux.md)
- [Understand LoRA adapters](concepts/lora-adapters.md)
diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md
index bf377d3..566a0b5 100644
--- a/docs/getting-started/quickstart.md
+++ b/docs/getting-started/quickstart.md
@@ -59,6 +59,10 @@ Or make an image:
tapioca image sd-turbo --prompt "A friendly pearl astronaut"
```
+Standalone music and sound effects are covered in the
+[audio generation guide](../guides/audio-generation.md); those models are
+larger and gated, so they are better as a next step than a first-run smoke test.
+
See [Choosing models](../guides/choosing-models.md) before downloading a large
model and [Storage](../reference/storage.md) to find the downloaded files.
diff --git a/docs/guides/audio-generation.md b/docs/guides/audio-generation.md
new file mode 100644
index 0000000..caa2c0d
--- /dev/null
+++ b/docs/guides/audio-generation.md
@@ -0,0 +1,78 @@
+# Music and sound generation
+
+Tapioca can generate standalone music, ambience, and sound effects locally as
+WAV files. This is separate from text-to-speech and from video models that
+produce a synchronized soundtrack.
+
+## Choose a Stable Audio 3 profile
+
+| Model | Best for | Default |
+| --- | --- | --- |
+| `stable-audio-3:small-music` | Instrumental music, loops, and ambience | 30 seconds, 8 steps |
+| `stable-audio-3:small-sfx` | Foley, transitions, impacts, and environmental sounds | 30 seconds, 8 steps |
+
+Both profiles use official Stable Audio 3 Small checkpoints, support up to 120
+seconds, and save 44.1 kHz WAV output. Music and SFX are separate downloads, so
+install only the profile you need.
+
+## Accept access and download
+
+Stable Audio 3 is gated under the Stability AI Community License. Review and
+accept the current terms in your own Hugging Face account, then provide a read
+token for the first pull:
+
+```bash
+export HF_TOKEN=hf_your_read_token
+tapioca pull stable-audio-3:small-music --accept-license
+```
+
+On PowerShell:
+
+```powershell
+$env:HF_TOKEN = "hf_your_read_token"
+tapioca pull stable-audio-3:small-music --accept-license
+```
+
+Tapioca records the local license acknowledgement but does not store the token
+or accept provider terms on your behalf.
+
+## Generate music
+
+```bash
+tapioca audio stable-audio-3:small-music \
+ --prompt "Warm analog synthesizer, slow pulse, spacious, instrumental" \
+ --negative-prompt "vocals, speech, clipping" \
+ --seconds 30 --output ambient.wav
+```
+
+## Generate a sound effect
+
+```bash
+tapioca audio stable-audio-3:small-sfx \
+ --prompt "Heavy wooden door closing in a stone hall, short natural reverb" \
+ --seconds 6 --output door.wav
+```
+
+Use `--seed NUMBER` to repeat a configuration or `--random-seed` to generate
+and print a new seed. The flags cannot be combined. Duration must be between 1
+and 120 seconds, and `--steps` must be between 1 and 200.
+
+## Hardware and first run
+
+An NVIDIA CUDA machine is recommended for practical generation. Apple MPS and
+CPU fallback are supported by the runtime but can be substantially slower,
+especially for long output. The first command creates a private Python runtime
+under Tapioca's managed runtime directory and installs the pinned upstream
+Stable Audio 3 package; later runs reuse it.
+
+The downloaded checkpoint and runtime are large. Keep ample free disk space,
+close other GPU-heavy applications, and begin with a 5–10 second smoke test
+before attempting the 120-second maximum.
+
+## Desktop and control API
+
+Tapioca Desktop exposes the same models under **Audio**, with duration, steps,
+and seed controls. Control-protocol clients use `audio.generate` with
+`model`, `prompt`, optional `negative_prompt`, `duration_seconds`, `steps`,
+`seed`, and `output_name` parameters. Outputs are written to Tapioca's managed
+audio directory rather than transported as binary protocol data.
diff --git a/docs/guides/choosing-models.md b/docs/guides/choosing-models.md
index 566488c..5715c4b 100644
--- a/docs/guides/choosing-models.md
+++ b/docs/guides/choosing-models.md
@@ -57,10 +57,12 @@ are not yet qualified. Try `tapioca pull minicpm5-2b` or
| `qwen3-coder:30b-mlx` | ~16 GiB | Coding agents |
| `qwen3.6:35b-mlx` | ~20 GiB | Larger Qwen MLX option |
| `qwen3.8:27b-mlx` | ~15 GiB | Newest Qwen agentic coding and tool use |
+| `gemma4:e2b-mlx-4bit` | ~3.6 GiB | Compact current-generation Gemma |
On Windows or Linux, `qwen3.8:27b-q4_k_m` is the portable GGUF option and
needs at least 24 GiB of system memory. Qwen3.8 is a reasoning-heavy model;
start with a moderate context size and allow extra time for complex tasks.
-For image and video recommendations, use the
-[image](image-generation.md) and [video](video-generation.md) guides.
+For image, video, speech, and standalone audio recommendations, use the
+[image](image-generation.md), [video](video-generation.md),
+[voice](speech-and-voices.md), and [audio](audio-generation.md) guides.
diff --git a/docs/guides/image-generation.md b/docs/guides/image-generation.md
index 3f7b455..2b47289 100644
--- a/docs/guides/image-generation.md
+++ b/docs/guides/image-generation.md
@@ -95,7 +95,17 @@ required; verify it with `python -c "import platform; print(platform.machine())"
## Windows x64 with NVIDIA CUDA
-Start with SD Turbo:
+FLUX.2 Klein 4B is the current recommended quality path on a modern NVIDIA
+card. Use the official FP8 checkpoint to reduce memory pressure:
+
+```powershell
+tapioca image flux2-klein:4b-fp8-cuda `
+ --prompt "A red fox in snow, editorial photography" `
+ --output fox.png
+```
+
+Use `flux2-klein:4b-bf16-cuda` when VRAM permits and you want the official
+BF16 weights. Start with SD Turbo on smaller GPUs:
```powershell
tapioca image sd-turbo:fp16 `
@@ -109,6 +119,8 @@ tapioca image sd-turbo:fp16 `
| `sd-turbo:onnx-arm64` | ~4.8 GiB | 512×512 | Windows ARM64 CPU; slower |
| `sd-turbo:fp16` | ~3 GiB | 512×512 | Best first model |
| `sdxl-turbo:fp16` | ~7 GiB | 1024×1024 | Better detail |
+| `flux2-klein:4b-fp8-cuda` | ~12 GiB | 1024×1024 | Current FLUX.2 4B, reduced precision |
+| `flux2-klein:4b-bf16-cuda` | ~15 GiB | 1024×1024 | Official BF16 checkpoint; more VRAM |
| `qwen-image-flash:bf16` | ~58 GiB | 1024×1024 | Ampere+; very high memory |
| `krea-2-turbo:bf16-cuda` | ~34 GiB | 1024×1024 | NVIDIA 16 GiB minimum with CPU offload; 24 GiB recommended |
diff --git a/docs/guides/speech-and-voices.md b/docs/guides/speech-and-voices.md
index 669f074..d110e0e 100644
--- a/docs/guides/speech-and-voices.md
+++ b/docs/guides/speech-and-voices.md
@@ -14,6 +14,11 @@ contain the model's built-in imperceptible audio watermark.
| `chatterbox:multilingual` | Default voices and cloning across 23+ languages | CPU, Apple MPS, or NVIDIA CUDA |
| `qwen3-tts:0.6b-mlx` | High-quality 3-second cloning on Apple Silicon | Mac with 12 GiB+ unified memory |
| `qwen3-tts:0.6b` | High-quality cloning on GPU servers | Windows/Linux NVIDIA; CPU fallback |
+| `qwen3-tts:1.7b-base` | Higher-quality reference cloning | Windows/Linux NVIDIA |
+| `qwen3-tts:1.7b-custom-voice` | Named built-in speakers and instructed delivery | Windows/Linux NVIDIA |
+| `qwen3-tts:1.7b-voice-design` | Design a voice from a written description | Windows/Linux NVIDIA |
+| `audio8-tts:0.6b-mlx` | Compact default voice or transcript-assisted cloning | Apple Silicon |
+| `audio8-tts:0.6b` | Compact default voice or transcript-assisted cloning | Windows/Linux NVIDIA |
Running a missing model automatically downloads it. To download first:
@@ -111,6 +116,28 @@ tapioca tts qwen3-tts:0.6b \
--output one-off.wav
```
+Qwen CustomVoice and VoiceDesign do not use a reference recording:
+
+```bash
+tapioca tts qwen3-tts:1.7b-custom-voice \
+ --speaker Ryan --instruct "Speak warmly and at a measured pace" \
+ --text "The local studio is ready." --output custom.wav
+
+tapioca tts qwen3-tts:1.7b-voice-design \
+ --instruct "A calm middle-aged narrator with a low, clear voice" \
+ --text "This voice was designed from text." --output designed.wav
+```
+
+Audio8 can use its default voice without a reference, or clone a recording
+when the exact transcript is supplied:
+
+```bash
+tapioca tts audio8-tts \
+ --voice-sample ./reference.wav \
+ --transcript "The exact words in the reference recording." \
+ --text "Audio8 is running locally." --output audio8.wav
+```
+
## Files on disk
Voices are stored under `~/.tapioca/voices` on macOS/Linux and
@@ -137,14 +164,14 @@ few minutes. Later runs reuse both the runtime and model.
## Platform and acceleration matrix
-| Platform | Chatterbox | Qwen3-TTS |
-| --- | --- | --- |
-| Apple Silicon | PyTorch MPS or CPU | MLX recommended |
-| Windows x64 NVIDIA | CUDA or CPU | CUDA or CPU |
-| Windows x64 AMD/Intel | CPU | CPU |
-| Windows ARM64 | CPU | CPU |
-| Linux NVIDIA | CUDA or CPU | CUDA or CPU |
-| Linux without NVIDIA | CPU | CPU |
+| Platform | Chatterbox | Qwen3-TTS | Audio8 |
+| --- | --- | --- | --- |
+| Apple Silicon | PyTorch MPS or CPU | MLX recommended | MLX recommended |
+| Windows x64 NVIDIA | CUDA or CPU | CUDA or CPU | CUDA or CPU |
+| Windows x64 AMD/Intel | CPU | CPU | CPU |
+| Windows ARM64 | CPU | CPU | CPU |
+| Linux NVIDIA | CUDA or CPU | CUDA or CPU | CUDA or CPU |
+| Linux without NVIDIA | CPU | CPU | CPU |
MLX is selected automatically when you request `qwen3-tts` on Apple Silicon.
On Windows or Linux NVIDIA, use `qwen3-tts:0.6b`.
diff --git a/docs/guides/video-generation.md b/docs/guides/video-generation.md
index cefeae2..c52c217 100644
--- a/docs/guides/video-generation.md
+++ b/docs/guides/video-generation.md
@@ -138,6 +138,25 @@ LTX-Video 2B needs approximately 24 GiB system RAM and an 8 GiB CUDA GPU with
offload; 32 GiB RAM is recommended. Stable Video Diffusion is approximately
4.5 GiB and requires a starting image.
+### LTX-2.5 on a high-memory NVIDIA workstation
+
+`ltx-2.5:22b-bf16-cuda` uses the official Diffusers component layout and
+generates synchronized video and audio. It is a gated, approximately 68 GiB
+download intended for a machine with at least 96 GiB system RAM and 24 GiB
+VRAM:
+
+```powershell
+$env:HF_TOKEN = "hf_your_read_token"
+tapioca pull ltx-2.5:22b-bf16-cuda --accept-license
+tapioca video ltx-2.5:22b-bf16-cuda `
+ --prompt 'A close-up of rain on leaves with quiet natural ambience' `
+ --width 768 --height 512 --frames 121 --steps 30 --fps 24 `
+ --output ltx25.mp4
+```
+
+Use the included Windows validation script before treating this heavyweight
+profile as verified on a particular driver/GPU combination.
+
## Rules and flags
- The default seed is `0`. Use `--random-seed` to generate and print a seed,
diff --git a/docs/reference/commands.md b/docs/reference/commands.md
index 06ee7d5..c5093bc 100644
--- a/docs/reference/commands.md
+++ b/docs/reference/commands.md
@@ -54,7 +54,7 @@ tapioca pull MODEL[:VARIANT] --force
tapioca pull GATED_MODEL[:VARIANT] --accept-license
```
-Explicit pull is optional for `run`, `serve`, `launch`, `image`, `video`, and `tts`;
+Explicit pull is optional for `run`, `serve`, `launch`, `image`, `video`, `audio`, and `tts`;
they download a missing catalog model automatically.
Gated models do not auto-pull until their terms have been accepted explicitly.
@@ -151,7 +151,27 @@ tapioca tts MODEL --text TEXT [flags]
```
Flags include `--output`, `--language`, `--voice`, `--voice-sample`,
-`--transcript`, and `--transcript-file`.
+`--transcript`, `--transcript-file`, `--speaker`, and `--instruct`.
+
+Qwen3-TTS Base variants clone from a voice sample, CustomVoice variants use
+`--speaker`, and VoiceDesign variants use `--instruct`. Audio8 can speak with
+its default voice or clone a sample when its exact transcript is supplied.
+
+## `audio`
+
+Generate standalone music, ambience, or sound effects as a WAV file:
+
+```bash
+tapioca audio stable-audio-3:small-music \
+ --prompt "Warm analog synthesizer, slow pulse, no vocals" \
+ --seconds 30 --output ambient.wav
+```
+
+Flags include `--negative-prompt`, `--seconds`, `--steps`, `--seed`,
+`--random-seed`, and `--output`. Stable Audio 3 supports 1–120 seconds; the
+catalog default is 30 seconds. Its weights are gated, so provider access,
+`HF_TOKEN`, and an explicit `tapioca pull ... --accept-license` are required
+before the first generation.
## `voice`
diff --git a/internal/app/app.go b/internal/app/app.go
index df54b9b..a120558 100644
--- a/internal/app/app.go
+++ b/internal/app/app.go
@@ -48,6 +48,8 @@ func Run(args []string) error {
return edit(args[1:])
case "video":
return video(args[1:])
+ case "audio":
+ return audio(args[1:])
case "tts":
return tts(args[1:])
case "voice":
@@ -74,7 +76,7 @@ func Run(args []string) error {
}
func usage(w io.Writer) {
- fmt.Fprintln(w, `Tapioca runs local language, speech, image, and video models.
+ fmt.Fprintln(w, `Tapioca runs local language, speech, image, video, and audio models.
Usage:
tapioca pull MODEL[:QUANT] [--accept-license]
@@ -83,6 +85,7 @@ Usage:
tapioca image MODEL --prompt TEXT [--seed NUMBER | --random-seed] [--output image.png]
tapioca edit MODEL --image FILE [--image FILE] --prompt TEXT [--seed NUMBER | --random-seed]
tapioca video MODEL --prompt TEXT [--image start.png] [--seconds N | --frames N] [--seed NUMBER | --random-seed] [--output video.mp4]
+ tapioca audio MODEL --prompt TEXT [--seconds N] [--seed NUMBER | --random-seed] [--output audio.wav]
tapioca tts MODEL --text TEXT [--voice NAME | --voice-sample FILE] [--output speech.wav]
tapioca voice (create|list|inspect|remove) [NAME]
tapioca adapter (inspect|pull|import|list) [REFERENCE]
@@ -99,6 +102,7 @@ Examples:
tapioca pull qwen-image-flash:int8
tapioca image qwen-image-flash:int8 --prompt "A red fox in snow"
tapioca video wan2.2-video:5b-q8-mlx --prompt "A red fox running in snow" --seconds 5 --random-seed
+ tapioca audio stable-audio-3:small-sfx --prompt "A wooden door closing in a stone hall" --seconds 8
tapioca voice create narrator --model chatterbox:nano --audio voice.wav
tapioca tts chatterbox:nano --voice narrator --text "Hello from Tapioca"
tapioca adapter inspect hf://Alissonerdx/BFS-Best-Face-Swap
@@ -242,7 +246,7 @@ func pullResolvedWithContext(
}
}
dir := filepath.Join(home, "models", strings.ReplaceAll(resolved.Name, ":", "-"))
- if resolved.Kind == "image" || resolved.Kind == "video" || resolved.Kind == "speech" {
+ if resolved.Kind == "image" || resolved.Kind == "video" || resolved.Kind == "speech" || resolved.Kind == "audio" {
if err := pullSnapshotWithContext(ctx, resolved, dir, force, report); err != nil {
return config.Model{}, err
}
diff --git a/internal/app/audio.go b/internal/app/audio.go
new file mode 100644
index 0000000..5073192
--- /dev/null
+++ b/internal/app/audio.go
@@ -0,0 +1,93 @@
+package app
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "os"
+ "os/signal"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/carlos/tapioca/internal/audioruntime"
+ "github.com/carlos/tapioca/internal/config"
+)
+
+func audio(args []string) error {
+ if len(args) == 0 {
+ return errors.New("usage: tapioca audio MODEL --prompt TEXT [flags]")
+ }
+ profile, err := resolveMediaModel(args[0], "audio")
+ if err != nil {
+ return err
+ }
+ if profile.Kind != "audio" {
+ return fmt.Errorf("%s is not an audio-generation model", profile.Name)
+ }
+ fs := flag.NewFlagSet("audio", flag.ContinueOnError)
+ prompt := fs.String("prompt", "", "music or sound description")
+ negative := fs.String("negative-prompt", "", "qualities to steer away from")
+ output := fs.String("output", "", "output WAV path")
+ duration := fs.Int("seconds", profile.DurationSeconds, "audio duration in seconds")
+ steps := fs.Int("steps", profile.Steps, "sampling steps")
+ seed := fs.Uint64("seed", 0, "generation seed (default 0)")
+ randomSeed := fs.Bool("random-seed", false, "generate and print a random seed")
+ if err := fs.Parse(args[1:]); err != nil {
+ return err
+ }
+ if fs.NArg() != 0 || strings.TrimSpace(*prompt) == "" {
+ return errors.New("usage: tapioca audio MODEL --prompt TEXT [flags]")
+ }
+ seedSet := false
+ fs.Visit(func(f *flag.Flag) {
+ if f.Name == "seed" {
+ seedSet = true
+ }
+ })
+ effectiveSeed, err := resolveMediaSeed(*seed, seedSet, *randomSeed, os.Stderr)
+ if err != nil {
+ return err
+ }
+ if *duration <= 0 || (profile.MaxDuration > 0 && *duration > profile.MaxDuration) {
+ return fmt.Errorf("seconds must be between 1 and %d for %s", profile.MaxDuration, profile.Name)
+ }
+ if *steps <= 0 || *steps > 200 {
+ return errors.New("steps must be between 1 and 200")
+ }
+ model, err := ensureResolvedModel(profile)
+ if err != nil {
+ return err
+ }
+ target := *output
+ if target == "" {
+ target = fmt.Sprintf("tapioca-%d.wav", time.Now().Unix())
+ }
+ target, err = filepath.Abs(target)
+ if err != nil {
+ return err
+ }
+ if strings.ToLower(filepath.Ext(target)) != ".wav" {
+ return errors.New("audio generation currently outputs WAV files")
+ }
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return err
+ }
+ home, err := config.Home()
+ if err != nil {
+ return err
+ }
+ ctx, stop := signal.NotifyContext(context.Background(), terminationSignals()...)
+ defer stop()
+ fmt.Fprintf(os.Stderr, "generating %d-second audio with %s...\n", *duration, model.Name)
+ if err := audioruntime.Run(ctx, filepath.Join(home, "runtime"), audioruntime.Request{
+ ModelPath: model.Path, Prompt: strings.TrimSpace(*prompt),
+ NegativePrompt: strings.TrimSpace(*negative), Output: target,
+ Duration: *duration, Steps: *steps, Seed: effectiveSeed, Backend: model.Backend,
+ }); err != nil {
+ return err
+ }
+ fmt.Println(target)
+ return nil
+}
diff --git a/internal/app/snapshot.go b/internal/app/snapshot.go
index f8533af..722de5a 100644
--- a/internal/app/snapshot.go
+++ b/internal/app/snapshot.go
@@ -35,8 +35,13 @@ func pullSnapshotWithContext(
if len(model.Artifacts) > 0 {
return pullArtifactsWithContext(ctx, model, destination, force, report)
}
+ include := snapshotFileFilter(model)
+ return pullHubSnapshotWithContext(ctx, model, destination, force, include, report)
+}
+
+func snapshotFileFilter(model catalog.Resolved) func(string) bool {
include := imageSnapshotFile
- if model.Kind == "speech" {
+ if model.Kind == "speech" || model.Kind == "audio" {
include = textSnapshotFile
}
if model.Repo == "stabilityai/sd-turbo" ||
@@ -50,10 +55,10 @@ func pullSnapshotWithContext(
if model.Backend == "mflux" {
include = textSnapshotFile
}
- if model.Gated {
+ if model.Gated && model.Kind != "audio" && model.Kind != "speech" {
include = licensedImageSnapshotFile
}
- return pullHubSnapshotWithContext(ctx, model, destination, force, include, report)
+ return include
}
func pullArtifactsWithContext(
diff --git a/internal/app/snapshot_test.go b/internal/app/snapshot_test.go
index b48f55d..02da2dc 100644
--- a/internal/app/snapshot_test.go
+++ b/internal/app/snapshot_test.go
@@ -76,6 +76,23 @@ func TestLicensedImageSnapshotIncludesTermsButNotDuplicateWeights(t *testing.T)
}
}
+func TestGatedAudioSnapshotKeepsWeightsAndLocalTextEncoder(t *testing.T) {
+ filter := snapshotFileFilter(catalog.Resolved{Kind: "audio", Gated: true})
+ for _, name := range []string{
+ "model_config.json",
+ "model.safetensors",
+ "t5gemma-b-b-ul2/model-00001-of-00002.safetensors",
+ "t5gemma-b-b-ul2/tokenizer.json",
+ } {
+ if !filter(name) {
+ t.Errorf("gated audio filter excluded %q", name)
+ }
+ }
+ if filter("Stable_Audio_3.0_Thumbnail_1x1.png") {
+ t.Fatal("gated audio filter included a preview image")
+ }
+}
+
func TestPullArtifactsRejectsEscapingTarget(t *testing.T) {
err := pullArtifactsWithContext(
context.Background(),
diff --git a/internal/app/speech.go b/internal/app/speech.go
index 3870ba5..3b2406a 100644
--- a/internal/app/speech.go
+++ b/internal/app/speech.go
@@ -48,6 +48,8 @@ func tts(args []string) error {
language := fs.String("language", "", "language name or code")
voiceConsent := fs.Bool("voice-consent", false, "confirm permission to use the reference voice")
seed := fs.Uint64("seed", 0, "speech sampling seed (CPU backends)")
+ speaker := fs.String("speaker", "", "built-in speaker name for CustomVoice models")
+ instruct := fs.String("instruct", "", "voice style or VoiceDesign description")
if err := fs.Parse(args[1:]); err != nil {
return err
}
@@ -89,6 +91,21 @@ func tts(args []string) error {
return fmt.Errorf("voice sample: %w", err)
}
}
+ variant := strings.ToLower(profile.Name)
+ if strings.Contains(variant, "custom-voice") && strings.TrimSpace(*speaker) == "" {
+ return errors.New("Qwen3-TTS CustomVoice models require --speaker")
+ }
+ if strings.Contains(variant, "voice-design") && strings.TrimSpace(*instruct) == "" {
+ return errors.New("Qwen3-TTS VoiceDesign models require --instruct")
+ }
+ if strings.HasPrefix(variant, "qwen3-tts:") &&
+ !strings.Contains(variant, "custom-voice") &&
+ !strings.Contains(variant, "voice-design") && sample == "" {
+ return errors.New("Qwen3-TTS Base models require --voice or --voice-sample")
+ }
+ if strings.HasPrefix(variant, "audio8-tts:") && sample != "" && referenceText == "" {
+ return errors.New("Audio8 voice cloning requires --transcript or --transcript-file")
+ }
model, err := ensureResolvedModel(profile)
if err != nil {
@@ -120,6 +137,7 @@ func tts(args []string) error {
VoiceSample: sample, Transcript: referenceText, Language: *language,
Backend: model.Backend,
VoiceConsent: *voiceConsent, Seed: *seed,
+ Speaker: strings.TrimSpace(*speaker), Instruct: strings.TrimSpace(*instruct),
}); err != nil {
return err
}
diff --git a/internal/app/video.go b/internal/app/video.go
index 5ed58a6..2c35e75 100644
--- a/internal/app/video.go
+++ b/internal/app/video.go
@@ -111,8 +111,9 @@ func video(args []string) error {
} else if *frames <= 0 || (*frames-1)%4 != 0 {
return errors.New("frames must be positive and have the form 4n+1 (for example 17, 41, or 81)")
}
- if profile.Name == "ltx-video:2b-fp16" && (*frames-1)%8 != 0 {
- return errors.New("LTX-Video frames must have the form 8n+1 (for example 17, 49, or 97)")
+ if (profile.Name == "ltx-video:2b-fp16" || profile.Name == "ltx-2.5:22b-bf16-cuda") &&
+ (*frames-1)%8 != 0 {
+ return errors.New("LTX video frames must have the form 8n+1 (for example 17, 49, or 121)")
}
if *steps <= 0 || *fps <= 0 {
return errors.New("steps and fps must be positive")
@@ -219,7 +220,7 @@ func videoFrameRule(model catalog.Resolved) (step, offset int) {
if model.Backend == "comfy-h3-mps" || model.Backend == "comfy-h3-cuda" {
return 17, 5
}
- if model.Name == "ltx-video:2b-fp16" {
+ if model.Name == "ltx-video:2b-fp16" || model.Name == "ltx-2.5:22b-bf16-cuda" {
return 8, 1
}
return 4, 1
diff --git a/internal/audioruntime/requirements.txt b/internal/audioruntime/requirements.txt
new file mode 100644
index 0000000..b7903d4
--- /dev/null
+++ b/internal/audioruntime/requirements.txt
@@ -0,0 +1 @@
+stable-audio-3 @ https://github.com/Stability-AI/stable-audio-3/archive/a0b57f5483c4588f827f3552b7d5c6ca2a9687be.zip
diff --git a/internal/audioruntime/runtime.go b/internal/audioruntime/runtime.go
new file mode 100644
index 0000000..eb96593
--- /dev/null
+++ b/internal/audioruntime/runtime.go
@@ -0,0 +1,130 @@
+package audioruntime
+
+import (
+ "context"
+ "embed"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+
+ "github.com/carlos/tapioca/internal/pythonruntime"
+)
+
+//go:embed stable_audio3.py requirements.txt
+var source embed.FS
+
+type Request struct {
+ ModelPath string
+ Prompt string
+ NegativePrompt string
+ Output string
+ Duration int
+ Steps int
+ Seed uint64
+ Backend string
+}
+
+func Run(ctx context.Context, cacheDir string, request Request) error {
+ return RunWithWriters(ctx, cacheDir, request, os.Stdout, os.Stderr)
+}
+
+func RunWithWriters(
+ ctx context.Context,
+ cacheDir string,
+ request Request,
+ stdout io.Writer,
+ stderr io.Writer,
+) error {
+ if request.Backend != "stable-audio3" {
+ return fmt.Errorf("unsupported audio backend %q", request.Backend)
+ }
+ if request.Duration <= 0 || request.Duration > 120 {
+ return errors.New("Stable Audio 3 duration must be between 1 and 120 seconds")
+ }
+ root := filepath.Join(cacheDir, "audio-runtime", "0.1.0-stable-audio3")
+ for _, name := range []string{"stable_audio3.py", "requirements.txt"} {
+ data, err := source.ReadFile(name)
+ if err != nil {
+ return err
+ }
+ target := filepath.Join(root, name)
+ if err := os.MkdirAll(filepath.Dir(target), 0o755); err != nil {
+ return err
+ }
+ if err := os.WriteFile(target, data, 0o644); err != nil {
+ return err
+ }
+ }
+
+ venv := filepath.Join(root, "venv")
+ python := filepath.Join(venv, "bin", "python")
+ if runtime.GOOS == "windows" {
+ python = filepath.Join(venv, "Scripts", "python.exe")
+ }
+ ready := filepath.Join(venv, ".tapioca-ready")
+ if _, err := os.Stat(ready); err != nil {
+ system, prefix, err := pythonruntime.Find("Stable Audio 3 generation")
+ if err != nil {
+ return err
+ }
+ fmt.Fprintln(stderr, "creating the Stable Audio 3 runtime (first run only)...")
+ cmd := exec.CommandContext(ctx, system, append(prefix, "-m", "venv", venv)...)
+ cmd.Stdout, cmd.Stderr = stderr, stderr
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf("create Stable Audio 3 environment: %w", err)
+ }
+ cmd = exec.CommandContext(ctx, python, "-m", "pip", "install", "--upgrade", "pip")
+ cmd.Stdout, cmd.Stderr = stderr, stderr
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf("upgrade pip: %w", err)
+ }
+ if usesCUDATorch(runtime.GOOS, runtime.GOARCH) {
+ cmd = exec.CommandContext(ctx, python, "-m", "pip", "install",
+ "torch==2.7.1", "torchaudio==2.7.1",
+ "--index-url", "https://download.pytorch.org/whl/cu128")
+ cmd.Stdout, cmd.Stderr = stderr, stderr
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf("install CUDA-enabled Stable Audio 3 PyTorch: %w", err)
+ }
+ }
+ cmd = exec.CommandContext(ctx, python, "-m", "pip", "install", "-r", filepath.Join(root, "requirements.txt"))
+ cmd.Stdout, cmd.Stderr = stderr, stderr
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf("install Stable Audio 3 dependencies: %w", err)
+ }
+ if err := os.WriteFile(ready, []byte("ready\n"), 0o644); err != nil {
+ return err
+ }
+ }
+
+ cmd := exec.CommandContext(ctx, python, pythonArguments(root, request)...)
+ cmd.Stdout, cmd.Stderr = stdout, stderr
+ if err := cmd.Run(); err != nil {
+ return fmt.Errorf("Stable Audio 3 generation failed: %w", err)
+ }
+ return nil
+}
+
+func usesCUDATorch(goos, goarch string) bool {
+ return (goos == "windows" || goos == "linux") && goarch == "amd64"
+}
+
+func pythonArguments(root string, request Request) []string {
+ args := []string{
+ filepath.Join(root, "stable_audio3.py"),
+ "--model", request.ModelPath,
+ "--prompt", request.Prompt,
+ "--output", request.Output,
+ "--duration", fmt.Sprint(request.Duration),
+ "--steps", fmt.Sprint(request.Steps),
+ "--seed", fmt.Sprint(request.Seed),
+ }
+ if request.NegativePrompt != "" {
+ args = append(args, "--negative-prompt", request.NegativePrompt)
+ }
+ return args
+}
diff --git a/internal/audioruntime/runtime_test.go b/internal/audioruntime/runtime_test.go
new file mode 100644
index 0000000..dcdd0f5
--- /dev/null
+++ b/internal/audioruntime/runtime_test.go
@@ -0,0 +1,36 @@
+package audioruntime
+
+import (
+ "path/filepath"
+ "slices"
+ "testing"
+)
+
+func TestPythonArguments(t *testing.T) {
+ args := pythonArguments("/runtime", Request{
+ ModelPath: "/models/stable-audio", Prompt: "bright synth loop",
+ NegativePrompt: "vocals", Output: "/out.wav", Duration: 12, Steps: 8, Seed: 42,
+ })
+ for _, value := range []string{
+ "--model", "/models/stable-audio", "--prompt", "bright synth loop",
+ "--negative-prompt", "vocals", "--duration", "12", "--steps", "8", "--seed", "42",
+ } {
+ if !slices.Contains(args, value) {
+ t.Fatalf("arguments missing %q: %v", value, args)
+ }
+ }
+ if filepath.Base(args[0]) != "stable_audio3.py" {
+ t.Fatalf("unexpected runtime script: %v", args)
+ }
+}
+
+func TestCUDAWheelSelectionExcludesARM64(t *testing.T) {
+ if !usesCUDATorch("windows", "amd64") || !usesCUDATorch("linux", "amd64") {
+ t.Fatal("x64 NVIDIA platforms must install the CUDA wheel")
+ }
+ for _, goos := range []string{"windows", "linux", "darwin"} {
+ if usesCUDATorch(goos, "arm64") {
+ t.Fatalf("%s/arm64 must use the default CPU-compatible dependency", goos)
+ }
+ }
+}
diff --git a/internal/audioruntime/stable_audio3.py b/internal/audioruntime/stable_audio3.py
new file mode 100644
index 0000000..1a43da6
--- /dev/null
+++ b/internal/audioruntime/stable_audio3.py
@@ -0,0 +1,71 @@
+import argparse
+import json
+import os
+
+
+def load_local_model(model_path):
+ import torch
+ from stable_audio_3.loading_utils import load_diffusion_cond
+ from stable_audio_3.model import StableAudioModel
+
+ config_path = os.path.join(model_path, "model_config.json")
+ checkpoint_path = os.path.join(model_path, "model.safetensors")
+ with open(config_path, encoding="utf-8") as handle:
+ model_config = json.load(handle)
+
+ text_encoder = os.path.join(model_path, "t5gemma-b-b-ul2")
+ conditioning = model_config.get("model", {}).get("conditioning", {})
+ for item in conditioning.get("configs", []):
+ if item.get("type") == "t5gemma":
+ item.setdefault("config", {})["model_path"] = text_encoder
+ item["config"].pop("repo_id", None)
+ item["config"].pop("subfolder", None)
+
+ if torch.cuda.is_available():
+ device = "cuda"
+ elif hasattr(torch.backends, "mps") and torch.backends.mps.is_available():
+ device = "mps"
+ else:
+ device = "cpu"
+ model_half = device == "cuda"
+ loaded = load_diffusion_cond(
+ model_config,
+ checkpoint_path,
+ device=device,
+ model_half=model_half,
+ )
+ loaded.use_lora = False
+ loaded.lora_names = []
+ return StableAudioModel(loaded, model_config, device, model_half)
+
+
+def main():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("--model", required=True)
+ parser.add_argument("--prompt", required=True)
+ parser.add_argument("--negative-prompt")
+ parser.add_argument("--output", required=True)
+ parser.add_argument("--duration", required=True, type=int)
+ parser.add_argument("--steps", required=True, type=int)
+ parser.add_argument("--seed", required=True, type=int)
+ args = parser.parse_args()
+
+ if os.path.splitext(args.output)[1].lower() != ".wav":
+ raise SystemExit("Stable Audio 3 output must use the .wav extension")
+ import torchaudio
+
+ model = load_local_model(args.model)
+ audio = model.generate(
+ prompt=args.prompt,
+ negative_prompt=args.negative_prompt,
+ duration=args.duration,
+ steps=args.steps,
+ cfg_scale=1.0,
+ seed=args.seed,
+ batch_size=1,
+ )
+ torchaudio.save(args.output, audio[0].detach().cpu(), model.model.sample_rate)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go
index 8d3dc5e..6ab1650 100644
--- a/internal/catalog/catalog.go
+++ b/internal/catalog/catalog.go
@@ -34,6 +34,9 @@ type Model struct {
Steps int `json:"steps,omitempty"`
Frames int `json:"frames,omitempty"`
FPS int `json:"fps,omitempty"`
+ DurationSeconds int `json:"duration_seconds,omitempty"`
+ MaxDuration int `json:"max_duration_seconds,omitempty"`
+ SampleRate int `json:"sample_rate,omitempty"`
Sizes map[string]string `json:"sizes"`
Memory map[string]string `json:"memory,omitempty"`
GPUs map[string]string `json:"gpus,omitempty"`
@@ -319,10 +322,13 @@ func validateModel(name string, model Model) error {
allowedBackends := map[string]bool{
"": true, "mlx": true, "mlx-vlm": true, "mlx-video": true,
"mflux": true, "diffusers": true, "diffusers-mps": true,
- "diffusers-video": true, "onnx-directml": true, "onnx-cpu": true,
+ "diffusers-video": true, "diffusers-ltx2": true,
+ "onnx-directml": true, "onnx-cpu": true,
"comfy-h3-mps": true, "comfy-h3-cuda": true,
"speech-chatterbox": true, "speech-qwen": true, "speech-qwen-mlx": true,
"speech-audio8-onnx": true, "speech-pocket-tts": true, "speech-sopro": true,
+ "speech-audio8": true, "speech-audio8-mlx": true,
+ "stable-audio3": true,
}
for variant := range model.Files {
if !safeName.MatchString(variant) {
@@ -397,9 +403,86 @@ func validRelativePath(value string) bool {
!strings.HasPrefix(clean, ".."+string(filepath.Separator))
}
+func flux2KleinFP8Artifacts() []Artifact {
+ const base = "black-forest-labs/FLUX.2-klein-4B"
+ artifacts := []Artifact{
+ {Repo: base, Filename: "model_index.json", Target: "model_index.json"},
+ {Repo: base, Filename: "scheduler/scheduler_config.json", Target: "scheduler/scheduler_config.json"},
+ {Repo: base, Filename: "text_encoder/config.json", Target: "text_encoder/config.json"},
+ {Repo: base, Filename: "text_encoder/generation_config.json", Target: "text_encoder/generation_config.json"},
+ {Repo: base, Filename: "text_encoder/model-00001-of-00002.safetensors", Target: "text_encoder/model-00001-of-00002.safetensors"},
+ {Repo: base, Filename: "text_encoder/model-00002-of-00002.safetensors", Target: "text_encoder/model-00002-of-00002.safetensors"},
+ {Repo: base, Filename: "text_encoder/model.safetensors.index.json", Target: "text_encoder/model.safetensors.index.json"},
+ {Repo: base, Filename: "tokenizer/added_tokens.json", Target: "tokenizer/added_tokens.json"},
+ {Repo: base, Filename: "tokenizer/chat_template.jinja", Target: "tokenizer/chat_template.jinja"},
+ {Repo: base, Filename: "tokenizer/merges.txt", Target: "tokenizer/merges.txt"},
+ {Repo: base, Filename: "tokenizer/special_tokens_map.json", Target: "tokenizer/special_tokens_map.json"},
+ {Repo: base, Filename: "tokenizer/tokenizer.json", Target: "tokenizer/tokenizer.json"},
+ {Repo: base, Filename: "tokenizer/tokenizer_config.json", Target: "tokenizer/tokenizer_config.json"},
+ {Repo: base, Filename: "tokenizer/vocab.json", Target: "tokenizer/vocab.json"},
+ {Repo: base, Filename: "transformer/config.json", Target: "transformer/config.json"},
+ {Repo: base, Filename: "vae/config.json", Target: "vae/config.json"},
+ {Repo: base, Filename: "vae/diffusion_pytorch_model.safetensors", Target: "vae/diffusion_pytorch_model.safetensors"},
+ {
+ Repo: "black-forest-labs/FLUX.2-klein-4b-fp8", Filename: "flux-2-klein-4b-fp8.safetensors",
+ Target: "transformer/diffusion_pytorch_model.safetensors",
+ },
+ }
+ return artifacts
+}
+
+func ltx25DiffusersArtifacts() []Artifact {
+ const repo = "Lightricks/LTX-2.5-Diffusers"
+ files := []string{
+ "model_index.json",
+ "modular_model_index.json",
+ "audio_vae/config.json",
+ "audio_vae/diffusion_pytorch_model.safetensors",
+ "connectors/config.json",
+ "connectors/diffusion_pytorch_model.safetensors",
+ "diffusion_decoder/config.json",
+ "diffusion_decoder/diffusion_pytorch_model.safetensors",
+ "duration_head/config.json",
+ "duration_head/diffusion_pytorch_model.safetensors",
+ "latent_upsampler/config.json",
+ "latent_upsampler/diffusion_pytorch_model.safetensors",
+ "scheduler/scheduler_config.json",
+ "temporal_latent_upsampler/config.json",
+ "temporal_latent_upsampler/diffusion_pytorch_model.safetensors",
+ "text_encoder/config.json",
+ "text_encoder/generation_config.json",
+ "text_encoder/model-00001-of-00005.safetensors",
+ "text_encoder/model-00002-of-00005.safetensors",
+ "text_encoder/model-00003-of-00005.safetensors",
+ "text_encoder/model-00004-of-00005.safetensors",
+ "text_encoder/model-00005-of-00005.safetensors",
+ "text_encoder/model.safetensors.index.json",
+ "tokenizer/chat_template.jinja",
+ "tokenizer/tokenizer.json",
+ "tokenizer/tokenizer_config.json",
+ "vae/config.json",
+ "vae/diffusion_pytorch_model.safetensors",
+ "vocoder/config.json",
+ "vocoder/diffusion_pytorch_model.safetensors",
+ }
+ artifacts := make([]Artifact, 0, len(files)+6)
+ for _, filename := range files {
+ artifacts = append(artifacts, Artifact{Repo: repo, Filename: filename, Target: filename})
+ }
+ artifacts = append(artifacts,
+ Artifact{Repo: repo, Filename: "transformer_full/config.json", Target: "transformer/config.json"},
+ Artifact{Repo: repo, Filename: "transformer_full/diffusion_pytorch_model-00001-of-00004.safetensors", Target: "transformer/diffusion_pytorch_model-00001-of-00004.safetensors"},
+ Artifact{Repo: repo, Filename: "transformer_full/diffusion_pytorch_model-00002-of-00004.safetensors", Target: "transformer/diffusion_pytorch_model-00002-of-00004.safetensors"},
+ Artifact{Repo: repo, Filename: "transformer_full/diffusion_pytorch_model-00003-of-00004.safetensors", Target: "transformer/diffusion_pytorch_model-00003-of-00004.safetensors"},
+ Artifact{Repo: repo, Filename: "transformer_full/diffusion_pytorch_model-00004-of-00004.safetensors", Target: "transformer/diffusion_pytorch_model-00004-of-00004.safetensors"},
+ Artifact{Repo: repo, Filename: "transformer_full/diffusion_pytorch_model.safetensors.index.json", Target: "transformer/diffusion_pytorch_model.safetensors.index.json"},
+ )
+ return artifacts
+}
+
var builtInModels = map[string]Model{
- "granite-4.2": graniteModel(),
- "minicpm5-2b": miniCPM5Model(),
+ "granite-4.2": graniteModel(),
+ "minicpm5-2b": miniCPM5Model(),
"spark-x2.5-4b": sparkX25Model(),
"chatterbox": {
Name: "chatterbox",
@@ -443,37 +526,173 @@ var builtInModels = map[string]Model{
Name: "qwen3-tts",
Kind: "speech",
Default: "0.6b",
+ Files: map[string]string{
+ "0.6b": "",
+ "0.6b-mlx": "",
+ "1.7b-base": "",
+ "1.7b-base-mlx": "",
+ "1.7b-custom-voice": "",
+ "1.7b-custom-voice-mlx": "",
+ "1.7b-voice-design": "",
+ "1.7b-voice-design-mlx": "",
+ },
+ Backends: map[string]string{
+ "0.6b": "speech-qwen",
+ "0.6b-mlx": "speech-qwen-mlx",
+ "1.7b-base": "speech-qwen",
+ "1.7b-base-mlx": "speech-qwen-mlx",
+ "1.7b-custom-voice": "speech-qwen",
+ "1.7b-custom-voice-mlx": "speech-qwen-mlx",
+ "1.7b-voice-design": "speech-qwen",
+ "1.7b-voice-design-mlx": "speech-qwen-mlx",
+ },
+ Repos: map[string]string{
+ "0.6b": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
+ "0.6b-mlx": "mlx-community/Qwen3-TTS-12Hz-0.6B-Base-bf16",
+ "1.7b-base": "Qwen/Qwen3-TTS-12Hz-1.7B-Base",
+ "1.7b-base-mlx": "mlx-community/Qwen3-TTS-12Hz-1.7B-Base-bf16",
+ "1.7b-custom-voice": "Qwen/Qwen3-TTS-12Hz-1.7B-CustomVoice",
+ "1.7b-custom-voice-mlx": "mlx-community/Qwen3-TTS-12Hz-1.7B-CustomVoice-bf16",
+ "1.7b-voice-design": "Qwen/Qwen3-TTS-12Hz-1.7B-VoiceDesign",
+ "1.7b-voice-design-mlx": "mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-bf16",
+ },
+ Sizes: map[string]string{
+ "0.6b": "~2.5 GiB",
+ "0.6b-mlx": "~2.5 GiB",
+ "1.7b-base": "~4.3 GiB",
+ "1.7b-base-mlx": "~4.3 GiB",
+ "1.7b-custom-voice": "~4.3 GiB",
+ "1.7b-custom-voice-mlx": "~4.3 GiB",
+ "1.7b-voice-design": "~4.3 GiB",
+ "1.7b-voice-design-mlx": "~4.3 GiB",
+ },
+ Memory: map[string]string{
+ "0.6b": "12 GiB min; 16 GiB recommended",
+ "0.6b-mlx": "12 GiB min; 16 GiB recommended",
+ "1.7b-base": "16 GiB min; 24 GiB recommended",
+ "1.7b-base-mlx": "16 GiB min; 24 GiB recommended",
+ "1.7b-custom-voice": "16 GiB min; 24 GiB recommended",
+ "1.7b-custom-voice-mlx": "16 GiB min; 24 GiB recommended",
+ "1.7b-voice-design": "16 GiB min; 24 GiB recommended",
+ "1.7b-voice-design-mlx": "16 GiB min; 24 GiB recommended",
+ },
+ GPUs: map[string]string{
+ "0.6b": "NVIDIA CUDA recommended; CPU supported",
+ "0.6b-mlx": "Apple Silicon GPU",
+ "1.7b-base": "NVIDIA CUDA recommended; CPU supported",
+ "1.7b-base-mlx": "Apple Silicon GPU",
+ "1.7b-custom-voice": "NVIDIA CUDA recommended; CPU supported",
+ "1.7b-custom-voice-mlx": "Apple Silicon GPU",
+ "1.7b-voice-design": "NVIDIA CUDA recommended; CPU supported",
+ "1.7b-voice-design-mlx": "Apple Silicon GPU",
+ },
+ Languages: map[string]string{
+ "0.6b": "10 languages",
+ "0.6b-mlx": "10 languages",
+ "1.7b-base": "10 languages",
+ "1.7b-base-mlx": "10 languages",
+ "1.7b-custom-voice": "10 languages",
+ "1.7b-custom-voice-mlx": "10 languages",
+ "1.7b-voice-design": "10 languages",
+ "1.7b-voice-design-mlx": "10 languages",
+ },
+ Features: map[string]string{
+ "0.6b": "3-second voice cloning, streaming architecture",
+ "0.6b-mlx": "3-second voice cloning, Apple Silicon optimized",
+ "1.7b-base": "higher-quality 3-second voice cloning, streaming architecture",
+ "1.7b-base-mlx": "higher-quality voice cloning, Apple Silicon optimized",
+ "1.7b-custom-voice": "9 built-in speakers, instruction-based emotion and style",
+ "1.7b-custom-voice-mlx": "9 built-in speakers, instruction-based emotion and style, Apple Silicon optimized",
+ "1.7b-voice-design": "natural-language voice design without a reference recording",
+ "1.7b-voice-design-mlx": "natural-language voice design without a reference recording, Apple Silicon optimized",
+ },
+ License: "Apache-2.0",
+ LicenseURL: "https://huggingface.co/Qwen/Qwen3-TTS-12Hz-1.7B-Base",
+ },
+ "audio8-tts": {
+ Name: "audio8-tts",
+ Kind: "speech",
+ Default: "0.6b",
+ PlatformDefaults: map[string]string{
+ "darwin/arm64": "0.6b-mlx",
+ },
Files: map[string]string{
"0.6b": "",
"0.6b-mlx": "",
},
Backends: map[string]string{
- "0.6b": "speech-qwen",
- "0.6b-mlx": "speech-qwen-mlx",
+ "0.6b": "speech-audio8",
+ "0.6b-mlx": "speech-audio8-mlx",
},
Repos: map[string]string{
- "0.6b": "Qwen/Qwen3-TTS-12Hz-0.6B-Base",
- "0.6b-mlx": "mlx-community/Qwen3-TTS-12Hz-0.6B-Base-bf16",
+ "0.6b": "Audio8/Audio8-TTS-Preview-0.6b",
+ "0.6b-mlx": "mlx-community/Audio8-TTS-Preview-0.6b-bf16",
},
Sizes: map[string]string{
- "0.6b": "~2.5 GiB",
- "0.6b-mlx": "~2.5 GiB",
+ "0.6b": "~2.4 GiB",
+ "0.6b-mlx": "~2.4 GiB",
},
Memory: map[string]string{
- "0.6b": "12 GiB min; 16 GiB recommended",
- "0.6b-mlx": "12 GiB min; 16 GiB recommended",
+ "0.6b": "8 GiB min; 12 GiB recommended",
+ "0.6b-mlx": "8 GiB min; 12 GiB recommended",
},
GPUs: map[string]string{
"0.6b": "NVIDIA CUDA recommended; CPU supported",
"0.6b-mlx": "Apple Silicon GPU",
},
Languages: map[string]string{
- "0.6b": "10 languages",
- "0.6b-mlx": "10 languages",
+ "0.6b": "11 languages",
+ "0.6b-mlx": "11 languages",
},
Features: map[string]string{
- "0.6b": "3-second voice cloning, streaming architecture",
- "0.6b-mlx": "3-second voice cloning, Apple Silicon optimized",
+ "0.6b": "preview, 44.1 kHz speech, zero-shot voice cloning, default voice",
+ "0.6b-mlx": "preview, 44.1 kHz speech, zero-shot voice cloning, Apple Silicon optimized",
+ },
+ License: "Apache-2.0",
+ LicenseURL: "https://huggingface.co/Audio8/Audio8-TTS-Preview-0.6b",
+ },
+ "stable-audio-3": {
+ Name: "stable-audio-3",
+ Kind: "audio",
+ Default: "small-music",
+ DurationSeconds: 30,
+ MaxDuration: 120,
+ SampleRate: 44100,
+ Steps: 8,
+ Gated: true,
+ License: "Stability AI Community License",
+ LicenseURL: "https://huggingface.co/collections/stabilityai/stable-audio-3",
+ Files: map[string]string{
+ "small-music": "",
+ "small-sfx": "",
+ },
+ Backends: map[string]string{
+ "small-music": "stable-audio3",
+ "small-sfx": "stable-audio3",
+ },
+ Repos: map[string]string{
+ "small-music": "stabilityai/stable-audio-3-small-music",
+ "small-sfx": "stabilityai/stable-audio-3-small-sfx",
+ },
+ Sizes: map[string]string{
+ "small-music": "~3.3 GiB",
+ "small-sfx": "~3.3 GiB",
+ },
+ Memory: map[string]string{
+ "small-music": "16 GiB min; 24 GiB recommended",
+ "small-sfx": "16 GiB min; 24 GiB recommended",
+ },
+ GPUs: map[string]string{
+ "small-music": "Apple Silicon MPS, NVIDIA CUDA, or CPU",
+ "small-sfx": "Apple Silicon MPS, NVIDIA CUDA, or CPU",
+ },
+ Languages: map[string]string{
+ "small-music": "English prompts",
+ "small-sfx": "English prompts",
+ },
+ Features: map[string]string{
+ "small-music": "text-to-music, 44.1 kHz stereo, variable duration up to 120 seconds",
+ "small-sfx": "text-to-sound-effects, 44.1 kHz stereo, variable duration up to 120 seconds",
},
},
"glm-4.7-flash": {
@@ -524,26 +743,73 @@ var builtInModels = map[string]Model{
},
},
"flux2-klein": {
- Name: "flux2-klein",
- Repo: "mlx-community/flux2-klein-4b-4bit",
- Kind: "image",
- Default: "4b-q4-mlx",
- Width: 1024,
- Height: 1024,
- Steps: 4,
- Files: map[string]string{"4b-q4-mlx": ""},
+ Name: "flux2-klein",
+ Repo: "mlx-community/flux2-klein-4b-4bit",
+ Kind: "image",
+ Default: "4b-q4-mlx",
+ Width: 1024,
+ Height: 1024,
+ Steps: 4,
+ GuidanceScale: 1.0,
+ GuidanceScaleSet: true,
+ PlatformDefaults: map[string]string{
+ "windows": "4b-bf16-cuda",
+ "linux": "4b-bf16-cuda",
+ "darwin/arm64": "4b-q4-mlx",
+ },
+ Files: map[string]string{
+ "4b-q4-mlx": "",
+ "4b-bf16-cuda": "",
+ "4b-fp8-cuda": "",
+ },
Backends: map[string]string{
- "4b-q4-mlx": "mflux",
+ "4b-q4-mlx": "mflux",
+ "4b-bf16-cuda": "diffusers",
+ "4b-fp8-cuda": "diffusers",
+ },
+ Repos: map[string]string{
+ "4b-q4-mlx": "mlx-community/flux2-klein-4b-4bit",
+ "4b-bf16-cuda": "black-forest-labs/FLUX.2-klein-4B",
+ "4b-fp8-cuda": "black-forest-labs/FLUX.2-klein-4b-fp8",
},
- Sizes: map[string]string{"4b-q4-mlx": "~4.6 GiB"},
- Memory: map[string]string{"4b-q4-mlx": "16 GiB min; 24 GiB recommended"},
- GPUs: map[string]string{"4b-q4-mlx": "Apple Silicon GPU"},
+ Sizes: map[string]string{
+ "4b-q4-mlx": "~4.6 GiB",
+ "4b-bf16-cuda": "~15 GiB",
+ "4b-fp8-cuda": "~11 GiB",
+ },
+ Memory: map[string]string{
+ "4b-q4-mlx": "16 GiB min; 24 GiB recommended",
+ "4b-bf16-cuda": "24 GiB system RAM; 13 GiB VRAM min",
+ "4b-fp8-cuda": "24 GiB system RAM; 12 GiB VRAM recommended",
+ },
+ GPUs: map[string]string{
+ "4b-q4-mlx": "Apple Silicon GPU",
+ "4b-bf16-cuda": "NVIDIA RTX 3090/4070 or newer",
+ "4b-fp8-cuda": "NVIDIA Ada or newer recommended; Ampere supported with reduced benefit",
+ },
+ Features: map[string]string{
+ "4b-q4-mlx": "4-step generation, image editing, multi-reference editing",
+ "4b-bf16-cuda": "4-step generation, image editing, multi-reference editing, Apache-2.0",
+ "4b-fp8-cuda": "reduced-storage CUDA bundle, image editing, multi-reference editing, Apache-2.0",
+ },
+ Artifacts: map[string][]Artifact{
+ "4b-fp8-cuda": flux2KleinFP8Artifacts(),
+ },
+ License: "Apache-2.0",
+ LicenseURL: "https://huggingface.co/black-forest-labs/FLUX.2-klein-4B",
},
"qwen3.6": {
Name: "qwen3.6",
Kind: "text",
Default: "35b-mlx",
+ PlatformDefaults: map[string]string{
+ "windows": "27b-q4_k_m",
+ "linux": "27b-q4_k_m",
+ "darwin/arm64": "35b-mlx",
+ },
Files: map[string]string{
+ "27b-q4_k_m": "Qwen3.6-27B-Q4_K_M.gguf",
+ "27b-q8_0": "Qwen3.6-27B-Q8_0.gguf",
"35b-mlx": "",
"35b-mlx-4bit": "",
"35b-mlx-6bit": "",
@@ -556,17 +822,71 @@ var builtInModels = map[string]Model{
"35b-mlx-8bit": "mlx-vlm",
},
Repos: map[string]string{
+ "27b-q4_k_m": "ggml-org/Qwen3.6-27B-GGUF",
+ "27b-q8_0": "ggml-org/Qwen3.6-27B-GGUF",
"35b-mlx": "mlx-community/Qwen3.6-35B-A3B-4bit",
"35b-mlx-4bit": "mlx-community/Qwen3.6-35B-A3B-4bit",
"35b-mlx-6bit": "mlx-community/Qwen3.6-35B-A3B-6bit",
"35b-mlx-8bit": "mlx-community/Qwen3.6-35B-A3B-8bit",
},
Sizes: map[string]string{
+ "27b-q4_k_m": "~19.1 GiB",
+ "27b-q8_0": "~28.6 GiB",
"35b-mlx": "~20 GiB",
"35b-mlx-4bit": "~20 GiB",
"35b-mlx-6bit": "~27 GiB",
"35b-mlx-8bit": "~36 GiB",
},
+ Memory: map[string]string{
+ "27b-q4_k_m": "24 GiB min; 32 GiB recommended",
+ "27b-q8_0": "40 GiB min; 48 GiB recommended",
+ },
+ GPUs: map[string]string{
+ "27b-q4_k_m": "Optional Metal, CUDA, Vulkan, or ROCm acceleration",
+ "27b-q8_0": "Optional Metal, CUDA, Vulkan, or ROCm acceleration",
+ },
+ Features: map[string]string{
+ "27b-q4_k_m": "reasoning, coding, tool use, image understanding with optional mmproj",
+ "27b-q8_0": "higher-fidelity reasoning, coding, tool use, image understanding with optional mmproj",
+ },
+ },
+ "gemma4": {
+ Name: "gemma4",
+ Repo: "ggml-org/gemma-4-E2B-it-GGUF",
+ Kind: "text",
+ Default: "e2b-q4_0",
+ PlatformDefaults: map[string]string{
+ "darwin/arm64": "e2b-mlx-4bit",
+ },
+ Files: map[string]string{
+ "e2b-q4_0": "gemma-4-E2B-it-Q4_0.gguf",
+ "e2b-mlx-4bit": "",
+ },
+ Backends: map[string]string{
+ "e2b-mlx-4bit": "mlx-vlm",
+ },
+ Repos: map[string]string{
+ "e2b-q4_0": "ggml-org/gemma-4-E2B-it-GGUF",
+ "e2b-mlx-4bit": "mlx-community/gemma-4-e2b-it-4bit",
+ },
+ Sizes: map[string]string{
+ "e2b-q4_0": "~2.8 GiB",
+ "e2b-mlx-4bit": "~3.6 GiB",
+ },
+ Memory: map[string]string{
+ "e2b-q4_0": "8 GiB min; 12 GiB recommended",
+ "e2b-mlx-4bit": "12 GiB min; 16 GiB recommended",
+ },
+ GPUs: map[string]string{
+ "e2b-q4_0": "Optional Metal, CUDA, Vulkan, or ROCm acceleration",
+ "e2b-mlx-4bit": "Apple Silicon GPU",
+ },
+ Features: map[string]string{
+ "e2b-q4_0": "reasoning, system prompts, function calling, 128K context",
+ "e2b-mlx-4bit": "reasoning, system prompts, function calling, image understanding",
+ },
+ License: "Apache-2.0",
+ LicenseURL: "https://huggingface.co/google/gemma-4-E2B-it",
},
"qwen3.8": {
Name: "qwen3.8",
@@ -929,6 +1249,41 @@ var builtInModels = map[string]Model{
Memory: map[string]string{"2b-fp16": "24 GiB min; 32 GiB recommended"},
GPUs: map[string]string{"2b-fp16": "NVIDIA CUDA, 8 GiB+ VRAM; CPU offload"},
},
+ "ltx-2.5": {
+ Name: "ltx-2.5",
+ Repo: "Lightricks/LTX-2.5-Diffusers",
+ Kind: "video",
+ Default: "22b-bf16-cuda",
+ Width: 768,
+ Height: 512,
+ Steps: 30,
+ Frames: 121,
+ FPS: 24,
+ Gated: true,
+ License: "LTX-2 Community License",
+ LicenseURL: "https://huggingface.co/Lightricks/LTX-2.5-Diffusers",
+ Files: map[string]string{
+ "22b-bf16-cuda": "",
+ },
+ Backends: map[string]string{
+ "22b-bf16-cuda": "diffusers-ltx2",
+ },
+ Sizes: map[string]string{
+ "22b-bf16-cuda": "~68 GiB",
+ },
+ Memory: map[string]string{
+ "22b-bf16-cuda": "96 GiB system RAM recommended; sequential CPU offload enabled",
+ },
+ GPUs: map[string]string{
+ "22b-bf16-cuda": "NVIDIA CUDA, 24 GiB VRAM minimum; 48 GiB recommended",
+ },
+ Features: map[string]string{
+ "22b-bf16-cuda": "text-to-video, synchronized stereo audio, multishot prompting, 1-20 second clips",
+ },
+ Artifacts: map[string][]Artifact{
+ "22b-bf16-cuda": ltx25DiffusersArtifacts(),
+ },
+ },
"stable-video-diffusion": {
Name: "stable-video-diffusion",
Repo: "stabilityai/stable-video-diffusion-img2vid-xt",
@@ -962,6 +1317,9 @@ type Resolved struct {
Steps int
Frames int
FPS int
+ DurationSeconds int
+ MaxDuration int
+ SampleRate int
Size string
Platform string
Memory string
@@ -1060,9 +1418,9 @@ func ResolveForPlatform(ref, goos, goarch string) (Resolved, error) {
backend := m.Backends[strings.ToLower(tag)]
platform := "Windows, macOS, Linux"
switch backend {
- case "mlx", "mlx-vlm", "mlx-video", "mflux", "speech-qwen-mlx":
+ case "mlx", "mlx-vlm", "mlx-video", "mflux", "speech-qwen-mlx", "speech-audio8-mlx":
platform = "macOS Apple Silicon"
- case "diffusers", "diffusers-video":
+ case "diffusers", "diffusers-video", "diffusers-ltx2":
platform = "Windows/Linux NVIDIA"
case "diffusers-mps":
platform = "macOS Apple Silicon"
@@ -1078,6 +1436,10 @@ func ResolveForPlatform(ref, goos, goarch string) (Resolved, error) {
platform = "macOS Apple Silicon CPU (experimental)"
case "speech-qwen":
platform = "Windows/Linux NVIDIA or CPU"
+ case "speech-audio8":
+ platform = "Windows, macOS, Linux; CUDA recommended"
+ case "stable-audio3":
+ platform = "Windows, macOS, Linux"
case "onnx-directml":
platform = "Windows x64 AMD/Intel/NVIDIA"
case "onnx-cpu":
@@ -1085,26 +1447,29 @@ func ResolveForPlatform(ref, goos, goarch string) (Resolved, error) {
}
memory, gpu := requirements(m, strings.ToLower(tag), backend)
return Resolved{
- Download: download,
- Name: m.Name + ":" + strings.ToLower(tag),
- Repo: repo,
- Filename: filename,
- URL: url,
- Kind: m.Kind,
- Backend: backend,
- Width: m.Width,
- Height: m.Height,
- Steps: m.Steps,
- Frames: m.Frames,
- FPS: m.FPS,
- Size: m.Sizes[strings.ToLower(tag)],
- Platform: platform,
- Memory: memory,
- GPU: gpu,
- Languages: m.Languages[strings.ToLower(tag)],
- Features: m.Features[strings.ToLower(tag)],
- Artifacts: append([]Artifact(nil), m.Artifacts[strings.ToLower(tag)]...),
- GuidanceScale: m.GuidanceScale, GuidanceScaleSet: m.GuidanceScaleSet,
+ Download: download,
+ Name: m.Name + ":" + strings.ToLower(tag),
+ Repo: repo,
+ Filename: filename,
+ URL: url,
+ Kind: m.Kind,
+ Backend: backend,
+ Width: m.Width,
+ Height: m.Height,
+ Steps: m.Steps,
+ Frames: m.Frames,
+ FPS: m.FPS,
+ DurationSeconds: m.DurationSeconds,
+ MaxDuration: m.MaxDuration,
+ SampleRate: m.SampleRate,
+ Size: m.Sizes[strings.ToLower(tag)],
+ Platform: platform,
+ Memory: memory,
+ GPU: gpu,
+ Languages: m.Languages[strings.ToLower(tag)],
+ Features: m.Features[strings.ToLower(tag)],
+ Artifacts: append([]Artifact(nil), m.Artifacts[strings.ToLower(tag)]...),
+ GuidanceScale: m.GuidanceScale, GuidanceScaleSet: m.GuidanceScaleSet,
Gated: m.Gated, License: m.License, LicenseURL: m.LicenseURL,
}, nil
}
@@ -1115,7 +1480,7 @@ func requirements(model Model, variant, backend string) (string, string) {
}
size := model.Sizes[variant]
switch backend {
- case "mlx", "mlx-vlm", "mflux", "speech-qwen-mlx":
+ case "mlx", "mlx-vlm", "mflux", "speech-qwen-mlx", "speech-audio8-mlx":
switch {
case strings.Contains(size, "~36"):
return "48 GiB min; 64 GiB recommended", "Apple Silicon GPU"
diff --git a/internal/catalog/catalog_test.go b/internal/catalog/catalog_test.go
index 43a8777..83d535d 100644
--- a/internal/catalog/catalog_test.go
+++ b/internal/catalog/catalog_test.go
@@ -1,6 +1,9 @@
package catalog
-import "testing"
+import (
+ "strings"
+ "testing"
+)
func TestResolveGLM(t *testing.T) {
got, err := Resolve("glm-4.7-flash:q8_0")
@@ -47,6 +50,26 @@ func TestResolveImageDefaultsToDiffusersOnWindows(t *testing.T) {
}
}
+func TestResolveFlux2KleinCUDAProfiles(t *testing.T) {
+ windows, err := ResolveForPlatform("flux2-klein", "windows", "amd64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if windows.Name != "flux2-klein:4b-bf16-cuda" ||
+ windows.Backend != "diffusers" || windows.GuidanceScale != 1.0 ||
+ !windows.GuidanceScaleSet || windows.Repo != "black-forest-labs/FLUX.2-klein-4B" {
+ t.Fatalf("unexpected FLUX.2 CUDA default: %#v", windows)
+ }
+ fp8, err := ResolveForPlatform("flux2-klein:4b-fp8-cuda", "windows", "amd64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fp8.Backend != "diffusers" || len(fp8.Artifacts) != 18 ||
+ fp8.Artifacts[len(fp8.Artifacts)-1].Target != "transformer/diffusion_pytorch_model.safetensors" {
+ t.Fatalf("unexpected FLUX.2 FP8 bundle: %#v", fp8)
+ }
+}
+
func TestResolveQwenMLXAlias(t *testing.T) {
got, err := ResolveFor("qwen3.6:35b-mlx", "darwin")
if err != nil {
@@ -81,6 +104,57 @@ func TestResolveQwen38PlatformDefaults(t *testing.T) {
}
}
+func TestResolveCurrentGenerationTextModels(t *testing.T) {
+ windowsQwen, err := ResolveForPlatform("qwen3.6", "windows", "amd64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if windowsQwen.Name != "qwen3.6:27b-q4_k_m" ||
+ windowsQwen.Filename != "Qwen3.6-27B-Q4_K_M.gguf" ||
+ windowsQwen.Repo != "ggml-org/Qwen3.6-27B-GGUF" {
+ t.Fatalf("unexpected Windows Qwen3.6 resolution: %#v", windowsQwen)
+ }
+
+ macGemma, err := ResolveForPlatform("gemma4", "darwin", "arm64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if macGemma.Name != "gemma4:e2b-mlx-4bit" ||
+ macGemma.Backend != "mlx-vlm" ||
+ macGemma.Repo != "mlx-community/gemma-4-e2b-it-4bit" {
+ t.Fatalf("unexpected macOS Gemma 4 resolution: %#v", macGemma)
+ }
+
+ linuxGemma, err := ResolveForPlatform("gemma4", "linux", "amd64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if linuxGemma.Name != "gemma4:e2b-q4_0" ||
+ linuxGemma.Filename != "gemma-4-E2B-it-Q4_0.gguf" {
+ t.Fatalf("unexpected Linux Gemma 4 resolution: %#v", linuxGemma)
+ }
+}
+
+func TestResolveCurrentSpeechModels(t *testing.T) {
+ qwen, err := ResolveForPlatform("qwen3-tts:1.7b-voice-design-mlx", "darwin", "arm64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if qwen.Backend != "speech-qwen-mlx" ||
+ qwen.Repo != "mlx-community/Qwen3-TTS-12Hz-1.7B-VoiceDesign-bf16" ||
+ !strings.Contains(qwen.Features, "voice design") {
+ t.Fatalf("unexpected Qwen voice design profile: %#v", qwen)
+ }
+ audio8, err := ResolveForPlatform("audio8-tts", "darwin", "arm64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if audio8.Name != "audio8-tts:0.6b-mlx" ||
+ audio8.Backend != "speech-audio8-mlx" || audio8.Languages != "11 languages" {
+ t.Fatalf("unexpected Audio8 profile: %#v", audio8)
+ }
+}
+
func TestResolveCompactWindowsModels(t *testing.T) {
tests := map[string]string{
"qwen3:4b-q4_k_m": "Qwen3-4B-Q4_K_M.gguf",
@@ -248,6 +322,36 @@ func TestLowMemoryVideoProfiles(t *testing.T) {
}
}
+func TestResolveLTX25CuratedBundle(t *testing.T) {
+ model, err := ResolveForPlatform("ltx-2.5", "windows", "amd64")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if model.Backend != "diffusers-ltx2" || !model.Gated || model.FPS != 24 ||
+ model.Frames != 121 || len(model.Artifacts) != 36 {
+ t.Fatalf("unexpected LTX-2.5 profile: %#v", model)
+ }
+ seenTargets := map[string]bool{}
+ for _, artifact := range model.Artifacts {
+ if seenTargets[artifact.Target] {
+ t.Fatalf("duplicate LTX-2.5 artifact target %q", artifact.Target)
+ }
+ seenTargets[artifact.Target] = true
+ if strings.Contains(artifact.Target, "transformer_full/") {
+ t.Fatalf("LTX-2.5 exposes source-only transformer_full path: %#v", artifact)
+ }
+ }
+ for _, target := range []string{
+ "audio_vae/diffusion_pytorch_model.safetensors",
+ "transformer/diffusion_pytorch_model.safetensors.index.json",
+ "vocoder/diffusion_pytorch_model.safetensors",
+ } {
+ if !seenTargets[target] {
+ t.Errorf("LTX-2.5 bundle is missing %s", target)
+ }
+ }
+}
+
func TestResolveMiniMaxH3PlatformBundles(t *testing.T) {
mac, err := ResolveForPlatform("minimax-h3", "darwin", "arm64")
if err != nil {
diff --git a/internal/control/creator.go b/internal/control/creator.go
index c82a47e..202775b 100644
--- a/internal/control/creator.go
+++ b/internal/control/creator.go
@@ -14,6 +14,7 @@ import (
"time"
"github.com/carlos/tapioca/internal/adapter"
+ "github.com/carlos/tapioca/internal/audioruntime"
"github.com/carlos/tapioca/internal/catalog"
"github.com/carlos/tapioca/internal/config"
"github.com/carlos/tapioca/internal/imageruntime"
@@ -24,6 +25,7 @@ import (
type ImageRunFunc func(context.Context, string, imageruntime.Request, io.Writer, io.Writer) error
type VideoRunFunc func(context.Context, string, videoruntime.Request, io.Writer, io.Writer) error
type SpeechRunFunc func(context.Context, string, speechruntime.Request, io.Writer, io.Writer) error
+type AudioRunFunc func(context.Context, string, audioruntime.Request, io.Writer, io.Writer) error
type LoRASelection struct {
Reference string `json:"reference"`
@@ -86,9 +88,21 @@ type SpeechGenerateParams struct {
VoiceSample string `json:"voice_sample,omitempty"`
Transcript string `json:"transcript,omitempty"`
Language string `json:"language,omitempty"`
+ Speaker string `json:"speaker,omitempty"`
+ Instruct string `json:"instruct,omitempty"`
OutputName string `json:"output_name,omitempty"`
}
+type AudioGenerateParams struct {
+ Model string `json:"model"`
+ Prompt string `json:"prompt"`
+ NegativePrompt string `json:"negative_prompt,omitempty"`
+ Duration int `json:"duration_seconds,omitempty"`
+ Steps int `json:"steps,omitempty"`
+ Seed uint64 `json:"seed,omitempty"`
+ OutputName string `json:"output_name,omitempty"`
+}
+
func (h *Handler) handleCreator(
ctx context.Context,
request Request,
@@ -109,7 +123,7 @@ func (h *Handler) handleCreator(
}
filtered := make([]CreatorCatalogModel, 0, len(models))
for _, model := range models {
- if model.Kind == "image" || model.Kind == "video" || model.Kind == "speech" {
+ if model.Kind == "image" || model.Kind == "video" || model.Kind == "speech" || model.Kind == "audio" {
item := CreatorCatalogModel{
CatalogModel: model,
Operation: model.Kind + ".generate",
@@ -153,6 +167,13 @@ func (h *Handler) handleCreator(
}
result, err := h.generateSpeech(ctx, request.ID, params, request.Method == "voice.clone")
return result, err, true
+ case "audio.generate":
+ var params AudioGenerateParams
+ if err := decodeParams(request.Params, ¶ms); err != nil {
+ return nil, err, true
+ }
+ result, err := h.generateAudio(ctx, request.ID, params)
+ return result, err, true
case "lora.list":
if err := decodeNoParams(request.Params); err != nil {
return nil, err, true
@@ -226,6 +247,14 @@ func creatorCapabilities() map[string]any {
"available": true, "method": "speech.generate",
"supports_voice_reference": true,
},
+ "audio": map[string]any{
+ "available": true, "method": "audio.generate",
+ "parameters": map[string]any{
+ "duration_seconds": "1 through the model maximum; omit for the catalog default",
+ "steps": "1 through 200; omit for the catalog default",
+ "output_name": "optional .wav filename without directories",
+ },
+ },
"voice_clone": map[string]any{
"available": true, "method": "voice.clone",
"requires_voice_reference": true,
@@ -237,7 +266,7 @@ func creatorCapabilities() map[string]any {
"progress": map[string]any{
"mode": "indeterminate",
"numeric_when_available": false,
- "reason": "current image and video runtimes expose log streams but no numeric progress callback",
+ "reason": "current media runtimes expose log streams but no numeric progress callback",
},
}
}
@@ -334,6 +363,9 @@ func (h *Handler) generateVideo(
if err := validateVideo(params); err != nil {
return nil, err
}
+ if protocolError := validateVideoModelFrames(model.Name, params.Frames); protocolError != nil {
+ return nil, protocolError
+ }
if (model.Backend == "comfy-h3-mps" || model.Backend == "comfy-h3-cuda") &&
(params.Frames < 5 || (params.Frames-5)%17 != 0) {
return nil, invalidParams("MiniMax-H3 frames must have the form 17n+5", nil)
@@ -376,6 +408,17 @@ func (h *Handler) generateVideo(
return creatorOutput(output, "video", model.Name, h.dependencies.Now())
}
+func validateVideoModelFrames(modelName string, frames int) *ProtocolError {
+ if (modelName == "ltx-video:2b-fp16" || modelName == "ltx-2.5:22b-bf16-cuda") &&
+ (frames-1)%8 != 0 {
+ return invalidParams(
+ "LTX video frames must have the form 8n+1 (for example 17, 49, or 121)",
+ nil,
+ )
+ }
+ return nil
+}
+
func runImage(
ctx context.Context,
cacheDir string,
@@ -406,6 +449,68 @@ func runSpeech(
return speechruntime.RunWithWriters(ctx, cacheDir, request, stdout, stderr)
}
+func runAudio(
+ ctx context.Context,
+ cacheDir string,
+ request audioruntime.Request,
+ stdout io.Writer,
+ stderr io.Writer,
+) error {
+ return audioruntime.RunWithWriters(ctx, cacheDir, request, stdout, stderr)
+}
+
+func (h *Handler) generateAudio(
+ ctx context.Context,
+ requestID string,
+ params AudioGenerateParams,
+) (any, *ProtocolError) {
+ model, resolved, protocolError := resolveCreatorModel(params.Model, "audio")
+ if protocolError != nil {
+ return nil, protocolError
+ }
+ params.Prompt = strings.TrimSpace(params.Prompt)
+ if params.Prompt == "" {
+ return nil, invalidParams("params.prompt is required", nil)
+ }
+ if params.Duration == 0 {
+ params.Duration = resolved.DurationSeconds
+ }
+ if params.Steps == 0 {
+ params.Steps = resolved.Steps
+ }
+ if params.Duration < 1 || params.Duration > resolved.MaxDuration {
+ return nil, invalidParams(fmt.Sprintf(
+ "duration_seconds must be between 1 and %d", resolved.MaxDuration,
+ ), nil)
+ }
+ if params.Steps < 1 || params.Steps > 200 {
+ return nil, invalidParams("steps must be between 1 and 200", nil)
+ }
+ output, err := managedOutputPath("audio", requestID, params.OutputName, ".wav")
+ if err != nil {
+ return nil, err
+ }
+ reportProgress(ctx, map[string]any{
+ "stage": "starting", "determinate": false, "output": output,
+ })
+ logs := creatorLogWriter{ctx: ctx}
+ cacheDir, cacheError := runtimeCacheDir()
+ if cacheError != nil {
+ return nil, cacheError
+ }
+ runError := h.dependencies.Audio(ctx, cacheDir, audioruntime.Request{
+ ModelPath: model.Path, Prompt: params.Prompt,
+ NegativePrompt: params.NegativePrompt, Output: output,
+ Duration: params.Duration, Steps: params.Steps, Seed: params.Seed,
+ Backend: model.Backend,
+ }, logs, logs)
+ if runError != nil {
+ _ = os.Remove(output)
+ return nil, operationError(ctx, "audio_generation_failed", runError)
+ }
+ return creatorOutput(output, "audio", model.Name, h.dependencies.Now())
+}
+
func (h *Handler) generateSpeech(
ctx context.Context,
requestID string,
@@ -438,10 +543,22 @@ func (h *Handler) generateSpeech(
if clone && voiceSample == "" {
return nil, invalidParams("params.voice_sample is required for voice cloning", nil)
}
- if (model.Backend == "speech-qwen" || model.Backend == "speech-qwen-mlx" ||
- strings.Contains(model.Name, "chatterbox:nano")) && voiceSample == "" {
+ qwenBase := strings.HasPrefix(model.Name, "qwen3-tts:") &&
+ !strings.Contains(model.Name, "custom-voice") &&
+ !strings.Contains(model.Name, "voice-design")
+ if (qwenBase || strings.Contains(model.Name, "chatterbox:nano")) && voiceSample == "" {
return nil, invalidParams(model.Name+" requires a voice reference", nil)
}
+ if strings.Contains(model.Name, "custom-voice") && strings.TrimSpace(params.Speaker) == "" {
+ return nil, invalidParams("params.speaker is required for Qwen3-TTS CustomVoice", nil)
+ }
+ if strings.Contains(model.Name, "voice-design") && strings.TrimSpace(params.Instruct) == "" {
+ return nil, invalidParams("params.instruct is required for Qwen3-TTS VoiceDesign", nil)
+ }
+ if strings.HasPrefix(model.Name, "audio8-tts:") && voiceSample != "" &&
+ strings.TrimSpace(params.Transcript) == "" {
+ return nil, invalidParams("params.transcript is required for Audio8 voice cloning", nil)
+ }
output, err := managedOutputPath("audio", requestID, params.OutputName, ".wav")
if err != nil {
return nil, err
@@ -459,6 +576,8 @@ func (h *Handler) generateSpeech(
Output: output, VoiceSample: voiceSample, Transcript: params.Transcript,
Language: params.Language, Backend: model.Backend,
VoiceConsent: params.VoiceConsent, Seed: params.Seed,
+ Speaker: strings.TrimSpace(params.Speaker),
+ Instruct: strings.TrimSpace(params.Instruct),
}, logs, logs)
if runError != nil {
_ = os.Remove(output)
diff --git a/internal/control/creator_test.go b/internal/control/creator_test.go
index 8b1d3cf..8a44149 100644
--- a/internal/control/creator_test.go
+++ b/internal/control/creator_test.go
@@ -13,6 +13,7 @@ import (
"testing"
"github.com/carlos/tapioca/internal/adapter"
+ "github.com/carlos/tapioca/internal/audioruntime"
"github.com/carlos/tapioca/internal/config"
"github.com/carlos/tapioca/internal/imageruntime"
"github.com/carlos/tapioca/internal/speechruntime"
@@ -27,6 +28,9 @@ func TestCreatorCapabilitiesReportProtocolSafeAvailability(t *testing.T) {
if capabilities["video"].(map[string]any)["available"] != true {
t.Fatal("video capability is not available")
}
+ if capabilities["audio"].(map[string]any)["available"] != true {
+ t.Fatal("audio capability is not available")
+ }
speech := capabilities["speech"].(map[string]any)
if speech["available"] != true || speech["supports_voice_reference"] != true {
t.Fatalf("speech capability = %#v", speech)
@@ -52,7 +56,7 @@ func TestCreatorCatalogContainsCompatibilityMetadata(t *testing.T) {
if len(models) == 0 {
t.Fatal("creator catalog is empty")
}
- var foundSpeech bool
+ var foundSpeech, foundAudio bool
for _, model := range models {
if model.Kind == "speech" {
foundSpeech = true
@@ -60,10 +64,19 @@ func TestCreatorCatalogContainsCompatibilityMetadata(t *testing.T) {
t.Fatalf("speech compatibility = %#v", model)
}
}
+ if model.Kind == "audio" {
+ foundAudio = true
+ if !model.Available || model.Operation != "audio.generate" || model.MaxDurationSeconds != 120 {
+ t.Fatalf("audio compatibility = %#v", model)
+ }
+ }
}
if !foundSpeech {
t.Fatal("creator catalog did not include speech compatibility records")
}
+ if !foundAudio {
+ t.Fatal("creator catalog did not include audio compatibility records")
+ }
}
func TestImageGenerateUsesRuntimeAdapterAndManagedOutput(t *testing.T) {
@@ -289,6 +302,17 @@ func TestValidateVideoUsesRuntimeFrameLimit(t *testing.T) {
}
}
+func TestValidateLTXVideoFrameShape(t *testing.T) {
+ for _, model := range []string{"ltx-video:2b-fp16", "ltx-2.5:22b-bf16-cuda"} {
+ if err := validateVideoModelFrames(model, 49); err != nil {
+ t.Fatalf("%s rejected valid frame count: %v", model, err)
+ }
+ if err := validateVideoModelFrames(model, 13); err == nil || err.Code != "invalid_params" {
+ t.Fatalf("%s accepted invalid frame count: %#v", model, err)
+ }
+ }
+}
+
func TestSpeechGenerateUsesRuntimeAdapterAndManagedOutput(t *testing.T) {
home := t.TempDir()
t.Setenv("TAPIOCA_HOME", home)
@@ -323,6 +347,40 @@ func TestSpeechGenerateUsesRuntimeAdapterAndManagedOutput(t *testing.T) {
}
}
+func TestAudioGenerateUsesRuntimeAdapterAndCatalogDefaults(t *testing.T) {
+ home := t.TempDir()
+ t.Setenv("TAPIOCA_HOME", home)
+ modelPath := filepath.Join(home, "models", "stable-audio-3-small-music")
+ if err := os.MkdirAll(modelPath, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ saveCreatorModel(t, config.Model{
+ Name: "stable-audio-3:small-music", Path: modelPath,
+ Kind: "audio", Backend: "stable-audio3",
+ })
+ var received audioruntime.Request
+ handler := NewHandler(Dependencies{
+ Audio: func(_ context.Context, _ string, request audioruntime.Request, _, _ io.Writer) error {
+ received = request
+ return os.WriteFile(request.Output, []byte("RIFF-test"), 0o600)
+ },
+ })
+ result, err := handler.Handle(context.Background(), Request{
+ ID: "audio-request", Method: "audio.generate",
+ Params: []byte(`{"model":"stable-audio-3:small-music","prompt":"warm ambient synth"}`),
+ })
+ if err != nil {
+ t.Fatalf("audio.generate error = %v", err)
+ }
+ output := result.(CreatorOutput)
+ if received.Prompt != "warm ambient synth" || received.Duration != 30 || received.Steps != 8 {
+ t.Fatalf("audio request = %#v", received)
+ }
+ if output.Kind != "audio" || !withinRoot(filepath.Join(home, "outputs", "audio"), output.Path) {
+ t.Fatalf("audio output = %#v", output)
+ }
+}
+
func TestLoRAListDiscoversOnlySafetensors(t *testing.T) {
home := t.TempDir()
t.Setenv("TAPIOCA_HOME", home)
diff --git a/internal/control/methods.go b/internal/control/methods.go
index aedcd20..3aaf14f 100644
--- a/internal/control/methods.go
+++ b/internal/control/methods.go
@@ -14,24 +14,27 @@ import (
)
type CatalogModel struct {
- Name string `json:"name"`
- Kind string `json:"kind"`
- Backend string `json:"backend"`
- Repo string `json:"repo"`
- Size string `json:"size,omitempty"`
- Memory string `json:"memory,omitempty"`
- GPU string `json:"gpu,omitempty"`
- Platforms []string `json:"platforms"`
- Languages string `json:"languages,omitempty"`
- Features string `json:"features,omitempty"`
- Width int `json:"width,omitempty"`
- Height int `json:"height,omitempty"`
- Steps int `json:"steps,omitempty"`
- Frames int `json:"frames,omitempty"`
- FPS int `json:"fps,omitempty"`
- Gated bool `json:"gated,omitempty"`
- License string `json:"license,omitempty"`
- LicenseURL string `json:"license_url,omitempty"`
+ Name string `json:"name"`
+ Kind string `json:"kind"`
+ Backend string `json:"backend"`
+ Repo string `json:"repo"`
+ Size string `json:"size,omitempty"`
+ Memory string `json:"memory,omitempty"`
+ GPU string `json:"gpu,omitempty"`
+ Platforms []string `json:"platforms"`
+ Languages string `json:"languages,omitempty"`
+ Features string `json:"features,omitempty"`
+ Width int `json:"width,omitempty"`
+ Height int `json:"height,omitempty"`
+ Steps int `json:"steps,omitempty"`
+ Frames int `json:"frames,omitempty"`
+ FPS int `json:"fps,omitempty"`
+ DurationSeconds int `json:"duration_seconds,omitempty"`
+ MaxDurationSeconds int `json:"max_duration_seconds,omitempty"`
+ SampleRate int `json:"sample_rate,omitempty"`
+ Gated bool `json:"gated,omitempty"`
+ License string `json:"license,omitempty"`
+ LicenseURL string `json:"license_url,omitempty"`
}
type InstalledModel struct {
@@ -53,6 +56,7 @@ type Dependencies struct {
Image ImageRunFunc
Video VideoRunFunc
Speech SpeechRunFunc
+ Audio AudioRunFunc
Now func() time.Time
}
@@ -95,6 +99,9 @@ func NewHandler(dependencies Dependencies) *Handler {
if dependencies.Speech == nil {
dependencies.Speech = runSpeech
}
+ if dependencies.Audio == nil {
+ dependencies.Audio = runAudio
+ }
return &Handler{
dependencies: dependencies,
cancellations: NewCancellationRegistry(),
@@ -211,6 +218,7 @@ func capabilities() map[string]any {
"image.generate",
"video.generate",
"speech.generate",
+ "audio.generate",
"voice.clone",
"lora.list",
"lora.inspect",
@@ -289,24 +297,27 @@ func loadCatalog(ctx context.Context) ([]CatalogModel, error) {
backend = "llama.cpp"
}
models = append(models, CatalogModel{
- Name: resolved.Name,
- Kind: kind,
- Backend: backend,
- Repo: resolved.Repo,
- Size: resolved.Size,
- Memory: resolved.Memory,
- GPU: resolved.GPU,
- Platforms: normalizePlatforms(resolved.Platform),
- Languages: resolved.Languages,
- Features: resolved.Features,
- Width: resolved.Width,
- Height: resolved.Height,
- Steps: resolved.Steps,
- Frames: resolved.Frames,
- FPS: resolved.FPS,
- Gated: resolved.Gated,
- License: resolved.License,
- LicenseURL: resolved.LicenseURL,
+ Name: resolved.Name,
+ Kind: kind,
+ Backend: backend,
+ Repo: resolved.Repo,
+ Size: resolved.Size,
+ Memory: resolved.Memory,
+ GPU: resolved.GPU,
+ Platforms: normalizePlatforms(resolved.Platform),
+ Languages: resolved.Languages,
+ Features: resolved.Features,
+ Width: resolved.Width,
+ Height: resolved.Height,
+ Steps: resolved.Steps,
+ Frames: resolved.Frames,
+ FPS: resolved.FPS,
+ DurationSeconds: resolved.DurationSeconds,
+ MaxDurationSeconds: resolved.MaxDuration,
+ SampleRate: resolved.SampleRate,
+ Gated: resolved.Gated,
+ License: resolved.License,
+ LicenseURL: resolved.LicenseURL,
})
}
return models, nil
diff --git a/internal/control/server.go b/internal/control/server.go
index 9f83442..9ed8379 100644
--- a/internal/control/server.go
+++ b/internal/control/server.go
@@ -224,7 +224,7 @@ func (s *Server) markRequestSeen(id string) bool {
func isMutating(method string) bool {
switch method {
case "model.pull", "model.remove", "server.start", "server.stop",
- "image.generate", "video.generate", "speech.generate", "voice.clone":
+ "image.generate", "video.generate", "speech.generate", "audio.generate", "voice.clone":
return true
default:
return false
diff --git a/internal/imageruntime/image_diffusion.py b/internal/imageruntime/image_diffusion.py
index 252792c..cec784f 100644
--- a/internal/imageruntime/image_diffusion.py
+++ b/internal/imageruntime/image_diffusion.py
@@ -39,6 +39,7 @@ def main():
"PyTorch build; AMD and CPU diffusion backends are not supported yet."
)
is_qwen_image = "qwen-image" in args.model.lower()
+ is_flux2_klein = "flux2-klein" in args.model.lower()
if is_qwen_image and not use_mps and not torch.cuda.is_bf16_supported():
raise SystemExit(
"Qwen-Image-Flash requires a CUDA GPU with bfloat16 support "
@@ -58,7 +59,7 @@ def main():
)
dtype = (
torch.bfloat16
- if (is_qwen_image or "krea-2" in args.model.lower())
+ if (is_qwen_image or is_flux2_klein or "krea-2" in args.model.lower())
else torch.float16
)
load_options = {
diff --git a/internal/imageruntime/requirements.txt b/internal/imageruntime/requirements.txt
index 3b53d7b..fec5623 100644
--- a/internal/imageruntime/requirements.txt
+++ b/internal/imageruntime/requirements.txt
@@ -1,5 +1,5 @@
diffusers==0.40.0
-transformers>=4.51
-accelerate>=1.6
-safetensors>=0.5
-sentencepiece>=0.2
+transformers==5.15.1
+accelerate==1.14.0
+safetensors==0.8.0
+sentencepiece==0.2.2
diff --git a/internal/speechruntime/requirements-audio8-torch.txt b/internal/speechruntime/requirements-audio8-torch.txt
new file mode 100644
index 0000000..e9b8efd
--- /dev/null
+++ b/internal/speechruntime/requirements-audio8-torch.txt
@@ -0,0 +1,5 @@
+torch>=2.5,<2.12
+torchaudio>=2.5,<2.12
+transformers==4.57.6
+soundfile>=0.12
+safetensors>=0.4
diff --git a/internal/speechruntime/requirements-mlx.txt b/internal/speechruntime/requirements-mlx.txt
index 3b4e3f3..edb6680 100644
--- a/internal/speechruntime/requirements-mlx.txt
+++ b/internal/speechruntime/requirements-mlx.txt
@@ -1 +1 @@
-mlx-audio>=0.3.0
+mlx-audio==0.5.0
diff --git a/internal/speechruntime/requirements-qwen.txt b/internal/speechruntime/requirements-qwen.txt
index 8a6685e..862ea74 100644
--- a/internal/speechruntime/requirements-qwen.txt
+++ b/internal/speechruntime/requirements-qwen.txt
@@ -1,2 +1,2 @@
-qwen-tts
-soundfile
+qwen-tts==0.1.1
+soundfile>=0.12
diff --git a/internal/speechruntime/runtime.go b/internal/speechruntime/runtime.go
index 5928c35..d654afa 100644
--- a/internal/speechruntime/runtime.go
+++ b/internal/speechruntime/runtime.go
@@ -25,6 +25,8 @@ type Request struct {
VoiceSample string
Transcript string
Language string
+ Speaker string
+ Instruct string
Backend string
VoiceConsent bool
Seed uint64
@@ -74,6 +76,13 @@ func RunWithWriters(
return errors.New("the MLX speech backend requires macOS on Apple Silicon")
}
flavor = "mlx"
+ case "speech-audio8":
+ flavor = "audio8-torch"
+ case "speech-audio8-mlx":
+ if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" {
+ return errors.New("the Audio8 MLX speech backend requires macOS on Apple Silicon")
+ }
+ flavor = "mlx"
default:
return fmt.Errorf("unsupported speech backend %q", request.Backend)
}
@@ -88,7 +97,7 @@ func runPython(
stdout io.Writer,
stderr io.Writer,
) error {
- root := filepath.Join(cacheDir, "speech-runtime", "0.1.3-"+flavor)
+ root := filepath.Join(cacheDir, "speech-runtime", "0.3.0-"+flavor)
requirementsName := "requirements-" + flavor + ".txt"
names := []string{"speech.py", requirementsName, "cpu_speech.py", "pocket_qualification.py", "sopro_qualification.py"}
if flavor == "audio8" {
@@ -178,5 +187,11 @@ func pythonArguments(root string, request Request) []string {
args = append(args, "--voice-consent")
}
args = append(args, "--seed", fmt.Sprint(request.Seed))
+ if request.Speaker != "" {
+ args = append(args, "--speaker", request.Speaker)
+ }
+ if request.Instruct != "" {
+ args = append(args, "--instruct", request.Instruct)
+ }
return args
}
diff --git a/internal/speechruntime/runtime_test.go b/internal/speechruntime/runtime_test.go
index fa6ce81..4f5bd56 100644
--- a/internal/speechruntime/runtime_test.go
+++ b/internal/speechruntime/runtime_test.go
@@ -14,12 +14,14 @@ func TestPythonArguments(t *testing.T) {
Text: "hello", Output: "/tmp/hello.wav", VoiceSample: "/tmp/voice.wav",
Transcript: "reference words", Language: "English", Backend: "speech-qwen-mlx",
VoiceConsent: true, Seed: 42,
+ Speaker: "Ryan", Instruct: "speak warmly",
})
for _, value := range []string{
"--model", "/models/qwen", "--model-name", "qwen3-tts:0.6b-mlx",
"--text", "hello", "--output", "/tmp/hello.wav", "--voice-sample",
"/tmp/voice.wav", "--transcript", "reference words", "--language", "English",
"--voice-consent", "--seed", "42",
+ "--speaker", "Ryan", "--instruct", "speak warmly",
} {
if !slices.Contains(args, value) {
t.Fatalf("arguments missing %q: %#v", value, args)
diff --git a/internal/speechruntime/speech.py b/internal/speechruntime/speech.py
index 91763aa..8ef1a56 100644
--- a/internal/speechruntime/speech.py
+++ b/internal/speechruntime/speech.py
@@ -133,10 +133,6 @@ def qwen(args):
import torch
from qwen_tts import Qwen3TTSModel
- if not args.voice_sample:
- raise SystemExit(
- "qwen3-tts Base requires --voice-sample or --voice for voice cloning"
- )
device = "cuda:0" if torch.cuda.is_available() else "cpu"
dtype = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float32
model = Qwen3TTSModel.from_pretrained(
@@ -146,21 +142,123 @@ def qwen(args):
attn_implementation="sdpa",
local_files_only=True,
)
- wavs, sample_rate = model.generate_voice_clone(
- text=args.text,
- language=args.language or "Auto",
- ref_audio=args.voice_sample,
- ref_text=args.transcript or None,
- x_vector_only_mode=not bool(args.transcript),
- )
+ variant = args.model_name.lower()
+ common = {"text": args.text, "language": args.language or "Auto"}
+ if "custom-voice" in variant:
+ if not args.speaker:
+ raise SystemExit("Qwen3-TTS CustomVoice requires --speaker")
+ wavs, sample_rate = model.generate_custom_voice(
+ **common, speaker=args.speaker, instruct=args.instruct or None
+ )
+ elif "voice-design" in variant:
+ if not args.instruct:
+ raise SystemExit("Qwen3-TTS VoiceDesign requires --instruct")
+ wavs, sample_rate = model.generate_voice_design(
+ **common, instruct=args.instruct
+ )
+ else:
+ if not args.voice_sample:
+ raise SystemExit(
+ "Qwen3-TTS Base requires --voice-sample or --voice for voice cloning"
+ )
+ wavs, sample_rate = model.generate_voice_clone(
+ **common,
+ ref_audio=args.voice_sample,
+ ref_text=args.transcript or None,
+ x_vector_only_mode=not bool(args.transcript),
+ )
sf.write(args.output, wavs[0], sample_rate)
def qwen_mlx(args):
- if not args.voice_sample:
- raise SystemExit(
- "qwen3-tts Base requires --voice-sample or --voice for voice cloning"
+ from mlx_audio.tts.generate import generate_audio
+ from mlx_audio.tts.utils import load_model
+
+ model = load_model(args.model)
+ with tempfile.TemporaryDirectory(prefix="tapioca-speech-") as output_dir:
+ options = dict(
+ model=model,
+ text=args.text,
+ output_path=output_dir,
+ file_prefix="speech",
+ audio_format="wav",
+ join_audio=True,
+ lang_code=args.language or "auto",
+ play=False,
+ verbose=False,
+ )
+ variant = args.model_name.lower()
+ if "custom-voice" in variant:
+ if not args.speaker:
+ raise SystemExit("Qwen3-TTS CustomVoice requires --speaker")
+ options.update(voice=args.speaker, instruct=args.instruct or None)
+ elif "voice-design" in variant:
+ if not args.instruct:
+ raise SystemExit("Qwen3-TTS VoiceDesign requires --instruct")
+ options.update(voice=None, instruct=args.instruct)
+ else:
+ if not args.voice_sample:
+ raise SystemExit(
+ "Qwen3-TTS Base requires --voice-sample or --voice for voice cloning"
+ )
+ options.update(
+ voice=None,
+ ref_audio=args.voice_sample,
+ ref_text=args.transcript or None,
+ )
+ generate_audio(**options)
+ candidates = glob.glob(os.path.join(output_dir, "*.wav"))
+ if not candidates:
+ raise SystemExit("MLX Audio did not produce a WAV file")
+ shutil.move(candidates[0], args.output)
+
+
+def audio8(args):
+ import soundfile as sf
+ import torch
+ from transformers import AutoModel, AutoProcessor
+
+ if args.voice_sample and not args.transcript:
+ raise SystemExit("Audio8 voice cloning requires --transcript")
+ device = "cuda" if torch.cuda.is_available() else "cpu"
+ dtype = torch.bfloat16 if device == "cuda" else torch.float32
+ processor = AutoProcessor.from_pretrained(
+ args.model, trust_remote_code=True, local_files_only=True
+ )
+ model = AutoModel.from_pretrained(
+ args.model,
+ trust_remote_code=True,
+ local_files_only=True,
+ dtype=dtype,
+ ).eval().to(device)
+ processor_options = {"text": [args.text], "return_tensors": "pt"}
+ if args.voice_sample:
+ processor_options.update(
+ reference_audio=[args.voice_sample], reference_text=[args.transcript]
)
+ inputs = processor(**processor_options)
+ inputs = {
+ name: value.to(device) if hasattr(value, "to") else value
+ for name, value in inputs.items()
+ }
+ with torch.inference_mode():
+ generated = model.generate(
+ **inputs,
+ max_new_tokens=1024,
+ temperature=0.8,
+ top_p=0.95,
+ top_k=50,
+ do_sample=True,
+ return_dict_in_generate=True,
+ )
+ waveforms, lengths = model.decode_audio(generated.codes)
+ audio = waveforms[0, : int(lengths[0])].float().cpu().numpy()
+ sf.write(args.output, audio, model.config.codec_sample_rate)
+
+
+def audio8_mlx(args):
+ if args.voice_sample and not args.transcript:
+ raise SystemExit("Audio8 voice cloning requires --transcript")
from mlx_audio.tts.generate import generate_audio
from mlx_audio.tts.utils import load_model
@@ -169,7 +267,8 @@ def qwen_mlx(args):
generate_audio(
model=model,
text=args.text,
- ref_audio=args.voice_sample,
+ voice=None,
+ ref_audio=args.voice_sample or None,
ref_text=args.transcript or None,
output_path=output_dir,
file_prefix="speech",
@@ -196,6 +295,8 @@ def main():
parser.add_argument("--language", default="")
parser.add_argument("--voice-consent", action="store_true")
parser.add_argument("--seed", type=int, default=0)
+ parser.add_argument("--speaker", default="")
+ parser.add_argument("--instruct", default="")
args = parser.parse_args()
if os.path.splitext(args.output)[1].lower() != ".wav":
@@ -216,6 +317,10 @@ def main():
elif args.backend == "speech-sopro":
from sopro_qualification import run
run(args)
+ elif args.backend == "speech-audio8":
+ audio8(args)
+ elif args.backend == "speech-audio8-mlx":
+ audio8_mlx(args)
else:
raise SystemExit(f"unsupported speech backend: {args.backend}")
diff --git a/internal/videoruntime/diffusers_video.py b/internal/videoruntime/diffusers_video.py
index bfc3f49..a87b89b 100644
--- a/internal/videoruntime/diffusers_video.py
+++ b/internal/videoruntime/diffusers_video.py
@@ -1,10 +1,11 @@
import argparse
import inspect
import os
+import sys
import torch
from diffusers import DiffusionPipeline, StableVideoDiffusionPipeline
-from diffusers.utils import export_to_video, load_image
+from diffusers.utils import encode_video, export_to_video, load_image
def main():
@@ -28,7 +29,15 @@ def main():
raise RuntimeError("a CUDA-capable NVIDIA GPU and driver are required")
is_svd = "stable-video-diffusion" in args.model.lower()
- pipeline_class = StableVideoDiffusionPipeline if is_svd else DiffusionPipeline
+ is_ltx2 = "ltx-2.5" in args.model.lower()
+ if is_ltx2 and sys.version_info < (3, 12):
+ raise SystemExit("LTX-2.5 requires Python 3.12 or newer")
+ if is_ltx2:
+ from diffusers import LTX2ImageToVideoPipeline, LTX2Pipeline
+
+ pipeline_class = LTX2ImageToVideoPipeline if args.image else LTX2Pipeline
+ else:
+ pipeline_class = StableVideoDiffusionPipeline if is_svd else DiffusionPipeline
load_options = {
"torch_dtype": (
torch.float16
@@ -72,13 +81,37 @@ def main():
"decode_chunk_size": 2,
"generator": torch.Generator().manual_seed(args.seed),
}
+ if is_ltx2:
+ call_args.update(
+ frame_rate=float(args.fps),
+ guidance_scale=3.0,
+ stg_scale=1.0,
+ modality_scale=3.0,
+ guidance_rescale=0.7,
+ audio_guidance_scale=7.0,
+ audio_stg_scale=1.0,
+ audio_modality_scale=3.0,
+ audio_guidance_rescale=0.7,
+ output_type="np",
+ return_dict=False,
+ )
if args.image:
image = load_image(args.image)
call_args["image"] = image.resize((args.width, args.height))
accepted = inspect.signature(pipe.__call__).parameters
call_args = {key: value for key, value in call_args.items() if key in accepted}
- frames = pipe(**call_args).frames[0]
- export_to_video(frames, args.output, fps=args.fps)
+ result = pipe(**call_args)
+ if is_ltx2:
+ video, audio = result
+ encode_video(
+ video[0],
+ fps=args.fps,
+ audio=audio[0].float().cpu(),
+ audio_sample_rate=pipe.vocoder.config.output_sampling_rate,
+ output_path=args.output,
+ )
+ else:
+ export_to_video(result.frames[0], args.output, fps=args.fps)
if __name__ == "__main__":
diff --git a/internal/videoruntime/engine.go b/internal/videoruntime/engine.go
index d88af4e..bc447ec 100644
--- a/internal/videoruntime/engine.go
+++ b/internal/videoruntime/engine.go
@@ -30,7 +30,7 @@ func engineFor(backend string) (Engine, error) {
return engineFunc{name: "mlx", run: func(ctx context.Context, cache string, request Request) error {
return runPython(ctx, cache, request, "mlx")
}}, nil
- case "diffusers-video":
+ case "diffusers-video", "diffusers-ltx2":
return engineFunc{name: "diffusers", run: func(ctx context.Context, cache string, request Request) error {
return runPython(ctx, cache, request, "diffusers")
}}, nil
diff --git a/internal/videoruntime/requirements-diffusers.txt b/internal/videoruntime/requirements-diffusers.txt
index 9bd89be..3d06860 100644
--- a/internal/videoruntime/requirements-diffusers.txt
+++ b/internal/videoruntime/requirements-diffusers.txt
@@ -1,7 +1,8 @@
diffusers==0.40.0
-transformers>=4.51
-accelerate>=1.6
-safetensors>=0.5
-sentencepiece>=0.2
+transformers==5.15.1
+accelerate==1.14.0
+safetensors==0.8.0
+sentencepiece==0.2.2
imageio>=2.37
imageio-ffmpeg>=0.6
+av>=14
diff --git a/internal/videoruntime/runtime.go b/internal/videoruntime/runtime.go
index fefb773..c5b8367 100644
--- a/internal/videoruntime/runtime.go
+++ b/internal/videoruntime/runtime.go
@@ -51,7 +51,7 @@ func Run(ctx context.Context, cacheDir string, request Request) error {
if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" {
return errors.New("the MLX video backend requires macOS on Apple Silicon")
}
- case "diffusers-video":
+ case "diffusers-video", "diffusers-ltx2":
if (runtime.GOOS != "windows" && runtime.GOOS != "linux") || runtime.GOARCH != "amd64" {
return errors.New("the CUDA video backend requires Windows or Linux x64 with an NVIDIA GPU")
}
@@ -624,12 +624,7 @@ func waitForServer(ctx context.Context, base string, timeout time.Duration) erro
}
func runPython(ctx context.Context, cacheDir string, request Request, flavor string) error {
- version := "0.1.1"
- if flavor == "diffusers" {
- // Do not reuse an environment with the vulnerable pre-0.40 shard loader.
- version = "0.1.2-diffusers040"
- }
- root := filepath.Join(cacheDir, "video-runtime", version+"-"+flavor)
+ root := filepath.Join(cacheDir, "video-runtime", "0.2.0-"+flavor)
script := flavor + "_video.py"
requirements := "requirements-" + flavor + ".txt"
for _, name := range []string{script, requirements} {
diff --git a/internal/videoruntime/runtime_test.go b/internal/videoruntime/runtime_test.go
index 90c55eb..29bdc5a 100644
--- a/internal/videoruntime/runtime_test.go
+++ b/internal/videoruntime/runtime_test.go
@@ -24,6 +24,20 @@ func TestDiffusersVideoUsesFixedShardLoader(t *testing.T) {
}
}
+func TestLTX2SelectsImageConditionedPipeline(t *testing.T) {
+ script, err := source.ReadFile("diffusers_video.py")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, required := range []string{
+ "LTX2ImageToVideoPipeline", "if args.image else LTX2Pipeline",
+ } {
+ if !strings.Contains(string(script), required) {
+ t.Fatalf("LTX-2.5 image conditioning is not wired through: %s", script)
+ }
+ }
+}
+
func TestH3GraphChainsAdaptersIntoSchedulerAndGuider(t *testing.T) {
python, err := exec.LookPath("python3")
if err != nil {
@@ -172,7 +186,8 @@ func TestExtractZipRejectsParentTraversal(t *testing.T) {
func TestEngineForKeepsBackendDetailsBehindStableBoundary(t *testing.T) {
tests := map[string]string{
"mlx-video": "mlx", "diffusers-video": "diffusers",
- "comfy-h3-mps": "comfy-h3", "comfy-h3-cuda": "comfy-h3",
+ "diffusers-ltx2": "diffusers",
+ "comfy-h3-mps": "comfy-h3", "comfy-h3-cuda": "comfy-h3",
}
for backend, want := range tests {
engine, err := engineFor(backend)
diff --git a/scripts/validate-windows-model-upgrade.ps1 b/scripts/validate-windows-model-upgrade.ps1
new file mode 100644
index 0000000..3d4843f
--- /dev/null
+++ b/scripts/validate-windows-model-upgrade.ps1
@@ -0,0 +1,104 @@
+[CmdletBinding()]
+param(
+ [ValidateSet("Catalog", "Smoke", "Heavy")]
+ [string]$Tier = "Catalog",
+ [string]$Tapioca = "tapioca",
+ [string]$OutputDirectory = (Join-Path $PWD "tapioca-model-validation"),
+ [switch]$AcceptGatedLicenses
+)
+
+$ErrorActionPreference = "Stop"
+Set-StrictMode -Version Latest
+
+function Invoke-Tapioca {
+ param([Parameter(Mandatory)][string[]]$Arguments)
+ Write-Host "tapioca $($Arguments -join ' ')" -ForegroundColor Cyan
+ & $Tapioca @Arguments
+ if ($LASTEXITCODE -ne 0) {
+ throw "Tapioca exited with code $LASTEXITCODE"
+ }
+}
+
+function Invoke-GatedPull {
+ param([Parameter(Mandatory)][string]$Model)
+ if (-not $AcceptGatedLicenses) {
+ Write-Warning "Skipping gated model $Model. Re-run with -AcceptGatedLicenses after reviewing its terms and setting HF_TOKEN."
+ return $false
+ }
+ if ([string]::IsNullOrWhiteSpace($env:HF_TOKEN) -and
+ [string]::IsNullOrWhiteSpace($env:HUGGING_FACE_HUB_TOKEN)) {
+ throw "HF_TOKEN or HUGGING_FACE_HUB_TOKEN is required for gated validation"
+ }
+ Invoke-Tapioca -Arguments @("pull", $Model, "--accept-license")
+ return $true
+}
+
+Write-Host "Tapioca Windows/NVIDIA model validation" -ForegroundColor Green
+& nvidia-smi
+if ($LASTEXITCODE -ne 0) {
+ throw "nvidia-smi failed; install or update the NVIDIA driver first"
+}
+Invoke-Tapioca -Arguments @("version")
+Invoke-Tapioca -Arguments @("catalog")
+
+if ($Tier -eq "Catalog") {
+ Write-Host "Catalog validation complete. Use -Tier Smoke or -Tier Heavy to download and run models." -ForegroundColor Green
+ exit 0
+}
+
+New-Item -ItemType Directory -Force -Path $OutputDirectory | Out-Null
+
+# P0: current llama.cpp plus compact current-generation text model.
+Invoke-Tapioca -Arguments @("pull", "gemma4:e2b-q4_0")
+"Reply with only: GEMMA4_OK" | & $Tapioca run "gemma4:e2b-q4_0"
+if ($LASTEXITCODE -ne 0) { throw "Gemma 4 inference smoke test failed" }
+
+# P1: current CUDA image and voice runtimes.
+Invoke-Tapioca -Arguments @(
+ "image", "flux2-klein:4b-fp8-cuda",
+ "--prompt", "A single pearl on black velvet, studio product photograph",
+ "--width", "512", "--height", "512", "--steps", "4",
+ "--output", (Join-Path $OutputDirectory "flux2-klein-fp8.png")
+)
+Invoke-Tapioca -Arguments @(
+ "tts", "audio8-tts:0.6b", "--text", "Audio eight validation succeeded.",
+ "--output", (Join-Path $OutputDirectory "audio8.wav")
+)
+Invoke-Tapioca -Arguments @(
+ "tts", "qwen3-tts:1.7b-custom-voice", "--speaker", "Ryan",
+ "--instruct", "Speak clearly and calmly", "--text", "Qwen custom voice validation succeeded.",
+ "--output", (Join-Path $OutputDirectory "qwen3-custom-voice.wav")
+)
+
+if ($Tier -eq "Smoke") {
+ Write-Host "Smoke validation complete: $OutputDirectory" -ForegroundColor Green
+ exit 0
+}
+
+# P0 heavyweight text profile.
+Invoke-Tapioca -Arguments @("pull", "qwen3.6:27b-q4_k_m")
+"Reply with only: QWEN36_OK" | & $Tapioca run "qwen3.6:27b-q4_k_m"
+if ($LASTEXITCODE -ne 0) { throw "Qwen 3.6 inference validation failed" }
+
+# P2 standalone audio. Stable Audio 3 is gated.
+if (Invoke-GatedPull "stable-audio-3:small-music") {
+ Invoke-Tapioca -Arguments @(
+ "audio", "stable-audio-3:small-music",
+ "--prompt", "Soft analog synthesizer chord, no vocals",
+ "--seconds", "5", "--steps", "4",
+ "--output", (Join-Path $OutputDirectory "stable-audio-3.wav")
+ )
+}
+
+# P2 synchronized video/audio. LTX-2.5 is gated and intentionally last because
+# its curated snapshot is approximately 68 GiB.
+if (Invoke-GatedPull "ltx-2.5:22b-bf16-cuda") {
+ Invoke-Tapioca -Arguments @(
+ "video", "ltx-2.5:22b-bf16-cuda",
+ "--prompt", "Rain falling on green leaves with quiet natural ambience",
+ "--width", "768", "--height", "512", "--frames", "9", "--fps", "24", "--steps", "4",
+ "--output", (Join-Path $OutputDirectory "ltx-2.5.mp4")
+ )
+}
+
+Write-Host "Heavy validation complete: $OutputDirectory" -ForegroundColor Green