diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e62bcea..e931a7e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -31,6 +31,8 @@ jobs: go-version-file: go.mod - run: go vet ./... - run: go test ./... + - name: Test Sopro candidate validation + run: python3 -m unittest discover -s internal/speechruntime -p 'test_sopro_qualification.py' - name: Verify published catalog manifest run: | go run ./cmd/catalog-manifest validate catalog/catalog.json diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index aa43664..bb83275 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -13,6 +13,10 @@ The Audio8 CPU speech adapter includes Apache-2.0 source from are retained in `internal/speechruntime/arktts_runtime/` and embedded in the managed runtime. The separate Pocket TTS runtime is MIT licensed; its gated weights are CC-BY-4.0 and require Kyutai attribution and accepted access terms. +The experimental Sopro CPU speech adapter installs `sopro==2.2.0` on demand; +its runtime and separately downloaded model weights are Apache-2.0 licensed. +Voice references must be used with permission. The public catalog does not yet +activate this candidate. Release bundles include a pinned `llama.cpp` runtime for local text-model inference. `llama.cpp` is licensed under the MIT License: diff --git a/catalog/candidates/sopro-v2-turbo.json b/catalog/candidates/sopro-v2-turbo.json new file mode 100644 index 0000000..2bf696a --- /dev/null +++ b/catalog/candidates/sopro-v2-turbo.json @@ -0,0 +1,31 @@ +{ + "schema": 1, + "generated_at": "2026-09-18T22:52:00Z", + "models": { + "sopro-v2-turbo": { + "name": "sopro-v2-turbo", + "repo": "samuel-vitorino/sopro-v2-turbo", + "files": {"cpu-fp32": ""}, + "kind": "speech", + "backends": {"cpu-fp32": "speech-sopro"}, + "default": "cpu-fp32", + "sizes": {"cpu-fp32": "~605 MiB"}, + "memory": {"cpu-fp32": "unmeasured; Mac CPU qualification only"}, + "gpus": {"cpu-fp32": "CPU on Apple Silicon macOS; Windows/Linux not qualified"}, + "languages": {"cpu-fp32": "English, European Portuguese, French, German"}, + "features": {"cpu-fp32": "experimental local voice cloning with explicit reference consent; 24 kHz WAV"}, + "license": "Apache-2.0", + "license_url": "https://huggingface.co/samuel-vitorino/sopro-v2-turbo", + "artifacts": { + "cpu-fp32": [ + {"repo": "samuel-vitorino/sopro-v2-turbo", "filename": "config.json", "target": "config.json", "revision": "f747f9edfb7b0233a3b7105af3a75603a7213d26", "sha256": "b8593ff96b2976215ff64e58e784f4d667f4f0f205ae8686b57e78c10f42d6d7", "size_bytes": 4482}, + {"repo": "samuel-vitorino/sopro-v2-turbo", "filename": "model.safetensors", "target": "model.safetensors", "revision": "f747f9edfb7b0233a3b7105af3a75603a7213d26", "sha256": "5ddc2e905ce68e015b3760f9437969f6fd7ef0af601b8e457fc18344bb908ffa", "size_bytes": 486332212}, + {"repo": "samuel-vitorino/sopro-v2-turbo", "filename": "semantic_encoder.safetensors", "target": "semantic_encoder.safetensors", "revision": "f747f9edfb7b0233a3b7105af3a75603a7213d26", "sha256": "d1d1bccdfffdcd6d01d802a3e5dbefa0674c660f5022d87ec14e3271596c5c5f", "size_bytes": 82497708}, + {"repo": "samuel-vitorino/sopro-v2-turbo", "filename": "speaker_encoder.safetensors", "target": "speaker_encoder.safetensors", "revision": "f747f9edfb7b0233a3b7105af3a75603a7213d26", "sha256": "67422a45d801542ee0aa55ce824f570ceb1c6b2283b4e74978c1b4673b49acd7", "size_bytes": 10784100}, + {"repo": "samuel-vitorino/sopro-v2-turbo", "filename": "tokenizer.model", "target": "tokenizer.model", "revision": "f747f9edfb7b0233a3b7105af3a75603a7213d26", "sha256": "2d76e7a4e8dbd0a4d2137c13200ac9710ad3fd93fad20a61f71102ab7bae754d", "size_bytes": 370821}, + {"repo": "samuel-vitorino/sopro-v2-turbo", "filename": "vocoder.safetensors", "target": "vocoder.safetensors", "revision": "f747f9edfb7b0233a3b7105af3a75603a7213d26", "sha256": "07e5561491cce41f7f90cfdb94b2ff263ff5742c3d89339db99b17ad82cc3f44", "size_bytes": 54348240} + ] + } + } + } +} diff --git a/docs/guides/model-integration-qualification.md b/docs/guides/model-integration-qualification.md index b89c5c7..88a027e 100644 --- a/docs/guides/model-integration-qualification.md +++ b/docs/guides/model-integration-qualification.md @@ -1,6 +1,6 @@ # Model integration qualification -Research/validation: 2026-09-05. Addresses [#30](https://github.com/kyndlo/tapioca/issues/30), [#31](https://github.com/kyndlo/tapioca/issues/31), and [#32](https://github.com/kyndlo/tapioca/issues/32). Issue #29 stays historical. Daily discovery remains local Codex scheduling, not GitHub cron. +Research/validation: 2026-09-05 and 2026-09-18. Addresses [#30](https://github.com/kyndlo/tapioca/issues/30), [#31](https://github.com/kyndlo/tapioca/issues/31), [#32](https://github.com/kyndlo/tapioca/issues/32), and [#36](https://github.com/kyndlo/tapioca/issues/36). Issue #29 stays historical. Daily discovery remains local Codex scheduling, not GitHub cron. ## Release scope and downloads @@ -49,6 +49,36 @@ Runner rewrites the pinned English configuration to local paths, sets offline fl Sources: [runtime](https://github.com/kyutai-labs/pocket-tts/tree/7809e76aa94d5e841a006a27c1b31f07c60d22d4), [release](https://kyutai.org/blog/2026-01-13-pocket-tts), [non-authoritative CPU discussion](https://www.reddit.com/r/LocalLLaMA/comments/1vzyuyv/local_tts_models_not_sure_if_this_is_the_place/). +## Sopro V2 Turbo — #36 + +Research and Apple Silicon CPU smoke date: 2026-09-18. The public, ungated +Apache-2.0 checkpoint is pinned to `f747f9edfb7b0233a3b7105af3a75603a7213d26`. +All six artifacts (~605 MiB total) were downloaded and matched the SHA-256 +values in `catalog/candidates/sopro-v2-turbo.json`. The isolated +`speech-sopro` candidate uses `sopro==2.2.0`, CPU inference, offline local paths, +incremental 24 kHz mono PCM16 WAV writing, an explicit 5–20-second reference, +and `--voice-consent`. It is **not in the public picker** and currently gates to +Apple Silicon macOS; Windows/Linux have not been qualified. + +The official runtime generated a 3.23-second valid WAV from a locally +synthesized 11.99-second reference in 1.67 seconds total on this Mac. The +Tapioca CLI pulled/verified the cached bundle and generated a valid 3.10-second +WAV with 1.13 seconds to first audio and 1.78 seconds of model load/inference +after dependency installation. The first run separately created its Python +environment. These are one short fixture, not a throughput or voice-quality +guarantee. An opt-in Tapioca hardware test exercises the packaged speech +adapter with a local model and reference. Remaining: assess intelligibility +and voice similarity in all four languages, reference formats/quality, +long-text chunk ordering, cancellation, +peak memory, and Windows/Linux CPU behavior before public catalog activation. +Users should clone only voices they have permission to use; the model does not +watermark output. + +Sources: [official model card](https://huggingface.co/samuel-vitorino/sopro-v2-turbo), +[official runtime at the pinned commit](https://github.com/samuel-vitorino/sopro/tree/7bfcf9a0539d274f6593959a21da948f506ceee1), +[PyPI 2.2.0](https://pypi.org/project/sopro/2.2.0/), and +[non-authoritative r/LocalLLaMA discussion](https://www.reddit.com/r/LocalLLaMA/comments/1vzvtgv/sopro_v2_sotalevel_voice_cloning_tts_at_120m/). + ## Repeat validation `powershell -NoProfile -File scripts/test-windows.ps1` records hardware/commands in a unique temporary directory and runs affected checks from a checkout. Executed successfully on the authorized Windows PC: 64 GB, RTX 4070 Ti SUPER, Python 3.12.10, Node 22.20.0, Go 1.26.1. Inference/visual QA remain separate from the script. diff --git a/internal/catalog/catalog.go b/internal/catalog/catalog.go index 52ac00f..ff8d422 100644 --- a/internal/catalog/catalog.go +++ b/internal/catalog/catalog.go @@ -322,7 +322,7 @@ func validateModel(name string, model Model) error { "diffusers-video": true, "onnx-directml": true, "onnx-cpu": true, "comfy-h3-mps": true, "comfy-h3-cuda": true, "speech-chatterbox": true, "speech-qwen": true, "speech-qwen-mlx": true, - "speech-audio8-onnx": true, "speech-pocket-tts": true, + "speech-audio8-onnx": true, "speech-pocket-tts": true, "speech-sopro": true, } for variant := range model.Files { if !safeName.MatchString(variant) { @@ -1072,6 +1072,8 @@ func ResolveForPlatform(ref, goos, goarch string) (Resolved, error) { platform = "Windows, macOS, Linux" case "speech-audio8-onnx", "speech-pocket-tts": platform = "macOS Apple Silicon; Windows/Linux x64 CPU" + case "speech-sopro": + platform = "macOS Apple Silicon CPU (experimental)" case "speech-qwen": platform = "Windows/Linux NVIDIA or CPU" case "onnx-directml": diff --git a/internal/catalog/sopro_candidate_test.go b/internal/catalog/sopro_candidate_test.go new file mode 100644 index 0000000..56550e0 --- /dev/null +++ b/internal/catalog/sopro_candidate_test.go @@ -0,0 +1,31 @@ +package catalog + +import ( + "os" + "testing" +) + +func TestSoproCandidateManifest(t *testing.T) { + data, err := os.ReadFile("../../catalog/candidates/sopro-v2-turbo.json") + if err != nil { + t.Fatal(err) + } + manifest, err := decodeManifest(data) + if err != nil { + t.Fatal(err) + } + model := manifest.Models["sopro-v2-turbo"] + if model.Repo != "samuel-vitorino/sopro-v2-turbo" || model.Backends[model.Default] != "speech-sopro" { + t.Fatalf("incorrect Sopro candidate: %#v", model) + } + artifacts := model.Artifacts[model.Default] + if len(artifacts) != 6 { + t.Fatalf("want six pinned Sopro artifacts, got %d", len(artifacts)) + } + for _, artifact := range artifacts { + if artifact.Revision != "f747f9edfb7b0233a3b7105af3a75603a7213d26" || + artifact.SizeBytes <= 0 || len(artifact.SHA256) != 64 { + t.Fatalf("artifact is not pinned: %#v", artifact) + } + } +} diff --git a/internal/speechruntime/requirements-sopro-qualification.txt b/internal/speechruntime/requirements-sopro-qualification.txt new file mode 100644 index 0000000..96b9db4 --- /dev/null +++ b/internal/speechruntime/requirements-sopro-qualification.txt @@ -0,0 +1,3 @@ +sopro==2.2.0 +torch==2.14.0 +torchaudio==2.11.0 diff --git a/internal/speechruntime/runtime.go b/internal/speechruntime/runtime.go index a3ff3a9..5928c35 100644 --- a/internal/speechruntime/runtime.go +++ b/internal/speechruntime/runtime.go @@ -14,7 +14,7 @@ import ( "github.com/carlos/tapioca/internal/pythonruntime" ) -//go:embed speech.py cpu_speech.py pocket_qualification.py arktts_runtime requirements-*.txt +//go:embed speech.py cpu_speech.py pocket_qualification.py sopro_qualification.py arktts_runtime requirements-*.txt var source embed.FS type Request struct { @@ -43,6 +43,14 @@ func RunWithWriters( ) error { flavor := "" switch request.Backend { + case "speech-sopro": + if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" { + return errors.New("experimental Sopro CPU speech currently requires Apple Silicon macOS") + } + if request.VoiceSample == "" || !request.VoiceConsent { + return errors.New("Sopro requires a reference recording and explicit voice-use permission") + } + flavor = "sopro-qualification" case "speech-audio8-onnx", "speech-pocket-tts": if runtime.GOARCH != "amd64" && !(runtime.GOOS == "darwin" && runtime.GOARCH == "arm64") { return errors.New("this CPU speech backend requires x64 Windows/Linux or Apple Silicon macOS") @@ -82,7 +90,7 @@ func runPython( ) error { root := filepath.Join(cacheDir, "speech-runtime", "0.1.3-"+flavor) requirementsName := "requirements-" + flavor + ".txt" - names := []string{"speech.py", requirementsName, "cpu_speech.py", "pocket_qualification.py"} + names := []string{"speech.py", requirementsName, "cpu_speech.py", "pocket_qualification.py", "sopro_qualification.py"} if flavor == "audio8" { entries, err := source.ReadDir("arktts_runtime") if err != nil { diff --git a/internal/speechruntime/sopro_integration_test.go b/internal/speechruntime/sopro_integration_test.go new file mode 100644 index 0000000..8b47b09 --- /dev/null +++ b/internal/speechruntime/sopro_integration_test.go @@ -0,0 +1,57 @@ +package speechruntime + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "os" + "path/filepath" + "runtime" + "testing" + "time" +) + +func TestSoproHardwareSmoke(t *testing.T) { + if runtime.GOOS != "darwin" || runtime.GOARCH != "arm64" { + t.Skip("Sopro CPU candidate is currently qualified on Apple Silicon macOS only") + } + model := os.Getenv("TAPIOCA_SOPRO_SMOKE_MODEL") + reference := os.Getenv("TAPIOCA_SOPRO_SMOKE_REFERENCE") + if model == "" || reference == "" { + t.Skip("set TAPIOCA_SOPRO_SMOKE_MODEL and TAPIOCA_SOPRO_SMOKE_REFERENCE") + } + output := filepath.Join(t.TempDir(), "sopro.wav") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + var stdout, stderr bytes.Buffer + err := RunWithWriters(ctx, t.TempDir(), Request{ + ModelPath: model, ModelName: "sopro-v2-turbo:cpu-fp32", + Text: "Hello from Tapioca. Sopro is ready to roll.", Output: output, + VoiceSample: reference, VoiceConsent: true, Language: "en", + Backend: "speech-sopro", Seed: 3, + }, &stdout, &stderr) + if err != nil { + t.Fatalf("Sopro generation: %v\n%s", err, stderr.String()) + } + var metrics struct { + FirstAudioSeconds float64 `json:"first_audio_seconds"` + ElapsedSeconds float64 `json:"elapsed_seconds"` + AudioSeconds float64 `json:"audio_seconds"` + } + if err := json.Unmarshal(stdout.Bytes(), &metrics); err != nil || + metrics.FirstAudioSeconds <= 0 || metrics.AudioSeconds <= 0 { + t.Fatalf("invalid Sopro metrics %q: %v", stdout.String(), err) + } + t.Logf("first audio %.2fs, total %.2fs, output %.2fs", metrics.FirstAudioSeconds, metrics.ElapsedSeconds, metrics.AudioSeconds) + data, err := os.ReadFile(output) + if err != nil { + t.Fatal(err) + } + if len(data) < 45 || string(data[:4]) != "RIFF" || string(data[8:12]) != "WAVE" || + binary.LittleEndian.Uint16(data[22:24]) != 1 || + binary.LittleEndian.Uint32(data[24:28]) != 24000 || + binary.LittleEndian.Uint16(data[34:36]) != 16 { + t.Fatalf("invalid Sopro WAV (%d bytes)", len(data)) + } +} diff --git a/internal/speechruntime/sopro_qualification.py b/internal/speechruntime/sopro_qualification.py new file mode 100644 index 0000000..14a5d1e --- /dev/null +++ b/internal/speechruntime/sopro_qualification.py @@ -0,0 +1,103 @@ +"""Experimental, offline-only CPU qualification for Sopro V2 Turbo. + +This runner requires a consented reference recording. It is not a public +catalog backend until voice quality and additional platforms are qualified. +""" + +import json +import os +from pathlib import Path +import tempfile +import time +import wave + + +ARTIFACTS = ( + "config.json", + "model.safetensors", + "semantic_encoder.safetensors", + "speaker_encoder.safetensors", + "tokenizer.model", + "vocoder.safetensors", +) +LANGUAGES = {"", "en", "pt", "fr", "de"} + + +def validate_inputs(args): + if not args.text.strip(): + raise ValueError("Sopro text must not be empty") + if args.language not in LANGUAGES: + raise ValueError("Sopro language must be en, pt, fr, or de") + if not args.voice_sample or not args.voice_consent: + raise ValueError("Sopro requires a reference recording and explicit voice-use permission") + root = Path(args.model).resolve() + for name in ARTIFACTS: + item = root / name + if not item.is_file() or item.stat().st_size == 0: + raise ValueError(f"Missing Sopro artifact: {item}") + sample = Path(args.voice_sample).resolve() + if not sample.is_file() or sample.stat().st_size == 0: + raise ValueError(f"Missing or empty Sopro reference recording: {sample}") + if sample.stat().st_size > 50 * 1024 * 1024: + raise ValueError("Sopro reference recording exceeds 50 MiB") + output = Path(args.output).resolve() + if output.suffix.lower() != ".wav": + raise ValueError("Sopro output must be a WAV file") + return root, sample, output + + +def write_pcm_stream(chunks, output, sample_rate, started, clock=time.monotonic): + first_audio = None + samples = 0 + with wave.open(str(output), "wb") as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(sample_rate) + for pcm in chunks: + if len(pcm) % 2: + raise ValueError("Sopro returned an incomplete PCM16 sample") + if not pcm: + continue + if first_audio is None: + first_audio = clock() - started + wav.writeframes(pcm) + samples += len(pcm) // 2 + if not samples: + raise ValueError("Sopro produced no audio") + elapsed = clock() - started + duration = samples / sample_rate + return {"first_audio_seconds": first_audio, "elapsed_seconds": elapsed, + "audio_seconds": duration, "realtime_factor": elapsed / duration, + "sample_rate": sample_rate, "samples": samples} + + +def run(args): + root, sample, output = validate_inputs(args) + os.environ["HF_HUB_OFFLINE"] = "1" + os.environ["TRANSFORMERS_OFFLINE"] = "1" + import torch + import soundfile + from sopro import SoproTTS + + info = soundfile.info(str(sample)) + seconds = info.frames / info.samplerate + if not 5 <= seconds <= 20: + raise ValueError("Sopro reference recording must be 5–20 seconds") + torch.manual_seed(args.seed) + torch.set_num_threads(4) + started = time.monotonic() + model = SoproTTS.from_pretrained(root, device="cpu") + chunks = ( + chunk.detach().cpu().clamp(-1, 1).mul(32767) + .to(torch.int16).numpy().astype("