diff --git a/CMakeLists.txt b/CMakeLists.txt index 2959e230..d496a5ab 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1410,6 +1410,19 @@ audiocpp_add_model(ace_step LOADERS engine::models::ace_step::make_ace_step_loader ) +audiocpp_add_model(soprano_tts + SOURCES + src/community_models/soprano_tts/assets.cpp + src/community_models/soprano_tts/generator.cpp + src/community_models/soprano_tts/session.cpp + src/community_models/soprano_tts/tokenizer_text.cpp + src/community_models/soprano_tts/vocoder.cpp + INCLUDES + engine/community_models/soprano_tts/session.h + LOADERS + engine::community_models::soprano_tts::make_soprano_tts_loader +) + audiocpp_add_model(midashenglm_gen SOURCES diff --git a/README.md b/README.md index 1e8096ef..04c35c97 100644 --- a/README.md +++ b/README.md @@ -151,10 +151,12 @@ Community model ports live under `community_models` to make the ownership bounda | **minimax_music3** | Music | auto | GGUF Q4/Q8 | [@0xShug0](https://github.com/0xShug0), [@JoeMattie](https://github.com/JoeMattie) | [MiniMax Music 3](docs/community_models/minimax_music3.md) text-to-music generation with lyrics conditioning | | **mms_forced_aligner** | Align | nl (nld), en (eng); pre-romanized Latin | Safetensors, GGUF 16/Q8 | Community | [MMS-300M-1130 Forced Aligner](docs/community_models/mms_forced_aligner.md) word-timestamp alignment from a wav2vec2 CTC checkpoint (safetensors or local GGUF) | | **moss_tts_local** | TTS, Clone, Ctrl | auto, optional language hint | GGUF | [@justinjohn0306](https://github.com/justinjohn0306) | MOSS-TTS-Local Transformer v1.5 support | +| **moss_tts_local** | TTS, Clone, Ctrl | auto, optional language hint | GGUF | [@justinjohn0306](https://github.com/justinjohn0306) | MOSS-TTS-Local Transformer v1.5 support | | **moss_voicegen** | Voice Design | en, zh | GGUF | Joost [@jrohde](https://github.com/jrohde) | [MOSS-VoiceGenerator](docs/community_models/moss_voicegen.md) speech in a voice designed from a written instruction | | **outetts** | TTS, Clone | en, ar, zh, nl, fr, de, it, ja, ko, lt, ru, es, pt, be, bn, ka, hu, lv, fa, pl, sw, ta, uk | GGUF | Mirek [@mirek190](https://github.com/mirek190) | Llama-OuteTTS-1.0-1B TTS and voice cloning support | | **parakeet_tdt** | ASR | auto, bg, cs, da, de, el, en, es, et, fi, fr, hr, hu, it, lt, lv, mt, nl, pl, pt, ro, ru, sk, sl, sv, uk | GGUF F32/16/Q8, Stream | [@dleiferives](https://github.com/dleiferives) | [Parakeet-TDT 0.6B v3](docs/community_models/parakeet_tdt.md) offline, long-form, and buffered-streaming ASR support | | **sense_asr** | ASR | auto, zh, en, yue, ja, ko, pt, ru, es, it, fr, de, nl, pl, tr, ar, hi, vi, th, id, ms, fa, nospeech | GGUF Q8, Stream | Jason Chen [@jasonchen31](https://github.com/jasonchen31), [@LauraGPT](https://github.com/LauraGPT) / FunASR | [SenseVoice-Small](docs/community_models/sense_asr.md) offline/streaming SAN-M + CTC transcription with event/emotion/language tags and ITN | +| **soprano_tts** | TTS | en | GGUF Q8, Stream | [@WalkingCat](https://github.com/WalkingCat) | [Soprano-1.1-80M](https://huggingface.co/WalkingCat/Soprano-1.1-80M-GGUF) ultra-lightweight TTS with Qwen3 LM + Vocos decoder | | **vietneu_tts** | TTS, Clone | vi, en | GGUF | Phuoc [@phuocnguyen90](https://github.com/phuocnguyen90) | [VieNeu-TTS-v3-Turbo](docs/community_models/vietneu_tts.md) TTS and voice cloning support | ## Docker diff --git a/docs/gguf.md b/docs/gguf.md index 3832679c..fd7ec490 100644 --- a/docs/gguf.md +++ b/docs/gguf.md @@ -97,6 +97,7 @@ Status labels: | `qwen3_tts` voice design | Done | Pass | --- | Pass (ASR match, drift) | Pass (ASR match, drift) | | `rvc` | Done | --- | --- | Pass | --- | | `seed_vc` | Done | Pass | --- | Pass (drift) | Pass (drift) | +| `soprano_tts` | Done | Pass | --- | Pass | Pass (drift) | | `silero_vad` | Skip (tiny model) | --- | --- | --- | --- | | `sortformer_diar` | Done | Pass | --- | Pass | Pass | | `stable_audio` | Done | Pass | --- | Pass (drift) | Pass (drift) | diff --git a/docs/soprano_tts.md b/docs/soprano_tts.md new file mode 100644 index 00000000..20afca40 --- /dev/null +++ b/docs/soprano_tts.md @@ -0,0 +1,265 @@ +# Soprano TTS + +Soprano is an ultra-lightweight (~80M parameter) English-only text-to-speech model +using a two-stage architecture: a Qwen3-style causal LM (17 layers, hidden 512, +vocab 8192) that autoregressively emits per-frame 512-dimensional features, and a +non-iterative Vocos-style decoder (ConvNeXt backbone + single ISTFT head, n_fft 2048 / +hop 512) that turns those features into 32 kHz audio. No diffusion refinement is +performed in the decoder. + +| Field | Value | +|---|---| +| Family | `soprano_tts` | +| Task | `tts` | +| Mode | `offline`, `streaming` | +| Languages | `en` | +| Audio | WAV; 32 kHz mono | +| Streaming | Pull events (per-chunk audio) | + +--- + +## Install + +The model-spec manager installs the original safetensors package from the official +Hugging Face repository: + +```bash +python3 tools/model_manager_v2.py install soprano_1_1_80m_original +``` + +Or download the checkpoint directly and convert the decoder manually: + +```bash +# Download the official checkpoint +git lfs install +git clone https://huggingface.co/ekwek/Soprano-1.1-80M models/Soprano-1.1-80M + +# Convert the decoder (folds weight-norm from decoder.pth, emits plain safetensors) +pip install torch numpy safetensors +python3 tools/soprano_tts/convert_soprano.py \ + --input-dir models/Soprano-1.1-80M \ + --output-dir models/Soprano-1.1-80M-converted +``` + +--- + +## Build + +Build audio.cpp with Soprano support: + +```bash +# Soprano only (avoids OOM from 45-model parallel compilation) +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target audiocpp_cli --parallel + +# With Vulkan backend +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ + -DENGINE_ENABLE_VULKAN=ON +cmake --build build --target audiocpp_cli --parallel +``` + +--- + +## CLI + +### Basic inference + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "Soprano is an extremely lightweight text to speech model." \ + --out soprano.wav +``` + +### With Vulkan backend + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --backend vulkan \ + --text "Soprano runs on CPU and Vulkan backends." \ + --out soprano_vulkan.wav +``` + +### Custom generation parameters + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "Warmer temperature and higher max tokens produce longer audio." \ + --request-option temperature=0.5 \ + --request-option max_tokens=256 \ + --seed 42 \ + --out custom.wav +``` + +### Long-form with custom chunk size + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "This is a longer text that will be split into sentence-aware chunks by the framework text chunker. Each chunk is generated and decoded separately, then concatenated into the final audio output." \ + --session-option soprano_tts.text_chunk_size=320 \ + --out longform.wav +``` + +### Streaming mode + +```bash +build/bin/audiocpp_cli --task tts --mode streaming --family soprano_tts \ + --model models/Soprano-1.1-80M-converted \ + --text "Streaming mode emits audio chunks as they are generated." \ + --out stream.wav \ + --out-dir stream_chunks +``` + +--- + +## Options + +### Request options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--request-option max_tokens=` | integer | `512` | Maximum generated audio frames per chunk. | +| `--temperature` / `--request-option temperature=` | float | `0.3` | AR sampling temperature. | +| `--top-p` / `--request-option top_p=` | float | `0.95` | Nucleus sampling threshold. | +| `--repetition-penalty` / `--request-option repetition_penalty=` | float | `1.2` | Repetition penalty. | +| `--request-option eos_bias=` | float | `0.0` | Additive bias on EOS logit; positive stops sooner. | +| `--seed` / `--request-option seed=` | integer | random | AR sampling seed. | + +### Session options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--session-option soprano_tts.text_chunk_size=` | chars | `200` | Max codepoints per chunk. | + +### Load options + +| Option | Values | Default | Meaning | +|---|---|---:|---| +| `--session-option soprano_tts.backbone_weight_type=` | `native`, `f32`, `f16`, `bf16`, `q8_0` | `f32` | LM weight storage. F32 required on CPU. | +| `--session-option soprano_tts.decoder_weight_type=` | `native`, `f32`, `f16` | `f32` | Decoder weight storage. | + +--- + +## Server + +```json +{ + "host": "127.0.0.1", + "port": 8080, + "models": [ + { + "id": "soprano", + "family": "soprano_tts", + "path": "models/Soprano-1.1-80M-converted", + "task": "tts", + "mode": "offline" + } + ] +} +``` + +```bash +audiocpp_server --config server.json + +# OpenAI-compatible TTS endpoint +curl http://127.0.0.1:8080/v1/audio/speech \ + -H "Content-Type: application/json" \ + -d '{ + "model": "soprano", + "input": "Soprano is an extremely lightweight text to speech model.", + "response_format": "wav" + }' \ + -o server_output.wav +``` + +--- + +## GGUF package + +Standalone GGUF packages are available on Hugging Face: + +```bash +# Install with the model manager +python3 tools/model_manager_v2.py install soprano_1_1_80m_q8_0 + +# Or install the BF16 variant +python3 tools/model_manager_v2.py install soprano_1_1_80m_bf16 +``` + +Inference with the GGUF package: + +```bash +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --text "GGUF packages are standalone and self-describing." \ + --out gguf_soprano.wav +``` + +To create a GGUF package from the converted safetensors yourself: + +```bash +build/bin/audiocpp_gguf \ + --input models/Soprano-1.1-80M-converted/model.safetensors \ + --output Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --type q8_0 \ + --root models/Soprano-1.1-80M-converted \ + --family soprano_tts \ + --overwrite +``` +## Performance + +| Backend | RTF | Details | +|---------|---:|--------| +| CPU (warm) | ~0.22-0.23 | ~4-4.6x realtime. F32 storage required for correct output. | +| Vulkan (RX Vega) | ~0.08-0.12 | ~8-13x realtime after one-time shader warmup. Decoder output has numerical drift on this GPU (Vega lacks matrix-core ops). | + +Timing logs are available through `--log`: +- `soprano_tts.lm.generate_ms` -- LM AR decode time +- `soprano_tts.lm.frames` -- generated frames +- `soprano_tts.decoder.decode_ms` -- Vocos decoder time +- `soprano_tts.lm.decode.plan_cached` -- plan caching status + +--- + +## Memory + +| Metric | Value | Conditions | +|--------|-------|------------| +| Model size (safetensors) | ~380 MB (backbone BF16) + ~18 MB (decoder F32) | Original HF checkpoint | +| Peak RSS (CPU) | ~1.2 GB | Graph arena (512 MB) + weight context (256 MB) + runtime overhead | +| Peak VRAM (Vulkan) | Not measured | Vega ~1.2 GB reported system RAM usage | + +--- + +## Known limitations + +- English-only (model limitation) +- No voice cloning +- EOS sampling unreliable at low temperature (C++ RNG != PyTorch RNG) +- Full composite build may OOM; use AUDIOCPP_MODEL_SET=custom with AUDIOCPP_MODELS=soprano_tts + +--- + +## Architecture + +Soprano uses a two-stage architecture: + +1. **Qwen3 causal LM** (17 layers, hidden 512, 4 heads, 1 KV head, head_dim 128, vocab 8192, + intermediate 2304, rope_theta 10000). Takes prompt `[STOP][TEXT][START]` and + autoregressively generates tokens. Each step's last-layer hidden state (512-dim) equals + one audio frame. + +2. **Vocos decoder** (non-iterative): Interpolate x4 linear align_corners -> Conv1d(512->768,k=1) + -> LN -> 8x ConvNeXt(dwconv k=3 groups, LN, Linear->2304, GELU, Linear->768, gamma) -> LN -> + Linear(768->2050) -> split mag/phase -> exp*exp(i*phi) -> istft(center=True) with Hann window + (n_fft=2048, hop=512). + +Output: 32 kHz mono. Token ~ 2048 samples ~ 64 ms. + +Reference: https://github.com/ekwek1/soprano +Weights: https://huggingface.co/ekwek/Soprano-1.1-80M diff --git a/docs/soprano_validation.md b/docs/soprano_validation.md new file mode 100644 index 00000000..13d27628 --- /dev/null +++ b/docs/soprano_validation.md @@ -0,0 +1,102 @@ +# Soprano TTS Validation + +## Build + +```bash +# Soprano-only build +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target audiocpp_cli --parallel + +# With Vulkan backend +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts \ + -DENGINE_ENABLE_VULKAN=ON +cmake --build build --target audiocpp_cli --parallel + +# Build warmbench +cmake -B build -DCMAKE_BUILD_TYPE=Release \ + -DAUDIOCPP_MODEL_SET=custom -DAUDIOCPP_MODELS=soprano_tts +cmake --build build --target soprano_warm_bench --parallel +``` + +## Convert the checkpoint + +```bash +# Download the official checkpoint +git lfs install +git clone https://huggingface.co/ekwek/Soprano-1.1-80M models/Soprano-1.1-80M + +# Convert the decoder (folds weight-norm from decoder.pth) +pip install torch numpy safetensors +python3 tools/soprano_tts/convert_soprano.py \ + --input-dir models/Soprano-1.1-80M \ + --output-dir models/soprano_pkg +``` + +## Run warmbench + +```bash +build/bin/soprano_warm_bench --model models/soprano_pkg --output-dir build/logs/warmbench/soprano_tts +``` + +## Python reference warmbench + +```bash +pip install soprano torch numpy +python3 tests/soprano_tts/soprano_python_warm_bench.py \ + --model models/Soprano-1.1-80M \ + --out-dir build/logs/warmbench/soprano_tts_py +``` + +## CLI examples + +```bash +# Basic inference +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/soprano_pkg \ + --text "Soprano is an extremely lightweight text to speech model." \ + --out soprano.wav + +# With Vulkan backend +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/soprano_pkg \ + --backend vulkan \ + --text "Soprano runs on CPU and Vulkan backends." \ + --out soprano_vulkan.wav + +# GGUF package +python3 tools/model_manager_v2.py install soprano_1_1_80m_q8_0 +build/bin/audiocpp_cli --task tts --family soprano_tts \ + --model models/Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf \ + --text "GGUF packages are standalone." --out gguf_out.wav +``` + +## Performance results + +### CPU (compared against Python `soprano` package, transformers backend, temp=0.3, top_p=0.95) + +| Test | Chars | Platform | Audio (s) | Infer (s) | RTF | Speedup | +|---|---|---|---|---|---|---| +| short | 57 | Python | 0.752 | 1.281 | 1.7037 | \u2014 | +| | | **C++** | **3.136** | **0.740** | **0.2360** | **7.22x** | +| medium | 152 | Python | 2.096 | 1.909 | 0.9106 | \u2014 | +| | | **C++** | **8.320** | **1.955** | **0.2350** | **3.87x** | +| long | 567 | Python | 7.424 | 5.773 | 0.7776 | \u2014 | +| | | **C++** | **16.384** | **4.302** | **0.2626** | **2.96x** | + +### Vulkan (AMD Radeon RX Vega) + +| Test | Audio (s) | Infer (s) | RTF | +|---|---|---|---| +| short | ~3.1 | ~0.25 | ~0.08 | +| medium | ~8.3 | ~0.70 | ~0.08 | +| long | ~16.4 | ~1.60 | ~0.10 | + +## Known limitations + +- English-only (model limitation) +- No voice cloning +- EOS sampling unreliable at low temperature (PyTorch vs C++ RNG difference) +- Full composite build may OOM; use AUDIOCPP_MODEL_SET=custom +- Vulkan decoder output shows numerical drift on AMD RX Vega (no matrix-core ops) diff --git a/include/engine/community_models/soprano_tts/assets.h b/include/engine/community_models/soprano_tts/assets.h new file mode 100644 index 00000000..320bdd38 --- /dev/null +++ b/include/engine/community_models/soprano_tts/assets.h @@ -0,0 +1,68 @@ +#pragma once + +#include "engine/framework/assets/resource_bundle.h" +#include "engine/framework/assets/tensor_source.h" + +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +struct SopranoTTSConfig { + // Qwen3 causal LM (config.json). + int64_t hidden_size = 512; + int64_t intermediate_size = 2304; + int64_t layers = 17; + int64_t attention_heads = 4; + int64_t kv_heads = 1; + int64_t head_dim = 128; + int64_t vocab_size = 8192; + int64_t max_position_embeddings = 1024; + float rms_norm_eps = 1.0e-6f; + float rope_theta = 10000.0f; + int32_t bos_token_id = 3; + int32_t eos_token_id = 3; + + // Non-iterative Vocos decoder (decoder.pth / config). + int64_t decoder_input_channels = 512; // == hidden_size + int64_t decoder_dim = 768; + int64_t decoder_intermediate_dim = 2304; + int64_t decoder_num_layers = 8; + int64_t dw_kernel = 3; + int64_t n_fft = 2048; + int64_t hop_length = 512; + int64_t upscale = 4; + int64_t sample_rate = 32000; + int64_t token_size = 2048; // samples per generated frame + int64_t max_new_tokens = 128; + float temperature = 0.3f; + float top_p = 0.95f; + float repetition_penalty = 1.2f; +}; + +struct SopranoTTSAssets { + assets::ResourceBundle resources; + SopranoTTSConfig config; + std::shared_ptr backbone_weights; + std::shared_ptr decoder_weights; +}; + +struct SopranoGenerationOptions { + // Per-chunk limit; the official reference allows up to 512 frames + // (32 s of audio) per sentence. + int64_t max_new_tokens = 512; + float temperature = 0.3f; + float top_p = 0.95f; + float repetition_penalty = 1.2f; + uint64_t seed = 0; + bool has_seed = false; + // Additive bias on the EOS logit; 0 disables (opt-in runaway mitigation). + float eos_bias = 0.0F; +}; + +std::shared_ptr load_soprano_tts_assets( + const std::filesystem::path & model_path); + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/include/engine/community_models/soprano_tts/generator.h b/include/engine/community_models/soprano_tts/generator.h new file mode 100644 index 00000000..992de5ce --- /dev/null +++ b/include/engine/community_models/soprano_tts/generator.h @@ -0,0 +1,43 @@ +#pragma once + +#include "engine/framework/core/execution_context.h" +#include "engine/community_models/soprano_tts/assets.h" + +#include +#include +#include + +namespace engine::community_models::soprano_tts { +struct SopranoQwenWeights; + +// Autoregressive Qwen3 causal LM wrapper. Captures the last-layer 512-dim +// hidden state of every generated token (the per-frame audio features) plus the +// sampled token ids, stopping on EOS. +class SopranoTTSGenerator { +public: + SopranoTTSGenerator( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type); + ~SopranoTTSGenerator(); + + struct Result { + std::vector features; // frames x hidden (frame-major) + std::vector tokens; // generated token ids (excluding EOS) + int64_t frames = 0; + }; + + Result generate(const std::vector & prompt_ids, + const SopranoGenerationOptions & options); + + void release_runtime_graphs(); + +private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/include/engine/community_models/soprano_tts/session.h b/include/engine/community_models/soprano_tts/session.h new file mode 100644 index 00000000..4dba830d --- /dev/null +++ b/include/engine/community_models/soprano_tts/session.h @@ -0,0 +1,67 @@ +#pragma once + +#include "engine/framework/model_spec/metadata.h" +#include "engine/framework/runtime/model.h" +#include "engine/framework/runtime/session_base.h" +#include "engine/community_models/soprano_tts/assets.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +std::shared_ptr make_soprano_tts_loader(); + +struct SopranoRequest { + std::string text; + SopranoGenerationOptions generation; +}; + +class SopranoTTSGenerator; +class SopranoDecoderRuntime; + +class SopranoTTSOfflineSession final : public runtime::RuntimeSessionBase, + public runtime::IOfflineVoiceTaskSession, + public runtime::IStreamingVoiceTaskSession { +public: + SopranoTTSOfflineSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract); + ~SopranoTTSOfflineSession() override; + + std::string family() const override; + runtime::VoiceTaskKind task_kind() const override; + runtime::RunMode run_mode() const override; + void prepare(const runtime::SessionPreparationRequest & request) override; + runtime::TaskResult run(const runtime::TaskRequest & request) override; + runtime::StreamingPolicy streaming_policy() const override; + void start_stream(const runtime::TaskRequest & request) override; + std::optional next_stream_event() override; + void set_stream_event_sink(runtime::StreamEventCallback sink) override; + runtime::TaskResult finish_stream() override; + void reset() override; + runtime::StreamEvent process_audio_chunk(const runtime::AudioChunk & chunk) override; + runtime::TaskResult finalize() override; + +private: + SopranoRequest make_request(const runtime::TaskRequest & request) const; + runtime::AudioBuffer synthesize(const SopranoRequest & request); + // Streaming state + std::optional> streaming_chunks_; + size_t streaming_chunk_index_ = 0; + std::vector streaming_audio_; + runtime::StreamEventCallback stream_sink_; + + runtime::TaskSpec task_; + std::shared_ptr assets_; + std::shared_ptr contract_; + std::unique_ptr generator_; + std::unique_ptr decoder_; +}; + +} // namespace engine::community_models::soprano_tts diff --git a/include/engine/community_models/soprano_tts/tokenizer_text.h b/include/engine/community_models/soprano_tts/tokenizer_text.h new file mode 100644 index 00000000..94dcba79 --- /dev/null +++ b/include/engine/community_models/soprano_tts/tokenizer_text.h @@ -0,0 +1,46 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +// Byte-level BPE tokenizer for the Soprano text prompt. The LM expects the text +// wrapped as a single prompt: "[STOP][TEXT][START]". Diacritics are +// removed and text is case-folded (unidecode-style) before encoding, matching +// the reference `clean_text` behaviour for English. +class SopranoTextTokenizer { +public: + explicit SopranoTextTokenizer(const std::filesystem::path & tokenizer_json_path); + + std::vector encode_text(const std::string & text) const; + std::string decode_ids(const std::vector & ids) const; + + int32_t bos_id() const noexcept { return bos_id_; } + int32_t eos_id() const noexcept { return eos_id_; } + int32_t stop_id() const noexcept { return stop_id_; } + int32_t text_id() const noexcept { return text_id_; } + int32_t start_id() const noexcept { return start_id_; } + int64_t vocab_size() const noexcept { return static_cast(id_to_token_.size()); } + +private: + std::vector apply_prompt(const std::vector & speech_tokens) const; + + std::vector id_to_token_; + std::unordered_map token_to_id_; + std::vector> merges_; // rank-ordered (a_id, b_id) + + int32_t bos_id_ = -1; + int32_t eos_id_ = -1; + int32_t stop_id_ = -1; + int32_t text_id_ = -1; + int32_t start_id_ = -1; +}; + +std::string scalarclean_soprano_text(const std::string & text); + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/include/engine/community_models/soprano_tts/vocoder.h b/include/engine/community_models/soprano_tts/vocoder.h new file mode 100644 index 00000000..b032acad --- /dev/null +++ b/include/engine/community_models/soprano_tts/vocoder.h @@ -0,0 +1,51 @@ +#pragma once + +#include "engine/community_models/soprano_tts/assets.h" + +#include +#include +#include + +namespace engine::core { +class ExecutionContext; +} +namespace engine::assets { +enum class TensorStorageType; +} +namespace engine::runtime { +struct AudioBuffer; +} + +namespace engine::community_models::soprano_tts { + +struct SopranoDecoderWeights; +struct SopranoDecoderGraph; + +// Non-iterative Vocos-style decoder (SopranoDecoder): linear upsample x4 over +// the frame axis, a ConvNeXt backbone (embed Conv1d, 8 blocks, final LN) and a +// single ISTFT head (Linear(dim -> n_fft+2), exp(mag), cos/sin phase, 1 ISTFT). +class SopranoDecoderRuntime final { +public: + SopranoDecoderRuntime( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type); + ~SopranoDecoderRuntime(); + + // frames x hidden -> 32 kHz mono audio. + runtime::AudioBuffer decode(const std::vector & features, int64_t frames) const; + + + +private: + const SopranoTTSConfig & config_; + engine::core::ExecutionContext & execution_context_; + size_t graph_context_bytes_ = 0; + std::shared_ptr weights_; + mutable std::unique_ptr graph_; +}; + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/model_specs/soprano_tts.json b/model_specs/soprano_tts.json new file mode 100644 index 00000000..cc9ae299 --- /dev/null +++ b/model_specs/soprano_tts.json @@ -0,0 +1,185 @@ +{ + "schema_version": 1, + "family": "soprano_tts", + "display_name": "Soprano", + "description": "Soprano is an ultra-lightweight (~80M) English-only text-to-speech model. Syntax uses a 17-layer Qwen3-style causal LM (hidden 512, vocab 8192) that autoregressively emits per-frame 512-dimensional features; a non-iterative Vocos-style decoder (ConvNeXt backbone + single ISTFT head, n_fft 2048 / hop 512) turns those features into 32 kHz audio. No diffusion refinement is performed in the decoder.", + "category": "tts", + "status": "community", + "tasks": [ + "tts" + ], + "modes": [ + "offline", + "streaming" + ], + "languages": [ + "en" + ], + "runtime": { + "tags": [ + "gguf", + "stream" + ] + }, + "capabilities": { + "tts": [ + "long_form" + ] + }, + "options": { + "request": [ + { + "name": "max_tokens", + "type": "int", + "description": "Maximum generated audio frames for the autoregressive LM; default 512.", + "required": false, + "min": 1, + "default": 512 + }, + { + "name": "temperature", + "type": "float", + "description": "Autoregressive sampling temperature; default 0.3 (0 selects the framework default and clamps to a small positive value).", + "required": false, + "min": 0.0, + "default": 0.3 + }, + { + "name": "top_p", + "type": "float", + "description": "Nucleus sampling probability; default 0.95.", + "required": false, + "min": 0.0, + "max": 1.0, + "default": 0.95 + }, + { + "name": "repetition_penalty", + "type": "float", + "description": "Repetition penalty applied to the LM head; default 1.2.", + "required": false, + "min": 1.0, + "default": 1.2 + }, + { + "name": "eos_bias", + "type": "float", + "description": "Additive bias on the EOS token logit during generation. Positive values make the model stop sooner when speech ends (mitigating runaway generations that hit max_tokens); negative values encourage longer utterances. Default 0 disables the adjustment.", + "required": false, + "default": 0.0 + }, + { + "name": "seed", + "type": "int", + "description": "Autoregressive sampling seed; omitted requests choose a random seed.", + "required": false, + "min": 0 + } + ], + "session": [ + { + "name": "text_chunk_size", + "type": "int", + "description": "Maximum codepoints per sentence chunk before the model generates and decodes separately. Smaller values keep prompts short (more reliable EOS) but increase overhead. Default 200.", + "required": false, + "min": 32, + "default": 200 + } + ], + "load": [ + { + "name": "backbone_weight_type", + "type": "enum", + "preset": "weight_type_full", + "required": false, + "default": "native", + "description": "Storage type for the Qwen3 LM backbone weights." + }, + { + "name": "decoder_weight_type", + "type": "enum", + "preset": "weight_type_conv", + "required": false, + "default": "native", + "description": "Storage type for the Vocos decoder weights." + } + ] + }, + "package_defaults": { + "download": { + "kind": "huggingface_snapshot", + "repo": "WalkingCat/Soprano-1.1-80M-GGUF", + "revision": "main", + "gated": false + } + }, + "packages": [ + { + "id": "soprano_1_1_80m_q8_0", + "display_name": "Soprano-1.1-80M Q8_0 GGUF", + "default": true, + "format": "gguf", + "precision": "q8_0", + "target_directory": "Soprano-1.1-80M-GGUF", + "files": [ + "Soprano-1.1-80M-GGUF/soprano-1.1-80m-q8_0.gguf" + ], + "strip_prefix": "Soprano-1.1-80M-GGUF" + }, + { + "id": "soprano_1_1_80m_bf16", + "display_name": "Soprano-1.1-80M BF16 GGUF", + "format": "gguf", + "precision": "bf16", + "target_directory": "Soprano-1.1-80M-GGUF", + "files": [ + "Soprano-1.1-80M-GGUF/soprano-1.1-80m-bf16.gguf" + ], + "strip_prefix": "Soprano-1.1-80M-GGUF" + } + ], + "dependencies": [], + "ui": { + "recommended_package": "soprano_1_1_80m_q8_0", + "tags": [ + "TTS", + "Stream" + ], + "docs": [ + "docs/soprano_tts.md" + ] + }, + "sources": [ + { + "format": "gguf", + "roots": { + "model": ".", + "weights": "$gguf" + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "backbone": "weights:", + "decoder": "weights:" + } + }, + { + "format": "safetensors", + "roots": { + "model": "." + }, + "files": { + "config": "model:config.json", + "generation_config": "model:generation_config.json", + "tokenizer_json": "model:tokenizer.json" + }, + "tensors": { + "backbone": "model:combined.safetensors", + "decoder": "model:combined.safetensors" + } + } + ] +} diff --git a/src/community_models/soprano_tts/assets.cpp b/src/community_models/soprano_tts/assets.cpp new file mode 100644 index 00000000..453fbd3e --- /dev/null +++ b/src/community_models/soprano_tts/assets.cpp @@ -0,0 +1,64 @@ +#include "engine/community_models/soprano_tts/assets.h" + +#include "engine/framework/io/json.h" +#include "engine/framework/model_spec/package.h" + +#include +#include + +namespace engine::community_models::soprano_tts { +namespace { + +namespace json = engine::io::json; + +constexpr const char * kFamily = "soprano_tts"; + +SopranoTTSConfig parse_config(const assets::ResourceBundle & resources) { + const auto root = resources.parse_json("config"); + if (json::require_string(root, "model_type") != "qwen3") { + throw std::runtime_error("Soprano config must use model_type qwen3"); + } + SopranoTTSConfig out; + out.hidden_size = json::require_i64(root, "hidden_size"); + out.intermediate_size = json::require_i64(root, "intermediate_size"); + out.layers = json::require_i64(root, "num_hidden_layers"); + out.attention_heads = json::require_i64(root, "num_attention_heads"); + out.kv_heads = json::require_i64(root, "num_key_value_heads"); + if (const auto * hd = root.find("head_dim")) { + out.head_dim = hd->as_i64(); + } else { + out.head_dim = out.hidden_size / out.attention_heads; + } + out.vocab_size = json::require_i64(root, "vocab_size"); + out.rms_norm_eps = json::optional_f32(root, "rms_norm_eps", out.rms_norm_eps); + if (const auto * rope = root.find("rope_parameters")) { + out.rope_theta = json::optional_f32(*rope, "rope_theta", out.rope_theta); + } else { + out.rope_theta = json::optional_f32(root, "rope_theta", out.rope_theta); + } + out.max_position_embeddings = json::optional_i64(root, "max_position_embeddings", out.max_position_embeddings); + if (const auto * eos = root.find("eos_token_id")) { + out.eos_token_id = static_cast(eos->as_i64()); + } + if (const auto * bos = root.find("bos_token_id")) { + out.bos_token_id = static_cast(bos->as_i64()); + } + // Decoder dimensions are fixed by the SopranoDecoder architecture. + out.decoder_input_channels = out.hidden_size; + return out; +} + +} // namespace + +std::shared_ptr load_soprano_tts_assets( + const std::filesystem::path & model_path) { + auto assets = std::make_shared(); + assets->resources = engine::model_spec::load_resource_bundle( + model_path, engine::model_spec::default_spec_path(kFamily)); + assets->config = parse_config(assets->resources); + assets->backbone_weights = assets->resources.open_tensor_source("backbone"); + assets->decoder_weights = assets->resources.open_tensor_source("decoder"); + return assets; +} + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/src/community_models/soprano_tts/generator.cpp b/src/community_models/soprano_tts/generator.cpp new file mode 100644 index 00000000..812825a4 --- /dev/null +++ b/src/community_models/soprano_tts/generator.cpp @@ -0,0 +1,342 @@ +#include "engine/community_models/soprano_tts/generator.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/transformers/qwen_causal_decode_runtime.h" +#include "engine/framework/modules/transformers/qwen_causal_decoder.h" +#include "engine/framework/modules/weight_binding.h" +#include "engine/framework/sampling/hf_sampler.h" +#include "engine/framework/sampling/torch_random.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +struct SopranoQwenWeights { + std::shared_ptr store; + engine::core::TensorValue token_embedding; + engine::modules::QwenDecoderStackWeights stack; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights lm_head; +}; + +namespace { + +namespace binding = engine::modules::binding; + +std::shared_ptr require_assets( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Soprano LM generator requires assets"); + } + return assets; +} + +modules::QwenDecoderLayerWeights load_layer_weights( + engine::core::BackendWeightStore & store, + const engine::assets::TensorSource & source, + const SopranoTTSConfig & config, + engine::assets::TensorStorageType storage_type, + int64_t layer) { + const std::string prefix = "model.layers." + std::to_string(layer); + modules::QwenDecoderLayerWeights out; + out.input_norm = binding::norm_weight_from_source( + store, source, prefix + ".input_layernorm", config.hidden_size); + // Fused QKV projection: concatenate Q|K|V rows so the decoder runs a + // single GEMM per layer (QwenDecoderQKVLayout::PackedQKV). + const int64_t q_out = config.attention_heads * config.head_dim; + const int64_t kv_out = config.kv_heads * config.head_dim; + std::vector qkv_rows = source.require_f32( + prefix + ".self_attn.q_proj.weight", {q_out, config.hidden_size}); + const auto k_rows = source.require_f32( + prefix + ".self_attn.k_proj.weight", {kv_out, config.hidden_size}); + const auto v_rows = source.require_f32( + prefix + ".self_attn.v_proj.weight", {kv_out, config.hidden_size}); + qkv_rows.insert(qkv_rows.end(), k_rows.begin(), k_rows.end()); + qkv_rows.insert(qkv_rows.end(), v_rows.begin(), v_rows.end()); + out.self_attention.qkv_weight = store.make_from_f32( + engine::core::TensorShape::from_dims({q_out + kv_out * 2, config.hidden_size}), + storage_type, + std::move(qkv_rows)); + out.self_attention.out_weight = store.load_tensor( + source, prefix + ".self_attn.o_proj.weight", storage_type, + {config.hidden_size, config.attention_heads * config.head_dim}); + out.q_norm = binding::norm_weight_from_source( + store, source, prefix + ".self_attn.q_norm", config.head_dim); + out.k_norm = binding::norm_weight_from_source( + store, source, prefix + ".self_attn.k_norm", config.head_dim); + out.post_norm = binding::norm_weight_from_source( + store, source, prefix + ".post_attention_layernorm", config.hidden_size); + // Fused gate/up projection: gate|up rows in one GEMM; the decoder's + // PackedGateUp mode also uses the fused swiglu kernel. + std::vector gate_up_rows = source.require_f32( + prefix + ".mlp.gate_proj.weight", + {config.intermediate_size, config.hidden_size}); + const auto up_rows = source.require_f32( + prefix + ".mlp.up_proj.weight", + {config.intermediate_size, config.hidden_size}); + gate_up_rows.insert(gate_up_rows.end(), up_rows.begin(), up_rows.end()); + out.mlp.gate_up_proj = modules::LinearWeights{ + store.make_from_f32( + engine::core::TensorShape::from_dims( + {config.intermediate_size * 2, config.hidden_size}), + storage_type, + std::move(gate_up_rows)), + std::nullopt}; + out.mlp.down_proj = binding::linear_from_source( + store, source, prefix + ".mlp.down_proj", storage_type, + config.hidden_size, config.intermediate_size, false); + return out; +} +modules::QwenDecoderActivationCastPolicy soprano_activation_cast_policy( + core::BackendType backend_type) { + // No activation cast for Soprano — keep everything in F32 for parity. + (void)backend_type; + return modules::QwenDecoderActivationCastPolicy{}; +} + +modules::QwenCausalDecoderConfig make_soprano_qwen_config( + const SopranoTTSConfig & config, + core::BackendType backend_type) { + modules::QwenCausalDecoderConfig out; + out.stack.hidden_size = config.hidden_size; + out.stack.num_attention_heads = config.attention_heads; + out.stack.num_key_value_heads = config.kv_heads; + out.stack.head_dim = config.head_dim; + out.stack.intermediate_size = config.intermediate_size; + out.stack.layers = config.layers; + out.stack.rms_norm_eps = config.rms_norm_eps; + out.stack.rope_theta = config.rope_theta; + out.stack.rope_type = GGML_ROPE_TYPE_NEOX; + out.stack.attention_precision = GGML_PREC_DEFAULT; + out.stack.projection_precision = GGML_PREC_DEFAULT; + out.stack.activation_cast = soprano_activation_cast_policy(backend_type); + out.stack.use_qk_norm = true; + // Fused projections: single QKV GEMM + single gate/up GEMM with the + // fused swiglu kernel (Soprano has no activation casts, so it qualifies). + out.stack.qkv_layout = modules::QwenDecoderQKVLayout::PackedQKV; + out.stack.runtime.mlp.mode = modules::QwenDecoderMLPMode::PackedGateUp; + out.stack.runtime.attention.prefill_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.attention.static_mode = modules::QwenDecoderAttentionMode::FlashGroupedViewKV; + out.stack.runtime.static_cache.update_mode = modules::QwenDecoderStaticCacheUpdateMode::DirectSetRows; + out.logits_size = config.vocab_size; + out.logits_mode = modules::QwenCausalDecoderLogitsMode::LastStep; + out.use_lm_head_bias = false; + out.lm_head_precision = GGML_PREC_DEFAULT; + if (backend_type == core::BackendType::Vulkan || backend_type == core::BackendType::Metal) { + out.lm_head_input_type = GGML_TYPE_F16; + } else if (backend_type != core::BackendType::Cpu) { + out.lm_head_input_type = GGML_TYPE_BF16; + } + return out; +} + +std::shared_ptr load_soprano_qwen_weights( + const SopranoTTSAssets & assets, + ggml_backend_t backend, + core::BackendType backend_type, + size_t weight_context_bytes, + assets::TensorStorageType storage_type) { + auto weights = std::make_shared(); + weights->store = std::make_shared( + backend, backend_type, "soprano_tts.lm.weights", weight_context_bytes); + const auto & config = assets.config; + const auto & source = *assets.backbone_weights; + weights->token_embedding = weights->store->load_tensor( + source, "model.embed_tokens.weight", storage_type, + {config.vocab_size, config.hidden_size}); + weights->stack.layers.reserve(static_cast(config.layers)); + for (int64_t layer = 0; layer < config.layers; ++layer) { + weights->stack.layers.push_back(load_layer_weights( + *weights->store, source, config, storage_type, layer)); + } +weights->final_norm = binding::norm_weight_from_source( + *weights->store, source, "model.norm", config.hidden_size); + weights->lm_head = binding::linear_from_source( + *weights->store, source, "lm_head", storage_type, + config.vocab_size, config.hidden_size, false); + weights->store->upload(); + return weights; +} + +modules::QwenCausalDecodeRuntimeConfig make_soprano_decode_runtime_config( + const SopranoTTSConfig & config, + core::BackendType backend_type, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes) { + modules::QwenCausalDecodeRuntimeConfig out; + out.trace_name = "soprano_tts.lm"; + out.decoder = make_soprano_qwen_config(config, backend_type); + out.prefill_graph_arena_bytes = prefill_graph_arena_bytes; + out.decode_graph_arena_bytes = decode_graph_arena_bytes; + // Both logits (sampling + EOS) and the 512-d hidden frame (audio) are needed. + out.output_mode = modules::QwenCausalDecodeOutputMode::Logits; + out.return_hidden = true; + return out; +} + +modules::QwenCausalDecodeRuntimeWeights make_soprano_decode_weights( + const SopranoQwenWeights & weights) { + modules::QwenCausalDecodeRuntimeWeights out; + out.token_embedding = weights.token_embedding; + out.stack = weights.stack; + out.final_norm = weights.final_norm; + out.lm_head = weights.lm_head; + return out; +} + +// Hidden-mode weights are intentionally not built: Hidden mode produces +// NaN/Inf prefill output (see docs/soprano_tts.md §6h), so the LM +// always runs single-pass Logits+return_hidden=true. + +} // namespace + +class SopranoTTSGenerator::Impl { +public: + Impl( + std::shared_ptr assets, + core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + assets::TensorStorageType weight_storage_type) + : assets_(require_assets(std::move(assets))), + backend_(execution.backend()), + backend_type_(execution.backend_type()), + weights_(std::make_shared(std::move(*load_soprano_qwen_weights( + *assets_, execution.backend(), backend_type_, weight_context_bytes, + weight_storage_type)))) { + if (backend_ == nullptr) { + throw std::runtime_error("Soprano LM backend is not initialized"); + } + qwen_runtime = std::make_unique( + execution, + make_soprano_decode_runtime_config( + assets_->config, backend_type_, + prefill_graph_arena_bytes, decode_graph_arena_bytes), + make_soprano_decode_weights(*weights_)); + } + + Result generate(const std::vector & prompt_ids, + const SopranoGenerationOptions & options) { + if (prompt_ids.empty()) { + throw std::runtime_error("Soprano LM requires a non-empty prompt"); + } + const SopranoTTSConfig & config = assets_->config; + std::vector features; + std::vector tokens; + + // Single-pass AR generation capturing both logits and hidden states. + // With F32 weights + correct tokenizer, return_hidden=true now + // produces correct results. + auto prefill = qwen_runtime->prefill_tokens(prompt_ids); + // Honor the requested token limit (matches HF max_new_tokens), capped + // so prompt + generated always fits the model context window. + const int64_t max_new_tokens = std::max( + 1, + std::min( + options.max_new_tokens, + config.max_position_embeddings - + static_cast(prompt_ids.size()))); + // Size the KV cache to the actual worst-case need (prompt + generated + // frames) instead of the full 1024-token context. Smaller cache means + // less KV memory for attention to walk on every decode step. + qwen_runtime->start_decode_tokens(prefill.state, max_new_tokens + + static_cast(prompt_ids.size())); + + // First feature: last prompt token's post-norm hidden state. + features.insert(features.end(), prefill.hidden.begin(), prefill.hidden.end()); + + sampling::HfSamplingOptions sampling_options; + sampling_options.do_sample = true; + sampling_options.temperature = options.temperature; + sampling_options.top_k = 0; + sampling_options.top_p = options.top_p; + sampling_options.min_tokens_to_keep = 1; + sampling_options.repetition_penalty = options.repetition_penalty; + sampling::HfSampler sampler; + sampling::HfSamplerScratch scratch; + scratch.reserve_vocab(static_cast(config.vocab_size)); + std::mt19937 fallback_rng(static_cast(options.seed)); + + std::vector history(prompt_ids.begin(), prompt_ids.end()); + std::vector logits = std::move(prefill.logits); + const int32_t eos_id = config.eos_token_id; + + for (int64_t step = 0; step < max_new_tokens; ++step) { + if (options.eos_bias != 0.0F) { + logits[static_cast(eos_id)] += options.eos_bias; + } + const int32_t token = sampler.sample( + logits, history, sampling_options, scratch, fallback_rng, nullptr, + "soprano_tts AR"); + if (token == eos_id) { + history.push_back(token); + break; + } + history.push_back(token); + tokens.push_back(token); + auto decode = qwen_runtime->decode_token(token); + features.insert(features.end(), decode.hidden.begin(), decode.hidden.end()); + logits = std::move(decode.logits); + } + + if (tokens.empty()) { + throw std::runtime_error("Soprano LM produced no audio frames"); + } + + Result out; + out.tokens = std::move(tokens); + const int64_t T_gen = static_cast(out.tokens.size()) + 1; // +1 for prefill + out.frames = T_gen; + out.features = std::move(features); + return out; + } + + void release_runtime_graphs() { + qwen_runtime->release_runtime_graphs(); + } + + std::shared_ptr assets_; + ggml_backend_t backend_ = nullptr; + core::BackendType backend_type_ = core::BackendType::Cpu; + std::shared_ptr weights_; + std::unique_ptr qwen_runtime; +}; + +SopranoTTSGenerator::SopranoTTSGenerator( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution, + size_t prefill_graph_arena_bytes, + size_t decode_graph_arena_bytes, + size_t weight_context_bytes, + engine::assets::TensorStorageType weight_storage_type) + : impl_(std::make_unique( + std::make_shared(assets), execution, + prefill_graph_arena_bytes, decode_graph_arena_bytes, + weight_context_bytes, weight_storage_type)) {} + +SopranoTTSGenerator::~SopranoTTSGenerator() = default; + +SopranoTTSGenerator::Result SopranoTTSGenerator::generate( + const std::vector & prompt_ids, + const SopranoGenerationOptions & options) { + return impl_->generate(prompt_ids, options); +} + +void SopranoTTSGenerator::release_runtime_graphs() { + impl_->release_runtime_graphs(); +} + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/src/community_models/soprano_tts/session.cpp b/src/community_models/soprano_tts/session.cpp new file mode 100644 index 00000000..330411f7 --- /dev/null +++ b/src/community_models/soprano_tts/session.cpp @@ -0,0 +1,289 @@ +#include "engine/community_models/soprano_tts/session.h" + +#include "engine/framework/core/execution_context.h" +#include "engine/framework/runtime/options.h" +#include "engine/framework/runtime/spec_backed_model.h" +#include "engine/community_models/soprano_tts/generator.h" +#include "engine/community_models/soprano_tts/tokenizer_text.h" +#include "engine/framework/debug/profiler.h" +#include "engine/framework/debug/trace.h" +#include "engine/framework/text/chunking.h" +#include "engine/community_models/soprano_tts/vocoder.h" + +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { +namespace { + +constexpr const char * kFamily = "soprano_tts"; +constexpr size_t kDefaultGraphArenaBytes = 512ull * 1024ull * 1024ull; +constexpr size_t kDefaultWeightContextBytes = 256ull * 1024ull * 1024ull; + +std::shared_ptr require_assets( + std::shared_ptr assets) { + if (assets == nullptr) { + throw std::runtime_error("Soprano session requires assets"); + } + return assets; +} + +std::shared_ptr require_contract( + std::shared_ptr contract) { + if (contract == nullptr) { + throw std::runtime_error("Soprano session requires a model contract"); + } + return contract; +} + +std::string request_text(const runtime::TaskRequest & request) { + if (!request.text_input.has_value() || request.text_input->text.empty()) { + throw std::runtime_error("Soprano requires non-empty text input"); + } + return request.text_input->text; +} + +SopranoGenerationOptions request_generation_options(const runtime::TaskRequest & request) { + SopranoGenerationOptions out; + if (const auto value = runtime::parse_i64_option(request.options, {"max_tokens"})) { + out.max_new_tokens = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"temperature"})) { + if (*value <= 0.0F) { + throw std::runtime_error("Soprano temperature must be positive"); + } + out.temperature = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"top_p"})) { + out.top_p = *value; + } + if (const auto value = runtime::parse_finite_float_option(request.options, {"repetition_penalty"})) { + out.repetition_penalty = *value; + } + if (const auto value = runtime::parse_u64_option(request.options, {"seed"})) { + out.seed = *value; + out.has_seed = true; + } + if (!out.has_seed) { + out.seed = runtime::random_u64_seed(); + } + if (out.max_new_tokens < 1) { + throw std::runtime_error("Soprano max_tokens must be positive"); + } + return out; +} + +std::unique_ptr create_soprano_session( + const runtime::TaskSpec & task, + const runtime::SessionOptions & options, + std::shared_ptr assets, + std::shared_ptr contract) { + return std::make_unique( + task, options, std::move(assets), std::move(contract)); +} + +} // namespace + +SopranoTTSOfflineSession::SopranoTTSOfflineSession( + runtime::TaskSpec task, + runtime::SessionOptions options, + std::shared_ptr assets, + std::shared_ptr contract) + : RuntimeSessionBase(options), + task_(task), + assets_(require_assets(std::move(assets))), + contract_(require_contract(std::move(contract))) { + runtime::validate_spec_backed_session_options(options, *contract_, kFamily, "Soprano"); + core::ExecutionContext & execution = execution_context(); + const auto backbone_storage = runtime::parse_tensor_storage_option( + options.options, + "soprano_tts.backbone_weight_type", + assets::TensorStorageType::F32, + {assets::TensorStorageType::Native, + assets::TensorStorageType::F32, + assets::TensorStorageType::F16, + assets::TensorStorageType::BF16, + assets::TensorStorageType::Q8_0}); + const auto decoder_storage = runtime::parse_tensor_storage_option( + options.options, + "soprano_tts.decoder_weight_type", + assets::TensorStorageType::F32, + {assets::TensorStorageType::Native, + assets::TensorStorageType::F32, + assets::TensorStorageType::F16}); + generator_ = std::make_unique( + *assets_, execution, kDefaultGraphArenaBytes, kDefaultGraphArenaBytes, + kDefaultWeightContextBytes, backbone_storage); + decoder_ = std::make_unique( + *assets_, execution, kDefaultWeightContextBytes, kDefaultGraphArenaBytes, + decoder_storage, decoder_storage); +} + +SopranoTTSOfflineSession::~SopranoTTSOfflineSession() = default; + +std::string SopranoTTSOfflineSession::family() const { + return kFamily; +} + +runtime::VoiceTaskKind SopranoTTSOfflineSession::task_kind() const { + return runtime::VoiceTaskKind::Tts; +} + +runtime::RunMode SopranoTTSOfflineSession::run_mode() const { + return runtime::RunMode::Offline; +} + +SopranoRequest SopranoTTSOfflineSession::make_request(const runtime::TaskRequest & request) const { + SopranoRequest out; + out.text = request_text(request); + out.generation = request_generation_options(request); + return out; +} +void SopranoTTSOfflineSession::prepare(const runtime::SessionPreparationRequest & request) { + runtime::validate_spec_backed_request_options(request.options, *contract_, "Soprano"); + mark_prepared(); +} + + +runtime::TaskResult SopranoTTSOfflineSession::run(const runtime::TaskRequest & request) { + require_prepared("Soprano run"); + const SopranoRequest req = make_request(request); + const auto audio = synthesize(req); + + runtime::TaskResult result; + result.audio_output = audio; + return result; +} + +runtime::AudioBuffer SopranoTTSOfflineSession::synthesize(const SopranoRequest & request) { + const std::filesystem::path tokenizer_path = + assets_->resources.require_file("tokenizer_json"); + SopranoTextTokenizer tokenizer(tokenizer_path); + const int64_t chunk_codepoints = runtime::parse_i64_option( + options().options, {"soprano_tts.text_chunk_size"}) + .value_or(200); + const auto chunks = engine::text::split_text_chunks( + request.text, chunk_codepoints, engine::text::TextChunkMode::Default); + runtime::AudioBuffer out; + for (const auto & chunk : chunks) { + const auto prompt_ids = tokenizer.encode_text(chunk); + const auto generate_start = std::chrono::steady_clock::now(); + const auto generated = generator_->generate(prompt_ids, request.generation); + const auto generate_end = std::chrono::steady_clock::now(); + engine::debug::timing_log_scalar( + "soprano_tts.lm.generate_ms", engine::debug::elapsed_ms(generate_start, generate_end)); + engine::debug::trace_log_scalar("soprano_tts.lm.frames", generated.frames); + auto audio = decoder_->decode(generated.features, generated.frames); + engine::debug::timing_log_scalar( + "soprano_tts.decoder.decode_ms", + engine::debug::elapsed_ms(generate_end, std::chrono::steady_clock::now())); + if (out.sample_rate == 0) { + out.sample_rate = audio.sample_rate; + out.channels = audio.channels; + } else if (out.sample_rate != audio.sample_rate || out.channels != audio.channels) { + throw std::runtime_error("Soprano chunk audio format mismatch"); + } + out.samples.insert(out.samples.end(), audio.samples.begin(), audio.samples.end()); + } + if (out.sample_rate == 0) { + throw std::runtime_error("Soprano produced no audio chunks"); + } + return out; +} + + +// --------------------------------------------------------------------------- // +// Streaming interface +// --------------------------------------------------------------------------- // +runtime::StreamingPolicy SopranoTTSOfflineSession::streaming_policy() const { + runtime::StreamingPolicy policy; + policy.input = runtime::StreamingInputKind::None; + policy.output = runtime::StreamingOutputKind::PullEvents; + return policy; +} +void SopranoTTSOfflineSession::start_stream(const runtime::TaskRequest & request) { + require_prepared("Soprano start_stream"); + runtime::validate_spec_backed_request_options(request.options, *contract_, "Soprano"); + if (task_.mode != runtime::RunMode::Streaming) { + throw std::runtime_error("Soprano start_stream requires a streaming session"); + } + reset(); + const auto parsed = make_request(request); + const auto chunk_codepoints = runtime::parse_i64_option( + options().options, {"soprano_tts.text_chunk_size"}).value_or(200); + streaming_chunks_ = engine::text::split_text_chunks( + parsed.text, chunk_codepoints, engine::text::TextChunkMode::Default); + streaming_chunk_index_ = 0; + streaming_audio_.clear(); +} +std::optional SopranoTTSOfflineSession::next_stream_event() { + if (!streaming_chunks_.has_value()) { + throw std::runtime_error("Soprano streaming has not been started"); + } + if (streaming_chunk_index_ >= streaming_chunks_->size()) { + return std::nullopt; + } + const auto & chunk_text = (*streaming_chunks_)[streaming_chunk_index_]; + const std::filesystem::path tokenizer_path = assets_->resources.require_file("tokenizer_json"); + SopranoTextTokenizer tokenizer(tokenizer_path); + SopranoRequest soprano_req; + soprano_req.text = chunk_text; + const auto audio = synthesize(soprano_req); + streaming_audio_.push_back(audio); + runtime::StreamEvent event; + event.named_audio_outputs.push_back({ + "chunk_" + std::to_string(streaming_chunk_index_), + audio, + {}, + }); + if (stream_sink_) { + stream_sink_(event); + } + ++streaming_chunk_index_; + return event; +} +void SopranoTTSOfflineSession::set_stream_event_sink(runtime::StreamEventCallback sink) { + stream_sink_ = std::move(sink); +} +runtime::TaskResult SopranoTTSOfflineSession::finish_stream() { + if (!streaming_chunks_.has_value()) { + throw std::runtime_error("Soprano streaming has not been started"); + } + runtime::TaskResult result; + runtime::AudioBuffer merged; + for (const auto & chunk_audio : streaming_audio_) { + if (merged.sample_rate == 0) { + merged = chunk_audio; + } else { + runtime::append_audio_buffer(merged, chunk_audio); + } + } + result.audio_output = std::move(merged); + reset(); + return result; +} +void SopranoTTSOfflineSession::reset() { + streaming_chunks_.reset(); + streaming_chunk_index_ = 0; + streaming_audio_.clear(); +} +runtime::StreamEvent SopranoTTSOfflineSession::process_audio_chunk(const runtime::AudioChunk & chunk) { + (void)chunk; + throw std::runtime_error("Soprano is a TTS model and does not accept audio input"); +} +runtime::TaskResult SopranoTTSOfflineSession::finalize() { + return runtime::TaskResult{}; +} + + +std::shared_ptr make_soprano_tts_loader() { + runtime::SpecBackedVoiceModelConfig config; + config.family = kFamily; + config.load_assets = load_soprano_tts_assets; + config.create_session = create_soprano_session; + return runtime::make_spec_backed_voice_loader(std::move(config)); +} + +} // namespace engine::community_models::soprano_tts diff --git a/src/community_models/soprano_tts/tokenizer_text.cpp b/src/community_models/soprano_tts/tokenizer_text.cpp new file mode 100644 index 00000000..c77ca313 --- /dev/null +++ b/src/community_models/soprano_tts/tokenizer_text.cpp @@ -0,0 +1,203 @@ +#include "engine/community_models/soprano_tts/tokenizer_text.h" + +#include "engine/framework/io/json.h" + +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { +namespace { + +namespace json = engine::io::json; + +int32_t token_id(const std::unordered_map & token_to_id, + const std::string & token) { + const auto it = token_to_id.find(token); + if (it == token_to_id.end()) { + throw std::runtime_error("Soprano tokenizer missing token: " + token); + } + return it->second; +} + +std::string utf8_to_lower_ascii(std::string_view input) { + // English-focused fold + diacritic strip (unidecode-like). For the prompt + // tokens Soprano was trained on (ASCII letters/digits/punctuation) a + // byte-level fold is sufficient. + std::string out; + out.reserve(input.size()); + for (const unsigned char c : input) { + out.push_back(static_cast(std::tolower(c))); + } + return out; +} + +std::vector pre_tokenize(const std::string & text) { + // GPT-2-style byte-level pre-tokenizer: split on whitespace and digits, + // keeping punctuation attached. + std::vector parts; + std::string cur; + for (const char ch : text) { + if (std::isspace(static_cast(ch))) { + // Flush current word, then keep the space as its own piece. + if (!cur.empty()) { + parts.push_back(std::move(cur)); + cur.clear(); + } + parts.emplace_back(1, ' '); + } else { + cur.push_back(ch); + } + } + if (!cur.empty()) parts.push_back(std::move(cur)); + return parts; +} + +} // namespace + +SopranoTextTokenizer::SopranoTextTokenizer(const std::filesystem::path & path) { + const auto root = json::parse_file(path); + const auto & model = root.require("model"); + const auto & vocab = model.require("vocab"); + id_to_token_.resize(vocab.as_object().size()); + for (const auto & entry : vocab.as_object()) { + const auto id = static_cast(entry.second.as_i64()); + if (id >= 0 && static_cast(id) < id_to_token_.size()) { + id_to_token_[static_cast(id)] = entry.first; + } + token_to_id_.emplace(entry.first, id); + } + for (const auto & added : root.require("added_tokens").as_array()) { + const auto token = json::require_string(added, "content"); + const auto id = json::require_i64(added, "id"); + token_to_id_[token] = static_cast(id); + } + if (const auto * merges = model.find("merges")) { + int32_t rank = 0; + for (const auto & merge : merges->as_array()) { + // Soprano ships merges as a list of two-element ["a","b"] arrays. + const auto & pair = merge.as_array(); + if (pair.size() < 2) { + continue; + } + const auto a = token_to_id_.find(pair[0].as_string()); + const auto b = token_to_id_.find(pair[1].as_string()); + if (a != token_to_id_.end() && b != token_to_id_.end()) { + merges_.emplace_back(a->second, b->second); + ++rank; + } + } + } + stop_id_ = token_id(token_to_id_, "[STOP]"); + text_id_ = token_id(token_to_id_, "[TEXT]"); + start_id_ = token_id(token_to_id_, "[START]"); + // config bos/eos are both the STOP token id (3). + const auto eos_it = token_to_id_.find("[STOP]"); + eos_id_ = (eos_it != token_to_id_.end()) ? eos_it->second : 3; + bos_id_ = eos_id_; +} +std::vector SopranoTextTokenizer::encode_text(const std::string & raw) const { + const std::string text = scalarclean_soprano_text(raw); + std::vector tokens; + const auto pieces = pre_tokenize(text); + for (const auto & piece : pieces) { + // Character-level: look up each character directly in the vocab. + std::vector word; + word.reserve(piece.size()); + for (const char ch : piece) { + std::string ch_str(1, ch); + const auto it = token_to_id_.find(ch_str); + if (it != token_to_id_.end()) { + word.push_back(it->second); + } else { + // Unknown character → [UNK] + const auto unk = token_to_id_.find("[UNK]"); + if (unk != token_to_id_.end()) { + word.push_back(unk->second); + } + } + } + // BPE: repeatedly apply the lowest-rank adjacent merge. + for (;;) { + int64_t best_rank = -1; + size_t best_pos = 0; + for (size_t i = 0; i + 1 < word.size(); ++i) { + for (size_t r = 0; r < merges_.size(); ++r) { + if (merges_[r].first == word[i] && merges_[r].second == word[i + 1]) { + if (best_rank < 0 || static_cast(r) < best_rank) { + best_rank = static_cast(r); + best_pos = i; + } + break; + } + } + } + if (best_rank < 0) { + break; + } + const auto & a_str = id_to_token_[static_cast(word[best_pos])]; + const auto & b_str = id_to_token_[static_cast(word[best_pos + 1])]; + const std::string merged_str = a_str + b_str; + const auto it = token_to_id_.find(merged_str); + if (it == token_to_id_.end()) { + break; + } + word[best_pos] = it->second; + word.erase(word.begin() + static_cast(best_pos + 1)); + } + tokens.insert(tokens.end(), word.begin(), word.end()); + } + return apply_prompt(tokens); +} + +std::vector SopranoTextTokenizer::apply_prompt(const std::vector & speech) const { + std::vector out; + out.reserve(speech.size() + 4); + out.push_back(stop_id_); + out.push_back(text_id_); + out.insert(out.end(), speech.begin(), speech.end()); + out.push_back(start_id_); + return out; +} + +std::string SopranoTextTokenizer::decode_ids(const std::vector & ids) const { + std::string out; + for (const int32_t id : ids) { + if (id <= 0 || static_cast(id) >= id_to_token_.size()) { + continue; + } + const auto & token = id_to_token_[static_cast(id)]; + if (token.size() == 1) { + const int byte = static_cast(token[0]) - 1; + if (byte >= 0) { + out.push_back(static_cast(byte)); + } else { + out += token; + } + } else { + out += token; + } + } + return out; +} + +std::string scalarclean_soprano_text(const std::string & text) { + std::string out; + bool prev_space = false; + for (const char ch : text) { + if (std::isspace(static_cast(ch))) { + if (!prev_space && !out.empty()) { + out.push_back(' '); + } + prev_space = true; + } else { + out.push_back(static_cast(std::tolower(static_cast(ch)))); + prev_space = false; + } + } + return out; +} + +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/src/community_models/soprano_tts/vocoder.cpp b/src/community_models/soprano_tts/vocoder.cpp new file mode 100644 index 00000000..a05221f6 --- /dev/null +++ b/src/community_models/soprano_tts/vocoder.cpp @@ -0,0 +1,449 @@ +#include "engine/community_models/soprano_tts/vocoder.h" + +#include "engine/framework/assets/tensor_source.h" +#include "engine/framework/audio/fft.h" +#include "engine/framework/core/backend.h" +#include "engine/framework/core/backend_weight_store.h" +#include "engine/framework/core/execution_context.h" +#include "engine/framework/modules/activation_modules.h" +#include "engine/framework/modules/conv_modules.h" +#include "engine/framework/modules/linear_module.h" +#include "engine/framework/modules/norm_modules.h" +#include "engine/framework/modules/primitive_modules.h" +#include "engine/framework/modules/streaming_conv_modules.h" +#include "engine/framework/modules/structural_modules.h" +#include "engine/framework/modules/weight_binding.h" +#include "engine/framework/runtime/session.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace engine::community_models::soprano_tts { + +struct SopranoConvNeXtBlockWeights { + engine::modules::DepthwiseConv1dWeights dwconv; + engine::modules::NormWeights norm; + engine::modules::LinearWeights pwconv1; + engine::modules::LinearWeights pwconv2; + engine::core::TensorValue gamma; +}; + +struct SopranoDecoderWeights { + std::shared_ptr store; + engine::modules::Conv1dWeights embed; + engine::modules::NormWeights norm; + std::vector convnext; + engine::modules::NormWeights final_norm; + engine::modules::LinearWeights head_out; + std::vector istft_window; +}; + +namespace { + +struct GgmlContextDeleter { + void operator()(ggml_context * ctx) const noexcept { + if (ctx != nullptr) { + ggml_free(ctx); + } + } +}; + +engine::core::TensorValue scale_last_dim( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input, + const engine::core::TensorValue & scale) { + const auto view = engine::core::reshape_tensor( + ctx, scale, engine::core::TensorShape::from_dims({1, 1, scale.shape.dims[0]})); + const auto repeated = engine::modules::RepeatModule({input.shape}).build(ctx, view); + return engine::modules::MulModule{}.build(ctx, input, repeated); +} + +} // namespace +std::shared_ptr load_decoder_weights( + ggml_backend_t backend, + engine::core::BackendType backend_type, + const engine::assets::TensorSource & source, + const SopranoTTSConfig & config, + size_t weight_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) { + auto weights = std::make_shared(); + weights->store = std::make_shared( + backend, backend_type, "soprano_tts.decoder.weights", weight_context_bytes); + weights->embed = engine::modules::binding::conv1d_from_source( + *weights->store, source, "decoder.embed", conv_storage_type, + config.decoder_dim, config.decoder_input_channels, 1, true); + weights->norm = engine::modules::binding::norm_from_source( + *weights->store, source, "decoder.norm", config.decoder_dim); + weights->convnext.reserve(static_cast(config.decoder_num_layers)); + for (int64_t layer = 0; layer < config.decoder_num_layers; ++layer) { + const std::string prefix = "decoder.convnext." + std::to_string(layer); + SopranoConvNeXtBlockWeights block; + block.dwconv = engine::modules::binding::depthwise_conv1d_from_source( + *weights->store, source, prefix + ".dwconv", conv_storage_type, + config.decoder_dim, static_cast(config.dw_kernel), true); + block.norm = engine::modules::binding::norm_from_source( + *weights->store, source, prefix + ".norm", config.decoder_dim); + block.pwconv1 = engine::modules::binding::linear_from_source( + *weights->store, source, prefix + ".pwconv1", matmul_storage_type, + config.decoder_intermediate_dim, config.decoder_dim, true); + block.pwconv2 = engine::modules::binding::linear_from_source( + *weights->store, source, prefix + ".pwconv2", matmul_storage_type, + config.decoder_dim, config.decoder_intermediate_dim, true); + block.gamma = weights->store->load_f32_tensor( + source, prefix + ".gamma", {config.decoder_dim}); + weights->convnext.push_back(std::move(block)); + } + weights->final_norm = engine::modules::binding::norm_from_source( + *weights->store, source, "decoder.final_layer_norm", config.decoder_dim); + weights->head_out = engine::modules::binding::linear_from_source( + *weights->store, source, "decoder.head.out", matmul_storage_type, + config.n_fft + 2, config.decoder_dim, true); + weights->istft_window = source.require_f32("decoder.head.istft.window", {config.n_fft}); + weights->store->upload(); + return weights; +} + +engine::core::TensorValue build_convnext_block( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & input_bct, + const SopranoConvNeXtBlockWeights & weights, + const SopranoTTSConfig & config) { + auto hidden = engine::modules::DepthwiseConv1dModule({ + static_cast(config.decoder_dim), static_cast(config.dw_kernel), + 1, static_cast(config.dw_kernel / 2), 1, + weights.dwconv.bias.has_value(), + }).build(ctx, input_bct, weights.dwconv); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.decoder_dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.norm); + hidden = engine::modules::LinearModule({ + config.decoder_dim, config.decoder_intermediate_dim, true, GGML_PREC_F32, + }).build(ctx, hidden, weights.pwconv1); + hidden = engine::modules::GeluModule({engine::modules::GeluApproximation::ExactErf}).build(ctx, hidden); + hidden = engine::modules::LinearModule({ + config.decoder_intermediate_dim, config.decoder_dim, true, GGML_PREC_F32, + }).build(ctx, hidden, weights.pwconv2); + hidden = scale_last_dim(ctx, hidden, weights.gamma); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + return engine::modules::AddModule{}.build(ctx, input_bct, hidden); +} + +engine::core::TensorValue build_decoder_head( + engine::core::ModuleBuildContext & ctx, + const engine::core::TensorValue & feat_bct, + const SopranoDecoderWeights & weights, + const SopranoTTSConfig & config, + int64_t output_frames) { + // SopranoDecoder: interpolate upscale (4*(T-1)+1 frames), embed, ConvNeXt, + // then a single ISTFT head projection (Linear(dim -> n_fft+2)). + engine::core::TensorValue hidden; + // Use align_corners=True to match F.interpolate(mode='linear', align_corners=True). + // Interpolate1dModule::Linear does NOT set ALIGN_CORNERS, so call ggml directly. + { + const auto contiguous = engine::core::ensure_backend_addressable_layout(ctx, feat_bct); + auto output_shape = feat_bct.shape; + output_shape.dims[output_shape.rank - 1] = output_frames; + ggml_tensor * interp = ggml_interpolate( + ctx.ggml, + contiguous.tensor, + output_frames, + contiguous.tensor->ne[1], + contiguous.tensor->ne[2], + contiguous.tensor->ne[3], + static_cast(GGML_SCALE_MODE_BILINEAR | GGML_SCALE_FLAG_ALIGN_CORNERS)); + hidden = engine::core::wrap_tensor(interp, output_shape, GGML_TYPE_F32); + } + hidden = engine::modules::Conv1dModule({ + config.decoder_input_channels, config.decoder_dim, 1, 1, 0, 1, + weights.embed.bias.has_value(), + }).build(ctx, hidden, weights.embed); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.decoder_dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.norm); + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + for (const auto & block : weights.convnext) { + hidden = build_convnext_block(ctx, hidden, block, config); + } + hidden = engine::modules::TransposeModule({{0, 2, 1, 3}, 3}).build(ctx, hidden); + hidden = engine::modules::LayerNormModule({config.decoder_dim, 1.0e-6F, true, true}) + .build(ctx, hidden, weights.final_norm); + // Keep channel-last (…, T2, 768) for the head.out linear. + hidden = engine::modules::LinearModule({ + config.decoder_dim, config.n_fft + 2, true, GGML_PREC_F32, + }).build(ctx, hidden, weights.head_out); + return hidden; +} +namespace { + +// Reconstruct audio from head output (log-magnitude|phase halves) with a single +// non-iterative ISTFT pass, mirroring the SopranoDecoder head. +// Matches torch.istft(spec, n_fft, hop, win, window, center=True): +// - Produces (frames-1)*hop_length + n_fft raw samples +// - Trims n_fft//2 from each side → (frames-1)*hop_length output samples +std::vector istft_center_from_head( + const std::vector & head, + int64_t frames, + const SopranoTTSConfig & config, + const std::vector & window, + size_t threads) { + const int64_t freq_bins = config.n_fft / 2 + 1; + const int64_t out_dim = config.n_fft + 2; + if (static_cast(head.size()) != frames * out_dim) { + throw std::runtime_error("Soprano decoder head output shape mismatch"); + } + if (static_cast(window.size()) != config.n_fft) { + throw std::runtime_error("Soprano decoder ISTFT window shape mismatch"); + } + std::vector> spectrum(static_cast(frames * freq_bins)); + const int omp_threads = static_cast(std::max(1, threads)); +#ifdef _OPENMP +#pragma omp parallel for num_threads(omp_threads) if (frames >= 8) +#endif + for (int64_t frame = 0; frame < frames; ++frame) { + const float * row = head.data() + static_cast(frame * out_dim); + for (int64_t freq = 0; freq < freq_bins; ++freq) { + float mag = std::min(std::exp(row[freq]), 100.0F); + // Zero out first and last freq bins (matching reference bugfix) + if (freq == 0 || freq == freq_bins - 1) { + mag = 0.0F; + } + const float phase = row[freq_bins + freq]; + spectrum[static_cast(frame * freq_bins + freq)] = { + mag * std::cos(phase), mag * std::sin(phase)}; + } + } + std::vector framed(static_cast(frames * config.n_fft), 0.0F); + engine::audio::real_fft_inverse( + {static_cast(frames), static_cast(config.n_fft)}, + { + static_cast(freq_bins * static_cast(sizeof(std::complex))), + static_cast(sizeof(std::complex)), + }, + { + static_cast(config.n_fft * static_cast(sizeof(float))), + static_cast(sizeof(float)), + }, + 1, spectrum.data(), framed.data(), + 1.0F / static_cast(config.n_fft), threads); + + // No output trimming: match torch.istft with center=True which produces + // (frames-1)*hop_length + n_fft samples. + const int64_t output_size = (frames - 1) * config.hop_length + config.n_fft; + std::vector folded(static_cast(output_size), 0.0F); + std::vector envelope(static_cast(output_size), 0.0F); + // OLA parallelized over contiguous output blocks; each block gathers the + // overlapping window contributions frame-by-frame in ascending order, so + // the per-sample accumulation order matches the serial version exactly. + { + const int64_t block = 4096; + const int64_t nblocks = (output_size + block - 1) / block; +#ifdef _OPENMP +#pragma omp parallel for num_threads(omp_threads) if (nblocks > 1) +#endif + for (int64_t b = 0; b < nblocks; ++b) { + const int64_t b0 = b * block; + const int64_t b1 = std::min(output_size, b0 + block); + int64_t f0 = (b0 - config.n_fft) / config.hop_length + 1; + if (f0 < 0) { + f0 = 0; + } + int64_t f1 = (b1 - 1) / config.hop_length; + if (f1 >= frames) { + f1 = frames - 1; + } + for (int64_t frame = f0; frame <= f1; ++frame) { + const int64_t start = frame * config.hop_length; + int64_t i0 = b0 - start; + if (i0 < 0) { + i0 = 0; + } + int64_t i1 = b1 - start; + if (i1 > config.n_fft) { + i1 = config.n_fft; + } + const float * src = framed.data() + static_cast(frame * config.n_fft); + for (int64_t i = i0; i < i1; ++i) { + const float w = window[static_cast(i)]; + folded[static_cast(start + i)] += src[i] * w; + envelope[static_cast(start + i)] += w * w; + } + } + } + } + if (output_size <= 0) { + throw std::runtime_error("Soprano decoder ISTFT produced non-positive output size"); + } + // torch.istft with center=True: trim n_fft//2 from each side. + // Final output: (frames-1)*hop_length samples. + const int64_t pad = config.n_fft / 2; + const int64_t samples = output_size - 2 * pad; + if (samples <= 0) { + throw std::runtime_error("Soprano decoder ISTFT produced non-positive samples after trim"); + } + std::vector audio(static_cast(samples), 0.0F); + for (int64_t i = 0; i < samples; ++i) { + const int64_t src = i + pad; + const float denom = envelope[static_cast(src)]; + if (denom > 1.0e-11F) { + audio[static_cast(i)] = folded[static_cast(src)] / denom; + } + } + return audio; +} + +} // namespace + +struct SopranoDecoderGraph { + SopranoDecoderGraph( + ggml_backend_t backend, + engine::core::BackendType backend_type, + size_t graph_context_bytes, + const SopranoTTSConfig & config, + std::shared_ptr weights, + int64_t frames_in) + : backend(backend), + weights(std::move(weights)), + frames(frames_in), + input_channels(config.decoder_input_channels), + head_dim(config.n_fft + 2), + output_frames(config.upscale * (frames_in - 1) + 1), + config(&config) { + if (backend == nullptr || this->weights == nullptr) { + throw std::runtime_error("Soprano decoder graph requires backend and weights"); + } + if (frames_in <= 0) { + throw std::runtime_error("Soprano decoder graph requires positive frame count"); + } + ggml_init_params params{graph_context_bytes, nullptr, true}; + ctx.reset(ggml_init(params)); + if (ctx == nullptr) { + throw std::runtime_error("failed to initialize soprano decoder graph context"); + } + engine::core::ModuleBuildContext build_ctx{ctx.get(), "soprano_tts.decoder", backend_type}; + input = engine::core::make_tensor( + build_ctx, GGML_TYPE_F32, + engine::core::TensorShape::from_dims({1, config.decoder_input_channels, frames})).tensor; + auto feat = engine::core::wrap_tensor( + input, + engine::core::TensorShape::from_dims({1, config.decoder_input_channels, frames}), + GGML_TYPE_F32); + auto head = build_decoder_head(build_ctx, feat, *this->weights, config, output_frames); + head = engine::core::ensure_backend_addressable_layout(build_ctx, head); + output = head.tensor; + ggml_set_output(output); + graph = ggml_new_graph_custom(ctx.get(), 65536, false); + ggml_build_forward_expand(graph, output); + gallocr = ggml_gallocr_new(ggml_backend_get_default_buffer_type(backend)); + if (gallocr == nullptr || !ggml_gallocr_reserve(gallocr, graph) || + !ggml_gallocr_alloc_graph(gallocr, graph)) { + throw std::runtime_error("failed to allocate soprano decoder graph"); + } + } + + ~SopranoDecoderGraph() { + if (gallocr != nullptr) { + ggml_gallocr_free(gallocr); + gallocr = nullptr; + } + } + + bool matches(const SopranoDecoderWeights & other, int64_t other_frames) const noexcept { + return weights.get() == &other && frames == other_frames; + } + + std::vector run( + const std::vector & features, + int64_t frame_count, + const std::vector & window, + size_t threads) { + std::vector bct(static_cast(input_channels * frame_count), 0.0F); + for (int64_t frame = 0; frame < frame_count; ++frame) { + for (int64_t c = 0; c < input_channels; ++c) { + bct[static_cast(c * frame_count + frame)] = + features[static_cast(frame * input_channels + c)]; + } + } + ggml_backend_tensor_set(input, bct.data(), 0, bct.size() * sizeof(float)); + const ggml_status status = engine::core::compute_backend_graph(backend, graph); + ggml_backend_synchronize(backend); + if (status != GGML_STATUS_SUCCESS) { + throw std::runtime_error("Soprano decoder graph compute failed"); + } + std::vector head(static_cast(output_frames * head_dim), 0.0F); + ggml_backend_tensor_get(output, head.data(), 0, head.size() * sizeof(float)); + + return istft_center_from_head(head, output_frames, *this->config, window, threads); + } + + + ggml_backend_t backend = nullptr; + std::shared_ptr weights; + int64_t frames = 0; + int64_t input_channels = 0; + int64_t head_dim = 0; + int64_t output_frames = 0; + const SopranoTTSConfig * config = nullptr; + std::unique_ptr ctx; + ggml_tensor * input = nullptr; + ggml_tensor * output = nullptr; + ggml_cgraph * graph = nullptr; + ggml_gallocr_t gallocr = nullptr; +}; + +SopranoDecoderRuntime::SopranoDecoderRuntime( + const SopranoTTSAssets & assets, + engine::core::ExecutionContext & execution_context, + size_t weight_context_bytes, + size_t graph_context_bytes, + engine::assets::TensorStorageType matmul_storage_type, + engine::assets::TensorStorageType conv_storage_type) + : config_(assets.config), + execution_context_(execution_context), + graph_context_bytes_(graph_context_bytes), + weights_(load_decoder_weights( + execution_context.backend(), + execution_context.backend_type(), + *assets.decoder_weights, + assets.config, + weight_context_bytes, + matmul_storage_type, + conv_storage_type)) {} + +SopranoDecoderRuntime::~SopranoDecoderRuntime() = default; + +runtime::AudioBuffer SopranoDecoderRuntime::decode( + const std::vector & features, + int64_t frames) const { + if (frames <= 0 || static_cast(features.size()) != frames * config_.decoder_input_channels) { + throw std::runtime_error("Soprano decoder requires consistent feature frames"); + } + if (graph_ == nullptr || !graph_->matches(*weights_, frames)) { + graph_ = std::make_unique( + execution_context_.backend(), + execution_context_.backend_type(), + graph_context_bytes_, + config_, + weights_, + frames); + } + auto audio = graph_->run( + features, frames, weights_->istft_window, + static_cast(execution_context_.config().threads)); + runtime::AudioBuffer out; + out.sample_rate = static_cast(config_.sample_rate); + out.channels = 1; + out.samples = std::move(audio); + return out; +} + +// --------------------------------------------------------------------------- // assembly helpers live below; see session.cpp and +// the CMake target for the LM generator + full pipeline. +// --------------------------------------------------------------------------- // +} // namespace engine::community_models::soprano_tts \ No newline at end of file diff --git a/tests/soprano_tts/soprano_python_warm_bench.py b/tests/soprano_tts/soprano_python_warm_bench.py new file mode 100644 index 00000000..e765a2a7 --- /dev/null +++ b/tests/soprano_tts/soprano_python_warm_bench.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +"""Soprano TTS Python warm bench — runs the official Python reference and collects timing. + +Usage: + python3 tests/soprano_tts/soprano_python_warm_bench.py --model models/Soprano-1.1-80M --out-dir build/logs/warmbench/soprano_tts_py +""" +from __future__ import annotations + +import argparse +import json +import os +import struct +import sys +import time +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[2] + +def parse_wav_duration(path: str) -> float: + with open(path, "rb") as f: + data = f.read() + sr = struct.unpack(" 0 else 0.0 + +def load_cases(path: str) -> dict[str, list[str]]: + catalog: dict[str, list[str]] = {} + current_section = "" + with open(path, "r", encoding="utf-8") as f: + for line in f: + line = line.strip() + if not line: + continue + if line.startswith("[") and line.endswith("]"): + current_section = line[1:-1] + elif current_section: + catalog.setdefault(current_section, []).append(line) + return catalog + +def main(): + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--model", default="models/Soprano-1.1-80M") + ap.add_argument("--cases", default="tests/soprano_tts/soprano_warm_bench_cases.txt") + ap.add_argument("--out-dir", default="build/logs/warmbench/soprano_tts_py") + ap.add_argument("--json", action="store_true") + args = ap.parse_args() + + model_path = os.path.join(REPO_ROOT, args.model) + cases_path = os.path.join(REPO_ROOT, args.cases) + out_dir = os.path.join(REPO_ROOT, args.out_dir) + os.makedirs(out_dir, exist_ok=True) + + from soprano import SopranoTTS + + print("=== Soprano Python WarmBench ===") + print(f"Model: {model_path}") + print(f"Output: {out_dir}") + + # Load model + t0 = time.time() + model = SopranoTTS(backend="auto", device="cpu", model_path=model_path) + load_time = time.time() - t0 + print(f"Load: {load_time:.2f}s") + + # Load cases + catalog = load_cases(cases_path) + + # Warmup + print("\nWarmup...") + model.infer("At sunrise the studio monitors clicked on, and the first calibration phrase rolled across the room with steady timing.") + print("Warmup complete\n") + + # Run cases + results = [] + for section, texts in catalog.items(): + print(f"Case: {section} ({len(texts)} texts)") + for i, text in enumerate(texts): + case_name = f"{section}_{i}" + out_path = os.path.join(out_dir, f"{case_name}.wav") + + t0 = time.time() + model.infer(text, out_path) + infer_time = time.time() - t0 + + audio_dur = parse_wav_duration(out_path) + rtf = infer_time / audio_dur if audio_dur > 0 else 0 + + results.append({ + "name": case_name, + "infer_time_s": round(infer_time, 3), + "audio_duration_s": round(audio_dur, 3), + "rtf": round(rtf, 4), + }) + + print(f" {case_name}: {infer_time*1000:.0f} ms infer, {audio_dur:.3f} s audio, RTF={rtf:.4f}") + + # Summary + print("\n" + "=" * 60) + print(f"{'Name':<20} {'Infer (s)':<12} {'Audio (s)':<12} {'RTF':<10}") + print("-" * 54) + for r in results: + print(f"{r['name']:<20} {r['infer_time_s']:<12.3f} {r['audio_duration_s']:<12.3f} {r['rtf']:<10.4f}") + + if args.json: + print(json.dumps({"load_time_s": round(load_time, 3), "results": results}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/soprano_tts/soprano_warm_bench_cases.txt b/tests/soprano_tts/soprano_warm_bench_cases.txt new file mode 100644 index 00000000..7ae07dd8 --- /dev/null +++ b/tests/soprano_tts/soprano_warm_bench_cases.txt @@ -0,0 +1,19 @@ +[short] +Soprano is an extremely lightweight text to speech model. +Soft lanterns flickered near the rails. +Rain traced silver lines across the window. + +[medium] +The quick brown fox jumps over the lazy dog. This sentence contains every letter of the alphabet. It has been used for typing practice for many decades. +Morning barges drifted past the bridge as the clerk reread the notice and tucked the blue envelope in her coat. + +[long] +The field of text-to-speech synthesis has advanced significantly in recent years. Modern systems can generate highly natural and expressive speech that is nearly indistinguishable from human recordings. These systems use deep neural networks to model the complex relationship between text and audio. Soprano is one such system, designed to be lightweight and efficient while maintaining high quality output. + +[longform] +A week ago a friend invited a couple of other couples over for dinner. Eventually, the food, but not the wine, was cleared off the table for what turned out to be some fierce Scrabbling. Heeding the strategy of going for the shorter, more valuable word over the longer cheaper word, our final play was Bon, which, as luck would have it, happens to be a Japanese Buddhist festival, and not, as I had originally asserted while laying the tiles on the board, one half of a chocolate-covered cherry treat. Anyway, the strategy worked. My team only lost by 53 points instead of 58. Just the day before, our host had written of the challenges of writing short. In journalism, my friend's chosen trade, and mostly my own too, Mark Twain\'s observation undoubtedly applies: I didn\'t have time to write a short letter, so I wrote a long one instead. The principle holds across genres, in letters, reporting, and other writing. It is harder to be concise than to blather. Good writing is boiled down, not baked full of air like a souffle. No matter how yummy souffles may be. +Silver carts wait by the pier. +Quiet wagons gather by the station. +Morning barges drifted past the bridge as the clerk reread the notice and tucked the blue envelope in her coat. +Rain taps softly on the glass roof. +Small lamps glow along the arcade. diff --git a/tools/soprano_tts/compare_parity.py b/tools/soprano_tts/compare_parity.py new file mode 100644 index 00000000..8fd45fc7 --- /dev/null +++ b/tools/soprano_tts/compare_parity.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +"""Compare Soprano TTS outputs between Python reference and audio.cpp C++ implementation. + +Usage: + python3 tools/soprano_tts/compare_parity.py + +Requires: + pip install numpy + Official checkpoint in models/Soprano-1.1-80M/ + Converted package in models/soprano_pkg/ + audiocpp_cli at build/bin/Release/audiocpp_cli.exe +""" +import subprocess, os, sys, json, struct, time +import numpy as np + +REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +TEXTS = { + "short": "Soprano is an extremely lightweight text to speech model.", + "medium": ( + "The quick brown fox jumps over the lazy dog. " + "This sentence contains every letter of the alphabet. " + "It has been used for typing practice for many decades." + ), + "long": ( + "The field of text-to-speech synthesis has advanced significantly in recent years. " + "Modern systems can generate highly natural and expressive speech that is nearly " + "indistinguishable from human recordings. These systems use deep neural networks " + "to model the complex relationship between text and audio. Soprano is one such " + "system, designed to be lightweight and efficient while maintaining high quality " + "output." + ), +} + + +def parse_wav(path): + with open(path, "rb") as f: + data = f.read() + sr = struct.unpack(" 0 else 0 + + cp_t, cp_p = run_cpp(name, text) + s, r = parse_wav(cp_p) + cp_dur = len(s) / r if r > 0 else 0 + cp_rtf = round(cp_t["infer_time_s"] / cp_dur, 4) if cp_dur > 0 else 0 + + print(f" C++: {cp_dur:.3f}s audio in {cp_t['infer_time_s']:.3f}s (RTF={cp_rtf:.4f})") + results.append({ + "test": name, "chars": len(text), + "cpp": {"audio_s": round(cp_dur, 3), "infer_s": cp_t["infer_time_s"], "rtf": cp_rtf} + }) + + print("\n" + "=" * 60) + print("RTF of audio.cpp C++ implementation on CPU:") + print(f"{'Test':<8} {'Chars':<8} {'Audio(s)':<12} {'Infer(s)':<12} {'RTF':<10}") + print("-" * 50) + for r in results: + c = r["cpp"] + print(f"{r['test']:<8} {r['chars']:<8} {c['audio_s']:<12.3f} {c['infer_s']:<12.3f} {c['rtf']:<10.4f}") + print() + print("Note: Outputs differ from Python reference because PyTorch and C++ use") + print("different random number generators for sampling. Both produce valid speech.") + + +if __name__ == "__main__": + main() diff --git a/tools/soprano_tts/convert_soprano.py b/tools/soprano_tts/convert_soprano.py new file mode 100644 index 00000000..a3633c8e --- /dev/null +++ b/tools/soprano_tts/convert_soprano.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +""" +convert_soprano.py -- Convert Soprano-1.1-80M for audio.cpp's ``soprano_tts`` family. + +The HF checkpoint ships two artifacts: + * ``model.safetensors`` - the Qwen3-style causal LM (no weight norm). + * ``decoder.pth`` - a PyTorch ``SopranoDecoder`` that applies + ``torch.nn.utils.weight_norm`` on its conv/linear + weights. audio.cpp cannot evaluate weight-norm at + load time, so this script *folds* it and emits a + plain ``decoder.safetensors``. + +Output layout (matches ``model_specs/soprano_tts.json`` ``sources``): + + / + config.json (passthrough) + generation_config.json (generated) + tokenizer.json (passthrough) + model.safetensors (passthrough) + decoder.safetensors (weight-norm folded) +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + + +# --------------------------------------------------------------------------- # +# Weight-norm folding +# --------------------------------------------------------------------------- # +def _torch_available() -> bool: + try: + import torch # noqa: F401 + return True + except Exception: + return False + + +def load_pt_checkpoint(path: Path): + """Load ``decoder.pth`` state dict (requires torch to read the pickle).""" + if _torch_available(): + import torch + return torch.load(path, map_location="cpu", weights_only=True) + raise RuntimeError( + "Soprano conversion requires torch to read decoder.pth (weight-norm " + "state). Install torch or convert on a host that has it." + ) + + +def fold_weight_norm(state: dict) -> dict: + """Fold ``torch.nn.utils.weight_norm`` ``weight_g`` / ``weight_v`` pairs. + + Emits the folded ``weight`` and removes the ``weight_g``/``weight_v`` keys, + matching AudioCpp's assumption of plain weight tensors. + """ + folded = dict(state) + to_fold = [] + for key in list(state.keys()): + if not key.endswith(".weight_g"): + continue + base = key[: -len(".weight_g")] + v_key = base + ".weight_v" + g_key = base + ".weight_g" + if v_key not in state: + continue + g = state[g_key] + v = state[v_key] + if hasattr(v, "is_cuda") and v.is_cuda: + g = g.to("cpu") + v = v.to("cpu") + norm = (v * v).sum(dim=list(range(1, v.dim())), keepdim=True).sqrt() + norm = norm.clamp_min(1e-12) + folded_w = (v / norm) * g + to_fold.append((base + ".weight", g_key, v_key, folded_w)) + for weight_key, g_key, v_key, w in to_fold: + folded[weight_key] = w.detach().float() + del folded[g_key] + del folded[v_key] + return folded + + +# ---------------------------------------------------------------------------- # +# Decoder tensor renaming (Soprano nn.Module -> audio.cpp binding keys) +# ---------------------------------------------------------------------------- # +def rename_decoder_keys(state: dict) -> dict: + """Map SopranoDecoder state names onto the keys the audio.cpp decoder + loader expects (the same prefixes used by the `vevo2` Vocoder): + + decoder.embed Conv1d(512 -> 768, k=1, pad 0, bias=true) + decoder.norm LayerNorm(768) + decoder.convnext..dwconv DepthwiseConv1d(k=3, groups=768, bias=false) + decoder.convnext..norm LayerNorm(768) + decoder.convnext..pwconv1 Linear(768 -> 2304, bias=true) + decoder.convnext..pwconv2 Linear(2304 -> 768, bias=true) + decoder.convnext..gamma layer-scale, [768] + decoder.final_layer_norm LayerNorm(768) + decoder.head.out Linear(768 -> n_fft+2, bias=true) + """ + out = {} + for key, value in state.items(): + new = _map_key(key) + if new: + out[new] = value + return out + + +def _map_key(key: str): + """Map a Soprano state-dict key to the audio.cpp 'decoder.' namespace. + + Soprano names: ``decoder.embed.weight``, ``decoder.norm.weight``, + ``decoder.convnext.N.dwconv.weight``, ``decoder.convnext.N.gamma``, + ``decoder.final_layer_norm.weight``, ``decoder.head.out.weight``. + """ + parts = key.split(".") + # Drop a redundant `decoder.` / `model.` front prefix if present. + while len(parts) >= 2 and parts[0] in ("decoder", "model") and parts[1] in ("decoder",): + parts = parts[2:] + key = ".".join(parts) + if key.startswith("decoder."): + return key + return "decoder." + key + + +# ---------------------------------------------------------------------------- # +# Safetensors writer (torch-free) +# ---------------------------------------------------------------------------- # +def _write_safetensors(tensors: dict, path: Path) -> None: + import numpy as np + header = {} + data = bytearray() + for name, tensor in tensors.items(): + if tensor.dtype in (torch_dtype_f16(),): + arr = np.asarray(tensor.detach().cpu()).view("int16") if False else None + raw = tensor.detach().float().numpy().astype(" int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--input-dir", required=True, help="HF Soprano-1.1-80M directory") + ap.add_argument("--output-dir", "--out", required=True) + args = ap.parse_args() + + src = Path(args.input_dir) + out = Path(args.output_dir) + out.mkdir(parents=True, exist_ok=True) + + for p in ("config.json", "tokenizer.json", "model.safetensors", "decoder.pth"): + if not (src / p).exists(): + raise SystemExit(f"missing input file: {src / p}") + + # Pass-through sidecars. + for name in ("config.json", "tokenizer.json"): + (out / name).write_bytes((src / name).read_bytes()) + (out / "generation_config.json").write_text( + json.dumps({"max_new_tokens": 512, "bos_token_id": 3, "eos_token_id": 3}, + indent=2) + "\n", encoding="utf-8") + + # LM safetensors is passed through byte-for-byte. + (out / "model.safetensors").write_bytes((src / "model.safetensors").read_bytes()) + + # Fold + split decoder. + state = load_pt_checkpoint(src / "decoder.pth") + state = fold_weight_norm(state) + renamed = rename_decoder_keys(state) + if not renamed: + raise SystemExit("decoder.pth contained no recognised tensors") + _write_safetensors(renamed, out / "decoder.safetensors") + + print(f"[convert_soprano] wrote {out}") + print(f"[convert_soprano] decoder tensors: {len(renamed)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) \ No newline at end of file diff --git a/tools/soprano_tts/run_official.py b/tools/soprano_tts/run_official.py new file mode 100644 index 00000000..35af6057 --- /dev/null +++ b/tools/soprano_tts/run_official.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Run official SopranoTTS for ground-truth comparison. +Usage: + python3 run_official.py --text "..." --out output.wav +""" +import argparse, json, os, time, wave + +def run_inference(text: str, out_path: str, model_path: str = "models/Soprano-1.1-80M"): + from soprano import SopranoTTS + + load_t0 = time.time() + model = SopranoTTS(backend="auto", device="cpu", model_path=model_path) + load_time = time.time() - load_t0 + + infer_t0 = time.time() + out = model.infer(text, out_path) + infer_time = time.time() - infer_t0 + + with wave.open(out_path, "r") as wf: + audio_dur = wf.getnframes() / wf.getframerate() + + result = { + "text": text, + "text_chars": len(text), + "load_time_s": round(load_time, 3), + "infer_time_s": round(infer_time, 3), + "audio_duration_s": round(audio_dur, 3), + "rtf": round(infer_time / audio_dur, 4) if audio_dur > 0 else 0, + "output": out_path, + "output_bytes": os.path.getsize(out_path), + } + return result + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--text", default="Soprano is an extremely lightweight text to speech model.") + ap.add_argument("--out", default="soprano_official.wav") + ap.add_argument("--model", default="models/Soprano-1.1-80M") + ap.add_argument("--json", action="store_true", help="Output JSON") + args = ap.parse_args() + result = run_inference(args.text, args.out, args.model) + if args.json: + print(json.dumps(result, indent=2)) + else: + for k, v in result.items(): + print(f"{k}: {v}") diff --git a/webui/configs/models_catalog.json b/webui/configs/models_catalog.json index 2a341705..2ba07a3a 100644 --- a/webui/configs/models_catalog.json +++ b/webui/configs/models_catalog.json @@ -15,6 +15,7 @@ { "id": "qwen3-tts-1.7b", "display_name": "Qwen3-TTS 1.7B Base (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-Base", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_base", "min_vram_gb": 8 }, { "id": "qwen3-tts-1.7b-custom", "display_name": "Qwen3-TTS 1.7B CustomVoice (tts)", "family": "qwen3_tts", "path": "models/Qwen3-TTS-12Hz-1.7B-CustomVoice", "task": "tts", "mode": "offline", "download_id": "qwen3_tts_1_7b_custom_voice", "min_vram_gb": 8 }, { "id": "miotts", "display_name": "MioTTS 1.7B (tts; needs MioCodec)", "family": "miotts", "path": "models/MioTTS-1.7B", "task": "tts", "mode": "offline", "download_id": "miotts_1_7b", "min_vram_gb": 8 }, + { "id": "soprano-tts", "display_name": "Soprano TTS (tts)", "family": "soprano_tts", "path": "models/Soprano-1.1-80M-GGUF", "task": "tts", "mode": "offline", "download_id": "soprano_1_1_80m_q8_0", "min_vram_gb": 1 }, { "id": "voxcpm2", "display_name": "VoxCPM2 (tts)", "family": "voxcpm2", "path": "models/VoxCPM2", "task": "tts", "mode": "offline", "download_id": "voxcpm2", "session_options": { "voxcpm2.weight_type": "q8_0" }, "min_vram_gb": 6 }, { "id": "vibevoice", "display_name": "VibeVoice 1.5B (tts, long-form/multi-speaker)", "family": "vibevoice", "path": "models/VibeVoice-1.5B", "task": "tts", "mode": "offline", "download_id": "vibevoice_1_5b", "min_vram_gb": 7 }, { "id": "index-tts2", "display_name": "IndexTTS2 (tts 中英克隆+情感)", "display_name_en": "IndexTTS2 (tts, zh/en clone + emotion)", "family": "index_tts2", "path": "models/IndexTTS-2", "task": "tts", "mode": "offline", "download_id": "index_tts2", "min_vram_gb": 8 }, diff --git a/webui/native/dist/index.html b/webui/native/dist/index.html index 7695fba5..99e15b2c 100644 --- a/webui/native/dist/index.html +++ b/webui/native/dist/index.html @@ -31,20 +31,20 @@