Pascal/poc chatterbox

conversion/__init__.py

@@ -37,6 +37,7 @@
     "BloomModel": "bloom",
     "CamembertModel": "bert",
     "ChameleonForCausalLM": "chameleon",
+    "ChatterboxModel": "chatterbox",
     "ChameleonForConditionalGeneration": "chameleon",
     "ChatGLMForConditionalGeneration": "chatglm",
     "ChatGLMModel": "chatglm",
@@ -260,6 +261,7 @@
 
 
 MMPROJ_MODEL_MAP: dict[str, str] = {
+    "ChatterboxModel": "chatterbox",
     "AudioFlamingo3ForConditionalGeneration": "ultravox",
     "CogVLMForCausalLM": "cogvlm",
     "DeepseekOCR2ForCausalLM": "deepseek",

gguf-py/gguf/constants.py

@@ -5084,6 +5084,8 @@ class VisionProjectorType:
     NEMOTRON_V2_VL = "nemotron_v2_vl"
     QWEN3TTS_SPKENC = "qwen3tts_spkenc" # audio: ECAPA-TDNN speaker encoder
     QWEN3TTS_GEN = "qwen3tts_gen" # audio generation: code_predictor
+    CHATTERBOX_SPKENC = "chatterbox_spkenc" # audio: CAMPPlus speaker encoder
+    CHATTERBOX = "chatterbox" # audio generation: s3gen flow matching + HiFT vocoder
     HUNYUANVL      = "hunyuanvl"
     PARAKEET       = "parakeet"  # audio
     MINIMAXM3      = "minimax_m3"

tools/mtmd/CMakeLists.txt

@@ -27,6 +27,8 @@ add_library(mtmd
             clip-model.h
             clip-graph.h
             models/models.h
+            models/chatterbox-gen.cpp
+            models/chatterbox-spkenc.cpp
             models/cogvlm.cpp
             models/conformer.cpp
             models/dotsocr.cpp

tools/mtmd/clip-impl.h

@@ -455,6 +455,8 @@ enum projector_type {
     PROJECTOR_TYPE_MIMO_AUDIO,
     PROJECTOR_TYPE_QWEN3TTS_SPKENC,
     PROJECTOR_TYPE_QWEN3TTS_GEN,
+    PROJECTOR_TYPE_CHATTERBOX,
+    PROJECTOR_TYPE_CHATTERBOX_SPKENC,
     PROJECTOR_TYPE_UNKNOWN,
 };
 
@@ -514,6 +516,8 @@ static std::map<projector_type, std::string> PROJECTOR_TYPE_NAMES = {
     { PROJECTOR_TYPE_PARAKEET,          "parakeet"},
     { PROJECTOR_TYPE_QWEN3TTS_SPKENC,   "qwen3tts_spkenc"},
     { PROJECTOR_TYPE_QWEN3TTS_GEN,      "qwen3tts_gen"},
+    { PROJECTOR_TYPE_CHATTERBOX,        "chatterbox"},
+    { PROJECTOR_TYPE_CHATTERBOX_SPKENC, "chatterbox_spkenc"},
 };
 
 static projector_type clip_projector_type_from_string(const std::string & str) {

tools/mtmd/clip-model.h

@@ -6,6 +6,7 @@
 
 #include <algorithm>
 #include <array>
+#include <map>
 #include <vector>
 #include <unordered_set>
 #include <cstdint>
@@ -456,6 +457,237 @@ struct clip_code2wav {
     ggml_tensor * dac_post_conv_b      = nullptr;
 };
 
+struct clip_chatterbox {
+    // espnet rel-pos conformer layer of the flow encoder
+    struct enc_layer {
+        ggml_tensor * norm_mha_w = nullptr;
+        ggml_tensor * norm_mha_b = nullptr;
+        ggml_tensor * attn_q_w = nullptr;
+        ggml_tensor * attn_q_b = nullptr;
+        ggml_tensor * attn_k_w = nullptr;
+        ggml_tensor * attn_k_b = nullptr;
+        ggml_tensor * attn_v_w = nullptr;
+        ggml_tensor * attn_v_b = nullptr;
+        ggml_tensor * attn_pos_w = nullptr;      // linear_pos, no bias
+        ggml_tensor * attn_pos_bias_u = nullptr;
+        ggml_tensor * attn_pos_bias_v = nullptr;
+        ggml_tensor * attn_out_w = nullptr;
+        ggml_tensor * attn_out_b = nullptr;
+        ggml_tensor * norm_ff_w = nullptr;
+        ggml_tensor * norm_ff_b = nullptr;
+        ggml_tensor * ffn_1_w = nullptr;
+        ggml_tensor * ffn_1_b = nullptr;
+        ggml_tensor * ffn_2_w = nullptr;
+        ggml_tensor * ffn_2_b = nullptr;
+    };
+
+    // causal conv block of the estimator (conv k3 left padded + layer norm)
+    struct causal_block {
+        ggml_tensor * conv_w = nullptr;
+        ggml_tensor * conv_b = nullptr;
+        ggml_tensor * norm_w = nullptr;
+        ggml_tensor * norm_b = nullptr;
+    };
+
+    // time conditioned resnet of the estimator
+    struct resnet {
+        causal_block block1;
+        causal_block block2;
+        ggml_tensor * mlp_w = nullptr;      // time projection (mlp.1)
+        ggml_tensor * mlp_b = nullptr;
+        ggml_tensor * res_conv_w = nullptr;
+        ggml_tensor * res_conv_b = nullptr;
+    };
+
+    // diffusers style transformer block of the estimator
+    struct tfm_block {
+        ggml_tensor * norm1_w = nullptr;
+        ggml_tensor * norm1_b = nullptr;
+        ggml_tensor * attn_q_w = nullptr;   // no bias on qkv
+        ggml_tensor * attn_k_w = nullptr;
+        ggml_tensor * attn_v_w = nullptr;
+        ggml_tensor * attn_out_w = nullptr; // to_out.0
+        ggml_tensor * attn_out_b = nullptr;
+        ggml_tensor * norm3_w = nullptr;
+        ggml_tensor * norm3_b = nullptr;
+        ggml_tensor * ff_in_w = nullptr;    // ff.net.0.proj
+        ggml_tensor * ff_in_b = nullptr;
+        ggml_tensor * ff_out_w = nullptr;   // ff.net.2
+        ggml_tensor * ff_out_b = nullptr;
+    };
+
+    // one estimator stage: resnet, transformer stack, boundary causal conv
+    // (the boundary conv stays null on mid stages)
+    struct est_stage {
+        resnet res;
+        std::vector<tfm_block> tfm;
+        ggml_tensor * conv_w = nullptr;
+        ggml_tensor * conv_b = nullptr;
+    };
+
+    // snake resblock unit of the hift vocoder
+    struct hift_res_unit {
+        ggml_tensor * act1_alpha = nullptr;
+        ggml_tensor * act2_alpha = nullptr;
+        ggml_tensor * conv1_w = nullptr;
+        ggml_tensor * conv1_b = nullptr;
+        ggml_tensor * conv2_w = nullptr;
+        ggml_tensor * conv2_b = nullptr;
+    };
+    struct hift_res {
+        std::vector<hift_res_unit> units;   // dilations 1/3/5 on conv1
+    };
+
+    // one hift upsample stage: conv transpose, source injection, 3 resblocks
+    struct hift_up {
+        ggml_tensor * up_w = nullptr;
+        ggml_tensor * up_b = nullptr;
+        ggml_tensor * source_down_w = nullptr;
+        ggml_tensor * source_down_b = nullptr;
+        hift_res source_res;
+        hift_res res_0;
+        hift_res res_1;
+        hift_res res_2;
+    };
+
+    // f0 predictor conv of the hift vocoder
+    struct f0_conv {
+        ggml_tensor * w = nullptr;
+        ggml_tensor * b = nullptr;
+    };
+
+    // s3 tokenizer attention block (neox rope on q/k, fsmn memory on v)
+    struct s3tok_block {
+        ggml_tensor * attn_ln_w = nullptr;
+        ggml_tensor * attn_ln_b = nullptr;
+        ggml_tensor * attn_q_w = nullptr;
+        ggml_tensor * attn_q_b = nullptr;
+        ggml_tensor * attn_k_w = nullptr;   // no bias
+        ggml_tensor * attn_v_w = nullptr;
+        ggml_tensor * attn_v_b = nullptr;
+        ggml_tensor * fsmn_w = nullptr;
+        ggml_tensor * attn_out_w = nullptr;
+        ggml_tensor * attn_out_b = nullptr;
+        ggml_tensor * mlp_ln_w = nullptr;
+        ggml_tensor * mlp_ln_b = nullptr;
+        ggml_tensor * mlp_in_w = nullptr;
+        ggml_tensor * mlp_in_b = nullptr;
+        ggml_tensor * mlp_out_w = nullptr;
+        ggml_tensor * mlp_out_b = nullptr;
+    };
+
+    // folded batchnorm (w/b stay null on the affine free variant)
+    struct bn {
+        ggml_tensor * mean = nullptr;
+        ggml_tensor * var  = nullptr;
+        ggml_tensor * w    = nullptr;
+        ggml_tensor * b    = nullptr;
+    };
+
+    // fcm residual 2d block of the speaker encoder
+    // (shortcut stays null on the stride 1 blocks)
+    struct spk_res2d {
+        ggml_tensor * conv1_w = nullptr;
+        bn bn1;
+        ggml_tensor * conv2_w = nullptr;
+        bn bn2;
+        ggml_tensor * shortcut_w = nullptr;
+        bn shortcut_bn;
+    };
+
+    // cam dense tdnn layer of the speaker encoder
+    struct spk_cam_layer {
+        bn nl1_bn;
+        ggml_tensor * linear1_w = nullptr;
+        bn nl2_bn;
+        ggml_tensor * local_w = nullptr;    // cam_layer.linear_local
+        ggml_tensor * ctx1_w = nullptr;     // cam_layer.linear1
+        ggml_tensor * ctx1_b = nullptr;
+        ggml_tensor * ctx2_w = nullptr;     // cam_layer.linear2
+        ggml_tensor * ctx2_b = nullptr;
+    };
+    struct spk_cam_block {
+        std::vector<spk_cam_layer> layers;
+        bn transit_bn;
+        ggml_tensor * transit_w = nullptr;
+    };
+
+    // flow encoder
+    ggml_tensor * input_embedding_w = nullptr;  // flow.input_embedding
+    ggml_tensor * embed_linear_w = nullptr;     // fenc.embed.out.0
+    ggml_tensor * embed_linear_b = nullptr;
+    ggml_tensor * embed_norm_w = nullptr;       // fenc.embed.out.1
+    ggml_tensor * embed_norm_b = nullptr;
+    ggml_tensor * pre_conv1_w = nullptr;        // pre_lookahead_layer
+    ggml_tensor * pre_conv1_b = nullptr;
+    ggml_tensor * pre_conv2_w = nullptr;
+    ggml_tensor * pre_conv2_b = nullptr;
+    std::vector<enc_layer> enc;
+    ggml_tensor * up_conv_w = nullptr;          // fenc.up_layer.conv
+    ggml_tensor * up_conv_b = nullptr;
+    ggml_tensor * up_embed_linear_w = nullptr;  // fenc.up_embed.out.0
+    ggml_tensor * up_embed_linear_b = nullptr;
+    ggml_tensor * up_embed_norm_w = nullptr;    // fenc.up_embed.out.1
+    ggml_tensor * up_embed_norm_b = nullptr;
+    std::vector<enc_layer> up_enc;
+    ggml_tensor * after_norm_w = nullptr;
+    ggml_tensor * after_norm_b = nullptr;
+    ggml_tensor * encoder_proj_w = nullptr;     // flow.encoder_proj
+    ggml_tensor * encoder_proj_b = nullptr;
+
+    // cfm estimator
+    ggml_tensor * time_mlp_1_w = nullptr;
+    ggml_tensor * time_mlp_1_b = nullptr;
+    ggml_tensor * time_mlp_2_w = nullptr;
+    ggml_tensor * time_mlp_2_b = nullptr;
+    ggml_tensor * time_embed_mixer_w = nullptr; // meanflow variant only
+    est_stage est_down;
+    std::vector<est_stage> est_mid;
+    est_stage est_up;
+    causal_block est_final_block;
+    ggml_tensor * est_final_proj_w = nullptr;
+    ggml_tensor * est_final_proj_b = nullptr;
+
+    // hift vocoder
+    ggml_tensor * hift_pre_w = nullptr;
+    ggml_tensor * hift_pre_b = nullptr;
+    std::vector<hift_up> hift_ups;
+    ggml_tensor * hift_post_w = nullptr;
+    ggml_tensor * hift_post_b = nullptr;
+    std::vector<f0_conv> f0_condnet;
+    ggml_tensor * f0_classifier_w = nullptr;
+    ggml_tensor * f0_classifier_b = nullptr;
+
+    // s3 tokenizer
+    ggml_tensor * s3tok_conv1_w = nullptr;
+    ggml_tensor * s3tok_conv1_b = nullptr;
+    ggml_tensor * s3tok_conv2_w = nullptr;
+    ggml_tensor * s3tok_conv2_b = nullptr;
+    std::vector<s3tok_block> s3tok_blocks;
+    ggml_tensor * s3tok_down_w = nullptr;       // quantizer._codebook.project_down
+    ggml_tensor * s3tok_down_b = nullptr;
+
+    // CAMPPlus speaker encoder
+    ggml_tensor * spk_conv1_w = nullptr;
+    bn spk_bn1;
+    spk_res2d spk_layer1_0;
+    spk_res2d spk_layer1_1;
+    spk_res2d spk_layer2_0;
+    spk_res2d spk_layer2_1;
+    ggml_tensor * spk_conv2_w = nullptr;
+    bn spk_bn2;
+    ggml_tensor * spk_tdnn_w = nullptr;
+    bn spk_tdnn_bn;
+    spk_cam_block spk_block1;
+    spk_cam_block spk_block2;
+    spk_cam_block spk_block3;
+    bn spk_out_bn;
+    ggml_tensor * spk_dense_w = nullptr;
+    bn spk_dense_bn;
+    ggml_tensor * spk_affine_w = nullptr;
+    ggml_tensor * spk_affine_b = nullptr;
+};
+
 struct clip_model {
     clip_modality modality = CLIP_MODALITY_VISION;
     projector_type proj_type = PROJECTOR_TYPE_MLP;
@@ -687,6 +919,15 @@ struct clip_model {
     // qwen3tts code2wav: RVQ codes -> raw PCM
     clip_code2wav c2w;
 
+    // chatterbox flow encoder, cfm estimator, hift vocoder, s3 tokenizer
+    // and CAMPPlus speaker encoder
+    clip_chatterbox cbx;
+
+    // chatterbox host-read side data (conditioning defaults, embedding
+    // tables, filterbanks, voice/conditioning encoders run on the host),
+    // accessed by name through clip_cbx_read_tensor
+    std::map<std::string, ggml_tensor *> cbx_tensors;
+
     // cogvlm
     ggml_tensor * mm_post_fc_norm_w = nullptr;
     ggml_tensor * mm_post_fc_norm_b = nullptr;