From 69e96dc4f27662fb7cebb61503671393c70b3183 Mon Sep 17 00:00:00 2001 From: Mukkesh Ganesh Date: Fri, 23 Oct 2026 15:34:35 -0700 Subject: [PATCH 1/2] feat(gguf): serve image input on deepseek4 ggufs from a deepseek4v mmproj --- docs/models.md | 4 + python/freetoken/mm/processors/deepseek_v4.py | 5 + .../freetoken/models/deepseek_v4/__init__.py | 4 +- python/freetoken/models/deepseek_v4/gguf.py | 205 +++++++++++++++++- python/freetoken/models/register.py | 7 +- 5 files changed, 213 insertions(+), 12 deletions(-) diff --git a/docs/models.md b/docs/models.md index 914e813bca..705d19f2f5 100644 --- a/docs/models.md +++ b/docs/models.md @@ -124,6 +124,10 @@ unset on PCIe GPUs: wider grids add no bandwidth there. with `ft checkpoint`, or add the encoder in place with [scripts/ftw_hotfix.py](ftw-hotfix.md). - DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — the authoritative model args are read from there. +- A DeepSeek-V4 GGUF takes image input from the Vision-Exp text file paired with its + `deepseek4v` mmproj; the text-only 0731 release has no image token and refuses the + mmproj. ggml-org's Vision-Exp text GGUF ships a chat template that renders no image + placeholder — serve unsloth's (or override `--chat-template`) for image requests. - DeepSeek-V4.1-Flash serves text and images from the HF checkpoint as shipped (fp8 block-32 dense weights, fp4 experts on the offload cache, 890 B/token global KV in packed fp4/fp8 pools). Its two 98 GiB Engram tables stream from the checkpoint shards on diff --git a/python/freetoken/mm/processors/deepseek_v4.py b/python/freetoken/mm/processors/deepseek_v4.py index 5cea991994..96528fd42e 100644 --- a/python/freetoken/mm/processors/deepseek_v4.py +++ b/python/freetoken/mm/processors/deepseek_v4.py @@ -103,6 +103,11 @@ def apply(self, input_ids: torch.Tensor, images: list[bytes]) -> MMResult: positions, delta = self.positions(len(out), items) or (None, 0) return MMResult(new_ids, items, positions, delta) + def _image_processor(self) -> None: + """None: this family's preprocessing is FreeToken-local (:func:`load_image`); no + transformers image-processor class exists to build from a preprocessor_config.""" + return None + def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]: # The smallest block the tower can run: one 2x2 merge grid out of a 6x6 patch grid. r, p = self.vc.vision_downsample_ratio, self.vc.vision_patch_size diff --git a/python/freetoken/models/deepseek_v4/__init__.py b/python/freetoken/models/deepseek_v4/__init__.py index 31931ede49..65d0e5f8ea 100644 --- a/python/freetoken/models/deepseek_v4/__init__.py +++ b/python/freetoken/models/deepseek_v4/__init__.py @@ -15,7 +15,7 @@ from .args import DeepseekV4Args, load_args from .config import DSV4VisionConfig, model_config_from_args, parse_config, parse_vision_config -from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf +from .gguf import gguf_mm_config, iter_gguf_weights, iter_mmproj_weights, parse_gguf_config, to_gguf from .model import DeepseekV4ForCausalLM from .vision import DSV4Aligner, DSV4Vision, DSV4VisionTower from .weight import iter_expert_pieces, iter_vision_weights, iter_weights @@ -31,8 +31,10 @@ "parse_config", "parse_gguf_config", "parse_vision_config", + "gguf_mm_config", "DeepseekV4ForCausalLM", "iter_gguf_weights", + "iter_mmproj_weights", "iter_weights", "iter_expert_pieces", "iter_vision_weights", diff --git a/python/freetoken/models/deepseek_v4/gguf.py b/python/freetoken/models/deepseek_v4/gguf.py index 4f90137d8b..7b980bed12 100644 --- a/python/freetoken/models/deepseek_v4/gguf.py +++ b/python/freetoken/models/deepseek_v4/gguf.py @@ -1,5 +1,5 @@ -"""DeepSeek-V4-Flash from a llama.cpp ``deepseek4`` GGUF (unsloth ``DeepSeek-V4-Flash-0731-GGUF``, -text only): config from the GGUF metadata, tensor names, and the dense state dict. +"""DeepSeek-V4-Flash from a llama.cpp ``deepseek4`` GGUF (unsloth ``DeepSeek-V4-Flash-0731-GGUF`` +and the Vision-Exp release): config from the GGUF metadata, tensor names, and the dense state dict. The GGUF geometry matches the HF checkpoint (43 MLA blocks with sliding-window sparse attention, per-layer KV compressors and a Lightning Indexer on the ratio-4 blocks, mHC x4 @@ -13,6 +13,12 @@ ``indexer.*`` tensor means ratio 4, a compressor without an indexer means 128, unless its ``wkv`` carries the CSA overlap's double rows (also ratio 4). Routed experts serve from the GGUF expert banks (``freetoken.models.gguf.experts``) like every GGUF family. + +A ``deepseek4v`` mmproj beside the Vision-Exp text file adds the vision tower under +``visual.``: the converter splits the fused SwiGLU ``w1`` into ``ffn_gate`` / ``ffn_up`` +rows and reshapes the patch embedding into the conv layout (both undone at load); ggml-org +writes the attention block fused and unsloth split. A vision trunk also carries the +``exp_probs_b_vl`` router biases and drops the hash layers' never-applied text bias. """ from __future__ import annotations @@ -25,12 +31,13 @@ import torch from freetoken.layers.quantization.ggml_types import is_quantized -from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names +from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor +from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index, gguf_tensor_names, load_gguf_metadata from freetoken.models.gguf.tensors import check_scales, fused, matrix from freetoken.utils import init_logger from .args import DeepseekV4Args -from .config import model_config_from_args +from .config import IMAGE_TOKEN_ID, model_config_from_args, parse_vision_config if TYPE_CHECKING: from freetoken.models.config import ModelConfig @@ -111,7 +118,18 @@ def need(key: str): names = gguf_tensor_names(shim.model_path) if any(n.startswith(("vision.", "aligner.")) for n in names): raise NotImplementedError( - f"{shim.model_path}: deepseek4 GGUF with a vision tower is not served (the text-only 0731 release has none)" + f"{shim.model_path}: deepseek4 GGUF with an inline vision tower is not served " + "(llama.cpp ships the tower as a separate mmproj file)" + ) + + # a deepseek4v mmproj fills the shim's flat vision fields; a text-only process nulls + # vision_n_layers on its copy. The weight iterator re-parses a fresh shim, so the + # check below still fires there for a broken pairing. + vc = parse_vision_config(shim) + if vc is not None and not any(n.endswith(".exp_probs_b_vl.bias") for n in names): + raise NotImplementedError( + f"{shim.model_path}: the text trunk predates vision routing (no exp_probs_b_vl; " + "converted before llama.cpp #28154) -- pair a Vision-Exp text GGUF reconverted after 2026-09-02" ) # the clamp is per-layer in the GGUF (and the shared expert carries its own array); the model @@ -209,12 +227,171 @@ def need(key: str): hc_sinkhorn_iters=int(need("hyper_connection.sinkhorn_iterations")), hc_eps=float(need("hyper_connection.epsilon")), n_mtp_layers=0, # the converter drops the MTP blocks - vision_n_layers=0, + # vision fields only when a mmproj filled them (the defaults are the text model's) + **({} if vc is None else dict( + vision_n_layers=vc.vision_n_layers, + vision_dim=vc.vision_dim, + vision_n_heads=vc.vision_n_heads, + vision_inter_dim=vc.vision_inter_dim, + vision_patch_size=vc.vision_patch_size, + vision_rope_theta=vc.vision_rope_theta, + vision_downsample_ratio=vc.vision_downsample_ratio, + vision_max_n_token=vc.vision_max_n_token, + vision_min_pixels=vc.vision_min_pixels, + vision_max_wh_ratio=vc.vision_max_wh_ratio, + )), ) - config = model_config_from_args(args, max_position=int(need("context_length"))) + config = model_config_from_args(args, max_position=int(need("context_length")), vision_config=vc) return replace(config, expert_quant="gguf") +# ---------------------------------------------------------------------------------------------- +# Vision tower (a deepseek4v mmproj) +# ---------------------------------------------------------------------------------------------- + +_PROJECTOR = "deepseek4v" +_IMAGE_TOKEN = "<｜deepseek_image｜>" +# no mmproj key carries these; llama.cpp hardcodes them and its converter asserts them +_ROPE_THETA = 10000.0 +_MAX_N_TOKEN = 384 +_MAX_WH_RATIO = 8 + +# top-level mmproj tensor -> the tower name under ``visual.`` +_MM_TOP: dict[str, str] = { + "v.patch_embd.bias": "vision.patch_embed.proj.bias", + "v.post_ln.weight": "vision.norm.weight", + "mm.1.weight": "aligner.w1.weight", + "mm.1.bias": "aligner.w1.bias", + "mm.2.weight": "aligner.w2.weight", + "mm.2.bias": "aligner.w2.bias", + "v.token_embd.img_start": "image_start", + "v.token_embd.img_end": "image_end", + "v.image_newline": "image_newline", + "v.token_embd.img_pad": "image_pad", +} + + +def gguf_mm_config(text_md: dict, mmproj: Mmproj, vocab_size: int) -> tuple[dict, dict]: + """The flat HF config keys (the vision dims sit at the top level on this family, like the + checkpoint's inference config) and the preprocessor dict of the tower in ``mmproj``, for + the text GGUF whose metadata is ``text_md``.""" + if mmproj.projector != _PROJECTOR: + raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; DeepSeek-V4 needs {_PROJECTOR!r}") + md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path) + # the converter writes use_silu=True only for the SwiGLU tower; the module has no gelu MLP + if md.get("clip.use_silu") is not True: + raise NotImplementedError(f"mmproj {mmproj.path}: only the SwiGLU tower is served") + + def kv(key: str): + return md[f"clip.vision.{key}"] + + arch = text_md["general.architecture"] + text_hidden = int(text_md[f"{arch}.embedding_length"]) + if int(kv("projection_dim")) != text_hidden: + raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide") + tokens = text_md["tokenizer.ggml.tokens"] + if _IMAGE_TOKEN not in tokens: + raise ValueError( + f"the text GGUF tokenizer has no {_IMAGE_TOKEN} token (the 0731 text-only release " + "uses another dialect; pair the mmproj with a Vision-Exp text GGUF)" + ) + # the model config and the processor hardcode the family id; a file with the token at + # another id would fail per request instead of here + if tokens.index(_IMAGE_TOKEN) != IMAGE_TOKEN_ID: + raise ValueError( + f"the text GGUF carries {_IMAGE_TOKEN} at id {tokens.index(_IMAGE_TOKEN)}, not the family's {IMAGE_TOKEN_ID}" + ) + for sentinel in ("v.token_embd.img_start", "v.token_embd.img_end", "v.image_newline", "v.token_embd.img_pad"): + if sentinel not in index: + raise ValueError(f"mmproj {mmproj.path}: sentinel tensor {sentinel!r} is missing") + # the tower hardcodes eps 1e-6 (vision.py); the converter warns it is NOT the text 1e-20 + if abs(float(kv("attention.layer_norm_epsilon")) - 1e-6) > 1e-9: + raise NotImplementedError(f"mmproj {mmproj.path}: vision layer_norm_epsilon {kv('attention.layer_norm_epsilon')} is not the served 1e-6") + # qkv dialect: ggml-org writes the fused attn_qkv, unsloth the split attn_q/k/v (clip.cpp loads both) + split = all(f"v.blk.0.attn_{q}.{part}" in index for q in "qkv" for part in ("weight", "bias")) + if split == ("v.blk.0.attn_qkv.weight" in index): + raise ValueError(f"mmproj {mmproj.path}: the attn tensors are neither the fused attn_qkv nor the split attn_q/k/v layout (or carry both)") + patch, dim = int(kv("patch_size")), int(kv("embedding_length")) + # the converter reshapes the Linear's [dim, 3*p*p] kernel into the conv [p, p, 3, dim] + if tuple(index["v.patch_embd.weight"].ne) != (patch, patch, 3, dim): + raise ValueError(f"mmproj {mmproj.path}: v.patch_embd.weight {tuple(index['v.patch_embd.weight'].ne)} is not the [p, p, 3, dim] conv layout") + # clip.vision.image_size (672) is deliberately unread: a converter compat/warmup value, + # the tower is native-resolution + + hf_fields = { + "vision_n_layers": int(kv("block_count")), + "vision_dim": dim, + "vision_n_heads": int(kv("attention.head_count")), + "vision_inter_dim": int(kv("feed_forward_length")), + "vision_patch_size": patch, + "vision_rope_theta": _ROPE_THETA, + "vision_downsample_ratio": int(kv("projector.scale_factor")), + "vision_max_n_token": _MAX_N_TOKEN, + "vision_min_pixels": int(kv("image_min_pixels")), + "vision_max_wh_ratio": _MAX_WH_RATIO, + # parse_vision_config reads the aligner's target width off hidden_size + "hidden_size": text_hidden, + "image_token_id": tokens.index(_IMAGE_TOKEN), + } + preprocessor = { + # no transformers class exists; DSV4MMProcessor._image_processor returns None + "image_processor_type": "DeepseekV4ImageProcessor", + "patch_size": patch, + "temporal_patch_size": 1, + "merge_size": int(kv("projector.scale_factor")), + "image_mean": [float(v) for v in kv("image_mean")], + "image_std": [float(v) for v in kv("image_std")], + "min_pixels": int(kv("image_min_pixels")), + "max_n_token": _MAX_N_TOKEN, + } + return hf_fields, preprocessor + + +def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]: + """The tower state dict under ``visual.``, named as the HF checkpoints load.""" + index = gguf_tensor_index(mmproj.path) + fused = "v.blk.0.attn_qkv.weight" in index + n_blocks = max(int(n.split(".")[2]) for n in index if n.startswith("v.blk.")) + 1 + + pe = index["v.patch_embd.weight"] + # ggml order [p, p, 3, out] -> torch [out, 3, p, p] -> the module's flat [out, 3*p*p] + yield "visual.vision.patch_embed.proj.weight", mmproj_tensor(pe).reshape(pe.ne[3], 3 * pe.ne[0] * pe.ne[0]) + for gguf_name, ft_name in _MM_TOP.items(): + yield f"visual.{ft_name}", mmproj_tensor(index[gguf_name]) + + consumed = {"v.patch_embd.weight", *_MM_TOP} + for layer in range(n_blocks): + blk, dst = f"v.blk.{layer}", f"visual.vision.blocks.{layer}" + if fused: + for part in ("weight", "bias"): + yield f"{dst}.attn.wqkv.{part}", mmproj_tensor(index[f"{blk}.attn_qkv.{part}"]) + consumed.update((f"{blk}.attn_qkv.weight", f"{blk}.attn_qkv.bias")) + else: + for part in ("weight", "bias"): + yield f"{dst}.attn.wqkv.{part}", torch.cat( + [mmproj_tensor(index[f"{blk}.attn_{q}.{part}"]) for q in "qkv"], dim=0 + ) + consumed.update(f"{blk}.attn_{q}.{part}" for q in "qkv" for part in ("weight", "bias")) + for part in ("weight", "bias"): + yield f"{dst}.attn.wo.{part}", mmproj_tensor(index[f"{blk}.attn_out.{part}"]) + # the converter chunks the fused SwiGLU w1 rows into gate / up; cat them back + yield f"{dst}.mlp.w1.weight", torch.cat( + [mmproj_tensor(index[f"{blk}.ffn_gate.weight"]), mmproj_tensor(index[f"{blk}.ffn_up.weight"])], dim=0 + ) + yield f"{dst}.mlp.w2.weight", mmproj_tensor(index[f"{blk}.ffn_down.weight"]) + yield f"{dst}.norm1.weight", mmproj_tensor(index[f"{blk}.ln1.weight"]) + yield f"{dst}.norm2.weight", mmproj_tensor(index[f"{blk}.ln2.weight"]) + consumed.update( + f"{blk}.attn_out.{part}" for part in ("weight", "bias") + ) + consumed.update(f"{blk}.ffn_{m}.weight" for m in ("gate", "up", "down")) + consumed.update((f"{blk}.ln1.weight", f"{blk}.ln2.weight")) + + leftovers = set(index) - consumed + if leftovers: + raise ValueError(f"mmproj {mmproj.path}: unexpected tensors {sorted(leftovers)[:5]}") + + # ---------------------------------------------------------------------------------------------- # Weights # ---------------------------------------------------------------------------------------------- @@ -267,6 +444,8 @@ def iter_gguf_weights( from freetoken.utils import cached_load_hf_config config = parse_gguf_config(cached_load_hf_config(model_path)) + vision = include_vision and config.is_multimodal + log_mmproj(model_path, vision) args = config.dsv4_args index = gguf_tensor_index(model_path) hidden = config.hidden_size @@ -331,14 +510,22 @@ def bf16(name: str) -> torch.Tensor: g = f"{pre}.ffn.gate" yield f"{g}.weight", bf16(f"{blk}.ffn_gate_inp") if layer < args.n_hash_layers: - # a vision checkpoint also gives its hash layers a text bias; the text model has no slot for it yield f"{g}.tid2eid", _tid2eid(t(f"{blk}.ffn_gate_tid2eid")) + if vision: + # the converter drops the hash layers' text bias (the reference never applies + # it); the module builds the slot on a vision build, so it zero-fills + yield f"{g}.bias", torch.zeros(args.n_routed_experts, dtype=torch.float32) else: yield f"{g}.bias", _float(t(f"{blk}.exp_probs_b.bias")) + if vision: + yield f"{g}.bias_vl", _float(t(f"{blk}.exp_probs_b_vl.bias")) for proj, src in (("w1", "ffn_gate_shexp"), ("w3", "ffn_up_shexp"), ("w2", "ffn_down_shexp")): info = t(f"{blk}.{src}") k = hidden if proj != "w2" else args.moe_inter_dim yield from fused(f"{pre}.ffn.shared_experts.{proj}", [info], [matrix(info)], k) + if vision: + yield from iter_mmproj_weights(find_mmproj(model_path)) + -__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"] +__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"] diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py index d78a85486f..ef3fe48b8f 100644 --- a/python/freetoken/models/register.py +++ b/python/freetoken/models/register.py @@ -232,8 +232,9 @@ class ModelSpec: mm_processor=_DSV4_PROCESSOR, encoders=_DSV4_ENCODERS, ), - # llama.cpp deepseek4 GGUF (unsloth DeepSeek-V4-Flash-0731-GGUF; text only): the same model - # classes with every tensor in its packed ggml type + # llama.cpp deepseek4 GGUF (unsloth DeepSeek-V4-Flash-0731-GGUF and the Vision-Exp + # release + its deepseek4v mmproj): the same model classes with every tensor in its + # packed ggml type "DeepseekV4GGUFForCausalLM": ModelSpec( "freetoken.models.deepseek_v4", "DeepseekV4ForCausalLM", @@ -244,6 +245,8 @@ class ModelSpec: # the KV compressors read their weights through a dense bf16 einsum, so the reader # dequantizes them instead of serving the packed ggml rows unquantized_modules=("*.compressor.wkv", "*.compressor.wgate"), + mm_processor=_DSV4_PROCESSOR, + encoders=_DSV4_ENCODERS, ), # DeepSeek-V4.1-Flash (model_type deepseek_v41): multimodal wrapper config (text tower in # text_config). The checkpoint has no ``model.`` root; the compressors, the indexer's key projection / From 3304d95b0910473a13cf03042f32fa5eaf37c2a7 Mon Sep 17 00:00:00 2001 From: Mukkesh Ganesh Date: Fri, 23 Oct 2026 23:47:58 -0700 Subject: [PATCH 2/2] fix(gguf): skip an unusable deepseek4v mmproj instead of refusing the trunk --- docs/models.md | 2 +- python/freetoken/models/deepseek_v4/gguf.py | 20 +++++++++++--------- python/freetoken/models/gemma4/gguf.py | 2 +- python/freetoken/models/gguf/config.py | 4 +++- python/freetoken/models/glm5_next/gguf.py | 2 +- python/freetoken/models/qwen3_vl/gguf.py | 2 +- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/docs/models.md b/docs/models.md index 705d19f2f5..05a79aec14 100644 --- a/docs/models.md +++ b/docs/models.md @@ -127,7 +127,7 @@ unset on PCIe GPUs: wider grids add no bandwidth there. - A DeepSeek-V4 GGUF takes image input from the Vision-Exp text file paired with its `deepseek4v` mmproj; the text-only 0731 release has no image token and refuses the mmproj. ggml-org's Vision-Exp text GGUF ships a chat template that renders no image - placeholder — serve unsloth's (or override `--chat-template`) for image requests. + placeholder; serve unsloth's release for image requests. - DeepSeek-V4.1-Flash serves text and images from the HF checkpoint as shipped (fp8 block-32 dense weights, fp4 experts on the offload cache, 890 B/token global KV in packed fp4/fp8 pools). Its two 98 GiB Engram tables stream from the checkpoint shards on diff --git a/python/freetoken/models/deepseek_v4/gguf.py b/python/freetoken/models/deepseek_v4/gguf.py index 7b980bed12..29055e51ce 100644 --- a/python/freetoken/models/deepseek_v4/gguf.py +++ b/python/freetoken/models/deepseek_v4/gguf.py @@ -23,6 +23,7 @@ from __future__ import annotations +import os import re from dataclasses import replace from typing import TYPE_CHECKING, Iterator @@ -122,15 +123,9 @@ def need(key: str): "(llama.cpp ships the tower as a separate mmproj file)" ) - # a deepseek4v mmproj fills the shim's flat vision fields; a text-only process nulls - # vision_n_layers on its copy. The weight iterator re-parses a fresh shim, so the - # check below still fires there for a broken pairing. + # a deepseek4v mmproj beside the file fills the shim's flat vision fields; a text-only + # process nulls vision_n_layers on its copy, and None here is the same answer vc = parse_vision_config(shim) - if vc is not None and not any(n.endswith(".exp_probs_b_vl.bias") for n in names): - raise NotImplementedError( - f"{shim.model_path}: the text trunk predates vision routing (no exp_probs_b_vl; " - "converted before llama.cpp #28154) -- pair a Vision-Exp text GGUF reconverted after 2026-09-02" - ) # the clamp is per-layer in the GGUF (and the shared expert carries its own array); the model # takes one value, so anything non-uniform is refused instead of silently taking the first @@ -271,7 +266,7 @@ def need(key: str): } -def gguf_mm_config(text_md: dict, mmproj: Mmproj, vocab_size: int) -> tuple[dict, dict]: +def gguf_mm_config(text_md: dict, mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict, dict]: """The flat HF config keys (the vision dims sit at the top level on this family, like the checkpoint's inference config) and the preprocessor dict of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``.""" @@ -301,6 +296,13 @@ def kv(key: str): raise ValueError( f"the text GGUF carries {_IMAGE_TOKEN} at id {tokens.index(_IMAGE_TOKEN)}, not the family's {IMAGE_TOKEN_ID}" ) + # a ValueError so auto-discovery skips this mmproj (text only); only --mmproj fails + if not any(n.endswith(".exp_probs_b_vl.bias") for n in gguf_tensor_names(model_path)): + raise ValueError( + f"the text trunk {os.path.basename(model_path)} predates vision routing " + "(no exp_probs_b_vl; converted before llama.cpp #28154) -- pair a Vision-Exp " + "text GGUF reconverted after 2026-09-02" + ) for sentinel in ("v.token_embd.img_start", "v.token_embd.img_end", "v.image_newline", "v.token_embd.img_pad"): if sentinel not in index: raise ValueError(f"mmproj {mmproj.path}: sentinel tensor {sentinel!r} is missing") diff --git a/python/freetoken/models/gemma4/gguf.py b/python/freetoken/models/gemma4/gguf.py index 047d089da1..ce806ba8f3 100644 --- a/python/freetoken/models/gemma4/gguf.py +++ b/python/freetoken/models/gemma4/gguf.py @@ -105,7 +105,7 @@ def _f32_value(v: float) -> float: return float(str(np.float32(v))) -def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]: +def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]: """The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``.""" if mmproj.projector not in (_VIT, _UNIFIED): raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; Gemma 4 needs {_VIT!r} or {_UNIFIED!r}") diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py index 61198ead29..680988389a 100644 --- a/python/freetoken/models/gguf/config.py +++ b/python/freetoken/models/gguf/config.py @@ -95,7 +95,9 @@ def _vocab_size(model_path: str) -> int: @functools.cache def _mm_fields(module: str, model_path: str, mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any] | None]: - return importlib.import_module(module).gguf_mm_config(load_gguf_metadata(model_path), mmproj, vocab_size) + return importlib.import_module(module).gguf_mm_config( + load_gguf_metadata(model_path), mmproj, vocab_size, model_path=model_path + ) def mm_fields(model_path: str, mmproj: Mmproj) -> tuple[dict[str, Any], dict[str, Any] | None]: diff --git a/python/freetoken/models/glm5_next/gguf.py b/python/freetoken/models/glm5_next/gguf.py index c918225396..97c5703d1e 100644 --- a/python/freetoken/models/glm5_next/gguf.py +++ b/python/freetoken/models/glm5_next/gguf.py @@ -281,7 +281,7 @@ def need(key: str): _BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$") -def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]: +def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]: """The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``.""" if mmproj.projector != _PROJECTOR: raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; GLM-5.3-Flash needs {_PROJECTOR!r}") diff --git a/python/freetoken/models/qwen3_vl/gguf.py b/python/freetoken/models/qwen3_vl/gguf.py index 52f1edd7c7..6fa124a30e 100644 --- a/python/freetoken/models/qwen3_vl/gguf.py +++ b/python/freetoken/models/qwen3_vl/gguf.py @@ -55,7 +55,7 @@ _PATCH = ("v.patch_embd.weight", "v.patch_embd.weight.1") -def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int) -> tuple[dict[str, Any], dict[str, Any]]: +def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]: """The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``.""" if mmproj.projector != _PROJECTOR: raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; Qwen VL needs {_PROJECTOR!r}")