From b94c1b7975b9e26f69a60a0ff49479a84d8fb4c1 Mon Sep 17 00:00:00 2001 From: Ogundepo Odunayo Date: Sun, 26 Jul 2026 15:41:34 -0400 Subject: [PATCH 1/3] Add first-class quantized inference --- CONTRIBUTING.md | 2 +- README.md | 72 ++++++++-- SUPPORT.md | 25 +++- docs/load_model.md | 119 ++++++++++++++++ pyproject.toml | 3 + src/mlx_transformers/__init__.py | 25 +++- src/mlx_transformers/cli.py | 162 ++++++++++++++++++++++ src/mlx_transformers/inference.py | 190 ++++++++++++++++++++++++++ src/mlx_transformers/models/base.py | 73 +++++++--- src/mlx_transformers/quantization.py | 127 +++++++++++++++++ tests/test_from_pretrained.py | 45 ++++++ tests/test_quantization.py | 71 ++++++++++ tests/test_quantized_inference.py | 196 +++++++++++++++++++++++++++ 13 files changed, 1079 insertions(+), 31 deletions(-) create mode 100644 src/mlx_transformers/cli.py create mode 100644 src/mlx_transformers/inference.py create mode 100644 src/mlx_transformers/quantization.py create mode 100644 tests/test_quantization.py create mode 100644 tests/test_quantized_inference.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index e666a77..6aaf3df 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ python -m unittest discover -s tests -v ``` -The expected baseline is 98 discovered tests: 76 pass and 22 Hub tests skip. +The expected baseline is 113 discovered tests: 91 pass and 22 Hub tests skip. The exact count should change only when tests are intentionally added or removed. diff --git a/README.md b/README.md index 50b148e..eb1a85a 100644 --- a/README.md +++ b/README.md @@ -56,18 +56,74 @@ inputs = {k: mx.array(v) for k, v in inputs.items()} outputs = model(**inputs) ``` -Quantized loading: +## Quantized Inference + +MLX Transformers can auto-detect an MLX pre-quantized checkpoint and run it +without extra loader flags: + +```python +from mlx_transformers import generate_text, load_causal_model + +loaded = load_causal_model( + "mlx-community/Phi-3-mini-4k-instruct-4bit", +) +result = generate_text( + loaded.model, + loaded.tokenizer, + "Explain weight quantization in one sentence.", + max_new_tokens=64, +) +print(loaded.quantization) +print(result.text) +``` + +To quantize a regular safetensors checkpoint in memory after loading: ```python -model.from_pretrained( +import mlx.core as mx + +from mlx_transformers import QuantizationConfig, load_causal_model + +loaded = load_causal_model( model_name, - quantize=True, - group_size=64, - bits=4, - mode="affine", + dtype=mx.float16, + quantization=QuantizationConfig( + group_size=64, + bits=4, + mode="affine", + ), ) ``` +The existing model-specific loader flags remain supported: + +```python +model.from_pretrained(model_name, quantize=True, group_size=64, bits=4) +``` + +The installed CLI offers the same two paths: + +```bash +# Auto-detect a pre-quantized MLX checkpoint. +mlx-transformers-generate \ + --model mlx-community/Phi-3-mini-4k-instruct-4bit \ + --prompt "Explain attention masking." \ + --max-new-tokens 64 + +# Quantize a regular checkpoint after loading it. +mlx-transformers-generate \ + --model meta-llama/Llama-3.2-1B-Instruct \ + --prompt "Explain attention masking." \ + --quantize --group-size 64 --bits 4 \ + --max-new-tokens 64 +``` + +On-load quantization temporarily materializes the regular checkpoint before +replacing supported layers, so peak memory is higher than loading an already +quantized checkpoint. Prefer a reviewed pre-quantized MLX checkpoint for large +models. See [docs/load_model.md](docs/load_model.md) for supported modes, +offline use, metadata inspection, and safety constraints. + Generation is finite and uses Hugging Face-style `max_new_tokens`: ```python @@ -184,5 +240,5 @@ Tests that use external checkpoints are skipped unless download sizes before opting in. Some checkpoints are gated; set `HF_TOKEN` only for an explicitly reviewed integration run. -The verified 2026-07-26 Apple-silicon baseline is 98 discovered tests: -76 pass and 22 Hub integration tests skip. +The verified 2026-07-26 Apple-silicon baseline is 113 discovered tests: +91 pass and 22 Hub integration tests skip. diff --git a/SUPPORT.md b/SUPPORT.md index 0f35ae6..3421a63 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -75,9 +75,19 @@ Gemma 3 4B multimodal inference produced non-finite values in float16 during validation. Use float32 or `mx.bfloat16` for that path. Float16 is not a verified Gemma 3 4B multimodal configuration. -Quantization structure and error paths have bounded tests. Real quantized -generation is currently verified only for the tested 4-bit Phi-3 checkpoint -with group size 64. +Quantized inference is verified through two bounded paths: + +- a regular synthetic Llama checkpoint loaded and quantized at runtime with + affine 4-bit weights and group size 32; +- saved/reloaded pre-quantized synthetic Llama weights plus the cached + `mlx-community/Phi-3-mini-4k-instruct-4bit` checkpoint with affine 4-bit + weights and group size 64. + +Both paths pass finite forward execution and bounded cached generation. The +public CLI was run offline against the cached Phi-3 checkpoint on Apple +silicon. Affine, MXFP4, MXFP8, and NVFP4 settings are validated against the +MLX 0.31/0.32 contract, but real-checkpoint evidence is currently limited to +affine 4-bit weights. ## Generation contract @@ -102,7 +112,14 @@ Transformers stopping-criteria API are not implemented. `from_pretrained` supports local checkpoint directories and Hugging Face Hub model IDs. It validates safetensors shards and indexes, rejects missing or unexpected model parameters, and supports explicit offline, cache, revision, -token, worker, and dtype controls. +token, worker, dtype, and quantization controls. Pre-quantized MLX checkpoints +are detected from checkpoint metadata and scale tensors. Regular checkpoints +can be quantized after loading through `QuantizationConfig` or the compatible +legacy flags. + +Runtime quantization has a higher peak-memory requirement because the regular +checkpoint is materialized before its supported modules are replaced. It does +not convert and publish a reusable checkpoint. PyTorch `.bin` checkpoints and remote model code are unsupported. `trust_remote_code=True` is ignored with a warning rather than executing code diff --git a/docs/load_model.md b/docs/load_model.md index e69de29..85473b4 100644 --- a/docs/load_model.md +++ b/docs/load_model.md @@ -0,0 +1,119 @@ +# Loading and running quantized models + +MLX Transformers 0.3.0 supports two bounded quantized-inference workflows on +Apple silicon: + +1. Load an MLX checkpoint that already contains quantized weights. +2. Load a regular safetensors checkpoint and quantize supported modules in + memory before inference. + +## Pre-quantized checkpoints + +Use `load_causal_model` for a supported text-generation family. Quantization +metadata and `.scales` tensors are detected automatically: + +```python +from mlx_transformers import generate_text, load_causal_model + +loaded = load_causal_model( + "mlx-community/Phi-3-mini-4k-instruct-4bit", +) + +print(loaded.quantization) +result = generate_text( + loaded.model, + loaded.tokenizer, + "Reply with one word: ready", + max_new_tokens=8, + temperature=0.0, +) +print(result.text) +``` + +For a local or cached checkpoint with networking disabled: + +```python +loaded = load_causal_model( + "/path/to/checkpoint", + local_files_only=True, +) +``` + +The loader preserves the stored quantized tensor dtypes and exposes a +`QuantizationInfo` object whose `source` is `"checkpoint"`. + +## Quantize after loading + +Pass a validated `QuantizationConfig` for a regular checkpoint: + +```python +import mlx.core as mx + +from mlx_transformers import QuantizationConfig, load_causal_model + +loaded = load_causal_model( + "meta-llama/Llama-3.2-1B-Instruct", + dtype=mx.float16, + quantization=QuantizationConfig( + group_size=64, + bits=4, + mode="affine", + ), +) +``` + +The resulting `QuantizationInfo.source` is `"runtime"`. This path temporarily +holds the regular checkpoint in memory, so its peak memory can be materially +higher than direct pre-quantized loading. It changes the in-memory model only; +it does not write or publish a converted checkpoint. + +The legacy `from_pretrained(..., quantize=True, group_size=..., bits=...)` +arguments remain supported for compatibility. Do not combine them with +`quantization=QuantizationConfig(...)`. + +## Supported settings + +Settings are validated before checkpoint loading: + +| Mode | Group size | Bits | Quantized inputs | +| --- | --- | --- | --- | +| `affine` | 32, 64, or 128 | 2, 3, 4, 5, 6, or 8 | No | +| `mxfp4` | 32 | 4 | No | +| `mxfp8` | 32 | 8 | Optional | +| `nvfp4` | 16 | 4 | Optional | + +The dimension quantized by MLX must be divisible by the chosen group size. +Runtime support also depends on the installed MLX version and Apple hardware. +The package bounds its supported MLX versions in `pyproject.toml`. + +## Command-line generation + +Pre-quantized checkpoint: + +```bash +mlx-transformers-generate \ + --model mlx-community/Phi-3-mini-4k-instruct-4bit \ + --prompt "Reply with one word: ready" \ + --max-new-tokens 8 +``` + +Runtime quantization: + +```bash +mlx-transformers-generate \ + --model meta-llama/Llama-3.2-1B-Instruct \ + --prompt "Reply with one word: ready" \ + --quantize --mode affine --group-size 64 --bits 4 \ + --max-new-tokens 8 +``` + +The CLI requires both `--model` and `--prompt`, always uses a finite token +bound, never enables remote model code, and prints the effective quantization +metadata to standard error. Add `--local-files-only` to prohibit Hub network +resolution. Use `--raw-prompt` when a tokenizer chat template should not be +applied. + +Supported causal model types are Gemma 3 text, Llama, OpenELM, Persimmon, Phi, +Phi-3, and Qwen3. OpenELM requires an explicit `--tokenizer` because its model +checkpoint does not bundle one. OpenELM and Persimmon are maintenance-only; +new verification work prioritizes active families listed in `SUPPORT.md`. diff --git a/pyproject.toml b/pyproject.toml index 6963ed4..94926fd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,9 @@ dependencies = [ "transformers>=4.57.6,<6", ] +[project.scripts] +mlx-transformers-generate = "mlx_transformers.cli:main" + [project.optional-dependencies] tokenizers = [ "sentencepiece>=0.2.0", diff --git a/src/mlx_transformers/__init__.py b/src/mlx_transformers/__init__.py index 8d0471a..2f73194 100644 --- a/src/mlx_transformers/__init__.py +++ b/src/mlx_transformers/__init__.py @@ -1,8 +1,31 @@ from importlib.metadata import PackageNotFoundError, version +from .inference import ( + GenerationResult, + LoadedCausalModel, + generate_text, + load_causal_model, + resolve_causal_model_class, +) +from .quantization import ( + QuantizationConfig, + QuantizationInfo, + get_quantization_info, +) + try: __version__ = version("mlx-transformers") except PackageNotFoundError: __version__ = "0+unknown" -__all__ = ["__version__"] +__all__ = [ + "GenerationResult", + "LoadedCausalModel", + "QuantizationConfig", + "QuantizationInfo", + "__version__", + "generate_text", + "get_quantization_info", + "load_causal_model", + "resolve_causal_model_class", +] diff --git a/src/mlx_transformers/cli.py b/src/mlx_transformers/cli.py new file mode 100644 index 0000000..0b5a5e3 --- /dev/null +++ b/src/mlx_transformers/cli.py @@ -0,0 +1,162 @@ +import argparse +import json +import sys +from typing import List, Optional + +import mlx.core as mx + +from .inference import generate_text, load_causal_model +from .quantization import QuantizationConfig + + +_DTYPES = { + "float16": mx.float16, + "bfloat16": mx.bfloat16, + "float32": mx.float32, +} + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="mlx-transformers-generate", + description=( + "Run bounded text generation with a supported MLX checkpoint. " + "Pre-quantized MLX checkpoints are detected automatically." + ), + ) + parser.add_argument( + "--model", + required=True, + help="Hugging Face model ID or local checkpoint directory.", + ) + parser.add_argument("--prompt", required=True, help="Prompt to generate from.") + parser.add_argument( + "--tokenizer", + help="Optional tokenizer ID/path; required for OpenELM.", + ) + parser.add_argument("--revision", default="main", help="Hub revision to load.") + parser.add_argument("--cache-dir", help="Hugging Face cache directory.") + parser.add_argument( + "--local-files-only", + action="store_true", + help="Use only a local directory or an already-cached Hub snapshot.", + ) + parser.add_argument( + "--dtype", + choices=sorted(_DTYPES), + default="float32", + help=( + "Floating dtype used while loading a regular checkpoint. " + "Pre-quantized checkpoint tensors keep their stored dtypes." + ), + ) + parser.add_argument( + "--quantize", + action="store_true", + help="Quantize a regular checkpoint in memory after loading it.", + ) + parser.add_argument( + "--group-size", + type=int, + help="Quantization group size; the selected MLX mode default is used if omitted.", + ) + parser.add_argument( + "--bits", + type=int, + help="Bits per weight; the selected MLX mode default is used if omitted.", + ) + parser.add_argument( + "--mode", + choices=["affine", "mxfp4", "mxfp8", "nvfp4"], + default="affine", + help="MLX quantization mode.", + ) + parser.add_argument( + "--quantize-input", + action="store_true", + help="Quantize Linear inputs; supported only by mxfp8 and nvfp4.", + ) + parser.add_argument( + "--max-new-tokens", + type=int, + default=64, + help="Finite maximum number of generated tokens.", + ) + parser.add_argument( + "--temperature", + type=float, + default=0.0, + help="Sampling temperature; zero selects greedy decoding.", + ) + parser.add_argument("--seed", type=int, default=0, help="MLX PRNG seed.") + parser.add_argument( + "--raw-prompt", + action="store_true", + help="Do not apply the tokenizer's chat template.", + ) + return parser + + +def main(argv: Optional[List[str]] = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + + if args.max_new_tokens < 0: + parser.error("--max-new-tokens must be non-negative.") + if args.temperature < 0: + parser.error("--temperature must be non-negative.") + has_quantization_options = ( + args.group_size is not None + or args.bits is not None + or args.mode != "affine" + or args.quantize_input + ) + if has_quantization_options and not args.quantize: + parser.error( + "--group-size, --bits, --mode, and --quantize-input require --quantize. " + "Omit them when loading an already-quantized MLX checkpoint." + ) + + quantization = None + if args.quantize: + try: + quantization = QuantizationConfig( + group_size=args.group_size, + bits=args.bits, + mode=args.mode, + quantize_input=args.quantize_input, + ) + except ValueError as error: + parser.error(str(error)) + + mx.random.seed(args.seed) + loaded = load_causal_model( + args.model, + tokenizer_name_or_path=args.tokenizer, + cache_dir=args.cache_dir, + revision=args.revision, + local_files_only=args.local_files_only, + dtype=_DTYPES[args.dtype], + quantization=quantization, + ) + status = ( + {"source": "none"} + if loaded.quantization is None + else loaded.quantization.as_dict() + ) + print(f"quantization={json.dumps(status, sort_keys=True)}", file=sys.stderr) + + result = generate_text( + loaded.model, + loaded.tokenizer, + args.prompt, + max_new_tokens=args.max_new_tokens, + temperature=args.temperature, + use_chat_template=not args.raw_prompt, + ) + print(result.text) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/mlx_transformers/inference.py b/src/mlx_transformers/inference.py new file mode 100644 index 0000000..edfc8a8 --- /dev/null +++ b/src/mlx_transformers/inference.py @@ -0,0 +1,190 @@ +from dataclasses import dataclass +from typing import Any, Dict, List, Optional, Type + +import mlx.core as mx +from transformers import AutoConfig, AutoTokenizer + +from .models import ( + Gemma3ForCausalLM, + LlamaForCausalLM, + OpenELMForCausalLM, + PersimmonForCausalLM, + Phi3ForCausalLM, + PhiForCausalLM, + Qwen3ForCausalLM, +) +from .quantization import ( + QuantizationConfig, + QuantizationInfo, + get_quantization_info, +) + + +ARCHITECTURE_TO_CAUSAL_MODEL = { + "Gemma3ForCausalLM": Gemma3ForCausalLM, + "LlamaForCausalLM": LlamaForCausalLM, + "OpenELMForCausalLM": OpenELMForCausalLM, + "PersimmonForCausalLM": PersimmonForCausalLM, + "Phi3ForCausalLM": Phi3ForCausalLM, + "PhiForCausalLM": PhiForCausalLM, + "Qwen3ForCausalLM": Qwen3ForCausalLM, +} + +MODEL_TYPE_TO_CAUSAL_MODEL = { + "gemma3_text": Gemma3ForCausalLM, + "llama": LlamaForCausalLM, + "openelm": OpenELMForCausalLM, + "persimmon": PersimmonForCausalLM, + "phi": PhiForCausalLM, + "phi3": Phi3ForCausalLM, + "qwen3": Qwen3ForCausalLM, +} + + +@dataclass(frozen=True) +class LoadedCausalModel: + model: Any + tokenizer: Any + quantization: Optional[QuantizationInfo] + + +@dataclass(frozen=True) +class GenerationResult: + text: str + token_ids: List[int] + + +def resolve_causal_model_class(config: Any) -> Type[Any]: + """Resolve an exported MLX causal model from a Transformers config.""" + + architectures = getattr(config, "architectures", []) or [] + for architecture in architectures: + model_class = ARCHITECTURE_TO_CAUSAL_MODEL.get(architecture) + if model_class is not None: + return model_class + + model_type = getattr(config, "model_type", None) + model_class = MODEL_TYPE_TO_CAUSAL_MODEL.get(model_type) + if model_class is not None: + return model_class + + supported = ", ".join(sorted(MODEL_TYPE_TO_CAUSAL_MODEL)) + raise ValueError( + "Unsupported causal model architecture. " + f"architectures={architectures!r}, model_type={model_type!r}. " + f"Supported model types: {supported}." + ) + + +def load_causal_model( + model_name_or_path: str, + *, + tokenizer_name_or_path: Optional[str] = None, + cache_dir: Optional[str] = None, + revision: str = "main", + local_files_only: bool = False, + token: Optional[str] = None, + dtype: Optional[Any] = None, + quantization: Optional[QuantizationConfig] = None, +) -> LoadedCausalModel: + """Load a supported causal model and tokenizer for bounded inference. + + Pre-quantized MLX checkpoints are detected automatically. Pass + ``quantization`` to quantize a regular safetensors checkpoint after load. + """ + + config = AutoConfig.from_pretrained( + model_name_or_path, + cache_dir=cache_dir, + revision=revision, + local_files_only=local_files_only, + token=token, + trust_remote_code=False, + ) + model_class = resolve_causal_model_class(config) + if ( + getattr(config, "model_type", None) == "openelm" + and tokenizer_name_or_path is None + ): + raise ValueError( + "OpenELM checkpoints do not bundle a tokenizer. Pass " + "tokenizer_name_or_path explicitly." + ) + + model = model_class(config) + model.from_pretrained( + model_name_or_path, + cache_dir=cache_dir, + revision=revision, + local_files_only=local_files_only, + token=token, + dtype=dtype, + quantization=quantization, + ) + + tokenizer_source = tokenizer_name_or_path or model_name_or_path + tokenizer = AutoTokenizer.from_pretrained( + tokenizer_source, + cache_dir=cache_dir, + revision=revision, + local_files_only=local_files_only, + token=token, + trust_remote_code=False, + ) + return LoadedCausalModel( + model=model, + tokenizer=tokenizer, + quantization=get_quantization_info(model), + ) + + +def prepare_text_inputs( + tokenizer: Any, + prompt: str, + *, + use_chat_template: bool = True, +) -> Dict[str, mx.array]: + """Tokenize one prompt for generation without creating Torch tensors.""" + + if use_chat_template and getattr(tokenizer, "chat_template", None): + encoded = tokenizer.apply_chat_template( + [{"role": "user", "content": prompt}], + add_generation_prompt=True, + tokenize=True, + return_dict=True, + return_tensors="np", + ) + else: + encoded = tokenizer(prompt, return_tensors="np") + return {key: mx.array(value) for key, value in encoded.items()} + + +def generate_text( + model: Any, + tokenizer: Any, + prompt: str, + *, + max_new_tokens: int = 64, + temperature: float = 0.0, + use_chat_template: bool = True, +) -> GenerationResult: + """Generate a bounded single-prompt response.""" + + inputs = prepare_text_inputs( + tokenizer, + prompt, + use_chat_template=use_chat_template, + ) + token_ids = [] + for token in model.generate( + inputs, + max_new_tokens=max_new_tokens, + temp=temperature, + ): + mx.eval(token) + if token.size != 1: + raise ValueError("generate_text currently supports one prompt at a time.") + token_ids.append(int(token.item())) + + text = tokenizer.decode(token_ids, skip_special_tokens=True) + return GenerationResult(text=text, token_ids=token_ids) diff --git a/src/mlx_transformers/models/base.py b/src/mlx_transformers/models/base.py index 5740ad0..f4734b1 100644 --- a/src/mlx_transformers/models/base.py +++ b/src/mlx_transformers/models/base.py @@ -17,6 +17,8 @@ import mlx.nn as nn from mlx.utils import tree_unflatten, tree_flatten +from ..quantization import QuantizationConfig, QuantizationInfo + logger = logging.getLogger(__name__) QuantizationPredicate = Optional[Callable[[str, Any], Union[bool, Dict[str, Any]]]] @@ -390,6 +392,7 @@ def _is_prequantized_checkpoint( def _quantize_model_for_checkpoint( model: "MlxPretrainedMixin", checkpoint_quantization: Dict[str, Any], + quantization_config: QuantizationConfig, tensor_keys: Set[str], ) -> None: def class_predicate(path: str, module: Any): @@ -401,9 +404,10 @@ def class_predicate(path: str, module: Any): nn.quantize( model, - group_size=checkpoint_quantization["group_size"], - bits=checkpoint_quantization["bits"], - mode=checkpoint_quantization.get("mode", "affine"), + group_size=quantization_config.group_size, + bits=quantization_config.bits, + mode=quantization_config.mode, + quantize_input=quantization_config.quantize_input, class_predicate=class_predicate, ) @@ -431,6 +435,7 @@ def from_pretrained( mode: str = "affine", quantize_input: bool = False, class_predicate: QuantizationPredicate = None, + quantization: Optional[QuantizationConfig] = None, ) -> "MlxPretrainedMixin": """ Load a pretrained model from HuggingFace Hub or local path. @@ -451,6 +456,8 @@ def from_pretrained( mode: Quantization mode passed to ``mlx.nn.quantize`` quantize_input: Whether to quantize supported layer inputs class_predicate: Optional predicate selecting which modules to quantize + quantization: Validated quantization settings. Do not combine this + with the legacy quantize/group_size/bits/mode arguments. Returns: Self with loaded model weights @@ -473,19 +480,36 @@ def from_pretrained( "dtype must be one of mx.float16, mx.bfloat16, or mx.float32." ) - should_quantize = ( + legacy_quantization_requested = ( quantize or group_size is not None or bits is not None or mode != "affine" or quantize_input - or class_predicate is not None ) - if should_quantize and quantize_input and mode not in {"nvfp4", "mxfp8"}: + if quantization is not None and not isinstance( + quantization, QuantizationConfig + ): + raise TypeError("quantization must be a QuantizationConfig instance.") + if quantization is not None and legacy_quantization_requested: raise ValueError( - "quantize_input=True is only supported for mode='nvfp4' or " - "mode='mxfp8'." + "Pass either quantization=QuantizationConfig(...) or the legacy " + "quantize/group_size/bits/mode arguments, not both." ) + runtime_quantization = quantization + should_quantize = ( + runtime_quantization is not None + or legacy_quantization_requested + or class_predicate is not None + ) + if should_quantize and runtime_quantization is None: + runtime_quantization = QuantizationConfig( + group_size=group_size, + bits=bits, + mode=mode, + quantize_input=quantize_input, + ) + self.quantization_info = None logger.info( f"Loading model from '{model_name_or_path}' " @@ -525,6 +549,7 @@ def from_pretrained( prequantized_checkpoint = self._is_prequantized_checkpoint( tensors, checkpoint_quantization ) + checkpoint_quantization_config = None if prequantized_checkpoint: if should_quantize: raise ValueError( @@ -532,10 +557,14 @@ def from_pretrained( "Load it without quantize/group_size/bits/mode arguments." ) + checkpoint_quantization_config = QuantizationConfig.from_mapping( + checkpoint_quantization + ) setattr(self.config, "quantization", checkpoint_quantization) self._quantize_model_for_checkpoint( self, checkpoint_quantization, + checkpoint_quantization_config, set(tensors), ) else: @@ -627,7 +656,7 @@ def from_pretrained( self._apply_pretrained_tensors(tensors) if should_quantize and not prequantized_checkpoint: - if quantize_input and class_predicate is None: + if runtime_quantization.quantize_input and class_predicate is None: linear_cls = nn.Linear def linear_only_predicate(_, module, linear_cls=linear_cls): @@ -642,18 +671,28 @@ def linear_only_predicate(_, module, linear_cls=linear_cls): logger.info( "Applying MLX quantization " "(group_size=%s, bits=%s, mode=%s, quantize_input=%s)", - group_size, - bits, - mode, - quantize_input, + runtime_quantization.group_size, + runtime_quantization.bits, + runtime_quantization.mode, + runtime_quantization.quantize_input, ) nn.quantize( self, - group_size=group_size, - bits=bits, - mode=mode, - quantize_input=quantize_input, + group_size=runtime_quantization.group_size, + bits=runtime_quantization.bits, + mode=runtime_quantization.mode, + quantize_input=runtime_quantization.quantize_input, class_predicate=class_predicate, ) + setattr(self.config, "quantization", runtime_quantization.as_dict()) + self.quantization_info = QuantizationInfo.from_config( + runtime_quantization, + source="runtime", + ) + elif prequantized_checkpoint: + self.quantization_info = QuantizationInfo.from_config( + checkpoint_quantization_config, + source="checkpoint", + ) self.eval() return self diff --git a/src/mlx_transformers/quantization.py b/src/mlx_transformers/quantization.py new file mode 100644 index 0000000..0158fee --- /dev/null +++ b/src/mlx_transformers/quantization.py @@ -0,0 +1,127 @@ +from dataclasses import asdict, dataclass +from typing import Any, Dict, Literal, Mapping, Optional + + +QuantizationMode = Literal["affine", "mxfp4", "mxfp8", "nvfp4"] +QuantizationSource = Literal["checkpoint", "runtime"] + +_MODE_DEFAULTS = { + "affine": (64, 4), + "mxfp4": (32, 4), + "mxfp8": (32, 8), + "nvfp4": (16, 4), +} +_AFFINE_GROUP_SIZES = {32, 64, 128} +_AFFINE_BITS = {2, 3, 4, 5, 6, 8} + + +@dataclass(frozen=True) +class QuantizationConfig: + """Validated MLX weight-quantization settings. + + Defaults match ``mlx.nn.quantize``. The normalized values are explicit so + callers can inspect and persist the exact runtime configuration. + """ + + group_size: Optional[int] = None + bits: Optional[int] = None + mode: QuantizationMode = "affine" + quantize_input: bool = False + + def __post_init__(self) -> None: + normalized_mode = str(self.mode).lower() + if normalized_mode not in _MODE_DEFAULTS: + supported = ", ".join(_MODE_DEFAULTS) + raise ValueError( + f"Unsupported quantization mode '{self.mode}'. " + f"Choose one of: {supported}." + ) + + default_group_size, default_bits = _MODE_DEFAULTS[normalized_mode] + group_size = ( + default_group_size if self.group_size is None else self.group_size + ) + bits = default_bits if self.bits is None else self.bits + + if not isinstance(group_size, int) or isinstance(group_size, bool): + raise ValueError("group_size must be an integer.") + if not isinstance(bits, int) or isinstance(bits, bool): + raise ValueError("bits must be an integer.") + + if normalized_mode == "affine": + if group_size not in _AFFINE_GROUP_SIZES: + raise ValueError( + "Affine quantization group_size must be 32, 64, or 128." + ) + if bits not in _AFFINE_BITS: + raise ValueError( + "Affine quantization bits must be one of 2, 3, 4, 5, 6, or 8." + ) + else: + expected_group_size, expected_bits = _MODE_DEFAULTS[normalized_mode] + if (group_size, bits) != (expected_group_size, expected_bits): + raise ValueError( + f"{normalized_mode} quantization requires " + f"group_size={expected_group_size} and bits={expected_bits}." + ) + + if self.quantize_input and normalized_mode not in {"mxfp8", "nvfp4"}: + raise ValueError( + "quantize_input=True is only supported for mode='nvfp4' or " + "mode='mxfp8'." + ) + + object.__setattr__(self, "mode", normalized_mode) + object.__setattr__(self, "group_size", group_size) + object.__setattr__(self, "bits", bits) + + @classmethod + def from_mapping(cls, values: Mapping[str, Any]) -> "QuantizationConfig": + if not isinstance(values, Mapping): + raise ValueError("Checkpoint quantization metadata must be a mapping.") + return cls( + group_size=values.get("group_size"), + bits=values.get("bits"), + mode=values.get("mode", "affine"), + quantize_input=values.get("quantize_input", False), + ) + + def as_dict(self) -> Dict[str, Any]: + return asdict(self) + + +@dataclass(frozen=True) +class QuantizationInfo: + """The quantization configuration applied to a loaded model.""" + + source: QuantizationSource + group_size: int + bits: int + mode: QuantizationMode + quantize_input: bool = False + + @classmethod + def from_config( + cls, + config: QuantizationConfig, + source: QuantizationSource, + ) -> "QuantizationInfo": + return cls( + source=source, + group_size=config.group_size, + bits=config.bits, + mode=config.mode, + quantize_input=config.quantize_input, + ) + + def as_dict(self) -> Dict[str, Any]: + return asdict(self) + + +def get_quantization_info(model: Any) -> Optional[QuantizationInfo]: + """Return normalized quantization metadata for a loaded model, if any.""" + + info = getattr(model, "quantization_info", None) + if info is not None and not isinstance(info, QuantizationInfo): + raise TypeError("model.quantization_info is not a QuantizationInfo instance.") + return info diff --git a/tests/test_from_pretrained.py b/tests/test_from_pretrained.py index 90fac8f..97b68a0 100644 --- a/tests/test_from_pretrained.py +++ b/tests/test_from_pretrained.py @@ -10,6 +10,7 @@ import mlx.core as mx from src.mlx_transformers.models.base import MlxPretrainedMixin +from src.mlx_transformers.quantization import QuantizationConfig class DummyTensor: @@ -85,6 +86,9 @@ def test_from_pretrained_quantizes_loaded_model(self): self.assertEqual(model.loaded_weights, ({"layer.weight": tensor}, True)) self.assertIsNone(model.updated_weights) self.assertTrue(model.eval_called) + self.assertEqual(model.quantization_info.source, "runtime") + self.assertEqual(model.quantization_info.group_size, 64) + self.assertEqual(model.config.quantization["bits"], 4) mock_quantize.assert_called_once_with( model, group_size=64, @@ -196,9 +200,12 @@ def test_from_pretrained_loads_prequantized_mlx_checkpoint(self): self.assertEqual(kwargs["group_size"], 64) self.assertEqual(kwargs["bits"], 4) self.assertEqual(kwargs["mode"], "affine") + self.assertFalse(kwargs["quantize_input"]) self.assertTrue(predicate("layer", FakeQuantizableModule())) self.assertFalse(predicate("other", FakeQuantizableModule())) self.assertFalse(predicate("layer", object())) + self.assertEqual(model.quantization_info.source, "checkpoint") + self.assertEqual(model.quantization_info.group_size, 64) def test_from_pretrained_rejects_requantizing_prequantized_checkpoint(self): model = DummyModel() @@ -224,6 +231,44 @@ def test_from_pretrained_rejects_requantizing_prequantized_checkpoint(self): ): model.from_pretrained(tmpdir, quantize=True, bits=4) + def test_from_pretrained_accepts_typed_quantization_config(self): + tensor = DummyTensor() + model = DummyModel() + + with tempfile.TemporaryDirectory() as tmpdir: + Path(tmpdir, "model.safetensors").touch() + with patch( + "src.mlx_transformers.models.base.mx.load", + return_value={"layer.weight": tensor}, + ), patch( + "src.mlx_transformers.models.base.nn.quantize", + ) as mock_quantize: + model.from_pretrained( + tmpdir, + quantization=QuantizationConfig(group_size=32, bits=3), + ) + + mock_quantize.assert_called_once_with( + model, + group_size=32, + bits=3, + mode="affine", + quantize_input=False, + class_predicate=None, + ) + self.assertEqual(model.quantization_info.source, "runtime") + self.assertEqual(model.config.quantization["group_size"], 32) + + def test_from_pretrained_rejects_mixed_quantization_apis(self): + model = DummyModel() + + with self.assertRaisesRegex(ValueError, "either quantization="): + model.from_pretrained( + "unused", + quantize=True, + quantization=QuantizationConfig(), + ) + class TestFromPretrainedLoadingContract(unittest.TestCase): def test_from_pretrained_accepts_explicit_bfloat16_dtype(self): diff --git a/tests/test_quantization.py b/tests/test_quantization.py new file mode 100644 index 0000000..d44504b --- /dev/null +++ b/tests/test_quantization.py @@ -0,0 +1,71 @@ +import unittest + +from src.mlx_transformers.quantization import ( + QuantizationConfig, + QuantizationInfo, +) + + +class TestQuantizationConfig(unittest.TestCase): + def test_affine_defaults_are_explicit(self): + config = QuantizationConfig() + + self.assertEqual(config.group_size, 64) + self.assertEqual(config.bits, 4) + self.assertEqual(config.mode, "affine") + self.assertEqual( + config.as_dict(), + { + "group_size": 64, + "bits": 4, + "mode": "affine", + "quantize_input": False, + }, + ) + + def test_specialized_modes_normalize_their_defaults(self): + self.assertEqual( + (QuantizationConfig(mode="mxfp4").group_size, + QuantizationConfig(mode="mxfp4").bits), + (32, 4), + ) + self.assertEqual( + (QuantizationConfig(mode="mxfp8").group_size, + QuantizationConfig(mode="mxfp8").bits), + (32, 8), + ) + self.assertEqual( + (QuantizationConfig(mode="nvfp4").group_size, + QuantizationConfig(mode="nvfp4").bits), + (16, 4), + ) + + def test_invalid_affine_settings_are_rejected(self): + with self.assertRaisesRegex(ValueError, "group_size must be"): + QuantizationConfig(group_size=16) + with self.assertRaisesRegex(ValueError, "bits must be"): + QuantizationConfig(bits=7) + + def test_invalid_specialized_mode_settings_are_rejected(self): + with self.assertRaisesRegex(ValueError, "mxfp4.*requires"): + QuantizationConfig(mode="mxfp4", group_size=64) + with self.assertRaisesRegex(ValueError, "mxfp8.*requires"): + QuantizationConfig(mode="mxfp8", bits=4) + + def test_quantize_input_is_restricted(self): + with self.assertRaisesRegex(ValueError, "quantize_input=True"): + QuantizationConfig(quantize_input=True) + + self.assertTrue( + QuantizationConfig(mode="nvfp4", quantize_input=True).quantize_input + ) + + def test_checkpoint_mapping_and_info_are_normalized(self): + config = QuantizationConfig.from_mapping( + {"group_size": 32, "bits": 4} + ) + info = QuantizationInfo.from_config(config, source="checkpoint") + + self.assertEqual(info.source, "checkpoint") + self.assertEqual(info.mode, "affine") + self.assertEqual(info.as_dict()["group_size"], 32) diff --git a/tests/test_quantized_inference.py b/tests/test_quantized_inference.py new file mode 100644 index 0000000..6b28d6b --- /dev/null +++ b/tests/test_quantized_inference.py @@ -0,0 +1,196 @@ +import json +import tempfile +import unittest +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +import mlx.core as mx +import mlx.nn as nn +from mlx.utils import tree_flatten +from transformers import LlamaConfig + +from src.mlx_transformers.cli import main +from src.mlx_transformers.inference import ( + GenerationResult, + LoadedCausalModel, + load_causal_model, + resolve_causal_model_class, +) +from src.mlx_transformers.models import LlamaForCausalLM +from src.mlx_transformers.quantization import ( + QuantizationConfig, + QuantizationInfo, +) + + +def tiny_llama_config() -> LlamaConfig: + return LlamaConfig( + architectures=["LlamaForCausalLM"], + vocab_size=64, + hidden_size=32, + intermediate_size=64, + num_hidden_layers=1, + num_attention_heads=4, + num_key_value_heads=4, + max_position_embeddings=32, + bos_token_id=1, + eos_token_id=2, + pad_token_id=0, + tie_word_embeddings=False, + ) + + +def save_model(model, config, directory: str, *, quantization=None) -> None: + mx.eval(model.parameters()) + mx.save_safetensors( + str(Path(directory, "model.safetensors")), + dict(tree_flatten(model.parameters())), + ) + config_data = config.to_dict() + if quantization is not None: + config_data["quantization"] = quantization + Path(directory, "config.json").write_text( + json.dumps(config_data), + encoding="utf-8", + ) + + +class TestQuantizedCheckpointExecution(unittest.TestCase): + def assert_model_executes(self, model) -> None: + inputs = {"input_ids": mx.array([[1, 5, 6]], dtype=mx.int32)} + output = model(**inputs) + mx.eval(output.logits) + self.assertTrue(bool(mx.all(mx.isfinite(output.logits)).item())) + + tokens = list( + model.generate( + inputs, + max_new_tokens=2, + temp=0.0, + ) + ) + mx.eval(tokens) + self.assertGreaterEqual(len(tokens), 1) + self.assertLessEqual(len(tokens), 2) + + def test_regular_checkpoint_can_be_quantized_and_generated(self): + config = tiny_llama_config() + source = LlamaForCausalLM(config) + + with tempfile.TemporaryDirectory() as tmpdir: + save_model(source, config, tmpdir) + loaded = LlamaForCausalLM(config) + loaded.from_pretrained( + tmpdir, + quantization=QuantizationConfig(group_size=32, bits=4), + ) + + parameter_keys = dict(tree_flatten(loaded.parameters())) + self.assertTrue(any(key.endswith(".scales") for key in parameter_keys)) + self.assertEqual(loaded.quantization_info.source, "runtime") + self.assert_model_executes(loaded) + + def test_prequantized_checkpoint_can_be_loaded_and_generated(self): + config = tiny_llama_config() + source = LlamaForCausalLM(config) + nn.quantize(source, group_size=32, bits=4) + + with tempfile.TemporaryDirectory() as tmpdir: + save_model( + source, + config, + tmpdir, + quantization={"group_size": 32, "bits": 4}, + ) + loaded = LlamaForCausalLM(config) + loaded.from_pretrained(tmpdir) + + self.assertEqual(loaded.quantization_info.source, "checkpoint") + self.assertEqual(loaded.quantization_info.group_size, 32) + self.assert_model_executes(loaded) + + +class TestInferenceAPIAndCLI(unittest.TestCase): + def test_resolver_uses_architecture_then_model_type(self): + by_architecture = resolve_causal_model_class( + SimpleNamespace( + architectures=["LlamaForCausalLM"], + model_type="unknown", + ) + ) + by_model_type = resolve_causal_model_class( + SimpleNamespace(architectures=[], model_type="llama") + ) + + self.assertIs(by_architecture, LlamaForCausalLM) + self.assertIs(by_model_type, LlamaForCausalLM) + + def test_resolver_rejects_unsupported_model(self): + with self.assertRaisesRegex(ValueError, "Unsupported causal model"): + resolve_causal_model_class( + SimpleNamespace(architectures=[], model_type="bert") + ) + + def test_openelm_requires_tokenizer_before_loading_weights(self): + config = SimpleNamespace( + architectures=["OpenELMForCausalLM"], + model_type="openelm", + ) + with patch( + "src.mlx_transformers.inference.AutoConfig.from_pretrained", + return_value=config, + ), self.assertRaisesRegex(ValueError, "OpenELM.*tokenizer"): + load_causal_model("/model") + + def test_cli_reports_quantization_and_generates(self): + info = QuantizationInfo.from_config( + QuantizationConfig(group_size=32, bits=4), + source="checkpoint", + ) + loaded = LoadedCausalModel( + model=object(), + tokenizer=object(), + quantization=info, + ) + result = GenerationResult(text="generated text", token_ids=[1, 2]) + + with patch( + "src.mlx_transformers.cli.load_causal_model", + return_value=loaded, + ) as mock_load, patch( + "src.mlx_transformers.cli.generate_text", + return_value=result, + ) as mock_generate, patch("sys.stdout") as stdout, patch( + "sys.stderr" + ) as stderr: + exit_code = main( + [ + "--model", + "/model", + "--prompt", + "hello", + "--local-files-only", + ] + ) + + self.assertEqual(exit_code, 0) + self.assertTrue(mock_load.call_args.kwargs["local_files_only"]) + self.assertEqual(mock_load.call_args.kwargs["dtype"], mx.float32) + self.assertIsNone(mock_load.call_args.kwargs["quantization"]) + mock_generate.assert_called_once() + self.assertIn("generated text", stdout.method_calls[0].args[0]) + self.assertIn('"source": "checkpoint"', stderr.method_calls[0].args[0]) + + def test_cli_requires_quantize_for_runtime_options(self): + with self.assertRaises(SystemExit): + main( + [ + "--model", + "/model", + "--prompt", + "hello", + "--bits", + "4", + ] + ) From dc72a2be273035c0709bd6a26314bec16b50ae82 Mon Sep 17 00:00:00 2001 From: Ogundepo Odunayo Date: Sun, 26 Jul 2026 16:20:50 -0400 Subject: [PATCH 2/3] Apply Ruff formatting --- src/mlx_transformers/quantization.py | 4 +--- tests/test_quantization.py | 22 +++++++++++++--------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/src/mlx_transformers/quantization.py b/src/mlx_transformers/quantization.py index 0158fee..993acc2 100644 --- a/src/mlx_transformers/quantization.py +++ b/src/mlx_transformers/quantization.py @@ -38,9 +38,7 @@ def __post_init__(self) -> None: ) default_group_size, default_bits = _MODE_DEFAULTS[normalized_mode] - group_size = ( - default_group_size if self.group_size is None else self.group_size - ) + group_size = default_group_size if self.group_size is None else self.group_size bits = default_bits if self.bits is None else self.bits if not isinstance(group_size, int) or isinstance(group_size, bool): diff --git a/tests/test_quantization.py b/tests/test_quantization.py index d44504b..3344bf0 100644 --- a/tests/test_quantization.py +++ b/tests/test_quantization.py @@ -25,18 +25,24 @@ def test_affine_defaults_are_explicit(self): def test_specialized_modes_normalize_their_defaults(self): self.assertEqual( - (QuantizationConfig(mode="mxfp4").group_size, - QuantizationConfig(mode="mxfp4").bits), + ( + QuantizationConfig(mode="mxfp4").group_size, + QuantizationConfig(mode="mxfp4").bits, + ), (32, 4), ) self.assertEqual( - (QuantizationConfig(mode="mxfp8").group_size, - QuantizationConfig(mode="mxfp8").bits), + ( + QuantizationConfig(mode="mxfp8").group_size, + QuantizationConfig(mode="mxfp8").bits, + ), (32, 8), ) self.assertEqual( - (QuantizationConfig(mode="nvfp4").group_size, - QuantizationConfig(mode="nvfp4").bits), + ( + QuantizationConfig(mode="nvfp4").group_size, + QuantizationConfig(mode="nvfp4").bits, + ), (16, 4), ) @@ -61,9 +67,7 @@ def test_quantize_input_is_restricted(self): ) def test_checkpoint_mapping_and_info_are_normalized(self): - config = QuantizationConfig.from_mapping( - {"group_size": 32, "bits": 4} - ) + config = QuantizationConfig.from_mapping({"group_size": 32, "bits": 4}) info = QuantizationInfo.from_config(config, source="checkpoint") self.assertEqual(info.source, "checkpoint") From 61509290352dbb9a0267af1727031692e97036aa Mon Sep 17 00:00:00 2001 From: Ogundepo Odunayo Date: Sun, 26 Jul 2026 16:31:38 -0400 Subject: [PATCH 3/3] Keep package metadata import dependency-free --- CONTRIBUTING.md | 2 +- README.md | 4 +-- src/mlx_transformers/__init__.py | 42 ++++++++++++++++++++---------- tests/test_package_import.py | 44 ++++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 16 deletions(-) create mode 100644 tests/test_package_import.py diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6aaf3df..be55ec9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 \ python -m unittest discover -s tests -v ``` -The expected baseline is 113 discovered tests: 91 pass and 22 Hub tests skip. +The expected baseline is 114 discovered tests: 92 pass and 22 Hub tests skip. The exact count should change only when tests are intentionally added or removed. diff --git a/README.md b/README.md index eb1a85a..8dbf2c9 100644 --- a/README.md +++ b/README.md @@ -240,5 +240,5 @@ Tests that use external checkpoints are skipped unless download sizes before opting in. Some checkpoints are gated; set `HF_TOKEN` only for an explicitly reviewed integration run. -The verified 2026-07-26 Apple-silicon baseline is 113 discovered tests: -91 pass and 22 Hub integration tests skip. +The verified 2026-07-26 Apple-silicon baseline is 114 discovered tests: +92 pass and 22 Hub integration tests skip. diff --git a/src/mlx_transformers/__init__.py b/src/mlx_transformers/__init__.py index 2f73194..97e84c6 100644 --- a/src/mlx_transformers/__init__.py +++ b/src/mlx_transformers/__init__.py @@ -1,18 +1,6 @@ +from importlib import import_module from importlib.metadata import PackageNotFoundError, version -from .inference import ( - GenerationResult, - LoadedCausalModel, - generate_text, - load_causal_model, - resolve_causal_model_class, -) -from .quantization import ( - QuantizationConfig, - QuantizationInfo, - get_quantization_info, -) - try: __version__ = version("mlx-transformers") except PackageNotFoundError: @@ -29,3 +17,31 @@ "load_causal_model", "resolve_causal_model_class", ] + +_LAZY_IMPORTS = { + "GenerationResult": (".inference", "GenerationResult"), + "LoadedCausalModel": (".inference", "LoadedCausalModel"), + "QuantizationConfig": (".quantization", "QuantizationConfig"), + "QuantizationInfo": (".quantization", "QuantizationInfo"), + "generate_text": (".inference", "generate_text"), + "get_quantization_info": (".quantization", "get_quantization_info"), + "load_causal_model": (".inference", "load_causal_model"), + "resolve_causal_model_class": (".inference", "resolve_causal_model_class"), +} + + +def __getattr__(name): + try: + module_name, attribute_name = _LAZY_IMPORTS[name] + except KeyError as error: + raise AttributeError( + f"module {__name__!r} has no attribute {name!r}" + ) from error + + value = getattr(import_module(module_name, __name__), attribute_name) + globals()[name] = value + return value + + +def __dir__(): + return sorted(set(globals()) | set(__all__)) diff --git a/tests/test_package_import.py b/tests/test_package_import.py new file mode 100644 index 0000000..b8cb76d --- /dev/null +++ b/tests/test_package_import.py @@ -0,0 +1,44 @@ +import os +import subprocess +import sys +import textwrap +import unittest +from pathlib import Path + + +class TestPackageImportContract(unittest.TestCase): + def test_version_import_does_not_require_mlx_runtime(self): + script = textwrap.dedent( + """ + import sys + + class BlockMlxImports: + def find_spec(self, fullname, path=None, target=None): + if fullname == "mlx" or fullname.startswith("mlx."): + raise ModuleNotFoundError("MLX intentionally unavailable") + return None + + sys.meta_path.insert(0, BlockMlxImports()) + + import mlx_transformers + + assert mlx_transformers.__version__ + assert "mlx" not in sys.modules + """ + ) + environment = os.environ.copy() + source_root = str(Path(__file__).resolve().parents[1] / "src") + existing_python_path = environment.get("PYTHONPATH") + environment["PYTHONPATH"] = ( + source_root + if not existing_python_path + else os.pathsep.join((source_root, existing_python_path)) + ) + result = subprocess.run( + [sys.executable, "-c", script], + capture_output=True, + env=environment, + text=True, + ) + + self.assertEqual(result.returncode, 0, result.stderr)