diff --git a/common/fit.cpp b/common/fit.cpp index 55b457e1060d..064d766cdd69 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -11,6 +11,8 @@ #include #include #include +#include +#include #include #include #include @@ -81,6 +83,26 @@ static std::vector common_get_device_memory_data_impl( throw std::runtime_error("failed to create llama_context from model"); } + // a draft measured next to its target (ctx_other set by common_fit_draft_target): the speculative decoder turns on + // nextn embeddings for DFlash/DSpark/Eagle3 drafts after creating them and installs the backend samplers only on + // those (not on DFlash2 / LiliCorr), so do the same and reserve again: the breakdown below then covers the compute + // graph the draft actually runs with + if (cparams->ctx_other != nullptr) { + const bool spec_draft = llama_model_target_layer_ids_n(model) > 0; + char buf[32] = {}; + const bool is_lilicorr = llama_model_meta_val_str(model, "lilicorr.candidate_topk", buf, sizeof(buf)) >= 0 && std::atoi(buf) > 0; + const bool is_dflash2 = llama_model_dflash_selector_top_k(model) > 0; + if (!spec_draft || is_dflash2 || is_lilicorr) { + for (size_t i = 0; i < cparams->n_samplers; i++) { + llama_set_sampler(ctx, cparams->samplers[i].seq_id, nullptr); + } + } + if (spec_draft) { + llama_set_embeddings_nextn(ctx, true, /*masked*/ !is_dflash2 && !is_lilicorr); + } + llama_sched_reserve(ctx); + } + const size_t nd = llama_model_n_devices(model); std::vector ret(nd + 1); @@ -257,6 +279,67 @@ common_device_memory_data_vec common_get_device_memory_data( return ret; } +// drafts that borrow token_embd/output from their target (DFlash, DSpark, Eagle3) cannot create a context on their +// own: while measuring such a draft, give it an allocation-free target context as ctx_other (its memory is counted with +// the main model, not here) and the backend top-k samplers the speculative decoder installs on the draft +struct common_fit_draft_target { + llama_context_params * cparams_dft; + llama_context_params cparams_dft_orig; + llama_model * model = nullptr; + llama_context * ctx = nullptr; + std::vector chains; + std::vector configs; + + common_fit_draft_target(const char * path_model, const llama_model_params * mparams, const llama_context_params * cparams, + llama_context_params * cparams_dft, bool backend_sampling, ggml_log_level log_level) + : cparams_dft(cparams_dft), cparams_dft_orig(*cparams_dft) { + struct user_data_t { + ggml_log_callback callback; + void * user_data; + ggml_log_level min_level; + } ud; + llama_log_get(&ud.callback, &ud.user_data); + ud.min_level = log_level; + llama_log_set([](ggml_log_level level, const char * text, void * user_data) { + const user_data_t * ud = (const user_data_t *) user_data; + ud->callback(level >= ud->min_level ? level : GGML_LOG_LEVEL_DEBUG, text, ud->user_data); + }, &ud); + + llama_model_params mparams_copy = *mparams; + mparams_copy.no_alloc = true; + mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE; + model = llama_model_load_from_file(path_model, mparams_copy); + if (model != nullptr) { + ctx = llama_init_from_model(model, *cparams); + } + llama_log_set(ud.callback, ud.user_data); + if (ctx == nullptr) { + return; // the draft is measured on its own, as before + } + + cparams_dft->ctx_other = ctx; + if (backend_sampling && cparams_dft->n_samplers == 0) { + for (llama_seq_id seq_id = 0; seq_id < (llama_seq_id) std::max(1, cparams_dft->n_seq_max); ++seq_id) { + llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); + llama_sampler_chain_add(chain, llama_sampler_init_top_k(10)); + chains.push_back(chain); + configs.push_back({ seq_id, chain }); + } + cparams_dft->samplers = configs.data(); + cparams_dft->n_samplers = configs.size(); + } + } + + ~common_fit_draft_target() { + *cparams_dft = cparams_dft_orig; + for (llama_sampler * chain : chains) { + llama_sampler_free(chain); + } + llama_free(ctx); + llama_model_free(model); + } +}; + static void common_params_fit_impl( const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, @@ -299,6 +382,12 @@ static void common_params_fit_impl( LOG_TRC("%s: getting device memory data for the extra model at a context size of %" PRIu32 ":\n", __func__, cparams->n_ctx); + std::unique_ptr draft_target; + if (!extra->shares_model) { + draft_target.reset(new common_fit_draft_target( + path_model, mparams, cparams, extra->cparams, extra->backend_sampling, log_level)); + } + dmds_t measured; try { measured = common_get_device_memory_data_impl( diff --git a/common/fit.h b/common/fit.h index 47478bba12e0..249caf80afb1 100644 --- a/common/fit.h +++ b/common/fit.h @@ -15,11 +15,13 @@ enum common_params_fit_status { // a second model that shares the devices of the main model, e.g. a draft model // - its context follows the context of the main model, so its memory is measured again whenever that context changes // - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context +// - backend_sampling adds the backend sampler of the speculative decoder to the measured draft graph struct common_fit_extra_model { const char * path_model; llama_model_params * mparams; llama_context_params * cparams; bool shares_model; + bool backend_sampling = true; // the speculative decoder offloads draft sampling to the backend (its default) }; // fits mparams and cparams to free device memory (assumes system memory is unlimited) diff --git a/common/parsers/deepseek.cpp b/common/parsers/deepseek.cpp index dac180715a52..73bb4fce8ecf 100644 --- a/common/parsers/deepseek.cpp +++ b/common/parsers/deepseek.cpp @@ -195,6 +195,14 @@ common_chat_params common_chat_params_init_deepseek_v3_2(const common_chat_templ foreach_function(tools, [&](size_t tool_index, const json & tool) { const auto & function = tool.at("function"); std::string name = function.at("name"); + if (is_v41 && name.find("::") == std::string::npos) { + // Match qualified_name() in the V4.1 template; rendering above validates conflicts. + const auto ns = tool.contains("namespace") && !tool.at("namespace").is_null() + ? tool.at("namespace") : function.value("namespace", json()); + if (!ns.is_null()) { + name = (ns.is_object() ? ns.at("name") : ns).get() + "::" + name; + } + } std::vector required_parsers; std::vector optional_parsers; diff --git a/conversion/deepseek41.py b/conversion/deepseek41.py index 53980c1d041f..fa4f0879ef6c 100644 --- a/conversion/deepseek41.py +++ b/conversion/deepseek41.py @@ -2,6 +2,7 @@ import json import math +import os import re from pathlib import Path from typing import Any, Callable, Iterable, TYPE_CHECKING, cast @@ -117,9 +118,10 @@ def _e8m0_check(scale_u8: np.ndarray, name: str) -> None: @ModelBase.register("DeepseekV41ForCausalLM") class DeepseekV41Model(DeepseekV4Model): """DeepSeek-V4.1 text model, lossless: FP8 (e4m3, 32x32 ue8m0 blocks) linears dequantized to BF16 (F32 with - --outtype f32), routed experts copied bit-exact as MXFP4, Engram tables (e4m3 rows, one e8m0 scale per 32) as the - exact BF16 lookup (F32 with --outtype f32), router and Engram gate F32. DSpark (mtp.*) is not converted here - (--dspark, DeepseekV41DSparkModel); the vision tower goes to an mmproj (--mmproj, DeepseekV41VisionModel). + --outtype f32), routed experts copied bit-exact as MXFP4, Engram tables (e4m3 rows, one e8m0 scale per 32) copied + bit-exact as MXFP8 (DSV41_ENGRAM_TYPE=bf16: the exact BF16 lookup, F32 with --outtype f32), router and Engram gate + F32. DSpark (mtp.*) is not converted here (--dspark, DeepseekV41DSparkModel); the vision tower goes to an mmproj + (--mmproj, DeepseekV41VisionModel). The Engram tables of all layers are written as one per_layer_token_embd (rows concatenated in layer order) with the PLE keys of qwen4exp: absolute head offsets and bucket counts, so the runtime can read it lazily.""" model_arch = gguf.MODEL_ARCH.DEEPSEEK41 @@ -131,6 +133,7 @@ def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.hparams.setdefault("num_hash_layers", 0) # no hash-routed layers in V4.1 self._f32 = self.ftype == gguf.LlamaFileType.ALL_F32 + self._engram_mxfp8 = os.environ.get("DSV41_ENGRAM_TYPE", "mxfp8").lower() != "bf16" qc = self.hparams.get("quantization_config") or {} self._fp8_block = tuple(qc.get("weight_block_size") or (32, 32)) if qc.get("scale_fmt", "ue8m0") != "ue8m0": @@ -138,8 +141,10 @@ def __init__(self, *args, **kwargs): if self.hparams.get("gate_temp", 1.0) != 1.0: raise ValueError("gate_temp != 1 is not supported (the router logits would need dividing by it)") - # V4.1 does not use the DeepSeek-V4 template; a template shipped with the model is still added by set_vocab - self.gguf_writer.remove_key(gguf.Keys.Tokenizer.CHAT_TEMPLATE) + template_path = Path(__file__).parent.parent / "models" / "templates" / "deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja" + if template_path.is_file(): + self.gguf_writer.remove_key(gguf.Keys.Tokenizer.CHAT_TEMPLATE) + self.gguf_writer.add_chat_template(template_path.read_text(encoding="utf-8")) def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Callable[[], Tensor]]: # num_hidden_layers lives in text_config, which is merged only after indexing; MTP stages are skipped anyway @@ -221,7 +226,7 @@ def chunks() -> Iterable[np.ndarray]: self.gguf_writer.add_tensor(self.format_tensor_name(key, bid), data, raw_dtype=gguf.GGMLQuantizationType.MXFP4) def _write_engram_tables(self, layer_ids: list[int]) -> None: - # official lookup e4m3 * 2^(e8m0 - 127), exact in BF16 + # official lookup e4m3 * 2^(e8m0 - 127): stored as the source MXFP8 blocks (bit-exact) or as exact BF16 parts, dim = [], None for bid in layer_ids: wn, sn = f"layers.{bid}.engram.embed.weight", f"layers.{bid}.engram.embed.scale" @@ -236,6 +241,22 @@ def _write_engram_tables(self, layer_ids: list[int]) -> None: name = self.format_tensor_name(gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD) n_rows = sum(p[2] for p in parts) + if self._engram_mxfp8: + def blocks() -> Iterable[np.ndarray]: + from safetensors import safe_open + for wn, sn, rows, wpart, spart in parts: + with safe_open(wpart, framework="pt") as fw, safe_open(spart, framework="pt") as fs: + ws, ss = fw.get_slice(wn), fs.get_slice(sn) + for r0 in range(0, rows, 1 << 18): + q = ws[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 32).numpy() + e = ss[r0:r0 + (1 << 18)].view(torch.uint8).reshape(-1, 1).numpy() + if (e == 0xFF).any() or ((q & 0x7F) == 0x7F).any(): + raise ValueError(f"{wn}: NaN E8M0 scale or E4M3 code") + yield np.concatenate([e, q], axis=1) + data = cast(np.ndarray, _StreamedBytes((n_rows, dim // 32 * 33), blocks)) + self.gguf_writer.add_tensor(name, data, raw_dtype=gguf.GGMLQuantizationType.MXFP8) + return + def chunks() -> Iterable[np.ndarray]: from safetensors import safe_open for wn, sn, rows, wpart, spart in parts: diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h index 3ed4181408fb..0ecf5a0a3e9c 100644 --- a/ggml/include/ggml.h +++ b/ggml/include/ggml.h @@ -432,8 +432,18 @@ extern "C" { GGML_TYPE_NVFP4 = 40, // NVFP4 (4 blocks, E4M3 scale) GGML_TYPE_Q1_0 = 41, GGML_TYPE_Q2_0 = 42, + // MXFP8 (E4M3, E8M0 scale), gather-only. Fork-local, top of the reserved range so the + // count above the IQ1 types stays put; files written while it was 43 (before upstream + // took 43 for F8_E4M3) are told apart by their byte size when read (gguf.cpp). + GGML_TYPE_MXFP8 = 63, + // IQ1_S's 8-element ternary codebook with a narrower index; see ggml-common.h. + // 42..63 stay reserved for upstream (42 is already Q2_0), so fork-local types start + // at 64 and a stock build rejects these files instead of misreading them. + GGML_TYPE_IQ1_XS = 64, // 1.4375 bpw, 1024-entry grid + GGML_TYPE_IQ1_XXS = 65, // 1.3125 bpw, 512-entry grid + GGML_TYPE_IQ1_XXXS = 66, // 1.1875 bpw, 256-entry grid + GGML_TYPE_COUNT = 67, GGML_TYPE_F8_E4M3 = 43, - GGML_TYPE_COUNT = 44, }; // [TAG_GGML_PREC] @@ -489,6 +499,10 @@ extern "C" { GGML_FTYPE_MOSTLY_NVFP4 = 26, // except 1d tensors GGML_FTYPE_MOSTLY_Q1_0 = 27, // except 1d tensors GGML_FTYPE_MOSTLY_Q2_0 = 28, // except 1d tensors + // 28..39 are reserved for upstream ggml, which is already using 28 for Q2_0. + GGML_FTYPE_MOSTLY_IQ1_XS = 40, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_XXS = 41, // except 1d tensors + GGML_FTYPE_MOSTLY_IQ1_XXXS = 42, // except 1d tensors GGML_FTYPE_MOSTLY_F8_E4M3 = 29, // except 1d tensors }; diff --git a/ggml/src/ggml-common.h b/ggml/src/ggml-common.h index 1dbbe326d0fc..bf214e2f9d8a 100644 --- a/ggml/src/ggml-common.h +++ b/ggml/src/ggml-common.h @@ -157,6 +157,13 @@ typedef sycl::half2 ggml_half2; #define QI1_S (QK_K / (4*QR1_S)) #define QR1_S 8 +#define QI1_XS (QK_K / (4*QR1_XS)) +#define QR1_XS 8 +#define QI1_XXS (QK_K / (4*QR1_XXS)) +#define QR1_XXS 8 +#define QI1_XXXS (QK_K / (4*QR1_XXXS)) +#define QR1_XXXS 8 + #define QI1_M (QK_K / (4*QR1_M)) #define QR1_M 8 @@ -218,6 +225,14 @@ typedef struct { } block_mxfp4; static_assert(sizeof(block_mxfp4) == sizeof(uint8_t) + QK_MXFP4/2, "wrong mxfp4 block size/padding"); +// MXFP8 (OCP MX): 32 E4M3 (fn) values sharing one E8M0 scale, value = e4m3(q[j]) * 2^(e - 127) +#define QK_MXFP8 32 +typedef struct { + uint8_t e; // E8M0 + uint8_t qs[QK_MXFP8]; // E4M3 (fn) codes +} block_mxfp8; +static_assert(sizeof(block_mxfp8) == sizeof(uint8_t) + QK_MXFP8, "wrong mxfp8 block size/padding"); + #define QK_NVFP4 64 #define QK_NVFP4_SUB 16 // sub-block size for per-group scales typedef struct { @@ -437,6 +452,44 @@ typedef struct { } block_iq1_m; static_assert(sizeof(block_iq1_m) == QK_K/8 + QK_K/16 + QK_K/32, "wrong iq1_m block size/padding"); +// Below IQ1_S: the same 8-dimensional ternary codebook with a NARROWER index. +// +// IQ1_S spends 1.3750 of its 1.5625 bpw on an 11-bit index into a 2048-entry grid, i.e. 88% of +// the whole budget, so the index width is the only field worth shrinking. Every other field +// keeps its IQ1_S meaning exactly: `d` is the fp16 super-block scale, the 3-bit sub-block code +// gives `dl = d*(2*ls+1)`, and the sign bit selects `delta = +/-IQ1S_DELTA`. Dequantisation is +// therefore identical to `dequantize_row_iq1_s` apart from how the index is reassembled. +// +// Per 32-element sub-block (QK_K/32 = 8 of them) each type must store 4 index-high fields, a +// 3-bit scale and a sign bit. That is 12 / 8 / 4 bits for a 10 / 9 / 8 bit index, which packs +// byte-exactly at 46 / 42 / 38 bytes per 256 weights. + +// 1.4375 bpw, 1024-entry grid, 10-bit index +typedef struct { + ggml_half d; + uint8_t qs[QK_K/8]; // index, low 8 bits + uint8_t qh[QK_K/32]; // 4 x 2-bit index-high, one per group of 8 + uint8_t sc[QK_K/64]; // nibble per sub-block: bits0-2 scale, bit3 delta sign +} block_iq1_xs; +static_assert(sizeof(block_iq1_xs) == sizeof(ggml_half) + QK_K/8 + QK_K/32 + QK_K/64, "wrong iq1_xs block size/padding"); + +// 1.3125 bpw, 512-entry grid, 9-bit index. The one that packs perfectly: the single index-high +// bit per group leaves exactly room for the scale and sign in the same byte. +typedef struct { + ggml_half d; + uint8_t qs[QK_K/8]; // index, low 8 bits + uint8_t qh[QK_K/32]; // bits0-3 index-high (one per group of 8), bits4-6 scale, bit7 delta sign +} block_iq1_xxs; +static_assert(sizeof(block_iq1_xxs) == sizeof(ggml_half) + QK_K/8 + QK_K/32, "wrong iq1_xxs block size/padding"); + +// 1.1875 bpw, 256-entry grid, 8-bit index. The index fits in qs alone, so there is no qh at all. +typedef struct { + ggml_half d; + uint8_t qs[QK_K/8]; // the whole 8-bit index + uint8_t sc[QK_K/64]; // nibble per sub-block: bits0-2 scale, bit3 delta sign +} block_iq1_xxxs; +static_assert(sizeof(block_iq1_xxxs) == sizeof(ggml_half) + QK_K/8 + QK_K/64, "wrong iq1_xxxs block size/padding"); + // Used by IQ1_M quants typedef union { ggml_half f16; @@ -1131,6 +1184,9 @@ GGML_TABLE_END() #define NGRID_IQ1S 2048 #define IQ1S_DELTA 0.125f #define IQ1M_DELTA 0.125f +#define NGRID_IQ1XS 1024 +#define NGRID_IQ1XXS 512 +#define NGRID_IQ1XXXS 256 #if defined(GGML_COMMON_IMPL_C) || defined(GGML_COMMON_IMPL_CPP) GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S) 0xffffffffffffffff, 0xffffffffffffff01, 0xffffffffffff0000, 0xffffffffffff01ff, @@ -1646,6 +1702,471 @@ GGML_TABLE_BEGIN(uint64_t, iq1s_grid, NGRID_IQ1S) 0x0101010101ffff01, 0x0101010101ff01ff, 0x0101010101ff0101, 0x0101010101000000, 0x010101010101ffff, 0x010101010101ff01, 0x01010101010101ff, 0x0101010101010101, GGML_TABLE_END() + +// Generated by scripts/iqxs_gen_tables.py. Do not edit by hand. +// Subsets of iq1s_grid chosen by imatrix-weighted greedy facility-location on real +// Kimi-K3 expert weights; every entry is an existing ggml grid point, so the int8 +// representability ggml_vec_dot_iq1_s_q8_K relies on is inherited rather than assumed. + +GGML_TABLE_BEGIN(uint64_t, iq1_xs_grid, NGRID_IQ1XS) + 0xffffffffffffffff, 0xffffffffff000000, 0xffffffff000000ff, 0xffffffff00000001, + 0xffffffff01000000, 0xffffff00ff00ff00, 0xffffff0000ffff00, 0xffffff0000ff0100, + 0xffffff000000ff01, 0xffffff0000000000, 0xffffff0000000101, 0xffffff00000100ff, + 0xffffff0001010000, 0xffffff01ff000000, 0xffff00ffff0000ff, 0xffff00ffff000001, + 0xffff00ff00ffff00, 0xffff00ff00ff0100, 0xffff00ff00000000, 0xffff00ff00010000, + 0xffff0000ffff00ff, 0xffff0000ffff0001, 0xffff0000ff000000, 0xffff000000ff0000, + 0xffff000000ff0101, 0xffff00000000ffff, 0xffff00000000ff00, 0xffff0000000000ff, + 0xffff000000000000, 0xffff000000000001, 0xffff000000000100, 0xffff00000001ffff, + 0xffff00000001ff01, 0xffff000000010000, 0xffff0000000101ff, 0xffff00000100ff00, + 0xffff000001000000, 0xffff0000010001ff, 0xffff0001ff0000ff, 0xffff0001ff000100, + 0xffff000100ffff00, 0xffff000100000000, 0xffff0001000001ff, 0xffff000100010001, + 0xffff000101ff0000, 0xffff000101000100, 0xffff01ffff000000, 0xffff01ff00000001, + 0xffff01ff01000000, 0xffff0100ffff0000, 0xffff0100ff000100, 0xffff0100ff0100ff, + 0xffff01000000ff00, 0xffff010000000000, 0xffff0100010000ff, 0xffff0101ff000000, + 0xffff010101000000, 0xff00ffffff010000, 0xff00ffff00ff00ff, 0xff00ffff0000ffff, + 0xff00ffff00000000, 0xff00ffff00010100, 0xff00ffff0100ff00, 0xff00ffff01000001, + 0xff00ff00ffff00ff, 0xff00ff00ff00ffff, 0xff00ff00ff00ff01, 0xff00ff00ff000000, + 0xff00ff00ff0001ff, 0xff00ff0000ff0000, 0xff00ff000000ff00, 0xff00ff00000000ff, + 0xff00ff0000000000, 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff000001ffff, + 0xff00ff0000010000, 0xff00ff0001000000, 0xff00ff0100ffffff, 0xff00ff0100ff0001, + 0xff00ff0100ff0100, 0xff00ff0100000000, 0xff00ff0100010001, 0xff00ff010100ff00, + 0xff00ff01010000ff, 0xff00ff0101000001, 0xff0000ffffffff00, 0xff0000ffff0000ff, + 0xff0000ffff000000, 0xff0000ffff000100, 0xff0000ffff01ff00, 0xff0000ff00ff0000, + 0xff0000ff00ff0001, 0xff0000ff00ff01ff, 0xff0000ff0000ff00, 0xff0000ff000000ff, + 0xff0000ff00000000, 0xff0000ff00000001, 0xff0000ff00000100, 0xff0000ff00010000, + 0xff0000ff000101ff, 0xff0000ff01ff0100, 0xff0000ff010000ff, 0xff0000ff01000000, + 0xff0000ff01000100, 0xff000000ffff0000, 0xff000000ffff0101, 0xff000000ff00ff00, + 0xff000000ff0000ff, 0xff000000ff000000, 0xff000000ff000001, 0xff000000ff000100, + 0xff000000ff01ff01, 0xff000000ff010000, 0xff000000ff0101ff, 0xff000000ff010101, + 0xff00000000ffff00, 0xff00000000ff00ff, 0xff00000000ff0000, 0xff00000000ff0001, + 0xff0000000000ff00, 0xff0000000000ff01, 0xff000000000000ff, 0xff00000000000000, + 0xff00000000000001, 0xff00000000000100, 0xff00000000000101, 0xff0000000001ff00, + 0xff000000000100ff, 0xff00000000010000, 0xff00000000010001, 0xff00000000010100, + 0xff00000001ffff01, 0xff00000001ff00ff, 0xff00000001ff0000, 0xff0000000100ff00, + 0xff000000010000ff, 0xff00000001000000, 0xff00000001000001, 0xff00000001000100, + 0xff00000001000101, 0xff0000000101ffff, 0xff0000000101ff01, 0xff00000001010000, + 0xff000001ff000000, 0xff00000100ffffff, 0xff00000100ff0000, 0xff00000100ff01ff, + 0xff0000010000ff00, 0xff00000100000000, 0xff00000100000001, 0xff00000100000100, + 0xff0000010001ff00, 0xff000001000100ff, 0xff00000100010000, 0xff00000100010100, + 0xff00000100010101, 0xff00000101ff0001, 0xff00000101000000, 0xff000001010100ff, + 0xff0001ff00ffff00, 0xff0001ff00ff00ff, 0xff0001ff0000ffff, 0xff0001ff00000000, + 0xff0001ff00000101, 0xff0001ff000100ff, 0xff0001ff01ff0000, 0xff000100ff00ffff, + 0xff000100ff00ff01, 0xff000100ff000000, 0xff000100ff01ff00, 0xff00010000ff0000, + 0xff00010000ff01ff, 0xff0001000000ff00, 0xff000100000000ff, 0xff00010000000000, + 0xff00010000000001, 0xff00010000000100, 0xff00010000010000, 0xff0001000100ff01, + 0xff00010001000000, 0xff000100010001ff, 0xff00010001010001, 0xff00010001010100, + 0xff000101ff000001, 0xff000101ff0100ff, 0xff00010100ff0001, 0xff00010100ff0100, + 0xff0001010000ffff, 0xff00010100000000, 0xff000101000001ff, 0xff00010100010001, + 0xff00010101ff0000, 0xff01ffff0000ff00, 0xff01ffff00000100, 0xff01ff00ff0000ff, + 0xff01ff0000ffff01, 0xff01ff0000000000, 0xff01ff00000001ff, 0xff01ff0000000101, + 0xff01ff00000100ff, 0xff01ff000100ffff, 0xff01ff0001000001, 0xff01ff0001010000, + 0xff01ff01ff000000, 0xff01ff0100ff0000, 0xff0100ffff000001, 0xff0100ff00ff0000, + 0xff0100ff0000ff01, 0xff0100ff00000000, 0xff0100ff000001ff, 0xff0100ff00010001, + 0xff0100ff0100ff00, 0xff010000ffff0100, 0xff010000ff000000, 0xff01000000ffffff, + 0xff01000000ff0000, 0xff0100000000ff00, 0xff010000000000ff, 0xff01000000000000, + 0xff01000000000100, 0xff0100000001ff01, 0xff01000000010000, 0xff01000001ff0100, + 0xff010000010000ff, 0xff01000001000000, 0xff01000001010100, 0xff010001ffff0000, + 0xff010001ff00ffff, 0xff010001ff00ff01, 0xff010001ff000100, 0xff010001ff010000, + 0xff01000100000000, 0xff01000101ff00ff, 0xff0101ffff000000, 0xff0101ff00010000, + 0xff010100ff0000ff, 0xff010100ff000100, 0xff01010000ff0001, 0xff01010000000000, + 0xff01010000010100, 0xff0101000100ffff, 0xff01010001000001, 0xff0101010000ff00, + 0xff01010100000001, 0xff01010101000000, 0x00ffffffffff0000, 0x00ffffffff00ff00, + 0x00ffffff0000ff01, 0x00ffffff00000000, 0x00ffffff000001ff, 0x00ffffff000100ff, + 0x00ffffff00010001, 0x00ffffff01000100, 0x00ffff00ffff00ff, 0x00ffff00ff000000, + 0x00ffff00ff000001, 0x00ffff00ff0001ff, 0x00ffff00ff01ff00, 0x00ffff00ff010100, + 0x00ffff0000ff0000, 0x00ffff000000ff00, 0x00ffff00000000ff, 0x00ffff0000000000, + 0x00ffff0000000001, 0x00ffff0000000100, 0x00ffff0000000101, 0x00ffff0000010000, + 0x00ffff0001ff00ff, 0x00ffff0001ff0001, 0x00ffff000100ffff, 0x00ffff0001000000, + 0x00ffff000101ff01, 0x00ffff01ff00ff00, 0x00ffff01ff000001, 0x00ffff0100ffff00, + 0x00ffff0100000000, 0x00ffff0100010100, 0x00ffff0101ff0100, 0x00ff00ffffffff00, + 0x00ff00ffff000000, 0x00ff00ffff000100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, + 0x00ff00ff00ff0101, 0x00ff00ff0000ff00, 0x00ff00ff000000ff, 0x00ff00ff00000000, + 0x00ff00ff00000001, 0x00ff00ff0001ff00, 0x00ff00ff00010000, 0x00ff00ff00010101, + 0x00ff00ff01ffff00, 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff00ff01000000, + 0x00ff0000ffff0000, 0x00ff0000ff00ff00, 0x00ff0000ff0000ff, 0x00ff0000ff000000, + 0x00ff0000ff000001, 0x00ff0000ff000100, 0x00ff0000ff01ffff, 0x00ff0000ff010000, + 0x00ff0000ff010101, 0x00ff000000ffff00, 0x00ff000000ff00ff, 0x00ff000000ff0000, + 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff, 0x00ff00000000ff00, + 0x00ff0000000000ff, 0x00ff000000000000, 0x00ff000000000001, 0x00ff0000000001ff, + 0x00ff000000000100, 0x00ff00000001ff00, 0x00ff0000000100ff, 0x00ff000000010000, + 0x00ff000000010001, 0x00ff000000010100, 0x00ff000001ffff01, 0x00ff000001ff0000, + 0x00ff00000100ff00, 0x00ff0000010000ff, 0x00ff000001000000, 0x00ff000001000001, + 0x00ff000001000100, 0x00ff000001010000, 0x00ff0000010101ff, 0x00ff000001010101, + 0x00ff0001ffff0000, 0x00ff0001ff000000, 0x00ff0001ff000101, 0x00ff000100ff0000, + 0x00ff000100ff01ff, 0x00ff00010000ffff, 0x00ff00010000ff00, 0x00ff00010000ff01, + 0x00ff000100000000, 0x00ff000100000001, 0x00ff000100000100, 0x00ff000100010000, + 0x00ff0001010000ff, 0x00ff000101000000, 0x00ff00010101ff00, 0x00ff01ffff00ff00, + 0x00ff01ff00000000, 0x00ff01ff000001ff, 0x00ff01ff00010100, 0x00ff01ff01ff0000, + 0x00ff0100ffffff00, 0x00ff0100ffff0001, 0x00ff0100ff000000, 0x00ff0100ff0001ff, + 0x00ff0100ff010001, 0x00ff010000ffffff, 0x00ff010000ff0000, 0x00ff01000000ff00, + 0x00ff01000000ff01, 0x00ff0100000000ff, 0x00ff010000000000, 0x00ff010000000001, + 0x00ff010000000100, 0x00ff01000001ffff, 0x00ff010000010000, 0x00ff010000010001, + 0x00ff010001ff0100, 0x00ff010001000000, 0x00ff010001000001, 0x00ff0100010100ff, + 0x00ff010001010100, 0x00ff010100ff00ff, 0x00ff010100ff0001, 0x00ff010100000000, + 0x00ff010100000101, 0x00ff0101000100ff, 0x00ff010100010100, 0x0000ffffffffff00, + 0x0000ffffffff0001, 0x0000ffffffff0100, 0x0000ffffff00ff01, 0x0000ffffff000000, + 0x0000ffffff000101, 0x0000ffffff0100ff, 0x0000ffff00ffffff, 0x0000ffff00ff0000, + 0x0000ffff0000ff00, 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, + 0x0000ffff00000100, 0x0000ffff00010000, 0x0000ffff01ff0001, 0x0000ffff01ff0100, + 0x0000ffff01000000, 0x0000ffff0101ff00, 0x0000ff00ffff0000, 0x0000ff00ffff0100, + 0x0000ff00ff00ff00, 0x0000ff00ff0000ff, 0x0000ff00ff000000, 0x0000ff00ff000001, + 0x0000ff00ff000100, 0x0000ff00ff010000, 0x0000ff0000ffff00, 0x0000ff0000ff00ff, + 0x0000ff0000ff0000, 0x0000ff0000ff0001, 0x0000ff0000ff0100, 0x0000ff000000ffff, + 0x0000ff000000ff00, 0x0000ff000000ff01, 0x0000ff00000000ff, 0x0000ff0000000000, + 0x0000ff0000000001, 0x0000ff00000001ff, 0x0000ff0000000100, 0x0000ff0000000101, + 0x0000ff000001ff00, 0x0000ff00000100ff, 0x0000ff0000010000, 0x0000ff0000010001, + 0x0000ff0000010100, 0x0000ff0001ffff01, 0x0000ff0001ff0000, 0x0000ff000100ff00, + 0x0000ff00010000ff, 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, + 0x0000ff0001010000, 0x0000ff0001010101, 0x0000ff01ff000000, 0x0000ff01ff0100ff, + 0x0000ff0100ff0000, 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, + 0x0000ff0100000001, 0x0000ff0100000100, 0x0000ff010001ff01, 0x0000ff0100010000, + 0x0000ff010100ffff, 0x0000ff0101000000, 0x0000ff0101000100, 0x0000ff0101000101, + 0x0000ff01010100ff, 0x000000ffffff0000, 0x000000ffff00ff00, 0x000000ffff0000ff, + 0x000000ffff000000, 0x000000ffff000001, 0x000000ffff0001ff, 0x000000ffff000100, + 0x000000ffff010000, 0x000000ff00ffff00, 0x000000ff00ff00ff, 0x000000ff00ff0000, + 0x000000ff00ff0001, 0x000000ff00ff0100, 0x000000ff00ff0101, 0x000000ff0000ffff, + 0x000000ff0000ff00, 0x000000ff000000ff, 0x000000ff00000000, 0x000000ff00000001, + 0x000000ff000001ff, 0x000000ff00000100, 0x000000ff00000101, 0x000000ff0001ff00, + 0x000000ff0001ff01, 0x000000ff000100ff, 0x000000ff00010000, 0x000000ff00010001, + 0x000000ff00010100, 0x000000ff01ffffff, 0x000000ff0100ff00, 0x000000ff010000ff, + 0x000000ff01000000, 0x000000ff01000001, 0x000000ff01000100, 0x000000ff010100ff, + 0x000000ff01010000, 0x000000ff01010101, 0x00000000ffffff00, 0x00000000ffffff01, + 0x00000000ffff00ff, 0x00000000ffff0000, 0x00000000ffff0001, 0x00000000ffff0100, + 0x00000000ff00ffff, 0x00000000ff00ff00, 0x00000000ff00ff01, 0x00000000ff0000ff, + 0x00000000ff000000, 0x00000000ff000001, 0x00000000ff000100, 0x00000000ff000101, + 0x00000000ff01ff00, 0x00000000ff0100ff, 0x00000000ff010000, 0x00000000ff010001, + 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, 0x0000000000ffff01, + 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff, + 0x0000000000ff0100, 0x000000000000ffff, 0x000000000000ff00, 0x000000000000ff01, + 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x00000000000001ff, + 0x0000000000000100, 0x0000000000000101, 0x000000000001ffff, 0x000000000001ff00, + 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, 0x00000000000101ff, + 0x0000000000010100, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff00ff, + 0x0000000001ff0000, 0x0000000001ff0100, 0x0000000001ff0101, 0x000000000100ffff, + 0x000000000100ff00, 0x00000000010000ff, 0x0000000001000000, 0x0000000001000001, + 0x00000000010001ff, 0x0000000001000100, 0x000000000101ff00, 0x00000000010100ff, + 0x0000000001010000, 0x0000000001010001, 0x0000000001010100, 0x00000001ffffff00, + 0x00000001ffff00ff, 0x00000001ffff0001, 0x00000001ffff0100, 0x00000001ff00ff00, + 0x00000001ff0000ff, 0x00000001ff000000, 0x00000001ff0001ff, 0x00000001ff000100, + 0x00000001ff01ff00, 0x00000001ff010000, 0x00000001ff010001, 0x00000001ff010100, + 0x0000000100ffff00, 0x0000000100ff0000, 0x0000000100ff0001, 0x0000000100ff0101, + 0x000000010000ffff, 0x000000010000ff00, 0x000000010000ff01, 0x00000001000000ff, + 0x0000000100000000, 0x0000000100000001, 0x00000001000001ff, 0x0000000100000100, + 0x0000000100000101, 0x000000010001ff00, 0x00000001000100ff, 0x0000000100010000, + 0x0000000100010100, 0x0000000101ffff01, 0x0000000101ff0000, 0x0000000101ff01ff, + 0x0000000101ff0100, 0x000000010100ff00, 0x0000000101000000, 0x0000000101010000, + 0x0000000101010001, 0x0000000101010100, 0x000001ffffff00ff, 0x000001ffffff0001, + 0x000001ffffff0100, 0x000001ffff00ffff, 0x000001ffff000000, 0x000001ffff010101, + 0x000001ff00ff0000, 0x000001ff0000ff00, 0x000001ff000000ff, 0x000001ff00000000, + 0x000001ff00000001, 0x000001ff00000100, 0x000001ff00010000, 0x000001ff01ffff01, + 0x000001ff01ff0100, 0x000001ff01000000, 0x000001ff010001ff, 0x000001ff0101ff00, + 0x00000100ffff0000, 0x00000100ff00ff00, 0x00000100ff0000ff, 0x00000100ff000000, + 0x00000100ff000001, 0x00000100ff000100, 0x00000100ff010000, 0x0000010000ffff00, + 0x0000010000ff00ff, 0x0000010000ff0000, 0x0000010000ff0001, 0x0000010000ff0100, + 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01, 0x00000100000000ff, + 0x0000010000000000, 0x0000010000000001, 0x00000100000001ff, 0x0000010000000100, + 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff, 0x0000010000010000, + 0x0000010000010001, 0x0000010000010100, 0x0000010001ff0000, 0x000001000100ff00, + 0x00000100010000ff, 0x0000010001000000, 0x0000010001000001, 0x0000010001000100, + 0x0000010001010000, 0x00000101ffff00ff, 0x00000101ff000000, 0x00000101ff000101, + 0x00000101ff010100, 0x0000010100ff0000, 0x000001010000ff00, 0x0000010100000000, + 0x0000010100000001, 0x0000010100000100, 0x000001010001ff01, 0x0000010100010000, + 0x0000010101ffff00, 0x000001010100ff01, 0x0000010101000000, 0x00000101010001ff, + 0x0000010101000101, 0x000001010101ff00, 0x0001ffffffff0000, 0x0001ffffff0000ff, + 0x0001ffff00000000, 0x0001ffff00000001, 0x0001ffff000001ff, 0x0001ffff000100ff, + 0x0001ffff01ffff00, 0x0001ff00ffffff00, 0x0001ff00ffff00ff, 0x0001ff00ff000000, + 0x0001ff00ff01ff00, 0x0001ff00ff010100, 0x0001ff0000ff0000, 0x0001ff000000ff00, + 0x0001ff0000000000, 0x0001ff0000000001, 0x0001ff0000000100, 0x0001ff0000010000, + 0x0001ff0000010001, 0x0001ff0001ff00ff, 0x0001ff000100ff01, 0x0001ff0001000000, + 0x0001ff0001010100, 0x0001ff01ff000001, 0x0001ff0100ffff00, 0x0001ff0100000000, + 0x0001ff01000001ff, 0x0001ff0101ff0000, 0x0001ff010100ff00, 0x0001ff0101010000, + 0x000100ffff00ff00, 0x000100ffff000000, 0x000100ffff010100, 0x000100ff00ffffff, + 0x000100ff00ffff01, 0x000100ff00ff0000, 0x000100ff00ff01ff, 0x000100ff0000ff00, + 0x000100ff000000ff, 0x000100ff00000000, 0x000100ff00000100, 0x000100ff00000101, + 0x000100ff0001ffff, 0x000100ff00010000, 0x000100ff01ff0000, 0x000100ff010000ff, + 0x000100ff01000000, 0x000100ff01000001, 0x000100ff010001ff, 0x000100ff0101ff00, + 0x000100ff01010100, 0x00010000ffff0000, 0x00010000ffff0101, 0x00010000ff00ff00, + 0x00010000ff000000, 0x00010000ff000001, 0x00010000ff000100, 0x0001000000ff00ff, + 0x0001000000ff0000, 0x0001000000ff0001, 0x0001000000ff0100, 0x000100000000ffff, + 0x000100000000ff00, 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, + 0x0001000000000100, 0x000100000001ff00, 0x00010000000100ff, 0x0001000000010000, + 0x0001000000010001, 0x0001000000010100, 0x0001000001ff0001, 0x000100000100ff00, + 0x0001000001000000, 0x0001000001000001, 0x0001000001000100, 0x0001000001000101, + 0x000100000101ff01, 0x0001000001010000, 0x00010000010101ff, 0x00010001ff000000, + 0x0001000100ffffff, 0x0001000100ff0000, 0x000100010000ff00, 0x00010001000000ff, + 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff, 0x0001000100000101, + 0x000100010001ffff, 0x0001000100010000, 0x0001000101ff0000, 0x00010001010000ff, + 0x0001000101000001, 0x0001000101000100, 0x000101ffff000001, 0x000101ff0000ff01, + 0x000101ff00000000, 0x000101ff01010001, 0x00010100ffffff00, 0x00010100ff000000, + 0x00010100ff01ff00, 0x00010100ff0100ff, 0x00010100ff010001, 0x0001010000ff0000, + 0x0001010000ff01ff, 0x000101000000ff00, 0x00010100000000ff, 0x0001010000000000, + 0x0001010000000001, 0x0001010000000100, 0x0001010000010000, 0x0001010001ffff01, + 0x0001010001ff00ff, 0x0001010001ff0101, 0x0001010001000000, 0x00010100010100ff, + 0x0001010001010100, 0x00010101ff00ff00, 0x00010101ff0001ff, 0x0001010100ffff00, + 0x0001010100ff0100, 0x0001010100000000, 0x0001010100000101, 0x00010101000100ff, + 0x0001010100010100, 0x0001010101010001, 0x01ffffff00ff0000, 0x01ffffff00000001, + 0x01ffffff00010000, 0x01ffff00ff0000ff, 0x01ffff000000ffff, 0x01ffff000000ff01, + 0x01ffff0000000000, 0x01ffff0000000100, 0x01ffff0001010000, 0x01ffff01ff000000, + 0x01ffff01000000ff, 0x01ffff0101000000, 0x01ff00ffff0000ff, 0x01ff00ff0000ff00, + 0x01ff00ff00000000, 0x01ff00ff010000ff, 0x01ff00ff01000100, 0x01ff0000ff00ff01, + 0x01ff0000ff000000, 0x01ff000000ffff00, 0x01ff000000ff0000, 0x01ff000000ff01ff, + 0x01ff00000000ff00, 0x01ff0000000000ff, 0x01ff000000000000, 0x01ff000000000001, + 0x01ff000000000100, 0x01ff000000000101, 0x01ff000000010000, 0x01ff000000010001, + 0x01ff0000000101ff, 0x01ff00000100ffff, 0x01ff000001000000, 0x01ff0001ff000100, + 0x01ff0001ff010000, 0x01ff000100000000, 0x01ff00010001ff00, 0x01ff000100010101, + 0x01ff000101ff0000, 0x01ff000101000101, 0x01ff01ff00ff0000, 0x01ff01ff00010000, + 0x01ff0100ff0000ff, 0x01ff010000ff0001, 0x01ff010000ff0100, 0x01ff010000000000, + 0x01ff010001000100, 0x01ff010001010000, 0x01ff01010000ff00, 0x01ff010100000001, + 0x0100ffffff00ff00, 0x0100ffffff000100, 0x0100ffff00000000, 0x0100ffff00010101, + 0x0100ffff0100ff00, 0x0100ffff01010000, 0x0100ff00ffffff00, 0x0100ff00ffff0001, + 0x0100ff00ff000000, 0x0100ff00ff0100ff, 0x0100ff00ff010001, 0x0100ff0000ffffff, + 0x0100ff0000ff0000, 0x0100ff000000ff00, 0x0100ff00000000ff, 0x0100ff0000000000, + 0x0100ff0000000001, 0x0100ff0000000100, 0x0100ff000001ff01, 0x0100ff0000010000, + 0x0100ff0001000000, 0x0100ff00010001ff, 0x0100ff01ff00ff00, 0x0100ff01ff000100, + 0x0100ff0100ff00ff, 0x0100ff0100ff0001, 0x0100ff0100ff0100, 0x0100ff0100000000, + 0x0100ff0100010001, 0x0100ff0100010100, 0x0100ff0101000001, 0x010000ffffff00ff, + 0x010000ffffff0001, 0x010000ffff000000, 0x010000ffff010001, 0x010000ff0000ff00, + 0x010000ff000000ff, 0x010000ff00000000, 0x010000ff00000001, 0x010000ff000001ff, + 0x010000ff00000100, 0x010000ff0001ffff, 0x010000ff00010000, 0x010000ff01ff00ff, + 0x010000ff01ff0001, 0x010000ff0100ff01, 0x010000ff01000000, 0x01000000ffffffff, + 0x01000000ffff0000, 0x01000000ffff01ff, 0x01000000ffff0101, 0x01000000ff00ff00, + 0x01000000ff0000ff, 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100, + 0x01000000ff01ff00, 0x01000000ff010000, 0x01000000ff010101, 0x0100000000ffff00, + 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001, 0x0100000000ff0100, + 0x010000000000ffff, 0x010000000000ff00, 0x010000000000ff01, 0x01000000000000ff, + 0x0100000000000000, 0x0100000000000001, 0x01000000000001ff, 0x0100000000000100, + 0x0100000000000101, 0x010000000001ff00, 0x01000000000100ff, 0x0100000000010000, + 0x0100000000010001, 0x0100000000010100, 0x0100000001ff0000, 0x010000000100ff00, + 0x010000000100ff01, 0x01000000010000ff, 0x0100000001000000, 0x0100000001000001, + 0x0100000001000100, 0x0100000001000101, 0x010000000101ffff, 0x0100000001010000, + 0x01000001ff00ffff, 0x01000001ff000000, 0x0100000100ff0000, 0x010000010000ff00, + 0x010000010000ff01, 0x0100000100000000, 0x0100000100000001, 0x0100000100000100, + 0x0100000100010000, 0x01000001000101ff, 0x01000001010000ff, 0x0100000101000000, + 0x01000001010100ff, 0x0100000101010001, 0x010001ffff000100, 0x010001ffff010000, + 0x010001ff00ffff00, 0x010001ff0000ffff, 0x010001ff00000000, 0x010001ff00000001, + 0x010001ff0100ff00, 0x010001ff01000100, 0x010001ff01010000, 0x01000100ff00ff01, + 0x01000100ff000000, 0x01000100ff0100ff, 0x0100010000ffff01, 0x0100010000ff0000, + 0x0100010000ff0101, 0x010001000000ff00, 0x01000100000000ff, 0x0100010000000000, + 0x0100010000000001, 0x0100010000000100, 0x010001000001ff01, 0x0100010000010000, + 0x0100010000010101, 0x0100010001ff00ff, 0x0100010001000000, 0x01000101ff000000, + 0x0100010100ff00ff, 0x0100010100ff0100, 0x010001010000ffff, 0x0100010100000000, + 0x01000101000001ff, 0x010001010001ff00, 0x0100010101000000, 0x0101ffffff000000, + 0x0101ffff00ff0000, 0x0101ffff01000000, 0x0101ff0000ff0100, 0x0101ff0000000000, + 0x0101ff0000000101, 0x0101ff00000100ff, 0x0101ff000100ffff, 0x0101ff0001000001, + 0x0101ff0100010000, 0x010100ffff000100, 0x010100ff00ffff00, 0x010100ff000000ff, + 0x010100ff00000000, 0x010100ff0001ff00, 0x010100ff00010001, 0x010100ff000101ff, + 0x01010000ff00ffff, 0x01010000ff00ff01, 0x01010000ff000000, 0x01010000ff0001ff, + 0x0101000000ffff01, 0x0101000000ff0000, 0x010100000000ff00, 0x01010000000000ff, + 0x0101000000000000, 0x0101000000000001, 0x0101000000000100, 0x0101000000010000, + 0x0101000000010101, 0x0101000001ff0000, 0x0101000001000000, 0x01010000010001ff, + 0x01010001ffff0000, 0x01010001ff000001, 0x01010001ff010000, 0x0101000100ff00ff, + 0x0101000100ff0001, 0x0101000100000000, 0x010100010001ff01, 0x0101000101000000, + 0x0101000101000100, 0x010100010101ff00, 0x010101ff00000100, 0x010101ff01000000, + 0x01010100ffff0000, 0x01010100ff000100, 0x0101010000ffff00, 0x0101010000000000, + 0x0101010000000101, 0x010101000001ff00, 0x0101010000010001, 0x0101010000010100, + 0x010101000100ffff, 0x0101010001000001, 0x01010101000000ff, 0x0101010100000001, +GGML_TABLE_END() + + +GGML_TABLE_BEGIN(uint64_t, iq1_xxs_grid, NGRID_IQ1XXS) + 0xffffffffffffffff, 0xffffffffff000000, 0xffffffff01000000, 0xffffff0000ff0100, + 0xffffff0000000000, 0xffffff01ff000000, 0xffff00ff00ffff00, 0xffff00ff00010000, + 0xffff0000ff000000, 0xffff000000ff0000, 0xffff00000000ff00, 0xffff0000000000ff, + 0xffff000000000001, 0xffff000000000100, 0xffff00000001ff01, 0xffff000000010000, + 0xffff000001000000, 0xffff000100000000, 0xffff0100ff000100, 0xffff01000000ff00, + 0xffff0101ff000000, 0xffff010101000000, 0xff00ffff0000ffff, 0xff00ffff00000000, + 0xff00ffff0100ff00, 0xff00ff00ff0001ff, 0xff00ff0000ff0000, 0xff00ff000000ff00, + 0xff00ff00000000ff, 0xff00ff0000000001, 0xff00ff0000000100, 0xff00ff0000010000, + 0xff00ff010100ff00, 0xff00ff01010000ff, 0xff0000ffff0000ff, 0xff0000ffff000100, + 0xff0000ffff01ff00, 0xff0000ff00ff0001, 0xff0000ff00000000, 0xff0000ff000101ff, + 0xff0000ff01ff0100, 0xff0000ff010000ff, 0xff0000ff01000100, 0xff000000ffff0000, + 0xff000000ffff0101, 0xff000000ff00ff00, 0xff000000ff000000, 0xff000000ff000001, + 0xff000000ff010000, 0xff00000000ffff00, 0xff0000000000ff00, 0xff0000000000ff01, + 0xff000000000000ff, 0xff00000000000000, 0xff00000000000100, 0xff00000000000101, + 0xff0000000001ff00, 0xff00000000010001, 0xff00000000010100, 0xff00000001ff00ff, + 0xff0000000100ff00, 0xff00000001000000, 0xff00000001000001, 0xff00000001000101, + 0xff00000001010000, 0xff000001ff000000, 0xff00000100ffffff, 0xff00000100ff0000, + 0xff00000100ff01ff, 0xff00000100000000, 0xff00000100000001, 0xff0000010001ff00, + 0xff000001000100ff, 0xff00000100010100, 0xff00000101ff0001, 0xff0001ff00ffff00, + 0xff0001ff0000ffff, 0xff0001ff00000000, 0xff000100ff000000, 0xff00010000ff0000, + 0xff00010000ff01ff, 0xff000100000000ff, 0xff00010000000000, 0xff00010000000001, + 0xff00010000000100, 0xff00010000010000, 0xff00010001000000, 0xff01ff00ff0000ff, + 0xff01ff0000000000, 0xff01ff0001000001, 0xff01ff01ff000000, 0xff0100ffff000001, + 0xff0100ff00ff0000, 0xff0100ff000001ff, 0xff0100ff00010001, 0xff0100ff0100ff00, + 0xff01000000ff0000, 0xff0100000000ff00, 0xff01000000000000, 0xff01000000000100, + 0xff01000000010000, 0xff010000010000ff, 0xff010001ff000100, 0xff01000100000000, + 0xff010100ff0000ff, 0xff01010001000001, 0xff0101010000ff00, 0xff01010101000000, + 0x00ffff00ff000000, 0x00ffff00ff000001, 0x00ffff0000ff0000, 0x00ffff000000ff00, + 0x00ffff00000000ff, 0x00ffff0000000000, 0x00ffff0000000101, 0x00ffff0000010000, + 0x00ffff0001ff0001, 0x00ffff0001000000, 0x00ffff0100000000, 0x00ffff0100010100, + 0x00ff00ffff000000, 0x00ff00ffff000100, 0x00ff00ff00ff0000, 0x00ff00ff00ff01ff, + 0x00ff00ff000000ff, 0x00ff00ff00000000, 0x00ff00ff00000001, 0x00ff00ff00010101, + 0x00ff00ff0100ffff, 0x00ff00ff0100ff01, 0x00ff0000ffff0000, 0x00ff0000ff00ff00, + 0x00ff0000ff0000ff, 0x00ff0000ff000001, 0x00ff0000ff010000, 0x00ff000000ffff00, + 0x00ff000000ff00ff, 0x00ff000000ff0001, 0x00ff000000ff0100, 0x00ff00000000ffff, + 0x00ff000000000000, 0x00ff000000000001, 0x00ff0000000001ff, 0x00ff00000001ff00, + 0x00ff0000000100ff, 0x00ff000000010100, 0x00ff000001ff0000, 0x00ff00000100ff00, + 0x00ff0000010000ff, 0x00ff000001000100, 0x00ff000001010000, 0x00ff000100ff0000, + 0x00ff00010000ffff, 0x00ff00010000ff01, 0x00ff000100000000, 0x00ff000100000001, + 0x00ff000100000100, 0x00ff000100010000, 0x00ff0001010000ff, 0x00ff01ff00000000, + 0x00ff0100ff000000, 0x00ff010000ffffff, 0x00ff010000ff0000, 0x00ff01000000ff01, + 0x00ff0100000000ff, 0x00ff010000000000, 0x00ff010000000100, 0x00ff010000010001, + 0x00ff010001000000, 0x00ff010001000001, 0x00ff0101000100ff, 0x0000ffffffffff00, + 0x0000ffffff000000, 0x0000ffff00ff0000, 0x0000ffff0000ff00, 0x0000ffff000000ff, + 0x0000ffff00000000, 0x0000ffff00000001, 0x0000ffff00000100, 0x0000ffff00010000, + 0x0000ff00ffff0000, 0x0000ff00ffff0100, 0x0000ff00ff00ff00, 0x0000ff00ff0000ff, + 0x0000ff00ff000000, 0x0000ff00ff000001, 0x0000ff00ff010000, 0x0000ff0000ffff00, + 0x0000ff0000ff00ff, 0x0000ff0000ff0001, 0x0000ff000000ffff, 0x0000ff000000ff00, + 0x0000ff000000ff01, 0x0000ff0000000000, 0x0000ff00000001ff, 0x0000ff0000000100, + 0x0000ff0000000101, 0x0000ff000001ff00, 0x0000ff00000100ff, 0x0000ff0000010001, + 0x0000ff0000010100, 0x0000ff0001ff0000, 0x0000ff000100ff00, 0x0000ff00010000ff, + 0x0000ff0001000000, 0x0000ff0001000001, 0x0000ff0001000100, 0x0000ff0001010000, + 0x0000ff0100ff0000, 0x0000ff010000ff00, 0x0000ff01000000ff, 0x0000ff0100000000, + 0x0000ff0100000001, 0x0000ff0100010000, 0x0000ff0101000100, 0x000000ffffff0000, + 0x000000ffff00ff00, 0x000000ffff000001, 0x000000ffff0001ff, 0x000000ffff010000, + 0x000000ff00ff00ff, 0x000000ff00ff0000, 0x000000ff00ff0101, 0x000000ff0000ff00, + 0x000000ff000000ff, 0x000000ff00000000, 0x000000ff00000001, 0x000000ff00000100, + 0x000000ff0001ff01, 0x000000ff00010000, 0x000000ff00010100, 0x000000ff01000000, + 0x000000ff01000001, 0x000000ff010100ff, 0x000000ff01010000, 0x000000ff01010101, + 0x00000000ffffff01, 0x00000000ffff00ff, 0x00000000ff00ffff, 0x00000000ff00ff01, + 0x00000000ff000000, 0x00000000ff000100, 0x00000000ff000101, 0x00000000ff0100ff, + 0x00000000ff010001, 0x00000000ff010100, 0x0000000000ffffff, 0x0000000000ffff00, + 0x0000000000ff00ff, 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff01ff, + 0x0000000000ff0100, 0x000000000000ff00, 0x000000000000ff01, 0x00000000000000ff, + 0x0000000000000000, 0x0000000000000001, 0x0000000000000100, 0x0000000000000101, + 0x000000000001ffff, 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, + 0x00000000000101ff, 0x0000000000010101, 0x0000000001ffff00, 0x0000000001ff0000, + 0x0000000001ff0100, 0x000000000100ffff, 0x0000000001000000, 0x0000000001000001, + 0x00000000010001ff, 0x000000000101ff00, 0x00000000010100ff, 0x0000000001010100, + 0x00000001ffffff00, 0x00000001ffff0001, 0x00000001ffff0100, 0x00000001ff0000ff, + 0x00000001ff000000, 0x00000001ff01ff00, 0x00000001ff010001, 0x00000001ff010100, + 0x0000000100ff0101, 0x000000010000ffff, 0x000000010000ff00, 0x00000001000000ff, + 0x0000000100000000, 0x00000001000001ff, 0x0000000100000100, 0x0000000100000101, + 0x0000000100010000, 0x0000000101ffff01, 0x0000000101ff0000, 0x0000000101ff0100, + 0x000000010100ff00, 0x0000000101000000, 0x0000000101010001, 0x0000000101010100, + 0x000001ffff000000, 0x000001ff00ff0000, 0x000001ff000000ff, 0x000001ff00000001, + 0x000001ff00000100, 0x000001ff00010000, 0x000001ff01000000, 0x00000100ffff0000, + 0x00000100ff00ff00, 0x00000100ff0000ff, 0x00000100ff000001, 0x00000100ff000100, + 0x00000100ff010000, 0x0000010000ffff00, 0x0000010000ff00ff, 0x0000010000ff0001, + 0x0000010000ff0100, 0x000001000000ffff, 0x000001000000ff00, 0x000001000000ff01, + 0x00000100000000ff, 0x0000010000000000, 0x00000100000001ff, 0x0000010000000100, + 0x0000010000000101, 0x000001000001ff00, 0x00000100000100ff, 0x0000010000010100, + 0x0000010001ff0000, 0x000001000100ff00, 0x00000100010000ff, 0x0000010001000001, + 0x0000010001000100, 0x0000010001010000, 0x00000101ff000000, 0x0000010100ff0000, + 0x000001010000ff00, 0x0000010100000001, 0x0000010100000100, 0x0000010100010000, + 0x0000010101000000, 0x0001ffff00000000, 0x0001ffff000100ff, 0x0001ff00ff000000, + 0x0001ff0000ff0000, 0x0001ff000000ff00, 0x0001ff0000000100, 0x0001ff0000010001, + 0x0001ff0001ff00ff, 0x0001ff0001000000, 0x0001ff0100000000, 0x000100ffff00ff00, + 0x000100ffff000000, 0x000100ff00ffffff, 0x000100ff00ffff01, 0x000100ff00000000, + 0x000100ff00000101, 0x000100ff00010000, 0x000100ff01ff0000, 0x000100ff010001ff, + 0x00010000ffff0000, 0x00010000ff00ff00, 0x00010000ff000000, 0x00010000ff000001, + 0x00010000ff000100, 0x0001000000ff00ff, 0x0001000000ff0001, 0x0001000000ff0100, + 0x000100000000ffff, 0x00010000000000ff, 0x0001000000000000, 0x0001000000000001, + 0x000100000001ff00, 0x00010000000100ff, 0x0001000000010100, 0x0001000001ff0001, + 0x000100000100ff00, 0x0001000001000000, 0x0001000001000100, 0x0001000001000101, + 0x000100000101ff01, 0x0001000001010000, 0x0001000100ff0000, 0x000100010000ff00, + 0x0001000100000000, 0x0001000100000001, 0x00010001000001ff, 0x0001000100010000, + 0x00010001010000ff, 0x000101ff00000000, 0x00010100ff000000, 0x00010100ff010001, + 0x0001010000ff0000, 0x000101000000ff00, 0x00010100000000ff, 0x0001010000000001, + 0x0001010000000100, 0x0001010000010000, 0x0001010001000000, 0x0001010100000000, + 0x0001010100000101, 0x00010101000100ff, 0x01ffffff00ff0000, 0x01ffffff00010000, + 0x01ffff000000ff01, 0x01ffff0000000100, 0x01ffff01ff000000, 0x01ffff0101000000, + 0x01ff00ffff0000ff, 0x01ff00ff0000ff00, 0x01ff00ff01000100, 0x01ff0000ff000000, + 0x01ff000000ffff00, 0x01ff0000000000ff, 0x01ff000000000000, 0x01ff000000000101, + 0x01ff000000010001, 0x01ff000001000000, 0x01ff0001ff000100, 0x01ff01ff00010000, + 0x01ff010000ff0100, 0x01ff01010000ff00, 0x01ff010100000001, 0x0100ffffff000100, + 0x0100ffff0100ff00, 0x0100ff00ffffff00, 0x0100ff0000ff0000, 0x0100ff00000000ff, + 0x0100ff0000000000, 0x0100ff0000000001, 0x0100ff0000010000, 0x0100ff0001000000, + 0x0100ff01ff00ff00, 0x0100ff0100ff0100, 0x010000ffffff0001, 0x010000ffff000000, + 0x010000ff00000000, 0x010000ff00000001, 0x010000ff00000100, 0x010000ff0001ffff, + 0x010000ff01ff00ff, 0x010000ff01ff0001, 0x01000000ffff0000, 0x01000000ff0000ff, + 0x01000000ff000000, 0x01000000ff000001, 0x01000000ff000100, 0x01000000ff01ff00, + 0x0100000000ff00ff, 0x0100000000ff0000, 0x0100000000ff0001, 0x0100000000ff0100, + 0x010000000000ffff, 0x010000000000ff00, 0x0100000000000000, 0x01000000000001ff, + 0x0100000000000100, 0x010000000001ff00, 0x01000000000100ff, 0x0100000000010000, + 0x0100000000010001, 0x0100000000010100, 0x010000000100ff01, 0x01000000010000ff, + 0x0100000001000000, 0x0100000001000101, 0x0100000001010000, 0x0100000100ff0000, + 0x010000010000ff01, 0x0100000100000000, 0x0100000100000001, 0x0100000100000100, + 0x01000001010100ff, 0x010001ff00ffff00, 0x010001ff00000000, 0x010001ff0100ff00, + 0x01000100ff000000, 0x0100010000ff0000, 0x01000100000000ff, 0x0100010000000000, + 0x0100010000000001, 0x0100010000000100, 0x0100010000010000, 0x01000101ff000000, + 0x0100010100ff00ff, 0x010001010001ff00, 0x0100010101000000, 0x0101ffffff000000, + 0x0101ffff01000000, 0x0101ff0000000000, 0x0101ff0000000101, 0x0101ff0100010000, + 0x010100ff000000ff, 0x010100ff00010001, 0x01010000ff00ffff, 0x01010000ff0001ff, + 0x010100000000ff00, 0x0101000000000000, 0x0101000000000001, 0x0101000000000100, + 0x0101000000010000, 0x0101000001ff0000, 0x01010001ff000001, 0x01010001ff010000, + 0x0101000100ff00ff, 0x0101000101000000, 0x0101000101000100, 0x010101ff00000100, + 0x0101010000ffff00, 0x0101010000000000, 0x0101010000010001, 0x0101010001000001, +GGML_TABLE_END() + + +GGML_TABLE_BEGIN(uint64_t, iq1_xxxs_grid, NGRID_IQ1XXXS) + 0xffffffffffffffff, 0xffffffffff000000, 0xffffffff01000000, 0xffffff01ff000000, + 0xffff00ff00010000, 0xffff000000ff0000, 0xffff0000000000ff, 0xffff000000000001, + 0xffff000000000100, 0xffff0100ff000100, 0xffff01000000ff00, 0xffff010101000000, + 0xff00ff00ff0001ff, 0xff00ff0000ff0000, 0xff00ff00000000ff, 0xff00ff0000000001, + 0xff00ff0000010000, 0xff00ff010100ff00, 0xff0000ff00000000, 0xff0000ff01ff0100, + 0xff000000ffff0000, 0xff000000ff00ff00, 0xff000000ff000000, 0xff0000000000ff00, + 0xff0000000000ff01, 0xff000000000000ff, 0xff00000000000000, 0xff00000000000100, + 0xff00000000000101, 0xff0000000001ff00, 0xff00000000010001, 0xff00000001ff00ff, + 0xff00000001000000, 0xff00000100ffffff, 0xff00000100000000, 0xff000001000100ff, + 0xff00000100010100, 0xff00000101ff0001, 0xff0001ff00ffff00, 0xff00010000ff01ff, + 0xff00010000000000, 0xff00010000010000, 0xff01ff0000000000, 0xff0100ffff000001, + 0xff0100ff000001ff, 0xff0100ff0100ff00, 0xff01000000ff0000, 0xff01000000010000, + 0xff010001ff000100, 0xff010100ff0000ff, 0xff01010001000001, 0x00ffff00ff000001, + 0x00ffff000000ff00, 0x00ffff0000000000, 0x00ffff0000000101, 0x00ff00ff00ff01ff, + 0x00ff00ff00000000, 0x00ff00ff00000001, 0x00ff0000ffff0000, 0x00ff0000ff00ff00, + 0x00ff0000ff0000ff, 0x00ff0000ff010000, 0x00ff00000000ffff, 0x00ff000000000000, + 0x00ff000000000001, 0x00ff0000000001ff, 0x00ff00000001ff00, 0x00ff000000010100, + 0x00ff000001ff0000, 0x00ff00000100ff00, 0x00ff0000010000ff, 0x00ff000001000100, + 0x00ff000001010000, 0x00ff000100ff0000, 0x00ff00010000ff01, 0x00ff000100000000, + 0x00ff010000ffffff, 0x00ff010000000000, 0x00ff010000010001, 0x00ff010001000001, + 0x00ff0101000100ff, 0x0000ffff000000ff, 0x0000ffff00000000, 0x0000ffff00000001, + 0x0000ff00ffff0100, 0x0000ff00ff000000, 0x0000ff00ff010000, 0x0000ff0000ff00ff, + 0x0000ff0000ff0001, 0x0000ff000000ff00, 0x0000ff000000ff01, 0x0000ff0000000000, + 0x0000ff0000000100, 0x0000ff00000100ff, 0x0000ff0001ff0000, 0x0000ff0001000000, + 0x0000ff0001010000, 0x0000ff01000000ff, 0x0000ff0100000000, 0x0000ff0100000001, + 0x0000ff0101000100, 0x000000ffffff0000, 0x000000ffff000001, 0x000000ffff0001ff, + 0x000000ffff010000, 0x000000ff00ff0000, 0x000000ff00ff0101, 0x000000ff0000ff00, + 0x000000ff000000ff, 0x000000ff00000100, 0x000000ff0001ff01, 0x000000ff00010000, + 0x000000ff00010100, 0x000000ff01000000, 0x000000ff01000001, 0x000000ff010100ff, + 0x00000000ff00ffff, 0x00000000ff00ff01, 0x00000000ff000000, 0x00000000ff000100, + 0x00000000ff000101, 0x00000000ff0100ff, 0x0000000000ffff00, 0x0000000000ff00ff, + 0x0000000000ff0000, 0x0000000000ff0001, 0x0000000000ff0100, 0x000000000000ff00, + 0x00000000000000ff, 0x0000000000000000, 0x0000000000000001, 0x0000000000000100, + 0x000000000001ffff, 0x00000000000100ff, 0x0000000000010000, 0x0000000000010001, + 0x00000000000101ff, 0x0000000001ffff00, 0x000000000100ffff, 0x0000000001000000, + 0x0000000001000001, 0x00000000010001ff, 0x000000000101ff00, 0x0000000001010100, + 0x00000001ffffff00, 0x00000001ffff0001, 0x00000001ff0000ff, 0x00000001ff000000, + 0x00000001ff01ff00, 0x00000001ff010001, 0x000000010000ff00, 0x0000000100000000, + 0x00000001000001ff, 0x0000000100000101, 0x0000000100010000, 0x0000000101ff0100, + 0x0000000101000000, 0x0000000101010001, 0x000001ffff000000, 0x000001ff000000ff, + 0x000001ff00000001, 0x000001ff01000000, 0x00000100ffff0000, 0x00000100ff000001, + 0x00000100ff010000, 0x0000010000ff0001, 0x000001000000ff00, 0x000001000000ff01, + 0x00000100000000ff, 0x0000010000000000, 0x0000010000000100, 0x0000010001ff0000, + 0x00000100010000ff, 0x0000010001000100, 0x0000010001010000, 0x00000101ff000000, + 0x0000010100ff0000, 0x0000010100000001, 0x0000010100000100, 0x0001ff0000ff0000, + 0x0001ff000000ff00, 0x0001ff0000000100, 0x0001ff0000010001, 0x000100ffff00ff00, + 0x000100ff00ffffff, 0x000100ff00ffff01, 0x000100ff00000000, 0x00010000ffff0000, + 0x00010000ff000000, 0x0001000000ff0100, 0x00010000000000ff, 0x0001000000000000, + 0x0001000000000001, 0x000100000001ff00, 0x0001000000010100, 0x0001000001000000, + 0x0001000001000101, 0x0001000001010000, 0x0001000100ff0000, 0x000100010000ff00, + 0x0001000100000000, 0x00010001010000ff, 0x0001010000ff0000, 0x000101000000ff00, + 0x0001010000000100, 0x0001010000010000, 0x0001010100000000, 0x01ffffff00ff0000, + 0x01ffffff00010000, 0x01ffff0101000000, 0x01ff00ffff0000ff, 0x01ff000000ffff00, + 0x01ff000000000000, 0x01ff000000000101, 0x01ff0001ff000100, 0x01ff01ff00010000, + 0x01ff010000ff0100, 0x0100ff00ffffff00, 0x0100ff00000000ff, 0x0100ff0000000000, + 0x0100ff0000000001, 0x0100ff0100ff0100, 0x010000ff00000000, 0x01000000ff000000, + 0x01000000ff01ff00, 0x0100000000ff0000, 0x0100000000ff0001, 0x010000000000ffff, + 0x010000000000ff00, 0x0100000000000100, 0x0100000000010000, 0x0100000000010100, + 0x010000000100ff01, 0x01000000010000ff, 0x0100000001000000, 0x0100000001000101, + 0x0100000100000000, 0x0100000100000001, 0x01000001010100ff, 0x010001ff00ffff00, + 0x01000100ff000000, 0x01000100000000ff, 0x0100010000000001, 0x0100010100ff00ff, + 0x010001010001ff00, 0x0100010101000000, 0x0101ffffff000000, 0x0101ffff01000000, + 0x0101ff0100010000, 0x010100ff000000ff, 0x010100ff00010001, 0x01010000ff0001ff, + 0x0101000000000000, 0x0101000001ff0000, 0x01010001ff000001, 0x010101ff00000100, +GGML_TABLE_END() + #else GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S) 0x00000000, 0x00000002, 0x00000101, 0x00000200, 0x00000202, 0x00010001, 0x00010101, 0x00020000, @@ -1905,6 +2426,239 @@ GGML_TABLE_BEGIN(uint32_t, iq1s_grid_gpu, NGRID_IQ1S) 0x20222020, 0x20222022, 0x20222220, 0x20222222, 0x21212021, 0x21212120, 0x21212122, 0x22202020, 0x22202022, 0x22202220, 0x22202222, 0x22212121, 0x22222020, 0x22222022, 0x22222220, 0x22222222, GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq1_xs_grid_gpu, NGRID_IQ1XS) + 0x00000000, 0x00010101, 0x01010100, 0x01010102, 0x02010101, 0x00010011, 0x01000011, 0x01000211, + 0x01010012, 0x01010111, 0x01010212, 0x01020110, 0x02020111, 0x00010121, 0x00011100, 0x00011102, + 0x01001001, 0x01001201, 0x01011101, 0x01021101, 0x00001110, 0x00001112, 0x00011111, 0x01001111, + 0x01001212, 0x01011010, 0x01011011, 0x01011110, 0x01011111, 0x01011112, 0x01011211, 0x01021010, + 0x01021012, 0x01021111, 0x01021210, 0x02011011, 0x02011111, 0x02011210, 0x00011120, 0x00011221, + 0x01001021, 0x01011121, 0x01011220, 0x01021122, 0x02001121, 0x02011221, 0x00012101, 0x01012102, + 0x02012101, 0x00002111, 0x00012211, 0x00022110, 0x01012011, 0x01012111, 0x02012110, 0x00012121, + 0x02012121, 0x00120101, 0x01100100, 0x01110000, 0x01110101, 0x01120201, 0x02110001, 0x02110102, + 0x00100110, 0x00110010, 0x00110012, 0x00110111, 0x00110210, 0x01100111, 0x01110011, 0x01110110, + 0x01110111, 0x01110112, 0x01110211, 0x01120010, 0x01120111, 0x02110111, 0x01100020, 0x01100122, + 0x01100221, 0x01110121, 0x01120122, 0x02110021, 0x02110120, 0x02110122, 0x00101001, 0x00111100, + 0x00111101, 0x00111201, 0x00121001, 0x01101101, 0x01101102, 0x01101200, 0x01111001, 0x01111100, + 0x01111101, 0x01111102, 0x01111201, 0x01121101, 0x01121200, 0x02101201, 0x02111100, 0x02111101, + 0x02111201, 0x00101111, 0x00101212, 0x00111011, 0x00111110, 0x00111111, 0x00111112, 0x00111211, + 0x00121012, 0x00121111, 0x00121210, 0x00121212, 0x01101011, 0x01101110, 0x01101111, 0x01101112, + 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111112, 0x01111211, 0x01111212, 0x01121011, + 0x01121110, 0x01121111, 0x01121112, 0x01121211, 0x02101012, 0x02101110, 0x02101111, 0x02111011, + 0x02111110, 0x02111111, 0x02111112, 0x02111211, 0x02111212, 0x02121010, 0x02121012, 0x02121111, + 0x00111121, 0x01101020, 0x01101121, 0x01101220, 0x01111021, 0x01111121, 0x01111122, 0x01111221, + 0x01121021, 0x01121120, 0x01121121, 0x01121221, 0x01121222, 0x02101122, 0x02111121, 0x02121120, + 0x01102001, 0x01102100, 0x01112000, 0x01112101, 0x01112202, 0x01122100, 0x02102101, 0x00112010, + 0x00112012, 0x00112111, 0x00122011, 0x01102111, 0x01102210, 0x01112011, 0x01112110, 0x01112111, + 0x01112112, 0x01112211, 0x01122111, 0x02112012, 0x02112111, 0x02112210, 0x02122112, 0x02122211, + 0x00112122, 0x00122120, 0x01102122, 0x01102221, 0x01112020, 0x01112121, 0x01112220, 0x01122122, + 0x02102121, 0x01210001, 0x01210201, 0x00210110, 0x01200012, 0x01210111, 0x01210210, 0x01210212, + 0x01220110, 0x02210010, 0x02210112, 0x02220111, 0x00210121, 0x01200121, 0x00211102, 0x01201101, + 0x01211002, 0x01211101, 0x01211200, 0x01221102, 0x02211001, 0x00201211, 0x00211111, 0x01201010, + 0x01201111, 0x01211011, 0x01211110, 0x01211111, 0x01211211, 0x01221012, 0x01221111, 0x02201211, + 0x02211110, 0x02211111, 0x02221211, 0x00201121, 0x00211020, 0x00211022, 0x00211221, 0x00221121, + 0x01211121, 0x02201120, 0x00212101, 0x01222101, 0x00212110, 0x00212211, 0x01202112, 0x01212111, + 0x01222211, 0x02212010, 0x02212112, 0x01212021, 0x01212122, 0x02212121, 0x10000101, 0x10010001, + 0x11010002, 0x11010101, 0x11010200, 0x11020100, 0x11020102, 0x12010201, 0x10000110, 0x10010111, + 0x10010112, 0x10010210, 0x10020011, 0x10020211, 0x11000111, 0x11010011, 0x11010110, 0x11010111, + 0x11010112, 0x11010211, 0x11010212, 0x11020111, 0x12000110, 0x12000112, 0x12010010, 0x12010111, + 0x12020012, 0x10010021, 0x10010122, 0x11000021, 0x11010121, 0x11020221, 0x12000221, 0x10001001, + 0x10011101, 0x10011201, 0x11001101, 0x11001200, 0x11001202, 0x11011001, 0x11011100, 0x11011101, + 0x11011102, 0x11021001, 0x11021101, 0x11021202, 0x12001001, 0x12011000, 0x12011002, 0x12011101, + 0x10001111, 0x10011011, 0x10011110, 0x10011111, 0x10011112, 0x10011211, 0x10021010, 0x10021111, + 0x10021212, 0x11001011, 0x11001110, 0x11001111, 0x11001112, 0x11001211, 0x11011010, 0x11011011, + 0x11011110, 0x11011111, 0x11011112, 0x11011210, 0x11011211, 0x11021011, 0x11021110, 0x11021111, + 0x11021112, 0x11021211, 0x12001012, 0x12001111, 0x12011011, 0x12011110, 0x12011111, 0x12011112, + 0x12011211, 0x12021111, 0x12021210, 0x12021212, 0x10001121, 0x10011121, 0x10011222, 0x11001121, + 0x11001220, 0x11011020, 0x11011021, 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021121, + 0x12011120, 0x12011121, 0x12021021, 0x10012001, 0x11012101, 0x11012200, 0x11022201, 0x12002101, + 0x10002011, 0x10002112, 0x10012111, 0x10012210, 0x10022112, 0x11002010, 0x11002111, 0x11012011, + 0x11012012, 0x11012110, 0x11012111, 0x11012112, 0x11012211, 0x11022010, 0x11022111, 0x11022112, + 0x12002211, 0x12012111, 0x12012112, 0x12022110, 0x12022211, 0x11002120, 0x11002122, 0x11012121, + 0x11012222, 0x11022120, 0x11022221, 0x10100001, 0x10100102, 0x10100201, 0x10110002, 0x10110101, + 0x10110202, 0x10120100, 0x11100000, 0x11100101, 0x11110001, 0x11110100, 0x11110101, 0x11110102, + 0x11110201, 0x11120101, 0x12100102, 0x12100201, 0x12110101, 0x12120001, 0x10100111, 0x10100211, + 0x10110011, 0x10110110, 0x10110111, 0x10110112, 0x10110211, 0x10120111, 0x11100011, 0x11100110, + 0x11100111, 0x11100112, 0x11100211, 0x11110010, 0x11110011, 0x11110012, 0x11110110, 0x11110111, + 0x11110112, 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120111, 0x11120112, + 0x11120211, 0x12100012, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, + 0x12120111, 0x12120212, 0x10110121, 0x10120120, 0x11100121, 0x11110021, 0x11110120, 0x11110121, + 0x11110122, 0x11110221, 0x11120022, 0x11120121, 0x12110020, 0x12110121, 0x12110221, 0x12110222, + 0x12120120, 0x10101101, 0x10111001, 0x10111100, 0x10111101, 0x10111102, 0x10111200, 0x10111201, + 0x10121101, 0x11101001, 0x11101100, 0x11101101, 0x11101102, 0x11101201, 0x11101202, 0x11111000, + 0x11111001, 0x11111100, 0x11111101, 0x11111102, 0x11111200, 0x11111201, 0x11111202, 0x11121001, + 0x11121002, 0x11121100, 0x11121101, 0x11121102, 0x11121201, 0x12101000, 0x12111001, 0x12111100, + 0x12111101, 0x12111102, 0x12111201, 0x12121100, 0x12121101, 0x12121202, 0x10101011, 0x10101012, + 0x10101110, 0x10101111, 0x10101112, 0x10101211, 0x10111010, 0x10111011, 0x10111012, 0x10111110, + 0x10111111, 0x10111112, 0x10111211, 0x10111212, 0x10121011, 0x10121110, 0x10121111, 0x10121112, + 0x10121211, 0x11101010, 0x11101011, 0x11101012, 0x11101110, 0x11101111, 0x11101112, 0x11101210, + 0x11101211, 0x11111010, 0x11111011, 0x11111012, 0x11111110, 0x11111111, 0x11111112, 0x11111210, + 0x11111211, 0x11111212, 0x11121010, 0x11121011, 0x11121110, 0x11121111, 0x11121112, 0x11121210, + 0x11121211, 0x11121212, 0x12101011, 0x12101110, 0x12101111, 0x12101211, 0x12101212, 0x12111010, + 0x12111011, 0x12111110, 0x12111111, 0x12111112, 0x12111210, 0x12111211, 0x12121011, 0x12121110, + 0x12121111, 0x12121112, 0x12121211, 0x10101021, 0x10101120, 0x10101122, 0x10101221, 0x10111021, + 0x10111120, 0x10111121, 0x10111220, 0x10111221, 0x10121021, 0x10121121, 0x10121122, 0x10121221, + 0x11101021, 0x11101121, 0x11101122, 0x11101222, 0x11111020, 0x11111021, 0x11111022, 0x11111120, + 0x11111121, 0x11111122, 0x11111220, 0x11111221, 0x11111222, 0x11121021, 0x11121120, 0x11121121, + 0x11121221, 0x12101022, 0x12101121, 0x12101220, 0x12101221, 0x12111021, 0x12111121, 0x12121121, + 0x12121122, 0x12121221, 0x10102100, 0x10102102, 0x10102201, 0x10112000, 0x10112101, 0x10122202, + 0x11102101, 0x11112001, 0x11112100, 0x11112101, 0x11112102, 0x11112201, 0x11122101, 0x12102002, + 0x12102201, 0x12112101, 0x12112200, 0x12122001, 0x10102111, 0x10112011, 0x10112110, 0x10112111, + 0x10112112, 0x10112211, 0x10122111, 0x11102011, 0x11102110, 0x11102111, 0x11102112, 0x11102211, + 0x11112010, 0x11112011, 0x11112012, 0x11112110, 0x11112111, 0x11112112, 0x11112210, 0x11112211, + 0x11112212, 0x11122011, 0x11122110, 0x11122111, 0x11122112, 0x11122211, 0x12102111, 0x12112011, + 0x12112110, 0x12112111, 0x12112112, 0x12112211, 0x12122111, 0x10102120, 0x10112121, 0x10112222, + 0x10122221, 0x11102121, 0x11112021, 0x11112121, 0x11112122, 0x11112221, 0x11122022, 0x11122121, + 0x12102021, 0x12112022, 0x12112121, 0x12112220, 0x12112222, 0x12122021, 0x10200101, 0x10210100, + 0x11210101, 0x11210102, 0x11210200, 0x11220100, 0x12200001, 0x10200011, 0x10200110, 0x10210111, + 0x10220011, 0x10220211, 0x11200111, 0x11210011, 0x11210111, 0x11210112, 0x11210211, 0x11220111, + 0x11220112, 0x12200110, 0x12210012, 0x12210111, 0x12220211, 0x10210122, 0x11200021, 0x11210121, + 0x11210220, 0x12200121, 0x12210021, 0x12220121, 0x10211001, 0x10211101, 0x10221201, 0x11201000, + 0x11201002, 0x11201101, 0x11201200, 0x11211001, 0x11211100, 0x11211101, 0x11211201, 0x11211202, + 0x11221000, 0x11221101, 0x12201101, 0x12211100, 0x12211101, 0x12211102, 0x12211200, 0x12221001, + 0x12221201, 0x10201111, 0x10201212, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, + 0x11201111, 0x11201112, 0x11201211, 0x11211010, 0x11211011, 0x11211110, 0x11211111, 0x11211112, + 0x11211211, 0x11221011, 0x11221110, 0x11221111, 0x11221112, 0x11221211, 0x12201112, 0x12211011, + 0x12211111, 0x12211112, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x12221210, 0x10211121, + 0x11201020, 0x11201121, 0x11211021, 0x11211120, 0x11211121, 0x11211122, 0x11211220, 0x11211222, + 0x11221020, 0x11221121, 0x12201121, 0x12211120, 0x12211122, 0x12211221, 0x10212102, 0x11212002, + 0x11212101, 0x12222102, 0x10202011, 0x10212111, 0x10222011, 0x10222110, 0x10222112, 0x11202111, + 0x11202210, 0x11212011, 0x11212110, 0x11212111, 0x11212112, 0x11212211, 0x11222111, 0x12202012, + 0x12202110, 0x12202212, 0x12212111, 0x12222110, 0x12222211, 0x10212021, 0x10212220, 0x11202021, + 0x11202221, 0x11212121, 0x11212222, 0x11222120, 0x11222221, 0x12222122, 0x21000101, 0x21010102, + 0x21020101, 0x20010110, 0x21010010, 0x21010012, 0x21010111, 0x21010211, 0x22020111, 0x20010121, + 0x21010120, 0x22010121, 0x20011100, 0x21011001, 0x21011101, 0x22011100, 0x22011201, 0x20011012, + 0x20011111, 0x21001011, 0x21001111, 0x21001210, 0x21011011, 0x21011110, 0x21011111, 0x21011112, + 0x21011211, 0x21011212, 0x21021111, 0x21021112, 0x21021210, 0x22011010, 0x22011111, 0x20011221, + 0x20021121, 0x21011121, 0x21021021, 0x21021222, 0x22001121, 0x22011222, 0x21002101, 0x21022101, + 0x20012110, 0x21002112, 0x21002211, 0x21012111, 0x22012211, 0x22022111, 0x21012021, 0x21012122, + 0x20110001, 0x20110201, 0x21110101, 0x21120202, 0x22110001, 0x22120101, 0x20100011, 0x20100112, + 0x20110111, 0x20120110, 0x20120112, 0x21100010, 0x21100111, 0x21110011, 0x21110110, 0x21110111, + 0x21110112, 0x21110211, 0x21120012, 0x21120111, 0x22110111, 0x22110210, 0x20110021, 0x20110221, + 0x21100120, 0x21100122, 0x21100221, 0x21110121, 0x21120122, 0x21120221, 0x22110122, 0x20101100, + 0x20101102, 0x20111101, 0x20121102, 0x21111001, 0x21111100, 0x21111101, 0x21111102, 0x21111200, + 0x21111201, 0x21121000, 0x21121101, 0x22101100, 0x22101102, 0x22111002, 0x22111101, 0x20101010, + 0x20101111, 0x20101210, 0x20101212, 0x20111011, 0x20111110, 0x20111111, 0x20111112, 0x20111211, + 0x20121011, 0x20121111, 0x20121212, 0x21101011, 0x21101110, 0x21101111, 0x21101112, 0x21101211, + 0x21111010, 0x21111011, 0x21111012, 0x21111110, 0x21111111, 0x21111112, 0x21111210, 0x21111211, + 0x21111212, 0x21121011, 0x21121110, 0x21121111, 0x21121112, 0x21121211, 0x22101111, 0x22111011, + 0x22111012, 0x22111110, 0x22111111, 0x22111112, 0x22111211, 0x22111212, 0x22121010, 0x22121111, + 0x20111020, 0x20111121, 0x21101121, 0x21111021, 0x21111022, 0x21111121, 0x21111122, 0x21111221, + 0x21121121, 0x21121220, 0x22111120, 0x22111121, 0x22121120, 0x22121122, 0x20112201, 0x20122101, + 0x21102001, 0x21112000, 0x21112101, 0x21112102, 0x22112001, 0x22112201, 0x22122101, 0x20112012, + 0x20112111, 0x20122110, 0x21102012, 0x21102111, 0x21102212, 0x21112011, 0x21112110, 0x21112111, + 0x21112112, 0x21112211, 0x21122012, 0x21122111, 0x21122212, 0x22102110, 0x22112111, 0x20112121, + 0x21102120, 0x21102221, 0x21112020, 0x21112121, 0x21112220, 0x21122021, 0x22112121, 0x20210101, + 0x21200101, 0x22210101, 0x21200211, 0x21210111, 0x21210212, 0x21220110, 0x22210010, 0x22210112, + 0x21220121, 0x20211201, 0x21201001, 0x21211100, 0x21211101, 0x21221001, 0x21221102, 0x21221200, + 0x20211010, 0x20211012, 0x20211111, 0x20211210, 0x21201012, 0x21201111, 0x21211011, 0x21211110, + 0x21211111, 0x21211112, 0x21211211, 0x21221111, 0x21221212, 0x22201111, 0x22211111, 0x22211210, + 0x20201121, 0x20211122, 0x20221121, 0x21201120, 0x21201122, 0x21211121, 0x21221022, 0x22211121, + 0x22211221, 0x22221021, 0x21212201, 0x22212101, 0x20202111, 0x20212211, 0x21202011, 0x21212111, + 0x21212212, 0x21222011, 0x21222112, 0x21222211, 0x22212010, 0x22212112, 0x21212120, 0x21212122, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq1_xxs_grid_gpu, NGRID_IQ1XXS) + 0x00000000, 0x00010101, 0x02010101, 0x01000211, 0x01010111, 0x00010121, 0x01001001, 0x01021101, + 0x00011111, 0x01001111, 0x01011011, 0x01011110, 0x01011112, 0x01011211, 0x01021012, 0x01021111, + 0x02011111, 0x01011121, 0x00012211, 0x01012011, 0x00012121, 0x02012121, 0x01110000, 0x01110101, + 0x02110001, 0x00110210, 0x01100111, 0x01110011, 0x01110110, 0x01110112, 0x01110211, 0x01120111, + 0x02110021, 0x02110120, 0x00111100, 0x00111201, 0x00121001, 0x01101102, 0x01111101, 0x01121200, + 0x02101201, 0x02111100, 0x02111201, 0x00101111, 0x00101212, 0x00111011, 0x00111111, 0x00111112, + 0x00121111, 0x01101011, 0x01111011, 0x01111012, 0x01111110, 0x01111111, 0x01111211, 0x01111212, + 0x01121011, 0x01121112, 0x01121211, 0x02101110, 0x02111011, 0x02111111, 0x02111112, 0x02111212, + 0x02121111, 0x00111121, 0x01101020, 0x01101121, 0x01101220, 0x01111121, 0x01111122, 0x01121021, + 0x01121120, 0x01121221, 0x02101122, 0x01102001, 0x01112000, 0x01112101, 0x00112111, 0x01102111, + 0x01102210, 0x01112110, 0x01112111, 0x01112112, 0x01112211, 0x01122111, 0x02112111, 0x00210110, + 0x01210111, 0x02210112, 0x00210121, 0x00211102, 0x01201101, 0x01211200, 0x01221102, 0x02211001, + 0x01201111, 0x01211011, 0x01211111, 0x01211211, 0x01221111, 0x02211110, 0x00211221, 0x01211121, + 0x00212110, 0x02212112, 0x01212021, 0x02212121, 0x10010111, 0x10010112, 0x11000111, 0x11010011, + 0x11010110, 0x11010111, 0x11010212, 0x11020111, 0x12000112, 0x12010111, 0x11010121, 0x11020221, + 0x10011101, 0x10011201, 0x11001101, 0x11001200, 0x11011100, 0x11011101, 0x11011102, 0x11021202, + 0x12011000, 0x12011002, 0x10001111, 0x10011011, 0x10011110, 0x10011112, 0x10021111, 0x11001011, + 0x11001110, 0x11001112, 0x11001211, 0x11011010, 0x11011111, 0x11011112, 0x11011210, 0x11021011, + 0x11021110, 0x11021211, 0x12001111, 0x12011011, 0x12011110, 0x12011211, 0x12021111, 0x11001121, + 0x11011020, 0x11011022, 0x11011121, 0x11011122, 0x11011221, 0x11021121, 0x12011120, 0x11012101, + 0x10012111, 0x11002010, 0x11002111, 0x11012012, 0x11012110, 0x11012111, 0x11012211, 0x11022112, + 0x12012111, 0x12012112, 0x11022120, 0x10100001, 0x10110101, 0x11100101, 0x11110001, 0x11110100, + 0x11110101, 0x11110102, 0x11110201, 0x11120101, 0x10100111, 0x10100211, 0x10110011, 0x10110110, + 0x10110111, 0x10110112, 0x10120111, 0x11100011, 0x11100110, 0x11100112, 0x11110010, 0x11110011, + 0x11110012, 0x11110111, 0x11110210, 0x11110211, 0x11110212, 0x11120011, 0x11120110, 0x11120112, + 0x11120211, 0x12100111, 0x12110011, 0x12110110, 0x12110111, 0x12110112, 0x12110211, 0x12120111, + 0x11100121, 0x11110021, 0x11110120, 0x11110121, 0x11110122, 0x11120121, 0x12110221, 0x10101101, + 0x10111001, 0x10111102, 0x10111200, 0x10121101, 0x11101100, 0x11101101, 0x11101202, 0x11111001, + 0x11111100, 0x11111101, 0x11111102, 0x11111201, 0x11121002, 0x11121101, 0x11121201, 0x12111101, + 0x12111102, 0x12121100, 0x12121101, 0x12121202, 0x10101012, 0x10101110, 0x10111010, 0x10111012, + 0x10111111, 0x10111211, 0x10111212, 0x10121110, 0x10121112, 0x10121211, 0x11101010, 0x11101011, + 0x11101110, 0x11101111, 0x11101112, 0x11101210, 0x11101211, 0x11111011, 0x11111012, 0x11111110, + 0x11111111, 0x11111112, 0x11111211, 0x11111212, 0x11121010, 0x11121110, 0x11121111, 0x11121112, + 0x11121210, 0x11121212, 0x12101011, 0x12101111, 0x12101211, 0x12111010, 0x12111111, 0x12111112, + 0x12111210, 0x12121011, 0x12121110, 0x12121211, 0x10101021, 0x10101122, 0x10101221, 0x10111120, + 0x10111121, 0x10121021, 0x10121122, 0x10121221, 0x11101222, 0x11111020, 0x11111021, 0x11111120, + 0x11111121, 0x11111220, 0x11111221, 0x11111222, 0x11121121, 0x12101022, 0x12101121, 0x12101221, + 0x12111021, 0x12111121, 0x12121122, 0x12121221, 0x10112101, 0x11102101, 0x11112100, 0x11112102, + 0x11112201, 0x11122101, 0x12112101, 0x10102111, 0x10112011, 0x10112110, 0x10112112, 0x10112211, + 0x10122111, 0x11102011, 0x11102110, 0x11102112, 0x11102211, 0x11112010, 0x11112011, 0x11112012, + 0x11112110, 0x11112111, 0x11112210, 0x11112211, 0x11112212, 0x11122011, 0x11122110, 0x11122211, + 0x12102111, 0x12112011, 0x12112110, 0x12112112, 0x12112211, 0x12122111, 0x10112121, 0x11102121, + 0x11112021, 0x11112122, 0x11112221, 0x11122121, 0x12112121, 0x11210101, 0x11220100, 0x10210111, + 0x11200111, 0x11210011, 0x11210211, 0x11220112, 0x12200110, 0x12210111, 0x11210121, 0x10211001, + 0x10211101, 0x11201000, 0x11201002, 0x11211101, 0x11211202, 0x11221101, 0x12201101, 0x12211200, + 0x10201111, 0x10211011, 0x10211111, 0x10211112, 0x10211211, 0x11201110, 0x11201112, 0x11201211, + 0x11211010, 0x11211110, 0x11211111, 0x11211112, 0x11221011, 0x11221110, 0x11221211, 0x12201112, + 0x12211011, 0x12211111, 0x12211211, 0x12211212, 0x12221012, 0x12221111, 0x11201121, 0x11211021, + 0x11211121, 0x11211122, 0x11211220, 0x11221121, 0x12211120, 0x11212101, 0x10212111, 0x10222112, + 0x11202111, 0x11212011, 0x11212110, 0x11212112, 0x11212211, 0x11222111, 0x12212111, 0x11212121, + 0x11212222, 0x11222120, 0x21000101, 0x21020101, 0x21010012, 0x21010211, 0x20010121, 0x22010121, + 0x20011100, 0x21011001, 0x22011201, 0x20011111, 0x21001011, 0x21011110, 0x21011111, 0x21011212, + 0x21021112, 0x22011111, 0x20011221, 0x21022101, 0x21002211, 0x21012021, 0x21012122, 0x20110201, + 0x22110001, 0x20100011, 0x21100111, 0x21110110, 0x21110111, 0x21110112, 0x21120111, 0x22110111, + 0x20110021, 0x21100221, 0x20101102, 0x20111101, 0x21111101, 0x21111102, 0x21111201, 0x21121000, + 0x22101100, 0x22101102, 0x20101111, 0x20111110, 0x20111111, 0x20111112, 0x20111211, 0x20121011, + 0x21101110, 0x21101111, 0x21101112, 0x21101211, 0x21111010, 0x21111011, 0x21111111, 0x21111210, + 0x21111211, 0x21121011, 0x21121110, 0x21121111, 0x21121112, 0x21121211, 0x22111012, 0x22111110, + 0x22111111, 0x22111212, 0x22121111, 0x21101121, 0x21111022, 0x21111121, 0x21111122, 0x21111221, + 0x22121120, 0x21102001, 0x21112101, 0x22112001, 0x20112111, 0x21102111, 0x21112110, 0x21112111, + 0x21112112, 0x21112211, 0x21122111, 0x20112121, 0x21102120, 0x21122021, 0x22112121, 0x20210101, + 0x22210101, 0x21210111, 0x21210212, 0x21220121, 0x21211100, 0x21221102, 0x20211010, 0x20211210, + 0x21211011, 0x21211111, 0x21211112, 0x21211211, 0x21221111, 0x22201111, 0x20211122, 0x20221121, + 0x21201120, 0x22211121, 0x22211221, 0x21212201, 0x21202011, 0x21212111, 0x21222112, 0x22212112, +GGML_TABLE_END() + +GGML_TABLE_BEGIN(uint32_t, iq1_xxxs_grid_gpu, NGRID_IQ1XXXS) + 0x00000000, 0x00010101, 0x02010101, 0x00010121, 0x01021101, 0x01001111, 0x01011110, 0x01011112, + 0x01011211, 0x00012211, 0x01012011, 0x02012121, 0x00110210, 0x01100111, 0x01110110, 0x01110112, + 0x01120111, 0x02110021, 0x01111101, 0x02101201, 0x00101111, 0x00111011, 0x00111111, 0x01111011, + 0x01111012, 0x01111110, 0x01111111, 0x01111211, 0x01111212, 0x01121011, 0x01121112, 0x02101110, + 0x02111111, 0x01101020, 0x01111121, 0x01121120, 0x01121221, 0x02101122, 0x01102001, 0x01102210, + 0x01112111, 0x01122111, 0x01210111, 0x00211102, 0x01211200, 0x02211001, 0x01201111, 0x01221111, + 0x00211221, 0x00212110, 0x02212112, 0x10010112, 0x11010011, 0x11010111, 0x11010212, 0x11001200, + 0x11011101, 0x11011102, 0x10001111, 0x10011011, 0x10011110, 0x10021111, 0x11011010, 0x11011111, + 0x11011112, 0x11011210, 0x11021011, 0x11021211, 0x12001111, 0x12011011, 0x12011110, 0x12011211, + 0x12021111, 0x11001121, 0x11011022, 0x11011121, 0x11002010, 0x11012111, 0x11022112, 0x12012112, + 0x11022120, 0x11110100, 0x11110101, 0x11110102, 0x10100211, 0x10110111, 0x10120111, 0x11100110, + 0x11100112, 0x11110011, 0x11110012, 0x11110111, 0x11110211, 0x11120110, 0x12100111, 0x12110111, + 0x12120111, 0x11110120, 0x11110121, 0x11110122, 0x12110221, 0x10101101, 0x10111102, 0x10111200, + 0x10121101, 0x11101101, 0x11101202, 0x11111001, 0x11111100, 0x11111201, 0x11121002, 0x11121101, + 0x11121201, 0x12111101, 0x12111102, 0x12121100, 0x10111010, 0x10111012, 0x10111111, 0x10111211, + 0x10111212, 0x10121110, 0x11101011, 0x11101110, 0x11101111, 0x11101112, 0x11101211, 0x11111011, + 0x11111110, 0x11111111, 0x11111112, 0x11111211, 0x11121010, 0x11121110, 0x11121111, 0x11121112, + 0x11121210, 0x12101011, 0x12111010, 0x12111111, 0x12111112, 0x12111210, 0x12121011, 0x12121211, + 0x10101021, 0x10101122, 0x10111120, 0x10111121, 0x10121021, 0x10121122, 0x11111021, 0x11111121, + 0x11111220, 0x11111222, 0x11121121, 0x12101221, 0x12111121, 0x12121122, 0x10112101, 0x11112100, + 0x11112102, 0x12112101, 0x10102111, 0x10112112, 0x10122111, 0x11102112, 0x11112011, 0x11112012, + 0x11112110, 0x11112111, 0x11112211, 0x12102111, 0x12112110, 0x12112211, 0x12122111, 0x10112121, + 0x11102121, 0x11112122, 0x11112221, 0x11200111, 0x11210011, 0x11210211, 0x11220112, 0x10211001, + 0x11201000, 0x11201002, 0x11211101, 0x10201111, 0x10211111, 0x11201211, 0x11211110, 0x11211111, + 0x11211112, 0x11221011, 0x11221211, 0x12211111, 0x12211212, 0x12221111, 0x11201121, 0x11211021, + 0x11211121, 0x12211120, 0x11202111, 0x11212011, 0x11212211, 0x11222111, 0x11212121, 0x21000101, + 0x21020101, 0x22010121, 0x20011100, 0x21001011, 0x21011111, 0x21011212, 0x20011221, 0x21022101, + 0x21002211, 0x20100011, 0x21110110, 0x21110111, 0x21110112, 0x21100221, 0x21111101, 0x20111111, + 0x20121011, 0x21101111, 0x21101112, 0x21111010, 0x21111011, 0x21111211, 0x21121111, 0x21121211, + 0x22111012, 0x22111110, 0x22111111, 0x22111212, 0x21111121, 0x21111122, 0x22121120, 0x21102001, + 0x20112111, 0x21112110, 0x21112112, 0x21102120, 0x21122021, 0x22112121, 0x20210101, 0x22210101, + 0x21220121, 0x21211100, 0x21221102, 0x20211210, 0x21211111, 0x22201111, 0x20211122, 0x21212201, +GGML_TABLE_END() #endif #endif // GGML_COMMON_IMPL diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp index 554a2066a75c..f7cb0e40e84f 100644 --- a/ggml/src/ggml-cpu/amx/amx.cpp +++ b/ggml/src/ggml-cpu/amx/amx.cpp @@ -155,6 +155,10 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type { if (!ggml_is_contiguous(src0) || !ggml_is_contiguous(src1)) { return false; } + // 2D weights only: the packed buffer holds a single slice; a batched src1 is folded into M + if (ggml_nrows(src0) != src0->ne[1]) { + return false; + } if (!src0->buffer || src0->buffer->buft != ggml_backend_amx_buffer_type()) { return false; } diff --git a/ggml/src/ggml-cpu/amx/mmq.cpp b/ggml/src/ggml-cpu/amx/mmq.cpp index 9f3a744b5de3..af247a1d4fcc 100644 --- a/ggml/src/ggml-cpu/amx/mmq.cpp +++ b/ggml/src/ggml-cpu/amx/mmq.cpp @@ -2349,13 +2349,15 @@ void ggml_backend_amx_mul_mat(const ggml_compute_params * params, struct ggml_te // amx kernels will be added once 6th gen xeon is released. const bool is_floating_type = TYPE == GGML_TYPE_F16; - const int M = dst->ne[1]; + // single 2D src0 slice (supports_op) and contiguous src1/dst: all src1 batches are one gemm + GGML_ASSERT(ggml_nrows(src0) == src0->ne[1]); + const int M = ggml_nrows(dst); const int N = dst->ne[0]; const int K = src0->ne[0]; const int ldc = dst->nb[1] / dst->nb[0]; - const int64_t ne2 = dst->ne[2]; - const int64_t n_batch = ne2 * dst->ne[3]; + const int64_t ne2 = 1; + const int64_t n_batch = 1; if (is_floating_type) { constexpr int BLOCK_M = 4; diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 4e59d0ae0a1c..57b29e395a18 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -75,6 +75,10 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 @@ -85,6 +89,10 @@ #define ggml_gemm_iq4_nl_8x8_q8_0_generic ggml_gemm_iq4_nl_8x8_q8_0 #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q2_K_8x8_q8_K_generic ggml_gemm_q2_K_8x8_q8_K +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__x86_64__) || defined(__i386__) || defined(_M_IX86) || defined(_M_X64) // repack.cpp #define ggml_quantize_mat_q8_0_4x4_generic ggml_quantize_mat_q8_0_4x4 @@ -115,6 +123,10 @@ #define ggml_gemm_mxfp4_4x4_q8_0_generic ggml_gemm_mxfp4_4x4_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__POWERPC__) || defined(__powerpc__) // ref: https://github.com/ggml-org/llama.cpp/pull/14146#issuecomment-2972561679 // quants.c @@ -166,6 +178,10 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__loongarch64) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -217,6 +233,10 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__riscv) // quants.c #define ggml_vec_dot_nvfp4_q8_0_generic ggml_vec_dot_nvfp4_q8_0 @@ -261,6 +281,10 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__s390x__) // quants.c #define quantize_row_q8_K_generic quantize_row_q8_K @@ -314,6 +338,10 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #elif defined(__wasm__) // quants.c #define ggml_vec_dot_tq1_0_q8_K_generic ggml_vec_dot_tq1_0_q8_K @@ -372,4 +400,8 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 +#define ggml_vec_dot_iq1_xs_q8_K_generic ggml_vec_dot_iq1_xs_q8_K +#define ggml_vec_dot_iq1_xxs_q8_K_generic ggml_vec_dot_iq1_xxs_q8_K +#define ggml_vec_dot_iq1_xxxs_q8_K_generic ggml_vec_dot_iq1_xxxs_q8_K + #endif diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index f4eaea5dfe95..f5ae8de5fdfb 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -386,6 +386,24 @@ static const struct ggml_type_traits_cpu type_traits_cpu[GGML_TYPE_COUNT] = { .vec_dot_type = GGML_TYPE_Q8_K, .nrows = 1, }, + [GGML_TYPE_IQ1_XS] = { + .from_float = NULL, + .vec_dot = ggml_vec_dot_iq1_xs_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, + [GGML_TYPE_IQ1_XXS] = { + .from_float = NULL, + .vec_dot = ggml_vec_dot_iq1_xxs_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, + [GGML_TYPE_IQ1_XXXS] = { + .from_float = NULL, + .vec_dot = ggml_vec_dot_iq1_xxxs_q8_K, + .vec_dot_type = GGML_TYPE_Q8_K, + .nrows = 1, + }, [GGML_TYPE_IQ4_NL] = { .from_float = quantize_row_iq4_nl, .vec_dot = ggml_vec_dot_iq4_nl_q8_0, diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 81ff5d79fa98..a571fc2aec03 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -429,6 +429,16 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st return true; } + // MXFP8 is a gather-only type: GET_ROWS reads it, nothing else does (and nothing writes it) + if (op->type == GGML_TYPE_MXFP8) { + return false; + } + for (int i = 0; i < GGML_MAX_SRC; i++) { + if (op->src[i] && op->src[i]->type == GGML_TYPE_MXFP8 && !(op->op == GGML_OP_GET_ROWS && i == 0)) { + return false; + } + } + // check extra buffer types // note: only the first sources are checked for extra buffer types to reduce overhead, increase if necessary for (int i = 0; i < 4; i++) { @@ -446,9 +456,12 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st op->type != GGML_TYPE_IQ3_XXS && op->type != GGML_TYPE_IQ3_S && op->type != GGML_TYPE_IQ2_XXS && - op->type != GGML_TYPE_IQ2_XS && - op->type != GGML_TYPE_IQ2_S && - op->type != GGML_TYPE_IQ1_S && + op->type != GGML_TYPE_IQ2_XS && + op->type != GGML_TYPE_IQ2_S && + op->type != GGML_TYPE_IQ1_S && + op->type != GGML_TYPE_IQ1_XS && + op->type != GGML_TYPE_IQ1_XXS && + op->type != GGML_TYPE_IQ1_XXXS && op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float case GGML_OP_MUL_MAT: if (ggml_get_op_params_i32(op, 1) == GGML_HINT_SRC0_IS_HADAMARD && diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp index dbdde6997f48..c1f5c1170311 100644 --- a/ggml/src/ggml-cpu/ops.cpp +++ b/ggml/src/ggml-cpu/ops.cpp @@ -695,6 +695,9 @@ void ggml_compute_forward_add( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: @@ -1153,6 +1156,9 @@ void ggml_compute_forward_add1( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: @@ -1285,6 +1291,9 @@ void ggml_compute_forward_acc( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: @@ -4829,6 +4838,9 @@ void ggml_compute_forward_out_prod( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: @@ -5155,6 +5167,9 @@ void ggml_compute_forward_set( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: @@ -5368,6 +5383,7 @@ void ggml_compute_forward_get_rows( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: @@ -5381,6 +5397,9 @@ void ggml_compute_forward_get_rows( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: @@ -6129,6 +6148,7 @@ void ggml_compute_forward_clamp( case GGML_TYPE_Q8_1: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_MXFP8: case GGML_TYPE_F8_E4M3: case GGML_TYPE_Q2_K: case GGML_TYPE_Q3_K: @@ -6142,6 +6162,9 @@ void ggml_compute_forward_clamp( case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_NL: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ3_S: diff --git a/ggml/src/ggml-cpu/quants.c b/ggml/src/ggml-cpu/quants.c index 44ea9fae55f1..a45a803ec36c 100644 --- a/ggml/src/ggml-cpu/quants.c +++ b/ggml/src/ggml-cpu/quants.c @@ -1238,6 +1238,138 @@ void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, *s = sumf; } +void ggml_vec_dot_iq1_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_xs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0; + for (int i = 0; i < nb; i++) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint8_t * sc = x[i].sc; + + int sumi = 0, sumi1 = 0; + for (int ib = 0; ib < QK_K/32; ++ib) { + const int nib = (sc[ib/2] >> (4*(ib&1))) & 0xf; + const int ls = 2*(nib & 7) + 1; + const int delta = nib & 8 ? -1 : 1; + int lsum = 0; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1_xs_grid + (qs[l] | (((qh[ib] >> 2*l) & 3) << 8))); + for (int j = 0; j < 8; ++j) { + lsum += q8[j] * grid[j]; + } + q8 += 8; + } + sumi += ls * lsum; + sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]); + qs += 4; + } + + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1); + } + + *s = sumf; +} + +void ggml_vec_dot_iq1_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_xxs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0; + for (int i = 0; i < nb; i++) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + + int sumi = 0, sumi1 = 0; + for (int ib = 0; ib < QK_K/32; ++ib) { + const int ls = 2*((qh[ib] >> 4) & 7) + 1; + const int delta = qh[ib] & 0x80 ? -1 : 1; + int lsum = 0; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1_xxs_grid + (qs[l] | (((qh[ib] >> l) & 1) << 8))); + for (int j = 0; j < 8; ++j) { + lsum += q8[j] * grid[j]; + } + q8 += 8; + } + sumi += ls * lsum; + sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]); + qs += 4; + } + + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1); + } + + *s = sumf; +} + +void ggml_vec_dot_iq1_xxxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { + assert(n % QK_K == 0); + assert(nrc == 1); + UNUSED(nrc); + UNUSED(bx); + UNUSED(by); + UNUSED(bs); + + const block_iq1_xxxs * GGML_RESTRICT x = vx; + const block_q8_K * GGML_RESTRICT y = vy; + + const int nb = n / QK_K; + + float sumf = 0; + for (int i = 0; i < nb; i++) { + + const int8_t * q8 = y[i].qs; + const uint8_t * qs = x[i].qs; + const uint8_t * sc = x[i].sc; + + int sumi = 0, sumi1 = 0; + for (int ib = 0; ib < QK_K/32; ++ib) { + const int nib = (sc[ib/2] >> (4*(ib&1))) & 0xf; + const int ls = 2*(nib & 7) + 1; + const int delta = nib & 8 ? -1 : 1; + int lsum = 0; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1_xxxs_grid + qs[l]); + for (int j = 0; j < 8; ++j) { + lsum += q8[j] * grid[j]; + } + q8 += 8; + } + sumi += ls * lsum; + sumi1 += ls * delta * (y[i].bsums[2*ib+0] + y[i].bsums[2*ib+1]); + qs += 4; + } + + sumf += GGML_CPU_FP16_TO_FP32(x[i].d) * y[i].d * (sumi + IQ1S_DELTA * sumi1); + } + + *s = sumf; +} + void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) { assert(n % QK_K == 0); assert(nrc == 1); diff --git a/ggml/src/ggml-cpu/quants.h b/ggml/src/ggml-cpu/quants.h index f04be52f54af..5771a041fdd8 100644 --- a/ggml/src/ggml-cpu/quants.h +++ b/ggml/src/ggml-cpu/quants.h @@ -66,6 +66,9 @@ void ggml_vec_dot_iq2_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void ggml_vec_dot_iq3_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_m_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_xxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_xxxs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_nl_q8_0 (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_xs_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq3_s_q8_K (int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); @@ -100,6 +103,9 @@ void ggml_vec_dot_iq3_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs void ggml_vec_dot_iq3_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_s_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq1_m_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_xxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); +void ggml_vec_dot_iq1_xxxs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_nl_q8_0_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); void ggml_vec_dot_iq4_xs_q8_K_generic(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc); diff --git a/ggml/src/ggml-cuda/common.cuh b/ggml/src/ggml-cuda/common.cuh index a46c4d3f096f..68d72487a9fd 100644 --- a/ggml/src/ggml-cuda/common.cuh +++ b/ggml/src/ggml-cuda/common.cuh @@ -1151,6 +1151,27 @@ struct ggml_cuda_type_traits { static constexpr int bs = sizeof(block_iq1_m); }; +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR1_XS; + static constexpr int qi = QI1_XS; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR1_XXS; + static constexpr int qi = QI1_XXS; +}; + +template<> +struct ggml_cuda_type_traits { + static constexpr int qk = QK_K; + static constexpr int qr = QR1_XXXS; + static constexpr int qi = QI1_XXXS; +}; + template<> struct ggml_cuda_type_traits { static constexpr int qk = QK4_NL; diff --git a/ggml/src/ggml-cuda/convert.cu b/ggml/src/ggml-cuda/convert.cu index 77fd28894516..4636aa148d87 100644 --- a/ggml/src/ggml-cuda/convert.cu +++ b/ggml/src/ggml-cuda/convert.cu @@ -215,6 +215,69 @@ static __global__ void dequantize_block_iq1_s(const void * __restrict__ vx, dst_ dequantize_iq1_s(vx, i, yy + i*QK_K, threadIdx.x); } +template +static __global__ void dequantize_block_iq1_xs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int64_t i = blockIdx.x; + const block_iq1_xs * x = (const block_iq1_xs *) vx; + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const int nibble = (x[i].sc[ib/2] >> (4*(ib%2))) & 0xf; + const float delta = nibble & 0x8 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float)x[i].d * (2*(nibble & 0x7) + 1); + uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = iq1_xs_grid_gpu[x[i].qs[4*ib+il] | (((x[i].qh[ib] >> 2*il) & 3) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = d * (q[j] + delta); + } +} + +template +static __global__ void dequantize_block_iq1_xxs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int64_t i = blockIdx.x; + const block_iq1_xxs * x = (const block_iq1_xxs *) vx; + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const int qh = x[i].qh[ib]; + const float delta = qh & 0x80 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float)x[i].d * (2*((qh >> 4) & 7) + 1); + uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = iq1_xxs_grid_gpu[x[i].qs[4*ib+il] | (((qh >> il) & 1) << 8)]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = d * (q[j] + delta); + } +} + +template +static __global__ void dequantize_block_iq1_xxxs(const void * __restrict__ vx, dst_t * __restrict__ yy) { + + const int64_t i = blockIdx.x; + const block_iq1_xxxs * x = (const block_iq1_xxxs *) vx; + const int64_t tid = threadIdx.x; + const int64_t il = tid/8; // 0...3 + const int64_t ib = tid%8; // 0...7 + dst_t * y = yy + i*QK_K + 32*ib + 8*il; + const int nibble = (x[i].sc[ib/2] >> (4*(ib%2))) & 0xf; + const float delta = nibble & 0x8 ? -1 - IQ1S_DELTA : -1 + IQ1S_DELTA; + const float d = (float)x[i].d * (2*(nibble & 0x7) + 1); + uint32_t grid32[2]; const int8_t * q = (const int8_t *)grid32; + grid32[0] = iq1_xxxs_grid_gpu[x[i].qs[4*ib+il]]; + grid32[1] = (grid32[0] >> 4) & 0x0f0f0f0f; + grid32[0] &= 0x0f0f0f0f; + for (int j = 0; j < 8; ++j) { + y[j] = d * (q[j] + delta); + } +} + template static __global__ void dequantize_block_iq1_m(const void * __restrict__ vx, dst_t * __restrict__ yy) { const int64_t i = blockIdx.x; @@ -355,6 +418,24 @@ static void dequantize_row_iq1_s_cuda(const void * vx, dst_t * y, const int64_t dequantize_block_iq1_s<<>>(vx, y); } +template +static void dequantize_row_iq1_xs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_xs<<>>(vx, y); +} + +template +static void dequantize_row_iq1_xxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_xxs<<>>(vx, y); +} + +template +static void dequantize_row_iq1_xxxs_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { + const int nb = k / QK_K; + dequantize_block_iq1_xxxs<<>>(vx, y); +} + template static void dequantize_row_iq4_nl_cuda(const void * vx, dst_t * y, const int64_t k, cudaStream_t stream) { const int nb32 = k / QK4_NL; @@ -529,6 +610,12 @@ to_bf16_cuda_t ggml_get_to_bf16_cuda(ggml_type type) { return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ1_XS: + return dequantize_row_iq1_xs_cuda; + case GGML_TYPE_IQ1_XXS: + return dequantize_row_iq1_xxs_cuda; + case GGML_TYPE_IQ1_XXXS: + return dequantize_row_iq1_xxxs_cuda; case GGML_TYPE_IQ1_M: return dequantize_row_iq1_m_cuda; case GGML_TYPE_IQ4_NL: @@ -589,6 +676,12 @@ to_fp16_cuda_t ggml_get_to_fp16_cuda(ggml_type type) { return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ1_XS: + return dequantize_row_iq1_xs_cuda; + case GGML_TYPE_IQ1_XXS: + return dequantize_row_iq1_xxs_cuda; + case GGML_TYPE_IQ1_XXXS: + return dequantize_row_iq1_xxxs_cuda; case GGML_TYPE_IQ1_M: return dequantize_row_iq1_m_cuda; case GGML_TYPE_IQ4_NL: @@ -646,6 +739,12 @@ to_fp32_cuda_t ggml_get_to_fp32_cuda(ggml_type type) { return dequantize_row_iq3_xxs_cuda; case GGML_TYPE_IQ1_S: return dequantize_row_iq1_s_cuda; + case GGML_TYPE_IQ1_XS: + return dequantize_row_iq1_xs_cuda; + case GGML_TYPE_IQ1_XXS: + return dequantize_row_iq1_xxs_cuda; + case GGML_TYPE_IQ1_XXXS: + return dequantize_row_iq1_xxxs_cuda; case GGML_TYPE_IQ1_M: return dequantize_row_iq1_m_cuda; case GGML_TYPE_IQ4_NL: diff --git a/ggml/src/ggml-cuda/dequantize.cuh b/ggml/src/ggml-cuda/dequantize.cuh index 651524be7b3e..dc2cbe3d75ec 100644 --- a/ggml/src/ggml-cuda/dequantize.cuh +++ b/ggml/src/ggml-cuda/dequantize.cuh @@ -119,6 +119,24 @@ static __device__ __forceinline__ void dequantize_q8_0(const void * vx, const in v.y *= d; } +// E4M3 (fn) decode, exact in fp32; 0x7F/0xFF are NaN +static __device__ __forceinline__ float ggml_cuda_e4m3_to_fp32(const uint8_t x) { + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float v = exp == 0 ? ldexpf((float) man, -9) : ldexpf((float) (8 + man), exp - 10); + v = (exp == 0xF && man == 0x7) ? __int_as_float(0x7FC00000) : v; + return (x & 0x80) ? -v : v; +} + +static __device__ __forceinline__ void dequantize_mxfp8(const void * vx, const int64_t ib, const int iqs, float2 & v){ + const block_mxfp8 * x = (const block_mxfp8 *) vx; + + const float d = x[ib].e == 0xFF ? __int_as_float(0x7FC00000) : ggml_cuda_e8m0_to_fp32(x[ib].e); + + v.x = ggml_cuda_e4m3_to_fp32(x[ib].qs[iqs + 0]) * d; + v.y = ggml_cuda_e4m3_to_fp32(x[ib].qs[iqs + 1]) * d; +} + //================================== k-quants // Each call dequantizes one super-block of QK_K values into y using the diff --git a/ggml/src/ggml-cuda/getrows.cu b/ggml/src/ggml-cuda/getrows.cu index d97e8fdb0f40..95f6d2383a78 100644 --- a/ggml/src/ggml-cuda/getrows.cu +++ b/ggml/src/ggml-cuda/getrows.cu @@ -344,6 +344,10 @@ static void ggml_cuda_get_rows_switch_src0_type( get_rows_cuda_q(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); break; + case GGML_TYPE_MXFP8: + get_rows_cuda_q(src0_d, src1_d, dst_d, + ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); + break; case GGML_TYPE_Q2_K: get_rows_cuda_kq<64, dst_t, dequantize_q2_K>(src0_d, src1_d, dst_d, ne00, nb01, nb02, nb03, ne10, ne11, ne12, nb10, nb11, nb12, nb1, nb2, nb3, stream); diff --git a/ggml/src/ggml-cuda/ggml-cuda.cu b/ggml/src/ggml-cuda/ggml-cuda.cu index 06956ce7a3d9..66f53aa97285 100644 --- a/ggml/src/ggml-cuda/ggml-cuda.cu +++ b/ggml/src/ggml-cuda/ggml-cuda.cu @@ -5856,6 +5856,9 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_Q8_K: case GGML_TYPE_IQ1_M: case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_XXS: @@ -5898,6 +5901,7 @@ static bool ggml_backend_cuda_device_supports_op(ggml_backend_dev_t dev, const g case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: case GGML_TYPE_IQ4_XS: + case GGML_TYPE_MXFP8: return true; case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: diff --git a/ggml/src/ggml-cuda/mmq-config-ampere.cuh b/ggml/src/ggml-cuda/mmq-config-ampere.cuh index 9f9fd197382f..76dc37438930 100644 --- a/ggml/src/ggml-cuda/mmq-config-ampere.cuh +++ b/ggml/src/ggml-cuda/mmq-config-ampere.cuh @@ -224,6 +224,57 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); CASE(GGML_TYPE_IQ1_S, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 1, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_IQ2_XXS, 256, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); diff --git a/ggml/src/ggml-cuda/mmq-config-cdna.cuh b/ggml/src/ggml-cuda/mmq-config-cdna.cuh index 4a8d89f72019..b46080558061 100644 --- a/ggml/src/ggml-cuda/mmq-config-cdna.cuh +++ b/ggml/src/ggml-cuda/mmq-config-cdna.cuh @@ -107,6 +107,30 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); CASE(GGML_TYPE_IQ1_S, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ1_XXXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, false); + CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); CASE(GGML_TYPE_IQ2_XXS, 512, 1, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, true, true); diff --git a/ggml/src/ggml-cuda/mmq-config-gcn.cuh b/ggml/src/ggml-cuda/mmq-config-gcn.cuh index 24af2ef2b05b..b096c6bbdf16 100644 --- a/ggml/src/ggml-cuda/mmq-config-gcn.cuh +++ b/ggml/src/ggml-cuda/mmq-config-gcn.cuh @@ -167,6 +167,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 512, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 512, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh b/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh index 83eb7c146e11..e80a69243e90 100644 --- a/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh +++ b/ggml/src/ggml-cuda/mmq-config-pascal-dp4a.cuh @@ -159,6 +159,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh b/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh index 2a8dc9e1a93e..5873edb3e011 100644 --- a/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh +++ b/ggml/src/ggml-cuda/mmq-config-pascal-older.cuh @@ -159,6 +159,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh index 8324d9e1a830..6662dff951df 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna2.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna2.cuh @@ -159,6 +159,42 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 24, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 40, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 8, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh index ead3825d213c..400f6d64859a 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna3-5.cuh @@ -167,6 +167,45 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna3.cuh b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh index 3a3ef7bd9c09..d151b08afe30 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna3.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna3.cuh @@ -160,6 +160,45 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 128, 1, 64, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh index 9293d9d55885..c89b0bfc5d1b 100644 --- a/ggml/src/ggml-cuda/mmq-config-rdna4.cuh +++ b/ggml/src/ggml-cuda/mmq-config-rdna4.cuh @@ -167,6 +167,45 @@ static constexpr __host__ __device__ ggml_cuda_mmq_config ggml_cuda_mmq_get_conf CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); CASE(GGML_TYPE_IQ1_S, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 48, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 80, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 96, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 112, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ1_XXXS, 256, 2, 128, 128, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, false); + CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 16, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 32, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); CASE(GGML_TYPE_IQ2_XXS, 128, 2, 64, 64, GGML_CUDA_MMQ_SRAM_LAYOUT_Q8_0, MMQ_ITER_K, false, true); diff --git a/ggml/src/ggml-cuda/mmq-load-tiles.cuh b/ggml/src/ggml-cuda/mmq-load-tiles.cuh index 4951db72ebfc..90e8f1821b08 100644 --- a/ggml/src/ggml-cuda/mmq-load-tiles.cuh +++ b/ggml/src/ggml-cuda/mmq-load-tiles.cuh @@ -1133,6 +1133,199 @@ template static __device__ __forceinline_ } } +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_xs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_XS); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq1_xs * bxi = (const block_iq1_xs *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + const int nib = (bxi->sc[kqsx/2] >> (4*(kqsx%2))) & 0x0F; + + #pragma unroll + for (int l = 0; l < QR1_XS/2; ++l) { + const int grid = iq1_xs_grid_gpu[qs[l] | (((qh >> (2*l)) & 0x03) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const float d1q = __half2float(bxi->d) * (2*(nib & 0x07) + 1); + const float delta = nib & 0x08 ? -1.0f - IQ1S_DELTA : -1.0f + IQ1S_DELTA; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta); +#else + x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_xxs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_XXS); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq1_xxs * bxi = (const block_iq1_xxs *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bxi->qh[kqsx]; + + #pragma unroll + for (int l = 0; l < QR1_XXS/2; ++l) { + const int grid = iq1_xxs_grid_gpu[qs[l] | (((qh >> l) & 0x01) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const float d1q = __half2float(bxi->d) * (2*((qh >> 4) & 0x07) + 1); + const float delta = qh & 0x80 ? -1.0f - IQ1S_DELTA : -1.0f + IQ1S_DELTA; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta); +#else + x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + +template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq1_xxxs( + const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { + GGML_UNUSED(x_gate); + + constexpr int warp_size = ggml_cuda_get_physical_warp_size(); + constexpr int nwarps = ggml_cuda_mmq_get_nthreads(type, J, fallback, GGML_PREC_Q8) / warp_size; + constexpr int I = ggml_cuda_mmq_get_I(type, J, fallback, GGML_PREC_Q8); + constexpr int sram_stride = ggml_cuda_mmq_get_sram_stride(type, J, fallback, GGML_PREC_Q8); + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + MMQ_TILE_NE_K*2); +#else + constexpr tile_x_sizes txs = mmq_get_dp4a_tile_x_sizes(GGML_TYPE_IQ3_S, I); + int * x_qs = (int *) x_tile; + half2 * x_ds = (half2 *) (x_qs + txs.qs); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + + constexpr int threads_per_row = MMQ_ITER_K / (4 * QR1_XXXS); + constexpr int nrows = warp_size / threads_per_row; + const int kqsx = threadIdx.x % threads_per_row; + +#pragma unroll + for (int i0 = 0; i0 < I; i0 += nwarps * nrows) { + int i = i0 + threadIdx.y*nrows + threadIdx.x/threads_per_row; + + if (fallback) { + i = min(i, i_max); + } + + const block_iq1_xxxs * bxi = (const block_iq1_xxxs *) x + kbx0 + i*stride; + + const int qs_packed = get_int_b2(bxi->qs, kqsx); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int nib = (bxi->sc[kqsx/2] >> (4*(kqsx%2))) & 0x0F; + + #pragma unroll + for (int l = 0; l < QR1_XXXS/2; ++l) { + const int grid = iq1_xxxs_grid_gpu[qs[l]]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_qs[i*sram_stride + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*sram_stride + 8*kqsx + (2*l+1)] = grid1; +#else + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+0)] = grid0; + x_qs[i*(2*MMQ_TILE_NE_K + 1) + 8*kqsx + (2*l+1)] = grid1; +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } + + const float d1q = __half2float(bxi->d) * (2*(nib & 0x07) + 1); + const float delta = nib & 0x08 ? -1.0f - IQ1S_DELTA : -1.0f + IQ1S_DELTA; + +#if defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + x_ds[i*sram_stride + kqsx] = make_half2(d1q, d1q*delta); +#else + x_ds[i*(MMQ_TILE_NE_K/4) + i/4 + kqsx] = make_half2(d1q, d1q*delta); +#endif // defined(AMD_MFMA_AVAILABLE) || defined(TURING_MMA_AVAILABLE) || defined(AMD_WMMA_AVAILABLE) + } +} + template static __device__ __forceinline__ void ggml_cuda_mmq_load_tiles_iq2_xxs( const char * __restrict__ x, const char * __restrict__ x_gate, int * __restrict__ x_tile, const int kbx0, const int i_max, const int stride) { GGML_UNUSED(x_gate); diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 49e0ce9050c0..d765ca5a0aa6 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -63,6 +63,15 @@ static void ggml_cuda_mul_mat_q_switch_type(ggml_backend_cuda_context & ctx, con case GGML_TYPE_IQ3_S: mul_mat_q_case(ctx, args, stream); break; + case GGML_TYPE_IQ1_XS: + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_IQ1_XXS: + mul_mat_q_case(ctx, args, stream); + break; + case GGML_TYPE_IQ1_XXXS: + mul_mat_q_case(ctx, args, stream); + break; case GGML_TYPE_IQ4_XS: mul_mat_q_case(ctx, args, stream); break; @@ -407,6 +416,9 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ3_XXS: case GGML_TYPE_IQ3_S: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ4_NL: // ------------------------------------------------- diff --git a/ggml/src/ggml-cuda/mmq.cuh b/ggml/src/ggml-cuda/mmq.cuh index 308968be9dcb..2200fe91d319 100644 --- a/ggml/src/ggml-cuda/mmq.cuh +++ b/ggml/src/ggml-cuda/mmq.cuh @@ -93,6 +93,9 @@ static mmq_q8_1_ds_layout mmq_get_q8_1_ds_layout(const ggml_type type_x) { case GGML_TYPE_IQ3_S: return MMQ_Q8_1_DS_LAYOUT_D4; case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: return MMQ_Q8_1_DS_LAYOUT_DS4; case GGML_TYPE_IQ4_XS: case GGML_TYPE_IQ4_NL: @@ -417,6 +420,9 @@ static constexpr __host__ __device__ tile_x_sizes mmq_get_dp4a_tile_x_sizes(ggml case GGML_TYPE_IQ3_XXS: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ3_S: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ1_S: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ1_XS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ1_XXS: return MMQ_DP4A_TXS_Q8_0; + case GGML_TYPE_IQ1_XXXS:return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ4_XS: return MMQ_DP4A_TXS_Q8_0; case GGML_TYPE_IQ4_NL: return MMQ_DP4A_TXS_Q8_0; default: return tile_x_sizes{0, 0, 0}; @@ -660,6 +666,24 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func ggml_cuda_mmq_load_tiles_iq1_s, ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ1_XS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ1_XS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq1_xs, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ1_XXS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ1_XXS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq1_xxs, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); + case GGML_TYPE_IQ1_XXXS: + return ggml_cuda_mmq_util_funcs( + VDR_IQ1_XXXS_Q8_1_MMQ, + ggml_cuda_mmq_load_tiles_iq1_xxxs, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_dp4a, + ggml_cuda_mmq_write_back_dp4a); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( VDR_IQ2_XXS_Q8_1_MMQ, @@ -830,6 +854,24 @@ static constexpr __device__ ggml_cuda_mmq_util_funcs ggml_cuda_mmq_get_util_func ggml_cuda_mmq_load_tiles_iq1_s, ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ1_XS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq1_xs, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ1_XXS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq1_xxs, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); + case GGML_TYPE_IQ1_XXXS: + return ggml_cuda_mmq_util_funcs( + -1, + ggml_cuda_mmq_load_tiles_iq1_xxxs, + ggml_cuda_mmq_vec_dot_q8_1_q8_1_mma, + ggml_cuda_mmq_write_back_mma); case GGML_TYPE_IQ2_XXS: return ggml_cuda_mmq_util_funcs( -1, @@ -1640,6 +1682,9 @@ extern DECL_MMQ_CASE(GGML_TYPE_Q5_K); extern DECL_MMQ_CASE(GGML_TYPE_Q6_K); // ----------------------------------------- extern DECL_MMQ_CASE(GGML_TYPE_IQ1_S); +extern DECL_MMQ_CASE(GGML_TYPE_IQ1_XS); +extern DECL_MMQ_CASE(GGML_TYPE_IQ1_XXS); +extern DECL_MMQ_CASE(GGML_TYPE_IQ1_XXXS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XXS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_XS); extern DECL_MMQ_CASE(GGML_TYPE_IQ2_S); diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index 1a9cf55cb915..671549c011b1 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -58,6 +58,9 @@ static constexpr __device__ vec_dot_q_cuda_t get_vec_dot_q_cuda(ggml_type type) case GGML_TYPE_IQ2_S: return vec_dot_iq2_s_q8_1; case GGML_TYPE_IQ3_XXS: return vec_dot_iq3_xxs_q8_1; case GGML_TYPE_IQ1_S: return vec_dot_iq1_s_q8_1; + case GGML_TYPE_IQ1_XS: return vec_dot_iq1_xs_q8_1; + case GGML_TYPE_IQ1_XXS: return vec_dot_iq1_xxs_q8_1; + case GGML_TYPE_IQ1_XXXS: return vec_dot_iq1_xxxs_q8_1; case GGML_TYPE_IQ1_M: return vec_dot_iq1_m_q8_1; case GGML_TYPE_IQ4_NL: return vec_dot_iq4_nl_q8_1; case GGML_TYPE_IQ4_XS: return vec_dot_iq4_xs_q8_1; @@ -150,6 +153,9 @@ static __host__ mmvq_parameter_table_id get_device_table_id(int cc) { static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_pascal_older(ggml_type type) { switch (type) { case GGML_TYPE_IQ1_S: return 6; + case GGML_TYPE_IQ1_XS: return 6; + case GGML_TYPE_IQ1_XXS: return 6; + case GGML_TYPE_IQ1_XXXS: return 6; case GGML_TYPE_IQ1_M: return 6; case GGML_TYPE_IQ2_S: return 4; case GGML_TYPE_IQ2_XS: return 5; @@ -190,6 +196,9 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_turing_plus(ggm static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_gcn(ggml_type type) { switch (type) { case GGML_TYPE_IQ1_S: return 5; + case GGML_TYPE_IQ1_XS: return 5; + case GGML_TYPE_IQ1_XXS: return 5; + case GGML_TYPE_IQ1_XXXS: return 5; case GGML_TYPE_IQ1_M: return 5; case GGML_TYPE_IQ2_S: return 4; case GGML_TYPE_IQ2_XS: return 4; @@ -240,6 +249,9 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna1_rdna2(ggm static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna3(ggml_type type) { switch (type) { case GGML_TYPE_IQ1_S: return 6; + case GGML_TYPE_IQ1_XS: return 6; + case GGML_TYPE_IQ1_XXS: return 6; + case GGML_TYPE_IQ1_XXXS: return 6; case GGML_TYPE_IQ1_M: return 6; case GGML_TYPE_IQ2_S: return 4; case GGML_TYPE_IQ2_XS: return 4; @@ -258,6 +270,9 @@ static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna3(ggml_type static constexpr __host__ __device__ int get_mmvq_mmid_max_batch_rdna4(ggml_type type) { switch (type) { case GGML_TYPE_IQ1_S: return 7; + case GGML_TYPE_IQ1_XS: return 7; + case GGML_TYPE_IQ1_XXS: return 7; + case GGML_TYPE_IQ1_XXXS: return 7; case GGML_TYPE_IQ1_M: return 7; case GGML_TYPE_IQ2_S: return 4; case GGML_TYPE_IQ2_XS: return 4; @@ -402,6 +417,9 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { case GGML_TYPE_Q6_K: return ne11 <= 4; case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: return ne11 <= 5; case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ3_S: @@ -1149,9 +1167,10 @@ static void mul_mat_vec_q_switch_ncols_dst( GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, }; - constexpr std::array iq_slow_other = { - GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, - GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, + constexpr std::array iq_slow_other = { + GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, + GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, + GGML_TYPE_IQ1_XS, GGML_TYPE_IQ1_XXS, GGML_TYPE_IQ1_XXXS, }; constexpr std::array slow_pascal = { GGML_TYPE_IQ3_S, @@ -1423,6 +1442,24 @@ static void mul_mat_vec_q_switch_type( nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); break; + case GGML_TYPE_IQ1_XS: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_IQ1_XXS: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; + case GGML_TYPE_IQ1_XXXS: + mul_mat_vec_q_switch_ncols_dst + (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, + nchannels_x, nchannels_y, nchannels_dst, stride_channel_x, stride_channel_y, stride_channel_dst, + nsamples_x, nsamples_dst, stride_sample_x, stride_sample_y, stride_sample_dst, ids_stride, stream); + break; case GGML_TYPE_IQ1_M: mul_mat_vec_q_switch_ncols_dst (vx, vy, ids, fusion, dst, ncols_x, nrows_x, ncols_dst, stride_row_x, stride_col_y, stride_col_dst, diff --git a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py index 7be409bd36ab..c1f910792fde 100755 --- a/ggml/src/ggml-cuda/template-instances/generate_cu_files.py +++ b/ggml/src/ggml-cuda/template-instances/generate_cu_files.py @@ -40,7 +40,7 @@ "GGML_TYPE_Q4_0", "GGML_TYPE_Q4_1", "GGML_TYPE_Q5_0", "GGML_TYPE_Q5_1", "GGML_TYPE_Q8_0", "GGML_TYPE_Q2_K", "GGML_TYPE_Q3_K", "GGML_TYPE_Q4_K", "GGML_TYPE_Q5_K", "GGML_TYPE_Q6_K", "GGML_TYPE_IQ2_XXS", "GGML_TYPE_IQ2_XS", "GGML_TYPE_IQ2_S", "GGML_TYPE_IQ3_XXS", "GGML_TYPE_IQ3_S", - "GGML_TYPE_IQ1_S", "GGML_TYPE_IQ4_NL", "GGML_TYPE_IQ4_XS", "GGML_TYPE_MXFP4", "GGML_TYPE_NVFP4" + "GGML_TYPE_IQ1_S", "GGML_TYPE_IQ1_XS", "GGML_TYPE_IQ1_XXS", "GGML_TYPE_IQ1_XXXS", "GGML_TYPE_IQ4_NL", "GGML_TYPE_IQ4_XS", "GGML_TYPE_MXFP4", "GGML_TYPE_NVFP4" ] SOURCE_MMQ = """// This file has been autogenerated by generate_cu_files.py, do not edit manually. diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xs.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xs.cu new file mode 100644 index 000000000000..ec5a345ccaac --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xs.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_IQ1_XS); diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xxs.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xxs.cu new file mode 100644 index 000000000000..f441a614780f --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xxs.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_IQ1_XXS); diff --git a/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xxxs.cu b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xxxs.cu new file mode 100644 index 000000000000..ac30fac5de2e --- /dev/null +++ b/ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_xxxs.cu @@ -0,0 +1,5 @@ +// This file has been autogenerated by generate_cu_files.py, do not edit manually. + +#include "../mmq.cuh" + +DECL_MMQ_CASE(GGML_TYPE_IQ1_XXXS); diff --git a/ggml/src/ggml-cuda/vecdotq.cuh b/ggml/src/ggml-cuda/vecdotq.cuh index d5ff93b91241..b871b1831394 100644 --- a/ggml/src/ggml-cuda/vecdotq.cuh +++ b/ggml/src/ggml-cuda/vecdotq.cuh @@ -1343,6 +1343,110 @@ static __device__ __forceinline__ float vec_dot_iq1_s_q8_1( return d1q * (ds.x*sumi + ds.y*delta); } +// The narrow IQ1 types below differ from IQ1_S only in how the grid index is reassembled and where +// the per-sub-block scale/sign nibble is stored. Everything else, including the 4-bit grid packing +// and the (grid + delta) offset trick, is inherited unchanged. + +#define VDR_IQ1_XS_Q8_1_MMVQ 1 +#define VDR_IQ1_XS_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq1_xs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + const block_iq1_xs * bq1 = (const block_iq1_xs *) vbq + kbx; + + const int qs_packed = get_int_b2(bq1->qs, iqs); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bq1->qh[iqs]; + const int nib = (bq1->sc[iqs/2] >> (4*(iqs%2))) & 0x0F; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int grid = iq1_xs_grid_gpu[qs[l0/2] | (((qh >> 2*(l0/2)) & 0x03) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid0, u0, sumi); + sumi = ggml_cuda_dp4a(grid1, u1, sumi); + } + + const float d1q = __half2float(bq1->d) * (2*(nib & 0x07) + 1); + const float delta = nib & 0x08 ? -1.0f - IQ1S_DELTA : -1.0f + IQ1S_DELTA; + const float2 ds = __half22float2(bq8_1[iqs].ds); + return d1q * (ds.x*sumi + ds.y*delta); +} + +#define VDR_IQ1_XXS_Q8_1_MMVQ 1 +#define VDR_IQ1_XXS_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq1_xxs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + const block_iq1_xxs * bq1 = (const block_iq1_xxs *) vbq + kbx; + + const int qs_packed = get_int_b2(bq1->qs, iqs); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int qh = bq1->qh[iqs]; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int grid = iq1_xxs_grid_gpu[qs[l0/2] | (((qh >> (l0/2)) & 0x01) << 8)]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid0, u0, sumi); + sumi = ggml_cuda_dp4a(grid1, u1, sumi); + } + + const float d1q = __half2float(bq1->d) * (2*((qh >> 4) & 0x07) + 1); + const float delta = qh & 0x80 ? -1.0f - IQ1S_DELTA : -1.0f + IQ1S_DELTA; + const float2 ds = __half22float2(bq8_1[iqs].ds); + return d1q * (ds.x*sumi + ds.y*delta); +} + +#define VDR_IQ1_XXXS_Q8_1_MMVQ 1 +#define VDR_IQ1_XXXS_Q8_1_MMQ 1 + +static __device__ __forceinline__ float vec_dot_iq1_xxxs_q8_1( + const void * __restrict__ vbq, const block_q8_1 * __restrict__ bq8_1, const int & kbx, const int & iqs) { + const block_iq1_xxxs * bq1 = (const block_iq1_xxxs *) vbq + kbx; + + const int qs_packed = get_int_b2(bq1->qs, iqs); + const uint8_t * qs = (const uint8_t *) &qs_packed; + + const int nib = (bq1->sc[iqs/2] >> (4*(iqs%2))) & 0x0F; + + int sumi = 0; +#pragma unroll + for (int l0 = 0; l0 < 8; l0 += 2) { + const int grid = iq1_xxxs_grid_gpu[qs[l0/2]]; + + const int grid0 = (grid >> 0) & 0x0F0F0F0F; + const int grid1 = (grid >> 4) & 0x0F0F0F0F; + + const int u0 = get_int_b4(bq8_1[iqs].qs, l0 + 0); + const int u1 = get_int_b4(bq8_1[iqs].qs, l0 + 1); + + sumi = ggml_cuda_dp4a(grid0, u0, sumi); + sumi = ggml_cuda_dp4a(grid1, u1, sumi); + } + + const float d1q = __half2float(bq1->d) * (2*(nib & 0x07) + 1); + const float delta = nib & 0x08 ? -1.0f - IQ1S_DELTA : -1.0f + IQ1S_DELTA; + const float2 ds = __half22float2(bq8_1[iqs].ds); + return d1q * (ds.x*sumi + ds.y*delta); +} + #define VDR_IQ1_M_Q8_1_MMVQ 1 #define VDR_IQ1_M_Q8_1_MMQ 1 diff --git a/ggml/src/ggml-impl.h b/ggml/src/ggml-impl.h index 4dc55e78e218..bcc640172923 100644 --- a/ggml/src/ggml-impl.h +++ b/ggml/src/ggml-impl.h @@ -566,6 +566,50 @@ static inline uint8_t ggml_fp32_to_ue4m3(float x) { return (uint8_t) ((ue4m3_exp << 3) | ue4m3_man); } +// E4M3 (fn): signed, 4 exp bits (bias=7), 3 mantissa bits, no inf, 0x7F/0xFF = NaN (used by MXFP8) +static inline float ggml_e4m3_to_fp32(uint8_t x) { + const int sign = x & 0x80; + const int exp = (x >> 3) & 0xF; + const int man = x & 0x7; + float v; + if (exp == 0xF && man == 0x7) { + v = NAN; + } else if (exp == 0) { + v = ldexpf((float) man, -9); + } else { + v = ldexpf((float) (8 + man), exp - 10); + } + return sign ? -v : v; +} + +// fp32 -> E4M3 (fn), round to nearest even, saturating to +-448, NaN -> 0x7F +static inline uint8_t ggml_fp32_to_e4m3(float x) { + if (x != x) { + return 0x7F; + } + const uint8_t sign = signbit(x) ? 0x80 : 0x00; + float a = fabsf(x); + if (a >= 448.0f) { + return sign | 0x7E; + } + if (a < 0.015625f) { // subnormal grid 2^-9 + const int m = (int) nearbyintf(a * 512.0f); + return sign | (uint8_t) m; + } + int e; + const float f = frexpf(a, &e); + int m = (int) nearbyintf((f * 2.0f - 1.0f) * 8.0f); + int E = e - 1 + 7; + if (m == 8) { + m = 0; + E++; + } + if (E > 15 || (E == 15 && m == 7)) { + return sign | 0x7E; + } + return sign | (uint8_t) ((E << 3) | m); +} + static inline float ggml_f8_e4m3_to_fp32(uint8_t x) { const uint8_t ax = x & 0x7F; if (ax == 0x7F) { diff --git a/ggml/src/ggml-quants.c b/ggml/src/ggml-quants.c index 3586f67ae9fd..2da6b1755e30 100644 --- a/ggml/src/ggml-quants.c +++ b/ggml/src/ggml-quants.c @@ -344,6 +344,35 @@ static inline int best_index_mxfp4(float x, float e) { return best_index; } +void quantize_row_mxfp8_ref(const float * GGML_RESTRICT x, block_mxfp8 * GGML_RESTRICT y, int64_t k) { + static const int qk = QK_MXFP8; + + assert(k % qk == 0); + + const int64_t nb = k / qk; + + for (int64_t i = 0; i < nb; i++) { + float amax = 0.0f; + for (int j = 0; j < qk; j++) { + amax = MAX(amax, fabsf(x[i*qk + j])); + } + + // smallest power of two scale with amax/scale <= 448 (largest e4m3 value) + int e = 0; + if (amax > 0.0f) { + e = (int) ceilf(log2f(amax / 448.0f)) + 127; + e = MAX(0, MIN(254, e)); + } + const float d = GGML_E8M0_TO_FP32((uint8_t) e); + const float id = 1.0f/d; + + y[i].e = (uint8_t) e; + for (int j = 0; j < qk; ++j) { + y[i].qs[j] = ggml_fp32_to_e4m3(x[i*qk + j]*id); + } + } +} + void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k) { static const int qk = QK_MXFP4; @@ -589,6 +618,23 @@ void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_REST } } +void dequantize_row_mxfp8(const block_mxfp8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + static const int qk = QK_MXFP8; + + assert(k % qk == 0); + + const int64_t nb = k / qk; + + for (int64_t i = 0; i < nb; i++) { + // exact: e4m3 * 2^(e-127) is representable in fp32 for every code and scale + const float d = x[i].e == 0xFF ? NAN : GGML_E8M0_TO_FP32(x[i].e); + + for (int j = 0; j < qk; ++j) { + y[i*qk + j] = ggml_e4m3_to_fp32(x[i].qs[j])*d; + } + } +} + void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { static const int qk = QK_NVFP4; static const int qk_sub = QK_NVFP4_SUB; @@ -2315,6 +2361,12 @@ size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * ggml_row_size(GGML_TYPE_MXFP4, n_per_row); } +size_t quantize_mxfp8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_UNUSED(quant_weights); + quantize_row_mxfp8_ref(src, dst, (int64_t)nrow*n_per_row); + return nrow * ggml_row_size(GGML_TYPE_MXFP8, n_per_row); +} + size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { GGML_UNUSED(quant_weights); quantize_row_nvfp4_ref(src, dst, (int64_t)nrow*n_per_row); @@ -2688,6 +2740,84 @@ void dequantize_row_iq1_s(const block_iq1_s * GGML_RESTRICT x, float * GGML_REST } } +void dequantize_row_iq1_xs(const block_iq1_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + const uint8_t * sc = x[i].sc; + + for (int ib = 0; ib < QK_K/32; ++ib) { + const int nib = (sc[ib/2] >> (4*(ib&1))) & 0xf; + const float dl = d * (2*(nib & 7) + 1); + const float delta = nib & 8 ? -IQ1S_DELTA : IQ1S_DELTA; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1_xs_grid + (qs[l] | (((qh[ib] >> 2*l) & 3) << 8))); + for (int j = 0; j < 8; ++j) { + y[j] = dl * (grid[j] + delta); + } + y += 8; + } + qs += 4; + } + } +} + +void dequantize_row_iq1_xxs(const block_iq1_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * qs = x[i].qs; + const uint8_t * qh = x[i].qh; + + for (int ib = 0; ib < QK_K/32; ++ib) { + const float dl = d * (2*((qh[ib] >> 4) & 7) + 1); + const float delta = qh[ib] & 0x80 ? -IQ1S_DELTA : IQ1S_DELTA; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1_xxs_grid + (qs[l] | (((qh[ib] >> l) & 1) << 8))); + for (int j = 0; j < 8; ++j) { + y[j] = dl * (grid[j] + delta); + } + y += 8; + } + qs += 4; + } + } +} + +void dequantize_row_iq1_xxxs(const block_iq1_xxxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { + assert(k % QK_K == 0); + const int64_t nb = k / QK_K; + + for (int i = 0; i < nb; i++) { + + const float d = GGML_FP16_TO_FP32(x[i].d); + const uint8_t * qs = x[i].qs; + const uint8_t * sc = x[i].sc; + + for (int ib = 0; ib < QK_K/32; ++ib) { + const int nib = (sc[ib/2] >> (4*(ib&1))) & 0xf; + const float dl = d * (2*(nib & 7) + 1); + const float delta = nib & 8 ? -IQ1S_DELTA : IQ1S_DELTA; + for (int l = 0; l < 4; ++l) { + const int8_t * grid = (const int8_t *)(iq1_xxxs_grid + qs[l]); + for (int j = 0; j < 8; ++j) { + y[j] = dl * (grid[j] + delta); + } + y += 8; + } + qs += 4; + } + } +} + void dequantize_row_iq1_m(const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k) { assert(k % QK_K == 0); const int64_t nb = k / QK_K; @@ -2839,7 +2969,10 @@ typedef struct { uint16_t * neighbours; } iq2_entry_t; -static iq2_entry_t iq2_data[4] = { +static iq2_entry_t iq2_data[7] = { + {NULL, NULL, NULL}, + {NULL, NULL, NULL}, + {NULL, NULL, NULL}, {NULL, NULL, NULL}, {NULL, NULL, NULL}, {NULL, NULL, NULL}, @@ -2847,17 +2980,42 @@ static iq2_entry_t iq2_data[4] = { }; static inline int iq2_data_index(enum ggml_type type) { - GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S); - return type == GGML_TYPE_IQ2_XXS ? 0 : - type == GGML_TYPE_IQ2_XS ? 1 : - type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 2 : 3; + switch (type) { + case GGML_TYPE_IQ2_XXS: return 0; + case GGML_TYPE_IQ2_XS: return 1; + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: return 2; + case GGML_TYPE_IQ2_S: return 3; + case GGML_TYPE_IQ1_XS: return 4; + case GGML_TYPE_IQ1_XXS: return 5; + case GGML_TYPE_IQ1_XXXS: return 6; + default: GGML_ABORT("fatal error"); + } } static inline int iq2_grid_size(enum ggml_type type) { - GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S); - return type == GGML_TYPE_IQ2_XXS ? 256 : - type == GGML_TYPE_IQ2_XS ? 512 : - type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? NGRID_IQ1S : 1024; + switch (type) { + case GGML_TYPE_IQ2_XXS: return 256; + case GGML_TYPE_IQ2_XS: return 512; + case GGML_TYPE_IQ1_S: + case GGML_TYPE_IQ1_M: return NGRID_IQ1S; + case GGML_TYPE_IQ2_S: return 1024; + case GGML_TYPE_IQ1_XS: return NGRID_IQ1XS; + case GGML_TYPE_IQ1_XXS: return NGRID_IQ1XXS; + case GGML_TYPE_IQ1_XXXS: return NGRID_IQ1XXXS; + default: GGML_ABORT("fatal error"); + } +} + +// the narrow IQ1 grids are subsets of iq1s_grid and are stored in the -1/0/1 form that +// dequantization uses, not as the packed 2-bit levels the other grids are built from +static const uint64_t * iq1_narrow_grid(enum ggml_type type) { + switch (type) { + case GGML_TYPE_IQ1_XS: return iq1_xs_grid; + case GGML_TYPE_IQ1_XXS: return iq1_xxs_grid; + case GGML_TYPE_IQ1_XXXS: return iq1_xxxs_grid; + default: return NULL; + } } static int iq2_compare_func(const void * left, const void * right) { @@ -3122,8 +3280,9 @@ void iq2xs_init_impl(enum ggml_type type) { }; const int kmap_size = 43692; + const uint64_t * narrow = iq1_narrow_grid(type); //const int nwant = type == GGML_TYPE_IQ1_S ? 3 : 2; - const int nwant = type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2; + const int nwant = narrow || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? 3 : type == GGML_TYPE_IQ2_S ? 1 : 2; const uint16_t * kgrid = type == GGML_TYPE_IQ2_XXS ? kgrid_2bit_256 : type == GGML_TYPE_IQ2_XS ? kgrid_2bit_512 : type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M ? kgrid_1bit_2048 : kgrid_2bit_1024; @@ -3136,7 +3295,7 @@ void iq2xs_init_impl(enum ggml_type type) { for (int k = 0; k < grid_size; ++k) { int8_t * pos = (int8_t *)(the_grid + k); for (int i = 0; i < 8; ++i) { - int l = (kgrid[k] >> 2*i) & 0x3; + int l = narrow ? ((const int8_t *)(narrow + k))[i] + 1 : (kgrid[k] >> 2*i) & 0x3; pos[i] = 2*l + 1; } } @@ -3274,7 +3433,6 @@ void iq2xs_init_impl(enum ggml_type type) { } void iq2xs_free_impl(enum ggml_type type) { - GGML_ASSERT(type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || type == GGML_TYPE_IQ1_S || type == GGML_TYPE_IQ1_M || type == GGML_TYPE_IQ2_S); const int gindex = iq2_data_index(type); if (iq2_data[gindex].grid) { free(iq2_data[gindex].grid); iq2_data[gindex].grid = NULL; @@ -4705,6 +4863,257 @@ size_t quantize_iq1_s(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, return nrow * nblock * sizeof(block_iq1_s); } +// IQ1_XS / IQ1_XXS / IQ1_XXXS share the IQ1_S search verbatim and differ only in the grid +// they may select from and in how the resulting index is split between qs and qh. +static void quantize_row_iq1_narrow_impl(enum ggml_type type, const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, + const float * GGML_RESTRICT quant_weights, + float * scales, + float * weight, + float * sumx, + float * sumw, + float * pairs, + int8_t * L, + uint16_t * index, + int8_t * shifts) { + + const int gindex = iq2_data_index(type); + const int ngrid = iq2_grid_size(type); + + const uint64_t * kgrid_q2xs = iq2_data[gindex].grid; + const int * kmap_q2xs = iq2_data[gindex].map; + const uint16_t * kneighbors_q2xs = iq2_data[gindex].neighbours; + + GGML_ASSERT(quant_weights && "missing quantization weights"); + GGML_ASSERT(kgrid_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kmap_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(kneighbors_q2xs && "forgot to call ggml_quantize_init()?"); + GGML_ASSERT(n%QK_K == 0); + + block_iq1_xs * y_xs = vy; + block_iq1_xxs * y_xxs = vy; + block_iq1_xxxs * y_xxxs = vy; + + const int64_t nbl = n/QK_K; + + const int block_size = IQ1S_BLOCK_SIZE; + + const float x_p[3] = {-1 + IQ1S_DELTA, IQ1S_DELTA, 1 + IQ1S_DELTA}; + const float x_m[3] = {-1 - IQ1S_DELTA, -IQ1S_DELTA, 1 - IQ1S_DELTA}; + + int * idx = (int *)(pairs + 1); + + for (int ibl = 0; ibl < nbl; ++ibl) { + + float max_scale = 0; + + memset(index, 0, (QK_K/8)*sizeof(uint16_t)); + + const float * xbl = x + QK_K*ibl; + float sumx2 = 0; + for (int i = 0; i < QK_K; ++i) sumx2 += xbl[i]*xbl[i]; + float sigma2 = 2*sumx2/QK_K; + + for (int ib = 0; ib < QK_K/block_size; ++ib) { + const float * xb = xbl + block_size*ib; + const float * qw = quant_weights + QK_K*ibl + block_size*ib; + for (int i = 0; i < block_size; ++i) weight[i] = qw[i] * sqrtf(sigma2 + xb[i]*xb[i]); + float max = fabsf(xb[0]); + for (int i = 1; i < block_size; ++i) max = MAX(max, fabsf(xb[i])); + if (max < GROUP_MAX_EPS_IQ1_S) { + scales[ib] = 0; + shifts[ib] = 1; + memset(L, 1, block_size); + continue; + } + for (int j = 0; j < block_size; ++j) { + pairs[2*j] = xb[j]; + idx[2*j] = j; + } + qsort(pairs, block_size, 2*sizeof(float), iq1_sort_helper); + { + sumx[0] = sumw[0] = 0; + for (int j = 0; j < block_size; ++j) { + int i = idx[2*j]; + sumx[j+1] = sumx[j] + weight[i]*xb[i]; + sumw[j+1] = sumw[j] + weight[i]; + } + } + float best_score = -FLT_MAX, scale = max; + int besti1 = -1, besti2 = -1, best_shift = 0; + for (int i1 = 0; i1 <= block_size; ++i1) { + for (int i2 = i1; i2 <= block_size; ++i2) { + float sumqx = (sumx[i1] - sumx[0])*x_p[0] + (sumx[i2] - sumx[i1])*x_p[1] + (sumx[block_size] - sumx[i2])*x_p[2]; + float sumq2 = (sumw[i1] - sumw[0])*x_p[0]*x_p[0] + (sumw[i2] - sumw[i1])*x_p[1]*x_p[1] + (sumw[block_size] - sumw[i2])*x_p[2]*x_p[2]; + if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + scale = sumqx/sumq2; best_score = scale*sumqx; + besti1 = i1; besti2 = i2; best_shift = 1; + } + sumqx = (sumx[i1] - sumx[0])*x_m[0] + (sumx[i2] - sumx[i1])*x_m[1] + (sumx[block_size] - sumx[i2])*x_m[2]; + sumq2 = (sumw[i1] - sumw[0])*x_m[0]*x_m[0] + (sumw[i2] - sumw[i1])*x_m[1]*x_m[1] + (sumw[block_size] - sumw[i2])*x_m[2]*x_m[2]; + if (sumq2 > 0 && sumqx*sumqx > best_score*sumq2) { + scale = sumqx/sumq2; best_score = scale*sumqx; + besti1 = i1; besti2 = i2; best_shift = -1; + } + } + } + if (besti1 < 0 || besti2 < 0 || best_shift == 0) { + scales[ib] = 0; + shifts[ib] = 1; + memset(L, 1, block_size); + continue; + } + for (int j = 0; j < besti1; ++j) L[idx[2*j]] = 0; + for (int j = besti1; j < besti2; ++j) L[idx[2*j]] = 1; + for (int j = besti2; j < block_size; ++j) L[idx[2*j]] = 2; + if (scale < 0) { + for (int j = 0; j < block_size; ++j) L[j] = 2 - L[j]; + scale = -scale; best_shift = -best_shift; + } + bool all_on_grid = true; + const float * xx = best_shift == 1 ? x_p : x_m; + uint16_t * index_b = index + (block_size/8)*ib; + for (int k = 0; k < block_size/8; ++k) { + uint16_t u = 0; + for (int j = 0; j < 8; ++j) u |= (L[8*k+j] << 2*j); + int grid_index = kmap_q2xs[u]; + if (grid_index < 0) { + all_on_grid = false; + const uint16_t * neighbours = kneighbors_q2xs - kmap_q2xs[u] - 1; + grid_index = iq1_find_best_neighbour2(neighbours, kgrid_q2xs, xb + 8*k, weight + 8*k, scale, xx, L + 8*k, ngrid); + GGML_ASSERT(grid_index >= 0); + } + index_b[k] = grid_index; + } + if (!all_on_grid) { + float sumqx = 0, sumq2 = 0; + for (int k = 0; k < block_size/8; ++k) { + const int8_t * pg = (const int8_t *)(kgrid_q2xs + index_b[k]); + for (int j = 0; j < 8; ++j) { + float w = weight[8*k + j]; + float q = xx[(pg[j] - 1)/2]; + sumqx += w*q*xb[8*k+j]; + sumq2 += w*q*q; + } + } + if (sumqx > 0 && sumq2 > 0) scale = sumqx/sumq2; + } + GGML_ASSERT(scale >= 0); + scales[ib] = scale; + shifts[ib] = best_shift; + max_scale = MAX(max_scale, scale); + } + + // the index fields are written whatever the scales turn out to be, and the scale + // codes are left at zero when the whole super-block is degenerate, exactly as + // quantize_row_iq1_s_impl does + switch (type) { + case GGML_TYPE_IQ1_XS: + y_xs[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(y_xs[ibl].qh, 0, QK_K/32); + memset(y_xs[ibl].sc, 0, QK_K/64); + for (int ib = 0; ib < QK_K/block_size; ++ib) { + uint8_t h = 0; + for (int k = 0; k < block_size/8; ++k) { + const uint16_t gi = index[(block_size/8)*ib + k]; + GGML_ASSERT(gi < ngrid); // else the high bits corrupt the next index + y_xs[ibl].qs[(block_size/8)*ib + k] = gi & 255; + h |= (gi >> 8) << 2*k; + } + y_xs[ibl].qh[ib] = h; + } + break; + case GGML_TYPE_IQ1_XXS: + y_xxs[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(y_xxs[ibl].qh, 0, QK_K/32); + for (int ib = 0; ib < QK_K/block_size; ++ib) { + uint8_t h = 0; + for (int k = 0; k < block_size/8; ++k) { + const uint16_t gi = index[(block_size/8)*ib + k]; + GGML_ASSERT(gi < ngrid); + y_xxs[ibl].qs[(block_size/8)*ib + k] = gi & 255; + h |= (gi >> 8) << k; + } + y_xxs[ibl].qh[ib] = h; + } + break; + case GGML_TYPE_IQ1_XXXS: + y_xxxs[ibl].d = GGML_FP32_TO_FP16(0.f); + memset(y_xxxs[ibl].sc, 0, QK_K/64); + for (int j = 0; j < QK_K/8; ++j) { + GGML_ASSERT(index[j] < ngrid); + y_xxxs[ibl].qs[j] = index[j] & 255; + } + break; + default: GGML_ABORT("fatal error"); + } + + if (!max_scale) { + continue; + } + + float d = max_scale/15; + float id = 1/d; + for (int ib = 0; ib < QK_K/block_size; ++ib) { + int l = nearest_int(0.5f*(id*scales[ib]-1)); + l = MAX(0, MIN(7, l)); + if (shifts[ib] == -1) l |= 8; + switch (type) { + case GGML_TYPE_IQ1_XS: y_xs[ibl].sc[ib/2] |= l << (4*(ib&1)); break; + case GGML_TYPE_IQ1_XXS: y_xxs[ibl].qh[ib] |= l << 4; break; + case GGML_TYPE_IQ1_XXXS: y_xxxs[ibl].sc[ib/2] |= l << (4*(ib&1)); break; + default: GGML_ABORT("fatal error"); + } + } + + switch (type) { + case GGML_TYPE_IQ1_XS: y_xs[ibl].d = GGML_FP32_TO_FP16(d*1.125f); break; + case GGML_TYPE_IQ1_XXS: y_xxs[ibl].d = GGML_FP32_TO_FP16(d*1.125f); break; + case GGML_TYPE_IQ1_XXXS: y_xxxs[ibl].d = GGML_FP32_TO_FP16(d*1.125f); break; + default: GGML_ABORT("fatal error"); + } + } +} + +static size_t quantize_iq1_narrow(enum ggml_type type, const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, + int64_t nrow, int64_t n_per_row, const float * quant_weights) { + GGML_ASSERT(n_per_row%QK_K == 0); + float scales[QK_K/IQ1S_BLOCK_SIZE]; + float weight[IQ1S_BLOCK_SIZE]; + int8_t L[IQ1S_BLOCK_SIZE]; + float sumx[IQ1S_BLOCK_SIZE+1]; + float sumw[IQ1S_BLOCK_SIZE+1]; + float pairs[2*IQ1S_BLOCK_SIZE]; + uint16_t index[QK_K/8]; + int8_t shifts[QK_K/IQ1S_BLOCK_SIZE]; + size_t block_bytes = 0; + switch (type) { + case GGML_TYPE_IQ1_XS: block_bytes = sizeof(block_iq1_xs); break; + case GGML_TYPE_IQ1_XXS: block_bytes = sizeof(block_iq1_xxs); break; + case GGML_TYPE_IQ1_XXXS: block_bytes = sizeof(block_iq1_xxxs); break; + default: GGML_ABORT("fatal error"); + } + int64_t nblock = n_per_row/QK_K; + char * qrow = (char *)dst; + for (int64_t row = 0; row < nrow; ++row) { + quantize_row_iq1_narrow_impl(type, src, qrow, n_per_row, quant_weights, scales, weight, sumx, sumw, pairs, L, index, shifts); + src += n_per_row; + qrow += nblock*block_bytes; + } + return nrow * nblock * block_bytes; +} + +size_t quantize_iq1_xs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + return quantize_iq1_narrow(GGML_TYPE_IQ1_XS, src, dst, nrow, n_per_row, quant_weights); +} + +size_t quantize_iq1_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + return quantize_iq1_narrow(GGML_TYPE_IQ1_XXS, src, dst, nrow, n_per_row, quant_weights); +} + +size_t quantize_iq1_xxxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrow, int64_t n_per_row, const float * quant_weights) { + return quantize_iq1_narrow(GGML_TYPE_IQ1_XXXS, src, dst, nrow, n_per_row, quant_weights); +} + static void quantize_row_iq1_m_impl(const float * GGML_RESTRICT x, void * GGML_RESTRICT vy, int64_t n, const float * GGML_RESTRICT quant_weights, float * scales, float * weight, @@ -5549,6 +5958,10 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp4, data, nb); } break; + case GGML_TYPE_MXFP8: + { + VALIDATE_ROW_DATA_E_E8M0_IMPL(block_mxfp8, data, nb); + } break; case GGML_TYPE_NVFP4: { // TODO: validate each unsigned E4M3 scale; NaN encodings and scales with the sign bit set are invalid. @@ -5606,6 +6019,18 @@ bool ggml_validate_row_data(enum ggml_type type, const void * data, size_t nbyte { VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_s, data, nb); } break; + case GGML_TYPE_IQ1_XS: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_xs, data, nb); + } break; + case GGML_TYPE_IQ1_XXS: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_xxs, data, nb); + } break; + case GGML_TYPE_IQ1_XXXS: + { + VALIDATE_ROW_DATA_D_F16_IMPL(block_iq1_xxxs, data, nb); + } break; case GGML_TYPE_IQ1_M: { const block_iq1_m * q = (const block_iq1_m *) data; diff --git a/ggml/src/ggml-quants.h b/ggml/src/ggml-quants.h index ca36d13695c8..c11fe8acfab8 100644 --- a/ggml/src/ggml-quants.h +++ b/ggml/src/ggml-quants.h @@ -24,6 +24,7 @@ GGML_API void quantize_row_q8_0_ref(const float * GGML_RESTRICT x, block_q8_0 * GGML_API void quantize_row_q8_1_ref(const float * GGML_RESTRICT x, block_q8_1 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_mxfp4_ref(const float * GGML_RESTRICT x, block_mxfp4 * GGML_RESTRICT y, int64_t k); +GGML_API void quantize_row_mxfp8_ref(const float * GGML_RESTRICT x, block_mxfp8 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_nvfp4_ref(const float * GGML_RESTRICT x, block_nvfp4 * GGML_RESTRICT y, int64_t k); GGML_API void quantize_row_f8_e4m3_ref(const float * GGML_RESTRICT x, ggml_fp8_e4m3_t * GGML_RESTRICT y, int64_t k); @@ -54,6 +55,7 @@ GGML_API void dequantize_row_q8_0(const block_q8_0 * GGML_RESTRICT x, float * GG //GGML_API void dequantize_row_q8_1(const block_q8_1 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_mxfp4(const block_mxfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_mxfp8(const block_mxfp8 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_nvfp4(const block_nvfp4 * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_f8_e4m3(const ggml_fp8_e4m3_t * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -73,6 +75,9 @@ GGML_API void dequantize_row_iq2_s (const block_iq2_s * GGML_RESTRICT x, floa GGML_API void dequantize_row_iq3_xxs(const block_iq3_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq1_s (const block_iq1_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq1_m (const block_iq1_m * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_iq1_xs (const block_iq1_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_iq1_xxs (const block_iq1_xxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); +GGML_API void dequantize_row_iq1_xxxs(const block_iq1_xxxs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq4_nl (const block_iq4_nl * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq4_xs (const block_iq4_xs * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); GGML_API void dequantize_row_iq3_s (const block_iq3_s * GGML_RESTRICT x, float * GGML_RESTRICT y, int64_t k); @@ -84,6 +89,9 @@ GGML_API size_t quantize_iq2_s (const float * GGML_RESTRICT src, void * GGML_RE GGML_API size_t quantize_iq3_xxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq1_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq1_m (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_iq1_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_iq1_xxs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_iq1_xxxs(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq4_nl (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq4_xs (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_iq3_s (const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); @@ -105,6 +113,7 @@ GGML_API size_t quantize_q5_1(const float * GGML_RESTRICT src, void * GGML_RESTR GGML_API size_t quantize_q8_0(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_mxfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); +GGML_API size_t quantize_mxfp8(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_nvfp4(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); GGML_API size_t quantize_f8_e4m3(const float * GGML_RESTRICT src, void * GGML_RESTRICT dst, int64_t nrows, int64_t n_per_row, const float * imatrix); diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c index 3d60c44e594f..2eb143a00d5d 100644 --- a/ggml/src/ggml.c +++ b/ggml/src/ggml.c @@ -766,6 +766,14 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_mxfp4, .from_float_ref = (ggml_from_float_t)quantize_row_mxfp4_ref, }, + [GGML_TYPE_MXFP8] = { + .type_name = "mxfp8", + .blck_size = QK_MXFP8, + .type_size = sizeof(block_mxfp8), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_mxfp8, + .from_float_ref = (ggml_from_float_t)quantize_row_mxfp8_ref, + }, [GGML_TYPE_NVFP4] = { .type_name = "nvfp4", .blck_size = QK_NVFP4, @@ -870,6 +878,30 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = { .to_float = (ggml_to_float_t) dequantize_row_iq1_s, .from_float_ref = NULL, }, + [GGML_TYPE_IQ1_XS] = { + .type_name = "iq1_xs", + .blck_size = QK_K, + .type_size = sizeof(block_iq1_xs), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq1_xs, + .from_float_ref = NULL, + }, + [GGML_TYPE_IQ1_XXS] = { + .type_name = "iq1_xxs", + .blck_size = QK_K, + .type_size = sizeof(block_iq1_xxs), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq1_xxs, + .from_float_ref = NULL, + }, + [GGML_TYPE_IQ1_XXXS] = { + .type_name = "iq1_xxxs", + .blck_size = QK_K, + .type_size = sizeof(block_iq1_xxxs), + .is_quantized = true, + .to_float = (ggml_to_float_t) dequantize_row_iq1_xxxs, + .from_float_ref = NULL, + }, [GGML_TYPE_IQ1_M] = { .type_name = "iq1_m", .blck_size = QK_K, @@ -1474,6 +1506,9 @@ enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype) { case GGML_FTYPE_MOSTLY_IQ4_XS: wtype = GGML_TYPE_IQ4_XS; break; case GGML_FTYPE_MOSTLY_IQ3_S: wtype = GGML_TYPE_IQ3_S; break; case GGML_FTYPE_MOSTLY_IQ2_S: wtype = GGML_TYPE_IQ2_S; break; + case GGML_FTYPE_MOSTLY_IQ1_XS: wtype = GGML_TYPE_IQ1_XS; break; + case GGML_FTYPE_MOSTLY_IQ1_XXS: wtype = GGML_TYPE_IQ1_XXS; break; + case GGML_FTYPE_MOSTLY_IQ1_XXXS: wtype = GGML_TYPE_IQ1_XXXS; break; case GGML_FTYPE_UNKNOWN: wtype = GGML_TYPE_COUNT; break; case GGML_FTYPE_MOSTLY_Q4_1_SOME_F16: wtype = GGML_TYPE_COUNT; break; } @@ -8164,7 +8199,10 @@ void ggml_quantize_init(enum ggml_type type) { case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ1_S: - case GGML_TYPE_IQ1_M: iq2xs_init_impl(type); break; + case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: iq2xs_init_impl(type); break; case GGML_TYPE_IQ3_XXS: iq3xs_init_impl(256); break; case GGML_TYPE_IQ3_S: iq3xs_init_impl(512); break; default: // nothing @@ -8182,6 +8220,9 @@ void ggml_quantize_free(void) { iq2xs_free_impl(GGML_TYPE_IQ2_S); iq2xs_free_impl(GGML_TYPE_IQ1_S); iq2xs_free_impl(GGML_TYPE_IQ1_M); + iq2xs_free_impl(GGML_TYPE_IQ1_XS); + iq2xs_free_impl(GGML_TYPE_IQ1_XXS); + iq2xs_free_impl(GGML_TYPE_IQ1_XXXS); iq3xs_free_impl(256); iq3xs_free_impl(512); @@ -8192,6 +8233,9 @@ bool ggml_quantize_requires_imatrix(enum ggml_type type) { return type == GGML_TYPE_IQ2_XXS || type == GGML_TYPE_IQ2_XS || + type == GGML_TYPE_IQ1_XS || + type == GGML_TYPE_IQ1_XXS || + type == GGML_TYPE_IQ1_XXXS || type == GGML_TYPE_IQ1_S;// || //type == GGML_TYPE_IQ1_M; } @@ -8229,6 +8273,7 @@ size_t ggml_quantize_chunk( case GGML_TYPE_Q5_1: result = quantize_q5_1 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q8_0: result = quantize_q8_0 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_MXFP4: result = quantize_mxfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_MXFP8: result = quantize_mxfp8 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_NVFP4: result = quantize_nvfp4 (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_F8_E4M3: result = quantize_f8_e4m3(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_Q2_K: result = quantize_q2_K (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; @@ -8245,6 +8290,9 @@ size_t ggml_quantize_chunk( case GGML_TYPE_IQ2_S: result = quantize_iq2_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ1_S: result = quantize_iq1_s (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ1_M: result = quantize_iq1_m (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ1_XS: result = quantize_iq1_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ1_XXS: result = quantize_iq1_xxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; + case GGML_TYPE_IQ1_XXXS:result = quantize_iq1_xxxs(src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ4_NL: result = quantize_iq4_nl (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_IQ4_XS: result = quantize_iq4_xs (src + start, (char *) dst + start_row * row_size, nrows, n_per_row, imatrix); break; case GGML_TYPE_F16: diff --git a/ggml/src/gguf.cpp b/ggml/src/gguf.cpp index 75ceade7d5fb..962cca75f899 100644 --- a/ggml/src/gguf.cpp +++ b/ggml/src/gguf.cpp @@ -3,6 +3,7 @@ #include "ggml-impl.h" #include "gguf.h" +#include #include #include #include @@ -368,6 +369,11 @@ struct gguf_reader { return data_offset; } + // bytes left between the current position and the end of the input + uint64_t remain() const { + return nbytes_remain; + } + // position in the file where the GGUF data starts, alignment is relative to it, not to the file uint64_t start() const { return start_offset; @@ -787,6 +793,87 @@ static struct gguf_context * gguf_init_from_reader(const struct gguf_reader & gr // store the current file offset - this is where the data section starts ctx->offset = gr.tell(); + // MXFP8 was type 43 in this fork until upstream took 43 for F8_E4M3, so files written before that store their MXFP8 + // tensors as 43. Tell them apart by the bytes a type-43 tensor occupies in the data section: the span to the next + // tensor in data order (or to the end of the file for the last one) fits MXFP8 (33 bytes per 32 values) or F8_E4M3 + // (1 byte per value). Small tensors at an alignment above 32 can pad to the same span either way; the rest of the + // file decides those: a type-43 tensor that fits only one of the two, a type-63 (MXFP8) tensor or general.file_type + // MOSTLY_F8_E4M3. With nothing to go on, the file is refused instead of guessed. + { + enum { FIT_NONE, FIT_F8, FIT_MX, FIT_BOTH }; + const uint64_t data_end = gr.remain(); // the reader sits at the start of the data section + std::vector offs; // data offsets in data order: the header order is not trusted here + offs.reserve(ctx->info.size()); + for (const gguf_tensor_info & ti : ctx->info) { + offs.push_back(ti.offset); + } + std::sort(offs.begin(), offs.end()); + std::vector fit(ctx->info.size(), FIT_NONE); + bool old_ids = false; + bool new_ids = false; + for (size_t i = 0; i < ctx->info.size(); ++i) { + const gguf_tensor_info & ti = ctx->info[i]; + if (ti.t.type == GGML_TYPE_MXFP8) { + new_ids = true; + } + if (ti.t.type != GGML_TYPE_F8_E4M3) { + continue; + } + if (ti.t.ne[0] % ggml_blck_size(GGML_TYPE_MXFP8) != 0) { + fit[i] = FIT_F8; + new_ids = true; + continue; + } + const int64_t n_rows = ggml_nrows(&ti.t); + const uint64_t f8 = ggml_row_size(GGML_TYPE_F8_E4M3, ti.t.ne[0]) * n_rows; + const uint64_t mx = ggml_row_size(GGML_TYPE_MXFP8, ti.t.ne[0]) * n_rows; + const uint64_t pad_f8 = GGML_PAD(f8, ctx->alignment); + const uint64_t pad_mx = GGML_PAD(mx, ctx->alignment); + const auto next = std::upper_bound(offs.begin(), offs.end(), ti.offset); + bool fits_mx; + bool fits_f8; + if (next != offs.end()) { + const uint64_t span = *next - ti.offset; + fits_mx = span == pad_mx; + fits_f8 = span == pad_f8; + } else { + const uint64_t span = data_end > ti.offset ? data_end - ti.offset : 0; + fits_mx = span >= mx && span <= pad_mx; + fits_f8 = span >= f8 && span <= pad_f8; + } + fit[i] = fits_mx && fits_f8 ? FIT_BOTH : fits_mx ? FIT_MX : fits_f8 ? FIT_F8 : FIT_NONE; + old_ids = old_ids || fit[i] == FIT_MX; + new_ids = new_ids || fit[i] == FIT_F8; + } + const int64_t ftype_idx = gguf_find_key(ctx, "general.file_type"); + if (ftype_idx != -1 && gguf_get_kv_type(ctx, ftype_idx) == GGUF_TYPE_UINT32 && + gguf_get_val_u32(ctx, ftype_idx) == 42) { // LLAMA_FTYPE_MOSTLY_F8_E4M3 + new_ids = true; + } + for (size_t i = 0; i < ctx->info.size(); ++i) { + gguf_tensor_info & ti = ctx->info[i]; + if (fit[i] == FIT_BOTH) { + if (old_ids == new_ids) { + GGML_LOG_ERROR("%s: tensor '%s' has type 43 and its size fits both MXFP8 (type 43 in files this fork " + "wrote before MXFP8 moved to type 63) and F8_E4M3 at alignment %zu, and nothing else in the file " + "tells which one it is; convert the model again\n", __func__, ti.t.name, ctx->alignment); + gguf_free(ctx); + return nullptr; + } + fit[i] = old_ids ? FIT_MX : FIT_F8; + } + if (fit[i] != FIT_MX) { + continue; + } + ti.t.type = GGML_TYPE_MXFP8; + ti.t.nb[0] = ggml_type_size(GGML_TYPE_MXFP8); + ti.t.nb[1] = ti.t.nb[0]*(ti.t.ne[0]/ggml_blck_size(GGML_TYPE_MXFP8)); + for (int j = 2; j < GGML_MAX_DIMS; ++j) { + ti.t.nb[j] = ti.t.nb[j - 1]*ti.t.ne[j - 1]; + } + } + } + // compute the total size of the data section, taking into account the alignment { ctx->size = 0; diff --git a/gguf-py/gguf/constants.py b/gguf-py/gguf/constants.py index 4d057295c128..0d1e00da9903 100644 --- a/gguf-py/gguf/constants.py +++ b/gguf-py/gguf/constants.py @@ -6559,6 +6559,11 @@ class GGMLQuantizationType(IntEnum): NVFP4 = 40 Q1_0 = 41 Q2_0 = 42 + MXFP8 = 63 # fork-local; files written while it was 43 are read by size (GGUFReader) + # 42..63 reserved for upstream ggml (42 is Q2_0); fork-local types start at 64 + IQ1_XS = 64 + IQ1_XXS = 65 + IQ1_XXXS = 66 F8_E4M3 = 43 @@ -6616,6 +6621,10 @@ class LlamaFileType(IntEnum): MOSTLY_NVFP4 = 39 # except 1d tensors MOSTLY_Q1_0 = 40 # except 1d tensors MOSTLY_Q2_0 = 41 # except 1d tensors + # 41..63 reserved for upstream llama.cpp (41 is Q2_0); fork-local ftypes start at 64 + MOSTLY_IQ1_XS = 64 # except 1d tensors + MOSTLY_IQ1_XXS = 65 # except 1d tensors + MOSTLY_IQ1_XXXS = 66 # except 1d tensors MOSTLY_F8_E4M3 = 42 # except 1d tensors GUESSED = 1024 # not specified in the model file @@ -6781,6 +6790,9 @@ class VisionProjectorType: GGMLQuantizationType.I64: (1, 8), GGMLQuantizationType.F64: (1, 8), GGMLQuantizationType.IQ1_M: (256, QK_K // 8 + QK_K // 16 + QK_K // 32), + GGMLQuantizationType.IQ1_XS: (256, 2 + QK_K // 8 + QK_K // 32 + QK_K // 64), + GGMLQuantizationType.IQ1_XXS: (256, 2 + QK_K // 8 + QK_K // 32), + GGMLQuantizationType.IQ1_XXXS: (256, 2 + QK_K // 8 + QK_K // 64), GGMLQuantizationType.BF16: (1, 2), GGMLQuantizationType.TQ1_0: (256, 2 + 4 * 13), GGMLQuantizationType.TQ2_0: (256, 2 + 64), @@ -6788,6 +6800,7 @@ class VisionProjectorType: GGMLQuantizationType.NVFP4: (64, 4 + 32), GGMLQuantizationType.Q1_0: (128, 2 + 16), GGMLQuantizationType.Q2_0: (64, 2 + 16), + GGMLQuantizationType.MXFP8: (32, 1 + 32), GGMLQuantizationType.F8_E4M3: (1, 1), } diff --git a/gguf-py/gguf/gguf_reader.py b/gguf-py/gguf/gguf_reader.py index bf3e083800b0..67b5ae2ee668 100644 --- a/gguf-py/gguf/gguf_reader.py +++ b/gguf-py/gguf/gguf_reader.py @@ -30,6 +30,7 @@ GGMLQuantizationType, GGUFValueType, GGUFEndian, + LlamaFileType, ) # limits mirroring ggml/src/gguf.cpp (not part of gguf.h) @@ -332,10 +333,63 @@ def _build_tensor_info(self, offs: int, count: int) -> tuple[int, list[ReaderFie tensor_fields.append(field) return offs, tensor_fields + def _legacy_mxfp8(self, start_offs: int, fields: list[ReaderField]) -> set[int]: + # MXFP8 was type 43 in this fork until upstream took 43 for F8_E4M3, so files written before that store their + # MXFP8 tensors as 43. Tell them apart by the bytes a type-43 tensor occupies in the data section: the span to + # the next tensor in data order (or to the end of the file for the last one) fits MXFP8 (33 bytes per 32 + # values) or F8_E4M3 (1 byte per value). Small tensors at an alignment above 32 can pad to the same span either + # way; the rest of the file decides those: a type-43 tensor that fits only one of the two, a type-63 (MXFP8) + # tensor or general.file_type MOSTLY_F8_E4M3. With nothing to go on, the file is refused instead of guessed. + # Returns the indices of the tensors to read as MXFP8. + align = int(self.alignment) + offs = sorted({int(field.parts[5][0]) for field in fields}) + next_offs = dict(zip(offs, offs[1:] + [None])) + data_size = len(self.data) - start_offs + fits: dict[int, str | None] = {} + for i_field, field in enumerate(fields): + dims, raw_dtype, offset_tensor = field.parts[3], field.parts[4], field.parts[5] + if int(raw_dtype[0]) != GGMLQuantizationType.F8_E4M3: + continue + if int(dims[0]) % 32 != 0: + fits[i_field] = 'f8' + continue + n_f8 = 1 + for dim in dims.tolist(): + n_f8 *= int(dim) + n_mx = n_f8 // 32 * 33 + pad_f8, pad_mx = -(-n_f8 // align) * align, -(-n_mx // align) * align + off = int(offset_tensor[0]) + if next_offs[off] is not None: + span = next_offs[off] - off + fits_mx, fits_f8 = span == pad_mx, span == pad_f8 + else: + span = data_size - off + fits_mx, fits_f8 = n_mx <= span <= pad_mx, n_f8 <= span <= pad_f8 + fits[i_field] = 'both' if fits_mx and fits_f8 else 'mx' if fits_mx else 'f8' if fits_f8 else None + file_type = self.fields.get('general.file_type') + new_ids = ('f8' in fits.values() + or any(int(field.parts[4][0]) == GGMLQuantizationType.MXFP8 for field in fields) + or (file_type is not None and int(file_type.parts[-1][0]) == LlamaFileType.MOSTLY_F8_E4M3)) + old_ids = 'mx' in fits.values() + legacy = set() + for i_field, fit in fits.items(): + if fit == 'both': + if old_ids == new_ids: + name = str(bytes(fields[i_field].parts[1]), encoding = 'utf-8') + raise ValueError( + f'Tensor {name} has type 43 and its size fits both MXFP8 (type 43 in files this fork wrote ' + f'before MXFP8 moved to type 63) and F8_E4M3 at alignment {align}, and nothing else in the ' + 'file tells which one it is; convert the model again') + fit = 'mx' if old_ids else 'f8' + if fit == 'mx': + legacy.add(i_field) + return legacy + def _build_tensors(self, start_offs: int, fields: list[ReaderField]) -> None: tensors = [] tensor_names = set() # keep track of name to prevent duplicated tensors - for field in fields: + legacy_mxfp8 = self._legacy_mxfp8(start_offs, fields) + for i_field, field in enumerate(fields): _name_len, name_data, _n_dims, dims, raw_dtype, offset_tensor = field.parts # check if there's any tensor having same name already in the list tensor_name = str(bytes(name_data), encoding = 'utf-8') @@ -343,6 +397,8 @@ def _build_tensors(self, start_offs: int, fields: list[ReaderField]) -> None: raise ValueError(f'Found duplicated tensor with name {tensor_name}') tensor_names.add(tensor_name) ggml_type = GGMLQuantizationType(raw_dtype[0]) + if i_field in legacy_mxfp8: + ggml_type = GGMLQuantizationType.MXFP8 # use Python ints: np.prod on uint64 wraps silently on overflow n_elems = 1 for dim in dims.tolist(): diff --git a/gguf-py/gguf/quants.py b/gguf-py/gguf/quants.py index c7f97e512252..d0aaade2d3ab 100644 --- a/gguf-py/gguf/quants.py +++ b/gguf-py/gguf/quants.py @@ -759,6 +759,66 @@ def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: return (d * qs.astype(np.float32)) +class MXFP8(__Quant, qtype=GGMLQuantizationType.MXFP8): + # 32 E4M3 (fn) codes sharing one E8M0 scale: value = e4m3(q) * 2^(e - 127) + # ref: https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf + + @staticmethod + def e4m3_table() -> np.ndarray: + x = np.arange(256, dtype=np.uint32) + exp = ((x >> 3) & 0xF).astype(np.int32) + man = (x & 0x7).astype(np.float64) + v = np.where(exp == 0, man * 2.0**-9, (8.0 + man) * np.exp2(exp - 10.0)) + v = np.where((x & 0x7F) == 0x7F, np.nan, v) + return np.where(x & 0x80, -v, v).astype(np.float32) + + @staticmethod + def e8m0_to_fp32(x: np.ndarray) -> np.ndarray: + # see ggml_e8m0_to_fp32 in ggml-impl.h (0xFF is NaN) + x = x.astype(np.uint32) + bits = np.where(x == 0, np.uint32(0x00400000), x << np.uint32(23)) + return np.where(x == 0xFF, np.float32(np.nan), bits.view(np.float32)) + + @staticmethod + def fp32_to_e4m3(x: np.ndarray) -> np.ndarray: + # see ggml_fp32_to_e4m3 in ggml-impl.h (round to nearest even, saturating) + sign = np.where(np.signbit(x), np.uint8(0x80), np.uint8(0)) + a = np.abs(x).astype(np.float32) + sub = np.rint(a * np.float32(512)).astype(np.int32) + f, e = np.frexp(a) + m = np.rint((f * 2 - 1) * 8).astype(np.int32) + E = e - 1 + 7 + E = np.where(m == 8, E + 1, E) + m = np.where(m == 8, 0, m) + norm = (E << 3) | m + norm = np.where((E > 15) | ((E == 15) & (m == 7)), 0x7E, norm) + code = np.where(a < 2.0**-6, sub, norm) + code = np.where(a >= 448.0, 0x7E, code) + code = np.where(np.isnan(x), 0x7F, code | sign) + return code.astype(np.uint8) + + @classmethod + def quantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + amax = abs(blocks).max(axis=-1, keepdims=True) + + with np.errstate(divide="ignore"): + e = np.where(amax > 0, np.ceil(np.log2(amax / np.float32(448))) + 127, 0) + e = np.clip(e, 0, 254).astype(np.uint8) + + d = cls.e8m0_to_fp32(e) + qs = cls.fp32_to_e4m3(blocks * (np.float32(1) / d)) + + return np.concatenate([e, qs], axis=-1) + + @classmethod + def dequantize_blocks(cls, blocks: np.ndarray) -> np.ndarray: + e, qs = np.hsplit(blocks, [1]) + + d = cls.e8m0_to_fp32(e) + + return d * cls.e4m3_table()[qs] + + class NVFP4(__Quant, qtype=GGMLQuantizationType.NVFP4): # E2M1 values doubled (kvalues_mxfp4 convention) kvalues = (0, 1, 2, 3, 4, 6, 8, 12, 0, -1, -2, -3, -4, -6, -8, -12) diff --git a/include/llama.h b/include/llama.h index 3877b69e2d1e..73c3d1d5cb8c 100644 --- a/include/llama.h +++ b/include/llama.h @@ -158,6 +158,11 @@ extern "C" { LLAMA_FTYPE_MOSTLY_NVFP4 = 39, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q1_0 = 40, // except 1d tensors LLAMA_FTYPE_MOSTLY_Q2_0 = 41, // except 1d tensors + // 41..63 reserved for upstream (41 is Q2_0); this is written as general.file_type, + // so a collision would make a stock build report the wrong quantization. + LLAMA_FTYPE_MOSTLY_IQ1_XS = 64, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_XXS = 65, // except 1d tensors + LLAMA_FTYPE_MOSTLY_IQ1_XXXS = 66, // except 1d tensors LLAMA_FTYPE_MOSTLY_F8_E4M3 = 42, // except 1d tensors LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file diff --git a/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja b/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja new file mode 100644 index 000000000000..c597891ba11e --- /dev/null +++ b/models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja @@ -0,0 +1,275 @@ +{#- DeepSeek-V4.1 chat template (for DeepSeek-V4.1-Flash-1017), ported from encoding/encoding.py in the model repo. + Based on the deepseek-ai/DeepSeek-V4.1-Flash template, with fixes so it matches encoding.py: + tool-result-only user turns do not start a new turn, tool_result content lists keep "[Unsupported x]" blocks, + top-level response_format is attached to the leading system message. + Extra kwargs: thinking_mode ("thinking" | "chat", default "thinking"; enable_thinking / thinking = true/false are + accepted as aliases), drop_thinking (default true), reasoning_effort (int in [1, 100] or "low"/"medium"/"high"/"max", + default "high"). Assistant reasoning is read from reasoning_content, or from thinking as an alias. + Tool call arguments must be objects (string arguments are rendered as a single "arguments" parameter). -#} +{%- set IMG = "<｜deepseek_image｜>" -%} +{%- set DSML = "｜DSML｜" -%} + +{%- if thinking_mode is not defined or thinking_mode is none -%} + {%- if enable_thinking is defined and enable_thinking is not none -%} + {%- set thinking_mode = "thinking" if enable_thinking else "chat" -%} + {%- elif thinking is defined and thinking is not none and thinking is not string -%} + {%- set thinking_mode = "thinking" if thinking else "chat" -%} + {%- else -%} + {%- set thinking_mode = "thinking" -%} + {%- endif -%} +{%- endif -%} +{%- if thinking_mode not in ["chat", "thinking"] -%} + {{- raise_exception("Invalid thinking_mode `" ~ thinking_mode ~ "`, should be \"chat\" or \"thinking\"") -}} +{%- endif -%} +{%- if drop_thinking is not defined or drop_thinking is none -%}{%- set drop_thinking = true -%}{%- endif -%} +{%- if add_generation_prompt is not defined -%}{%- set add_generation_prompt = false -%}{%- endif -%} +{%- if tools is not defined -%}{%- set tools = none -%}{%- endif -%} +{%- if response_format is not defined -%}{%- set response_format = none -%}{%- endif -%} + +{%- if reasoning_effort is not defined or reasoning_effort is none -%}{%- set reasoning_effort = "high" -%}{%- endif -%} +{%- set effort_map = {"low": 50, "medium": 62, "high": 75, "max": 100} -%} +{%- if reasoning_effort is string and reasoning_effort in effort_map -%} + {%- set budget = effort_map[reasoning_effort] -%} +{%- elif reasoning_effort is integer and reasoning_effort >= 1 and reasoning_effort <= 100 -%} + {%- set budget = reasoning_effort -%} +{%- else -%} + {{- raise_exception("Invalid reasoning effort: " ~ reasoning_effort ~ ", should be int within [1,100] or one of \"low\", \"medium\", \"high\", \"max\"") -}} +{%- endif -%} + +{#- Splits/validates "ns::name" against an explicit namespace and returns the qualified name. -#} +{%- macro qualified_name(name, ns) -%} + {%- if ns is mapping -%}{%- set ns = ns.name -%}{%- endif -%} + {%- set parts = name.split("::", 1) -%} + {%- if parts | length == 2 -%} + {%- if ns is not none and ns != parts[0] -%}{{- raise_exception("Conflicting tool namespaces: " ~ ns ~ " != " ~ parts[0]) -}}{%- endif -%} + {%- set ns = parts[0] -%} + {%- set name = parts[1] -%} + {%- endif -%} + {%- if "::" in name -%}{{- raise_exception("Tool name must not contain '::': " ~ name) -}}{%- endif -%} + {%- if ns is not none and "::" in ns -%}{{- raise_exception("Tool namespace must not contain '::': " ~ ns) -}}{%- endif -%} + {{- name if ns is none else ns ~ "::" ~ name -}} +{%- endmacro -%} + +{%- macro render_tools(tool_list) -%} + {%- set ts = namespace(schemas=[]) -%} + {%- for tool in tool_list -%} + {%- set fn = tool.function if tool.function is defined else tool -%} + {%- set ns = tool.namespace if tool.namespace is defined and tool.namespace is not none else fn.get("namespace") -%} + {#- JSON assembled by hand (key order kept, name/description replaced in place): llama.cpp has no ** unpacking. -#} + {%- set qname = qualified_name(fn.name, ns) -%} + {%- set sch = namespace(parts=[]) -%} + {%- for k, v in fn.items() if k != "namespace" -%} + {%- if k == "name" -%}{%- set v = qname -%} + {%- elif k == "description" and ns is mapping and ns.description -%}{%- set v = ns.description ~ "\n" ~ (fn.description or "") -%} + {%- endif -%} + {%- set sch.parts = sch.parts + [(k | tojson) ~ ": " ~ (v | tojson)] -%} + {%- endfor -%} + {%- if "description" not in fn and ns is mapping and ns.description -%} + {%- set sch.parts = sch.parts + ['"description": ' ~ ((ns.description ~ "\n") | tojson)] -%} + {%- endif -%} + {%- set ts.schemas = ts.schemas + ["{" ~ sch.parts | join(", ") ~ "}"] -%} + {%- endfor -%} + {{- "## Tools\n\nYou have access to a set of tools to help answer the user's question. You can invoke tools by writing a \"<｜DSML｜ calls>\" block like the following:\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"$TOOL_NAME\">\n<｜DSML｜ parameter name=\"$PARAMETER_NAME\" string=\"true|false\">$PARAMETER_VALUE\n...\n\n<｜DSML｜ invoke name=\"$TOOL_NAME2\">\n...\n\n\n\nString parameters should be specified as is and set `string=\"true\"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string=\"false\"`.\n\nIf thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response.\n\nOtherwise, output directly after with tool calls or final response.\n\n### Available Tool Schemas\n\n" -}} + {{- ts.schemas | join("\n") -}} + {{- "\n\nYou MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls.\n" -}} +{%- endmacro -%} + +{#- 1. Merge tool results (and consecutive user turns) into user messages made of content blocks. -#} +{%- set mns = namespace(msgs=[]) -%} +{%- for msg in messages -%} + {%- set content = msg.get("content") -%} + {%- set is_blocks = content is iterable and content is not string and content is not mapping -%} + {%- if content is string and IMG in content -%} + {{- raise_exception("Message content contains image special token " ~ IMG ~ ". Images should be provided as image content blocks.") -}} + {%- endif -%} + {%- set rc = msg.reasoning_content if msg.get("reasoning_content") is not none else msg.get("thinking") -%} + {%- if rc is string and IMG in rc -%} + {{- raise_exception("reasoning_content contains image special token " ~ IMG) -}} + {%- endif -%} + {%- set prev = mns.msgs[-1] if mns.msgs else none -%} + {%- if msg.role == "tool" -%} + {%- set tblock = {"type": "tool_result", "tool_use_id": msg.get("tool_call_id", ""), "content": content if content is not none else ""} -%} + {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined -%} + {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + [tblock])] -%} + {%- else -%} + {%- set mns.msgs = mns.msgs + [{"role": "user", "content_blocks": [tblock]}] -%} + {%- endif -%} + {%- elif msg.role == "user" -%} + {%- set blocks = content if is_blocks else [{"type": "text", "text": content or ""}] -%} + {%- if prev is not none and prev.role == "user" and prev.content_blocks is defined and prev.get("task") is none -%} + {%- set mns.msgs = mns.msgs[:-1] + [dict(prev, content_blocks=prev.content_blocks + blocks)] -%} + {%- else -%} + {%- set mns.msgs = mns.msgs + [dict(msg, content_blocks=blocks)] -%} + {%- endif -%} + {%- else -%} + {%- set mns.msgs = mns.msgs + [msg] -%} + {%- endif -%} +{%- endfor -%} + +{#- Top-level tools / response_format attach to a leading system message, or to a new empty one. -#} +{%- if tools or response_format -%} + {%- if not (mns.msgs and mns.msgs[0].role == "system") -%} + {%- set mns.msgs = [{"role": "system", "content": ""}] + mns.msgs -%} + {%- endif -%} + {%- set first = mns.msgs[0] -%} + {%- if tools and not first.get("tools") -%}{%- set first = dict(first, tools=tools) -%}{%- endif -%} + {%- if response_format and not first.get("response_format") -%}{%- set first = dict(first, response_format=response_format) -%}{%- endif -%} + {%- set mns.msgs = [first] + mns.msgs[1:] -%} +{%- endif -%} + +{#- 2. Reorder tool results to match the tool_calls order of the preceding assistant message. -#} +{%- set sns = namespace(msgs=[], ids=[]) -%} +{%- for m in mns.msgs -%} + {%- if m.role == "assistant" and m.get("tool_calls") -%} + {%- set sns.ids = [] -%} + {%- for tc in m.tool_calls -%} + {%- set sns.ids = sns.ids + [tc.get("id") or (tc.function.get("id", "") if tc.function is defined else "")] -%} + {%- endfor -%} + {%- set sns.msgs = sns.msgs + [m] -%} + {%- elif m.role == "user" and m.content_blocks -%} + {%- set tb = namespace(blocks=[]) -%} + {%- for b in m.content_blocks if b.type == "tool_result" -%} + {%- set order = namespace(idx=0) -%} + {%- for tc_id in sns.ids -%} + {%- if tc_id and tc_id == b.get("tool_use_id", "") -%}{%- set order.idx = loop.index0 -%}{%- endif -%} + {%- endfor -%} + {%- set tb.blocks = tb.blocks + [dict(b, _order=order.idx)] -%} + {%- endfor -%} + {%- if tb.blocks | length > 1 and sns.ids -%} + {%- set sorted_blocks = tb.blocks | sort(attribute="_order") -%} + {%- set rb = namespace(blocks=[], i=0) -%} + {%- for b in m.content_blocks -%} + {%- if b.type == "tool_result" -%} + {%- set rb.blocks = rb.blocks + [sorted_blocks[rb.i]] -%} + {%- set rb.i = rb.i + 1 -%} + {%- else -%} + {%- set rb.blocks = rb.blocks + [b] -%} + {%- endif -%} + {%- endfor -%} + {%- set sns.msgs = sns.msgs + [dict(m, content_blocks=rb.blocks)] -%} + {%- else -%} + {%- set sns.msgs = sns.msgs + [m] -%} + {%- endif -%} + {%- else -%} + {%- set sns.msgs = sns.msgs + [m] -%} + {%- endif -%} +{%- endfor -%} +{%- set msgs = sns.msgs -%} + +{#- Last user turn: users and mid-conversation systems count, except user turns made only of tool results. + Tools anywhere disable drop_thinking. -#} +{%- set state = namespace(last_user=-1, has_tools=false) -%} +{%- for m in msgs -%} + {%- if m.role == "user" or (m.role == "system" and loop.index0 > 0) -%} + {%- set real = namespace(v=true) -%} + {%- if m.get("content_blocks") -%} + {%- set real.v = false -%} + {%- for b in m.content_blocks if b.type in ["text", "image", "image_url"] -%}{%- set real.v = true -%}{%- endfor -%} + {%- endif -%} + {%- if real.v -%}{%- set state.last_user = loop.index0 -%}{%- endif -%} + {%- endif -%} + {%- if m.get("tools") -%}{%- set state.has_tools = true -%}{%- endif -%} +{%- endfor -%} +{%- set effective_drop = drop_thinking and not state.has_tools -%} +{%- set is_thinking = thinking_mode == "thinking" -%} + +{{- "<｜begin▁of▁sentence｜>" -}} +{#- 3. Render each message. -#} +{%- for m in msgs -%} + {%- set idx = loop.index0 -%} + {%- set content = m.get("content") -%} + {%- if content is iterable and content is not string and content is not mapping -%} + {%- set sp = namespace(parts=[]) -%} + {%- for b in content -%} + {%- if b.type == "text" -%}{%- set sp.parts = sp.parts + [b.text or ""] -%} + {%- elif b.type in ["image", "image_url"] -%}{%- set sp.parts = sp.parts + [IMG] -%} + {%- endif -%} + {%- endfor -%} + {%- set content = sp.parts | join("\n\n") -%} + {%- endif -%} + + {%- if idx == 0 -%} + {%- if is_thinking or m.role == "system" -%}{{- "<｜System｜>" -}}{%- endif -%} + {%- if is_thinking -%}{{- "Reasoning Effort: " ~ budget ~ " (range 1-100, the higher the value, the more thorough the reasoning)\n\n" -}}{%- endif -%} + {%- endif -%} + + {%- if m.role == "system" -%} + {%- if idx > 0 -%}{{- "<｜System｜>" -}}{%- endif -%} + {{- content or "" -}} + {%- if m.get("tools") -%}{{- "\n\n" ~ render_tools(m.tools) -}}{%- endif -%} + {%- if m.get("response_format") -%} + {{- "\n\n## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n" ~ (m.response_format | tojson) -}} + {%- endif -%} + + {%- elif m.role == "user" -%} + {{- "<｜User｜>" -}} + {%- set up = namespace(parts=[]) -%} + {%- for b in m.content_blocks -%} + {%- if b.type == "text" -%} + {%- if IMG in (b.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} + {%- set up.parts = up.parts + [b.text or ""] -%} + {%- elif b.type in ["image", "image_url"] -%} + {%- set up.parts = up.parts + [IMG] -%} + {%- elif b.type == "tool_result" -%} + {%- set tc = b.get("content", "") -%} + {%- if tc is iterable and tc is not string and tc is not mapping -%} + {%- set sub = namespace(parts=[]) -%} + {%- for s in tc -%} + {%- if s.type == "text" -%} + {%- if IMG in (s.text or "") -%}{{- raise_exception("Text block contains image placeholder " ~ IMG ~ ". Images should be separate content blocks.") -}}{%- endif -%} + {%- set sub.parts = sub.parts + [s.get("text", "")] -%} + {%- elif s.type in ["image", "image_url"] -%}{%- set sub.parts = sub.parts + [IMG] -%} + {%- else -%}{%- set sub.parts = sub.parts + ["[Unsupported " ~ s.type ~ "]"] -%} + {%- endif -%} + {%- endfor -%} + {%- set tc = sub.parts | join("\n\n") -%} + {%- endif -%} + {%- set up.parts = up.parts + ["" ~ tc ~ ""] -%} + {%- else -%} + {%- set up.parts = up.parts + ["[Unsupported " ~ b.type ~ "]"] -%} + {%- endif -%} + {%- endfor -%} + {{- up.parts | join("\n\n") -}} + + {%- elif m.role == "latest_reminder" -%} + {{- "<｜latest_reminder｜>" ~ (content or "") -}} + + {%- elif m.role == "assistant" -%} + {%- set prev_has_task = idx > 0 and msgs[idx - 1].get("task") is not none -%} + {%- if is_thinking and not prev_has_task and (not effective_drop or idx > state.last_user) -%} + {{- ((m.reasoning_content if m.get("reasoning_content") is not none else m.get("thinking")) or "") ~ "" -}} + {%- endif -%} + {{- content or "" -}} + {%- if m.get("tool_calls") -%} + {%- set tcs = namespace(lines=[]) -%} + {%- for tc in m.tool_calls -%} + {%- set fn = tc.function if tc.function is defined else tc -%} + {%- set params = namespace(lines=[]) -%} + {%- set args = fn.arguments if fn.arguments is mapping else {"arguments": fn.arguments} -%} + {%- for k, v in args.items() -%} + {%- set params.lines = params.lines + ["<" ~ DSML ~ " parameter name=\"" ~ k ~ "\" string=\"" ~ ("true" if v is string else "false") ~ "\">" ~ (v if v is string else v | tojson) ~ ""] -%} + {%- endfor -%} + {%- set name = qualified_name(fn.name, tc.get("namespace") or fn.get("namespace")) -%} + {%- set tcs.lines = tcs.lines + ["<" ~ DSML ~ " invoke name=\"" ~ name ~ "\">\n" ~ params.lines | join("\n") ~ "\n"] -%} + {%- endfor -%} + {{- "\n\n<" ~ DSML ~ " calls>\n" ~ tcs.lines | join("\n") ~ "\n" -}} + {%- endif -%} + {%- if not m.get("wo_eos", false) -%}{{- "<｜end▁of▁sentence｜>" -}}{%- endif -%} + + {%- else -%} + {{- raise_exception("Unknown role: " ~ m.role) -}} + {%- endif -%} + + {#- Transition tokens, only when the next message is an assistant/latest_reminder turn or this is the last one. -#} + {%- if loop.last or msgs[idx + 1].role in ["assistant", "latest_reminder"] -%} + {%- set task = m.get("task") -%} + {%- if task is not none -%} + {%- set task_tokens = {"action": "<｜action｜>", "query": "<｜query｜>", "authority": "<｜authority｜>", "domain": "<｜domain｜>", "title": "<｜title｜>", "read_url": "<｜read_url｜>"} -%} + {%- if task not in task_tokens -%}{{- raise_exception("Invalid task: '" ~ task ~ "'") -}}{%- endif -%} + {%- if task == "action" -%}{{- "<｜Assistant｜>" ~ ("" if is_thinking else "") -}}{%- endif -%} + {{- task_tokens[task] -}} + {%- elif (m.role == "user" or (m.role == "system" and idx > 0)) and (add_generation_prompt or not loop.last) -%} + {{- "<｜Assistant｜>" ~ ("" if is_thinking and (not effective_drop or idx >= state.last_user) else "") -}} + {%- endif -%} + {%- endif -%} +{%- endfor -%} diff --git a/scripts/iq1_narrow_grids/k3_greedy1024.npy b/scripts/iq1_narrow_grids/k3_greedy1024.npy new file mode 100644 index 000000000000..8f0883d2a9d4 Binary files /dev/null and b/scripts/iq1_narrow_grids/k3_greedy1024.npy differ diff --git a/scripts/iq1_narrow_grids/k3_greedy256.npy b/scripts/iq1_narrow_grids/k3_greedy256.npy new file mode 100644 index 000000000000..5af65b466a00 Binary files /dev/null and b/scripts/iq1_narrow_grids/k3_greedy256.npy differ diff --git a/scripts/iq1_narrow_grids/k3_greedy512.npy b/scripts/iq1_narrow_grids/k3_greedy512.npy new file mode 100644 index 000000000000..6ffddb3da3ec Binary files /dev/null and b/scripts/iq1_narrow_grids/k3_greedy512.npy differ diff --git a/scripts/iqxs_gen_tables.py b/scripts/iqxs_gen_tables.py new file mode 100644 index 000000000000..6d8c6679cd13 --- /dev/null +++ b/scripts/iqxs_gen_tables.py @@ -0,0 +1,100 @@ +"""Emit the ggml C tables for IQ1_XS / IQ1_XXS / IQ1_XXXS. + +The three narrow types reuse ggml's own 8-dimensional ternary codebook. Each new grid is a +SUBSET of `iq1s_grid`, selected by imatrix-weighted greedy facility-location on real Kimi-K3 +expert weights (`scripts/iq1_narrow_grids/k3_greedy{1024,512,256}.npy`, stored as indices into +the 2048-entry table and verified here to be subsets). + +Reusing ggml's entries rather than training fresh ones is deliberate: + + * every entry is already known to be int8-representable, which `ggml_vec_dot_iq1_s_q8_K` + requires because it reaches the grid through `_mm256_maddubs_epi16`, an int8 dot product; + * the measured quality of these exact codebooks is already on record, so the C types + reproduce numbers we have rather than new ones; + * codebook selection was measured to be a weak lever anyway: three selection criteria agree + to within 0.5% and selecting from all 6561 ternary patterns instead of ggml's 2048 moves + weighted nrmse by 0.19%. + +Both the CPU (`uint64_t`, 8 packed int8) and CUDA (`uint32_t`, 8 packed 4-bit) mirrors are +emitted, matching the `iq1s_grid` / `iq1s_grid_gpu` pair at ggml-common.h:1135 and :1650. +""" + +from __future__ import annotations + +import argparse +import os +import re +import sys + +import numpy as np + +REPO = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +GRIDS = os.path.join(REPO, "scripts", "iq1_narrow_grids") + +# name -> (npy of indices into iq1s_grid, entries, index bits, block bytes, bpw) +TYPES = { + "IQ1_XS": ("k3_greedy1024.npy", 1024, 10, 46, 1.4375), + "IQ1_XXS": ("k3_greedy512.npy", 512, 9, 42, 1.3125), + "IQ1_XXXS": ("k3_greedy256.npy", 256, 8, 38, 1.1875), +} + + +def read_ggml_table(path: str, name: str, ctype: str) -> list[int]: + """Parse GGML_TABLE_BEGIN(ctype, name, N) ... GGML_TABLE_END() out of ggml-common.h.""" + src = open(path).read() + m = re.search(r"GGML_TABLE_BEGIN\(\s*%s\s*,\s*%s\s*,\s*[A-Z0-9_]+\s*\)(.*?)GGML_TABLE_END\(\)" + % (re.escape(ctype), re.escape(name)), src, re.S) + if not m: + raise SystemExit(f"{name} ({ctype}) not found in {path}") + body = m.group(1) + vals = [int(t, 16) for t in re.findall(r"0x([0-9a-fA-F]+)", body)] + return vals + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--common", default=os.path.join(REPO, "ggml", "src", "ggml-common.h")) + ap.add_argument("--out", required=True, help="C fragment to write") + a = ap.parse_args() + + cpu = read_ggml_table(a.common, "iq1s_grid", "uint64_t") + gpu = read_ggml_table(a.common, "iq1s_grid_gpu", "uint32_t") + if len(cpu) != 2048 or len(gpu) != 2048: + raise SystemExit(f"expected 2048 entries, got cpu={len(cpu)} gpu={len(gpu)}") + print(f"read iq1s_grid: {len(cpu)} uint64, iq1s_grid_gpu: {len(gpu)} uint32") + + out = ["// Generated by scripts/iqxs_gen_tables.py. Do not edit by hand.", + "// Subsets of iq1s_grid chosen by imatrix-weighted greedy facility-location on real", + "// Kimi-K3 expert weights; every entry is an existing ggml grid point, so the int8", + "// representability ggml_vec_dot_iq1_s_q8_K relies on is inherited rather than assumed.", + ""] + for tname, (npy, n, bits, blk, bpw) in TYPES.items(): + idx = np.load(os.path.join(GRIDS, npy)) + if len(idx) != n: + raise SystemExit(f"{npy}: expected {n} entries, got {len(idx)}") + if idx.min() < 0 or idx.max() > 2047: + raise SystemExit(f"{npy}: index out of range [{idx.min()},{idx.max()}]") + if len(set(idx.tolist())) != n: + raise SystemExit(f"{npy}: duplicate indices") + low = tname.lower() + macro = "NGRID_" + tname.replace("IQ1_", "IQ1") + out.append(f"#define {macro} {n}") + for arr, ct, suffix, per in ((cpu, "uint64_t", "", 4), (gpu, "uint32_t", "_gpu", 8)): + out.append(f"GGML_TABLE_BEGIN({ct}, {low}_grid{suffix}, {macro})") + vals = [arr[int(i)] for i in idx] + w = 16 if ct == "uint64_t" else 8 + for i in range(0, n, per): + row = ", ".join(f"0x{v:0{w}x}" for v in vals[i:i + per]) + out.append(" " + row + ",") + out.append("GGML_TABLE_END()") + out.append("") + print(f"{tname:9s} {n:5d} entries {bits} bit index {blk} B/block {bpw} bpw") + + os.makedirs(os.path.dirname(a.out) or ".", exist_ok=True) + open(a.out, "w").write("\n".join(out) + "\n") + print(f"wrote {a.out} ({len(out)} lines)") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/llama-context.cpp b/src/llama-context.cpp index 0cd795a5690b..5c6fbca39c4a 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -4963,3 +4963,7 @@ llama_memory_breakdown llama_get_memory_breakdown(const struct llama_context * c llama_context * llama_get_ctx_other(struct llama_context * ctx) { return ctx->get_cparams().ctx_other; } + +void llama_sched_reserve(struct llama_context * ctx) { + ctx->sched_reserve(); +} diff --git a/src/llama-ext.h b/src/llama-ext.h index 23a781404bd8..b9c7ad9c46bd 100644 --- a/src/llama-ext.h +++ b/src/llama-ext.h @@ -179,6 +179,10 @@ LLAMA_API int32_t llama_batch_ext_add_embd_row(struct llama_batch_ext * batch, l LLAMA_API llama_context * llama_get_ctx_other(struct llama_context * ctx); +// Run a pending scheduler reserve now instead of on the next decode, e.g. so that the memory breakdown of an +// allocation-free context reflects flags that change the graph shape +LLAMA_API void llama_sched_reserve(struct llama_context * ctx); + // // model/context data extraction // diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp index fe8bdebe5e2c..7ae6b7b7eb3b 100644 --- a/src/llama-graph.cpp +++ b/src/llama-graph.cpp @@ -17,11 +17,13 @@ #include "llama-memory-hybrid-iswa.h" #include "llama-memory-recurrent.h" +#include #include #include #include #include #include +#include #include #include @@ -131,8 +133,62 @@ static llm_attn_mask build_attn_inp_attn_mask( // impl +bool llm_graph_host_rows_supported(const ggml_tensor * tab) { + if (!tab || !tab->buffer || !tab->data || !ggml_backend_buffer_is_host(tab->buffer) || tab->view_src) { + return false; + } + if (tab->nb[0] != ggml_type_size(tab->type) || tab->ne[2] != 1 || tab->ne[3] != 1) { + return false; + } + return tab->type == GGML_TYPE_F32 || ggml_get_type_traits(tab->type)->to_float != nullptr; +} + +void llm_graph_host_get_rows(const ggml_tensor * tab, const int32_t * ids, int64_t n_ids, float * dst) { + const int64_t nc = tab->ne[0]; + const ggml_to_float_t to_float = tab->type == GGML_TYPE_F32 ? nullptr : ggml_get_type_traits(tab->type)->to_float; + + auto run = [&](int64_t i0, int64_t i1) { + for (int64_t i = i0; i < i1; ++i) { + const int64_t r = ids[i]; + GGML_ASSERT(r >= 0 && r < tab->ne[1]); + const char * src = (const char *) tab->data + r*tab->nb[1]; + if (to_float) { + to_float(src, dst + i*nc, nc); + } else { + memcpy(dst + i*nc, src, nc*sizeof(float)); + } + } + }; + + // split prompt-sized gathers over a few threads (the CPU GET_ROWS they replace is multi-threaded) + const int64_t n_elem = n_ids*nc; + const int n_thr = (int) std::min(8, std::max(1, n_elem >> 18)); + if (n_thr <= 1) { + run(0, n_ids); + return; + } + std::vector workers; + const int64_t per = (n_ids + n_thr - 1)/n_thr; + for (int t = 1; t < n_thr; ++t) { + const int64_t i0 = std::min(n_ids, t*per); + const int64_t i1 = std::min(n_ids, i0 + per); + workers.emplace_back(run, i0, i1); + } + run(0, std::min(n_ids, per)); + for (auto & w : workers) { + w.join(); + } +} + void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { - if (ubatch->token) { + if (ubatch->token && tok_host) { + GGML_ASSERT(!ubatch->is_mixed()); + const int64_t n_tokens = ubatch->n_tokens; + + tok_rows.resize(n_tokens*n_embd); + llm_graph_host_get_rows(tok_host, ubatch->token, n_tokens, tok_rows.data()); + ggml_backend_tensor_set(embd, tok_rows.data(), 0, tok_rows.size()*sizeof(float)); + } else if (ubatch->token) { const int64_t n_tokens = ubatch->n_tokens; ggml_backend_tensor_set(tokens, ubatch->token, 0, n_tokens*ggml_element_size(tokens)); @@ -144,7 +200,7 @@ void llm_graph_input_embd::set_input(const llama_ubatch * ubatch) { ggml_backend_tensor_set(embd_rows, ubatch->embd_row, 0, n_tokens*ggml_element_size(embd_rows)); } - if (ubatch->embd && embd && !ubatch->is_mixed()) { + if (ubatch->embd && embd && !ubatch->is_mixed() && !(tok_host && ubatch->token)) { GGML_ASSERT(n_embd == embd->ne[0]); const int64_t n_tokens = ubatch->n_tokens; @@ -2356,7 +2412,8 @@ ggml_tensor * llm_graph_context::build_moe_ffn( ggml_tensor * weights_sum = ggml_sum_rows(ctx0, weights); // [1, n_tokens] cb(weights_sum, "ffn_moe_weights_sum", il); - if (arch == LLM_ARCH_DEEPSEEK41) { + if (arch == LLM_ARCH_DEEPSEEK41 || + (arch == LLM_ARCH_DFLASH && hparams.dsv4_hc_single_pass)) { // model.py Gate.forward: weights / (sum + 1e-20); the sum can fall below the F16 clamp below weights_sum = ggml_scale_bias(ctx0, weights_sum, 1.0f, 1e-20f); } else { @@ -2649,7 +2706,7 @@ ggml_tensor * llm_graph_context::build_moe_cache_slots( } // input embeddings with optional lora -ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float tok_scale) const { +ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float tok_scale, bool host_gather) const { const int64_t n_embd_inp = hparams.n_embd_inp(); const int64_t n_embd = hparams.n_embd; @@ -2663,6 +2720,12 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to const int64_t n_tok_rows = has_mixed ? llm_graph_n_tok_rows(ubatch) : 0; + bool tok_host = host_gather && !has_mixed && n_embd_inp == n_embd && llm_graph_host_rows_supported(tok_embd) && + tok_embd->ne[0] == n_embd && !(hdmd && hdmd->inv.count(tok_embd)); + for (const auto & lora : *loras) { + tok_host = tok_host && lora.first->get_weight(tok_embd) == nullptr; + } + // we have 3 standard paths to produce the input embeddings for the first layer: // - embd0: extract from the token embeddings weight (`tok_embd`) using the input token ids // - embd1: pass raw embeddings, skipping the `tok_embd` @@ -2692,9 +2755,14 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to // the embeddings placeholders for the 3 paths // we use ggml_build_forward_order to make the GET_ROWS ops stick at the beginning of the compute graph // this way the embeddings remain in the host buffer, and the GET_ROWS run before any other computations - { + if (tok_host) { + inp->tok_host = tok_embd; + embd0 = inp->embd; + } else { embd0 = ggml_get_rows(ctx0, tok_embd, inp->tokens); ggml_build_forward_order(gf, embd0); + } + { embd1 = inp->embd; @@ -2748,7 +2816,7 @@ ggml_tensor * llm_graph_context::build_inp_embd(ggml_tensor * tok_embd, float to std::array inps = {}; // token embeddings path (ubatch.token != nullptr) - inps[0] = build_tok(embd0, inp->tokens); + inps[0] = tok_host ? embd0 : build_tok(embd0, inp->tokens); // vector embeddings path (ubatch.embd != nullptr) inps[1] = embd1; diff --git a/src/llama-graph.h b/src/llama-graph.h index 37b47acbb30a..9d1ef959e3d0 100644 --- a/src/llama-graph.h +++ b/src/llama-graph.h @@ -132,6 +132,10 @@ class llm_graph_input_i { using llm_graph_input_ptr = std::unique_ptr; +// dst[i] = row ids[i] of host table tab as F32, same conversion as CPU GET_ROWS (bitwise equal) +bool llm_graph_host_rows_supported(const ggml_tensor * tab); +void llm_graph_host_get_rows(const ggml_tensor * tab, const int32_t * ids, int64_t n_ids, float * dst); + class llm_graph_input_embd : public llm_graph_input_i { public: llm_graph_input_embd(int64_t n_embd) : n_embd(n_embd) {} @@ -149,6 +153,10 @@ class llm_graph_input_embd : public llm_graph_input_i { ggml_tensor * scale_rows = nullptr; // F32 [1, n_batch], per-row scale: scale_tok for token rows, 1 for embd rows ggml_tensor * embd_rows = nullptr; // I32 [n_batch], embd row path: rows of ctx_other's layer input device buffer + // host-gather mode: token rows are dequantized into `embd` in set_input (no CPU split before the first device) + const ggml_tensor * tok_host = nullptr; + std::vector tok_rows; + float scale_tok = 1.0f; const int64_t n_embd = 0; @@ -1245,7 +1253,9 @@ struct llm_graph_context { // // tok_scale: applied to token rows only - ggml_tensor * build_inp_embd(ggml_tensor * tok_embd, float tok_scale = 1.0f) const; + // host_gather: gather tok_embd rows on the host in set_input when it lives in host memory (bitwise equal to + // GET_ROWS); silently falls back to GET_ROWS when it cannot apply + ggml_tensor * build_inp_embd(ggml_tensor * tok_embd, float tok_scale = 1.0f, bool host_gather = false) const; ggml_tensor * build_inp_pos() const; ggml_tensor * build_inp_attn_scale() const; ggml_tensor * build_inp_out_ids() const; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 485483ba1f27..97ec44340fac 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -67,6 +67,9 @@ const char * llama_ftype_name(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = LLAMA_FTYPE_PREFIX "IQ3_XS - 3.3 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = LLAMA_FTYPE_PREFIX "IQ3_XXS - 3.0625 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ1_S: name = LLAMA_FTYPE_PREFIX "IQ1_S - 1.5625 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_XS: name = LLAMA_FTYPE_PREFIX "IQ1_XS - 1.4375 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_XXS: name = LLAMA_FTYPE_PREFIX "IQ1_XXS - 1.3125 bpw"; break; + case LLAMA_FTYPE_MOSTLY_IQ1_XXXS: name = LLAMA_FTYPE_PREFIX "IQ1_XXXS - 1.1875 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ1_M: name = LLAMA_FTYPE_PREFIX "IQ1_M - 1.75 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = LLAMA_FTYPE_PREFIX "IQ4_NL - 4.5 bpw"; break; case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = LLAMA_FTYPE_PREFIX "IQ4_XS - 4.25 bpw"; break; @@ -778,6 +781,9 @@ llama_model_loader::llama_model_loader( case GGML_TYPE_IQ3_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ3_XXS; break; case GGML_TYPE_IQ1_S: ftype = LLAMA_FTYPE_MOSTLY_IQ1_S; break; case GGML_TYPE_IQ1_M: ftype = LLAMA_FTYPE_MOSTLY_IQ1_M; break; + case GGML_TYPE_IQ1_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ1_XS; break; + case GGML_TYPE_IQ1_XXS: ftype = LLAMA_FTYPE_MOSTLY_IQ1_XXS; break; + case GGML_TYPE_IQ1_XXXS: ftype = LLAMA_FTYPE_MOSTLY_IQ1_XXXS; break; case GGML_TYPE_IQ4_NL: ftype = LLAMA_FTYPE_MOSTLY_IQ4_NL; break; case GGML_TYPE_IQ4_XS: ftype = LLAMA_FTYPE_MOSTLY_IQ4_XS; break; case GGML_TYPE_IQ3_S: ftype = LLAMA_FTYPE_MOSTLY_IQ3_S; break; @@ -1279,6 +1285,14 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } + // the deepseek41 Engram table (~200 GB) is a CPU get_rows gather table: never put it in device memory + if (get_arch() == LLM_ARCH_DEEPSEEK41 && tn.tensor == LLM_TENSOR_PER_LAYER_TOKEN_EMBD && !ggml_backend_buft_is_host(buft)) { + LLAMA_LOG_WARN("llama_model_loader: %s (%zu MiB %s) must stay in host memory; ignoring placement on %s, using %s\n", + tn.str().c_str(), ggml_nbytes(t_meta) / 1024 / 1024, ggml_type_name(t_meta->type), + ggml_backend_buft_name(buft), ggml_backend_buft_name(ggml_backend_cpu_buffer_type())); + buft = ggml_backend_cpu_buffer_type(); + } + // avoid using a host buffer when using mmap auto * buft_dev = ggml_backend_buft_get_device(buft); if (use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) { diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index 34c247858188..93935bf2e708 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -440,6 +440,9 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso // types on the left: block size 256 case GGML_TYPE_IQ1_S: case GGML_TYPE_IQ1_M: + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: case GGML_TYPE_IQ2_XXS: case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: @@ -480,6 +483,12 @@ static ggml_type tensor_type_fallback(quantize_state_impl & qs, const ggml_tenso return return_type; } +// the narrow-index IQ1 variants sit below IQ1_S and reuse its per-tensor type overrides unchanged, +// so that a file differs from the IQ1_S build only on the tensors that stay at the IQ1 type +static bool ftype_is_iq1_narrow(llama_ftype ftype) { + return ftype == LLAMA_FTYPE_MOSTLY_IQ1_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_XXXS; +} + // internal standard logic for selecting the target tensor type based on tensor category, ftype, and model arch static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type, const ggml_tensor * tensor, llama_ftype ftype, tensor_category category) { const std::string name = ggml_get_name(tensor); @@ -553,7 +562,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ3_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || - ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) { + ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || ftype_is_iq1_narrow(ftype)) { new_type = GGML_TYPE_Q5_K; } else if (new_type != GGML_TYPE_Q8_0) { @@ -573,7 +582,8 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type new_type = qs.params->token_embedding_type; } else { if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || - ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) { + ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || + ftype_is_iq1_narrow(ftype)) { new_type = GGML_TYPE_Q2_K; } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) { @@ -587,7 +597,8 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type } } } else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_XXS || ftype == LLAMA_FTYPE_MOSTLY_IQ2_XS || ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || - ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) { + ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || + ftype_is_iq1_narrow(ftype)) { if (category_is_attn_v(category)) { if (qs.model.hparams.n_gqa() >= 4 || qs.model.hparams.n_expert >= 4) new_type = GGML_TYPE_Q4_K; else new_type = ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M ? GGML_TYPE_IQ3_S : GGML_TYPE_Q2_K; @@ -606,7 +617,7 @@ static ggml_type llama_tensor_get_type_impl(quantize_state_impl & qs, ggml_type if (qs.model.hparams.n_expert == 8) { new_type = GGML_TYPE_Q5_K; } else { - if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M) new_type = GGML_TYPE_IQ2_XXS; + if (ftype == LLAMA_FTYPE_MOSTLY_IQ1_S || ftype == LLAMA_FTYPE_MOSTLY_IQ1_M || ftype_is_iq1_narrow(ftype)) new_type = GGML_TYPE_IQ2_XXS; else if (ftype == LLAMA_FTYPE_MOSTLY_IQ2_S || ftype == LLAMA_FTYPE_MOSTLY_IQ2_M) new_type = GGML_TYPE_IQ3_S; } } @@ -910,6 +921,11 @@ static bool tensor_requires_imatrix(const char * tensor_name, const ggml_type ds case GGML_TYPE_IQ2_S: case GGML_TYPE_IQ1_M: case GGML_TYPE_IQ1_S: + // the narrow-index IQ1 variants search the same lattice as IQ1_S, which is only defined + // against per-column importance, so there is no non-imatrix path for them either + case GGML_TYPE_IQ1_XS: + case GGML_TYPE_IQ1_XXS: + case GGML_TYPE_IQ1_XXXS: return true; case GGML_TYPE_Q2_K: // as a general rule, the k-type quantizations don't require imatrix data. @@ -960,6 +976,9 @@ ggml_type llama_ftype_get_default_type(llama_ftype ftype) { case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return GGML_TYPE_IQ3_XXS; case LLAMA_FTYPE_MOSTLY_IQ1_S: return GGML_TYPE_IQ1_S; case LLAMA_FTYPE_MOSTLY_IQ1_M: return GGML_TYPE_IQ1_M; + case LLAMA_FTYPE_MOSTLY_IQ1_XS: return GGML_TYPE_IQ1_XS; + case LLAMA_FTYPE_MOSTLY_IQ1_XXS: return GGML_TYPE_IQ1_XXS; + case LLAMA_FTYPE_MOSTLY_IQ1_XXXS: return GGML_TYPE_IQ1_XXXS; case LLAMA_FTYPE_MOSTLY_IQ4_NL: return GGML_TYPE_IQ4_NL; case LLAMA_FTYPE_MOSTLY_IQ4_XS: return GGML_TYPE_IQ4_XS; case LLAMA_FTYPE_MOSTLY_IQ3_S: diff --git a/src/models/deepseek41.cpp b/src/models/deepseek41.cpp index d8299897043b..3ae252620881 100644 --- a/src/models/deepseek41.cpp +++ b/src/models/deepseek41.cpp @@ -287,6 +287,7 @@ class llm_graph_input_dsv41 : public llm_graph_input_i { bool can_reuse(const llm_graph_params & params) override; std::vector engram_ids; + std::vector engram_tab; // non-null: host table, rows gathered in set_input ggml_tensor * engram_keep = nullptr; // 0 for embedding (image) tokens; only in mixed ubatches ggml_tensor * cand_pin = nullptr; bool cand_csa = false; @@ -315,9 +316,17 @@ void llm_graph_input_dsv41::set_input(const llama_ubatch * ubatch) { if (model.can_prefetch.count(model.per_layer_tok_embd)) { llama_prefetch_rows(model.per_layer_tok_embd, all.data(), all.size()); } + std::vector rows; for (size_t e = 0; e < engram_ids.size(); ++e) { if (engram_ids[e]->buffer) { - ggml_backend_tensor_set(engram_ids[e], ids[e].data(), 0, ids[e].size()*sizeof(int32_t)); + if (engram_tab[e]) { + // host gather: same rows and conversion as the CPU GET_ROWS + rows.resize(ids[e].size()*engram_tab[e]->ne[0]); + llm_graph_host_get_rows(engram_tab[e], ids[e].data(), (int64_t) ids[e].size(), rows.data()); + ggml_backend_tensor_set(engram_ids[e], rows.data(), 0, rows.size()*sizeof(float)); + } else { + ggml_backend_tensor_set(engram_ids[e], ids[e].data(), 0, ids[e].size()*sizeof(int32_t)); + } } } } @@ -354,7 +363,8 @@ bool llm_graph_input_dsv41::can_reuse(const llm_graph_params & params) { bool res = true; for (auto * t : engram_ids) { - res &= t->ne[0] == (int64_t) model.hparams.ple_n_heads*params.ubatch.n_tokens; + res &= t->type == GGML_TYPE_F32 ? t->ne[1] == (int64_t) params.ubatch.n_tokens + : t->ne[0] == (int64_t) model.hparams.ple_n_heads*params.ubatch.n_tokens; } res &= (engram_keep != nullptr) == (!engram_ids.empty() && params.ubatch.embd != nullptr); if (engram_keep) { @@ -777,7 +787,11 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g ggml_tensor * cur; - ggml_tensor * inp = build_inp_embd(model.tok_embd); + // LLAMA_DSV41_HOST_GATHER=0: in-graph GET_ROWS on the host tables (one CPU split each) instead of + // gathering token and Engram rows in set_input (bitwise the same rows) + static const bool host_gather = dsv41_env_on("LLAMA_DSV41_HOST_GATHER"); + + ggml_tensor * inp = build_inp_embd(model.tok_embd, 1.0f, host_gather); ggml_tensor * inp_pos = build_inp_pos(); ggml_tensor * inp_out_ids = build_inp_out_ids(); llm_graph_input_dsv4 * inp_dsv4 = build_inp_dsv4(); @@ -791,10 +805,22 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g if (!hparams.is_ple(il)) { continue; } + const ggml_tensor * tab = model.per_layer_tok_embd; + if (host_gather && llm_graph_host_rows_supported(tab) && tab->ne[0] == (int64_t) hparams.ple_head_dim) { + // host-resident table: rows are gathered in set_input + ggml_tensor * rows = ggml_new_tensor_2d(ctx0, GGML_TYPE_F32, (int64_t) hparams.ple_n_heads*tab->ne[0], n_tokens); + ggml_set_name(rows, ("engram_rows_in-" + std::to_string(il)).c_str()); + ggml_set_input(rows); + inp41->engram_ids.push_back(rows); + inp41->engram_tab.push_back(tab); + engram_rows.push_back(rows); + continue; + } ggml_tensor * ids = ggml_new_tensor_1d(ctx0, GGML_TYPE_I32, (int64_t) hparams.ple_n_heads*n_tokens); ggml_set_name(ids, ("engram_ids-" + std::to_string(il)).c_str()); ggml_set_input(ids); inp41->engram_ids.push_back(ids); + inp41->engram_tab.push_back(nullptr); ggml_tensor * rows = ggml_get_rows(ctx0, model.per_layer_tok_embd, ids); rows = ggml_reshape_2d(ctx0, rows, hparams.ple_head_dim*hparams.ple_n_heads, n_tokens); @@ -927,7 +953,7 @@ llama_model_deepseek41::graph::graph(const llama_model & model_base, const llm_g cb(cur, "result_norm", -1); res->t_embd = cur; - cur = ggml_mul_mat(ctx0, model.output, cur); + cur = build_lora_mm(model.output, cur); cb(cur, "result_output", -1); res->t_logits = cur; } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 64fdb8259a30..e5cfc15a7691 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -4240,6 +4240,64 @@ struct test_relu_sqr : public test_case { } }; +// softplus vs a double log1p(exp(x)) reference: for x < -16.6 the naive log(1 + exp(x)) returns 0 in F32 +struct test_softplus_range : public test_case { + const int64_t n; + std::vector x; + + std::string op_desc(ggml_tensor * t) override { + GGML_UNUSED(t); + return "SOFTPLUS_RANGE"; + } + + std::string vars() override { + return VARS_TO_STR1(n); + } + + test_softplus_range(int64_t n = 4001) : n(n) {} + + ggml_tensor * build_graph(ggml_context * ctx) override { + ggml_tensor * a = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, n); + ggml_set_name(a, "a"); + + ggml_tensor * out = ggml_softplus(ctx, a); + ggml_set_name(out, "out"); + + return out; + } + + void initialize_tensors(ggml_context * ctx) override { + x.resize(n); + for (int64_t i = 0; i < n; i++) { + x[i] = -100.0f + 200.0f*(float) i/(float) (n - 1); + } + ggml_tensor * a = ggml_get_tensor(ctx, "a"); + ggml_backend_tensor_set(a, x.data(), 0, ggml_nbytes(a)); + } + + // relative error vs the double reference; below FLT_MIN (x < -87.3) an absolute FLT_MIN error is allowed (FTZ) + // only the softplus output is scored, not the input or sentinels + double err(const float * a, const float * b, size_t nel) override { + if (nel != x.size() || memcmp(a, x.data(), nel*sizeof(float)) == 0) { + return 0.0; + } + double max_err = 0.0; + for (size_t i = 0; i < nel; i++) { + const double ref = std::log1p(std::exp((double) x[i])); + for (const float v : {a[i], b[i]}) { + const double d = std::fabs(v - ref); + const double e = ref < FLT_MIN ? (d <= FLT_MIN ? 0.0 : 1.0) : d/ref; + max_err = std::max(max_err, e); + } + } + return max_err; + } + + double max_nmse_err() override { + return 1e-4; + } +}; + // GGML_OP_UNARY(GELU|SILU|SIGMOID|SOFTPLUS) + GGML_OP_MUL (fused operation). struct test_unary_mul : public test_case { const ggml_unary_op op; @@ -10088,6 +10146,7 @@ static const ggml_type all_types[] = { GGML_TYPE_TQ1_0, GGML_TYPE_IQ2_XXS, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, + GGML_TYPE_IQ1_XS, GGML_TYPE_IQ1_XXS, GGML_TYPE_IQ1_XXXS, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, }; @@ -10117,6 +10176,7 @@ static const ggml_type other_types[] = { GGML_TYPE_TQ1_0, GGML_TYPE_IQ2_XS, GGML_TYPE_IQ2_S, GGML_TYPE_IQ3_XXS, GGML_TYPE_IQ1_S, GGML_TYPE_IQ1_M, + GGML_TYPE_IQ1_XS, GGML_TYPE_IQ1_XXS, GGML_TYPE_IQ1_XXXS, GGML_TYPE_IQ4_NL, GGML_TYPE_IQ3_S, GGML_TYPE_IQ4_XS, GGML_TYPE_BF16, }; @@ -10145,6 +10205,8 @@ static std::vector> make_test_cases_eval() { } } + test_cases.emplace_back(new test_softplus_range()); + // fused relu + sqr (squared ReLU) for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { test_cases.emplace_back(new test_relu_sqr(type, { 128, 2, 2, 2 })); @@ -10326,6 +10388,16 @@ static std::vector> make_test_cases_eval() { } test_cases.emplace_back(new test_get_rows(GGML_TYPE_F32, 256, 8, 2, 1, 1, false, true, 3)); + // MXFP8 is gather-only + for (int b : {1, 7}) { + for (bool v : {false, true}) { + for (bool vs0 : {false, true}) { + test_cases.emplace_back(new test_get_rows(GGML_TYPE_MXFP8, 256, 5, 4, b, 1, v, vs0)); + } + } + } + test_cases.emplace_back(new test_get_rows(GGML_TYPE_MXFP8, 256, 1000, 24, 1, 1, false)); + test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 8, 2, 1, false)); test_cases.emplace_back(new test_get_rows_back(GGML_TYPE_F32, 1, 70000, 4, 1, false)); // row count > CUDA grid-y limit (65535) for (ggml_type type : all_types) { diff --git a/tests/test-chat.cpp b/tests/test-chat.cpp index fa4525389976..98c2116470e9 100644 --- a/tests/test-chat.cpp +++ b/tests/test-chat.cpp @@ -4407,10 +4407,9 @@ static void test_template_output_peg_parsers(bool detailed_debug) { // DeepSeek V4.1: V4 behaviour, DSML tag names with a leading space { - auto tst = peg_tester("deepseek-v4.1-min", - common_chat_templates_ptr(common_chat_templates_init(nullptr, DEEPSEEK_V4_1_MIN_TEMPLATE)), - detailed_debug); + auto tst = peg_tester("models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja", detailed_debug); + // non-thinking mode: the generation prompt ends with tst.test("Hello, world!\nWhat's up?") .enable_thinking(false) .reasoning_format(COMMON_REASONING_FORMAT_DEEPSEEK) @@ -8106,6 +8105,69 @@ static void test_deepseek_v4_1_message_fields() { } } +// DeepSeek V4.1 shipped template (models/templates): extra message keys (encoding.py golden cases 3 and 4) and "\n\n" around media markers +static void test_deepseek_v4_1_shipped_template_fields() { + LOG_DBG("%s\n", __func__); + + auto tmpls = read_templates("models/templates/deepseek-ai-DeepSeek-V4.1-Flash-1017.jinja"); + auto render = [&](const char * messages_json, bool add_generation_prompt) { + common_chat_templates_inputs inputs; + inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(messages_json)); + inputs.add_generation_prompt = add_generation_prompt; + inputs.chat_template_kwargs["thinking_mode"] = "\"chat\""; + return common_chat_templates_apply(tmpls.get(), inputs).prompt.str(); + }; + + assert_equals(std::string("<｜begin▁of▁sentence｜><｜User｜>q<｜Assistant｜><｜action｜>"), + render(R"([{"role": "user", "content": "q", "task": "action"}])", false)); + + { + auto prompt = render(R"([ + {"role": "system", "content": "a"}, + {"role": "system", "content": "b", "tools": [{"type": "function", "function": {"name": "f", "parameters": {"type": "object"}}}]}, + {"role": "user", "content": "q"}, + {"role": "assistant", "content": "", "tool_calls": [{"type": "function", "namespace": "ns", "function": {"name": "f", "arguments": {"x": 1}}}]} + ])", false); + assert_contains(prompt, "<｜System｜>b\n\n## Tools"); + assert_contains(prompt, "{\"name\": \"f\", \"parameters\": {\"type\": \"object\"}}"); + assert_contains(prompt, "<｜DSML｜ invoke name=\"ns::f\">"); + } + + assert_equals(std::string("<｜begin▁of▁sentence｜><｜User｜>a\n\n<__media__>\n\nb<｜Assistant｜>"), + render(R"([{"role": "user", "content": [{"type": "text", "text": "a"}, {"type": "media_marker", "text": "<__media__>"}, {"type": "text", "text": "b"}]}])", true)); + + // OpenAI reasoning_effort values never fail + for (const auto & [effort, budget] : std::vector>{ + { "minimal", "50" }, { "low", "50" }, { "medium", "75" }, { "high", "75" }, { "xhigh", "100" }, { "max", "100" } }) { + common_chat_templates_inputs inputs; + inputs.messages = { message_user }; + inputs.add_generation_prompt = true; + inputs.chat_template_kwargs["reasoning_effort"] = json(effort).dump(); + auto prompt = common_chat_templates_apply(tmpls.get(), inputs).prompt.str(); + assert_contains(prompt, "<｜System｜>Reasoning Effort: " + budget + " (range 1-100"); + } + + { + common_chat_templates_inputs inputs; + inputs.messages = common_chat_msgs_parse_oaicompat(json::parse(R"([ + {"role": "system", "content": "s", "tools": [{"type": "function", "function": {"name": "get_time", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}}, "required": ["city"]}}}]}, + {"role": "user", "content": "q"} + ])")); + inputs.add_generation_prompt = true; + inputs.chat_template_kwargs["thinking_mode"] = "\"chat\""; + inputs.enable_thinking = false; + inputs.reasoning_format = COMMON_REASONING_FORMAT_DEEPSEEK; + common_chat_session session(tmpls.get(), nullptr, inputs, {}); + const auto & msg = session.finish(common_chat_input(std::string( + "\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_time\">\n" + "<｜DSML｜ parameter name=\"city\" string=\"true\">Tokyo\n\n"))); + assert_equals(size_t(1), msg.tool_calls.size()); + assert_equals(std::string("get_time"), msg.tool_calls[0].name); + assert_equals(std::string("{\"city\":\"Tokyo\"}"), msg.tool_calls[0].arguments); + } +} + static void test_prompt_parts() { LOG_DBG("%s\n", __func__); const std::string tmpl_src = @@ -8527,6 +8589,7 @@ int main(int argc, char ** argv) { test_deepseek_v4_thinking_retention(); test_deepseek_v4_tool_result_ordering(); test_deepseek_v4_1_message_fields(); + test_deepseek_v4_1_shipped_template_fields(); test_template_generation_prompt(); test_reasoning_effort_caps(); test_prompt_parts(); diff --git a/tests/test-quant-type-selection.cpp b/tests/test-quant-type-selection.cpp index 1696ec164c07..63894cc24d8d 100644 --- a/tests/test-quant-type-selection.cpp +++ b/tests/test-quant-type-selection.cpp @@ -57,6 +57,9 @@ static const ftype_name_entry ftype_name_table[] = { { "TQ2_0", LLAMA_FTYPE_MOSTLY_TQ2_0 }, { "MXFP4_MOE", LLAMA_FTYPE_MOSTLY_MXFP4_MOE }, { "NVFP4", LLAMA_FTYPE_MOSTLY_NVFP4 }, + { "IQ1_XS", LLAMA_FTYPE_MOSTLY_IQ1_XS }, + { "IQ1_XXS", LLAMA_FTYPE_MOSTLY_IQ1_XXS }, + { "IQ1_XXXS", LLAMA_FTYPE_MOSTLY_IQ1_XXXS }, }; static llama_ftype llama_ftype_from_name(const char * name) { diff --git a/tools/mtmd/mtmd-image.cpp b/tools/mtmd/mtmd-image.cpp index 2750bc3fc61b..3e6c60b0eb6b 100644 --- a/tools/mtmd/mtmd-image.cpp +++ b/tools/mtmd/mtmd-image.cpp @@ -1601,6 +1601,9 @@ mtmd_image_preproc_out mtmd_image_preprocessor_deepseek41v::preprocess(const cli // load_image rounds the normalized input to bf16 (round to nearest even) for (auto & e : out.entries) { + if (e.is_placeholder()) { + continue; + } std::vector buf = e.get_ro_buf(); for (float & v : buf) { v = ggml_bf16_to_fp32(ggml_fp32_to_bf16(v)); diff --git a/tools/quantize/quantize.cpp b/tools/quantize/quantize.cpp index 38950036cd82..597a222bb598 100644 --- a/tools/quantize/quantize.cpp +++ b/tools/quantize/quantize.cpp @@ -45,6 +45,9 @@ static const std::vector QUANT_OPTIONS = { { "IQ2_M", LLAMA_FTYPE_MOSTLY_IQ2_M, " 2.7 bpw quantization", }, { "IQ1_S", LLAMA_FTYPE_MOSTLY_IQ1_S, " 1.56 bpw quantization", }, { "IQ1_M", LLAMA_FTYPE_MOSTLY_IQ1_M, " 1.75 bpw quantization", }, + { "IQ1_XS", LLAMA_FTYPE_MOSTLY_IQ1_XS, " 1.44 bpw quantization", }, + { "IQ1_XXS", LLAMA_FTYPE_MOSTLY_IQ1_XXS, " 1.31 bpw quantization", }, + { "IQ1_XXXS", LLAMA_FTYPE_MOSTLY_IQ1_XXXS, " 1.19 bpw quantization", }, { "TQ1_0", LLAMA_FTYPE_MOSTLY_TQ1_0, " 1.69 bpw ternarization", }, { "TQ2_0", LLAMA_FTYPE_MOSTLY_TQ2_0, " 2.06 bpw ternarization", }, { "Q2_K", LLAMA_FTYPE_MOSTLY_Q2_K, " 2.96G, +3.5199 ppl @ Llama-3-8B", },