diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index b14c50c53..5a3aff067 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -102,7 +102,7 @@ lifecycle are unchanged. | `KV-SLIDING-LOCAL-SPECS` | Block row (claim the two leaves below, not this row): sliding-window and chunked-local KV specs | T1 | `vllm/v1/kv_cache_interface.py:205-307,480-586`; `tests/v1/test_kv_cache_spec_registry.py:174-306` | - | - | [sliding-local-yarn-long-context.md](specs/sliding-local-yarn-long-context.md) | `READY` | - | | `KV-SLIDING-WINDOW-SPEC` | `SlidingWindowSpec` sizing, grouping, admission, allocation, eviction, and prefix-cache policy; CPU G1/G2 green, while feature-positive attention/model/oracle/performance gates remain | T1 | `vllm/v1/kv_cache_interface.py:518-586`; `vllm/v1/core/single_type_kv_cache_manager.py:669-873`; `tests/v1/core/test_single_type_kv_cache_manager.py:127,259,380,413,489`; `tests/v1/core/test_prefix_caching.py:2457-3909` | `include/vllm/v1/kv_cache_interface.h:187`; `src/vllm/v1/kv_cache_spec_registry.cpp:69`; `src/vllm/v1/core/single_type_kv_cache_manager.cpp:350,377,470,920`; `src/vllm/v1/core/kv_cache_utils.cpp:21`; `src/vllm/v1/core/kv_cache_coordinator.cpp:36,119` | `tests/vllm/v1/test_kv_cache_interface.cpp:157,204,258`; `tests/vllm/v1/test_single_type_kv_cache_manager.cpp:283,331,368,411,453,476`; `tests/vllm/v1/test_kv_cache_utils.cpp:592,617`; `tests/vllm/v1/test_kv_cache_coordinator.cpp:163,238,357` | [sliding-local-yarn-long-context.md](specs/sliding-local-yarn-long-context.md) | `GATING` | - | | `KV-CHUNKED-LOCAL-SPEC` | `ChunkedLocalAttentionSpec` sizing, grouping, admission, allocation, fixed-chunk prefix-cache/recycling policy and hybrid-disabled fallback; CPU G1/G2 green, while W4/model/oracle/runtime gates remain | T1 | `vllm/v1/kv_cache_interface.py:480-514`; `vllm/v1/core/single_type_kv_cache_manager.py:876-1023`; `vllm/v1/core/kv_cache_utils.py:1403-1496`; `tests/v1/core/test_single_type_kv_cache_manager.py:54,198,456`; `tests/v1/test_kv_cache_spec_registry.py:174-315` | `include/vllm/v1/kv_cache_interface.h:219`; `src/vllm/v1/kv_cache_spec_registry.cpp:71`; `src/vllm/v1/core/single_type_kv_cache_manager.cpp:535,553,618,933`; `src/vllm/v1/core/kv_cache_utils.cpp:21`; `src/vllm/v1/core/kv_cache_coordinator.cpp:47` | `tests/vllm/v1/test_kv_cache_interface.cpp:188,204,258`; `tests/vllm/v1/test_single_type_kv_cache_manager.cpp:576,643,683,705,730,1072`; `tests/vllm/v1/test_kv_cache_utils.cpp:629,654,674,686`; `tests/vllm/v1/test_kv_cache_coordinator.cpp:188,258,380,524` | [sliding-local-yarn-long-context.md](specs/sliding-local-yarn-long-context.md) | `GATING` | - | -| `KV-FP8` | FP8 KV cache and scale handling. **W0 spike + W1 CPU brick LANDED 2026-07-29** — fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first. Storage is 1-byte fp8 (`DType::kI8`) + the `Fp8KVCacheDataType` interpretation enum (mirrors vLLM's `cache_t=uint8_t`+`KV_DTYPE`), per-tensor k/v scales (`kv_cache.py:108-191`). **W2 CUDA arm LANDED 2026-08-21** ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)) -- the fp8-e4m3 store kernel + the fp8 dequant on the paged-attention read, gated for parity against the W1 CPU oracle; the two W1 device-class refusals that made the CUDA arm unreachable are gone, and the READ keeps a NAMED CPU-or-CUDA refusal because it rides additive `PagedAttentionArgs` fields on an op `kMETAL`/`kROCM` register for the FLOAT path. **Its DEVICE cases are UNEXECUTED** (no device in the implementing session), though the CUDA TUs DO COMPILE: CI `cuda-fat-build` built them for ten architectures under `-Werror=all-warnings` on `4d71e776e` (run 32495320287). That job sets `-DVLLM_CPP_BUILD_TESTS=OFF`, so nothing has EXECUTED them -- see the spec's `## Owed`. **Residuals (honest, named):** the runner/spec integration (half-sized KV blocks + checkpoint-scale threading + `--kv-cache-dtype`/`--calculate-kv-scales`), fp8_e5m2 CPU compute + per-head scales — all W2-W5 in the spec | T1 | `vllm/config/cache.py:19-36,76`; `vllm/model_executor/layers/quantization/kv_cache.py:42,108-191`; store `csrc/libtorch_stable/cache_kernels.cu:241-252,314-401`; scale convention `csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh:296-308` | codec `include/vt/fp8_kv.h`; store `src/vt/cpu/cpu_cache.cpp:143`; wrapper `src/vt/ops.cpp:2255`; read dequant `src/vt/cpu/cpu_paged_attn.cpp:82`; config parse `include/vllm/v1/kv_cache_dtype.h:37` | `tests/vt/test_ops_fp8_kv_cache.cpp:1` (8 cases / 511 assertions; RED-first: wrong store direction fails 3/480) | [fp8-kv-cache](specs/fp8-kv-cache.md) | `ANCHOR-BACKFILL` | `CLAIM-KV-FP8` | +| `KV-FP8` | FP8 KV cache and scale handling. **W0 spike + W1 CPU brick LANDED 2026-07-29** — fp8-e4m3 K/V STORE (`Quantize(hp/scale)`) + the paged-attention READ dequant (`Dequant(fp8)*scale`) + the `cache_dtype` config parse, all CPU-gated RED-first. Storage is 1-byte fp8 (`DType::kI8`) + the `Fp8KVCacheDataType` interpretation enum (mirrors vLLM's `cache_t=uint8_t`+`KV_DTYPE`), per-tensor k/v scales (`kv_cache.py:108-191`). **W2 CUDA arm LANDED 2026-08-21** ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)) -- the fp8-e4m3 store kernel + the fp8 dequant on the paged-attention read, gated for parity against the W1 CPU oracle; the two W1 device-class refusals that made the CUDA arm unreachable are gone, and the READ keeps a NAMED CPU-or-CUDA refusal because it rides additive `PagedAttentionArgs` fields on an op `kMETAL`/`kROCM` register for the FLOAT path. **Its DEVICE cases are UNEXECUTED** (no device in the implementing session), though the CUDA TUs DO COMPILE: CI `cuda-fat-build` built them for ten architectures under `-Werror=all-warnings` on `4d71e776e` (run 32495320287). That job sets `-DVLLM_CPP_BUILD_TESTS=OFF`, so nothing has EXECUTED them -- see the spec's `## Owed`. **W3 runner integration LANDED 2026-08-22** ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)) -- `--kv-cache-dtype` on the server flag, the checkpoint's own `kv_cache_quant_algo` honoured when no flag is typed (read from `config.json:quantization_config` first and `hf_quant_config.json` as the fallback, which is upstream's order at `transformers_utils/config.py:751-761`), KV blocks sized at ONE byte per element so a fixed `--kv-cache-memory` buys exactly 2x the blocks, and the `k_scale`/`v_scale` path with its declared-but-absent arm named rather than defaulted. The store and the read normalise K and V to the MODEL dtype first, because the fp8 store quantizes from one source dtype and the attention preamble emits f32 K beside a bf16 V on every production weight arm. **Turning it on COSTS the fast attention kernels:** FA-2 prefill, FA-2 decode, the WMMA ladder and the vectorized decode-opt/GQA kernels are bf16-native by construction and an fp8 cache routes only through tiled prefill and block decode, so the memory win and the throughput cost have not been measured against each other -- recorded, not claimed, in the spec's `## W3`. **Residuals (honest, named):** the C ABI does not expose the flag, 16 architectures refuse rather than route, and only ONE of them (`nemotron_h_device`) names fp8 to the operator -- `qwen3_vl` alone reaches the store guard, 13 stop at their own `": KV cache must be bf16 or f32"` rule and `gemma4` stops one step earlier still, inside `vt::CastF32`, with a message that names no architecture at all, no weight loader extracts `k_scale`/`v_scale`, fp8_e5m2 CPU compute and per-head scales -- all in the spec's `## Owed` | T1 | `vllm/config/cache.py:19-36,76`; `vllm/model_executor/layers/quantization/kv_cache.py:42,108-191`; store `csrc/libtorch_stable/cache_kernels.cu:241-252,314-401`; scale convention `csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh:296-308` | codec `include/vt/fp8_kv.h`; store `src/vt/cpu/cpu_cache.cpp:143`; wrapper `src/vt/ops.cpp:2255`; read dequant `src/vt/cpu/cpu_paged_attn.cpp:82`; config parse `include/vllm/v1/kv_cache_dtype.h:37` | `tests/vt/test_ops_fp8_kv_cache.cpp:1` (8 cases / 511 assertions; RED-first: wrong store direction fails 3/480); W3 `tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp` (31 cases, G1-G12, entering through `LoadedEngine` and through `Qwen3DenseModel::Forward` for the shared seam; G12's third case bounds the fp8 pages against the bf16 run element by element, inside e4m3's own round-trip envelope, because a bound written on the logits measures the model's insensitivity and not the cache) + `tests/vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp` (the `--kv-cache-dtype` flag through the REAL `VllmServerMain`) | [fp8-kv-cache](specs/fp8-kv-cache.md) | `ANCHOR-BACKFILL` | `CLAIM-KV-FP8` | | `KV-NVFP4-TURBO` | NVFP4, per-token-head, and TurboQuant KV | T2 | `vllm/config/cache.py:14,28-35,272` | - | - | `planned: specs/nvfp4-kv-cache.md` | `INVENTORIED` | - | | `KV-OFFLOAD` | KV offload tiering: CPU primary tier plus secondary tiers, including the **filesystem (disk) tier that is vLLM's KV-persistence-to-disk answer**. **Record CORRECTED 2026-07-22 ([spike](specs/kv-persistence-lmcache.md)) — the prior row text named a class that does not exist and omitted the half the user asked for.** There is no `LRUOffloadingManager` at this pin: LRU and ARC are pluggable `CachePolicy` objects behind ONE `CPUOffloadingManager`, and the row's scope ('CPU tiering with LRU and ARC') left out the entire secondary-tier surface. Disk format enumerated: ONE RAW FILE PER BLOCK, no container and no index, `/__r//_g/.bin`, written via temp-file + atomic rename under `O_DIRECT` and self-healing by deleting unreadable files. Two upstream WEAKNESSES recorded as beyond-parity targets: `config.json` is written and NEVER read (the only identity check is a path digest omitting checkpoint content, weight quantization, rope config and `sliding_window`), and the disk tier has NO capacity accounting and NO eviction. Secondary tiers can never touch GPU memory — all traffic cascades through the CPU primary tier **W1-W3 IMPLEMENTED 2026-07-22.** Deterministic block hashes (W1), the CPU primary tier (W2: `CachePolicy` LRU+ARC with the `ref_cnt == -1` tri-state and the ATOMIC evict, `CPUOffloadingManager` incl. the `prepare_store -> nullopt` skip path, pinned backing store plus side-queue event-polled device/host transfer), and the DISK tier (W3: one raw file per block, temp-file + atomic rename publish, self-healing unlink, dual-queue read/write pool). **BOTH recorded upstream weaknesses are now EXCEEDED, not merely noted:** the identity block is a VERIFIED header read on every open that REFUSES on mismatch across 27 fields (upstream's `config.json` is never read), and the tier carries a byte budget with policy-driven eviction honoured across restarts (upstream has none). `O_DIRECT` is deliberately NOT ported — a header+payload file breaks its alignment requirement; recorded. **W4 IMPLEMENTED 2026-07-23.** The TIERING MANAGER (ONE manager over the CPU primary + disk secondary tier: disk→CPU promotion is RETRY this step / HIT the next with the reserved slot marked in-flight, cascade demotion on store, reset drains the secondary FIRST and DELIBERATELY never resets it so a persisted cache survives a prefix-cache reset) and the CONNECTOR/SCHEDULER HALF (`OffloadingConnector` mirroring `KVConnectorBase_V1`'s scheduler hooks — `get_num_new_matched_tokens` with the load-bearing NULLOPT third state, `Request::block_hashes` striding, load-before-compute ordering, `build_connector_meta` reset — wired OPT-IN and DEFAULT-OFF into the scheduler so a cross-request/restarted-process prefix HIT shortcuts prefill). The semantics are ported, NOT the Python plugin ABI (compile-time wiring replaces the `importlib` module path; the full 7-method abstract ABI + registration + `KVTransferConfig` is the W5 generalization behind the same seam). Deviation recorded: W4 ships the SYNCHRONOUS-load shape (async flag always false), the disk→CPU promotion being the async part handled by RETRY/re-ask; the cross-step `WAITING_FOR_REMOTE_KVS` GPU-load buffer is W5. **First measured offload speedup:** a restarted-prefix workload through the real scheduler saved 32/48 prefill tokens (2/3 blocks HIT from disk) with the promoted bytes proven byte-identical to the cold store. **W5 LANDED 2026-07-23** (the connector seam is now a first-class C++ ABI — abstract `KVConnector` base + `KVConnectorFactory` + `KVTransferConfig`, the disk connector refactored onto it behaviour-identically; see the `KV-CONNECTORS` row). **D1 CORRECTION 2026-07-24 (`CLAIM-DOCS-T2-FIXES`): the disk connector's WORKER HALF IS NOT IMPLEMENTED and is now REFUSED, not merely absent.** `OffloadingConnector` emits `ConnectorLoadJob`s that NOTHING consumes, and its bytes live in a host `PrimaryByteView` that is never copied into a KV page — on any device. Because its scheduler half DOES shortcut prefill for matched blocks, wiring it into an engine would have made the model attend over never-written KV (silently wrong output); `BuildKvConnector` previously built it for any device with no guard. It is now refused at construction by a per-connector capability predicate (`KVConnector::supports_worker_transfer_on` / the registered `KVConnectorWorkerTransferFn`, queried by name BEFORE construction via `KVConnectorFactory::WorkerTransferSupportedOn`), with an error naming the connector, the device, the consequence and the admissible connectors. The scheduler-side 32/48 e2e is UNAFFECTED (it never reaches a worker). Implementing the worker half remains OPEN work and is NOT claimed. W6 (LMCache study) and W7 (named save/restore) remain open | T2 | core `vllm/v1/kv_offload/base.py:27-47,88-108,177-347,486-588,536-549`; CPU tier `vllm/v1/kv_offload/cpu/manager.py:36,169-237`, policies `cpu/policies/base.py:10-33,36-92`, `lru.py:12`, `arc.py:12`; **disk tier** `vllm/v1/kv_offload/tiering/fs/io.py:32-72,75-101`, `tiering/fs/manager.py:95-103,131-137`, `tiering/fs/thread_pool.py:50-57,153-180`; naming/identity `vllm/v1/kv_offload/file_mapper.py:112-120,128-139`; tiering ordering `tiering/manager.py:238-329,408-459,498-556,643-681`; transfer `cpu/gpu_worker.py:240-421,388-394`; config `docs/features/kv_offloading_usage.md:64-82,95-121`; tests `tests/v1/kv_offload/tiering/test_fs_tier.py`, `tests/v1/kv_offload/test_file_mapper.py`, `tests/v1/kv_offload/cpu/test_manager.py` | **W1-W3 LANDED.** Core `include/vllm/v1/kv_offload/base.h` (OffloadKey verified byte-identical to upstream's packing); policies `include/vllm/v1/kv_offload/cache_policy.h` + `src/vllm/v1/kv_offload/cache_policy.cpp`; CPU tier `include/vllm/v1/kv_offload/cpu_manager.h` + `src/vllm/v1/kv_offload/cpu_manager.cpp`; transfer `include/vllm/v1/kv_offload/kv_block_transfer.h` + `src/vllm/v1/kv_offload/kv_block_transfer.cpp` (plus the new non-blocking `vt::Backend::QueryEvent` seam with its CUDA override in `src/vt/cuda/cuda_backend.cu`); disk byte path + naming `include/vllm/v1/kv_offload/fs_io.h` + `src/vllm/v1/kv_offload/fs_io.cpp`; tier `include/vllm/v1/kv_offload/fs_tier.h` + `src/vllm/v1/kv_offload/fs_tier.cpp`; the verified identity header `include/vllm/v1/kv_offload/cache_identity.h` + `src/vllm/v1/kv_offload/cache_identity.cpp`; determinism fix `src/vllm/v1/core/kv_cache_utils.cpp` (`init_none_hash` seed resolution + `none_hash_provenance`), caller `src/vllm/entrypoints/model_loader.cpp:140-152`; **W4** tiering manager `include/vllm/v1/kv_offload/tiering_manager.h` + `src/vllm/v1/kv_offload/tiering_manager.cpp`; connector/scheduler half `include/vllm/v1/kv_offload/kv_connector.h` + `src/vllm/v1/kv_offload/kv_connector.cpp`; scheduler wiring `src/vllm/v1/core/sched/scheduler.cpp` (`set_kv_connector`, null = zero change) + `include/vllm/v1/core/sched/scheduler.h`; `BlockPool::evict_blocks` `src/vllm/v1/core/block_pool.cpp:139-155` (1:1, replaces the throw) | `tests/vllm/v1/test_none_hash_determinism.cpp:108` 7/7 (cross-PROCESS byte-identical hash chains via a `/proc/self/exe` re-exec, both env escape hatches, and the `=random` negative control); `tests/vllm/v1/test_kv_offload_cpu.cpp` 21/21 (atomic evict, pinning, ARC promotion, HIT_PENDING, failed-store rollback, same-batch protection, store_threshold, events, transfer round-trip); `tests/vllm/v1/test_kv_offload_fs.cpp` 22/22 + 3 SKIP (byte-exact round trip for full attention AND MLA rank-3, truncation/foreign-magic/misfiled refusal with self-heal, a 27-field identity-refusal matrix with a positive control, the byte budget across a restart, and a 6/6 cross-restart hit measurement); the SKIPs are row-tagged to `KV-SLIDING-WINDOW-SPEC`, `KV-FP8`/`KV-NVFP4-TURBO` and `KV-MAMBA-ALIGN`; **W4** `tests/vllm/v1/test_kv_offload_tiering.cpp` 5/5 (promotion RETRY→HIT byte-identical, CPU-eviction→disk-survival→re-promotion, reset clears CPU but disk survives, a FRESH manager on the same directory promotes = restart, and identity REFUSAL through a promotion — a corrupt disk block is unlinked and treated as absent, never trusted) and `tests/vllm/v1/test_kv_offload_connector.cpp` 4/4 (null-connector inertness, external match shortcuts prefill by exactly ext, the nullopt third state defers then schedules next step, and the END-TO-END restarted-prefix disk HIT through the real scheduler: hit rate 2/3 blocks, 32/48 prefill tokens saved, promoted bytes byte-identical) | [kv-persistence-lmcache.md](specs/kv-persistence-lmcache.md) | `ANCHOR-BACKFILL` | `CLAIM-KV-PERSISTENCE-LMCACHE` | | `KV-EXTERNAL-CACHE` | External KV-cache provider ABI plus LMCache interoperability: producer/consumer/both roles, the scheduler/worker metadata split, cache registration, block-hash lookup, asynchronous load/store and completion/free ownership. **SPIKED 2026-07-22 ([spike](specs/kv-persistence-lmcache.md)) — the ABI is smaller than the row implied and the LMCache half is larger.** The minimum viable connector is **exactly 7 abstract methods** (worker `start_load_kv`/`wait_for_layer_load`/`save_kv_layer`/`wait_for_save`, scheduler `get_num_new_matched_tokens`/`update_state_after_alloc`/`build_connector_meta`); roughly thirty further hooks all have safe defaults. Three traps recorded: `get_num_new_matched_tokens` has a THIRD state (`None` = deschedule and re-ask, not zero), `request_finished` returning True transfers block-freeing OWNERSHIP to the connector, and non-HMA connectors ASSERT a single KV cache group while our gate models are two-group hybrids. **LMCache determination: it is an EXTERNAL PyPI package** (`lmcache >= 0.3.9` in an opt-in extras file that `setup.py`/`pyproject.toml` never reference; not installed on any of this project's boxes). vLLM vendors roughly 2396 lines of `lmcache_integration/` glue, but every one of those files imports the external package at module scope — the storage engine, the paged-memory GPU connectors, the config schema, the ZMQ message queue and the **CUDA-IPC** handoff are all outside the tree, and no upstream test exercises it without importing `lmcache`. Scoped as an interop STUDY, not a from-scratch client, and gated on two blockers we own: our `sha256_cbor` hashes are not byte-compatible with vLLM's default, and our `NONE_HASH` is per-process random. **REOPENED 2026-07-23 ([client spike](specs/lmcache-cpp-client-connector.md)) on the user's connect-as-client hypothesis, and the prior "no specified wire protocol" verdict is REFUTED by reading the LMCache package (`LMCache/LMCache@8570aad`).** vLLM connects to a RUNNING LMCache instance over two fully-specified, language-agnostic wires: (1) the `lm://` remote-store server — **plain TCP + a fixed `struct.pack` header + raw KV bytes**, no ZMQ/msgpack/pickle/CUDA-IPC (`lmcache/v1/protocol.py:214-321`, `server/__main__.py:24-147`, `lm_connector.py:28-177`); and (2) the MP server — **ZMQ DEALER↔ROUTER + `msgspec.msgpack` control + CUDA-IPC data** (`multiprocess/mq.py:270-353`, `custom_types.py:120-234`), the mode the user recalled as "zmq". BOTH need ZERO `lmcache` in our process and BOTH sidestep the R1 hash blocker — LMCache keys on its OWN blake3 rolling token hash (`token_hasher.py:54-79`), never vLLM block hashes. Pickle appears ONLY in the MP one-time IPC-wrapper registration (`platform/base/ipc_wrapper.py` Serialize); CUDA-IPC ONLY in MP data (portable via `RawCudaIPCWrapper` `cudaIpcGetMemHandle`, but co-located). Verdict: a C++ client is FEASIBLE — recommend MODE (1) first (stabler/simpler); the standing risk is LMCache being an unpinned moving target, so it is an interop feature with a version-sync cost, not a mechanical core port | T2 | ABI `vllm/distributed/kv_transfer/kv_connector/v1/base.py:171,293,311,325,347,454,489,510,542,585`; roles `:124`; HMA `:85,93`; factory + out-of-tree module seam `vllm/distributed/kv_transfer/kv_connector/factory.py:28,31,96,102-123,152-238`; config `vllm/config/kv_transfer.py:22-75,102-106`; MRV2 worker hooks `vllm/v1/worker/gpu/kv_connector.py:56,61-75,77-95`; scheduler call sites `vllm/v1/core/sched/scheduler.py:280,736-742,933-937,1118-1119,2340-2371`; LMCache `vllm/distributed/kv_transfer/kv_connector/v1/lmcache_connector.py:74-115,259,281`, `lmcache_mp_connector.py:1-50`, `lmcache_integration/vllm_v1_adapter.py:11-35,175-188,368-376,781`, external requirement `requirements/kv_connectors.txt:1`; tests `tests/v1/kv_connector/unit/test_lmcache_integration.py:60-223`, `test_kv_connector_lifecycle.py:37`, `test_config.py:51` | **W1 LANDED 2026-07-23 — the LMCache MODE-1 `lm://` wire CODEC (pure CPU, INERT: no call site routes to it, the connector is W3):** `src/vllm/v1/kv_offload/lmcache/remote_protocol.{h,cpp}` (186-byte `ClientMetaMessage` / 36-byte `ServerMetaMessage` fixed-`struct` framing + `ClientCommand`/`ServerReturnCode`/`DTYPE_TO_INT`/`Location` maps), `cache_engine_key.{h,cpp}` (`model@world@worker@chunk_hash_hex@dtype` to/from string), `token_hasher.{h,cpp}` (blake3 rolling chunk hash over vendored `third_party/blake3/` 1.5.5), `memory_format.{h,cpp}` (the `KV_2LTD` `[2,L,T,D]` repack); wired in `CMakeLists.txt` (`blake3_vendored` static lib). Later-connector seams still NAMED: `include/vllm/v1/core/kv_cache_manager.h:31` (`ext_comp`), `include/vllm/v1/core/single_type_kv_cache_manager.h:122`, `include/vllm/v1/core/sched/output.h:30-31`, `include/vllm/v1/engine/types.h:26,30`. **W5 worker-side store/load LANDED 2026-07-24 (the last open arm):** `src/vllm/v1/worker/gpu/runner.cpp` (`ConnectorLoadExternalKv` writes the external-prefix KV into the allocated GPU blocks BEFORE the forward = load-before-compute; `ConnectorStorePromptKv` stores each newly-complete prompt block AFTER the forward; both behind a `kv_connector_ != nullptr` guard so default-off is byte-identical) + `include/vllm/v1/worker/gpu/runner.h` (`set_kv_connector`), `src/vllm/entrypoints/model_loader.cpp` (`BuildKvConnector` builds the connector from `EngineParams::kv_transfer_config` via `KVConnectorFactory`, injects the runner's full-attention KV geometry, wires it to scheduler + runner) + `include/vllm/entrypoints/model_loader.h` (`EngineParams::kv_transfer_config`, `LoadedEngine::kv_connector()`) | **W1 byte/bit-exact gate GREEN (CPU): `tests/vllm/v1/kv_offload/lmcache/test_lmcache_codec.cpp:105` (6 cases / 2074 assertions) vs `tests/fixtures/lmcache/lmcache_fixtures.json` — our wire bytes == the real Python codec's (stdlib `struct` framing + `blake3` PyPI hashes + numpy KV_2LTD); blake3 digest VERIFIED byte-identical on x86-64 AND `dgx.casa` aarch64.** **W2 (client, CPU) GREEN — go/no-go PASSED:** `src/vllm/v1/kv_offload/lmcache/remote_client.{h,cpp}` (blocking POSIX-socket PUT/GET/EXIST/HEALTH/LIST + partial-read/write loops + `PutKv2ltd`/`GetKv2ltd` `KV_2LTD` repack + `LmcacheClientConfig`/`VT_LMCACHE_*` env); `tests/vllm/v1/kv_offload/lmcache/test_lmcache_client.cpp` round-trips a **REAL `lmcache.v1.server`** (`8570aad`, run headless from source in a throwaway venv — torch imported before lmcache to dodge a torch circular import, the compiled `c_ops` ext stubbed as unused by the lm:// CPU store) byte-identical (36/36), and interop is **BIDIRECTIONAL** with LMCache's OWN Python protocol codec (`scripts/lmcache/{lm_server,lm_interop_client}.py`+`run_live_roundtrip.sh`); always-on CI gate = a same-binary C++ mock-server round-trip (45/45, no Python). **W3 LANDED 2026-07-23 — the `lm://` client wired as a `KVConnector` over the W5 seam (the FIRST time engine -> connector -> W2 client -> a running lm:// server -> back runs):** `src/vllm/v1/kv_offload/lmcache/lmcache_connector.{h,cpp}` (`LMCacheConnector : KVConnector`, `REGISTER_KV_CONNECTOR("LMCacheConnector", …)`, selected by `KVTransferConfig{kv_connector="LMCacheConnector", kv_connector_extra_config={host,port,hash_algo,chunk_tokens,…}}`, default OFF). Scheduler side is real: `get_num_new_matched_tokens` computes the request's rolling-blake3 chunk hashes, builds the `CacheEngineKey` per chunk and `Exist`-probes the REMOTE store for the longest cached prefix (synchronous -> `(n, false)`, mirroring `lmcache_connector.py:230-259`); `update_state_after_alloc` records the load (drops `blocks` upstream, `:261-268`); worker `StoreChunk` (PUT KV_2LTD) / `LoadChunk` (GET+unpack, foreign-block REFUSAL via `GetKv2ltd`). **Gate ACHIEVED = the connector-level round-trip: store -> lookup -> prefill-shortcut through the REAL scheduler -> load byte-identical (32/48 prefill tokens saved), foreign/mismatched-key REFUSAL, default-off inertness** (`tests/vllm/v1/kv_offload/lmcache/test_lmcache_connector.cpp` 5 cases / 50 assertions vs an in-process mock; the store->load round-trip ALSO passes vs a REAL `lmcache.v1.server` 8570aad, 16 assertions, under `VT_LMCACHE_LIVE_*`). **W4 LANDED 2026-07-23 — REAL peer KEY-AGREEMENT + a peer->us interop LOAD, both PROVEN (the interop-correctness milestone is complete; the row stays `ACTIVE` only for the DGX full-model output-invariance + throughput arm, spec gates 4/6):** the actual `lm://` key derivation is NOT the blake3 MP `TokenHasher` (a different subsystem) but `ChunkedTokenDatabase` (`lmcache/v1/token_database.py:298-449`) — chunk_size 256, a rolling prefix-hash chain over the 3-tuple `(prefix_int, tuple(tokens), extra_keys=())`, keyed by vLLM's OWN hash function (`pre_caching_hash_algorithm`; the portable interop choice `sha256_cbor` = cbor2-canonical + SHA-256, `vllm/utils/hashing.py:43`), folded to uint64 each step (`_normalize_hash_to_int` `token_database.py:34-56`), with `NONE_HASH = fold8(sha256_cbor(str(PYTHONHASHSEED)))` (`kv_cache_utils.py:99-114`). Mirrored BYTE-EXACT in `src/vllm/v1/kv_offload/lmcache/chunked_token_database.{h,cpp}` (reusing the project's `CborValue`+`sha256_cbor`, already Python-cbor2/hashlib-exact), and wired into the connector as `key_mode=kVllmSha256Cbor` (`hash_algo="vllm"/"sha256_cbor"`, chunk 256) alongside W3's kept-green blake3 path. **Key-agreement gate GREEN:** `tests/vllm/v1/kv_offload/lmcache/test_lmcache_key_agreement.cpp` (4 cases / 85 assertions) asserts our `CacheEngineKey` strings + chunk boundaries + folded hashes are BYTE-IDENTICAL to the REAL lmcache `ChunkedTokenDatabase.process_tokens()` (fixtures `tests/fixtures/lmcache/key_agreement_fixtures.json` dumped by `scripts/lmcache/gen_key_agreement_fixtures.py` driving the unmodified real driver, with vLLM's pinned `sha256_cbor`/`init_none_hash`), incl. the connector's own peer-mode `ChunkKey`. Sample: tokens 1000..1511 -> `meta-llama/Llama-3.1-8B@1@0@33d6862800fff40c@bfloat16`. **Peer->us interop LOAD gate GREEN (over the wire, real server):** `scripts/lmcache/{lm_key_interop.py,run_key_interop.sh}` has the REAL lmcache `ChunkedTokenDatabase` derive a key from tokens and PUT KV to a REAL `lmcache.v1.server` (8570aad, headless); our C++ INDEPENDENTLY re-derives the SAME key and GETs the peer-written 512 B byte-identical (`test_lmcache_key_agreement` LIVE case under `VT_LMCACHE_LIVE_SPEC`). ASan+UBSan clean on the connector path. Text-only scope (mm-hash extra_keys deferred); the DGX full-model output-invariance + throughput are the W5 arm below. **W5 OUTPUT-INVARIANCE GATE GREEN 2026-07-24 (spec gates 4+6 met — the LAST open arm CLOSED):** `tests/vllm/models/test_lmcache_output_invariance.cpp` on a REAL OPT-125m bf16 loop vs a live `lmcache.v1.server` (8570aad, headless per the W2 recipe) proves connector-ON generated tokens are BIT-IDENTICAL to connector-OFF cold full prefill (first-divergence index -1) in BOTH modes — (a) store->restart->load within one process AND (b) a genuinely COLD second process that only hits the server (`VT_LMCACHE_OI_MODE=loadonly`) — with prefill SAVED on the hit = 48 tokens (3×16-token blocks) and chunks_stored>0; driven by `scripts/lmcache/run_output_invariance.sh` under `flock $HOME/gpu.lock`, `VT_ASYNC_SCHED=0`. Throughput reported HONESTLY: on a 125M model wall-clock is noise-dominated (fixed TCP/copy overhead ~ tiny compute saved) so NO binding speedup is claimed — a real speed number is owed by an every-axis grid on a larger model + long shared-prefix corpus (docs/BENCHMARKS.md). No-regression WITNESS: OPT SACRED gate UNCHANGED default-off (`test_opt_paged_engine` 6/6 prompts, 96/96 tokens, 63/63 assertions) with the connector code present; connector units green (codec 6/6·2074, client 3/3·45, connector 5/5·50, key-agreement 4/4·85, kv_offload_connector 11/11·80); ASan+UBSan clean on the connector path (0 sanitizer hits); CUDA `-Werror` 0 warnings. Additive + default-off inert (scheduler/worker/seam untouched) | [kv-persistence-lmcache.md](specs/kv-persistence-lmcache.md); LMCache client wire analysis + W-plan [lmcache-cpp-client-connector.md](specs/lmcache-cpp-client-connector.md) | `ANCHOR-BACKFILL` (W1-W5 landed; the connector-ON full-model OUTPUT-INVARIANCE arm is CLOSED — connector-ON == connector-OFF tokens BIT-IDENTICAL on a real OPT-125m loop vs a live `lmcache.v1.server`, both after an in-process restart and from a cold second process, spec gates 4/6 met; a BINDING every-axis LMCache throughput grid on a LARGER model stays PENDING, mirroring the Llama 'correctness DONE, speed PENDING' disposition — a 125M model's wall time is noise-dominated) | `CLAIM-LMCACHE-CPP-CLIENT` (W1 codec + W2 client + W3 connector + W4 key-agreement + W5 output-invariance); parent seam `CLAIM-KV-PERSISTENCE-LMCACHE` | diff --git a/.agents/quantization-matrix.md b/.agents/quantization-matrix.md index fed6b2919..40642d133 100644 --- a/.agents/quantization-matrix.md +++ b/.agents/quantization-matrix.md @@ -157,7 +157,7 @@ Pinned vLLM source: `vllm/config/cache.py:19-36`. | ID | Item | Upstream | Our code | Tests/evidence | Spike/spec | State | Owner | |---|---|---|---|---|---|---|---| -| `QUANT-KV-FP8` | fp8, fp8_e4m3, fp8_e5m2 | `vllm/config/cache.py:19-25`; `vllm/model_executor/layers/quantization/kv_cache.py:42-191`; store `cache_kernels.cu:241-252`; scale convention `quant_utils.cuh:296-308` | **W1 CPU fp8-e4m3 store+read LANDED**: [codec](../include/vt/fp8_kv.h#L39), [store kernel](../src/vt/cpu/cpu_cache.cpp#L143), [read dequant](../src/vt/cpu/cpu_paged_attn.cpp#L82), [config parse](../include/vllm/v1/kv_cache_dtype.h#L37). **W2 CUDA fp8-e4m3 store+read LANDED** ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)): [store kernel](../src/vt/cuda/cuda_cache.cu), [read dequant](../src/vt/cuda/cuda_paged_attn.cu) -- gate [test_cuda_fp8_kv_cache](../tests/vt/test_cuda_fp8_kv_cache.cpp), whose DEVICE cases are UNEXECUTED; the CUDA TUs COMPILE (CI `cuda-fat-build`, ten architectures, run 32495320287 on `4d71e776e`) but that job builds with tests OFF, so none has been executed (spec `## Owed`). e5m2 compute, per-head scales, the Metal/ROCm arms and the runner integration are named later bricks (see spec) | [test_ops_fp8_kv_cache](../tests/vt/test_ops_fp8_kv_cache.cpp#L1) — 8 cases / 511 assertions, round-trip within the e4m3 band + fp8-vs-bf16 NMSE<1% + paged-attention e2e; RED-first (wrong store direction fails 3/480) | [fp8-kv-cache](specs/fp8-kv-cache.md) | `PARTIAL` | - | +| `QUANT-KV-FP8` | fp8, fp8_e4m3, fp8_e5m2 | `vllm/config/cache.py:19-25`; `vllm/model_executor/layers/quantization/kv_cache.py:42-191`; store `cache_kernels.cu:241-252`; scale convention `quant_utils.cuh:296-308` | **W1 CPU fp8-e4m3 store+read LANDED**: [codec](../include/vt/fp8_kv.h#L39), [store kernel](../src/vt/cpu/cpu_cache.cpp#L143), [read dequant](../src/vt/cpu/cpu_paged_attn.cpp#L82), [config parse](../include/vllm/v1/kv_cache_dtype.h#L37). **W2 CUDA fp8-e4m3 store+read LANDED** ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)): [store kernel](../src/vt/cuda/cuda_cache.cu), [read dequant](../src/vt/cuda/cuda_paged_attn.cu) -- gate [test_cuda_fp8_kv_cache](../tests/vt/test_cuda_fp8_kv_cache.cpp), whose DEVICE cases are UNEXECUTED; the CUDA TUs COMPILE (CI `cuda-fat-build`, ten architectures, run 32495320287 on `4d71e776e`) but that job builds with tests OFF, so none has been executed (spec `## Owed`). **W3 runner integration LANDED** ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)): [`--kv-cache-dtype`](../src/vllm/entrypoints/openai/server_main.cpp) reaches `EngineParams`, the checkpoint's `kv_cache_quant_algo` is honoured when no flag is typed, [`ApplyCacheDType`](../src/vllm/v1/kv_cache_interface.cpp) retypes every attention spec (group and heterogeneous per-layer alike) so one byte budget buys exactly 2x the blocks, and [`kv_cache_route.h`](../include/vllm/model_executor/models/kv_cache_route.h) is the ONE place the store and the read decide float versus fp8. An fp8 cache DISABLES FA-2 prefill and decode, the WMMA ladder and the vectorized decode kernels, which are bf16-native; the resulting throughput cost is unmeasured and recorded as such. e5m2 compute, per-head scales, the Metal/ROCm arms, the C-ABI field, the 16 unrouted architectures (15 of which refuse with a message that names neither fp8 nor the flag) and the `k_scale`/`v_scale` weight-loader read are named later bricks (see spec) | [test_ops_fp8_kv_cache](../tests/vt/test_ops_fp8_kv_cache.cpp#L1) — 8 cases / 511 assertions, round-trip within the e4m3 band + fp8-vs-bf16 NMSE<1% + paged-attention e2e; RED-first (wrong store direction fails 3/480); W3 [test_kv_cache_fp8_wiring](../tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp) — 31 cases, G1-G12, entering through `LoadedEngine` rather than by building a spec by hand, and bounding the fp8 pages against a bf16 run inside e4m3's own round-trip envelope, and [test_serve_kv_cache_dtype](../tests/vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp) — the flag through the REAL `VllmServerMain` | [fp8-kv-cache](specs/fp8-kv-cache.md) | `PARTIAL` | - | | `QUANT-KV-FP8-VENDOR` | fp8_inc, fp8_ds_mla | `vllm/config/cache.py:24-25`; vendor KV implementations selected by attention backend | - | no quantized KV cache | `planned: specs/vendor-fp8-kv-cache.md` | `INVENTORIED` | - | | `QUANT-KV-TURBO` | k8v4, 4bit_nc, k3v4_nc, 3bit_nc | `vllm/config/cache.py:28-33`; TurboQuant dependency path | - | no quantized KV cache | `planned: specs/turboquant-kv-cache.md` | `INVENTORIED` | - | | `QUANT-KV-PER-HEAD` | int4/int8/fp8 per-token-head | `vllm/config/cache.py:34`; quantized cache kernels selected by backend | - | no quantized KV cache | `planned: specs/per-head-kv-cache.md` | `INVENTORIED` | - | diff --git a/.agents/specs/fp8-kv-cache.md b/.agents/specs/fp8-kv-cache.md index c75306498..90b8f5dc6 100644 --- a/.agents/specs/fp8-kv-cache.md +++ b/.agents/specs/fp8-kv-cache.md @@ -1,4 +1,4 @@ -# fp8 KV cache (`cache_dtype=fp8*`) — spike + W1 + W2 (`KV-FP8`, `QUANT-KV-FP8`) +# fp8 KV cache (`cache_dtype=fp8*`) — spike + W1 + W2 + W3 (`KV-FP8`, `QUANT-KV-FP8`) Rows: `KV-FP8` (engine-matrix, KV cache and memory) and `QUANT-KV-FP8` (quantization-matrix). HIGH-priority feature gap #5 @@ -22,11 +22,16 @@ re-port). - **In (W2, `## W2 — the CUDA arm` below):** the CUDA fp8-e4m3 K/V store kernel and the fp8 dequant on the CUDA paged-attention read, gated for parity against the W1 CPU reference. +- **In (W3, `## W3 — the runner integration` below):** half-sized KV blocks in + the real runner, `--kv-cache-dtype` threaded from the server flag through the + checkpoint's own `kv_cache_quant_algo` to the block sizing, and the checkpoint + `k_scale`/`v_scale` path with its declared-but-absent arm named rather than + defaulted. - **Out (named later bricks):** fp8_e5m2 compute on either backend, per-attention-head scales, the Metal and ROCm fp8-KV arms (both refuse by name - — see `## W2` below), the full engine-runner integration - (half-sized KV blocks in the real runner + checkpoint `k_scale`/`v_scale` - threading + `--kv-cache-dtype`/`--calculate-kv-scales` CLI), and the vendor + — see `## W2` below), `--calculate-kv-scales` (upstream's deprecated dynamic + scale), the C-ABI exposure of `--kv-cache-dtype`, the 16 architectures whose + attention blocks W3 refuses rather than routes, and the vendor KV dtypes (`fp8_inc`, `fp8_ds_mla` — `QUANT-KV-FP8-VENDOR`) and turboquant / nvfp4 / per-token-head KV (`KV-NVFP4-TURBO`). @@ -164,7 +169,7 @@ vendor/turbo/nvfp4 KV dtypes are separate rows. | W0 | this spike | DONE (this commit) | | W1 | CPU fp8-e4m3 store + read dequant + config parse + unit gate | DONE (this commit) | | W2 | CUDA fp8-e4m3 store + fp8 paged-attention read (parity vs W1) | DONE (code + gate landed; the DEVICE cases are UNEXECUTED — see `## Owed`) | -| W3 | runner/spec integration: half-sized KV blocks + checkpoint k/v_scale threading + `--kv-cache-dtype`/`--calculate-kv-scales` | later | +| W3 | runner/spec integration: half-sized KV blocks + checkpoint k/v_scale threading + `--kv-cache-dtype` | DONE (code + CPU gate landed; see `## W3` and `## Owed`) | | W4 | memory-halving e2e on a gate model (the binding gate, DGX) | later | | W5 | fp8_e5m2 CPU+CUDA compute; per-attention-head scales | later | @@ -267,6 +272,340 @@ then calls the same pointer with e4m3 to prove the guard refuses one kind rather than disabling the kernel. Bypassing the wrapper is deliberate and is the point of the case; the production path still goes through it, and G1/G1b/G2 gate that. +## W3 — the runner integration (#1593) + +Issue: [#1593](https://github.com/mudler/vllm.cpp/issues/1593), the same issue +that carries W2. W3 is what makes the fp8 KV cache a SERVED capability instead +of a pair of kernels: half-sized KV blocks in the real runner, +`--kv-cache-dtype` threaded from the flag to the block sizing, and the +checkpoint `k_scale`/`v_scale` path. + +**Why it is on the critical path.** Benchmark campaign +[#1574](https://github.com/mudler/vllm.cpp/issues/1574) measures us against vLLM +and SGLang on `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121`. We already beat vLLM's AR +baseline there (11.06 vs 9.71 tok/s) — with **bf16 KV against their fp8**, so we +move twice the KV bytes for the same tokens. Until W3 the comparison was not +matched, and [#415](https://github.com/mudler/vllm.cpp/issues/415) attributes a +prefill gap to exactly this. + +**What turning it on COSTS, unmeasured and therefore not claimed.** An fp8 KV +cache takes the subject OFF every fast attention kernel this engine has. +`qwen3_5.cpp` makes `kv.dtype == DType::kBF16` a term of both `fa2_prefill` and +`fa2_decode`, so FA-2 prefill and all three FA-2 decode topologies are disabled; +`src/vt/cuda/cuda_paged_attn.cu` routes an fp8 read only through tiled prefill +and block decode, because the WMMA ladder, the vendored FA-2 launchers and the +vectorized decode-opt/GQA kernels are bf16-native by construction. So the 11.06 +vs 9.71 tok/s lead above was measured on the bf16 path, and matching vLLM's KV +dtype plausibly LOSES it. Nothing here has measured which way the sum goes: half +the KV bytes and twice the pool against a slower attention kernel is an +empirical question and this session had no GPU. It is recorded so that the next +`--kv-cache-dtype fp8` benchmark is read as a NEW measurement rather than as a +regression, and `docs/USAGE.md` says the same thing to an operator. Routing the +fp8 read through the fast kernels is owed below. + +### The resolution chain, mirrored + +Four upstream steps, in upstream's own order. Every anchor was read in +`/home/mudler/_git/vllm` at `555967922` (`git rev-parse HEAD` = +`5559679229bc961848b121ccdeaa8fa5d79bec98`). + +1. **`--kv-cache-dtype` vs the checkpoint** — `resolve_kv_cache_dtype_string` + (`vllm/utils/torch_utils.py:374-392`) + `get_kv_cache_quant_algo_string` + (`:310-362`) + `MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP` (`:64-67`). Ported as + `vllm::ResolveKvCacheDTypeString` (`include/vllm/config/cache.h`, + `src/vllm/config/cache.cpp`) and called ONCE, from + `LoadedEngine::FromModelDir`, exactly where `EngineArgs.create_engine_config` + calls it before constructing `CacheConfig` (`vllm/engine/arg_utils.py: + 1915-1929`). An explicit value is returned unchanged and the checkpoint is + never consulted (`:380-381`) — the operator outranks the checkpoint, which + `attention.py:279-290` restates in its own comment. + + **Which file the declaration is read FROM is also upstream's, and W3 first + shipped it inverted.** `vllm/transformers_utils/config.py:751-761` takes + `config_dict["quantization_config"]` and consults `hf_quant_config.json` only + when that is `None`, under upstream's own comments: ModelOpt writes the + inline document from 0.31.0 on, and the standalone file is what 0.29.0 and + before wrote. `vllm::ReadQuantConfigJson` now reads them in that order. + Reversed, a repository that was re-quantized in place — inline document + added, stale `hf_quant_config.json` left beside it — resolves to the OLD + declaration, which for the KV half means quantizing a cache nobody asked to + quantize, at half the page, silently. G10 gates the order at both the + resolver and the loader. +2. **String to storage dtype** — `kv_cache_dtype_str_to_dtype` (`:394-401`) over + `STR_DTYPE_TO_TORCH_DTYPE` (`:32-52`), where every fp8 CacheDType maps to + `torch.uint8`. W1's `vllm::v1::ParseCacheDType` already did this; W3 adds no + parsing. +3. **Storage dtype to bytes** — `GPUModelRunner.__init__` resolves ONE + `self.kv_cache_dtype` (`vllm/v1/worker/gpu_model_runner.py:484-486`) and every + attention spec is built with it; `AttentionSpec.real_page_size_bytes` + (`vllm/v1/kv_cache_interface.py:204-218`) is linear in + `get_dtype_size(self.dtype)`. Ported as `vllm::v1::ApplyCacheDType` + (`src/vllm/v1/kv_cache_interface.cpp`), called from + `LoadedEngine::ApplyResolvedCacheDType` on the PROBE config **before** + `ResolveNumBlocks` reads its geometry. That ordering is the feature: the + probe's `KVBytesPerBlock` is the divisor knob 2 sizes the pool with, so an + fp8 page halves the divisor and doubles the block count at the same + `--kv-cache-memory`. Applying it afterwards would serve the same pool in half + the bytes instead of twice the pool. +4. **The scales** — `BaseKVCacheMethod.process_weights_after_loading` + (`vllm/model_executor/layers/quantization/kv_cache.py:74-156`), ported as + `vllm::ResolveKvCacheScales` + (`include/vllm/model_executor/layers/quantization/kv_cache.h`). + +### The trap this checkpoint sets + +**First, the fact that changes what the trap is.** +`r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` @ `36f717a2` declares +`kv_cache_quant_algo: "FP8"` in +`hf_quant_config.json` — and that file is NOT the one that gets read. +MEASURED from the live artifact 2026-08-22: its +`config.json:quantization_config` carries `quant_method: "modelopt"`, +`quant_algo: "MIXED_PRECISION"` and **no `kv_cache_*` key at all**, and an +inline document suppresses the legacy file entirely +(`transformers_utils/config.py:751-761`, and the same order in +`vllm::ReadQuantConfigJson`). Running the pinned +`get_kv_cache_quant_algo_string` over those exact bytes returns `None`. + +**So the #1574 subject auto-selects nothing, on either engine, and +`--kv-cache-dtype fp8` has to be typed on BOTH sides of the comparison.** The +competitors' own `serve.sh` already passes it, so the campaign is consistent +rather than blocked; what is wrong is any sentence that says this checkpoint +takes the declared-fp8 path by itself. G10's `#1574 checkpoint declares NOTHING` +case pins it with both real documents in one directory. + +**The scale trap is real and unchanged**, and it is what the flag runs into: the +checkpoint ships **ZERO** `k_scale`/`v_scale` tensors. MEASURED 2026-08-21 from +the public `model.safetensors.index.json`: 2001 tensors, none of them named +`k_scale`, `v_scale` or `kv_scale`. + +So the default scale 1.0 has to be reached DELIBERATELY, by a path that knows +the algorithm was declared and the tensors were absent — not by falling off the +end of a missing-tensor lookup. The two are indistinguishable at runtime and +produce identical output, right up to the first checkpoint that declares no +algorithm at all, at which point the accidental path silently invents a scale +for a cache nobody asked to quantize. + +Upstream keeps them apart STRUCTURALLY and this port mirrors that. +`process_weights_after_loading` reaches the scale block at all only under +`is_quantized_kv_cache(layer.kv_cache_dtype)` (`kv_cache.py:100-102`); INSIDE +it, both scales still holding the `KVCacheScaleParameter` sentinel `-1.0` +(`:18-30`) is the separate "no scales were loaded" arm that takes 1.0 and warns +(`:112-116`, `:150-156`). `KvScaleOrigin` names all four arms, and +`ScalesForFp8Store` REFUSES `kNotQuantized` by name rather than answering 1.0. +`G2` gates the difference: two calls with identical numbers out, distinguished +only by `origin`, and the refusal message names `kv_cache_quant_algo`. + +### The store and the read + +`PagedKvCache` gains `fp8_kind`/`k_scale`/`v_scale`, carried from the layer's +own `AttentionSpec` by the runner. `dense_attn::WriteKvCache` and +`dense_attn::ApplyKvCacheQuant` +(`include/vllm/model_executor/models/kv_cache_route.h`) are the ONE place that +decides float versus fp8, and `IsFp8KvCache` refuses a view whose storage dtype +and fp8 interpretation disagree — a `kI8` page with no fp8 kind, or an fp8 kind +over a float page, is a mis-sized cache and never a mode. + +**Routed in W3:** the shared seam `dense_attn::AttnBlock` +(`include/vllm/model_executor/models/dense_attn_block.h`) — reached by the +Qwen3 dense family (`qwen3.cpp:185`), Qwen3-MoE (`qwen3_moe.cpp:84`), Voxtral +(`voxtral.cpp:102`) and the Llama/Mistral/InternLM2 registries that share +`Qwen3DenseModel` — and `src/vllm/model_executor/models/qwen3_5.cpp`, the +Qwen3.5/3.8 family, which is the benchmark subject. + +**The seam's routing was DEAD until the second review.** Its preamble guard +still admitted only `kBF16` and `kF32`, so `IsFp8KvCache` was false at every +call and neither fp8 arm could execute; reverting the whole of the seam's +routing left the gate at 26/26 green, because every case entered through +`Qwen3_5DenseModel::Forward`. The guard is widened the same way +`qwen3_5.cpp:5313` was, and **G12** enters an fp8 cache through +`Qwen3DenseModel::Forward` — the function `ForwardQwen3ForCausalLM` calls under +`ModelRegistry::Forward` (`qwen3_dense.cpp:113`) — so the same revert now goes +red. + +**Every other architecture is refused BY NAME, and the name is usually its +OWN.** 16 architectures at 17 call sites keep their own attention preambles. +The third review traced each of them with an fp8 (`kI8`) page against a bf16 +model dtype, and the split is **14 / 1 / 1**: + +- **14 never reach the store.** Thirteen refuse at their own dtype guard — + `granite:95`, `minicpm:96`, `phi3:78`, `gemma3:121`, `opt:125`, + `stablelm:86`, `glm4:93`, `commandr:93`, `gemma:53`, `gemma2:135`, `phi:98`, + `muse_glimmer:144` and `olmo2:94`, each saying + `": KV cache must be bf16 or f32"`, which names the architecture but + neither fp8 nor the flag. `gemma4` (two sites) is the fourteenth and refuses + EARLIER and WORSE: `gemma4.cpp:306-315` takes `kv.dtype != adt`, allocates + `DBuf kcast(d, kv.dtype /* kI8 */, ...)`, and calls `vt::CastF32`, which + refuses at `src/vt/ops.cpp:4087` with `"cast_f32: out must be f32"` — a + message that names neither fp8, nor the flag, nor the architecture. +- **1 reaches the store guard.** Only `qwen3_vl:198-200` carries no guard and + no cast, so `vt::ReshapeAndCache` is what refuses it, naming + `vt::ReshapeAndCacheFp8` and saying the architecture is not routed for fp8 KV. +- **1 refuses with its own fp8-naming message.** `nemotron_h_device.cpp:1589-1593` + has an explicit `else { VT_CHECK(false, "NemotronH paged forward: ... The fp8 + KV scheme the checkpoint ships k_scale/v_scale for is a SEPARATE decision with + its own gate and is not selected here"); }` on the cast, which fires first and + is the ONE refusal in the sixteen that tells the operator what they asked for. + +Either way the failure is a sentence rather than a float path indexing a +half-sized page, which is the property that matters and which is unaffected by +the recount. What the recount changes is the message quality: the store guard's +better message is reached by 1 of the 16, not 3, and that is recorded under +`## Owed` rather than claimed away. **G7** (`test_kv_cache_fp8_wiring.cpp:910`) +hand-builds its K/V tensors and calls `vt::ReshapeAndCache` directly, so it +gates the store guard's MESSAGE and never the claim that any particular +architecture reaches it. + +### Gates + +`tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp` — **31 cases / 481 +assertions GREEN** on a CPU-only build, plus +`tests/vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp` — **3 cases / 26 +assertions GREEN**, which drives the REAL `VllmServerMain`. + +| Case | What it would let through if it were missing | +|---|---| +| G1 | the checkpoint declaration RESOLVES, an explicit flag outranks it, and the modelopt marker is exactly the three upstream can produce | +| G2 | a declared-but-absent scale collapsing into "nothing declared" | +| G3 | an fp8 page that is not EXACTLY half a bf16 page (closed form, not a ratio) | +| G4 | the same halving through the LOADER: one byte budget, 2x the blocks; and the Mamba state left alone | +| G5 | the fp8 path not being REACHED — the engine generates tokens over an fp8 cache | +| G6 | a storage dtype and an fp8 interpretation that disagree | +| G7 | an unrouted architecture writing floats into a half-sized page | +| G8 | MLA, `float16` and `fp8_e5m2` being mis-sized instead of refused | +| G9 | the store handed K and V in DIFFERENT float dtypes, which is every production weight arm | +| G10 | the loader's own resolution stanza — that it runs, which file it reads first, that the drafter-chain refusal still precedes it, and that the #1574 subject's own two documents resolve to `auto` | +| G11 | the heterogeneous per-layer specs (Gemma-4 G1b) left at full width while the pool is sized at half | +| G12 | the SHARED SEAM's fp8 routing being dead code, which it was — the guard above it admitted no fp8 cache; and, since the third review, an fp8 cache that RAN and stayed finite while holding the wrong tensor or the wrong scale in the STORE; and, since the fourth, one that dequantized with the wrong scale on the READ | +| serve | `--kv-cache-dtype` never reaching `EngineParams` from the command line | + +**G12's third case is the one that measures the cache rather than the model.** +The two logit assertions above it (`differing > 0`, `max_abs < 1.0`) were a +3000x-slack bound around a 3.4e-4 signal, and the third review walked two +mutations of `kv_cache_route.h:63` straight through a 30/30 green suite: storing +V into `k_cache` and K into `v_cache` (delta 0.0247, 73x) and storing with +`k_scale * 8` / `v_scale * 8` against an unscaled read (delta 0.0064, 19x). A +bound on the logits measures this toy model's insensitivity to its KV cache, not +the cache, and no number written on that axis would have been safe. + +The repair compares the CACHE BYTES of layer 0 against the bf16 run's page, +element by element, inside the envelope the FORMAT defines: e4m3fn has three +explicit mantissa bits and rounds to nearest even, so a normal is within +`2^-4 * |ref|` and a subnormal within `2^-10 * scale`. Layer 0 is the whole +population, because its K and V are functions of the embedding and the input +layernorm alone and the two runs therefore hand the store bit-identical floats; +from layer 1 on, the fp8 run's inputs already carry the previous layer's +dequantization. The scales are 0.125 (K) and 0.25 (V) — non-unit and unequal, so +a dropped, swapped, or one-sided scale IN THE STORE leaves the envelope. MEASURED +on the repaired tree: `0/320` elements outside, `281/320` normals over `10/10` +pages. Both mutations are RED against it — `316/320` outside at worst ratio +`416.9` for the K/V swap, `319/320` at worst ratio `13.9` for the scaled store. + +**That envelope gates the STORE, and the fourth review found the READ still +open.** The envelope decodes the cache bytes with the case's own +`vt::LoadKvFp8E4M3` and never enters the production dequant +(`cpu_paged_attn.cpp:167`), and the one value in the case that IS downstream of +that dequant carried no assertion. Every other case in the file that asserts a +number downstream of the read runs at `k_scale == v_scale == 1`, where a k/v +scale swap on the read is arithmetically inert. Mutating the production read to +`const float v_scale = args.k_scale;` therefore left `test_kv_cache_fp8_wiring` +at 31/31 and `test_ops_fp8_kv_cache` at 8/8, both SUCCESS, while every V the +softmax saw was halved. + +**The read's SCALE is closed by INVARIANCE, and EXACTLY rather than by a +tolerance** — its ROUTING is not, and that half is under `## Owed`. +e4m3fn's normal grid is relative — for `|y|` in `[2^e, 2^(e+1))` the +representable points are `m * 2^(e-3)` — and dividing by a power of two is exact +in binary floating point and shifts `e` without touching the mantissa. So for any +two power-of-two scales that both leave a value normal and unsaturated, +`s * Dequant(Quantize(x/s))` is the same float, bit for bit: the cache BYTES +differ in every element's exponent field and the floats the kernel is handed do +not. The case runs the seam twice more, at `(2^-7, 2^-13)` and `(2^-11, 2^-9)`, +and requires `memcmp`-level agreement on the logits. Both pairs move BOTH sides, +so no single wrong-scale formula reproduces itself across them and cancels out. + +**That exactness is ASSERTED for LAYER 0, and the logit equality is an EMPIRICAL +result for this fixture rather than a theorem.** The `scale_exact` `REQUIRE` +reads `bf16.buf[0]` and holds `320/320` LAYER-0 elements normal and unsaturated +at all four scales (the measured magnitudes are 1.76e-4 to 1.32e-1 for K and +5.41e-5 to 4.22e-2 for V, against all-normal windows of (2.94e-4, 1.13e-2] and +(9.43e-5, 3.46e-3]). `MakeSeamConfig` sets `num_hidden_layers = 2`, so that is +320 of the 640 elements each run stores, while the logits the case compares are a +function of BOTH layers' caches. Layer 1 does NOT satisfy the precondition: +decoding each run's own `buf[1]` at its own scales, `3/320` of its elements +disagree by up to `7.62939e-06`, all K-side and an order of magnitude below layer +0's 1.76e-4 minimum, so at `kInvAKScale = 2^-7` they fall in e4m3's SUBNORMAL +region where the grid is the absolute `2^-9` step and the power-of-two covariance +does not hold — and that layer carries 2 SATURATED elements besides, the other +escape. `CHECK(inv_differing == 0)` therefore holds by the format property for +layer 0 and by ABSORPTION for layer 1: a 7.6e-6 cache perturbation vanishing in +f32 accumulation before it reaches a logit. Extending `scale_exact` over every +layer was the preferred repair and the fixture cannot satisfy it, so the prose is +narrowed instead and the fragility is recorded under `## Owed`. + +The second anti-vacuity `REQUIRE` is that the two caches really do hold different +bytes. MEASURED on the repaired tree: `0/320` logits differ, max `|delta|` +exactly `0`. Three read-side mutations of `cpu_paged_attn.cpp:167` are RED — +`320/320` at max `|delta|` `0.0673` for `v_scale = args.k_scale`, `256/320` at +`1.08e-4` for `k_scale = args.v_scale`, and `320/320` at `2.47e-3` for +`v_scale = 1.0F` — and the store envelope above reads `0/320` under all three. +The K-side one is why this is stated as exact equality rather than a bound: the +three signals span 621x, so a bound sized against the largest keeps nothing for +the smallest. Even a tight 12.4x margin against `0.0673` puts the constant at +`5.4e-3`, which admits `1.08e-4` by 50x. An absolute `1e-4` would in fact have +CAUGHT the K-side one, by 8.4%, and a relative `1e-4` against the fixture's +largest logit (`0.0627671`) is `6.3e-6` absolute and catches it by 17x — the +argument is the 621x span and the refusal to fit a constant to whichever defect +was measured first, never that one number. + +A bf16-versus-fp8 comparison through `LoadedEngine` in G5/G9 was considered and +is NOT what closes this. Both mutations left the whole suite green EXCEPT the new +case, G5 and G9 included, which is the direct measurement that an engine-level +token or determinism comparison cannot see a dequant defect this model absorbs. + +G4, G5, G9 and G10 enter through the production entry point (the `LoadedEngine` +constructor or `LoadedEngine::FromModelDir` → `MakeKVCacheResolved` → +`ApplyResolvedCacheDType` → `ResolveNumBlocks` → the runner → +`Qwen3_5DenseModel::Forward`), not by constructing a spec or a `PagedKvCache` by +hand. The `serve` cases enter one step earlier still, at `argv`. + +**G12 enters through the OTHER registered forward**, and states its harness +adaptation once. `LoadedEngine` has no in-memory overload for +`Qwen3DenseWeights`, so its paged buffers are allocated by the case rather than +by the runner — at the width the cache dtype names, one byte per element for +fp8 and two for bf16, because allocating the fp8 arm at the float width would +hide the very mis-sizing this row exists to prevent. Everything else is +production code entered through `Qwen3DenseModel::Forward`, which is what +`ForwardQwen3ForCausalLM` calls under `ModelRegistry::Forward` +(`qwen3_dense.cpp:113`); this is the same entry +`tests/vllm/models/test_qwen3_break_point.cpp` uses for the graph-break +reachability gate. + +**G9 is the case the first version of this gate could not have.** Every case in +the file built its model with `MakeDenseWeights`, whose projection weights carry +no `nk` flag, so `ProjectFullAttnQkv` served them through `MatmulF32D` and the V +projection came out f32 — the same dtype the attention preamble gives K. A real +checkpoint ships raw torch Linear weights ([N=out, K=in], `nk`), which take +`MatmulBf16D` and emit the MODEL dtype, and the fp8 store then saw an f32 K +beside a bf16 V and refused the pair by name at the first forward. That is true +of the block-wise fp8 arm (`MatmulFp8BlockScaledD`), the NVFP4 arm under the +default `VT_BF16_GEMM_OUT`, and ordinary bf16 safetensors — which includes the +#1574 campaign checkpoint. The fix normalises both to bf16 on the fp8 route +exactly as the bf16-cache route already did, and bf16 rather than f32 because +bf16 is the dtype upstream quantizes from: its model IS bf16 where +`reshape_and_cache_flash` takes key/value (`cache_kernels.cu:314-401`). + +**G10 also holds an ORDERING that a merge can silently drop.** `main`'s +SPEC-DRAFTER-CHAIN W1 refusal (#1522) and this row's resolution stanza both +insert as the first statement of `FromModelDir`, and both are load-bearing: that +row's G5 requires the chain refusal before any weight I/O, and +`ReadQuantConfigJson` opens a file inside `model_dir`. Taking either side of that +textual conflict drops a guarantee. +`tests/vllm/entrypoints/test_drafter_chain_reach.cpp` cannot see an inversion, +because it points at a NONEXISTENT directory and `ReadQuantConfigJson` answers "" +for one of those without opening anything. G10's ordering case points at a +directory that EXISTS and declares fp8, so an inverted order announces the +declaration first and that line is the evidence. + ## Owed - **The W2 device gates are UNEXECUTED** (#1593). `tests/vt/test_cuda_fp8_kv_cache.cpp` @@ -306,12 +645,173 @@ of the case; the production path still goes through it, and G1/G1b/G2 gate that. owes the "CUDA TUs are UNCOMPILED" clause in [`.agents/engine-matrix.md`](../engine-matrix.md) and [`.agents/quantization-matrix.md`](../quantization-matrix.md). -- **Nothing reaches the fp8 KV path from a production entry point yet**, on - either backend. `vt::ReshapeAndCacheFp8` and `PagedAttentionArgs::kv_cache_dtype` - have no caller outside their tests; W1 landed in that state and W2 does not - change it. **`KV-FP8` W3 owns the wiring** — half-sized KV blocks in the runner, - `--kv-cache-dtype` threaded from the CLI, and the checkpoint `k_scale`/`v_scale` - path — and it is tracked by #1593 alongside W2. +- **RESOLVED by W3 on the CPU leg, still owed on the device.** W1 and W2 landed + with nothing reaching the fp8 KV path from a production entry point on either + backend. W3 wires it: `--kv-cache-dtype fp8` on the server flag now sizes + half-width KV blocks and the routed attention blocks call + `vt::ReshapeAndCacheFp8` and the scaled read, gated end to end on CPU by + `test_kv_cache_fp8_wiring` G5. The CUDA arm rides the SAME `PagedKvCache` + fields and the same two routing helpers, so it is wired by construction — and + it is still UNMEASURED for the reason the two bullets above give. +- **W3: the C ABI does not expose `--kv-cache-dtype`** (#1593). The flag reaches + the engine through `src/vllm/entrypoints/openai/server_main.cpp` and + `EngineParams::kv_cache_dtype`, and NOT through `include/vllm.h` / + `src/capi/vllm_c.cpp` / `examples/cli/main.cpp`, which the W3 dispatch's + authority did not cover (it named `src/vllm/**`, `include/vllm/**`, + `src/vt/**`). AGENTS.md requires every shipped capability to be reachable from + `include/vllm.h`, so this is debt and not a design: a C-ABI caller cannot ask + for an fp8 KV cache today unless the CHECKPOINT declares one, which the loader + does honour on every path including that one. The ABI field, its version bump + and its `test_capi` case are owed here. +- **W3: 16 architectures are refused rather than routed, at 17 call sites** + (#1593). W3 routes the shared seam `dense_attn::AttnBlock` and + `src/vllm/model_executor/models/qwen3_5.cpp`. The other direct + `vt::ReshapeAndCache` call sites — `glm4`, `minicpm`, `opt`, `gemma`, + `gemma2`, `gemma3`, `gemma4` (two sites), `commandr`, `phi`, `phi3`, + `muse_glimmer`, `stablelm`, `qwen3_vl`, `olmo2`, `granite` and + `nemotron_h_device` — keep their own attention preambles and refuse + `--kv-cache-dtype fp8` by name. Routing each is one call swapped for + `dense_attn::WriteKvCache` plus one `ApplyKvCacheQuant`, and each needs its + own gate. +- **W3: 15 of those 16 refuse with a message that names neither fp8 nor the + flag** (#1593). The refusal was first described as arriving at + `vt::ReshapeAndCache`, and the third review traced every arm with a `kI8` page + against a bf16 model dtype. It arrives there for exactly ONE architecture. + Thirteen carry their own `": KV cache must be bf16 or f32"` guard, which + fires first — `granite:95`, `minicpm:96`, `phi3:78`, `gemma3:121`, `opt:125`, + `stablelm:86`, `glm4:93`, `commandr:93`, `gemma:53`, `gemma2:135`, `phi:98`, + `muse_glimmer:144` and `olmo2:94`. `gemma4:306-315` is the fourteenth and is + the worst of the set: it casts into a `kI8` destination and dies inside + `vt::CastF32` (`src/vt/ops.cpp:4087`, `"cast_f32: out must be f32"`), which + names neither fp8, nor the flag, nor `gemma4`. Only `qwen3_vl:198-200` reaches + the store guard and gets the message that names `vt::ReshapeAndCacheFp8` and + the unrouted architecture. `nemotron_h_device:1589-1593` is the sixteenth and + the only good refusal in the set: an explicit `VT_CHECK(false, ...)` on the + cast that names the fp8 KV scheme and says it is not selected here. The SAFETY + property holds for all sixteen — nothing writes floats into a half-sized page + — but an operator who typed `--kv-cache-dtype fp8` on one of the 15 is told a + dtype or cast rule rather than what they asked for. Widening those 15 the way + `dense_attn_block.h:358` and `qwen3_5.cpp:5313` were widened is the same edit + that routes them, so this is owed together with the bullet above rather than + separately. +- **W3: G12's read-side exactness is ASSERTED for LAYER 0, and the logit + equality is EMPIRICAL for this fixture** (#1593). The `scale_exact` `REQUIRE` + decodes `bf16.buf[0]`, so it holds e4m3's normal-and-unsaturated precondition + on 320 elements. `MakeSeamConfig` sets `num_hidden_layers = 2`, so each run + stores 640, and `CHECK(inv_differing == 0)` compares LOGITS that are a function + of both layers' caches. Layer 1 measured `3/320` elements that dequantize + differently between the two invariance runs, by up to `7.62939e-06` — all + K-side, an order of magnitude below layer 0's 1.76e-4 minimum, and therefore + SUBNORMAL at `kInvAKScale = 2^-7`, where e4m3 rounds on the absolute `2^-9` + grid and the power-of-two covariance argument does not hold; the same layer + carries 2 SATURATED elements, the other escape. That `CHECK` passes for layer 1 + by ABSORPTION — a 7.6e-6 perturbation vanishing in f32 accumulation before it + reaches a logit — and not by the format property. It is not knife-edge today: a + third legal pair `(2^-9, 2^-11)` against pair A also measures `0/320` logits + differing at max `|delta|` `0`. It is fragile in one specific way: a change to + `MakeSeamWeights`, `kSeamTokens`, `num_hidden_layers`, the thread count or the + accumulation order could push a layer-1 discrepancy into a logit and redden a + CORRECT tree with a defect-shaped message. The repair is to extend + `scale_exact` over every layer, which needs a fixture whose layer-1 K clears + e4m3's smallest normal at every invariance scale — `2^-6 * 2^-7 = 1.22e-4` + against the measured 7.6e-5 is the margin that is missing. +- **W3: the fp8-vs-fp8 read comparison closes the SCALE half of the read and not + the ROUTING half** (#1593). G12's invariance case compares two fp8 runs, so it + is structurally blind to a read-side defect that is a function of BYTES and + INDICES rather than of scales: both runs commit it identically and it cancels. + Two mutations of production code were measured and both PASS the whole file at + `31/31`, `487/487`, with `0/320 logits differ` — serving V out of the K page + with K's scale (`cpu_paged_attn.cpp:174` `v_base = k_cache.data` together with + `:167` `v_scale = args.k_scale`), which is the read-side twin of the + `N1_KVSWAP` store mutation the envelope DOES catch; and dropping the in-page + token offset from the V read (`:270`, `off & 0`), which is pure indexing and + carries no scale at all. Closing that class needs a comparison against a + REFERENCE — the bf16 run's page, or the case's own decode through the same + math — and never a second fp8 run. The design paragraph above therefore claims + the SCALE half only, and this bullet is the other half. +- **W3: no weight loader extracts `k_scale`/`v_scale`** (#1593). `ResolveKvCacheScales` + mirrors all four of upstream's arms, and the loader calls it with the + `KVCacheScaleParameter` unloaded sentinel for both scales, so every declaring + checkpoint lands on `kDeclaredButAbsent` and serves at 1.0. That is CORRECT for + the #1574 gate checkpoint under an explicit `--kv-cache-dtype fp8`, since it + ships zero KV scales (and it never reaches the arm without the flag, because + its inline `config.json:quantization_config` declares no `kv_cache_*` key) — + but the two + checkpoint-loaded arms (`kCheckpoint`, `kCheckpointKvScale`) are unit-gated and + unreached, and a calibrated checkpoint would silently serve uncalibrated. The + per-layer scale-tensor read, and a per-layer (rather than per-engine) scale on + `AttentionSpec`, are owed. +- **W3: `--calculate-kv-scales` is refused, not implemented** (#1593). Upstream's + dynamic on-the-fly scale (`config/cache.py:111`) is deprecated there for + removal in v0.19; `ResolveKvCacheScales` refuses it BY NAME rather than + silently taking the static arm, and no flag exposes it. +- **W3: an fp8 KV cache is served by the SLOW attention kernels only** (#1593). + `fa2_prefill` and `fa2_decode` in `src/vllm/model_executor/models/qwen3_5.cpp` + both require `kv.dtype == DType::kBF16`, and + `src/vt/cuda/cuda_paged_attn.cu` routes an fp8 read to tiled prefill and block + decode only. Every other launcher — the WMMA ladder, the vendored FA-2 + kernels, the vectorized decode-opt and GQA kernels — is bf16-native by + construction and would need an fp8 instantiation. Owed by this row (`KV-FP8`) + under #1593. Until it lands, `--kv-cache-dtype fp8` trades attention + throughput for KV bytes at an unmeasured exchange rate, and the row says so in + `## W3`, in `docs/USAGE.md` and in `docs/FEATURES.md` rather than implying the + memory win is free. +- **W3: the heterogeneous per-layer fp8 arm is SIZING-only** (#1593). + `ApplyCacheDType` retypes `KVCacheConfig::per_layer_attn_specs` as well as the + group specs, which is what keeps `KVBytesPerBlock` and the runner's own + per-layer allocation agreeing, and G11 gates that arithmetic. No shipped + architecture can spend it yet: the only model that populates that vector is + Gemma-4 (G1b), and Gemma-4 is one of the 16 architectures whose attention block + refuses the fp8 store by name. So a Gemma-4 run with `--kv-cache-dtype fp8` + gets a correctly halved pool and then a named refusal at the first forward, + which is the intended order. Routing Gemma-4 is owed with the other 15 above. +- **W3: the resolver accepts one producer-only document upstream REFUSES** + (#1593). `GetKvCacheQuantAlgoString` takes the modelopt marker from + `producer["name"] == "modelopt"`, which mirrors the injection + `ModelArchConfigConvertorBase._normalize_quantization_config` + (`transformers_utils/model_arch_config_convertor.py:208-247`) performs before + `torch_utils.py:319` ever reads the key — RUN, not transcribed, on 2026-08-22: + `nvidia/Llama-3.3-70B-Instruct-FP8`'s producer-only `hf_quant_config.json` + answers `None` before normalization and `'fp8_e4m3'` after it, and G1's marker + case pins all three markers. + + **The third review found that acceptance was UNGUARDED, and W3 now mirrors the + guard.** `:224` injects only `if quant_algo is not None`, and it reads that key + out of `quant_cfg.get("quantization", {})` — an EMPTY-object fallback, unlike + the reader's `quant_cfg.get("quantization", quant_cfg)` at + `torch_utils.py:321`. Three documents therefore answer `None` upstream and used + to answer `fp8_e4m3` here: a `producer.name` of `modelopt` with no + `quant_algo`, a `quantization.modelopt_quant_config` with no `quant_algo`, and + a TOP-LEVEL `modelopt_quant_config` with no `quantization` key at all. All + three are now refused, arm (g) of G1's marker case pins each of them together + with the same document made acceptable by adding the `quant_algo`, and + restoring the pre-repair predicate turns seven of that case's assertions red. + No shipped fixture moved: every real document in this suite — + `kGateCheckpointQuantConfig`, `kNoKvDeclarationQuantConfig`, + `kInlineWeightsOnlyQuantConfig` and the #1574 subject's own two files — carries + a `quant_algo`. + + **One arm of that injection is still not mirrored:** + upstream RAISES `ValueError: Unknown ModelOpt quant algo: ` (`:235`) + when the producer is modelopt and the nested `quant_algo` is neither + FP8-family nor NVFP4, and we answer `fp8_e4m3` instead. This is why the + mirrored test above is exactly `quant_algo is not None` rather than the family + set: the only two upstream outcomes for a `quant_algo` that IS present are + "inject" and "raise", and taking the raise's arm collapses them into one. That + refusal is a + WEIGHT-half validation living in a config convertor this port does not have, + and moving it into the KV resolver would refuse a `MIXED_PRECISION` + checkpoint whose weights `modelopt_mixed_precision.h` loads. It is unreachable + for the #1574 subject, whose inline `config.json` document wins, so it is + recorded here rather than implemented; porting the convertor's validation is + its own row. +- **W3: no engine auto-selects fp8 KV for the #1574 subject** (#1593), and the + campaign has to type `--kv-cache-dtype fp8` on both sides. Not a defect — the + mirror is correct on both halves and the competitors' `serve.sh` already + passes the flag — but it means the checkpoint declaration path this row + builds is gated by G1/G10 rather than exercised by the benchmark it was built + for. A calibrated ModelOpt checkpoint that declares the algorithm INLINE is + what would exercise it end to end, and this row has none. - **Metal and ROCm have no fp8 KV arm.** Both refuse by name (see above). Neither has a row yet; they belong with W5's per-head/e5m2 work or a backend row. - fp8_e5m2 and per-attention-head scales stay refused on both backends (W5). diff --git a/.agents/specs/qwen38-27b-quant-arms.md b/.agents/specs/qwen38-27b-quant-arms.md index d877dc26e..9d86d5ca9 100644 --- a/.agents/specs/qwen38-27b-quant-arms.md +++ b/.agents/specs/qwen38-27b-quant-arms.md @@ -1828,9 +1828,13 @@ The restored tree is green at 22 cases / 1687 assertions. - **A locally computed sha256, and mirrored bytes.** Named under `## Owed`. - **Routing by the declared algorithm.** Named under `## Owed`. - **The FP8 KV arm.** `hf_quant_config.json` asks for `kv_cache_quant_algo: - "FP8"` and the checkpoint ships zero `k_scale`/`v_scale`; no production path - in this tree reads `hf_quant_config.json` at all, so the declaration is - invisible to the loader rather than ignored by it. Owned by `KV-FP8` + "FP8"` and the checkpoint ships zero `k_scale`/`v_scale`. `KV-FP8` W3 gave + that file its first production reader, `vllm::ReadQuantConfigJson` + (`src/vllm/config/cache.cpp:207`) under `LoadedEngine::FromModelDir`, but only + as the legacy fallback behind `config.json`'s inline `quantization_config` + (`config.py:751-761`). This artifact ships that inline document, so its legacy + file is never opened and the declaration stays invisible to the loader rather + than ignored by it. Owned by `KV-FP8` ([#1593](https://github.com/mudler/vllm.cpp/issues/1593)) and named under `## Owed`. W5 does not refuse it, because the identical declaration in `nvidia/Qwen3.6-27B-NVFP4`'s `config.json` would then refuse a gate model. @@ -2067,9 +2071,14 @@ them: `r0b0tlab/...-MTP-sm121` sets `kv_cache_quant_algo: "FP8"` in `hf_quant_config.json` and ships ZERO `k_scale`/`v_scale`; `nvidia/Qwen3.6-27B-NVFP4` declares an equivalent `kv_cache_scheme` in its - `config.json` and also ships zero. No production path in this tree reads - `hf_quant_config.json`, and this tree has no quantized KV cache to apply - either declaration to. Owned by `KV-FP8`, tracked by + `config.json` and also ships zero. `KV-FP8` W3 landed the first production + reader of `hf_quant_config.json` — `vllm::ReadQuantConfigJson` + (`src/vllm/config/cache.cpp:207`) under `LoadedEngine::FromModelDir` — but it + is the legacy fallback behind `config.json`'s inline `quantization_config` + (`config.py:751-761`), and BOTH artifacts carry that inline document, so + neither declaration reaches it, and no weight loader extracts the + `k_scale`/`v_scale` a calibrated checkpoint would ship. Owned by `KV-FP8`, + tracked by [#1593](https://github.com/mudler/vllm.cpp/issues/1593) — named here rather than refused, because refusing it would refuse a gate model this tree loads and measures today. diff --git a/CMakeLists.txt b/CMakeLists.txt index 0e7add54e..5ae1122ee 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -734,6 +734,7 @@ add_library(vllm STATIC src/vllm/sampling_params.cpp src/vllm/lora/punica_cpu.cpp src/vllm/lora/layers.cpp + src/vllm/config/cache.cpp src/vllm/config/scheduler.cpp src/vllm/config/device.cpp src/vllm/config/kv_transfer.cpp diff --git a/docs/FEATURES.md b/docs/FEATURES.md index f81c6e88c..aa65f801f 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -52,7 +52,7 @@ are our reading of their documented behavior, not measurements. | Block-paged KV with refcount and LRU evict | ✅ | ✅ | ✅ | ◐ | | Hybrid KV groups (full attention + GDN/Mamba) | ◐ GDN gate activation resolved from the checkpoint's `output_gate_type` (silu/swish/sigmoid; anything else refused at load, #489) | ✅ | ◐ | ◐ | | Sliding-window and chunked-local attention | ◐ | ✅ | ✅ | ✅ | -| fp8 KV cache | ◐ e4m3 store + read dequant on CPU and CUDA (#1593); nothing serves it yet: no runner block sizing and no `--kv-cache-dtype`. Metal/ROCm refused by name. CUDA gate UNRUN ([spec](../.agents/specs/fp8-kv-cache.md)) | ✅ | ✅ | ✅ | +| fp8 KV cache | ◐ `--kv-cache-dtype fp8` halves the block, so a fixed `--kv-cache-memory` buys 2x the blocks and the DEFAULT 256-block path halves the pool bytes instead. Costs the bf16-native FA-2/WMMA/vector kernels (net UNMEASURED). 16 archs, MLA, the C ABI are refused before any write; only 1 arch names fp8 back. CUDA UNRUN ([spec](../.agents/specs/fp8-kv-cache.md)) | ✅ | ✅ | ✅ | | KV offload to host memory | ✅ | ✅ | ✅ | ☐ | | External KV provider ABI (LMCache) | ☐ | ✅ | ◐ | ☐ | | KV events (block create / evict publish) | ◐ no transport | ✅ | ☐ | ☐ | diff --git a/docs/USAGE.md b/docs/USAGE.md index 643780843..c01bd23d3 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -119,6 +119,81 @@ reports every file as `already in the cache` and transfers no bytes. Before nothing at all, because the hub answers with a relative `Location` header that the client read as a URL. +### Halve the KV cache with `--kv-cache-dtype fp8` + +Store the paged K/V as 1-byte fp8-e4m3 instead of 2-byte bf16. The KV block +halves, so the same memory budget holds twice the context: + +```sh +build/examples/vllm-server \ + --model /path/to/model \ + --kv-cache-dtype fp8 \ + --kv-cache-memory 8589934592 +``` + +Values are vLLM's own `CacheDType` names. `auto` is the default and uses the +model dtype. `fp8` and `fp8_e4m3` select the quantized store; `bfloat16` names +the default storage dtype explicitly. `float16` and `fp8_e5m2` parse and are +then refused by name, because no attention block writes either yet. + +**The checkpoint can ask for it.** When you pass no flag, the server reads the +checkpoint's `config.json` `quantization_config` (falling back to a standalone +`hf_quant_config.json`, which is what ModelOpt 0.29.0 and before wrote) and +honours a declared `kv_cache_quant_algo`, printing one line naming what it +resolved. An explicit `--kv-cache-dtype` always wins over the declaration. Both +the order of the two files and the precedence mirror vLLM. + +Check which document your checkpoint declares in before you rely on this. A +repository can carry a current `config.json` beside a stale +`hf_quant_config.json` that disagrees with it, and the inline one is the one +that counts — on this server and on vLLM. `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` +is exactly that shape: only its legacy file mentions the KV cache, so neither +engine turns fp8 KV on for it and the flag has to be typed. + +Note that `--kv-cache-memory` is what turns the halved block into twice the +pool. Without it the server falls back to a fixed block count, and `fp8` then +halves the KV bytes for the same context instead. + +**It costs you the fast attention kernels, and we have not measured the net.** +An fp8 KV cache is read by the tiled prefill and block decode kernels only. +FA-2 prefill, all three FA-2 decode topologies, the WMMA ladder and the +vectorized decode-opt/GQA kernels are bf16-native and are skipped whenever the +cache is not bf16. So this flag buys half the KV bytes and twice the pool, and +spends an unmeasured amount of attention throughput to do it. Which way the sum +goes depends on your model, context length and concurrency; measure your own +workload both ways rather than assuming the memory win is free. + +**Accuracy.** A checkpoint that declares fp8 KV but ships no `k_scale`/`v_scale` +tensors serves on the default scale 1.0, and the server says so on stderr. That +is the documented default, not a silent one — and a checkpoint that declares +nothing never reaches it. + +**Coverage.** The store and the scaled read are routed for the Qwen3.5/3.8 +family and for the shared dense-attention seam, which serves Qwen3 dense, +Qwen3-MoE, Voxtral and the Llama, Mistral and InternLM2 registries. The other 16 +architectures carry their own attention preamble and refuse before writing +anything, rather than writing floats into a half-sized block. Only one of them +(Nemotron-H) tells you what you asked for: its refusal names the fp8 KV scheme. +Qwen3-VL reaches the store, which names the op that should have been called and +says the architecture is not routed for fp8 KV. The other 14 report a dtype rule +instead — 13 say `": KV cache must be bf16 or f32"`, and Gemma-4 dies one +step earlier inside a cast with `"cast_f32: out must be f32"`, which does not +even name the architecture. Every one of the 16 refuses before writing, so the +half-sized block is never fed floats; what differs is how much the message tells +you. Metal and ROCm refuse it too. See +[the row spec](../.agents/specs/fp8-kv-cache.md) for the exact list. + +A refusal arrives AFTER the pool has already been sized at half, which is the +intended order: the sizing is what a wrong answer would corrupt silently, so it +is made consistent first and the unrouted store then says so out loud. On a +heterogeneous-KV model such as Gemma-4, where each layer carries its own +attention spec, that means you see a doubled block count in the startup line and +then a named refusal at the first forward — not a served run. + +**Not on the C ABI yet.** `vllm_model_params` carries no `kv_cache_dtype` field, +so a C-ABI caller reaches the fp8 cache only through a checkpoint that declares +it. Tracked by [#1593](https://github.com/mudler/vllm.cpp/issues/1593). + ## Draft with a second checkpoint Speculative decoding runs a small draft model beside the target and verifies its diff --git a/include/vllm/config/cache.h b/include/vllm/config/cache.h new file mode 100644 index 000000000..bdf775a91 --- /dev/null +++ b/include/vllm/config/cache.h @@ -0,0 +1,94 @@ +// Ported from: vllm/config/cache.py @ 555967922 (CacheDType:19-36, +// cache_dtype:76, calculate_kv_scales:111) plus the two resolvers +// that turn a checkpoint's declaration into that string: +// vllm/utils/torch_utils.py:64-67 (MODELOPT_TO_VLLM_KV_CACHE_ +// DTYPE_MAP), :310-362 (get_kv_cache_quant_algo_string) and +// :374-392 (resolve_kv_cache_dtype_string). +// +// This is the CONFIG half of `KV-FP8` W3: how `--kv-cache-dtype` and the +// checkpoint's own `kv_cache_quant_algo` combine into ONE resolved CacheDType +// string, before anything sizes a block or writes a byte. `include/vllm/v1/ +// kv_cache_dtype.h` (W1) then turns that string into a storage dtype and an fp8 +// interpretation; this file decides WHICH string it is handed. +// +// THE ORDER IS UPSTREAM'S AND IT MATTERS. `resolve_kv_cache_dtype_string` +// (`torch_utils.py:374-392`) returns an explicit user value UNCHANGED and only +// consults the checkpoint when the user said "auto". So `--kv-cache-dtype +// bfloat16` on an FP8-declaring checkpoint serves bf16 — the operator wins — +// and `--kv-cache-dtype fp8` on a checkpoint that declares nothing is likewise +// honoured. `attention.py:279-290` re-applies the same precedence defensively +// and says so in its own comment ("an explicit choice (e.g. bfloat16) must +// win"). +// +// WHAT A DECLARATION IS AND IS NOT. `Declared()` answers "did the checkpoint ask +// for a quantized KV cache", and it is a DIFFERENT question from "did the +// checkpoint ship k/v scales". A checkpoint can answer yes to the first and no +// to the second; see +// `include/vllm/model_executor/layers/quantization/kv_cache.h` for why the two +// must not collapse into one fallthrough. +// +// `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` @ `36f717a2` — the #1574 campaign +// subject — is NOT that checkpoint, although its legacy `hf_quant_config.json` +// reads like it: its `config.json:quantization_config` declares no +// `kv_cache_*` key, and the inline document is the one that is read +// (`transformers_utils/config.py:751-761`). It therefore answers NO to the +// first question on both engines, and an fp8 KV run of it needs +// `--kv-cache-dtype fp8` typed explicitly on each side. MEASURED from the live +// artifact 2026-08-22. +#ifndef VLLM_CONFIG_CACHE_H_ +#define VLLM_CONFIG_CACHE_H_ + +#include +#include + +namespace vllm { + +// vllm/config/cache.py:76 — "Data type for kv cache storage. If 'auto', will use +// model data type." The default every surface starts from. +inline constexpr const char* kDefaultCacheDType = "auto"; + +// The outcome of resolving `--kv-cache-dtype` against the checkpoint. +struct ResolvedCacheDTypeString { + // The CacheDType string to hand to `vllm::v1::ParseCacheDType`. + std::string cache_dtype = kDefaultCacheDType; + // True when `cache_dtype` came from the CHECKPOINT's `kv_cache_quant_algo` + // rather than from the caller — i.e. the caller said "auto" and the checkpoint + // declared a KV quantization algorithm (`torch_utils.py:381-390`). False both + // when the caller named a dtype and when nothing declared one, because those + // are different facts from this one and the callers below need to tell them + // apart. + bool declared_by_checkpoint = false; +}; + +// `torch_utils.py:310-362` get_kv_cache_quant_algo_string. `quant_config_json` +// is the raw text of the checkpoint's `hf_quant_config.json` (or the +// `quantization_config` object out of `config.json`); an empty string means the +// checkpoint has neither. Returns nullopt when no algorithm is declared, and +// "auto" when one is declared in a shape this port does not recognize — +// upstream's own safe fallback, which it reaches with a warning rather than by +// guessing. +// +// Only `quant_method` values that start with "modelopt" are read, exactly as +// upstream (`:319`): the compressed-tensors `kv_cache_scheme` route is a +// different surface (`attention.py:283-290`) and is owed, not silently folded +// in here. +std::optional GetKvCacheQuantAlgoString( + const std::string& quant_config_json); + +// `torch_utils.py:374-392` resolve_kv_cache_dtype_string. `requested` is what +// the operator typed (or `kDefaultCacheDType`); `quant_config_json` is as above. +ResolvedCacheDTypeString ResolveKvCacheDTypeString( + const std::string& requested, const std::string& quant_config_json); + +// Read the `quantization_config` object inside `config.json` from a model +// directory, falling back to a standalone `hf_quant_config.json`. That order is +// upstream's (`transformers_utils/config.py:751-761`) and it is the inline +// document first: ModelOpt writes it into `config.json` from 0.31.0 on, and the +// separate file is what 0.29.0 and before wrote. Returns "" when the directory +// holds neither — which is not an error: most checkpoints declare no +// quantization at all. +std::string ReadQuantConfigJson(const std::string& model_dir); + +} // namespace vllm + +#endif // VLLM_CONFIG_CACHE_H_ diff --git a/include/vllm/entrypoints/model_loader.h b/include/vllm/entrypoints/model_loader.h index 42966cc74..a795a1cc6 100644 --- a/include/vllm/entrypoints/model_loader.h +++ b/include/vllm/entrypoints/model_loader.h @@ -109,6 +109,19 @@ struct EngineParams { // count directly and IGNORES gpu_memory_utilization, mirroring vLLM // CacheConfig.kv_cache_memory_bytes (cache.py:182,189). int64_t kv_cache_memory_bytes = 0; + // KV-cache STORAGE dtype, mirroring vLLM CacheConfig.cache_dtype + // (config/cache.py:19-36,76) and its `--kv-cache-dtype` flag. "auto" (the + // default) uses the model dtype and is byte-identical to before this field + // existed; "fp8"/"fp8_e4m3" stores 1-byte fp8 K/V, which HALVES the bytes per + // KV block and therefore doubles the pool at a fixed --kv-cache-memory. + // + // TWO-STAGE, exactly as upstream. `FromModelDir` RESOLVES this string once + // against the checkpoint's own `kv_cache_quant_algo` before anything reads it + // (`vllm::ResolveKvCacheDTypeString`, mirroring `arg_utils.py:1915-1918`), so + // every consumer downstream sees an already-resolved value and "auto" there + // means "nothing declared it either". An explicit value is never overridden by + // the checkpoint (`torch_utils.py:380-381`). + std::string kv_cache_dtype = "auto"; int max_model_len = 0; // 0 => config.max_position_embeddings. // max concurrent sequences. vLLM's default is 1024 (EngineArgs.max_num_seqs); // ours was 8, which put c8 EXACTLY on the batch ceiling so the 8th stream @@ -345,8 +358,13 @@ class LoadedEngine { // Load config.json + tokenizer.json + *.safetensors from `model_dir` and build // the stack. Throws std::runtime_error on any load failure (bad path, missing // shards, unparseable config). - static std::unique_ptr FromModelDir(const std::string& model_dir, - const EngineParams& params); + // + // KV-FP8 W3: this is where `params.kv_cache_dtype` is RESOLVED against the + // checkpoint's own `kv_cache_quant_algo` (mirroring `arg_utils.py:1915-1918`); + // the direct constructors below take the field verbatim because they are + // handed in-memory weights and have no checkpoint directory to ask. + static std::unique_ptr FromModelDir( + const std::string& model_dir, const EngineParams& params_in); // ── The `clip` mmproj vision tower (row `LOAD-GGUF-MMPROJ`, issue #821) ─── // @@ -459,6 +477,11 @@ class LoadedEngine { // the enablement gate can assert the C-ABI/C++/flag toggle took effect. bool jump_forward_enabled() const { return jump_forward_enabled_; } const vllm::v1::GPUModelRunner& runner() const { return runner_; } + // KV-FP8 W3: the RESOLVED KV-cache config — the block count the sizing knobs + // produced and the group specs carrying the storage dtype `--kv-cache-dtype` + // selected. Exposed so a gate reads what the loader actually sized instead of + // re-deriving the arithmetic it is supposed to be checking. + const vllm::v1::KVCacheConfig& kv_cache_config() const { return kv_cfg_; } // KV-EXTERNAL-CACHE (LMCache): the wired external KV connector, or null when // none was configured. Exposed so the output-invariance gate can read the @@ -571,6 +594,15 @@ class LoadedEngine { const LoadedModel& model, const HfConfig& config, int block_size, const EngineParams& params, const std::optional& spec); + // KV-FP8 W3: turn the (already checkpoint-resolved) `params.kv_cache_dtype` + // into the KV specs' storage dtype, fp8 interpretation and per-tensor scales. + // Runs on the PROBE config before ResolveNumBlocks reads its geometry, which + // is what makes an fp8 cache double the block count rather than halve the + // pool. A no-op on the "auto"/bf16 default. + static void ApplyResolvedCacheDType(const EngineParams& params, + vllm::v1::KVCacheConfig& cfg); + // The `kv_cache.py:150-156` uncalibrated-scale warning, once per LOAD. + static void WarnUncalibratedKvScales(const EngineParams& params); // Ensure NONE_HASH is initialized before the scheduler/hasher are built // (upstream global init). Idempotent; runs as the first member initializer. static bool EnsureNoneHash(); diff --git a/include/vllm/model_executor/layers/quantization/kv_cache.h b/include/vllm/model_executor/layers/quantization/kv_cache.h new file mode 100644 index 000000000..21ec5cd1e --- /dev/null +++ b/include/vllm/model_executor/layers/quantization/kv_cache.h @@ -0,0 +1,170 @@ +// Ported from: vllm/model_executor/layers/quantization/kv_cache.py @ 555967922 +// +// `BaseKVCacheMethod`'s k/v-scale resolution, which is the half of the fp8-KV +// feature that is NOT arithmetic: deciding which scale the store and the read +// multiply by, and — the part this file exists for — deciding whether there was +// an fp8 KV cache to scale at all. +// +// THE TWO STATES A FALLTHROUGH CANNOT TELL APART. `r0b0tlab/Qwen3.8-27B-NVFP4- +// MTP-sm121` @ `36f717a2` declares `kv_cache_quant_algo: "FP8"` in +// `hf_quant_config.json` and ships ZERO `k_scale`/`v_scale` tensors — 2001 +// entries in `model.safetensors.index.json`, none of them a KV scale (measured +// 2026-08-21 from the public index). It therefore serves on the DEFAULT scale +// 1.0, and so would a checkpoint that declared no KV quantization at all if the +// default were reached by falling off the end of a missing-tensor lookup. The +// two produce identical output today and stop being the same the moment a +// checkpoint declares nothing: one is a documented default, the other is an +// invented scale for a cache nobody asked to quantize. +// +// Upstream keeps them apart STRUCTURALLY rather than by a sentinel, and this +// file mirrors that. `process_weights_after_loading` reaches the scale block at +// all only under `is_quantized_kv_cache(layer.kv_cache_dtype)` +// (`kv_cache.py:100-102`); INSIDE that block, both scales still holding the +// `KVCacheScaleParameter` sentinel `-1.0` (`kv_cache.py:18-30`) is the separate +// "no scales were loaded" arm that takes 1.0 and warns (`:112-116`, `:150-156`). +// So `kNotQuantized` is not a value on the same axis as `kDeclaredButAbsent`: +// it is the state in which asking for a scale is a question that was never +// posed. `ScalesForFp8Store` refuses it by name rather than answering 1.0. +#ifndef VLLM_MODEL_EXECUTOR_LAYERS_QUANTIZATION_KV_CACHE_H_ +#define VLLM_MODEL_EXECUTOR_LAYERS_QUANTIZATION_KV_CACHE_H_ + +#include +#include +#include + +#include "vllm/v1/kv_cache_dtype.h" +#include "vt/dtype.h" + +namespace vllm { + +// kv_cache.py:18-30 `KVCacheScaleParameter.__new__` — the scalar parameter is +// initialized to -1.0, an INVALID sentinel, so "absent" is a value the resolver +// can read rather than an absence it has to infer. Mirror the constant, because +// every branch below keys on `< 0.0` exactly as upstream's does. +inline constexpr float kKvScaleUnloaded = -1.0F; + +// Which of `process_weights_after_loading`'s arms produced the pair. The point +// of naming all four is that `kNotQuantized` and `kDeclaredButAbsent` carry the +// same NUMBERS and are different FACTS. +enum class KvScaleOrigin { + // is_quantized_kv_cache(kv_cache_dtype) == false (kv_cache.py:100-102): the + // scale block never ran. There is no fp8 cache, so there is no scale — not a + // scale of 1.0. + kNotQuantized, + // kv_cache.py:104-111 — both `k_scale` and `v_scale` came from the checkpoint. + kCheckpoint, + // kv_cache.py:117-127 — the checkpoint carried a single `kv_scale`, remapped + // to `k_scale` at load and duplicated to `v_scale` here. + kCheckpointKvScale, + // kv_cache.py:112-116 — an fp8 KV cache WAS declared and both scales are the + // unloaded sentinel, so the documented default 1.0 applies and the + // uncalibrated warning fires (`:150-156`). This is the arm the + // `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` gate checkpoint takes WHEN + // `--kv-cache-dtype fp8` is typed. It does not reach it on its own: + // MEASURED 2026-08-22 from the live artifact @ `36f717a2`, that repository's + // `config.json:quantization_config` carries `quant_method: "modelopt"` and + // `quant_algo: "MIXED_PRECISION"` and NO `kv_cache_*` key at all, and only the + // legacy `hf_quant_config.json` declares `kv_cache_quant_algo: "FP8"`. The + // inline document wins (`transformers_utils/config.py:751-761`), on both + // engines, so neither vLLM nor this port auto-selects an fp8 KV cache for it. + kDeclaredButAbsent, +}; + +struct ResolvedKvCacheScales { + float k_scale = 1.0F; + float v_scale = 1.0F; + KvScaleOrigin origin = KvScaleOrigin::kNotQuantized; + // kv_cache.py:150-156: `k_scale == 1.0 and v_scale == 1.0 and "e5m2" not in + // kv_cache_dtype` -> warn that the cache is being quantized against an + // uncalibrated scale. Carried as a flag rather than printed here so the caller + // decides once per engine instead of once per layer (upstream's own + // `warning_once`). + bool uncalibrated = false; +}; + +// kv_cache.py:74-156 `BaseKVCacheMethod.process_weights_after_loading`, reduced +// to the per-tensor k/v half this port serves. `loaded_k_scale`/`loaded_v_scale` +// are `kKvScaleUnloaded` when the checkpoint carried no such tensor, exactly as +// `KVCacheScaleParameter` leaves them. +// +// `calculate_kv_scales` is upstream's DEPRECATED dynamic path (`cache.py:111`, +// removal announced for v0.19): when it is set, upstream skips this whole block +// and computes the scales from the first forward instead. We do not implement +// that, so it is refused BY NAME rather than silently taking the static arm. +inline ResolvedKvCacheScales ResolveKvCacheScales(std::string_view kv_cache_dtype, + bool calculate_kv_scales, + float loaded_k_scale, + float loaded_v_scale) { + ResolvedKvCacheScales r; + // kv_cache.py:100-102 — the guard, and the reason kNotQuantized is a state + // rather than a default. It is checked BEFORE anything reads a scale, so a + // checkpoint that declares no KV quantization cannot reach a default value. + if (!v1::IsQuantizedKvCache(kv_cache_dtype)) { + r.origin = KvScaleOrigin::kNotQuantized; + return r; + } + VT_CHECK(!calculate_kv_scales, + "kv_cache scales: --calculate-kv-scales (the on-the-fly dynamic k/v " + "scale, deprecated upstream at config/cache.py:111) is not " + "implemented; the checkpoint scale path is (KV-FP8 W3)"); + if (loaded_k_scale > 0.0F && loaded_v_scale > 0.0F) { + // kv_cache.py:104-111 — prefer separate k_scale and v_scale when present. + r.k_scale = loaded_k_scale; + r.v_scale = loaded_v_scale; + r.origin = KvScaleOrigin::kCheckpoint; + } else if (loaded_k_scale < 0.0F && loaded_v_scale < 0.0F) { + // kv_cache.py:112-116 — BOTH sentinels: no scale was loaded, so the + // documented default 1.0 applies. Reached only because the branch above + // proved an fp8 KV cache was declared. + r.k_scale = 1.0F; + r.v_scale = 1.0F; + r.origin = KvScaleOrigin::kDeclaredButAbsent; + } else { + // kv_cache.py:117-127 — a single `kv_scale`, remapped to k_scale at load + // and duplicated here. Upstream asserts `layer.k_scale > 0.0` and takes + // `max(k_scale, v_scale)`. + VT_CHECK(loaded_k_scale > 0.0F, + "kv_cache scales: a single checkpoint kv_scale must land on " + "k_scale (kv_cache.py:120)"); + const float dup = std::max(loaded_k_scale, loaded_v_scale); + r.k_scale = dup; + r.v_scale = dup; + r.origin = KvScaleOrigin::kCheckpointKvScale; + } + // kv_cache.py:150-156. e5m2's dynamic range makes 1.0 the ordinary choice, so + // upstream suppresses the warning there; e5m2 compute is refused elsewhere in + // this port, and the condition is mirrored anyway so the two agree. + r.uncalibrated = r.k_scale == 1.0F && r.v_scale == 1.0F && + kv_cache_dtype.find("e5m2") == std::string_view::npos; + return r; +} + +// The CONSUMER guard. `kNotQuantized` has no scale to give, and a caller that +// asks for one has decided to quantize a cache the configuration never declared +// — the exact failure the origin enum exists to make impossible. Refuse rather +// than return the 1.0 that would make the mistake invisible. +inline void ScalesForFp8Store(const ResolvedKvCacheScales& scales, float* k_out, + float* v_out) { + VT_CHECK(scales.origin != KvScaleOrigin::kNotQuantized, + "kv_cache scales: no fp8 KV cache was declared (cache_dtype is not " + "an fp8 dtype and the checkpoint declares no kv_cache_quant_algo), " + "so there is no k/v scale to apply; a default 1.0 here would " + "quantize a cache nobody asked to quantize"); + VT_CHECK(scales.k_scale > 0.0F && scales.v_scale > 0.0F, + "kv_cache scales: k_scale/v_scale must be > 0"); + *k_out = scales.k_scale; + *v_out = scales.v_scale; +} + +// The one line upstream prints when the scales defaulted (kv_cache.py:150-156), +// as a string so the caller can log it once per engine. +inline std::string UncalibratedKvScaleWarning(std::string_view kv_cache_dtype) { + return std::string("vllm.cpp: WARNING using KV cache scaling factor 1.0 for ") + + std::string(kv_cache_dtype) + + ". If this is unintended, verify that k/v_scale scaling factors are " + "properly set in the checkpoint.\n"; +} + +} // namespace vllm + +#endif // VLLM_MODEL_EXECUTOR_LAYERS_QUANTIZATION_KV_CACHE_H_ diff --git a/include/vllm/model_executor/models/dense_attn_block.h b/include/vllm/model_executor/models/dense_attn_block.h index fbd3cebc0..1bd6969f7 100644 --- a/include/vllm/model_executor/models/dense_attn_block.h +++ b/include/vllm/model_executor/models/dense_attn_block.h @@ -35,6 +35,7 @@ #include "vllm/model_executor/models/dense_device_glue.h" // Dev/DBuf/MakeTensor/Reshape #include "vllm/model_executor/models/dense_nvfp4_gemm.h" // NVFP4 W4A16 dispatch #include "vllm/model_executor/models/device_pool.h" // DevicePool/Pool/ActivePool (shared) +#include "vllm/model_executor/models/kv_cache_route.h" // KV-FP8 W3 store/read route #include "vllm/model_executor/models/qwen3.h" // Qwen3DenseAttnWeights, PagedKvCache #include "vllm/model_executor/models/tensor_parallel.h" // TensorParallel/TpAllReduceSum (W2) #include "vllm/platforms/interface.h" @@ -346,8 +347,18 @@ inline DBuf AttnBlock(Dev d, const Qwen3DenseAttnWeights& w, const HfConfig& cfg const int64_t qdim = Hq * Dh, kdim = Hkv * Dh; VT_CHECK(w.qkv_bias.Empty(), "qwen3 dense forward: attention_bias not supported yet (Qwen3-0.6B has none)"); - VT_CHECK(kv.dtype == DType::kBF16 || kv.dtype == DType::kF32, - "qwen3 dense: KV cache must be bf16 or f32"); + // KV-FP8 W3: a third storage dtype joins the two float ones — 1-byte fp8 + // (`vt::DType::kI8`), which `IsFp8KvCache` admits only together with a + // matching fp8 interpretation, so a bare `kI8` view still fails here. This + // guard and the routing at the store/read below are ONE decision: leaving it + // at the two float dtypes made `fp8_kv` provably false at every call and the + // fp8 arms of `WriteKvCache`/`ApplyKvCacheQuant` unreachable, which is the + // shape G12 exists to keep out (`qwen3_5.cpp:5313` is the same widening on + // the other routed family). + VT_CHECK(kv.dtype == DType::kBF16 || kv.dtype == DType::kF32 || + IsFp8KvCache(kv), + "qwen3 dense: KV cache must be bf16, f32, or 1-byte fp8 " + "(--kv-cache-dtype fp8)"); VT_CHECK(kv.num_kv_heads == Hkv && kv.head_size == Dh, "qwen3 dense: KV cache head dims mismatch config"); @@ -510,9 +521,16 @@ inline DBuf AttnBlock(Dev d, const Qwen3DenseAttnWeights& w, const HfConfig& cfg // no-op, the common production case). Tensor kw = k3; Tensor vw = v3; - DBuf kcast(d, kv.dtype, {T, Hkv, Dh}); - DBuf vcast(d, kv.dtype, {T, Hkv, Dh}); - if (kv.dtype != adt) { + // KV-FP8 W3: on an fp8 cache there is NO cast to do. `vt::ReshapeAndCacheFp8` + // takes the model-dtype K/V and performs `Quantize(hp / k_scale|v_scale)` + // itself (`quant_utils.cuh:296-300`), so the buffers below are allocated at + // the SOURCE dtype and the branch is skipped — casting to `kI8` would be + // meaningless, and allocating a `kI8` scratch here would silently halve it. + const bool fp8_kv = IsFp8KvCache(kv); + const DType cast_dt = fp8_kv ? adt : kv.dtype; + DBuf kcast(d, cast_dt, {T, Hkv, Dh}); + DBuf vcast(d, cast_dt, {T, Hkv, Dh}); + if (!fp8_kv && kv.dtype != adt) { if (kv.dtype == DType::kBF16) { vt::CastBf16(d.q, kcast.t(), k3); vt::CastBf16(d.q, vcast.t(), v3); @@ -525,13 +543,14 @@ inline DBuf AttnBlock(Dev d, const Qwen3DenseAttnWeights& w, const HfConfig& cfg } Tensor k_cache = KvSlice(kv, d.q.device, 0); Tensor v_cache = KvSlice(kv, d.q.device, 1); - vt::ReshapeAndCache(d.q, kw, vw, k_cache, v_cache, si.slot_mapping.t()); + WriteKvCache(d.q, kv, kw, vw, k_cache, v_cache, si.slot_mapping.t()); DBuf attn(d, adt, {T, Hq, Dh}); const float scale = 1.0F / std::sqrt(static_cast(Dh)); vt::PagedAttentionArgs pa{scale, meta.causal}; pa.query_start_loc_host = meta.query_start_loc.data(); pa.max_seq_len = meta.max_seq_len; + ApplyKvCacheQuant(pa, kv); vt::PagedAttention(d.q, attn.t(), q3, k_cache, v_cache, si.block_table.t(), si.seq_lens.t(), si.query_start_loc.t(), pa); diff --git a/include/vllm/model_executor/models/kv_cache_route.h b/include/vllm/model_executor/models/kv_cache_route.h new file mode 100644 index 000000000..004f60d64 --- /dev/null +++ b/include/vllm/model_executor/models/kv_cache_route.h @@ -0,0 +1,86 @@ +// KV-FP8 W3 — the ONE place a model's attention block decides whether it is +// writing and reading a float KV cache or an fp8 one. +// +// Ported from: vllm/model_executor/layers/attention/attention.py @ 555967922 — +// `Attention.forward` hands `layer._k_scale`/`layer._v_scale` and the layer's +// `kv_cache_dtype` to the backend impl, which routes to +// `reshape_and_cache_flash`'s fp8 branch (`csrc/libtorch_stable/ +// cache_kernels.cu:241-252,314-401`) and to the scaled read +// (`csrc/quantization/w8a8/fp8/nvidia/quant_utils.cuh:419-429`). Upstream's +// routing lives inside one `Attention` module that every model instantiates; +// ours lives here because our attention preambles are per-architecture free +// functions, and a decision copied into each of them is a decision that will +// drift. +// +// WHY A HELPER AND NOT AN IF AT EACH SITE. The store and the read must agree +// with the BLOCK SIZING about how wide a KV element is. `ApplyCacheDType` +// (`kv_cache_interface.cpp`) sizes the page at `vt::SizeOf(spec->dtype)`; these +// two functions are what spend it. If one of them takes the float path against a +// half-sized page, the writes land at the wrong offsets and the model emits +// wrong tokens — no bounds check fires, because the buffer is exactly as large +// as the sizing said. So both decisions read the SAME `PagedKvCache::fp8_kind` +// field, and neither re-derives it from the storage dtype. +#ifndef VLLM_MODEL_EXECUTOR_MODELS_KV_CACHE_ROUTE_H_ +#define VLLM_MODEL_EXECUTOR_MODELS_KV_CACHE_ROUTE_H_ + +#include "vllm/model_executor/models/qwen3_5.h" // PagedKvCache +#include "vt/dtype.h" +#include "vt/fp8_kv.h" +#include "vt/ops.h" + +namespace vllm { +namespace dense_attn { + +// True when this layer's paged cache holds fp8 bytes rather than floats. The +// two facts are asserted to agree here, once, rather than at every caller: a +// 1-byte storage dtype with no fp8 interpretation, or an fp8 interpretation over +// a float page, is a mis-sized cache and never a mode. +inline bool IsFp8KvCache(const PagedKvCache& kv) { + const bool byte_storage = kv.dtype == vt::DType::kI8; + const bool fp8_kind = kv.fp8_kind != vt::Fp8KVCacheDataType::kAuto; + VT_CHECK(byte_storage == fp8_kind, + "kv cache route: the paged cache's storage dtype and its fp8 " + "interpretation disagree — a 1-byte page must carry an fp8 kind and " + "a float page must not (KV-FP8 W3)"); + return fp8_kind; +} + +// The KV STORE. `k`/`v` are the model-dtype [T, Hkv, Dh] tensors the attention +// preamble produced; `k_cache`/`v_cache` are this layer's `KvSlice` views. +// +// On the float path this is `vt::ReshapeAndCache` verbatim — the same op, the +// same arguments, in the same order — so every existing caller is byte-identical +// once routed through here. On the fp8 path it is `vt::ReshapeAndCacheFp8`, +// which takes the FLOAT k/v directly and does the `Quantize(hp / scale)` +// conversion itself (`quant_utils.cuh:296-300`); there is no cast to the cache +// dtype to do first, and a caller that tried would be casting to `kI8`. +inline void WriteKvCache(vt::Queue& q, const PagedKvCache& kv, + const vt::Tensor& k, const vt::Tensor& v, + vt::Tensor& k_cache, vt::Tensor& v_cache, + const vt::Tensor& slot_mapping) { + if (IsFp8KvCache(kv)) { + vt::ReshapeAndCacheFp8(q, k, v, k_cache, v_cache, slot_mapping, kv.fp8_kind, + kv.k_scale, kv.v_scale); + return; + } + vt::ReshapeAndCache(q, k, v, k_cache, v_cache, slot_mapping); +} + +// The KV READ. Sets the three additive `PagedAttentionArgs` fields W1 added, so +// the paged kernel dequantizes each cache read as `Dequant(fp8) * k_scale| +// v_scale`. Inert on a float cache: the fields keep their defaults and the op +// takes the same branch it always did (`src/vt/ops.cpp` — the read guard refuses +// an fp8 cache read with `kv_cache_dtype == kAuto`, which is what makes a +// forgotten call here a loud failure rather than silent garbage). +inline void ApplyKvCacheQuant(vt::PagedAttentionArgs& args, + const PagedKvCache& kv) { + if (!IsFp8KvCache(kv)) return; + args.kv_cache_dtype = kv.fp8_kind; + args.k_scale = kv.k_scale; + args.v_scale = kv.v_scale; +} + +} // namespace dense_attn +} // namespace vllm + +#endif // VLLM_MODEL_EXECUTOR_MODELS_KV_CACHE_ROUTE_H_ diff --git a/include/vllm/model_executor/models/qwen3_5.h b/include/vllm/model_executor/models/qwen3_5.h index 48260e04a..ae881a361 100644 --- a/include/vllm/model_executor/models/qwen3_5.h +++ b/include/vllm/model_executor/models/qwen3_5.h @@ -36,6 +36,7 @@ #include "vllm/v1/attention/backend.h" #include "vllm/v1/attention/backends/gdn_attn.h" #include "vt/device.h" +#include "vt/fp8_kv.h" // KV-FP8 W3: the PagedKvCache fp8 interpretation #include "vt/tensor.h" namespace vllm { @@ -65,6 +66,19 @@ struct PagedKvCache { int64_t block_size = 0; int64_t num_kv_heads = 0; int64_t head_size = 0; + // KV-FP8 W3 — carried straight from the layer's `AttentionSpec` by the runner + // and consumed by `dense_attn::WriteKvCache` / `dense_attn::ApplyKvCacheQuant`. + // ADDITIVE and default-inert: `kAuto` means `dtype` is a float cache and the + // store/read take the byte-identical float path they always did. + // + // `dtype` and `fp8_kind` travel TOGETHER on purpose. `dtype == kI8` sizes the + // page at one byte per element; `fp8_kind` says what that byte means. A view + // that carried only the first would read fp8 bytes as int8 and only the second + // would index a half-sized page at full width — both are wrong tokens rather + // than a crash, which is why they are one struct and not two. + vt::Fp8KVCacheDataType fp8_kind = vt::Fp8KVCacheDataType::kAuto; + float k_scale = 1.0F; + float v_scale = 1.0F; }; // Per-GDN-layer PERSISTENT mamba state (device buffers, updated in place). Rows diff --git a/include/vllm/v1/kv_cache_interface.h b/include/vllm/v1/kv_cache_interface.h index e020995cf..d4b5cf309 100644 --- a/include/vllm/v1/kv_cache_interface.h +++ b/include/vllm/v1/kv_cache_interface.h @@ -73,7 +73,9 @@ #include #include +#include "vllm/v1/kv_cache_dtype.h" #include "vt/dtype.h" +#include "vt/fp8_kv.h" namespace vllm::v1 { @@ -146,6 +148,27 @@ struct AttentionSpec : KVCacheSpec { std::optional page_size_padded; bool indexes_kv_by_block_stride; + // KV-FP8 W3 — the fp8 INTERPRETATION of a 1-byte (`vt::DType::kI8`) storage + // dtype, plus the per-tensor dequant scales. ADDITIVE and default-inert: every + // existing producer constructs a float spec and leaves these at kAuto/1.0, so + // nothing about the bf16 default changes. + // + // WHY THE SPEC AND NOT A SIDE TABLE. `dtype` alone cannot answer "which fp8", + // because upstream stores every fp8 flavour as `torch.uint8` + // (`torch_utils.py:38-40`) and carries the flavour on the layer instead. Our + // vt ops take the flavour and the scales as ARGUMENTS, and the runner builds + // its `PagedKvCache` view from this spec and nothing else — the header's own + // rule that "the KV cache SPEC is the single source of truth for the storage + // dtype" (`kv_cache_dtype.h`). Splitting the interpretation away from the + // storage dtype is exactly how a half-sized block and a full-sized store come + // to disagree, which is silent corruption rather than a crash. + // + // Written ONCE, by `ApplyCacheDType` below, after the model's factory has + // built the spec. Nothing else may set them. + vt::Fp8KVCacheDataType fp8_kind = vt::Fp8KVCacheDataType::kAuto; + float k_scale = 1.0F; + float v_scale = 1.0F; + int64_t page_size_bytes() const override; // The raw (unpadded) K+V bytes per page. Overridden by FullAttentionSpec. @@ -397,6 +420,32 @@ struct KVCacheConfig { // spec's own `page_size_bytes()` throws (deferred quantized-KV math). int64_t KVBytesPerBlock(const KVCacheConfig& config); +// KV-FP8 W3 — HALF-SIZED KV BLOCKS. Rewrite every ATTENTION spec in `config` to +// the resolved KV storage dtype, then hand the fp8 interpretation and the +// per-tensor scales to the same specs. +// +// This is the single place the resolved `cache_dtype` becomes bytes, and it +// mirrors where upstream does it: `GPUModelRunner.__init__` resolves +// `self.kv_cache_dtype = kv_cache_dtype_str_to_dtype(cache_config.cache_dtype, +// model_config)` (`gpu_model_runner.py:484-486`) and every attention spec is +// then built with `dtype=self.kv_cache_dtype`. `AttentionSpec. +// real_page_size_bytes` is linear in `get_dtype_size(self.dtype)` +// (`kv_cache_interface.py:204-218`), so an fp8 store dtype (1 byte) against +// bf16 (2 bytes) halves the page and, at a fixed byte budget, doubles the block +// count. Nothing else in the sizing chain needs to know. +// +// A NO-OP on the default path, deliberately: `resolved.storage` for "auto" is +// the model dtype the factory already used, and the function returns before it +// touches a spec unless the caller named a non-auto dtype. `MambaSpec` is never +// rewritten — recurrent state is not the KV cache and upstream keeps it on its +// own `mamba_cache_dtype` knob (`config/cache.py:131-138`). +// +// Refuses BY NAME rather than mis-sizing: an MLA spec (upstream's fp8 arm there +// is `fp8_ds_mla`, a different page formula, `kv_cache_interface.py:398-410`), a +// storage dtype no store is wired for, and an fp8 kind the kernels refuse. +void ApplyCacheDType(KVCacheConfig& config, const ResolvedCacheDType& resolved, + float k_scale, float v_scale); + } // namespace vllm::v1 #endif // VLLM_V1_KV_CACHE_INTERFACE_H_ diff --git a/src/vllm/config/cache.cpp b/src/vllm/config/cache.cpp new file mode 100644 index 000000000..61ce51bce --- /dev/null +++ b/src/vllm/config/cache.cpp @@ -0,0 +1,298 @@ +// Ported from: vllm/utils/torch_utils.py @ 555967922 — :64-67 +// MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP, :310-362 +// get_kv_cache_quant_algo_string, :374-392 +// resolve_kv_cache_dtype_string. See include/vllm/config/cache.h. +#include "vllm/config/cache.h" + +#include +#include +#include +#include +#include +#include + +#include + +namespace vllm { + +namespace { + +using nlohmann::json; + +std::string Lower(std::string s) { + std::transform(s.begin(), s.end(), s.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + return s; +} + +// torch_utils.py:64-67 MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP. Two entries, and the +// second one is deliberately kept: `nvfp4` resolves to the string `nvfp4`, which +// `vllm::v1::ParseCacheDType` then REFUSES by name (KV-NVFP4-TURBO owns it). +// Dropping it here would turn a declared-and-unimplemented KV format into +// "nothing declared", which is the silent-default failure this whole path is +// built to avoid. +std::optional MapModeloptKvAlgo(const std::string& algo_lower) { + if (algo_lower == "fp8") return std::string("fp8_e4m3"); + if (algo_lower == "nvfp4") return std::string("nvfp4"); + return std::nullopt; +} + +// torch_utils.py:329-346 — the DICT spelling of `kv_cache_scheme`. +std::optional KvAlgoFromObject(const json& kv_algo) { + const bool dynamic_false = kv_algo.contains("dynamic") && + kv_algo["dynamic"].is_boolean() && + !kv_algo["dynamic"].get(); + // Written as a statement rather than a conditional expression: GCC 13 at -O2 + // cannot see through the inlined `std::optional` ternary and reports + // `-Wmaybe-uninitialized` on the `*num_bits` reads below, which is -Werror in + // this tree's Release and RelWithDebInfo builds. + std::optional num_bits; + if (kv_algo.contains("num_bits") && kv_algo["num_bits"].is_number_integer()) { + num_bits = kv_algo["num_bits"].get(); + } + const std::string type = kv_algo.contains("type") && kv_algo["type"].is_string() + ? kv_algo["type"].get() + : std::string(); + if (dynamic_false && num_bits.has_value() && *num_bits == 8 && type == "float") { + return std::string("fp8"); + } + if (num_bits.has_value() && *num_bits == 4 && type == "float") { + return std::string("nvfp4"); + } + return std::nullopt; +} + +} // namespace + +std::optional GetKvCacheQuantAlgoString( + const std::string& quant_config_json) { + if (quant_config_json.empty()) return std::nullopt; + json cfg; + try { + cfg = json::parse(quant_config_json); + } catch (const json::exception&) { + // A malformed quantization config is not this resolver's error to raise — + // the weight loader reports it with far more context. "Nothing declared" is + // the honest answer here. + return std::nullopt; + } + if (!cfg.is_object()) return std::nullopt; + + // torch_utils.py:319 — only modelopt configs carry `kv_cache_quant_algo`. + const std::string quant_method = + cfg.contains("quant_method") && cfg["quant_method"].is_string() + ? cfg["quant_method"].get() + : std::string(); + const json& inner = (cfg.contains("quantization") && cfg["quantization"].is_object()) + ? cfg["quantization"] + : cfg; + // THE MARKER IS TWO UPSTREAM FUNCTIONS, NOT ONE, and reading only the first + // gets this wrong. `get_kv_cache_quant_algo_string` tests + // `quant_cfg.get("quant_method", "").startswith("modelopt")` at the TOP level + // (`torch_utils.py:319`) — which a ModelOpt 0.29.0-and-before + // `hf_quant_config.json` does not carry, because that file nests the algorithm + // under `"quantization"` and names the producer at the top + // (`{"producer":{"name":"modelopt"},"quantization":{...}}`). Upstream STILL + // resolves it, because `ModelArchConfigConvertorBase. + // _normalize_quantization_config` + // (`vllm/transformers_utils/model_arch_config_convertor.py:208-247`) runs + // FIRST, at `ModelConfig.__post_init__` (`vllm/config/model.py:577` -> + // `get_model_arch_config`), and INJECTS `quant_cfg["quant_method"]` when + // `producer["name"] == "modelopt"` OR the nested document carries + // `modelopt_quant_config` (`:216-235`); it mutates the very dict + // `hf_config.quantization_config` names, so the test at `:319` sees the + // injected marker. `:238-246` then lower-cases it, which is what `Lower()` + // mirrors here. + // + // MEASURED 2026-08-22 by running BOTH pinned functions (extracted with `ast` + // from the files at `555967922`, not transcribed) over the live documents: + // `nvidia/Llama-3.3-70B-Instruct-FP8`'s producer-only `hf_quant_config.json` + // answers `None` before normalization and `'fp8_e4m3'` after it. Accepting + // the producer name here is that two-step chain collapsed into one function, + // and it is a MIRROR rather than a widening. + // + // ONE DIFFERENCE REMAINS and is recorded rather than copied: the injector + // RAISES `ValueError: Unknown ModelOpt quant algo: ` (`:235`) when the + // producer is modelopt and the nested `quant_algo` is neither FP8-family nor + // NVFP4 — which is what `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121`'s legacy file + // (`MIXED_PRECISION`) gets, and we answer `fp8_e4m3` for it instead. That + // refusal is a WEIGHT-half validation living in a config convertor this port + // does not have, and importing it into the KV resolver would refuse a + // checkpoint whose weights we load; the spec's `## Owed` names it. It is + // unreachable for that checkpoint anyway, because its `config.json` carries an + // inline `quantization_config` and that document wins. + const std::string producer = + cfg.contains("producer") && cfg["producer"].is_object() && + cfg["producer"].contains("name") && cfg["producer"]["name"].is_string() + ? cfg["producer"]["name"].get() + : std::string(); + // THE INJECTOR'S NESTED DOCUMENT IS NOT THE READER'S. `:217` is + // `quant_cfg.get("quantization", {})` and falls back to an EMPTY object, + // while `torch_utils.py:321` is `quant_cfg.get("quantization", quant_cfg)` and + // falls back to the whole document. Two different fallbacks in two different + // functions, so two variables here: `modelopt_inner` decides whether a marker + // is injected, `inner` (above) decides where the KV key is looked up. Reading + // the legacy key out of `inner` made a TOP-LEVEL `modelopt_quant_config` with + // no `quantization` object a marker, which upstream never treats as one. + static const json kEmptyObject = json::object(); + const json& modelopt_inner = + (cfg.contains("quantization") && cfg["quantization"].is_object()) + ? cfg["quantization"] + : kEmptyObject; + // `_normalize_quantization_config:218-220` — the legacy nested shape, which + // names no producer and is recognised by the key alone. + const bool legacy_modelopt = modelopt_inner.contains("modelopt_quant_config"); + const auto starts_with_modelopt = [](const std::string& s) { + return s.rfind("modelopt", 0) == 0; + }; + // AND THE INJECTION IS GUARDED. `:224` is `if quant_algo is not None`, read + // out of that same nested document, and only then does `:225-235` write a + // marker: `modelopt` for {FP8, FP8_PER_CHANNEL_PER_TOKEN, FP8_PB_WO}, + // `modelopt_fp4` for NVFP4, and a `ValueError` for anything else. A modelopt + // producer that declares NO `quant_algo` therefore gets no marker at all, and + // `torch_utils.py:319` answers `None` for it. + // + // The two outcomes for a `quant_algo` that IS present are "inject" and + // "raise", and the paragraph above records why this port takes the raise's arm + // rather than importing it. So the mirrored test is exactly + // `quant_algo is not None`, and what it newly refuses are the three shapes + // that used to resolve to `fp8_e4m3` here while upstream answered `None`: a + // `producer.name` of `modelopt` with no `quant_algo`, a + // `quantization.modelopt_quant_config` with no `quant_algo`, and a top-level + // `modelopt_quant_config` with no `quantization` key. + const bool has_quant_algo = modelopt_inner.contains("quant_algo") && + !modelopt_inner["quant_algo"].is_null(); + // `quant_method` is prefix-matched and case-folded because upstream lower-cases + // it before testing (`:238-246` then `torch_utils.py:319`); the producer name + // is neither, because `:222` is a raw `==` against the literal and nothing + // normalises it first. Same file, two different tests, mirrored separately. + const bool injected_marker = + (producer == "modelopt" || legacy_modelopt) && has_quant_algo; + if (!starts_with_modelopt(Lower(quant_method)) && !injected_marker) { + return std::nullopt; + } + + // torch_utils.py:322-328 — the four spellings, in upstream's own order. + const json* kv_algo = nullptr; + const json* const candidates[] = {&inner, &cfg}; + for (const json* obj : candidates) { + for (const char* key : {"kv_cache_scheme", "kv_cache_quant_algo"}) { + if (obj->contains(key) && !(*obj)[key].is_null()) { + kv_algo = &(*obj)[key]; + break; + } + } + if (kv_algo != nullptr) break; + } + // Upstream's order is scheme(inner), scheme(outer), algo(inner), algo(outer); + // the loop above is scheme(inner), algo(inner), scheme(outer), algo(outer). + // They differ only for a config that carries an inner `kv_cache_quant_algo` + // AND an outer `kv_cache_scheme`, which no shipped checkpoint does — recorded + // rather than silently equated. + if (kv_algo == nullptr) return std::nullopt; + + if (kv_algo->is_object()) { + const std::optional named = KvAlgoFromObject(*kv_algo); + if (!named.has_value()) { + std::cerr << "vllm.cpp: WARNING unknown kv_cache_quant_algo object in the " + "model quantization config; falling back to 'auto' " + "(torch_utils.py:339-346)\n"; + return std::string("auto"); + } + const std::optional mapped = MapModeloptKvAlgo(*named); + return mapped.has_value() ? mapped : std::optional("auto"); + } + if (kv_algo->is_string()) { + const std::string algo_lower = Lower(kv_algo->get()); + const std::optional mapped = MapModeloptKvAlgo(algo_lower); + if (mapped.has_value()) return mapped; + std::cerr << "vllm.cpp: WARNING unknown kv_cache_quant_algo '" + << kv_algo->get() + << "' in the model quantization config (supported: fp8, nvfp4); " + "falling back to 'auto' (torch_utils.py:351-361)\n"; + return std::string("auto"); + } + return std::nullopt; +} + +ResolvedCacheDTypeString ResolveKvCacheDTypeString( + const std::string& requested, const std::string& quant_config_json) { + ResolvedCacheDTypeString out; + // torch_utils.py:380-381 — an explicit choice is returned UNCHANGED and the + // checkpoint is never consulted. The operator outranks the checkpoint. + if (!requested.empty() && requested != kDefaultCacheDType) { + out.cache_dtype = requested; + out.declared_by_checkpoint = false; + return out; + } + const std::optional declared = + GetKvCacheQuantAlgoString(quant_config_json); + if (declared.has_value() && *declared != kDefaultCacheDType) { + out.cache_dtype = *declared; + out.declared_by_checkpoint = true; + return out; + } + // torch_utils.py:391-392 — nothing declared, or declared in a shape that fell + // back to "auto". Either way the model dtype wins downstream. + out.cache_dtype = kDefaultCacheDType; + out.declared_by_checkpoint = false; + return out; +} + +std::string ReadQuantConfigJson(const std::string& model_dir) { + namespace fs = std::filesystem; + if (model_dir.empty()) return std::string(); + std::error_code ec; + const fs::path dir(model_dir); + if (!fs::is_directory(dir, ec)) return std::string(); + + // UPSTREAM ORDER, and it is the CURRENT file first. `config.py:751-761`: + // + // quantization_config = config_dict.get("quantization_config", None) + // if quantization_config is None and file_or_path_exists( + // model, "hf_quant_config.json", revision): + // quantization_config = get_hf_file_to_dict("hf_quant_config.json", ...) + // + // carrying upstream's own two comments: "ModelOpt 0.31.0 and after saves the + // quantization config in the model config file", and the separate file is + // "ModelOpt 0.29.0 and before". So the legacy file is a FALLBACK and is never + // opened when `config.json` carries the inline document. That matters because + // checkpoints get re-quantized in place: a repository that grew an inline + // `quantization_config` and kept its old `hf_quant_config.json` beside it + // resolves to what the CURRENT file says rather than to the stale one. W3 + // shipped this pair in the other order and nothing gated it; the case that + // does now is G10 in tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp. + const fs::path config = dir / "config.json"; + if (fs::is_regular_file(config, ec)) { + std::ifstream in(config, std::ios::binary); + if (in) { + try { + nlohmann::json doc = nlohmann::json::parse(in); + if (doc.is_object() && doc.contains("quantization_config") && + doc["quantization_config"].is_object()) { + return doc["quantization_config"].dump(); + } + } catch (const nlohmann::json::exception&) { + // A malformed `config.json` is not this resolver's error to raise, and + // it is not a reason to reach for the legacy file either: upstream + // parses this file once and would have failed there. "Nothing declared" + // is the honest answer; the loader reports the parse with its own + // context. + return std::string(); + } + } + } + const fs::path hf_quant = dir / "hf_quant_config.json"; + if (fs::is_regular_file(hf_quant, ec)) { + std::ifstream in(hf_quant, std::ios::binary); + if (in) { + std::ostringstream ss; + ss << in.rdbuf(); + return ss.str(); + } + } + return std::string(); +} + +} // namespace vllm diff --git a/src/vllm/entrypoints/model_loader.cpp b/src/vllm/entrypoints/model_loader.cpp index 0f6fa27c7..ef6f1ae31 100644 --- a/src/vllm/entrypoints/model_loader.cpp +++ b/src/vllm/entrypoints/model_loader.cpp @@ -22,6 +22,8 @@ #include +#include "vllm/config/cache.h" // KV-FP8 W3: --kv-cache-dtype vs the checkpoint +#include "vllm/model_executor/layers/quantization/kv_cache.h" // k/v scale arms #include "vllm/model_executor/weight_offloader.h" #include "vllm/model_executor/model_loader/gguf_device_fit.h" #include "vllm/model_executor/model_loader/gguf_reader.h" @@ -1487,14 +1489,83 @@ vllm::v1::KVCacheConfig LoadedEngine::MakeKVCacheResolved( // The per-block byte geometry is independent of the block count, so build a // probe at the override-or-256 count, read its geometry to resolve the real // count, and only rebuild when the resolved count differs. + // Once per load — this function is the single call site (the LoadedEngine + // ctor's kv_cfg_ member initializer), and ApplyResolvedCacheDType below runs + // up to twice. + WarnUncalibratedKvScales(params); const int probe_blocks = params.num_blocks > 0 ? params.num_blocks : 256; vllm::v1::KVCacheConfig probe = MakeKVCacheMaybeSpec(model, config, block_size, probe_blocks, spec); + // KV-FP8 W3 — the storage dtype is applied to the PROBE, before + // ResolveNumBlocks reads its geometry. That ordering IS the halved-block + // feature: `KVBytesPerBlock(probe)` is the divisor knob 2 sizes the pool with, + // so an fp8 page (1 byte/element vs bf16's 2) halves the divisor and doubles + // the block count at the same --kv-cache-memory. Applying it after would size + // the pool from a bf16 page and then serve it as fp8, which is the same pool + // in half the bytes rather than twice the pool. + ApplyResolvedCacheDType(params, probe); const int resolved = ResolveNumBlocks(params, probe); if (resolved == probe_blocks) { return probe; } - return MakeKVCacheMaybeSpec(model, config, block_size, resolved, spec); + vllm::v1::KVCacheConfig sized = + MakeKVCacheMaybeSpec(model, config, block_size, resolved, spec); + ApplyResolvedCacheDType(params, sized); + return sized; +} + +// Upstream's `logger.warning_once` for the defaulted-scale case +// (`kv_cache.py:150-156`), lifted out of ApplyResolvedCacheDType because that +// function runs up to TWICE per load (probe, then the resized config) and a +// warning that fires twice reads as two engines. Once per LOAD, not once per +// process: two engines in one process both deserve the line. +void LoadedEngine::WarnUncalibratedKvScales(const EngineParams& params) { + const vllm::ResolvedKvCacheScales scales = vllm::ResolveKvCacheScales( + params.kv_cache_dtype, /*calculate_kv_scales=*/false, + vllm::kKvScaleUnloaded, vllm::kKvScaleUnloaded); + if (scales.origin == vllm::KvScaleOrigin::kNotQuantized || !scales.uncalibrated) { + return; + } + std::cerr << vllm::UncalibratedKvScaleWarning(params.kv_cache_dtype); + std::cerr.flush(); +} + +void LoadedEngine::ApplyResolvedCacheDType(const EngineParams& params, + vllm::v1::KVCacheConfig& cfg) { + // `params.kv_cache_dtype` is already resolved against the checkpoint by + // FromModelDir; here it only has to become bytes. + const vllm::v1::ResolvedCacheDType resolved = vllm::v1::ParseCacheDType( + params.kv_cache_dtype, vllm::v1::ResolveKvCacheDType()); + // The scale pair, resolved through the SAME four-arm mirror of + // `BaseKVCacheMethod.process_weights_after_loading` that upstream uses. Both + // loaded scales are the `KVCacheScaleParameter` sentinel because no model's + // weight loader extracts `k_scale`/`v_scale` yet (owed, see the spec's + // `## Owed`), so a declaring checkpoint lands on `kDeclaredButAbsent`, which + // is a DIFFERENT state from a checkpoint that declared nothing. The #1574 + // subject `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` reaches this arm only when an + // operator types `--kv-cache-dtype fp8`: its inline + // `config.json:quantization_config` declares no `kv_cache_*` key, and that is + // the document both engines read (`transformers_utils/config.py:751-761`). + const vllm::ResolvedKvCacheScales scales = vllm::ResolveKvCacheScales( + params.kv_cache_dtype, /*calculate_kv_scales=*/false, + vllm::kKvScaleUnloaded, vllm::kKvScaleUnloaded); + if (scales.origin == vllm::KvScaleOrigin::kNotQuantized) { + // Nothing declared an fp8 KV cache. Do NOT reach for a scale: ApplyCacheDType + // returns immediately on the auto/bf16 path anyway, and asking + // ScalesForFp8Store here would be the invented-default this port refuses. + vllm::v1::ApplyCacheDType(cfg, resolved, /*k_scale=*/1.0F, /*v_scale=*/1.0F); + return; + } + float k_scale = 0.0F; + float v_scale = 0.0F; + // The uncalibrated line is NOT printed here. This function runs up to twice + // per load (the probe, then the resized config), and a process-static + // `call_once` would fix that by printing once for the process instead — which + // silences the SECOND engine in the same process rather than the second call + // of the same load. `WarnUncalibratedKvScales` above is the once-per-load + // owner, and it is the only caller of the message. + vllm::ScalesForFp8Store(scales, &k_scale, &v_scale); + vllm::v1::ApplyCacheDType(cfg, resolved, k_scale, v_scale); } int LoadedEngine::ResolveMaxModelLen(const EngineParams& params, @@ -1865,26 +1936,67 @@ vllm::v1::AsyncLLM& LoadedEngine::async_engine() { } std::unique_ptr LoadedEngine::FromModelDir( - const std::string& model_dir, const EngineParams& params) { + const std::string& model_dir, const EngineParams& params_in) { // SPEC-DRAFTER-CHAIN W1 (#1522): refuse a drafter chain BEFORE anything else // this function does. // // G5 requires the refusal to land "before any weight I/O", and the ordering is - // the requirement rather than tidiness. Below this line the function installs - // two process-global configurations, resolves a device, stats the model - // directory, parses a config, builds a tokenizer and maps weights. A chain - // config cannot produce an engine, so every one of those is work spent on a - // load that will not happen — and the two installs are not free, because one - // of them latches decisions that a later engine in the same process cannot - // retake. + // the requirement rather than tidiness. Below this line the function reads the + // checkpoint's quantization config, installs two process-global + // configurations, resolves a device, stats the model directory, parses a + // config, builds a tokenizer and maps weights. A chain config cannot produce + // an engine, so every one of those is work spent on a load that will not + // happen, and the two installs are not free, because one of them latches + // decisions that a later engine in the same process cannot retake. // // It delegates to `ResolveSpecConfig` rather than repeating the test, so there // is ONE chain refusal with ONE message. `ResolveSpecConfig` runs again in the // constructor further down, which is what refuses a chain on the loader paths // that do not come through this function. - if (params.speculative_config.has_value() && - params.speculative_config->use_drafter_chain()) { - (void)ResolveSpecConfig(params, vllm::HfConfig{}); + // + // It reads `params_in`, the UNRESOLVED argument, deliberately: it must run + // ahead of the KV-FP8 W3 stanza below, whose `ReadQuantConfigJson` opens a + // file inside `model_dir`. The case + // "kv-fp8 W3 G10: the drafter-chain refusal runs BEFORE the KV resolution" in + // `tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp` gates that order, by + // pointing at a directory that EXISTS and declares fp8 -- which + // `test_drafter_chain_reach.cpp` cannot do, because its nonexistent path makes + // `ReadQuantConfigJson` answer "" without opening anything. + if (params_in.speculative_config.has_value() && + params_in.speculative_config->use_drafter_chain()) { + (void)ResolveSpecConfig(params_in, vllm::HfConfig{}); + } + // KV-FP8 W3: resolve `--kv-cache-dtype` against the CHECKPOINT once, here, + // before any consumer reads it. Upstream does exactly this and in exactly this + // position: `EngineArgs.create_engine_config` calls + // `resolve_kv_cache_dtype_string(self.kv_cache_dtype, model_config)` and hands + // the RESULT to `CacheConfig(cache_dtype=...)` (`arg_utils.py:1915-1929`), so + // nothing downstream of the config ever sees the unresolved "auto". + // + // `params` shadows the argument from here on, so every later reference in this + // function reads the resolved value and no call site had to change. The only + // field that differs is `kv_cache_dtype`. + // + // THIS IS THE ONLY PLACE A CHECKPOINT'S DECLARATION IS READ. The direct + // `LoadedEngine(config, weights, ...)` constructors take `kv_cache_dtype` + // verbatim, which is right: they are handed weights that are already in + // memory and there is no checkpoint directory to ask. + EngineParams params = params_in; + { + const vllm::ResolvedCacheDTypeString resolved = + vllm::ResolveKvCacheDTypeString(params.kv_cache_dtype, + vllm::ReadQuantConfigJson(model_dir)); + if (resolved.declared_by_checkpoint) { + // Say it. An operator who typed no flag and gets a quantized KV cache + // because the checkpoint asked for one deserves to read that sentence + // rather than infer it from a block count that doubled. + std::cerr << "engine: the checkpoint declares kv_cache_quant_algo -> " + "--kv-cache-dtype " + << resolved.cache_dtype + << " (pass an explicit --kv-cache-dtype to override)" + << std::endl; + } + params.kv_cache_dtype = resolved.cache_dtype; } // ENG-RESIDENCY-CONFIG (#1110): install the host-RAM -> DISK residency config // FIRST — before the offloader below, before the device resolution, before any diff --git a/src/vllm/entrypoints/openai/server_main.cpp b/src/vllm/entrypoints/openai/server_main.cpp index 2f1950221..ab7a5ecf4 100644 --- a/src/vllm/entrypoints/openai/server_main.cpp +++ b/src/vllm/entrypoints/openai/server_main.cpp @@ -10,6 +10,7 @@ // [--served-model-name ] // [--block-size N] [--num-blocks N] [--max-model-len N] // [--gpu-memory-utilization F] [--kv-cache-memory BYTES] +// [--kv-cache-dtype auto|bfloat16|fp8|fp8_e4m3] // [--max-num-seqs N] [--max-num-batched-tokens N] // [--enable-force-include-usage] // [--[no-]enable-prefix-caching] @@ -204,6 +205,14 @@ struct Args { // double pre-filled with 0.92 could not express the difference. std::optional gpu_memory_utilization = std::nullopt; long long kv_cache_memory_bytes = 0; + // --kv-cache-dtype: vLLM CacheConfig.cache_dtype (config/cache.py:19-36,76). + // "auto" (the default) uses the model dtype and is byte-identical to before + // the flag existed; "fp8"/"fp8_e4m3" stores the paged K/V as 1-byte fp8, which + // HALVES the bytes per KV block and so doubles the pool at the same + // --kv-cache-memory. The value is resolved against the checkpoint's own + // `kv_cache_quant_algo` inside LoadedEngine::FromModelDir, and an explicit + // value always beats the checkpoint (torch_utils.py:380-381). + std::string kv_cache_dtype = "auto"; int max_model_len = 0; // 0 => config.max_position_embeddings int max_num_seqs = 32; // see model_loader.h: 8 clamped c8 batching. int max_num_batched_tokens = 0; // 0 => per-architecture default. @@ -410,6 +419,7 @@ const InertArg* FindAcceptedInertArg(const std::string& flag) { "[--num-blocks N] [--max-model-len N]\n" " [--gpu-memory-utilization F] " "[--kv-cache-memory BYTES]\n" + " [--kv-cache-dtype auto|bfloat16|fp8|fp8_e4m3]\n" " [--max-num-seqs N] " "[--max-num-batched-tokens N]\n" " [--device auto|cpu|cuda]\n" @@ -490,6 +500,8 @@ Args ParseArgs(int argc, char** argv) { a.gpu_memory_utilization = std::stod(NextArg(argc, argv, i, argv[0])); } else if (flag == "--kv-cache-memory") { a.kv_cache_memory_bytes = std::stoll(NextArg(argc, argv, i, argv[0])); + } else if (flag == "--kv-cache-dtype") { + a.kv_cache_dtype = NextArg(argc, argv, i, argv[0]); } else if (flag == "--max-model-len") { a.max_model_len = std::stoi(NextArg(argc, argv, i, argv[0])); } else if (flag == "--max-num-seqs") { @@ -1234,6 +1246,7 @@ int VllmServerMain(int argc, char** argv) { engine_params.num_blocks = args.num_blocks; engine_params.gpu_memory_utilization = args.gpu_memory_utilization; engine_params.kv_cache_memory_bytes = args.kv_cache_memory_bytes; + engine_params.kv_cache_dtype = args.kv_cache_dtype; engine_params.max_model_len = args.max_model_len; // 0 => from config. engine_params.max_num_seqs = args.max_num_seqs; engine_params.max_num_batched_tokens = args.max_num_batched_tokens; diff --git a/src/vllm/model_executor/layers/quantization/modelopt_mixed_precision.h b/src/vllm/model_executor/layers/quantization/modelopt_mixed_precision.h index e1477de06..d5c3c0102 100644 --- a/src/vllm/model_executor/layers/quantization/modelopt_mixed_precision.h +++ b/src/vllm/model_executor/layers/quantization/modelopt_mixed_precision.h @@ -1003,8 +1003,14 @@ inline std::string JoinModules(const std::set& modules, // loads and measures today — declares `kv_cache_scheme` 8-bit float static and // ships ZERO `k_scale`/`v_scale` tensors, and // `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` declares `kv_cache_quant_algo: "FP8"` -// in `hf_quant_config.json`, a file no production path in this tree reads at -// all. A refusal here would refuse a checkpoint that loads today. The FP8 KV +// in `hf_quant_config.json`. Since `KV-FP8` W3 (#1593) that file DOES have a +// production reader — `vllm::ReadQuantConfigJson` (`src/vllm/config/cache.cpp` +// :207), called from `LoadedEngine::FromModelDir` (`model_loader.cpp:1988`) — +// but only as the LEGACY FALLBACK behind `config.json`'s inline +// `quantization_config` (`config.py:751-761`), and only to resolve the KV cache +// dtype. This checkpoint ships that inline document, so its legacy file is +// never opened and the declaration reaches nothing. +// A refusal here would refuse a checkpoint that loads today. The FP8 KV // arm is owned by `KV-FP8` (issue #1593) and is listed under `## Owed` in // `.agents/specs/qwen38-27b-quant-arms.md`. `ModuleOperands` RECORDS a KV scale // and leaves it out of `AnyQuantOperand`, which is where that decision is diff --git a/src/vllm/model_executor/models/qwen3_5.cpp b/src/vllm/model_executor/models/qwen3_5.cpp index 9f32a034d..eeec46b9f 100644 --- a/src/vllm/model_executor/models/qwen3_5.cpp +++ b/src/vllm/model_executor/models/qwen3_5.cpp @@ -17,6 +17,7 @@ #include "vllm/model_executor/models/qwen3_5.h" #include "vllm/model_executor/models/decode_graph_sizes.h" +#include "vllm/model_executor/models/kv_cache_route.h" // KV-FP8 W3 store/read route #include "vllm/model_executor/models/dense_fp8_block_gemm.h" // MODEL-FP8-BLOCK-LINEAR (#1189 M4) #include "vllm/model_executor/models/device_pool.h" // DevicePool/Pool/AuxPool/ActivePool (shared) @@ -5306,8 +5307,13 @@ DBuf FullAttnBlockPaged(Dev d, const FullAttnLayerWeights& w, const HfConfig& cf const int rot = static_cast(cfg.rotary_dim); const float base = static_cast(cfg.rope_theta); const float eps = static_cast(cfg.rms_norm_eps); - VT_CHECK(kv.dtype == DType::kBF16 || kv.dtype == DType::kF32, - "full-attn paged: KV cache must be bf16 or f32"); + // KV-FP8 W3: a third storage dtype joins the two float ones — 1-byte fp8 + // (`vt::DType::kI8`), which `dense_attn::IsFp8KvCache` admits only together + // with a matching fp8 interpretation, so a bare `kI8` view still fails here. + VT_CHECK(kv.dtype == DType::kBF16 || kv.dtype == DType::kF32 || + dense_attn::IsFp8KvCache(kv), + "full-attn paged: KV cache must be bf16, f32, or 1-byte fp8 " + "(--kv-cache-dtype fp8)"); VT_CHECK(kv.num_kv_heads == Hkv && kv.head_size == Dh, "full-attn paged: KV cache head dims mismatch config"); @@ -5416,11 +5422,25 @@ DBuf FullAttnBlockPaged(Dev d, const FullAttnLayerWeights& w, const HfConfig& cf // V to bf16 only when the cache is bf16. The query stays f32 either way // (Phase 1: f32 query · cache, f32-accumulate softmax — the // attention kernel converts bf16 cache reads to f32). + // + // KV-FP8 W3 (#1593): the fp8 cache takes the SAME bf16 normalisation, and it + // has to. `vt::ReshapeAndCacheFp8` quantizes from ONE source dtype + // (`k.dtype == v.dtype`, ops.cpp), and K and V do not arrive in the same one: + // K is `attn_dt`, which is f32 for every fp8 cache because `kv.dtype == + // DType::kBF16` is a term of both FA2 eligibility tests above, while V is + // whatever the v_proj GEMM emitted — bf16 on the block-wise fp8 arm + // (`MatmulFp8BlockScaledD`), bf16 on the NVFP4 arm under the default + // `VT_BF16_GEMM_OUT`, and bf16 on ordinary torch safetensors + // (`MatmulBf16D`). Only the per-tensor fp8 arm and the transposed + // GGUF/synthetic path pair f32 with f32, which is why a CPU gate over + // synthetic weights could not see this. bf16 rather than f32 because bf16 is + // the dtype upstream quantizes from: its model IS bf16 where + // `reshape_and_cache_flash` takes key/value (`cache_kernels.cu:314-401`). Tensor kw = kn3; Tensor vw = v3; DBuf kbf(d, DType::kBF16, {T, Hkv, Dh}); DBuf vbf(d, DType::kBF16, {T, Hkv, Dh}); - if (kv.dtype == DType::kBF16) { + if (kv.dtype == DType::kBF16 || dense_attn::IsFp8KvCache(kv)) { // K may already be bf16 (an FA2 preamble emits bf16 k directly — // the RN round of the same f32 value this CastBf16 would produce); only // down-cast when the preamble/fallback produced f32 K. @@ -5448,7 +5468,10 @@ DBuf FullAttnBlockPaged(Dev d, const FullAttnLayerWeights& w, const HfConfig& cf Tensor dblk = sdi.block_table.t(); Tensor dsl = sdi.seq_lens.t(); Tensor dqsl = sdi.query_start_loc.t(); - vt::ReshapeAndCache(d.q, kw, vw, k_cache, v_cache, dslot); + // KV-FP8 W3: routes to `vt::ReshapeAndCacheFp8` when this layer's cache is + // 1-byte fp8, and the cast block above has already put K and V into the ONE + // model dtype that store quantizes from. + dense_attn::WriteKvCache(d.q, kv, kw, vw, k_cache, v_cache, dslot); // bf16 attention out on an FA2 path (FA2 writes bf16; the sigmoid // gate upcast is exact) — f32 everywhere else, byte-identical to today. @@ -5465,6 +5488,7 @@ DBuf FullAttnBlockPaged(Dev d, const FullAttnLayerWeights& w, const HfConfig& cf vt::PagedAttentionArgs pa_args{scale, meta.causal}; pa_args.query_start_loc_host = meta.query_start_loc.data(); pa_args.max_seq_len = meta.max_seq_len; + dense_attn::ApplyKvCacheQuant(pa_args, kv); vt::PagedAttention(d.q, dattn.t(), qn3, k_cache, v_cache, dblk, dsl, dqsl, pa_args); // VT_DUMP_ATTN (issue #41, 0.8B ROCm divergence spike W1/W2): dump the diff --git a/src/vllm/v1/kv_cache_interface.cpp b/src/vllm/v1/kv_cache_interface.cpp index 01fdf37d6..6143a7902 100644 --- a/src/vllm/v1/kv_cache_interface.cpp +++ b/src/vllm/v1/kv_cache_interface.cpp @@ -159,4 +159,84 @@ int64_t KVBytesPerBlock(const KVCacheConfig& config) { return bytes; } +namespace { + +// The ONE arithmetic statement W3 makes about block sizing, written where it can +// be read beside the specs it rewrites: an fp8 KV element is 1 byte where bf16 +// is 2 (`torch_utils.py:38-40` maps every fp8 CacheDType to `torch.uint8`), and +// `AttentionSpec::real_page_size_bytes` is linear in `vt::SizeOf(dtype)`, so the +// page halves. If this and the store ever disagree about the element size the +// result is wrong tokens rather than a crash, so the storage dtype is asserted +// here rather than assumed. +void RetypeAttentionSpec(AttentionSpec& spec, const ResolvedCacheDType& resolved, + float k_scale, float v_scale) { + VT_CHECK(dynamic_cast(&spec) == nullptr, + "cache_dtype: an MLA KV cache has its own quantized page formula " + "upstream (fp8_ds_mla, kv_cache_interface.py:398-410) and no " + "cache_dtype override is wired for it; run the MLA model on " + "--kv-cache-dtype auto"); + if (resolved.is_fp8) { + VT_CHECK(resolved.storage == vt::DType::kI8, + "cache_dtype: an fp8 KV cache stores 1 byte per element " + "(vt::DType::kI8); any other storage dtype would size the block " + "for one element width and store another"); + VT_CHECK(vt::SizeOf(resolved.storage) == 1, + "cache_dtype: the fp8 KV storage dtype must be exactly 1 byte"); + VT_CHECK(resolved.fp8_kind == vt::Fp8KVCacheDataType::kFp8E4M3, + "cache_dtype: only fp8_e4m3 is implemented on the KV store and " + "read (fp8_e5m2 compute is a named later brick, KV-FP8 W5)"); + VT_CHECK(k_scale > 0.0F && v_scale > 0.0F, + "cache_dtype: an fp8 KV cache needs k_scale/v_scale > 0"); + } else { + // Explicit float overrides. bfloat16 is the storage dtype the KV path + // already writes; float16 parses (the CacheDType surface is mirrored in + // full) but no model's attention block casts K/V to f16 before the store, + // so it is refused by name instead of reaching a dtype mismatch deep in an + // op wrapper. + VT_CHECK(resolved.storage == vt::DType::kBF16, + "cache_dtype: only 'auto', 'bfloat16', 'fp8' and 'fp8_e4m3' are " + "wired to the KV store; 'float16' parses but no attention block " + "casts K/V to f16 before the store (KV-FP8 W3, owed)"); + } + spec.dtype = resolved.storage; + spec.fp8_kind = resolved.fp8_kind; + spec.k_scale = k_scale; + spec.v_scale = v_scale; +} + +} // namespace + +void ApplyCacheDType(KVCacheConfig& config, const ResolvedCacheDType& resolved, + float k_scale, float v_scale) { + const auto retype = [&](AttentionSpec& spec) { + // NOTHING TO APPLY, and this is the whole default path. "auto" resolves to + // the model dtype, which is exactly what every KV-cache factory already + // built the spec with (`ResolveKvCacheDType()`), so the write would set the + // field to the value it holds. Returning first keeps the default load + // byte-identical AND keeps the refusals below out of its way: an MLA model + // on `--kv-cache-dtype auto`, and the `VT_KV_CACHE_F32=1` A/B cache, must + // both keep loading, and neither is asking for anything to change. + if (!resolved.is_fp8 && spec.dtype == resolved.storage && + spec.fp8_kind == vt::Fp8KVCacheDataType::kAuto) { + return; + } + RetypeAttentionSpec(spec, resolved, k_scale, v_scale); + }; + for (auto& group : config.kv_cache_groups) { + auto* attn = dynamic_cast(group.kv_cache_spec.get()); + // MambaSpec (recurrent conv/SSM state) is NOT the KV cache: upstream sizes + // it from its own `mamba_cache_dtype`/`mamba_ssm_cache_dtype` knobs + // (`config/cache.py:131-138`) and `--kv-cache-dtype` never touches it. + if (attn == nullptr) continue; + retype(*attn); + } + // The heterogeneous-per-layer seam (Gemma-4) allocates from these instead of + // the group spec, so a rewrite that skipped them would half-size the pool and + // leave the layers writing full-width bytes into it. + for (auto& spec : config.per_layer_attn_specs) { + if (spec == nullptr) continue; + retype(*spec); + } +} + } // namespace vllm::v1 diff --git a/src/vllm/v1/worker/gpu/runner.cpp b/src/vllm/v1/worker/gpu/runner.cpp index 697cfc460..75848736c 100644 --- a/src/vllm/v1/worker/gpu/runner.cpp +++ b/src/vllm/v1/worker/gpu/runner.cpp @@ -687,6 +687,11 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { int64_t Dh = 0; int64_t fa_page_bytes = 0; vt::DType kv_dtype = ResolveKvCacheDType(); + // KV-FP8 W3: the group-level fp8 interpretation + scales, defaulting to the + // inert float path. Read off the SAME AttentionSpec as `kv_dtype` below. + vt::Fp8KVCacheDataType kv_fp8_kind = vt::Fp8KVCacheDataType::kAuto; + float kv_k_scale = 1.0F; + float kv_v_scale = 1.0F; if (full_attn_group_id_ >= 0) { const KVCacheSpec* fa_spec = kv_cache_config.kv_cache_groups[static_cast(full_attn_group_id_)] @@ -698,6 +703,9 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { Hkv = attn_spec->num_kv_heads; Dh = attn_spec->head_size; kv_dtype = attn_spec->dtype; + kv_fp8_kind = attn_spec->fp8_kind; + kv_k_scale = attn_spec->k_scale; + kv_v_scale = attn_spec->v_scale; fa_page_bytes = attn_spec->page_size_bytes(); // The PagedKvCache view carries ONE head_size, so an asymmetric-V full // attention layer cannot be viewed by it. MLA's own view (a later W) is a @@ -853,6 +861,12 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { int64_t num_kv_heads; int64_t head_size; vt::DType dtype; + // KV-FP8 W3: the fp8 interpretation + per-tensor scales, carried from the + // same spec that supplied `dtype` and `page_size_bytes()`. They travel with + // the dtype because the page width and the byte's meaning are one decision. + vt::Fp8KVCacheDataType fp8_kind; + float k_scale; + float v_scale; }; std::vector fa_dims; // Parallel to fa_dims: 1 when the layer's spec kind is kMlaAttention (the @@ -912,6 +926,9 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { int64_t l_Dh = Dh; int64_t l_page = fa_page_bytes; vt::DType l_dtype = kv_dtype; + vt::Fp8KVCacheDataType l_fp8_kind = kv_fp8_kind; + float l_k_scale = kv_k_scale; + float l_v_scale = kv_v_scale; if (has_per_layer) { const std::shared_ptr& sp = kv_cache_config.per_layer_attn_specs[static_cast(l)]; @@ -920,6 +937,9 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { l_Hkv = sp->num_kv_heads; l_Dh = sp->head_size; l_dtype = sp->dtype; + l_fp8_kind = sp->fp8_kind; + l_k_scale = sp->k_scale; + l_v_scale = sp->v_scale; l_page = sp->page_size_bytes(); // Same guard as the group spec: the PagedKvCache view carries ONE // head_size, so an asymmetric-V layer is not expressible in it. @@ -935,7 +955,8 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { dev, queue_, static_cast(num_blocks_) * static_cast(l_page), kv_cache_backend_resident_)); - fa_dims.push_back(FaDims{l_Hkv, l_Dh, l_dtype}); + fa_dims.push_back( + FaDims{l_Hkv, l_Dh, l_dtype, l_fp8_kind, l_k_scale, l_v_scale}); // Per-layer MLA flag, parallel to fa_dims: the view loop picks the right // backend name (TRITON_MLA for an MLA group) and the right expected KV // shape (fused 3-dim, not the NHD 5-dim) per group. @@ -972,6 +993,12 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { kv.block_size = fa_block_size; kv.num_kv_heads = fa_dims[i].num_kv_heads; kv.head_size = fa_dims[i].head_size; + // KV-FP8 W3: the fp8 interpretation + scales reach the model's attention + // block ONLY through this view, which is what makes `--kv-cache-dtype fp8` + // a served capability rather than a resized allocation. + kv.fp8_kind = fa_dims[i].fp8_kind; + kv.k_scale = fa_dims[i].k_scale; + kv.v_scale = fa_dims[i].v_scale; // M3: the backend selection resolved for THIS group must describe the view // geometry the engine allocates + KvSlice reads — the NHD 5-dim // (num_blocks, 2, block_size, num_kv_heads, head_size) for a dense group, @@ -998,7 +1025,17 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { cfg.head_size = static_cast(fa_dims[i].head_size); cfg.num_heads = static_cast(fa_dims[i].num_kv_heads); cfg.block_size = static_cast(fa_block_size); - cfg.kv_cache_dtype = vllm::v1::KvCacheDTypeName(fa_dims[i].dtype); + // KV-FP8 W3: `KvCacheDTypeName` cannot answer this one, and deliberately so + // — a bare `kI8` byte does not know its semantic type (`vt/dtype.h:20-32`), + // exactly as upstream stores every fp8 flavour as `torch.uint8` + // (`torch_utils.py:38-40`) and reads the flavour off the layer. The + // interpretation is the thing that knows, so ask it. + cfg.kv_cache_dtype = + fa_dims[i].fp8_kind == vt::Fp8KVCacheDataType::kFp8E4M3 + ? "fp8_e4m3" + : (fa_dims[i].fp8_kind == vt::Fp8KVCacheDataType::kFp8E5M2 + ? "fp8_e5m2" + : vllm::v1::KvCacheDTypeName(fa_dims[i].dtype)); cfg.quantized_kv_cache = vllm::v1::IsQuantizedKvCacheName(cfg.kv_cache_dtype); std::string name; @@ -1082,6 +1119,12 @@ void GPUModelRunner::initialize_kv_cache(const KVCacheConfig& kv_cache_config) { dkv.block_size = fa_block_size; dkv.num_kv_heads = Hkv; dkv.head_size = Dh; + // The draft layer is sized from `fa_page_bytes` — the TARGET group's page + // — so it must carry the target's storage dtype AND its fp8 interpretation + // or the two disagree about element width over one shared block table. + dkv.fp8_kind = kv_fp8_kind; + dkv.k_scale = kv_k_scale; + dkv.v_scale = kv_v_scale; draft_attn_kv_.push_back(dkv); break; // exactly one fa_draft group at k=1. } diff --git a/src/vt/ops.cpp b/src/vt/ops.cpp index 092150546..0961a0807 100644 --- a/src/vt/ops.cpp +++ b/src/vt/ops.cpp @@ -3389,6 +3389,19 @@ void ReshapeAndCache(Queue& q, const Tensor& k, const Tensor& v, Tensor& k_cache // trailing rows (CUDA-graph padding) that are ignored. VT_CHECK(k.shape[0] >= slot_mapping.shape[0], "reshape_and_cache: num_tokens (k.shape[0]) must be >= slot_mapping length"); + // KV-FP8 W3 — the loud end of the half-sized-block hazard. A `kI8` cache page + // is one byte per element because the KV-cache spec was sized that way; a + // float store into it would write two bytes per element at offsets computed + // for one, which is wrong tokens rather than an out-of-bounds. Every + // attention block that has been routed calls `vt::ReshapeAndCacheFp8` here + // instead (`include/vllm/model_executor/models/kv_cache_route.h`), so reaching + // this line means THIS architecture's attention block has not been routed — + // and the message says so rather than reporting a dtype mismatch. + VT_CHECK(k_cache.dtype != DType::kI8 && v_cache.dtype != DType::kI8, + "reshape_and_cache: this cache is 1-byte fp8 storage (DType::kI8) but " + "this is the float (auto) store; an fp8 KV cache must be written " + "through vt::ReshapeAndCacheFp8. This model's attention block is not " + "routed for fp8 KV (KV-FP8 W3) — run it on --kv-cache-dtype auto"); VT_CHECK(IsFloat(k.dtype) && k.dtype == v.dtype && k_cache.dtype == k.dtype && v_cache.dtype == k.dtype, "reshape_and_cache: k/v/k_cache/v_cache must share one float dtype (auto cache path)"); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 0a5fd872a..6f4810550 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1450,6 +1450,9 @@ vllm_cpp_add_test(test_output_processor vllm/v1/test_output_processor.cpp) vllm_cpp_add_test(test_async_llm vllm/v1/test_async_llm.cpp) vllm_cpp_add_test(test_llm_engine vllm/v1/test_llm_engine.cpp) vllm_cpp_add_test(test_loaded_engine_dense vllm/entrypoints/test_loaded_engine_dense.cpp) +# KV-FP8 W3 (#1593): half-sized KV blocks, --kv-cache-dtype threading and the +# checkpoint k/v-scale path, entered through the LoadedEngine loader. +vllm_cpp_add_test(test_kv_cache_fp8_wiring vllm/entrypoints/test_kv_cache_fp8_wiring.cpp) vllm_cpp_add_test(test_dspark_block_size_guard vllm/entrypoints/test_dspark_block_size_guard.cpp) # ENG-HF-MODEL-DOWNLOAD W2 (#1280): the HuggingFace cache walk entered through # the production loader, with a repository identifier rather than a path. The @@ -1690,6 +1693,11 @@ if(VLLM_CPP_SERVER) vllm_cpp_add_test(test_serve_residency_config vllm/entrypoints/openai/test_serve_residency_config.cpp) target_include_directories(test_serve_residency_config PRIVATE ${CMAKE_SOURCE_DIR}/src) + # KV-FP8 W3 (#1593): --kv-cache-dtype reaching EngineParams through the REAL + # VllmServerMain. Same shape and the same reason as the row above. + vllm_cpp_add_test(test_serve_kv_cache_dtype + vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp) + target_include_directories(test_serve_kv_cache_dtype PRIVATE ${CMAKE_SOURCE_DIR}/src) endif() vllm_cpp_add_test(test_outputs vllm/test_outputs.cpp) diff --git a/tests/vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp b/tests/vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp new file mode 100644 index 000000000..5d08eba75 --- /dev/null +++ b/tests/vllm/entrypoints/openai/test_serve_kv_cache_dtype.cpp @@ -0,0 +1,221 @@ +// KV-FP8 W3 (#1593) — the SERVER end of the chain: does `--kv-cache-dtype` +// actually reach `EngineParams` when a real `vllm serve` line carries it? +// +// This is the row's reachability gate for its own headline flag, in the sense of +// .agents/reachability.md. `tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp` +// proves the resolver, the block arithmetic, the store and the read, and it +// enters through `LoadedEngine`; none of it touches argument parsing. Deleting +// BOTH the `--kv-cache-dtype` arm of `ParseArgs` and the +// `engine_params.kv_cache_dtype = args.kv_cache_dtype` assignment left that file +//19/19, 89/89 SUCCESS, and no other test in the tree named the flag. A flag +// nothing reads is exactly the shape AGENTS.md `## Nothing lands dead` is about. +// +// WHAT MAKES IT OBSERVABLE WITHOUT A CHECKPOINT. `LoadedEngine::FromModelDir` +// resolves `--kv-cache-dtype` against the checkpoint's own `kv_cache_quant_algo` +// in its second statement, before any weight operation, and it SAYS SO on stderr +// when the checkpoint is what decided. So a model directory that carries only a +// `hf_quant_config.json` produces that line and then fails on the absent weights. +// The line is present when no flag was typed, and ABSENT when one was — because +// an explicit value is returned unchanged and the checkpoint is never consulted +// (`vllm/utils/torch_utils.py:380-381`). +// +// That polarity is the gate. A run whose flag never arrived would carry the +// default "auto" into the resolver, the checkpoint would win, and CASE 2's line +// would appear. Deleting either half of the chain therefore reddens CASE 2. +// +// WHY A SUBPROCESS. `ParseArgs` reports a bad argument through `Usage()`, which +// calls `std::exit`, so an in-process call would take the test binary with it. +// Each case re-execs this binary into a skip-decorated child that calls +// `VllmServerMain` on argv assembled from `VLLM_TEST_SERVE_ARGS` — the harness +// tests/vllm/entrypoints/openai/test_serve_residency_config.cpp establishes and +// this file mirrors, including its no-spaces-in-arguments limitation. +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "vllm/entrypoints/openai/server_main.h" + +namespace { + +// The gate checkpoint's own declaration, in the file ModelOpt 0.29.0 and before +// wrote it to. Transcribed from `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` +// @ `36f717a22990e82c54c1d48ee77c491b87825680` and trimmed to the keys the +// resolver reads; `test_kv_cache_fp8_wiring.cpp` carries the same bytes and the +// provenance note. +constexpr const char* kDeclaresFp8 = + R"({"producer":{"name":"modelopt"},)" + R"("quantization":{"quant_algo":"MIXED_PRECISION","kv_cache_quant_algo":"FP8"}})"; + +// Printed by VllmServerMain AFTER ParseArgs returns. Its presence proves +// argument parsing succeeded and control reached engine construction. +constexpr const char* kPostParseBanner = "server: request logging"; + +// The loader's resolution line. It names the checkpoint as the decider, which is +// the only reason it can be used as a signal for "no explicit flag arrived". +constexpr const char* kCheckpointDecided = + "the checkpoint declares kv_cache_quant_algo"; + +constexpr const char* kUnknownArgument = "server: unknown argument"; + +bool Contains(const std::string& haystack, const std::string& needle) { + return haystack.find(needle) != std::string::npos; +} + +// A throwaway model directory carrying only `hf_quant_config.json`. No weights, +// no tokenizer: the load is MEANT to fail, after the resolution stanza. +class DeclaringCheckpoint { + public: + DeclaringCheckpoint() { + static int counter = 0; + path_ = std::filesystem::temp_directory_path() / + ("vllm_serve_kvfp8_" + std::to_string(counter++)); + std::error_code ec; + std::filesystem::remove_all(path_, ec); + std::filesystem::create_directories(path_); + std::ofstream(path_ / "hf_quant_config.json", std::ios::binary) + << kDeclaresFp8; + } + ~DeclaringCheckpoint() { + std::error_code ec; + std::filesystem::remove_all(path_, ec); + } + DeclaringCheckpoint(const DeclaringCheckpoint&) = delete; + DeclaringCheckpoint& operator=(const DeclaringCheckpoint&) = delete; + + std::string str() const { return path_.string(); } + + private: + std::filesystem::path path_; +}; + +struct ChildRun { + std::string output; // stdout + stderr, combined + int status = -1; +}; + +ChildRun RunServer(const std::string& serve_args) { + // Resolve our own path in the PARENT: popen runs under /bin/sh, so a literal + // /proc/self/exe inside the command would resolve to the shell. + char exe[4096]; + const ssize_t n = ::readlink("/proc/self/exe", exe, sizeof(exe) - 1); + REQUIRE(n > 0); + exe[n] = '\0'; + const std::string cmd = "VLLM_TEST_SERVE_ARGS='" + serve_args + "' " + + std::string(exe) + + " --no-skip --test-case='serve_kv_cache_dtype_child'" + " 2>&1"; + FILE* pipe = ::popen(cmd.c_str(), "r"); + REQUIRE(pipe != nullptr); + ChildRun run; + std::array buf{}; + while (std::fgets(buf.data(), static_cast(buf.size()), pipe) != nullptr) { + run.output += buf.data(); + } + const int closed = ::pclose(pipe); + REQUIRE(closed != -1); + run.status = WIFEXITED(closed) ? WEXITSTATUS(closed) : -1; + return run; +} + +std::vector SplitOnSpaces(const std::string& text) { + std::vector out; + std::string current; + for (const char c : text) { + if (c == ' ') { + if (!current.empty()) out.push_back(current); + current.clear(); + } else { + current.push_back(c); + } + } + if (!current.empty()) out.push_back(current); + return out; +} + +} // namespace + +// The CHILD case, filtered out of a normal run and executed only when a parent +// re-execs it by name. +TEST_CASE("serve_kv_cache_dtype_child" * doctest::skip()) { + const char* raw = std::getenv("VLLM_TEST_SERVE_ARGS"); + REQUIRE(raw != nullptr); + std::vector args{"vllm-server"}; + for (std::string& token : SplitOnSpaces(raw)) { + args.push_back(std::move(token)); + } + std::vector argv; + argv.reserve(args.size()); + for (std::string& arg : args) argv.push_back(arg.data()); + const int rc = vllm::entrypoints::openai::VllmServerMain( + static_cast(argv.size()), argv.data()); + std::cout << "SERVE_RC=" << rc << "\n" << std::flush; + // Leave immediately: doctest's own summary would otherwise be mistaken for the + // server's verdict, and the parent reads this process's exit status. + std::exit(0); +} + +// CASE 1 — THE CONTROL. No `--kv-cache-dtype` at all, and the checkpoint's own +// declaration decides. It establishes that the signal CASE 2 waits for is really +// produced by this directory, so CASE 2's absence means "the flag won" rather +// than "the line never prints here". +TEST_CASE("serve: with no --kv-cache-dtype the checkpoint's declaration decides") { + const DeclaringCheckpoint model; + const ChildRun run = RunServer("--model " + model.str()); + INFO("child output:\n" << run.output); + + CHECK_FALSE(Contains(run.output, kUnknownArgument)); + CHECK(Contains(run.output, kPostParseBanner)); + CHECK(Contains(run.output, kCheckpointDecided)); + // The RESOLVED value, which is what every consumer downstream sees. + CHECK(Contains(run.output, "fp8_e4m3")); + // The load then failed on the deliberately absent weights, which is what makes + // the stanza observable without a checkpoint. + CHECK(Contains(run.output, "SERVE_RC=")); + CHECK_FALSE(Contains(run.output, "SERVE_RC=0")); + CHECK(run.status == 0); +} + +// CASE 2 — THE REACHABILITY CASE. The same directory, one typed +// `--kv-cache-dtype`, and the checkpoint is not consulted at all. This is the +// case that reddens when the `--kv-cache-dtype` arm of `ParseArgs` is deleted, +// and again when `engine_params.kv_cache_dtype = args.kv_cache_dtype` is +// deleted: either way the default "auto" arrives, the checkpoint wins, and the +// line comes back. +TEST_CASE("serve: --kv-cache-dtype reaches EngineParams and OUTRANKS the checkpoint") { + const DeclaringCheckpoint model; + const ChildRun run = + RunServer("--model " + model.str() + " --kv-cache-dtype bfloat16"); + INFO("child output:\n" << run.output); + + CHECK_FALSE(Contains(run.output, kUnknownArgument)); + CHECK(Contains(run.output, kPostParseBanner)); + CHECK_FALSE(Contains(run.output, kCheckpointDecided)); + CHECK(Contains(run.output, "SERVE_RC=")); + CHECK(run.status == 0); +} + +// CASE 3 — and `fp8` is a value the flag accepts, not only a value the loader +// resolves to. Without it CASE 2 could be satisfied by a parser that took the +// flag and refused every fp8 spelling, which is the one arm this row ships for. +TEST_CASE("serve: --kv-cache-dtype fp8 parses and reaches the loader") { + const DeclaringCheckpoint model; + const ChildRun run = + RunServer("--model " + model.str() + " --kv-cache-dtype fp8"); + INFO("child output:\n" << run.output); + + CHECK_FALSE(Contains(run.output, kUnknownArgument)); + CHECK(Contains(run.output, kPostParseBanner)); + CHECK(Contains(run.output, "server: loading model from")); + CHECK_FALSE(Contains(run.output, kCheckpointDecided)); + CHECK(run.status == 0); +} diff --git a/tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp b/tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp new file mode 100644 index 000000000..aa3b38397 --- /dev/null +++ b/tests/vllm/entrypoints/test_kv_cache_fp8_wiring.cpp @@ -0,0 +1,1907 @@ +// KV-FP8 W3 gate — the RUNNER integration: half-sized KV blocks, the +// `--kv-cache-dtype` thread from the flag to the block sizing, and the +// checkpoint `k_scale`/`v_scale` path. +// +// Upstream anchors, all verified in /home/mudler/_git/vllm at the parity pin +// `555967922`: +// * `vllm/config/cache.py:19-36` CacheDType, `:76` cache_dtype default, +// `:111` calculate_kv_scales (deprecated). +// * `vllm/utils/torch_utils.py:32-52` STR_DTYPE_TO_TORCH_DTYPE (every fp8 +// CacheDType maps to `torch.uint8` — ONE byte), `:64-67` +// MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP, `:75-80` is_quantized_kv_cache, +// `:310-362` get_kv_cache_quant_algo_string, `:374-392` +// resolve_kv_cache_dtype_string, `:394-401` kv_cache_dtype_str_to_dtype. +// * `vllm/v1/worker/gpu_model_runner.py:484-486` — the runner resolves ONE +// kv_cache_dtype and every attention spec is built with it. +// * `vllm/v1/kv_cache_interface.py:204-218` AttentionSpec.real_page_size_bytes +// — linear in `get_dtype_size(self.dtype)`, which is the whole halving. +// * `vllm/model_executor/layers/quantization/kv_cache.py:18-30` +// KVCacheScaleParameter (the -1.0 unloaded sentinel), `:100-102` the +// is_quantized_kv_cache guard, `:104-127` the three loaded arms, `:150-156` +// the uncalibrated warning. +// * `vllm/engine/arg_utils.py:1915-1929` — the resolution happens ONCE, at +// config construction, and CacheConfig receives the resolved string. +// +// THE CASES ARE ORDERED BY WHAT THEY WOULD LET THROUGH IF THEY WERE MISSING: +// G1 the checkpoint declaration resolves, and an explicit flag outranks it +// G2 a declared-but-absent scale is NOT the same state as no declaration +// G3 the block arithmetic — an fp8 page is EXACTLY half a bf16 page +// G4 the same halving through the LOADER: the same byte budget buys 2x blocks +// G5 the fp8 KV path is REACHED from a production entry point (generation) +// G6 storage dtype and fp8 interpretation cannot disagree +// G7 an unrouted attention block is refused BY NAME, never silently +// G8 the refusals: MLA, float16, e5m2, and a Mamba state left alone +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "vllm/config/cache.h" +#include "vllm/config/speculative.h" +#include "vllm/entrypoints/model_loader.h" +#include "vllm/model_executor/layers/quantization/kv_cache.h" +#include "vllm/model_executor/models/kv_cache_route.h" +#include "vllm/model_executor/models/qwen3.h" +#include "vllm/model_executor/models/qwen3_5_dense.h" +#include "vllm/sampling_params.h" +#include "vllm/tokenizer/bpe.h" +#include "vllm/tokenizer/tokenizer.h" +#include "vllm/transformers_utils/hf_config.h" +#include "vllm/v1/kv_cache_dtype.h" +#include "vllm/v1/kv_cache_interface.h" +#include "vt/backend.h" +#include "vt/dtype.h" +#include "vt/ops.h" + +using nlohmann::json; +using vllm::HfConfig; +using vllm::KvScaleOrigin; +using vllm::OwnedTensor; +using vllm::entrypoints::EngineParams; +using vllm::entrypoints::LoadedEngine; +using vt::DType; + +namespace { + +// ─── The gate checkpoint's own declaration, transcribed ────────────────────── +// +// `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` @ `36f717a22990e82c54c1d48ee77c491b8 +// 7825680`, the subject of benchmark campaign #1574. Fetched from the public +// `hf_quant_config.json` on 2026-08-21 and trimmed to the three keys this +// resolver reads; the `quantized_layers` map (1900+ entries) is the WEIGHT half +// and is read elsewhere. +// +// The same revision's `model.safetensors.index.json` lists 2001 tensors and +// ZERO named `k_scale`, `v_scale` or `kv_scale` — measured, not assumed. That +// pair of facts is the whole reason G2 exists. +constexpr const char* kGateCheckpointQuantConfig = R"({ + "producer": {"name": "modelopt", "version": "0.46.0rc1"}, + "quantization": { + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": "FP8", + "quantized_layers": { + "model.language_model.layers.0.mlp.gate_proj": + {"quant_algo": "W4A16_NVFP4", "group_size": 16} + } + } +})"; + +// A modelopt checkpoint that quantizes WEIGHTS and declares nothing about the +// KV cache — the case that must not reach a default scale. +constexpr const char* kNoKvDeclarationQuantConfig = R"({ + "producer": {"name": "modelopt", "version": "0.46.0rc1"}, + "quantization": {"quant_algo": "FP8", "quantized_layers": {}} +})"; + +// The SAME declaration in the place ModelOpt 0.31.0 and after writes it: inline +// under `config.json:quantization_config`, flat, with `quant_method` beside the +// algorithm (`vllm/transformers_utils/config.py:751-753`). It quantizes weights +// and says nothing about the KV cache, so a loader that reads it takes the model +// dtype — and a loader that reaches past it to a stale `hf_quant_config.json` +// does not. +constexpr const char* kInlineWeightsOnlyQuantConfig = R"({ + "model_type": "qwen3_5_text", + "quantization_config": {"quant_method": "modelopt", "quant_algo": "FP8"} +})"; + +// AND THE ONE THE GATE CHECKPOINT ACTUALLY SHIPS. Fetched from the same +// revision's `config.json` on 2026-08-22 and trimmed to the keys this resolver +// reads (`config_groups`, `quantized_layers` and `ignore` are the WEIGHT half; +// `quantized_layers` holds 401 entries). It carries `quant_method` and +// `producer` — so the modelopt marker is present twice over — and NO +// `kv_cache_*` key anywhere. Because `config.json:quantization_config` is read +// in preference to `hf_quant_config.json`, THIS is the document that decides, +// and it declares nothing. The transcription above is the file that does not +// get read for this checkpoint. +constexpr const char* kGateCheckpointInlineConfig = R"({ + "model_type": "qwen3_5", + "quantization_config": { + "quant_algo": "MIXED_PRECISION", + "producer": {"name": "modelopt", "version": "0.46.0rc1"}, + "quant_method": "modelopt", + "quantized_layers": { + "model.language_model.layers.0.mlp.gate_proj": + {"quant_algo": "W4A16_NVFP4", "group_size": 16} + } + } +})"; + +// ─── Synthetic dense-hybrid model (the same shape as +// tests/vllm/entrypoints/test_loaded_engine_dense.cpp, which is the file whose +// LOADER path these cases enter through) ───────────────────────────────────── +uint64_t Mix(uint64_t x) { + x += 0x9E3779B97F4A7C15ULL; + x = (x ^ (x >> 30)) * 0xBF58476D1CE4E5B9ULL; + x = (x ^ (x >> 27)) * 0x94D049BB133111EBULL; + return x ^ (x >> 31); +} +float RandV(uint64_t seed) { + const double u = + static_cast(Mix(seed) >> 40) / static_cast(1 << 24); + return static_cast(u * 0.16 - 0.08); +} +OwnedTensor MakeOwned(DType dt, std::vector shape, uint64_t seed) { + OwnedTensor t; + t.dtype = dt; + t.rank = static_cast(shape.size()); + int64_t n = 1; + for (int i = 0; i < t.rank; ++i) { + t.shape[i] = shape[static_cast(i)]; + n *= shape[static_cast(i)]; + } + if (dt == DType::kBF16) { + t.bytes.resize(static_cast(n) * 2); + auto* p = reinterpret_cast(t.bytes.data()); + for (int64_t i = 0; i < n; ++i) + p[i] = vt::F32ToBF16(RandV(seed + static_cast(i))); + } else { + t.bytes.resize(static_cast(n) * 4); + auto* p = reinterpret_cast(t.bytes.data()); + for (int64_t i = 0; i < n; ++i) p[i] = RandV(seed + static_cast(i)); + } + return t; +} + +constexpr int kVocab = 24; +constexpr int kMaxModelLen = 32; + +HfConfig MakeDenseConfig() { + HfConfig c; + c.model_type = "qwen3_5_text"; + c.architectures = {"Qwen3_5ForConditionalGeneration"}; + c.hidden_size = 32; + c.num_hidden_layers = 4; + c.vocab_size = kVocab; + c.num_attention_heads = 6; + c.num_key_value_heads = 2; + c.head_dim = 8; + c.layer_types = {"linear_attention", "linear_attention", "linear_attention", + "full_attention"}; + c.intermediate_size = 16; + c.num_experts = 0; + c.linear_num_key_heads = 2; + c.linear_num_value_heads = 6; + c.linear_key_head_dim = 8; + c.linear_value_head_dim = 8; + c.linear_conv_kernel_dim = 4; + c.rope_theta = 10000.0; + c.rotary_dim = 4; + c.rms_norm_eps = 1e-6; + c.max_position_embeddings = kMaxModelLen; + c.raw = json::object(); + return c; +} + +vllm::DenseMlpWeights MakeMlp(const HfConfig& c, uint64_t s) { + vllm::DenseMlpWeights m; + const int64_t H = c.hidden_size, I = c.intermediate_size; + m.gate_proj = MakeOwned(DType::kBF16, {H, I}, s + 1); + m.up_proj = MakeOwned(DType::kBF16, {H, I}, s + 2); + m.down_proj = MakeOwned(DType::kBF16, {I, H}, s + 3); + return m; +} + +vllm::Qwen3_5DenseWeights MakeDenseWeights(const HfConfig& c) { + vllm::Qwen3_5DenseWeights w; + const int64_t H = c.hidden_size, V = c.vocab_size; + const int64_t Hq = c.num_attention_heads, Hkv = c.num_key_value_heads, + Dh = c.head_dim; + const int64_t Hk = c.linear_num_key_heads, Hv = c.linear_num_value_heads, + Dk = c.linear_key_head_dim, Dv = c.linear_value_head_dim, + Kw = c.linear_conv_kernel_dim; + const int64_t key_dim = Hk * Dk, value_dim = Hv * Dv, + conv_dim = 2 * key_dim + value_dim; + w.embed_tokens = MakeOwned(DType::kBF16, {V, H}, 11); + w.final_norm = MakeOwned(DType::kBF16, {H}, 12); + w.lm_head = MakeOwned(DType::kBF16, {H, V}, 13); + for (int64_t l = 0; l < c.num_hidden_layers; ++l) { + const uint64_t s = 1000 + static_cast(l) * 5000; + vllm::Qwen3_5DenseLayerWeights lw; + lw.is_linear_attention = + (c.layer_types[static_cast(l)] == "linear_attention"); + lw.input_layernorm = MakeOwned(DType::kBF16, {H}, s + 1); + lw.post_attention_layernorm = MakeOwned(DType::kBF16, {H}, s + 2); + if (lw.is_linear_attention) { + lw.gdn.in_proj_qkv = MakeOwned(DType::kBF16, {H, conv_dim}, s + 10); + lw.gdn.in_proj_z = MakeOwned(DType::kBF16, {H, value_dim}, s + 20); + lw.gdn.in_proj_b = MakeOwned(DType::kBF16, {H, Hv}, s + 30); + lw.gdn.in_proj_a = MakeOwned(DType::kBF16, {H, Hv}, s + 40); + lw.gdn.conv1d_weight = MakeOwned(DType::kBF16, {conv_dim, Kw}, s + 50); + lw.gdn.a_log = MakeOwned(DType::kF32, {Hv}, s + 60); + lw.gdn.dt_bias = MakeOwned(DType::kF32, {Hv}, s + 70); + lw.gdn.norm_weight = MakeOwned(DType::kBF16, {Dv}, s + 80); + lw.gdn.out_proj = MakeOwned(DType::kBF16, {value_dim, H}, s + 90); + } else { + lw.attn.q_proj = MakeOwned(DType::kBF16, {H, 2 * Hq * Dh}, s + 10); + lw.attn.k_proj = MakeOwned(DType::kBF16, {H, Hkv * Dh}, s + 20); + lw.attn.v_proj = MakeOwned(DType::kBF16, {H, Hkv * Dh}, s + 30); + lw.attn.o_proj = MakeOwned(DType::kBF16, {Hq * Dh, H}, s + 40); + lw.attn.q_norm = MakeOwned(DType::kBF16, {Dh}, s + 50); + lw.attn.k_norm = MakeOwned(DType::kBF16, {Dh}, s + 60); + } + lw.mlp = MakeMlp(c, s + 500); + w.layers.push_back(std::move(lw)); + } + return w; +} + +// The same model with its full-attention K and V projections in the RAW torch +// Linear layout every real safetensors checkpoint ships: [N=out, K=in] with +// `nk` set. `ProjectFullAttnQkv` serves those through `MatmulBf16D`, so the +// projection emits the MODEL dtype (bf16) instead of the f32 that `MakeDenseWeights` +// happens to produce — which is the difference the fp8 store sees. See G9. +vllm::Qwen3_5DenseWeights MakeDenseWeightsTorchKv(const HfConfig& c) { + vllm::Qwen3_5DenseWeights w = MakeDenseWeights(c); + const int64_t H = c.hidden_size; + const int64_t Hkv = c.num_key_value_heads, Dh = c.head_dim; + for (size_t l = 0; l < w.layers.size(); ++l) { + vllm::Qwen3_5DenseLayerWeights& lw = w.layers[l]; + if (lw.is_linear_attention) continue; + const uint64_t s = 1000 + static_cast(l) * 5000; + const int64_t Hq = c.num_attention_heads; + // All THREE, because that is the only shape a checkpoint comes in: the fused + // preamble requires `qgate` and `kf` to share one dtype + // (`vt::AttnQkNormRopeGate`), so a half-converted layer is a harness defect + // rather than a case. + lw.attn.q_proj = MakeOwned(DType::kBF16, {2 * Hq * Dh, H}, s + 10); + lw.attn.q_proj.nk = true; + lw.attn.k_proj = MakeOwned(DType::kBF16, {Hkv * Dh, H}, s + 20); + lw.attn.k_proj.nk = true; + lw.attn.v_proj = MakeOwned(DType::kBF16, {Hkv * Dh, H}, s + 30); + lw.attn.v_proj.nk = true; + } + return w; +} + +// A throwaway model directory carrying only the files the loader's resolution +// stanza opens. `LoadedEngine::FromModelDir` is the only caller of +// `vllm::ReadQuantConfigJson`, so a case that wants to measure the LOADER has to +// hand it a directory rather than a string. +class CheckpointDir { + public: + CheckpointDir() { + static int counter = 0; + path_ = std::filesystem::temp_directory_path() / + ("vllm_kvfp8_ckpt_" + std::to_string(counter++)); + std::error_code ec; + std::filesystem::remove_all(path_, ec); + std::filesystem::create_directories(path_); + } + ~CheckpointDir() { + std::error_code ec; + std::filesystem::remove_all(path_, ec); + } + CheckpointDir(const CheckpointDir&) = delete; + CheckpointDir& operator=(const CheckpointDir&) = delete; + + void Write(const char* name, const std::string& body) const { + std::ofstream(path_ / name, std::ios::binary) << body; + } + std::string str() const { return path_.string(); } + + private: + std::filesystem::path path_; +}; + +vllm::tok::Tokenizer BuildFixture() { + static int counter = 0; + const std::string path = + (std::filesystem::temp_directory_path() / + ("vllm_kvfp8_tok_" + std::to_string(counter++) + ".json")) + .string(); + json doc; + doc["version"] = "1.0"; + doc["added_tokens"] = json::array( + {{{"id", 19}, {"content", "<|end|>"}, {"special", true}}, + {{"id", 20}, {"content", ""}, {"special", false}}, + {{"id", 21}, {"content", "<|end|>of"}, {"special", true}}}); + doc["normalizer"] = nullptr; + doc["pre_tokenizer"] = { + {"type", "Sequence"}, + {"pretokenizers", + json::array( + {{{"type", "Split"}, + {"pattern", + {{"Regex", + R"((?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?[\p{L}\p{M}]+|\p{N}| ?[^\s\p{L}\p{M}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+)"}}}, + {"behavior", "Isolated"}, + {"invert", false}}, + {{"type", "ByteLevel"}, + {"add_prefix_space", false}, + {"trim_offsets", false}, + {"use_regex", false}}})}}; + json vocab = {{"h", 0}, {"e", 1}, {"l", 2}, {"o", 3}, {"w", 4}, + {"r", 5}, {"d", 6}, {"Ġ", 7}, {"1", 8}, {"2", 9}, + {"ll", 10}, {"he", 11}, {"llo", 12}, {"hello", 13}, + {"Ġw", 14}, {"or", 15}, {"orld", 16}, {"Ġworld", 17}, + {"ld", 18}}; + vocab[vllm::tok::MapBytesToUnicode("\xF0\x9F")] = 22; + vocab[vllm::tok::MapBytesToUnicode("\x8C\x8D")] = 23; + doc["model"] = { + {"type", "BPE"}, + {"ignore_merges", false}, + {"vocab", vocab}, + {"merges", + json::array({json::array({"l", "l"}), json::array({"h", "e"}), + json::array({"ll", "o"}), json::array({"he", "llo"}), + json::array({"Ġ", "w"}), json::array({"o", "r"}), + json::array({"l", "d"}), json::array({"or", "ld"}), + json::array({"Ġw", "orld"})})}}; + std::ofstream(path, std::ios::binary) << doc.dump(); + vllm::tok::Tokenizer tok = vllm::tok::Tokenizer::FromHfJson(path); + std::remove(path.c_str()); + return tok; +} + +// The absolute KV budget both arms of G4 are given. Large enough that the fp8 +// arm's doubled block count is well inside the pool the tiny model needs, and +// EXACTLY divisible by both per-block sizes so the 2x is an equality rather +// than a rounding coincidence. +constexpr int64_t kKvBudgetBytes = 1 << 20; + +EngineParams ParamsWithCacheDType(const std::string& cache_dtype) { + EngineParams p; + p.kv_cache_memory_bytes = kKvBudgetBytes; + p.kv_cache_dtype = cache_dtype; + return p; +} + +// The single full-attention group's spec out of a loaded engine's RESOLVED KV +// config. The synthetic model has exactly one (three GDN layers + one full +// attention layer), so "the first attention spec" is unambiguous. +const vllm::v1::AttentionSpec* SoleAttentionSpec(const LoadedEngine& eng) { + for (const auto& group : eng.kv_cache_config().kv_cache_groups) { + const auto* attn = + dynamic_cast(group.kv_cache_spec.get()); + if (attn != nullptr) return attn; + } + return nullptr; +} + +vllm::SamplingParams Greedy(int max_tokens) { + vllm::SamplingParams sp; + sp.temperature = 0.0; + sp.max_tokens = max_tokens; + sp.output_kind = vllm::RequestOutputKind::kCumulative; + return sp; +} + +class CerrRedirect { + public: + explicit CerrRedirect(std::streambuf* target) + : previous_(std::cerr.rdbuf(target)) {} + ~CerrRedirect() { std::cerr.rdbuf(previous_); } + CerrRedirect(const CerrRedirect&) = delete; + CerrRedirect& operator=(const CerrRedirect&) = delete; + + private: + std::streambuf* previous_; +}; + +// A bare NHD KV cache pair for the op-level cases, on the CPU queue. +struct HostKvPair { + std::vector storage; + vt::Tensor k; + vt::Tensor v; +}; + +} // namespace + +// ─── G1. The checkpoint's declaration, and who outranks whom ───────────────── +TEST_CASE("kv-fp8 W3 G1: the gate checkpoint's kv_cache_quant_algo resolves") { + // torch_utils.py:374-392 + :310-362 + :64-67. "FP8" (the modelopt spelling, + // upper case) maps to vLLM's own `fp8_e4m3`, not to the bare "fp8" alias. + // + // This document names NO top-level `quant_method`, and upstream's + // `get_kv_cache_quant_algo_string` tests exactly that key (`:319`). It still + // resolves upstream, because `_normalize_quantization_config` + // (`transformers_utils/model_arch_config_convertor.py:208-247`) INJECTS the + // marker from `producer.name` into the same dict first — see the long comment + // in `src/vllm/config/cache.cpp` for the measurement. Reading only the first + // function makes this case look like a divergence; running both says it is + // the mirror. + const vllm::ResolvedCacheDTypeString r = + vllm::ResolveKvCacheDTypeString("auto", kGateCheckpointQuantConfig); + CHECK(r.cache_dtype == "fp8_e4m3"); + // The FACT that separates this from an operator who typed the flag. + CHECK(r.declared_by_checkpoint); +} + +TEST_CASE("kv-fp8 W3 G1: the modelopt marker is upstream's THREE, and no more") { + // `_normalize_quantization_config:216-235` injects `quant_method` on exactly + // two conditions — `producer["name"] == "modelopt"` (an equality, not a + // prefix) and a nested `modelopt_quant_config` key — AND ONLY WHEN the same + // nested document carries a `quant_algo` (`:224`). `torch_utils.py:319` reads + // the top-level key itself. Nothing upstream can put a marker anywhere else, + // so nothing else may be accepted here: a resolver that reads a marker + // upstream cannot see turns on an fp8 KV cache vLLM would not, at half the + // page, on a checkpoint nobody flagged. + + // (a) The flat 0.31.0-and-after shape: the marker upstream reads directly. + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"quant_method":"modelopt","quant_algo":"FP8",)" + R"("kv_cache_quant_algo":"FP8"})") + .cache_dtype == "fp8_e4m3"); + + // (b) The legacy nested shape, recognised by the key alone (`:218-220`) — + // WITH the `quant_algo` `:224` requires before it injects anything. + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"quantization":{"modelopt_quant_config":{},"quant_algo":"FP8",)" + R"("kv_cache_quant_algo":"FP8"}})") + .cache_dtype == "fp8_e4m3"); + + // (c) A marker only INSIDE `quantization`. Upstream never looks there — its + // injector writes the top level and `:319` reads the top level — so neither + // do we. + const vllm::ResolvedCacheDTypeString inner = vllm::ResolveKvCacheDTypeString( + "auto", + R"({"quantization":{"quant_method":"modelopt",)" + R"("kv_cache_quant_algo":"FP8"}})"); + CHECK(inner.cache_dtype == "auto"); + CHECK_FALSE(inner.declared_by_checkpoint); + + // (d) A producer that merely STARTS with "modelopt". `:222` is `==`, so + // `modelopt_fp4` as a PRODUCER name is not the marker (it is a `quant_method` + // VALUE the injector writes, which arm (a) already covers). The `quant_algo` + // is present in this document and in (e) so that the PRODUCER test is what + // refuses them, rather than the `:224` guard arm (g) covers. + const vllm::ResolvedCacheDTypeString near = vllm::ResolveKvCacheDTypeString( + "auto", + R"({"producer":{"name":"modelopt_fp4"},)" + R"("quantization":{"quant_algo":"FP8","kv_cache_quant_algo":"FP8"}})"); + CHECK(near.cache_dtype == "auto"); + CHECK_FALSE(near.declared_by_checkpoint); + + // (e) A non-modelopt producer with the same key is still nothing. + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"producer":{"name":"llm-compressor"},)" + R"("quantization":{"quant_algo":"FP8","kv_cache_quant_algo":"FP8"}})") + .cache_dtype == "auto"); + + // (f) The two markers are normalised DIFFERENTLY, because upstream normalises + // them differently: `quant_method` is lower-cased before the prefix test + // (`:238-246`), the producer name is compared raw against the literal + // (`:222`). A `MODELOPT` quant_method resolves; a `ModelOpt` producer does + // not. + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"quant_method":"MODELOPT","kv_cache_quant_algo":"FP8"})") + .cache_dtype == "fp8_e4m3"); + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"producer":{"name":"ModelOpt"},)" + R"("quantization":{"quant_algo":"FP8","kv_cache_quant_algo":"FP8"}})") + .cache_dtype == "auto"); + + // (g) THE `:224` GUARD. The injector writes a marker only `if quant_algo is + // not None`, read out of `quant_cfg.get("quantization", {})` — an EMPTY-object + // fallback, unlike the reader's `quant_cfg.get("quantization", quant_cfg)` at + // `torch_utils.py:321`. Three shapes therefore answer `None` upstream, and + // each of them resolved to `fp8_e4m3` here before the third review: + // + // (g1) a `modelopt` producer whose nested document declares no `quant_algo` + // (g2) a legacy `modelopt_quant_config` with the same omission + // (g3) a TOP-LEVEL `modelopt_quant_config`, with no `quantization` key at + // all — upstream's `{}` fallback means the key is not even looked for + // + // A resolver that accepts any of these halves the KV page on a checkpoint vLLM + // would run at the model dtype. + const vllm::ResolvedCacheDTypeString g1 = vllm::ResolveKvCacheDTypeString( + "auto", + R"({"producer":{"name":"modelopt"},)" + R"("quantization":{"kv_cache_quant_algo":"FP8"}})"); + CHECK(g1.cache_dtype == "auto"); + CHECK_FALSE(g1.declared_by_checkpoint); + + const vllm::ResolvedCacheDTypeString g2 = vllm::ResolveKvCacheDTypeString( + "auto", + R"({"quantization":{"modelopt_quant_config":{},)" + R"("kv_cache_quant_algo":"FP8"}})"); + CHECK(g2.cache_dtype == "auto"); + CHECK_FALSE(g2.declared_by_checkpoint); + + const vllm::ResolvedCacheDTypeString g3 = vllm::ResolveKvCacheDTypeString( + "auto", + R"({"modelopt_quant_config":{},"quant_algo":"FP8",)" + R"("kv_cache_quant_algo":"FP8"})"); + CHECK(g3.cache_dtype == "auto"); + CHECK_FALSE(g3.declared_by_checkpoint); + + // And the SAME three documents with a `quant_algo` where `:224` reads it do + // resolve, so what refuses (g1) and (g2) is the guard and not the shape. + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"producer":{"name":"modelopt"},)" + R"("quantization":{"quant_algo":"FP8","kv_cache_quant_algo":"FP8"}})") + .cache_dtype == "fp8_e4m3"); + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"quantization":{"modelopt_quant_config":{},"quant_algo":"NVFP4",)" + R"("kv_cache_quant_algo":"FP8"}})") + .cache_dtype == "fp8_e4m3"); + // (g3) has no `quantization` object, so no `quant_algo` can rescue it: the + // legacy key at the top level is not a marker in any spelling. + CHECK(vllm::ResolveKvCacheDTypeString( + "auto", + R"({"modelopt_quant_config":{"quant_algo":"FP8"},)" + R"("kv_cache_quant_algo":"FP8"})") + .cache_dtype == "auto"); +} + +TEST_CASE("kv-fp8 W3 G1: an explicit --kv-cache-dtype outranks the checkpoint") { + // torch_utils.py:380-381 returns the explicit value UNCHANGED without ever + // reading the config, and attention.py:279-290 re-applies the same precedence + // with the comment "an explicit choice (e.g. bfloat16) must win". + const vllm::ResolvedCacheDTypeString r = + vllm::ResolveKvCacheDTypeString("bfloat16", kGateCheckpointQuantConfig); + CHECK(r.cache_dtype == "bfloat16"); + CHECK_FALSE(r.declared_by_checkpoint); +} + +TEST_CASE("kv-fp8 W3 G1: a checkpoint that declares no KV algo resolves auto") { + const vllm::ResolvedCacheDTypeString none = + vllm::ResolveKvCacheDTypeString("auto", kNoKvDeclarationQuantConfig); + CHECK(none.cache_dtype == "auto"); + CHECK_FALSE(none.declared_by_checkpoint); + + // No quantization config at all — the ordinary bf16 checkpoint. + const vllm::ResolvedCacheDTypeString empty = + vllm::ResolveKvCacheDTypeString("auto", ""); + CHECK(empty.cache_dtype == "auto"); + CHECK_FALSE(empty.declared_by_checkpoint); +} + +TEST_CASE("kv-fp8 W3 G1: an unrecognized kv_cache_quant_algo falls back to auto") { + // torch_utils.py:351-361 — upstream's own safe fallback. It must NOT become + // "declared", because a KV format we cannot serve is not a declaration we can + // honour. + const vllm::ResolvedCacheDTypeString r = vllm::ResolveKvCacheDTypeString( + "auto", + R"({"producer":{"name":"modelopt"}, + "quantization":{"quant_algo":"FP8","kv_cache_quant_algo":"INT3"}})"); + CHECK(r.cache_dtype == "auto"); + CHECK_FALSE(r.declared_by_checkpoint); +} + +// ─── G2. Declared-but-absent is NOT the same state as never declared ───────── +TEST_CASE("kv-fp8 W3 G2: a DECLARED fp8 cache with no scale tensors takes 1.0") { + // kv_cache.py:112-116 — this is the arm the gate checkpoint takes: it declares + // `kv_cache_quant_algo: "FP8"` and ships zero k/v scale tensors, so both + // sentinels survive and the documented default applies. + const vllm::ResolvedKvCacheScales r = vllm::ResolveKvCacheScales( + "fp8_e4m3", /*calculate_kv_scales=*/false, vllm::kKvScaleUnloaded, + vllm::kKvScaleUnloaded); + CHECK(r.origin == KvScaleOrigin::kDeclaredButAbsent); + CHECK(r.k_scale == doctest::Approx(1.0F)); + CHECK(r.v_scale == doctest::Approx(1.0F)); + // kv_cache.py:150-156 — and it says so. + CHECK(r.uncalibrated); + + // The consumer is happy to be handed this pair: it was DECLARED. + float k = 0.0F; + float v = 0.0F; + vllm::ScalesForFp8Store(r, &k, &v); + CHECK(k == doctest::Approx(1.0F)); + CHECK(v == doctest::Approx(1.0F)); +} + +TEST_CASE( + "kv-fp8 W3 G2: NO declaration is a different state and yields NO scale") { + // THE CASE THIS WHOLE FILE EXISTS FOR. Identical inputs to the one above + // except the declaration, and identical NUMBERS out — 1.0/1.0 are the struct's + // defaults — so a gate that only read k_scale/v_scale could not tell them + // apart. `origin` can, and the consumer refuses on it. + const vllm::ResolvedKvCacheScales r = vllm::ResolveKvCacheScales( + "auto", /*calculate_kv_scales=*/false, vllm::kKvScaleUnloaded, + vllm::kKvScaleUnloaded); + CHECK(r.origin == KvScaleOrigin::kNotQuantized); + // kv_cache.py:100-102: the scale block never ran, so the uncalibrated warning + // is not owed either. + CHECK_FALSE(r.uncalibrated); + + float k = 0.0F; + float v = 0.0F; + CHECK_THROWS_AS(vllm::ScalesForFp8Store(r, &k, &v), std::runtime_error); + // And the refusal NAMES what is missing, so the next reader does not have to + // rediscover the distinction. + try { + vllm::ScalesForFp8Store(r, &k, &v); + FAIL("ScalesForFp8Store accepted a kNotQuantized pair"); + } catch (const std::runtime_error& e) { + const std::string msg = e.what(); + CHECK(msg.find("no fp8 KV cache was declared") != std::string::npos); + CHECK(msg.find("kv_cache_quant_algo") != std::string::npos); + } + // Nothing was written into the outputs. + CHECK(k == doctest::Approx(0.0F)); + CHECK(v == doctest::Approx(0.0F)); +} + +TEST_CASE("kv-fp8 W3 G2: the three LOADED arms mirror kv_cache.py:104-127") { + // Both scales present (:104-111). + const vllm::ResolvedKvCacheScales both = vllm::ResolveKvCacheScales( + "fp8_e4m3", /*calculate_kv_scales=*/false, 0.5F, 0.25F); + CHECK(both.origin == KvScaleOrigin::kCheckpoint); + CHECK(both.k_scale == doctest::Approx(0.5F)); + CHECK(both.v_scale == doctest::Approx(0.25F)); + CHECK_FALSE(both.uncalibrated); // not 1.0/1.0 + + // A single `kv_scale`, remapped to k_scale at load and duplicated (:117-127). + const vllm::ResolvedKvCacheScales dup = vllm::ResolveKvCacheScales( + "fp8_e4m3", /*calculate_kv_scales=*/false, 0.5F, vllm::kKvScaleUnloaded); + CHECK(dup.origin == KvScaleOrigin::kCheckpointKvScale); + CHECK(dup.k_scale == doctest::Approx(0.5F)); + CHECK(dup.v_scale == doctest::Approx(0.5F)); + + // e5m2 suppresses the uncalibrated warning (:153) — 1.0 is ordinary there. + const vllm::ResolvedKvCacheScales e5m2 = vllm::ResolveKvCacheScales( + "fp8_e5m2", /*calculate_kv_scales=*/false, vllm::kKvScaleUnloaded, + vllm::kKvScaleUnloaded); + CHECK(e5m2.origin == KvScaleOrigin::kDeclaredButAbsent); + CHECK_FALSE(e5m2.uncalibrated); + + // The deprecated dynamic path is refused BY NAME rather than silently taking + // the static arm (cache.py:111). + CHECK_THROWS_AS(vllm::ResolveKvCacheScales("fp8_e4m3", + /*calculate_kv_scales=*/true, + vllm::kKvScaleUnloaded, + vllm::kKvScaleUnloaded), + std::runtime_error); +} + +// ─── G3. The block arithmetic ──────────────────────────────────────────────── +TEST_CASE("kv-fp8 W3 G3: an fp8 KV page is EXACTLY half a bf16 page") { + // kv_cache_interface.py:204-218 — `real_page_size_bytes` is + // `2 * block_size * num_kv_heads * head_dim * get_dtype_size(dtype)`, and + // torch_utils.py:38-40 makes every fp8 CacheDType one byte. Assert the CLOSED + // FORM, not just the ratio: a ratio alone is satisfied by any pair of widths + // in 2:1, including a pair that is wrong on both sides. + constexpr int kBlock = 16; + constexpr int kHkv = 4; + constexpr int kDh = 64; + constexpr int64_t kElems = 2LL * kBlock * kHkv * kDh; // K + V + + vllm::v1::KVCacheConfig cfg; + cfg.num_blocks = 8; + cfg.kv_cache_groups.emplace_back( + std::vector{"fa"}, + std::make_shared(kBlock, kHkv, kDh, + DType::kBF16)); + const auto* spec = dynamic_cast( + cfg.kv_cache_groups[0].kv_cache_spec.get()); + REQUIRE(spec != nullptr); + const int64_t bf16_page = spec->page_size_bytes(); + CHECK(bf16_page == kElems * 2); + + vllm::v1::ApplyCacheDType(cfg, vllm::v1::ParseCacheDType("fp8", DType::kBF16), + 1.0F, 1.0F); + const int64_t fp8_page = spec->page_size_bytes(); + CHECK(fp8_page == kElems * 1); + CHECK(fp8_page * 2 == bf16_page); + // The storage dtype and the interpretation both landed, on the SAME spec. + CHECK(spec->dtype == DType::kI8); + CHECK(spec->fp8_kind == vt::Fp8KVCacheDataType::kFp8E4M3); + // KVBytesPerBlock — the divisor the pool sizing actually uses — halves too. + CHECK(vllm::v1::KVBytesPerBlock(cfg) == kElems); +} + +TEST_CASE("kv-fp8 W3 G3: an auto cache_dtype leaves every spec untouched") { + // The byte-identical default. `ApplyCacheDType` must not rewrite a spec the + // model's factory already built at the model dtype. + constexpr int kBlock = 16; + vllm::v1::KVCacheConfig cfg; + cfg.num_blocks = 8; + cfg.kv_cache_groups.emplace_back( + std::vector{"fa"}, + std::make_shared(kBlock, 4, 64, + DType::kBF16)); + const int64_t before = vllm::v1::KVBytesPerBlock(cfg); + vllm::v1::ApplyCacheDType(cfg, vllm::v1::ParseCacheDType("auto", DType::kBF16), + 1.0F, 1.0F); + CHECK(vllm::v1::KVBytesPerBlock(cfg) == before); + const auto* spec = dynamic_cast( + cfg.kv_cache_groups[0].kv_cache_spec.get()); + REQUIRE(spec != nullptr); + CHECK(spec->dtype == DType::kBF16); + CHECK(spec->fp8_kind == vt::Fp8KVCacheDataType::kAuto); +} + +TEST_CASE("kv-fp8 W3 G3: the f32 A/B cache and an MLA spec still load on auto") { + // The regression this early return exists for. `ApplyCacheDType` refuses + // float16 and refuses MLA — correctly — so it must not REACH those refusals on + // the default path. `VT_KV_CACHE_F32=1` builds an f32 KV spec and "auto" + // resolves to f32; an MLA model on "auto" resolves to its own model dtype. + // Both are asking for nothing to change, and both must survive. + vllm::v1::KVCacheConfig f32; + f32.num_blocks = 4; + f32.kv_cache_groups.emplace_back( + std::vector{"fa"}, + std::make_shared(16, 4, 64, DType::kF32)); + vllm::v1::ApplyCacheDType( + f32, vllm::v1::ParseCacheDType("auto", DType::kF32), 1.0F, 1.0F); + const auto* f32_spec = dynamic_cast( + f32.kv_cache_groups[0].kv_cache_spec.get()); + REQUIRE(f32_spec != nullptr); + CHECK(f32_spec->dtype == DType::kF32); + + vllm::v1::KVCacheConfig mla; + mla.num_blocks = 4; + mla.kv_cache_groups.emplace_back( + std::vector{"mla"}, + std::make_shared(16, 576, DType::kBF16)); + // No throw: the MLA refusal is for an fp8 REQUEST, not for every load. + vllm::v1::ApplyCacheDType( + mla, vllm::v1::ParseCacheDType("auto", DType::kBF16), 1.0F, 1.0F); + const auto* mla_spec = dynamic_cast( + mla.kv_cache_groups[0].kv_cache_spec.get()); + REQUIRE(mla_spec != nullptr); + CHECK(mla_spec->dtype == DType::kBF16); +} + +// ─── G4. The same halving, through the LOADER ──────────────────────────────── +TEST_CASE( + "kv-fp8 W3 G4: --kv-cache-dtype fp8 buys EXACTLY 2x the blocks at one " + "--kv-cache-memory") { + // This case enters through the production entry point — the LoadedEngine + // constructor -> MakeKVCacheResolved -> ApplyResolvedCacheDType -> + // ResolveNumBlocks — rather than calling the resolver, because what is under + // test is that the storage dtype reaches the sizing BEFORE the sizing reads + // the geometry. Applying it afterwards would leave this equality at 1x and is + // the ordering mistake the comment in MakeKVCacheResolved names. + const HfConfig c = MakeDenseConfig(); + + LoadedEngine bf16(c, MakeDenseWeights(c), BuildFixture(), + ParamsWithCacheDType("auto")); + LoadedEngine fp8(c, MakeDenseWeights(c), BuildFixture(), + ParamsWithCacheDType("fp8")); + + const int bf16_blocks = bf16.kv_cache_config().num_blocks; + const int fp8_blocks = fp8.kv_cache_config().num_blocks; + REQUIRE(bf16_blocks > 0); + CHECK(fp8_blocks == 2 * bf16_blocks); + + // And the specs say why. + const auto* bf16_spec = SoleAttentionSpec(bf16); + const auto* fp8_spec = SoleAttentionSpec(fp8); + REQUIRE(bf16_spec != nullptr); + REQUIRE(fp8_spec != nullptr); + CHECK(bf16_spec->dtype == DType::kBF16); + CHECK(fp8_spec->dtype == DType::kI8); + CHECK(fp8_spec->page_size_bytes() * 2 == bf16_spec->page_size_bytes()); + + // The POOL is the same size in bytes — that is the point of the feature: the + // same memory now holds twice the context. + CHECK(static_cast(fp8_blocks) * vllm::v1::KVBytesPerBlock( + fp8.kv_cache_config()) == + static_cast(bf16_blocks) * + vllm::v1::KVBytesPerBlock(bf16.kv_cache_config())); +} + +TEST_CASE("kv-fp8 W3 G4: the GDN/Mamba state is NOT retyped") { + // config/cache.py:131-138 — recurrent state has its own mamba_cache_dtype + // knob and `--kv-cache-dtype` never touches it. The synthetic model has three + // linear-attention layers, so a rewrite that walked every group would corrupt + // them. + const HfConfig c = MakeDenseConfig(); + LoadedEngine fp8(c, MakeDenseWeights(c), BuildFixture(), + ParamsWithCacheDType("fp8")); + bool saw_mamba = false; + for (const auto& group : fp8.kv_cache_config().kv_cache_groups) { + const auto* mamba = + dynamic_cast(group.kv_cache_spec.get()); + if (mamba == nullptr) continue; + saw_mamba = true; + for (const vt::DType dt : mamba->dtypes) { + CHECK(dt != DType::kI8); + } + } + CHECK(saw_mamba); // the case would be vacuous without one +} + +// ─── G5. Reachability: the fp8 KV path SERVES ──────────────────────────────── +TEST_CASE("kv-fp8 W3 G5: an fp8 KV engine generates through the real forward") { + // The reachability gate. Nothing here constructs a vt op or a PagedKvCache by + // hand: the engine is built from the loader, the request goes through + // LLMEngine, and the tokens come out of Qwen3_5DenseModel::Forward, whose + // full-attention layer writes and reads THIS cache. If the store or the read + // were not routed, `vt::ReshapeAndCache` refuses the kI8 page by name (G7) + // and this case throws instead of counting tokens. + const HfConfig c = MakeDenseConfig(); + EngineParams params = ParamsWithCacheDType("fp8"); + LoadedEngine eng(c, MakeDenseWeights(c), BuildFixture(), params); + + // The cache really is one byte per element on the layer that serves. + const auto* spec = SoleAttentionSpec(eng); + REQUIRE(spec != nullptr); + REQUIRE(spec->dtype == DType::kI8); + REQUIRE(spec->fp8_kind == vt::Fp8KVCacheDataType::kFp8E4M3); + + constexpr int kMaxTokens = 4; + const vllm::RequestOutput run1 = + eng.engine().generate("hello world", Greedy(kMaxTokens), "req"); + REQUIRE(run1.finished); + REQUIRE(run1.outputs.size() == 1); + CHECK(static_cast(run1.outputs[0].token_ids.size()) == kMaxTokens); + + // Deterministic across two fresh stacks on the fp8 cache — greedy decode over + // an fp8 KV store is still a function of the inputs. + LoadedEngine again(c, MakeDenseWeights(c), BuildFixture(), params); + const vllm::RequestOutput run2 = + again.engine().generate("hello world", Greedy(kMaxTokens), "req"); + REQUIRE(run2.outputs.size() == 1); + CHECK(run2.outputs[0].token_ids == run1.outputs[0].token_ids); +} + +TEST_CASE("kv-fp8 W3 G5: the uncalibrated-scale warning fires ONCE per load") { + // kv_cache.py:150-156. The gate checkpoint's own case: an fp8 cache serving on + // the default 1.0, said out loud. + const HfConfig c = MakeDenseConfig(); + std::ostringstream captured; + { + CerrRedirect guard(captured.rdbuf()); + LoadedEngine eng(c, MakeDenseWeights(c), BuildFixture(), + ParamsWithCacheDType("fp8")); + std::cerr.flush(); + } + const std::string logged = captured.str(); + const std::string needle = "KV cache scaling factor 1.0"; + const size_t first = logged.find(needle); + REQUIRE(first != std::string::npos); + // Once, not once per ApplyResolvedCacheDType call (there are two per load). + CHECK(logged.find(needle, first + needle.size()) == std::string::npos); + // It names the checkpoint as the place to fix it. + CHECK(logged.find("checkpoint") != std::string::npos); + + // A SECOND engine in the SAME process gets its own line. Upstream's + // `logger.warning_once` is per-process; ours is per LOAD, because a + // process-static latch would silence the second engine rather than the second + // of the two ApplyResolvedCacheDType calls one load makes. Getting that wrong + // reads as "the warning fired once" on both counts. + std::ostringstream second; + { + CerrRedirect guard(second.rdbuf()); + LoadedEngine eng(c, MakeDenseWeights(c), BuildFixture(), + ParamsWithCacheDType("fp8")); + std::cerr.flush(); + } + const std::string logged2 = second.str(); + const size_t only = logged2.find(needle); + REQUIRE(only != std::string::npos); + CHECK(logged2.find(needle, only + needle.size()) == std::string::npos); + + // And an auto engine says nothing, so the line is a warning rather than noise. + std::ostringstream quiet; + { + CerrRedirect guard(quiet.rdbuf()); + LoadedEngine eng(c, MakeDenseWeights(c), BuildFixture(), + ParamsWithCacheDType("auto")); + std::cerr.flush(); + } + CHECK(quiet.str().find(needle) == std::string::npos); +} + +// ─── G6. Storage dtype and interpretation cannot disagree ──────────────────── +TEST_CASE("kv-fp8 W3 G6: a half-sized page with no fp8 kind is refused") { + // The silent-corruption shape, asserted at the routing seam. A `kI8` page is + // sized at one byte per element; an fp8 kind of kAuto would send it to the + // float store, which indexes at the source width. Neither half of the pair is + // allowed to travel alone. + vllm::PagedKvCache kv; + kv.dtype = DType::kI8; + kv.fp8_kind = vt::Fp8KVCacheDataType::kAuto; + CHECK_THROWS_AS(vllm::dense_attn::IsFp8KvCache(kv), std::runtime_error); + + vllm::PagedKvCache other; + other.dtype = DType::kBF16; + other.fp8_kind = vt::Fp8KVCacheDataType::kFp8E4M3; + CHECK_THROWS_AS(vllm::dense_attn::IsFp8KvCache(other), std::runtime_error); + + // The two consistent states answer without throwing. + vllm::PagedKvCache floatkv; + CHECK_FALSE(vllm::dense_attn::IsFp8KvCache(floatkv)); + vllm::PagedKvCache fp8kv; + fp8kv.dtype = DType::kI8; + fp8kv.fp8_kind = vt::Fp8KVCacheDataType::kFp8E4M3; + CHECK(vllm::dense_attn::IsFp8KvCache(fp8kv)); +} + +TEST_CASE("kv-fp8 W3 G6: ApplyKvCacheQuant is inert on a float cache") { + // Every existing caller must be byte-identical: the three additive + // PagedAttentionArgs fields keep their defaults on a float cache, so the op + // takes exactly the branch it took before W3. + vllm::PagedKvCache floatkv; + vt::PagedAttentionArgs args{0.125F, true}; + vllm::dense_attn::ApplyKvCacheQuant(args, floatkv); + CHECK(args.kv_cache_dtype == vt::Fp8KVCacheDataType::kAuto); + CHECK(args.k_scale == doctest::Approx(1.0F)); + CHECK(args.v_scale == doctest::Approx(1.0F)); + + vllm::PagedKvCache fp8kv; + fp8kv.dtype = DType::kI8; + fp8kv.fp8_kind = vt::Fp8KVCacheDataType::kFp8E4M3; + fp8kv.k_scale = 0.5F; + fp8kv.v_scale = 0.25F; + vt::PagedAttentionArgs fp8_args{0.125F, true}; + vllm::dense_attn::ApplyKvCacheQuant(fp8_args, fp8kv); + CHECK(fp8_args.kv_cache_dtype == vt::Fp8KVCacheDataType::kFp8E4M3); + CHECK(fp8_args.k_scale == doctest::Approx(0.5F)); + CHECK(fp8_args.v_scale == doctest::Approx(0.25F)); +} + +// ─── G7. An unrouted attention block is refused BY NAME ────────────────────── +TEST_CASE("kv-fp8 W3 G7: the float store refuses a 1-byte fp8 cache by name") { + // W3 routes `dense_attn::AttnBlock` (the shared seam) and `qwen3_5.cpp`. Every + // other architecture still calls `vt::ReshapeAndCache` directly, and this is + // what happens when one of them is served `--kv-cache-dtype fp8`: a named + // refusal at the first store, not a half-width write into a half-sized page. + vt::Queue q{vt::Device{vt::DeviceType::kCPU, 0}, nullptr}; + + constexpr int64_t T = 2, Hkv = 2, Dh = 4, kBlocks = 2, kBlockSize = 4; + std::vector k_src(static_cast(T * Hkv * Dh), 0); + std::vector v_src(k_src.size(), 0); + // The NHD unbind-slice cache: ONE (num_blocks, 2, block_size, H, D) byte + // allocation, k/v are the two dim-1 slices. + std::vector cache( + static_cast(kBlocks * 2 * kBlockSize * Hkv * Dh), 0); + std::vector slots{0, 1}; + + vt::Tensor k = vt::Tensor::Contiguous(k_src.data(), DType::kBF16, q.device, + {T, Hkv, Dh}); + vt::Tensor v = vt::Tensor::Contiguous(v_src.data(), DType::kBF16, q.device, + {T, Hkv, Dh}); + vt::Tensor slot_mapping = vt::Tensor::Contiguous( + slots.data(), DType::kI64, q.device, {static_cast(slots.size())}); + + const auto kv_slice = [&](int which) { + vt::Tensor t; + t.data = cache.data() + static_cast(which) * + static_cast(kBlockSize * Hkv * Dh); + t.dtype = DType::kI8; + t.device = q.device; + t.rank = 4; + t.shape[0] = kBlocks; + t.shape[1] = kBlockSize; + t.shape[2] = Hkv; + t.shape[3] = Dh; + t.stride[0] = 2 * kBlockSize * Hkv * Dh; + t.stride[1] = Hkv * Dh; + t.stride[2] = Dh; + t.stride[3] = 1; + return t; + }; + vt::Tensor k_cache = kv_slice(0); + vt::Tensor v_cache = kv_slice(1); + + try { + vt::ReshapeAndCache(q, k, v, k_cache, v_cache, slot_mapping); + FAIL("the float store accepted a 1-byte fp8 cache"); + } catch (const std::runtime_error& e) { + const std::string msg = e.what(); + // It names the OP that should have been called... + CHECK(msg.find("vt::ReshapeAndCacheFp8") != std::string::npos); + // ...and the missing part, so the reader knows this is an unwired + // architecture rather than a corrupt tensor. + CHECK(msg.find("not routed for fp8 KV") != std::string::npos); + } +} + +// ─── G8. The refusals that stop a mis-sized pool ───────────────────────────── +TEST_CASE("kv-fp8 W3 G8: an MLA cache refuses --kv-cache-dtype fp8 by name") { + // kv_cache_interface.py:398-410 — upstream's MLA fp8 arm is `fp8_ds_mla` with + // a different page formula (656 B/token on V3.2), not this one. Retyping an + // MLA spec to kI8 would size the latent page at half and store it at full. + vllm::v1::KVCacheConfig cfg; + cfg.num_blocks = 4; + cfg.kv_cache_groups.emplace_back( + std::vector{"mla"}, + std::make_shared(16, 576, DType::kBF16)); + try { + vllm::v1::ApplyCacheDType( + cfg, vllm::v1::ParseCacheDType("fp8", DType::kBF16), 1.0F, 1.0F); + FAIL("ApplyCacheDType retyped an MLA spec"); + } catch (const std::runtime_error& e) { + const std::string msg = e.what(); + CHECK(msg.find("MLA") != std::string::npos); + CHECK(msg.find("fp8_ds_mla") != std::string::npos); + } +} + +TEST_CASE("kv-fp8 W3 G8: float16 and fp8_e5m2 are refused, not mis-stored") { + const auto fresh = [] { + vllm::v1::KVCacheConfig cfg; + cfg.num_blocks = 4; + cfg.kv_cache_groups.emplace_back( + std::vector{"fa"}, + std::make_shared(16, 4, 64, DType::kBF16)); + return cfg; + }; + + // float16 PARSES — the CacheDType surface is mirrored in full (cache.py:19-36) + // — but no attention block casts K/V to f16 before the store, so applying it + // would reach a dtype mismatch deep inside the op instead of a sentence. + vllm::v1::KVCacheConfig f16 = fresh(); + try { + vllm::v1::ApplyCacheDType( + f16, vllm::v1::ParseCacheDType("float16", DType::kBF16), 1.0F, 1.0F); + FAIL("ApplyCacheDType accepted float16"); + } catch (const std::runtime_error& e) { + CHECK(std::string(e.what()).find("float16") != std::string::npos); + } + + // fp8_e5m2 likewise: W1/W2 refuse the compute, so the SIZING must refuse too + // rather than allocate a half-sized pool nothing can write. + vllm::v1::KVCacheConfig e5m2 = fresh(); + try { + vllm::v1::ApplyCacheDType( + e5m2, vllm::v1::ParseCacheDType("fp8_e5m2", DType::kBF16), 1.0F, 1.0F); + FAIL("ApplyCacheDType accepted fp8_e5m2"); + } catch (const std::runtime_error& e) { + CHECK(std::string(e.what()).find("fp8_e5m2") != std::string::npos); + } +} + +// ─── G9. The store's TWO sources must share one float dtype ────────────────── +TEST_CASE("kv-fp8 W3 G9: the op refuses a mixed f32 K / bf16 V pair by name") { + // The contract the model has to satisfy, asserted where it lives, so the case + // below cannot be read as being about a layout. `vt::ReshapeAndCacheFp8` takes + // the model floats and quantizes them itself; it has ONE source-dtype template + // instantiation, so a K and a V of different widths is a refusal and never a + // mode. + vt::Queue q{vt::Device{vt::DeviceType::kCPU, 0}, nullptr}; + + constexpr int64_t T = 2, Hkv = 2, Dh = 4, kBlocks = 2, kBlockSize = 4; + std::vector k_src(static_cast(T * Hkv * Dh), 0.5F); + std::vector v_src(static_cast(T * Hkv * Dh), + vt::F32ToBF16(0.5F)); + std::vector cache( + static_cast(kBlocks * 2 * kBlockSize * Hkv * Dh), 0); + std::vector slots{0, 1}; + + vt::Tensor k = vt::Tensor::Contiguous(k_src.data(), DType::kF32, q.device, + {T, Hkv, Dh}); + vt::Tensor v = vt::Tensor::Contiguous(v_src.data(), DType::kBF16, q.device, + {T, Hkv, Dh}); + vt::Tensor slot_mapping = vt::Tensor::Contiguous( + slots.data(), DType::kI64, q.device, {static_cast(slots.size())}); + + const auto kv_slice = [&](int which) { + vt::Tensor t; + t.data = cache.data() + static_cast(which) * + static_cast(kBlockSize * Hkv * Dh); + t.dtype = DType::kI8; + t.device = q.device; + t.rank = 4; + t.shape[0] = kBlocks; + t.shape[1] = kBlockSize; + t.shape[2] = Hkv; + t.shape[3] = Dh; + t.stride[0] = 2 * kBlockSize * Hkv * Dh; + t.stride[1] = Hkv * Dh; + t.stride[2] = Dh; + t.stride[3] = 1; + return t; + }; + vt::Tensor k_cache = kv_slice(0); + vt::Tensor v_cache = kv_slice(1); + + try { + vt::ReshapeAndCacheFp8(q, k, v, k_cache, v_cache, slot_mapping, + vt::Fp8KVCacheDataType::kFp8E4M3, 1.0F, 1.0F); + FAIL("the fp8 store accepted a mixed-dtype k/v pair"); + } catch (const std::runtime_error& e) { + CHECK(std::string(e.what()).find("must share one float dtype") != + std::string::npos); + } +} + +TEST_CASE("kv-fp8 W3 G9: an fp8 engine serves TORCH-LAYOUT K/V projections") { + // THE PRODUCTION ARM, and the one the CPU gate could not see. + // + // `ProjectFullAttnQkv` picks the projection GEMM's OUTPUT dtype from the + // weight's own layout: a raw torch Linear weight ([N=out, K=in], `nk`) — which + // is what every real safetensors checkpoint ships — takes `MatmulBf16D` and + // therefore emits the MODEL dtype, bf16. `MakeDenseWeights` builds [K,N] + // weights with `nk` unset, so every other case in this file takes `MatmulF32D` + // and gets an f32 V. That is the ONLY reason they pass. + // + // K never has the choice. The attention preamble writes K into `dk3`, whose + // dtype is `attn_dt`, and `attn_dt` is f32 for EVERY fp8 cache because + // `kv.dtype == DType::kBF16` is a term of both FA2 eligibility tests + // (qwen3_5.cpp, `fa2_prefill` and `fa2_decode`). So the fp8 route hands the + // store an f32 K beside a bf16 V, and the case above is what it then gets. + // + // The bf16 cache route never had this problem: it casts BOTH K and V to the + // cache dtype. The fp8 route now normalises the same way, to the same bf16 — + // which is also the dtype upstream quantizes from, because upstream's model IS + // bf16 at `reshape_and_cache_flash` (`cache_kernels.cu:314-401`). + const HfConfig c = MakeDenseConfig(); + constexpr int kMaxTokens = 4; + + LoadedEngine fp8(c, MakeDenseWeightsTorchKv(c), BuildFixture(), + ParamsWithCacheDType("fp8")); + const auto* spec = SoleAttentionSpec(fp8); + REQUIRE(spec != nullptr); + REQUIRE(spec->dtype == DType::kI8); + const vllm::RequestOutput out = + fp8.engine().generate("hello world", Greedy(kMaxTokens), "req"); + REQUIRE(out.finished); + REQUIRE(out.outputs.size() == 1); + CHECK(static_cast(out.outputs[0].token_ids.size()) == kMaxTokens); + + // NOT a case about the layout: the same weights serve the default cache too. + // Without this the case could go green by the torch layout becoming + // unloadable, which is a different failure wearing the same colour. + LoadedEngine automatic(c, MakeDenseWeightsTorchKv(c), BuildFixture(), + ParamsWithCacheDType("auto")); + const vllm::RequestOutput ref = + automatic.engine().generate("hello world", Greedy(kMaxTokens), "req"); + REQUIRE(ref.outputs.size() == 1); + CHECK(static_cast(ref.outputs[0].token_ids.size()) == kMaxTokens); +} + +// ─── G10. The loader stanza, entered through the loader ────────────────────── +TEST_CASE("kv-fp8 W3 G10: FromModelDir READS the checkpoint's declaration") { + // `LoadedEngine::FromModelDir` is the ONLY caller of + // `vllm::ReadQuantConfigJson` and of `vllm::ResolveKvCacheDTypeString` on a + // model directory. G1 calls the resolver with a string; that measures the + // resolver. This case measures the LOADER, which is the thing an operator + // reaches: it hands `FromModelDir` a directory and reads what it said. + // + // The directory carries no weights, so the load fails — after the stanza, + // which is what makes the stanza observable without a checkpoint (the pattern + // tests/vllm/entrypoints/openai/test_serve_residency_config.cpp establishes). + CheckpointDir dir; + dir.Write("hf_quant_config.json", kGateCheckpointQuantConfig); + + std::ostringstream captured; + std::string thrown; + { + CerrRedirect guard(captured.rdbuf()); + try { + (void)LoadedEngine::FromModelDir(dir.str(), EngineParams{}); + } catch (const std::exception& e) { + thrown = e.what(); + } + std::cerr.flush(); + } + CHECK_FALSE(thrown.empty()); + const std::string logged = captured.str(); + CHECK(logged.find("the checkpoint declares kv_cache_quant_algo") != + std::string::npos); + // It names the RESOLVED value, not the modelopt spelling, because that is the + // string every consumer downstream will see. + CHECK(logged.find("fp8_e4m3") != std::string::npos); +} + +TEST_CASE("kv-fp8 W3 G10: an explicit flag stops the loader consulting the checkpoint") { + // torch_utils.py:380-381 through the loader rather than through the resolver. + // The same directory, one explicit `--kv-cache-dtype`, and the line is gone: + // the operator outranks the checkpoint, and the checkpoint is not even read. + CheckpointDir dir; + dir.Write("hf_quant_config.json", kGateCheckpointQuantConfig); + + EngineParams params; + params.kv_cache_dtype = "bfloat16"; + + std::ostringstream captured; + { + CerrRedirect guard(captured.rdbuf()); + try { + (void)LoadedEngine::FromModelDir(dir.str(), params); + } catch (const std::exception&) { + } + std::cerr.flush(); + } + CHECK(captured.str().find("the checkpoint declares") == std::string::npos); +} + +TEST_CASE("kv-fp8 W3 G10: config.json's quantization_config OUTRANKS hf_quant_config.json") { + // MIRROR, and it used to be inverted. `vllm/transformers_utils/config.py:751-761`: + // + // quantization_config = config_dict.get("quantization_config", None) + // if quantization_config is None and file_or_path_exists(hf_quant_config.json): + // quantization_config = get_hf_file_to_dict("hf_quant_config.json", ...) + // + // with upstream's own comments naming the two producers: "ModelOpt 0.31.0 and + // after saves the quantization config in the model config file", and the + // separate file is "ModelOpt 0.29.0 and before". So an inline + // `quantization_config` means the legacy file is NEVER opened, and a + // re-quantized checkpoint that still carries a stale `hf_quant_config.json` + // beside a current `config.json` resolves to what the CURRENT one says. + // + // The two files below disagree on purpose, and only on the KV half: the inline + // one quantizes weights and declares nothing about the KV cache, the legacy one + // declares FP8. Reading them in the wrong order quantizes a KV cache nobody + // asked for, at half the page, silently. + CheckpointDir dir; + dir.Write("config.json", kInlineWeightsOnlyQuantConfig); + dir.Write("hf_quant_config.json", kGateCheckpointQuantConfig); + + // The unit half: which bytes come back. + const std::string read = vllm::ReadQuantConfigJson(dir.str()); + CHECK(read.find("kv_cache_quant_algo") == std::string::npos); + CHECK(vllm::GetKvCacheQuantAlgoString(read).value_or("auto") == "auto"); + + // The loader half: and therefore nothing is declared. + std::ostringstream captured; + { + CerrRedirect guard(captured.rdbuf()); + try { + (void)LoadedEngine::FromModelDir(dir.str(), EngineParams{}); + } catch (const std::exception&) { + } + std::cerr.flush(); + } + CHECK(captured.str().find("the checkpoint declares") == std::string::npos); + + // And the legacy file IS read when `config.json` carries no inline document — + // the 0.29.0-and-before arm, which the inversion would have made unreachable. + CheckpointDir legacy; + legacy.Write("config.json", R"({"model_type":"qwen3_5_text"})"); + legacy.Write("hf_quant_config.json", kGateCheckpointQuantConfig); + CHECK(vllm::GetKvCacheQuantAlgoString(vllm::ReadQuantConfigJson(legacy.str())) + .value_or("auto") == "fp8_e4m3"); +} + +TEST_CASE("kv-fp8 W3 G10: the #1574 checkpoint declares NOTHING about the KV") { + // THE CAMPAIGN CONSEQUENCE, stated as a gate instead of as prose. The two + // documents above are shaped like the gate checkpoint's; these ARE the gate + // checkpoint's, both transcribed from + // `r0b0tlab/Qwen3.8-27B-NVFP4-MTP-sm121` @ `36f717a2` on 2026-08-22. The + // legacy file declares `kv_cache_quant_algo: "FP8"`, the inline one declares + // no `kv_cache_*` key, and the inline one is what gets read — so an fp8 KV + // run of the #1574 subject needs `--kv-cache-dtype fp8` typed EXPLICITLY, on + // this engine and on vLLM alike. Running the pinned + // `get_kv_cache_quant_algo_string` over the same bytes returns `None`, before + // and after `_normalize_quantization_config`, so the two engines agree. + CheckpointDir dir; + dir.Write("config.json", kGateCheckpointInlineConfig); + dir.Write("hf_quant_config.json", kGateCheckpointQuantConfig); + + const std::string read = vllm::ReadQuantConfigJson(dir.str()); + // It IS the inline document — the modelopt marker is there twice, so this is + // not "nothing was found". + CHECK(read.find("MIXED_PRECISION") != std::string::npos); + CHECK(read.find("kv_cache_quant_algo") == std::string::npos); + + const vllm::ResolvedCacheDTypeString r = + vllm::ResolveKvCacheDTypeString("auto", read); + CHECK(r.cache_dtype == "auto"); + CHECK_FALSE(r.declared_by_checkpoint); + + // And the flag still reaches it, which is the path the campaign uses. + const vllm::ResolvedCacheDTypeString typed = + vllm::ResolveKvCacheDTypeString("fp8", read); + CHECK(typed.cache_dtype == "fp8"); + CHECK_FALSE(typed.declared_by_checkpoint); +} + +TEST_CASE("kv-fp8 W3 G10: the drafter-chain refusal runs BEFORE the KV resolution") { + // THE ORDERING F0 had to keep. `main`'s SPEC-DRAFTER-CHAIN W1 refusal (#1522) + // and this row's resolution stanza both want to be the first statement of + // `FromModelDir`, and both are load-bearing: G5 of that row requires the chain + // refusal "before any weight I/O", and `ReadQuantConfigJson` opens a file + // inside `model_dir`. Taking either side of that conflict drops a guarantee. + // + // `tests/vllm/entrypoints/test_drafter_chain_reach.cpp` cannot see an + // inversion: it points at a NONEXISTENT directory, and `ReadQuantConfigJson` + // answers "" for one of those without opening anything, so the chain refusal + // still arrives. This case points at a directory that EXISTS and declares + // fp8 — so an inverted order announces the declaration first, and that line is + // the evidence. + CheckpointDir dir; + dir.Write("hf_quant_config.json", kGateCheckpointQuantConfig); + + EngineParams params; + params.speculative_config = vllm::ParseSpeculativeConfigJson( + R"({"vllm_cpp":{"drafter_chain":[)" + R"({"method":"ngram","num_speculative_tokens":4},)" + R"({"method":"mtp"}]}})"); + REQUIRE(params.speculative_config.has_value()); + REQUIRE(params.speculative_config->use_drafter_chain()); + + std::ostringstream captured; + std::string thrown; + { + CerrRedirect guard(captured.rdbuf()); + try { + (void)LoadedEngine::FromModelDir(dir.str(), params); + } catch (const std::exception& e) { + thrown = e.what(); + } + std::cerr.flush(); + } + CHECK(thrown.find("drafter_chain") != std::string::npos); + CHECK(captured.str().find("the checkpoint declares") == std::string::npos); +} + +// ─── G11. The heterogeneous-per-layer seam ─────────────────────────────────── +TEST_CASE("kv-fp8 W3 G11: per_layer_attn_specs are retyped, and the pool halves") { + // `gpu_model_runner.py:484-486` — upstream resolves ONE kv_cache_dtype and + // EVERY attention spec is built with it. Ours arrives after the specs exist, + // so `ApplyCacheDType` has to reach every one of them, and a heterogeneous + // model (Gemma-4 G1b) allocates from `per_layer_attn_specs` instead of from the + // group spec. A rewrite that reached only the groups would leave + // `KVBytesPerBlock` at the full width for exactly the models whose pool it + // sizes — the half-sizing this row calls silent-corruption territory, with the + // sign reversed. + const auto build = [] { + vllm::v1::KVCacheConfig cfg; + cfg.num_blocks = 4; + // Two full-attention layers and one GDN layer, which is the null entry the + // loop has to step over rather than dereference. + cfg.kv_cache_groups.emplace_back( + std::vector{"fa0", "fa1"}, + std::make_shared(16, 4, 64, DType::kBF16)); + cfg.per_layer_attn_specs = { + std::make_shared(16, 4, 64, DType::kBF16), + nullptr, + std::make_shared(16, 2, 64, DType::kBF16)}; + return cfg; + }; + + vllm::v1::KVCacheConfig bf16 = build(); + const int64_t bf16_bytes = vllm::v1::KVBytesPerBlock(bf16); + REQUIRE(bf16_bytes > 0); + + vllm::v1::KVCacheConfig fp8 = build(); + vllm::v1::ApplyCacheDType(fp8, vllm::v1::ParseCacheDType("fp8", DType::kBF16), + 0.5F, 0.25F); + + // Every non-null per-layer spec carries the fp8 storage AND its interpretation + // AND the scales, because the store and the read both read them off the spec. + int retyped = 0; + for (const auto& spec : fp8.per_layer_attn_specs) { + if (spec == nullptr) continue; + CHECK(spec->dtype == DType::kI8); + CHECK(spec->fp8_kind == vt::Fp8KVCacheDataType::kFp8E4M3); + CHECK(spec->k_scale == doctest::Approx(0.5F)); + CHECK(spec->v_scale == doctest::Approx(0.25F)); + ++retyped; + } + CHECK(retyped == 2); // the case would be vacuous without them + + // And the divisor the pool is sized with is EXACTLY half. An equality, not a + // ratio: `KVBytesPerBlock` reads `per_layer_attn_specs` in preference to the + // groups, so this number is what the runner's `--kv-cache-memory` buys. + CHECK(vllm::v1::KVBytesPerBlock(fp8) * 2 == bf16_bytes); +} + +// ─── G12. The SHARED SEAM, entered through a model that uses it ────────────── +// +// WHY THIS CASE EXISTS, and what its absence let through. `## Shared seams` in +// AGENTS.md names `dense_attn::AttnBlock` as the decode seam, and `## W3` of the +// spec lists it as ROUTED. It was not. The routing was written +// (`dense_attn_block.h:519,536,543`) behind a preamble guard that admits only +// `kBF16` and `kF32`, and `IsFp8KvCache` is true only for `kI8` — so `fp8_kv` +// was provably false at every call and the fp8 arms of `WriteKvCache` and +// `ApplyKvCacheQuant` could never execute. A fresh review reverted the WHOLE of +// the seam's routing in a scratch copy and this file stayed at 26/26 SUCCESS, +// because every case above enters through `Qwen3_5DenseModel::Forward` and none +// of them enters here. +// +// The seam is the production forward for the Qwen3 dense family +// (`qwen3.cpp:185`), Qwen3-MoE (`qwen3_moe.cpp:84`), Voxtral +// (`voxtral.cpp:102`) and the Llama/Mistral/InternLM2 registries that share +// `Qwen3DenseModel`. `Qwen3DenseModel::Forward` is the function +// `ForwardQwen3ForCausalLM` calls under `ModelRegistry::Forward` +// (`qwen3_dense.cpp:113`), which is the entry `.agents/reachability.md` names; +// the harness adaptation, stated once, is that `LoadedEngine` has no in-memory +// overload for `Qwen3DenseWeights`, so the cache is allocated here rather than +// by the runner. Everything the case measures — the preamble guard, the store +// and the read — is production code entered through the registered forward. +namespace { + +vllm::OwnedTensor MakeSeamBf16(const std::vector& shape, bool nk, + uint64_t seed) { + vllm::OwnedTensor t = MakeOwned(DType::kBF16, shape, seed); + t.nk = nk; + return t; +} + +constexpr int64_t kSeamVocab = 64; + +HfConfig MakeSeamConfig() { + HfConfig c; + c.model_type = "qwen3"; + c.architectures = {"Qwen3ForCausalLM"}; + c.num_hidden_layers = 2; + c.hidden_size = 64; + c.num_attention_heads = 4; + c.num_key_value_heads = 2; + c.head_dim = 16; + c.rotary_dim = 16; + c.intermediate_size = 128; + c.rms_norm_eps = 1e-6; + c.rope_theta = 1000000.0; + c.vocab_size = kSeamVocab; + c.max_position_embeddings = 128; + c.raw = json::object(); + return c; +} + +vllm::Qwen3DenseWeights MakeSeamWeights(const HfConfig& c) { + const int64_t H = c.hidden_size, Hq = c.num_attention_heads; + const int64_t Hkv = c.num_key_value_heads, Dh = c.head_dim; + const int64_t I = c.intermediate_size, V = c.vocab_size; + const int64_t qdim = Hq * Dh, kdim = Hkv * Dh; + vllm::Qwen3DenseWeights w; + w.tie_word_embeddings = true; + w.attention_bias = false; + w.embed_tokens = MakeSeamBf16({V, H}, /*nk=*/false, 7001); + w.final_norm = MakeSeamBf16({H}, false, 7002); + for (int64_t l = 0; l < c.num_hidden_layers; ++l) { + const uint64_t s = 8000 + static_cast(l) * 4000; + vllm::Qwen3DenseLayerWeights lw; + lw.input_layernorm = MakeSeamBf16({H}, false, s + 1); + lw.post_attention_layernorm = MakeSeamBf16({H}, false, s + 2); + lw.attn.qkv_proj = MakeSeamBf16({qdim + 2 * kdim, H}, /*nk=*/true, s + 3); + lw.attn.o_proj = MakeSeamBf16({H, qdim}, /*nk=*/true, s + 4); + lw.attn.q_norm = MakeSeamBf16({Dh}, false, s + 5); + lw.attn.k_norm = MakeSeamBf16({Dh}, false, s + 6); + lw.mlp.gate_up_proj = MakeSeamBf16({2 * I, H}, /*nk=*/true, s + 7); + lw.mlp.down_proj = MakeSeamBf16({H, I}, /*nk=*/true, s + 8); + w.layers.push_back(std::move(lw)); + } + return w; +} + +constexpr int64_t kSeamBlocks = 2; +constexpr int64_t kSeamBlockSize = 8; + +// One paged KV buffer per layer, at the STORAGE WIDTH the cache dtype names — +// one byte per element for fp8, two for bf16. Allocating the fp8 arm at the +// float width would hide exactly the mis-sizing this row exists to prevent. +struct SeamCachePool { + std::vector> buf; + std::vector attn_kv; + + SeamCachePool(const HfConfig& c, DType dt, vt::Fp8KVCacheDataType kind, + float k_scale, float v_scale) { + const int64_t Hkv = c.num_key_value_heads, Dh = c.head_dim; + const size_t elems = static_cast(kSeamBlocks * 2 * kSeamBlockSize * + Hkv * Dh); + for (int64_t l = 0; l < c.num_hidden_layers; ++l) + buf.emplace_back(elems * vt::SizeOf(dt), 0); + for (auto& b : buf) { + vllm::PagedKvCache kv; + kv.data = b.data(); + kv.dtype = dt; + kv.num_blocks = kSeamBlocks; + kv.block_size = kSeamBlockSize; + kv.num_kv_heads = Hkv; + kv.head_size = Dh; + kv.fp8_kind = kind; + kv.k_scale = k_scale; + kv.v_scale = v_scale; + attn_kv.push_back(kv); + } + } +}; + +vllm::v1::CommonAttentionMetadata SeamPrefillMeta(int64_t T) { + vllm::v1::CommonAttentionMetadata m; + m.num_reqs = 1; + m.num_actual_tokens = static_cast(T); + m.query_start_loc = {0, static_cast(T)}; + m.query_start_loc_cpu = m.query_start_loc; + m.seq_lens = {static_cast(T)}; + m.seq_lens_cpu = m.seq_lens; + m.max_query_len = static_cast(T); + m.max_seq_len = static_cast(T); + m.block_table_num_cols = 1; + m.block_table_tensor = {0}; + for (int64_t t = 0; t < T; ++t) + m.slot_mapping.push_back(static_cast(t % kSeamBlockSize)); + m.causal = true; + return m; +} + +const std::vector kSeamTokens = {3, 17, 42, 8, 61}; +const std::vector kSeamPositions = {0, 1, 2, 3, 4}; + +std::vector RunSeamForward(const HfConfig& c, + const vllm::Qwen3DenseWeights& w, + SeamCachePool& pool) { + vt::Queue q{vt::Device{vt::DeviceType::kCPU, 0}, nullptr}; + const vllm::v1::CommonAttentionMetadata meta = + SeamPrefillMeta(static_cast(kSeamTokens.size())); + return vllm::Qwen3DenseModel::Forward(kSeamTokens, kSeamPositions, meta, + pool.attn_kv, w, c, q); +} + +size_t NonZeroBytes(const std::vector& b) { + size_t n = 0; + for (uint8_t x : b) + if (x != 0) ++n; + return n; +} + +} // namespace + +TEST_CASE("kv-fp8 W3 G12: the SHARED SEAM serves an fp8 KV cache") { + const HfConfig c = MakeSeamConfig(); + const vllm::Qwen3DenseWeights w = MakeSeamWeights(c); + + SeamCachePool fp8(c, DType::kI8, vt::Fp8KVCacheDataType::kFp8E4M3, 1.0F, + 1.0F); + // BEFORE the repair this line threw + // "qwen3 dense: KV cache must be bf16 or f32" — the preamble guard refused the + // cache the routing below it was written to serve. + const std::vector logits = RunSeamForward(c, w, fp8); + REQUIRE(logits.size() == + kSeamTokens.size() * static_cast(c.vocab_size)); + for (float v : logits) REQUIRE(std::isfinite(v)); + + // The STORE ran through the seam: the half-width pages carry bytes now. Zero + // here is what a store that silently skipped the fp8 arm would leave. + for (const auto& page : fp8.buf) CHECK(NonZeroBytes(page) > 0); + + // Deterministic over a fresh cache — an fp8 KV store is still a function of + // its inputs. + SeamCachePool again(c, DType::kI8, vt::Fp8KVCacheDataType::kFp8E4M3, 1.0F, + 1.0F); + const std::vector repeat = RunSeamForward(c, w, again); + REQUIRE(repeat.size() == logits.size()); + CHECK(std::memcmp(repeat.data(), logits.data(), + logits.size() * sizeof(float)) == 0); +} + +TEST_CASE("kv-fp8 W3 G12: the seam's fp8 cache is really QUANTIZED, not float") { + // The counter-case to the one above, and the reason "it ran and produced + // finite numbers" is not enough. An fp8 cache that behaved identically to a + // bf16 one would mean the read never dequantized — and `ApplyKvCacheQuant` is + // the only thing that tells the paged kernel to. Four mantissa bits against + // bf16's eight is a difference the logits carry. + const HfConfig c = MakeSeamConfig(); + const vllm::Qwen3DenseWeights w = MakeSeamWeights(c); + + SeamCachePool bf16(c, DType::kBF16, vt::Fp8KVCacheDataType::kAuto, 1.0F, 1.0F); + const std::vector float_logits = RunSeamForward(c, w, bf16); + + SeamCachePool fp8(c, DType::kI8, vt::Fp8KVCacheDataType::kFp8E4M3, 1.0F, 1.0F); + const std::vector fp8_logits = RunSeamForward(c, w, fp8); + + REQUIRE(float_logits.size() == fp8_logits.size()); + REQUIRE(!float_logits.empty()); + // The fp8 page is EXACTLY half the bf16 one, which is the memory the feature + // buys and the sizing the store has to agree with. + REQUIRE(fp8.buf[0].size() * 2 == bf16.buf[0].size()); + + size_t differing = 0; + double max_abs = 0.0; + for (size_t i = 0; i < fp8_logits.size(); ++i) { + if (fp8_logits[i] != float_logits[i]) ++differing; + max_abs = std::max(max_abs, static_cast(std::fabs( + fp8_logits[i] - float_logits[i]))); + } + CHECK(differing > 0); + // And it is a QUANTIZATION difference rather than a wrong-offset read: an + // fp8 store indexed at the float width would land the second half of every + // page outside the tokens it wrote and the logits would not track at all. + CHECK(max_abs < 1.0); + MESSAGE("fp8 vs bf16 cache: " << differing << "/" << fp8_logits.size() + << " logits differ, max |delta| " << max_abs); +} + +namespace { + +// The e4m3 ROUND-TRIP ENVELOPE. Every constant below is read off the FORMAT +// (`vt::F32ToF8E4M3`, `include/vt/fp8_kv.h`) rather than fitted to a measured +// delta, because a threshold sized to today's number is the same hole one +// decimal place tighter. +// +// fp8-e4m3fn carries three explicit mantissa bits and rounds to nearest even. +// For a NORMAL magnitude in [2^e, 2^(e+1)) the grid step is 2^(e-3), so the +// rounding error is at most the half step 2^(e-4), which is at most 2^-4 of the +// value. Below the smallest normal (2^-6) the grid is uniform at 2^-9, so the +// error is at most the half step 2^-10 in ABSOLUTE terms. The store divides by +// the scale and the read multiplies by it (`quant_utils.cuh:296-308`), so the +// absolute arm carries a factor of the scale and the relative arm does not. +constexpr double kE4m3RelHalfUlp = 1.0 / 16.0; // 2^-4 +constexpr double kE4m3AbsHalfStep = 1.0 / 1024.0; // 2^-10 +constexpr double kE4m3SmallestNormal = 1.0 / 64.0; // 2^-6 +constexpr double kE4m3Max = 448.0; + +// Non-unit, and DIFFERENT per side, so a k/v scale that is swapped, dropped, or +// applied at one end only leaves the envelope instead of staying inside it. +// Both are below one because the store divides by the scale: this toy model's +// layer-0 K and V land around 1e-2, and a scale above one would push most of +// them under e4m3's smallest normal (2^-6), where only the weaker absolute arm +// of the envelope applies. The values below keep every page's largest element a +// normal, which the `pages_with_a_normal` assertion holds them to. +constexpr float kEnvKScale = 0.125F; +constexpr float kEnvVScale = 0.25F; + +// Two INVARIANCE scale pairs, for the half the envelope above cannot reach: the +// production READ. Every one of the four is a power of two, and every one keeps +// EVERY layer-0 element a normal — the measured magnitudes are 1.76e-4 to 1.32e-1 +// for K and 5.41e-5 to 4.22e-2 for V, so the all-normal window (max/448, min*64] +// is (2.94e-4, 1.13e-2] for K and (9.43e-5, 3.46e-3] for V and all four sit +// inside it. That is what makes the comparison EXACT rather than a tolerance +// for the elements it covers. The `scale_exact` REQUIRE below holds LAYER 0's +// 320 elements to that window, which is 320 of the 640 each run stores; layer 1 +// does NOT satisfy it, and the comment on the comparison itself says what that +// costs. +// +// The two pairs differ on BOTH sides, and deliberately: a pair that moved only +// one side would let a read-side defect that depends on the OTHER scale +// reproduce itself identically in both runs and cancel out of the comparison. +constexpr float kInvAKScale = 1.0F / 128.0F; // 2^-7 +constexpr float kInvAVScale = 1.0F / 8192.0F; // 2^-13 +constexpr float kInvBKScale = 1.0F / 2048.0F; // 2^-11 +constexpr float kInvBVScale = 1.0F / 512.0F; // 2^-9 + +// Flat element index into a (num_blocks, 2, block_size, Hkv, Dh) contiguous KV +// buffer — the layout `KvSlice` (`src/vllm/model_executor/models/qwen3_5.cpp`) +// views, with `which` 0 = K and 1 = V. +size_t SeamKvIndex(const HfConfig& c, int which, int64_t slot, int64_t h, + int64_t d) { + const int64_t H = c.num_key_value_heads, D = c.head_dim; + const int64_t block = slot / kSeamBlockSize, off = slot % kSeamBlockSize; + return static_cast( + ((block * 2 + which) * kSeamBlockSize + off) * H * D + h * D + d); +} + +double SeamBf16At(const std::vector& page, size_t elem) { + uint16_t raw = 0; + std::memcpy(&raw, page.data() + elem * sizeof(uint16_t), sizeof(uint16_t)); + return static_cast(vt::BF16ToF32(raw)); +} + +} // namespace + +TEST_CASE( + "kv-fp8 W3 G12: the fp8 pages hold THIS layer's K and V, inside the e4m3 " + "round-trip envelope") { + // WHAT A LOGIT COMPARISON CANNOT SEE. "The fp8 logits differ from the bf16 + // logits by less than one" is satisfied by a cache that stores V where K + // belongs, and by a store that divides by a scale the read never multiplies + // back. Both leave this toy model's logits within a thousandth, so a bound + // stated on the logits measures the model's insensitivity and not the cache. + // This case compares the CACHE BYTES, where a mis-routed or mis-scaled store + // is an O(1) relative error and the correct answer is bounded by the format. + // + // LAYER 0 ONLY, and that is the point rather than a limitation: its K and V + // are functions of the embedding and the input layernorm alone, so the bf16 + // run and the fp8 run hand the store BIT-IDENTICAL floats and the float run's + // page IS the reference the fp8 page has to round. From layer 1 on, the fp8 + // run's inputs already carry the previous layer's dequantization and no + // per-element envelope holds. + const HfConfig c = MakeSeamConfig(); + const vllm::Qwen3DenseWeights w = MakeSeamWeights(c); + + SeamCachePool bf16(c, DType::kBF16, vt::Fp8KVCacheDataType::kAuto, kEnvKScale, + kEnvVScale); + const std::vector float_logits = RunSeamForward(c, w, bf16); + REQUIRE(!float_logits.empty()); + + SeamCachePool fp8(c, DType::kI8, vt::Fp8KVCacheDataType::kFp8E4M3, kEnvKScale, + kEnvVScale); + const std::vector fp8_logits = RunSeamForward(c, w, fp8); + REQUIRE(fp8_logits.size() == float_logits.size()); + + const int64_t Hkv = c.num_key_value_heads, Dh = c.head_dim; + size_t outside = 0, normals = 0, elems = 0; + size_t pages_with_a_normal = 0; + double worst_ratio = 0.0, worst_delta = 0.0; + double max_ref[2] = {0.0, 0.0}; + int worst_which = -1; + int64_t worst_slot = -1, worst_head = -1, worst_dim = -1; + + for (int which = 0; which < 2; ++which) { + const double scale = which == 0 ? kEnvKScale : kEnvVScale; + for (size_t t = 0; t < kSeamTokens.size(); ++t) { + const int64_t slot = static_cast(t) % kSeamBlockSize; + bool page_has_a_normal = false; + for (int64_t h = 0; h < Hkv; ++h) { + for (int64_t d = 0; d < Dh; ++d) { + const size_t i = SeamKvIndex(c, which, slot, h, d); + // The float run stored the model-dtype element verbatim, so this IS + // the value the fp8 store was handed. + const double ref = SeamBf16At(bf16.buf[0], i); + const double got = static_cast( + vt::LoadKvFp8E4M3(fp8.buf[0][i], static_cast(scale))); + const double bound = + kE4m3RelHalfUlp * std::fabs(ref) + kE4m3AbsHalfStep * scale; + const double delta = std::fabs(got - ref); + ++elems; + max_ref[which] = std::max(max_ref[which], std::fabs(ref)); + if (std::fabs(ref) >= kE4m3SmallestNormal * scale) { + ++normals; + page_has_a_normal = true; + } + if (delta > bound) { + ++outside; + if (delta / bound > worst_ratio) { + worst_ratio = delta / bound; + worst_delta = delta; + worst_which = which; + worst_slot = slot; + worst_head = h; + worst_dim = d; + } + } + } + } + if (page_has_a_normal) ++pages_with_a_normal; + } + } + + // Printed BEFORE the assertions, so a red run still says what it measured. + MESSAGE("layer-0 KV envelope: " + << outside << "/" << elems + << " elements outside 2^-4*|ref| + 2^-10*scale; worst ratio " + << worst_ratio << " (delta " << worst_delta + << ", which=" << worst_which << " slot=" << worst_slot + << " head=" << worst_head << " dim=" << worst_dim + << "), max|ref| k=" << max_ref[0] << " v=" << max_ref[1] + << ", normals " << normals << "/" << elems << " over " + << pages_with_a_normal << " pages"); + + // ANTI-VACUITY, stated structurally rather than as a fitted fraction. The + // relative arm only bites on magnitudes the format stores as NORMALS; a token + // page that held only subnormals — or that the store never wrote at all, and + // so reads back as zeros — would satisfy the absolute arm alone and prove + // nothing. Every one of the ten pages (five tokens, K and V) has to carry at + // least one normal. + REQUIRE(elems == 2 * kSeamTokens.size() * static_cast(Hkv * Dh)); + REQUIRE(pages_with_a_normal == 2 * kSeamTokens.size()); + // Nothing saturated, so the bound above is the pure rounding envelope and not + // a clamp: e4m3's finite maximum is 448 and the store divides by the scale. + REQUIRE(max_ref[0] < kE4m3Max * kEnvKScale); + REQUIRE(max_ref[1] < kE4m3Max * kEnvVScale); + + CHECK(outside == 0); + + // ---- AND NOW THE READ, which nothing above this line gates. ---- + // + // Everything above measures the STORE. It decodes the cache bytes with the + // test's own `vt::LoadKvFp8E4M3` and never enters the production dequant + // (`cpu_paged_attn.cpp:167`), and `fp8_logits` — the one value here that IS + // downstream of that dequant — carries no assertion at all. Every other case + // in this file that asserts a number downstream of the read runs at + // `k_scale == v_scale == 1`, where a k/v scale SWAP on the read is + // arithmetically inert. So a read that dequantizes V with K's scale, halving + // every V the softmax sees whenever the two scales differ, walks this whole + // file green. + // + // The gate below is EXACT rather than a tolerance. For the elements that meet + // its precondition that is a property of the FORMAT and not a lucky + // measurement; how far the precondition is actually ASSERTED is a separate + // question, and the block after the identity answers it. e4m3fn's NORMAL + // grid is relative: for |y| in [2^e, 2^(e+1)) the representable points are + // m*2^(e-3). Dividing by a power of two is exact in binary floating point and + // shifts `e` without touching the mantissa, so for any two power-of-two + // scales s and s' that both leave a value normal and unsaturated, + // + // s * Dequant(Quantize(x / s)) == s' * Dequant(Quantize(x / s')) + // + // bit for bit. The cache BYTES of the two runs differ — a different exponent + // field in every element — and the floats the attention kernel is handed do + // not. There is no constant to fit and none to widen later, which is the same + // discipline the envelope above is written to. + // + // WHAT IS ASSERTED, AND WHAT IS ONLY MEASURED. `scale_exact` below reads + // `bf16.buf[0]`, so it holds that precondition on LAYER 0 — 320 of the 640 + // elements each run stores, `MakeSeamConfig` setting num_hidden_layers = 2 — + // while `inv_differing` compares LOGITS, which are a function of BOTH layers' + // caches. Layer 1 does NOT satisfy the precondition. Decoding each run's own + // `buf[1]` at its own scales, 3 of its 320 elements disagree, by up to + // 7.62939e-06: all K-side, an order of magnitude below layer 0's 1.76e-4 + // minimum, so at kInvAKScale = 2^-7 they fall in e4m3's SUBNORMAL region + // where the grid is the absolute 2^-9 step and the power-of-two covariance + // does not hold. That layer also carries 2 SATURATED elements, the other + // escape. So `CHECK(inv_differing == 0)` is exact-by-format for layer 0 and, + // for layer 1, an EMPIRICAL result for this fixture: a 7.6e-6 cache + // perturbation absorbed in f32 accumulation before it reaches a logit. Not + // knife-edge today — a third legal pair (2^-9, 2^-11) against pair A also + // measures 0/320 logits differing at max |delta| 0 — but a change to + // MakeSeamWeights, kSeamTokens, num_hidden_layers, the thread count or the + // accumulation order could push a layer-1 discrepancy into a logit and redden + // a CORRECT tree with a defect-shaped message. Recorded under `## Owed` in + // `.agents/specs/fp8-kv-cache.md`. + // + // A read that uses the wrong scale breaks this and cannot hide, because the + // two pairs differ on both sides: dequantizing V with K's scale multiplies V + // by 64 in one run and by 1/4 in the other, and dropping a read scale + // altogether multiplies by 2^13 against 2^9. + + // ANTI-VACUITY for that exactness. The identity holds only while every element + // stays a NORMAL at all four scales and nothing saturates. One subnormal would + // round on the ABSOLUTE 2^-9 grid, which is not scale invariant, and the + // comparison below would then be measuring the data instead of the read. Count + // the elements the identity actually covers and require all of them. (An exact + // zero is scale invariant on its own and counts; the measured population has + // none.) LAYER 0 only, for the reason the block above gives. + size_t scale_exact = 0; + for (int which = 0; which < 2; ++which) { + const float sa = which == 0 ? kInvAKScale : kInvAVScale; + const float sb = which == 0 ? kInvBKScale : kInvBVScale; + for (size_t t = 0; t < kSeamTokens.size(); ++t) { + const int64_t slot = static_cast(t) % kSeamBlockSize; + for (int64_t h = 0; h < Hkv; ++h) { + for (int64_t d = 0; d < Dh; ++d) { + const double ref = std::fabs( + SeamBf16At(bf16.buf[0], SeamKvIndex(c, which, slot, h, d))); + const bool exact_a = + ref >= kE4m3SmallestNormal * sa && ref < kE4m3Max * sa; + const bool exact_b = + ref >= kE4m3SmallestNormal * sb && ref < kE4m3Max * sb; + if (ref == 0.0 || (exact_a && exact_b)) ++scale_exact; + } + } + } + } + REQUIRE(scale_exact == elems); + + SeamCachePool inv_a(c, DType::kI8, vt::Fp8KVCacheDataType::kFp8E4M3, + kInvAKScale, kInvAVScale); + const std::vector inv_a_logits = RunSeamForward(c, w, inv_a); + SeamCachePool inv_b(c, DType::kI8, vt::Fp8KVCacheDataType::kFp8E4M3, + kInvBKScale, kInvBVScale); + const std::vector inv_b_logits = RunSeamForward(c, w, inv_b); + REQUIRE(inv_a_logits.size() == float_logits.size()); + REQUIRE(inv_b_logits.size() == float_logits.size()); + + // The two runs really do hold DIFFERENT bytes, so the comparison below is a + // statement about the read and not about two identical buffers: a store that + // ignored its scale would write the same page twice and every logit would + // match while proving nothing. + REQUIRE(inv_a.buf[0].size() == inv_b.buf[0].size()); + CHECK(std::memcmp(inv_a.buf[0].data(), inv_b.buf[0].data(), + inv_a.buf[0].size()) != 0); + + size_t inv_differing = 0; + double inv_max_abs = 0.0; + for (size_t i = 0; i < inv_a_logits.size(); ++i) { + if (inv_a_logits[i] != inv_b_logits[i]) ++inv_differing; + inv_max_abs = std::max( + inv_max_abs, + static_cast(std::fabs(inv_a_logits[i] - inv_b_logits[i]))); + } + MESSAGE("read-side scale invariance: " + << inv_differing << "/" << inv_a_logits.size() + << " logits differ between k/v scales (" << kInvAKScale << ", " + << kInvAVScale << ") and (" << kInvBKScale << ", " << kInvBVScale + << "), max |delta| " << inv_max_abs << "; scale-exact elements " + << scale_exact << "/" << elems); + + CHECK(inv_differing == 0); +} diff --git a/tests/vllm/models/test_qwen38_27b_modelopt_mtp_arm.cpp b/tests/vllm/models/test_qwen38_27b_modelopt_mtp_arm.cpp index ddb0eb2ee..abf4c1383 100644 --- a/tests/vllm/models/test_qwen38_27b_modelopt_mtp_arm.cpp +++ b/tests/vllm/models/test_qwen38_27b_modelopt_mtp_arm.cpp @@ -479,14 +479,19 @@ TEST_CASE("Qwen3.8-27B-NVFP4-MTP: the MTP shard is 15 unquantized bf16 tensors") CHECK(mtp == kShards[3].tensors); } -TEST_CASE("Qwen3.8-27B-NVFP4-MTP: the FP8 KV scheme is declared where NO production path reads it") { +TEST_CASE("Qwen3.8-27B-NVFP4-MTP: the FP8 KV scheme is declared in the file this artifact's config.json outranks") { // `hf_quant_config.json` asks for an FP8 KV cache. const nlohmann::json& hq = ReleasedHfQuantConfig().at("quantization"); CHECK(hq.at("quant_algo").get() == "MIXED_PRECISION"); CHECK(hq.at("kv_cache_quant_algo").get() == "FP8"); - // `config.json`'s `quantization_config` — the ONLY document any production - // path in this tree reads — declares no KV scheme in either spelling, so the - // parsed config carries none and no loader can act on it. + // `config.json`'s `quantization_config` OUTRANKS `hf_quant_config.json` on + // the one production path that reads either — `vllm::ReadQuantConfigJson` + // (`src/vllm/config/cache.cpp:207`), called from `LoadedEngine::FromModelDir` + // (`model_loader.cpp:1988`) since `KV-FP8` W3 (#1593), mirroring the + // current-file-first order at `config.py:751-761`. This artifact ships the + // inline document, so the legacy file is never opened for it. And the inline + // document declares no KV scheme in either spelling, so the parsed config + // carries none and no loader can act on it. CHECK_FALSE(ReleasedQuant().contains("kv_cache_quant_algo")); CHECK_FALSE(ReleasedQuant().contains("kv_cache_scheme")); CHECK(ReleasedMixed().kv_cache_quant_algo().empty());