From 10f947eba3dd8a286a1c1c4ab183c43b2ec6dedf Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 13:10:51 +0800 Subject: [PATCH 01/13] feat: upgrade the op collector to vLLM 0.30.0, SGLang 0.5.21 and TRT-LLM 1.3.0rc29 Main already ships the 2026-10 campaign tables for these runtimes (#517), but its collector still defaults to vLLM 0.24.0 / SGLang 0.5.14 / TRT-LLM 1.3.0rc20. This lands the collector that produced those tables. It is the collector half of #228; the op-probe harness follows separately. - Runtime pins and audits: framework_manifest defaults move to vLLM 0.30.0, SGLang 0.5.21 and TRT-LLM 1.3.0rc29. Per-op __compat__ spans keep main's GLM-5.3-Flash pins (vLLM 0.31.0, SGLang 0.5.20). The SGLang gemm fork for 0.5.20 applies to exactly 0.5.20, so the 0.5.21 default keeps its own path. - Op ports to the new runtimes (sglang runtime_compat, vLLM attention metadata, TRT-LLM GDN / DSV4 / MoE), new kernel_source label translations and the regenerated op_backend_facts registry. - vLLM and SGLang MoE run FlashInfer's autotuner before timing and log the chosen tactic and fallback reasons per case (the fix behind the SM100 NVFP4 MoE data). - Collection cost: vLLM MLA/DSA modules sweep a whole seq_len or isl x prefix series per build; DSA modules are measured at the per-rank heads their TP shards produce; graph-timed modules run fewer eager warmups; TRT-LLM reuses GEMM modules and attention layers within a worker and recycles the MoE worker every N cases; SGLang MoE builds its layer once per token sweep; per-phase timing lines; the stall watchdog counts a worker heartbeat as progress. - The retired DSV4 hca_attn / csa_attn sidecar ops are removed end to end (collector, Rust enum, data tables, reuse manifest). - Generator: a TRT-LLM 1.3.0rc29 template, the SGLang 0.5.20 template update, hardware and model facts from the op probes, and token budgets rounded up to a multiple of 64 on every backend (the Dynamo 1.2.0 golden contract now expects 4096 / 3584). - Nine bundled model configs (two with hf_quant_config) and their THIRD_PARTY_NOTICES entries. - benchmarks/evidence/collector/op-probe-2026-10: the probe findings and results the collector and generator comments cite as provenance. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- THIRD_PARTY_NOTICES.md | 43 + .../collector/op-probe-2026-10/README.md | 32 + .../collector/op-probe-2026-10/findings.yaml | 3104 ++++ .../retests/sm100/sglang-0.5.21.moe_auto.yaml | 602 + .../results/sm100/sglang-0.5.21.yaml | 803 + .../results/sm100/trtllm-1.3.0rc29.yaml | 734 + .../perf-data/perf-data-reuse-analysis.md | 2237 +-- crates/core/src/perfmodel/common/enums.rs | 4 - docs/perf-model/collector/upgrade.md | 6 +- .../.claude/skills/aic-auto-collect/SKILL.md | 31 + python/aisimulate/THIRD_PARTY_NOTICES.md | 43 + python/aisimulate/collector/README.md | 49 +- python/aisimulate/collector/case_generator.py | 105 +- .../collector/cases/base_ops/mla_module.yaml | 58 + .../collector/cases/capabilities.yaml | 12 +- .../models/DeepseekV32ForCausalLM_cases.yaml | 7 +- .../models/DeepseekV3ForCausalLM_cases.yaml | 9 +- .../models/DeepseekV4ForCausalLM_cases.yaml | 34 +- .../Gemma4ForConditionalGeneration_cases.yaml | 12 +- .../models/GlmMoeDsaForCausalLM_cases.yaml | 11 +- ...KimiK25ForConditionalGeneration_cases.yaml | 2 - .../Llama4ForConditionalGeneration_cases.yaml | 2 +- .../models/MiniMaxM2ForCausalLM_cases.yaml | 3 - .../models/MiniMaxM3ForCausalLM_cases.yaml | 7 + python/aisimulate/collector/collect.py | 594 +- .../collector/evidence_exceptions.yaml | 52 + .../collector/framework_manifest.yaml | 148 +- .../aisimulate/collector/hash_closures.yaml | 7 - python/aisimulate/collector/helper.py | 755 +- .../collector/kernel_source_backends.yaml | 25 +- .../network/slurm/collect_trtllm_alltoall.py | 59 +- .../collector/op_backend_catalog.yaml | 4 - .../collector/op_backend_facts.yaml | 354 +- python/aisimulate/collector/registry_types.py | 70 +- .../collector/sglang/collect_attn.py | 193 +- .../collector/sglang/collect_attn_encoder.py | 62 +- .../collector/sglang/collect_computescale.py | 66 +- .../collector/sglang/collect_dsv4_attn.py | 125 +- .../collector/sglang/collect_dsv4_megamoe.py | 5 +- .../collector/sglang/collect_gdn.py | 218 +- .../collector/sglang/collect_gemm.py | 10 +- .../collector/sglang/collect_kda.py | 112 +- .../collector/sglang/collect_mhc_module.py | 35 +- .../collector/sglang/collect_mla.py | 203 +- .../collector/sglang/collect_mla_bmm.py | 35 +- .../collector/sglang/collect_mla_module.py | 129 +- .../collector/sglang/collect_moe.py | 467 +- .../collector/sglang/collect_msa_module.py | 74 +- .../sglang/deepseekv4_sparse_modules.py | 438 +- .../sglang/glm5_dsa_sparse_modules.py | 4 +- .../aisimulate/collector/sglang/registry.py | 49 +- .../collector/sglang/runtime_compat.py | 181 + .../collector/sglang/runtime_limits.py | 27 + .../collector/trtllm/collect_attn.py | 131 +- .../collector/trtllm/collect_attn_encoder.py | 29 +- .../collector/trtllm/collect_computescale.py | 100 +- .../collector/trtllm/collect_dsv4_attn.py | 181 +- .../collector/trtllm/collect_gdn.py | 415 +- .../collector/trtllm/collect_gemm.py | 134 +- .../collector/trtllm/collect_mamba2.py | 307 +- .../collector/trtllm/collect_mla.py | 43 +- .../collector/trtllm/collect_mla_bmm.py | 5 +- .../collector/trtllm/collect_mla_module.py | 149 +- .../collector/trtllm/collect_moe.py | 324 +- .../collector/trtllm/collect_msa_module.py | 124 +- .../aisimulate/collector/trtllm/registry.py | 3 + .../aisimulate/collector/version_resolver.py | 3 +- .../aisimulate/collector/vllm/collect_attn.py | 117 +- .../collector/vllm/collect_attn_encoder.py | 2 +- .../collector/vllm/collect_computescale.py | 53 +- .../collector/vllm/collect_dsv4_attn.py | 242 +- .../aisimulate/collector/vllm/collect_gdn.py | 6 +- .../aisimulate/collector/vllm/collect_gemm.py | 25 +- .../aisimulate/collector/vllm/collect_kda.py | 217 +- .../collector/vllm/collect_mhc_module.py | 11 +- .../collector/vllm/collect_mla_bmm.py | 9 +- .../collector/vllm/collect_mla_module.py | 1204 +- .../collector/vllm/collect_mla_module_027.py | 36 - .../aisimulate/collector/vllm/collect_moe.py | 41 +- .../collector/vllm/collect_moe_xpu.py | 6 +- .../collector/vllm/collect_msa_module.py | 142 +- python/aisimulate/collector/vllm/registry.py | 14 +- python/aisimulate/collector/vllm/utils.py | 85 +- .../wideep/trtllm/collect_moe_compute.py | 14 +- .../config/backend_config_mapping.yaml | 41 +- .../sglang/cli_args.0.5.20.j2 | 9 + .../extra_engine_args.1.3.0rc29.yaml.j2 | 84 + .../src/aisimulate/generator/facts/apply.py | 57 +- .../aisimulate/generator/facts/hardware.yaml | 97 +- .../aisimulate/generator/facts/models.yaml | 108 +- .../generator/facts/request_resolution.py | 7 + .../src/aisimulate/generator/module_bridge.py | 36 + .../src/aisimulate/generator/naive.py | 14 +- .../aisimulate/generator/rendering/engine.py | 12 +- .../generator/rendering/rule_engine.py | 31 + .../generator/rule_plugin/trtllm.rule | 6 + .../generator/rule_plugin/vllm.rule | 12 + .../src/aisimulate/generator/utils.py | 143 + .../Qwen--Qwen3.8-27B-FP8_config.json | 1033 ++ .../Qwen--Qwen3.8-27B_config.json | 140 + .../TokenRhythm--NeoHorse-1-9B_config.json | 81 + ...dia--DeepSeek-V4.1-Flash-NVFP4_config.json | 365 + ...Seek-V4.1-Flash-NVFP4_hf_quant_config.json | 179 + .../nvidia--Kimi-K3-NVFP4_config.json | 14241 ++++++++++++++++ ...ron-3.5-Lightning-30B-A3B-BF16_config.json | 121 + ...vidia--Qwen3.8-2.4T-A95B-NVFP4_config.json | 2690 +++ ...en3.8-2.4T-A95B-NVFP4_hf_quant_config.json | 1902 +++ .../openbmb--MiniCPM5-2B_config.json | 27 + .../ornith-ai--Ornith-1.5-35B-A3B_config.json | 125 + .../src/aisimulate_core/sdk/common.py | 23 +- .../aisimulate_core/sdk/operations/dsv4.py | 6 +- .../src/aisimulate_core/sdk/utils.py | 3 + .../vllm/0.24.0/collection_meta.yaml | 2 - .../0.24.0/dsv4_hca_attn_module_perf.parquet | Bin 14637 -> 0 bytes .../vllm/0.25.0/collection_meta.yaml | 16 - .../0.25.0/dsv4_hca_attn_module_perf.parquet | Bin 21704 -> 0 bytes .../vllm-0.25.0-collection-report.json | 31 +- .../sparse_attention/vllm/0.22.0/reuse.yaml | 4 - .../vllm/0.24.0/collection_meta.yaml | 2 - .../0.24.0/dsv4_hca_attn_module_perf.parquet | Bin 14614 -> 0 bytes .../vllm/0.25.0/collection_meta.yaml | 10 - .../0.25.0/dsv4_hca_attn_module_perf.parquet | Bin 21277 -> 0 bytes .../vllm/0.24.0/collection_meta.yaml | 2 - .../0.24.0/dsv4_hca_attn_module_perf.parquet | Bin 14906 -> 0 bytes .../vllm/0.25.0/collection_meta.yaml | 10 - .../0.25.0/dsv4_hca_attn_module_perf.parquet | Bin 21113 -> 0 bytes .../sparse_attention/vllm/0.22.0/reuse.yaml | 4 - .../vllm/0.24.0/collection_meta.yaml | 2 - .../0.24.0/dsv4_hca_attn_module_perf.parquet | Bin 14943 -> 0 bytes .../vllm/0.25.0/collection_meta.yaml | 10 - .../0.25.0/dsv4_hca_attn_module_perf.parquet | Bin 20485 -> 0 bytes .../sparse_attention/vllm/0.22.0/reuse.yaml | 4 - .../vllm/0.24.0/collection_meta.yaml | 2 - .../0.24.0/dsv4_hca_attn_module_perf.parquet | Bin 15563 -> 0 bytes .../vllm/0.25.0/collection_meta.yaml | 10 - .../0.25.0/dsv4_hca_attn_module_perf.parquet | Bin 23869 -> 0 bytes .../vllm/0.24.0/collection_meta.yaml | 2 - .../0.24.0/dsv4_hca_attn_module_perf.parquet | Bin 15314 -> 0 bytes .../vllm/0.25.0/collection_meta.yaml | 10 - .../0.25.0/dsv4_hca_attn_module_perf.parquet | Bin 24178 -> 0 bytes .../systems/perf_data_reuse_manifest.yaml | 5 - .../test_release_1_2_0_backend_contracts.py | 6 +- .../test_collect_attn_encoder_sm_dispatch.py | 34 + .../test_collect_attn_flashinfer_workspace.py | 71 + .../sglang/test_collect_gdn_contract.py | 1 + .../sglang/test_collect_kda_contract.py | 20 +- .../sglang/test_collect_mla_graph_capture.py | 195 + .../sglang/test_collect_mla_module.py | 3 + .../sglang/test_collect_moe_population.py | 175 +- .../test_collect_moe_unverified_lane_guard.py | 38 +- .../sglang/test_moe_config_observation.py | 6 +- .../collector/test_b200_025_publication.py | 4 +- .../collector/test_benchmark_graph_scope.py | 37 +- .../test_collect_ops_not_in_plan_exit_code.py | 52 + .../test_collect_provenance_writer.py | 4 +- .../unit/collector/test_collect_replay.py | 110 + .../collector/test_collect_trtllm_alltoall.py | 91 + .../collector/test_dsv4_common_test_cases.py | 2 +- .../test_dsv4_retired_sidecar_ops_removed.py | 48 + .../collector/test_error_classification.py | 55 + .../unit/collector/test_framework_manifest.py | 75 +- .../collector/test_getter_deduplication.py | 1 + .../test_glm5next_collection_plan.py | 9 +- .../collector/test_glm5next_runtime_pins.py | 8 +- .../test_helper_flashinfer_autotune.py | 447 + .../unit/collector/test_helper_phase_clock.py | 107 + .../unit/collector/test_kv_block_size.py | 48 + .../tests/unit/collector/test_model_cases.py | 205 +- .../collector/test_msa_getter_contract.py | 29 + .../collector/test_msa_kernel_limit_scope.py | 45 + .../unit/collector/test_network_layout.py | 11 +- .../test_no_eager_graph_fallback_sites.py | 65 + .../tests/unit/collector/test_parallel_run.py | 145 +- .../test_perf_output_finalization.py | 133 + .../tests/unit/collector/test_provenance.py | 13 - .../test_registry_table_producers.py | 166 + .../test_resume_finalize_zero_passed.py | 90 + .../collector/test_sglang_glm5_sparse_0514.py | 15 +- .../unit/collector/test_version_resolver.py | 19 +- .../test_vllm_attn_kv_cache_layout.py | 67 + .../unit/collector/test_vllm_collect_attn.py | 18 +- .../unit/collector/test_vllm_dsv4_attn.py | 5 +- .../tests/unit/collector/test_vllm_mla_cli.py | 2 + .../collector/test_vllm_mla_graph_timing.py | 164 +- .../test_vllm_mla_input_population.py | 113 +- .../trtllm/test_collect_dsv4_attn.py | 55 +- .../trtllm/test_collect_gdn_fail_closed.py | 169 + .../trtllm/test_collect_gemm_op_cache.py | 178 + .../trtllm/test_collect_mamba2_fail_closed.py | 336 + .../trtllm/test_collect_module_layer_cache.py | 86 + .../collector/trtllm/test_collect_moe_ep.py | 35 +- .../trtllm/test_collect_moe_worker_recycle.py | 200 + .../test_mla_precision_mxfp4_padding.py | 12 +- .../vllm/test_collect_kda_contract.py | 9 +- .../vllm/test_rope_missing_key_shim.py | 261 + .../test_engine_limit_preservation.py | 2 +- .../generator/test_facts_hardware_l40s.py | 100 + .../generator/test_facts_hardware_sm120.py | 53 + .../unit/generator/test_llmd_deployment.py | 3 +- .../test_model_profile_deepseek_v41.py | 55 + .../test_model_profile_minimax_m3.py | 41 + .../test_sglang_cuda_graph_flags_0520.py | 83 + .../test_trtllm_msa_implementation.py | 57 + .../test_vllm_kda_decode_batch_cap.py | 69 + .../test_vllm_nvfp4_dsa_kv_cache_dtype.py | 117 + .../test_vllm_token_budget_alignment.py | 78 + .../unit/sdk/database/test_dsv4_sparse.py | 12 +- .../test_modelopt_mixed_precision_targets.py | 2 + .../aisimulate/tests/unit/sdk/test_utils.py | 45 + .../tests/unit/test_support_config_profile.py | 2 + .../tests/unit/tools/test_collect_campaign.py | 81 + .../tools/perf_database/collect_campaign.py | 383 + 212 files changed, 40192 insertions(+), 3626 deletions(-) create mode 100644 benchmarks/evidence/collector/op-probe-2026-10/README.md create mode 100644 benchmarks/evidence/collector/op-probe-2026-10/findings.yaml create mode 100644 benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml create mode 100644 benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml create mode 100644 benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml create mode 100644 python/aisimulate/collector/sglang/runtime_compat.py delete mode 100644 python/aisimulate/collector/vllm/collect_mla_module_027.py create mode 100644 python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json create mode 100644 python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/h200_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet delete mode 100644 python/aisimulate/src/aisimulate_core/systems/data/h200_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet create mode 100644 python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py create mode 100644 python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py create mode 100644 python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py create mode 100644 python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py create mode 100644 python/aisimulate/tests/unit/collector/test_collect_replay.py create mode 100644 python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py create mode 100644 python/aisimulate/tests/unit/collector/test_error_classification.py create mode 100644 python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py create mode 100644 python/aisimulate/tests/unit/collector/test_helper_phase_clock.py create mode 100644 python/aisimulate/tests/unit/collector/test_kv_block_size.py create mode 100644 python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py create mode 100644 python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py create mode 100644 python/aisimulate/tests/unit/collector/test_registry_table_producers.py create mode 100644 python/aisimulate/tests/unit/collector/test_resume_finalize_zero_passed.py create mode 100644 python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py create mode 100644 python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py create mode 100644 python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py create mode 100644 python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py create mode 100644 python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py create mode 100644 python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py create mode 100644 python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py create mode 100644 python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py create mode 100644 python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py create mode 100644 python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py create mode 100644 python/aisimulate/tests/unit/generator/test_model_profile_minimax_m3.py create mode 100644 python/aisimulate/tests/unit/generator/test_sglang_cuda_graph_flags_0520.py create mode 100644 python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py create mode 100644 python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py create mode 100644 python/aisimulate/tests/unit/generator/test_vllm_token_budget_alignment.py create mode 100644 python/aisimulate/tests/unit/tools/test_collect_campaign.py create mode 100644 python/aisimulate/tools/perf_database/collect_campaign.py diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 2fd2b58b6b..1f2c508bd2 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -851,6 +851,49 @@ THE SOFTWARE AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED "AS IS", WITHOUT For any questions regarding this license, please contact model-business@notice.qwencloud.com. ``` +## Qwen3.8-27B model configuration files + +The following model configuration files are copied byte-for-byte from the +named repositories at the immutable revisions shown (onboarding 2026-09-25): + +| Packaged file | Upstream revision | +| --- | --- | +| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json` | `Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | +| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json` | `Qwen/Qwen3.8-27B-FP8@017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` | + +Copyright owner: Qwen. Both upstream revisions are licensed under the Apache +License 2.0 (`LICENSE` in each repository); the Apache License 2.0 terms are +reproduced in this distribution's `LICENSE` file. + +## Ornith 1.5 model configuration + +`aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json` is +an unmodified copy of `config.json` from +https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B at immutable revision +`10fbf86fed7ecee4a061f8b499a618f46001cac1`. The repository declares the MIT +License in its model card (no separate LICENSE file is shipped); copyright +owner: ornith-ai. The MIT License terms are reproduced in the DeepSeek +section above. + +## MiniCPM5 model configuration + +`aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json` is an +unmodified copy of `config.json` from +https://huggingface.co/openbmb/MiniCPM5-2B at immutable revision +`12a3808a956f869c767195e9266b59c4d21d92e2`. The repository declares the +Apache License 2.0 in its model card (no separate LICENSE file is shipped); +copyright owner: OpenBMB. The Apache License 2.0 terms are reproduced in this +distribution's `LICENSE` file. + +## NeoHorse 1 model configuration + +`aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json` is an +unmodified copy of `config.json` from +https://huggingface.co/TokenRhythm/NeoHorse-1-9B at immutable revision +`ba5b6e40d88a6ddf4591e176738254a3bc715765`. Copyright owner: TokenRhythm. +License: Apache License 2.0 (`LICENSE` in the repository), reproduced in this +distribution's `LICENSE` file. + ## Meta Llama 4 The following bundled model configs are modified copies of Meta Llama 4 diff --git a/benchmarks/evidence/collector/op-probe-2026-10/README.md b/benchmarks/evidence/collector/op-probe-2026-10/README.md new file mode 100644 index 0000000000..1dd99f0616 --- /dev/null +++ b/benchmarks/evidence/collector/op-probe-2026-10/README.md @@ -0,0 +1,32 @@ + + +# Op-probe findings (2026-09 to 2026-10) + +This non-packaged bundle holds the serving observations that the collector +and generator changes for the vLLM 0.30.0, SGLang 0.5.21 and TensorRT-LLM +1.3.0rc29 pins cite as provenance. The op-probe harness (`opharness`) produced +them by booting the real serving framework per model and recording the +kernels and backends it selected. The harness lands separately. + +Citations in the code name either a finding key or a results file below. Grep +a finding key, such as `sm100_probe_failures_recheck_2026_10_04`, to reach its +statement. + +## Files + +- `findings.yaml`: curated conclusions, one entry per key. Each statement gives + its own versions, dates and hardware inline, and `applies_to` lists the + models it was observed on. Evidence paths inside statements refer to + workspace bundles that are not committed. +- `results/sm100/sglang-0.5.21.yaml`: B200 boot-and-run verdict per model on + SGLang 0.5.21 under the default generator render. +- `results/retests/sm100/sglang-0.5.21.moe_auto.yaml`: the B200 SGLang 0.5.21 + retest of the MoE models under an explicit `--moe-runner-backend auto`. +- `results/sm100/trtllm-1.3.0rc29.yaml`: B200 boot-and-run verdict per model on + TensorRT-LLM 1.3.0rc29. + +Findings describe framework releases and rot with them. Re-verify them on the +next version bump instead of extending them. diff --git a/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml b/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml new file mode 100644 index 0000000000..e695caea22 --- /dev/null +++ b/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml @@ -0,0 +1,3104 @@ +# OUTPUT: curated conclusions from probe campaigns (the facts archive's +# human layer). Each finding states its own evidence paths, versions and +# dates inline; applies_to lists where it was observed. Rots with +# framework releases — re-verify on version bumps. Inputs live in targets.yaml. +findings: + dummy_notes: + statement: layer-kind period is FRAMEWORK-HARDCODED (sglang llama4.py:217 use_rope=(layer_id+1)%4!=0) with no + config field -> _ARCH_IMPLICIT_PERIODS; aic-core config lacks vision_config -> merged from unsloth mirror + (vllm builds the vision tower and profiles it even for text-only probes). The manual mirror-merge step BIT + on B300 (2026-09-20) — the gated-config fallback used the core copy WITHOUT vision_config, vllm assumed + default vision sizes (1025 pos) vs the mirror preprocessor's 336px (577) and both Llama-4 probes 'failed'; + codified since as fetch_assets' mirror-config fallback (B300 patch, pending merge) — after rebuilding the + dummies both pass. + applies_to: + - meta-llama/Llama-4-Scout-17B-16E-Instruct + k3_deepep: + statement: 'NO working deepep mode on sglang 0.5.17 + bundled DeepEP @ SM90 tp8/ep8 (A/B 2026-08-24, both deterministic): + mode=auto -> AssertionError forward_deepgemm_contiguous is deprecated at the first MoE forward (normal dispatch + routed into the removed contiguous-deepgemm path; all 8 ranks); mode=low_latency -> DeepEP device assert internode_ll.cu:358 + num_topk <= kNumMaxTopK (K3 num_experts_per_token=16 exceeds the LL kernel cap). Additionally K3''s own SP-collective/MegaMoE + path is SM103-gated (startup log: requires SM103, TP4/TP8, MegaMoE/DeepEP, CustomAllReduceV2 multicast; falls + back to NCCL on SM90) — the independent kernel route is unreachable on H20 regardless. Non-deepep tp8/ep8 + (a2a=none) works and matches tp1 kernel families. Evidence facts/tepdep/k3_deepep_{auto_v2,ll}.rank*.json + + run logs.' + applies_to: + - moonshotai/Kimi-K3 + sglang_0516: + statement: 'CORRECTED 2026-08-23 (retracts the earlier framework_bug verdict): the ''Hidden size mismatch'' + chain was a DUMMY false positive for converted-FP8 checkpoints. sglang picks the DSV4 expert layout by probing + the safetensors HEADER dtype of one routed-expert tensor (configs/deepseek_v4.py try_detect_fp4_experts); + weightless dummies probed None -> env default fp4=True -> fp4-packed weights fed to the triton fp8 kernel. + Real header dtypes (fetched from HF): native deepseek-ai = I8 (fp4-packed), sgl-project FP8 = F8_E4M3, nvidia + NVFP4 = U8. With the dtype_probe.safetensors signal: converted-FP8 checkpoints RUN OUT OF THE BOX (ok); native/NVFP4 + (true fp4 experts) fail under runner=auto on SM90 and RUN with moe_runner_backend=marlin -> needs_flags (facts/bisect_dsv4_{fp8experts,native_marlin}.json). + The earlier five-runner-all-fail experiments ran under the wrong fp4 layout and are retracted. The ue8m0/deep_gemm + SM100-only branch remains a separate real limit for the deep_gemm runner choice.' + applies_to: + - family:dsv4 + sglang_0516_flaky: + statement: trace NONDETERMINISTICALLY fails (~1/3 of runs) in flashinfer cutlass-DSL fused_add_rmsnorm via tvm_ffi + 'Mismatched Tensor' — the same command text both passes and fails on the same GPU (12 controlled reruns 2026-08-21); + every generator flag excluded by bisection; suspected timing-dependent tensor layout under enable-mixed-chunk. + Also hits MiniMax-M2.5. Probe retries once and records warmup_retry; persistent failure is a record, not a + block. + applies_to: + - MiniMaxAI/MiniMax-M2 + sglang_marlin_ab: + statement: 'runner=marlin A/B''d 2026-08-24: SAME capability ValueError — the gate (loader.py:282) checks the + quant config''s declared min_capability=100 before model construction; runner choice never enters. Declaration-locked, + no flag escape. Evidence facts/bisect_m3mxfp8_marlin.json.' + applies_to: + - MiniMaxAI/MiniMax-M3-MXFP8 + sglang_silent_downgrade: + statement: 'SGLANG BUG (not ours), mechanism pinned: the artifact ships quant_method=''modelopt'' + quant_algo=''MIXED_PRECISION''; + sglang 0.5.16''s mixed detector matches only the literal ''modelopt_mixed'' (modelopt_quant.py:622) and the + generic modelopt fallback greps quant_algo for FP8/NVFP4/FP4 substrings (base_config.py:189-196) — MIXED_PRECISION + matches neither -> silently Unquantized, zero warnings. Our dummy carries the full real metadata and the SAME + metadata binds ModelOptNvFp4FusedMoE correctly on vllm. Worth reporting upstream; recheck on 0.5.17.' + applies_to: + - nvidia/MiniMax-M3-NVFP4 + sglang_sm90: + statement: 'NO working path: SM90 gates NVFP4 MoE to marlin only (flashinfer_mxfp4/flashinfer_cutlass/triton + all rejected ''platform does not support NVFP4 with selected backend''), and marlin fused_moe asserts ''Only + gated SiLU is supported'' — Gemma uses gelu_pytorch_tanh. framework gap: NVFP4xgelu-MoE unservable on sglang/SM90. + Dense 31B-IT-NVFP4 passes (no MoE). Evidence facts/bisect_gemma4nvfp4_*.json, 2026-08-24.' + applies_to: + - nvidia/Gemma-4-26B-A4B-NVFP4 + sglang_workaround: + statement: 'flake root-caused to flashinfer cutlass-DSL fused_add_rmsnorm tvm_ffi nondeterminism: SGLANG_IS_FLASHINFER_AVAILABLE=false + passes 4/4 (facts/bisect_m2_noflashinfer_*.json, 2026-08-22). Deterministic workaround exists; perf cost unmeasured.' + applies_to: + - MiniMaxAI/MiniMax-M2 + - MiniMaxAI/MiniMax-M2.5 + sm90_triple: + statement: 'Same checkpoint, three fates on SM90. vllm 0.24 RUNS but NOT at mxfp8 precision: binds ModelOptMxFp8FusedMoE/LinearMethod + yet executes marlin_moe_wna16::Marlin + marlin::Marlin — weight-only dequant (W8A16 semantics); perf collected + here is Marlin-dequant data, not mxfp8 tensor-core data (same class as NVFP4-on-SM90). sglang 0.5.16 DEFINITIVELY + unservable: explicit quant-method capability floor ''mxfp8 minimum capability 100, current 90'' — not runner-selectable, + no flag bypasses it. trtllm rc23 DEFINITIVELY unservable but for a NON-quant reason: all three M3 artifacts + die at the sparse-attention wiring wall before quant matters. Evidence: records + raws, 2026-08-24.' + applies_to: + - MiniMaxAI/MiniMax-M3-MXFP8 + support: + statement: 'Generator/SDK supports K3 since upstream rebase 2026-08-23 (KimiK3Config, KIMIK3 family); the golden + command renders. Failure now correctly sits at the FRAMEWORK layer: unregistered at the pins (sglang 0.5.16 + / vllm 0.24 / trtllm rc23); registered in latest releases (0.5.17 / 0.27.1 / rc24, probed).' + applies_to: + - moonshotai/Kimi-K3 + support_2: + statement: 'same as moonshotai/Kimi-K3: architecture unregistered at all three pinned framework versions (probe-verified + 2026-08-20).' + applies_to: + - nvidia/Kimi-K3-NVFP4 + topology_invariance: + statement: 'ep/tp sharding on the a2a=none (single-node default) path does NOT change kernel families — measured + across GLM-5.2-FP8 (tp2/tp8 x ep1/ep2/ep8: triton fused_moe throughout), DSV4-NVFP4 (tp2 x ep1/ep2: marlin), + Kimi-K2.5-FP8 (tp8 x ep1/ep8: fused_marlin_moe). Evidence facts/tepdep/*.rank0.json, 2026-08-21. Therefore + tp1 probes represent kernel identity for the default path; multi-rank probing is scoped to routes that change + kernels.' + applies_to: + - a2a=none sharding + topology_sm103: + statement: 'the owner-reported TEP8 kernel fork is SM-gated, from sglang''s own dispatch log: ''K3 SP collective + requires SM103, TP4/TP8, MegaMoE/DeepEP, and CustomAllReduceV2 with multicast; using NCCL.'' — the special + path never activates on SM90; verify on SM103 (Blackwell Ultra) during the Blackwell replay.' + applies_to: + - moonshotai/Kimi-K3 + topology_sm90: + statement: 'sglang 0.5.17 on SM90 (probed): tp1 and TEP8 and DEP8(a2a=none) all run Mxfp4MoEMethod -> bf16-dequant + triton fused_moe_kernel, IDENTICAL kernel sets (facts/tepdep/k3_{tep8,dep8}.rank0.json). DEP8+deepep (the + real DEP deployment route) CRASHES at warmup: ''AssertionError: forward_deepgemm_contiguous is deprecated'' + — the DeepEPMoE route for this quant recipe is broken at 0.5.17/SM90.' + applies_to: + - moonshotai/Kimi-K3 + topology_verdict: + statement: 'Kimi-K2.5 (compressed-tensors W4-packed -> marlin): TEP8 == DEP8 kernel family (sglang::fused_marlin_moe + + device::marlin_moe::Marlin, same align/reduce/activation set; only shape-tuning template args differ). facts/tepdep/k25_fp8_{tep8,dep8}.rank0.json, + 2026-08-21.' + applies_to: + - tp8/ep1 + trtllm_fp8kv: + statement: 'trtllm rc23/SM90 UNIFIED root cause: fp8 KV cache in the MLA/DSA path crashes natively in flash_mla + (frame#62 abort). Door 1 = engine kv_cache_config.dtype:fp8 (generator renders for fp8 profiles) -> config_gap, + kv-auto A/B passes (GLM-5/5.1/5.2-FP8, DSV3.2, MiniMax-M2/M2.5 whose ''Linear assert'' was the same trigger). + Door 2 = checkpoint hf_quant kv_cache_quant_algo:FP8 (all NVFP4 artifacts) -> framework_bug, kv-auto STILL + crashes (GLM-5/5.1-NVFP4, DSV3.2-NVFP4). Evidence facts/trt_ab_kvauto_*.json, 2026-08-22.' + applies_to: + - MiniMaxAI/MiniMax-M2 + - zai-org/GLM-5-FP8 + - zai-org/GLM-5.1-FP8 + - deepseek-ai/DeepSeek-V3.2 + - MiniMaxAI/MiniMax-M2.5 + - nvidia/GLM-5-NVFP4 + - nvidia/GLM-5.1-NVFP4 + - nvidia/DeepSeek-V3.2-NVFP4 + - family:glm + trtllm_layers: + statement: 'rc23 fails in QUANT PARSING before the arch check (quant_config_utils.py:43 NoneType on the 4-bit-float + compressed-tensors config) — two walls stacked: parser first, arch-unregistered behind it. rc24 passes parsing + AND registers the arch, then stops at the missing ''fla'' package in the image.' + applies_to: + - moonshotai/Kimi-K3 + trtllm_linear_assert: + statement: 'trtllm rc23: fp8 Linear.apply assertion (linear.py:1142) under the generator-rendered engine args; + probe-default args loaded fine — surfaced by the fidelity upgrade 2026-08-21.' + applies_to: + - MiniMaxAI/MiniMax-M2 + - MiniMaxAI/MiniMax-M2.5 + trtllm_quant_algo: + statement: hf_quant mixes NVFP4 experts (552 layers) with FP8_PB_WO attention projections (2790 layers, fp8 + per-block WEIGHT-ONLY). FP8_PB_WO is absent from the QuantAlgo enum in rc23 AND rc24; nearest member FP8_BLOCK_SCALES + is W8A8 (fp8 activations), NOT weight-only — no semantic equivalent exists, so this needs a new enum + fp8 + weight-only dequant path in trtllm, not a rename. W4A16 kernels exist but only apply to the expert half. Enum + grep-verified 2026-08-24. + applies_to: + - nvidia/Kimi-K3-NVFP4 + trtllm_quant_parser: + statement: rc23 quant_config_utils.py:43 crashes (NoneType) parsing the native compressed-tensors W4 quantization_config + — framework_bug; K2.5-NVFP4 sibling fails separately on NVFP4 MoE tactics (platform_floor SM100). + applies_to: + - moonshotai/Kimi-K2.5 + trtllm_rc23: + statement: 'trtllm rc23: NOT capacity (2026-08-21 explanation RETRACTED — the 64B/120GiB arithmetic used gated-3-mat + at hidden width, ignoring moe_latent_size=2048+relu2; true experts/layer ~10.7B). Measured 2026-08-23: peak + 33GB/96GB, weights fit; crash is a native ILLEGAL MEMORY ACCESS right after model init (cudaStreamDestroy + IMA -> abort in KVCacheManager teardown, exit 139, no raw written). Isolation: Ultra-BF16/FP8 pass trtllm, + Nano/Super-NVFP4 pass trtllm — Ultra-shape x NVFP4-path specific framework bug. Evidence facts/ultra_nvfp4_trt_ima.log + + mem monitor.' + applies_to: + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 + trtllm_rc23_2: + statement: 'DSV4 rejected at load: ''Missing COMPRESS pool metadata for compress ratio 4'' — CSA compression + unsupported on this path.' + applies_to: + - family:dsv4 + trtllm_rc24: + statement: rc24 registers KimiK3ForConditionalGeneration but the nvcr image lacks the 'fla' (flash-linear-attention) + package its model file imports — K3 is unserveable on the shipped image (probe-verified 2026-08-21). + applies_to: + - moonshotai/Kimi-K3 + trtllm_sparse_config: + statement: 'MiniMax-M3 on trtllm REQUIRES llmapi sparse_attention_config (algorithm: minimax_m3) to get MiniMaxM3KVCacheManagerV2 + — the generator does not render it (needs_flags). With it set, rc23/SM90 still fails deeper (MiniMaxM3SparseRuntimeBackend.forward + missing cache args — framework gap, recheck on rc24/Blackwell). Evidence facts/trt_ab_m3_sparse.json.' + applies_to: + - family:m3 + trtllm_tokens_per_block: + statement: DeepseekV4CacheManager requires tokens_per_block in [128,256]; the generator renders 64 (FPM example + value) — ILLEGAL FOR THIS ARCH ON ANY PLATFORM (needs_flags for the generator). With 128 the load proceeds + to the explicit pre-blackwell rejection -> final cause on SM90 is platform_floor. Evidence facts/trt_ab_dsv4_tpb128.json, + 2026-08-22. + applies_to: + - family:dsv4 + vllm_024: + statement: 'NotImplementedError at load: checkpoint hf_quant excludes lm_head but NOT model.embed_tokens while + weights are tied; vllm''s ModelOpt NVFP4 config has no embedding quant method. Real-artifact metadata x framework + combination — real deployment hits the same. Affects both 26B and 31B-IT.' + applies_to: + - nvidia/Gemma-4-26B-A4B-NVFP4 + vllm_024_2: + statement: same tied-embedding NotImplementedError as 26B (see its entry). + applies_to: + - nvidia/Gemma-4-31B-IT-NVFP4 + vllm_024_image_tilelang_stub: + statement: 'PRISTINE vllm/vllm-openai:v0.24.0 CANNOT load DSV4 on vllm: tilelang (hard dep of DSV4 mhc) ships + lib/libcudart_stub.so with undefined symbol cudaDeviceReset; vllm''s load path resolves the stub via ctypes + and crashes (A/B on pristine image: facts/ab_dsv4_vllm_pristine_image.json, 2026-08-24 — minimal ''import + tilelang; import flashinfer.comm'' does NOT trigger it, the engine load path does). Real 0.24.0-image deployments + of DSV4 hit the same. Upstream-fixed: v0.27.1 ships tilelang without stub files. The -fix image only symlinks + the stub to the real libcudart (no dispatch change); drop it when the vllm pin moves past 0.24.' + applies_to: + - vllm/vllm-openai:v0.24.0 + vllm_0271: + statement: 'vllm v0.27.1 probed OK: KimiK3DeltaAttention (+MTP model registered), Mxfp4MoEMethod — same quant + identity as sglang 0.5.17.' + applies_to: + - moonshotai/Kimi-K3 + vllm_0290_dsa_attention_swap: + statement: 'vllm 0.29.0/SM90: the sparse-MLA/DSA attention path changed implementation — probes now record FlashAttnMLASparseImpl + and the 0.24-era dsa_nsa/triton kernel-family mix no longer appears for DeepSeek-V3.2 and the GLM-5.x family + (7 models, all still pass). Consequences: (1) attention perf data for these models must be recollected on + 0.29; (2) kernel_taxonomy.yaml needs rules for the new kernel names before attention identity is machine-adjudicable + again (backlog). Also observed: Kimi-K2.5 W4A16 MoE binding class renamed (CompressedTensorsWNA16MarlinMoEMethod + -> CompressedTensorsWNA16MoEMethod) while the executed kernel is unchanged (marlin_moe_wna16::Marlin) — class + names alone are not identity. Evidence: records.jsonl 0.24-vs-0.29 diff, 2026-09-19.' + applies_to: + - deepseek-ai/DeepSeek-V3.2 + - zai-org/GLM-5 + - zai-org/GLM-5-FP8 + - zai-org/GLM-5.1 + - zai-org/GLM-5.1-FP8 + - zai-org/GLM-5.2 + - zai-org/GLM-5.2-FP8 + - moonshotai/Kimi-K2.5 + vllm_0290_dsa_blackwell_paths_unvalidated: + statement: 'HARDWARE-VALIDATED on B300/SM103 2026-09-20 (upgrades the source-derived pre-read; evidence + results/pathdiff/sm103/vllm-0.29.0/*.json in the B300 workspace, pending its patch landing): backend selection + matched the pre-read — DSA ctx bf16 -> FLASHMLA_SPARSE (2.42ms b1 s512), fp8 -> FLASHINFER_MLA_SPARSE (1.58ms); + MSA -> MiniMaxM3SparseMSAImpl; dense attn -> flashinfer trtllm prefill/decode. The 029 lane construction + carried over with ZERO arch edits — the one bug found was arch-NEUTRAL: the lane handed the indexer builder + block_table_width=get_block_table_width(max_model_len)=64 while its own block_table_tensor had ceil(seq/block)=32 + columns; serving allocates the table AT the width (v1/worker/block_table.py) so builders assume equality, and + only the family-100 decode path reads the expanded buffer (indexer.py:609,745) — latent on SM90. Fixed by + zero-padding the table to the width; SM90 re-smoke green (gen bf16 0.190ms FLASH_ATTN_MLA_SPARSE, fp8 0.187ms + FLASHMLA_SPARSE, ctx 1.97ms) and all 3 sm90 dsa verdicts re-emitted aligned. CORRECTION to the pre-read: + "B200 = B300 on the vllm chain" holds only at DISPATCH level (family-100, no 103 special-case) — at kernel + identity level B300 prefill runs fmhaSm103a* cubins while decode runs fmhaSm100f* (B300 probe evidence), so + SM100 and SM103 need separate perf rows. Prior pre-read kept for the record: FLASH_ATTN_MLA_SPARSE is + major==9-only (flashattn_mla_sparse.py:71-72); indexer family-100 uses varlen_paged_mqa_logits + (indexer.py:475-503); SM120: TRITON_MLA + FLASHINFER_MLA_SPARSE_SM120 (cuda.py:130-134). DSA attention perf + rows remain strictly per-SM.' + applies_to: + - family:sparse_attention + - platform:sm100 + - platform:sm103 + - platform:sm120 + vllm_0290_dsa_lane_indexer_drift: + statement: 'RESOLVED 2026-09-19 (same day): the 029 lane now constructs the serving-same DeepseekV32Attention + (vllm.models.deepseek_v32.attention:118 — 0.29 routes DeepseekV32ForCausalLM to the DSV32-specific fused implementation, + registry.py:94) instead of the generic DeepseekV2MLAAttention the 0.24 collector used. Adaptations, each serving-cited: + forward arity (positions, hidden_states — no residual arg), forward_context.slot_mapping dict populated per + layer name (attention.py:303-306), perf-log layer accessor (DSV32Attention IS the MLAAttention layer, no .mla_attn + nesting). path_diff verdicts ALL ALIGNED (bf16 s512, bf16 s4096 sparse branch, fp8-kv) with zero kernel drift + — the fused prelude (fused_norm_rope) now matches serving, and the short-seq dense/sparse branch puzzle dissolved + with the module swap (same class, same branch logic). Perf delta of construction parity (H20, same GPU, back-to-back): + context b1 s512 1.76→1.97 ms (+12% — the fused serving prelude is SLOWER at tiny seq here), context b1 s4096 + 26.25→26.88 ms (+2.4%), generation b4 s2048 0.209→0.189 ms (-9.6%). Non-trivial deltas = the old construction + was mispricing the module; serving-parity rows are the only trustworthy ones. Legacy construction remains + behind AIC_DSA_LEGACY_MODULE=1 for A/B only (loud banner, never for data).' + applies_to: + - family:sparse_attention + - collector.vllm.collect_mla_module_029 + vllm_0290_dsv4_cutedsl: + statement: 'CORRECTED 2026-09-19 (retracts the same-day ''framework regression, family dead'' verdict): DSV4 + RUNS on vllm 0.29.0/SM90 — the crash is a CONFIG-TRIGGERED alignment bug, single-variable isolated. Trigger: + the generator renders --max-num-batched-tokens 6012; the runner slices per-group slot mappings as rows of a + 2D buffer with row width = max_num_batched_tokens, so a non-multiple-of-8 budget puts every row after the + first at a 64B-misaligned int64 offset (6012*8=48096 bytes, not /64), and DSV4 is multi-cache-group (MLA main + + compressor state + indexer) so its state-cache slot_mapping lands on a misaligned row — cutedsl''s tvm_ffi + wrapper asserts 64B alignment on argument #3 (slot_mapping, NOT state_cache as first written). A/B on the + pristine 0.29 image, same dummy: minimal args PASS (full request, whole DSV4 path executes: cutedsl compress, + fused indexer_q, dsv4_topk, FlashMLA decode metadata, deep_gemm fp8 — facts/ab_dsv4_minargs_trace_0290.json, + probe_isl 4096); minimal + 6012 reproduces the Misaligned error verbatim (ab_dsv4_tok6012_0290.json); minimal + + 6016 (=64*94) PASSES (ab_dsv4_tok6016_0290.json). Routing is SM-agnostic (compressor.py:402 is_cuda & + head=512 -> cutedsl on every SM; triton only for AMD/XPU/head-128), so B200 would crash identically under + 6012 — not a Hopper-specific kernel bug. Dual attribution: vllm robustness bug (pad slot-mapping rows or + relax the assert; worth reporting upstream) + generator trigger (odd token budget). REPRODUCED ON B300/SM103 2026-09-20 + (same Misaligned crash on all 8 DSV4 checkpoints, same single-variable flip: 6012->6016 makes + DSV4-Pro-0813 serve with native fp4 kernels). Birthplace located: the vllm rule plugin''s agg formula + max_batch_size + isl + 1500 (generator/rule_plugin/vllm.rule) — and the rule runs AFTER --generator-set, + silently overriding a user-supplied params.agg.max_num_tokens (separate defect, still open). FIX SHIPPED on + branch fix/generator-round-vllm-batched-tokens: rule budgets rounded up to 64 at the source + the render-layer + mapping rounds as last-line guard; 6 unit tests. Consequence: the + sparse_attention family split is NO LONGER FORCED by dsv4 (see vllm_0290_sparse_family_split_needed).' + applies_to: + - deepseek-ai/DeepSeek-V4-Flash + - deepseek-ai/DeepSeek-V4-Pro + - deepseek-ai/DeepSeek-V4-Flash-0731 + - deepseek-ai/DeepSeek-V4-Pro-0813 + - nvidia/DeepSeek-V4-Flash-NVFP4 + - nvidia/DeepSeek-V4-Pro-NVFP4 + - sgl-project/DeepSeek-V4-Flash-FP8 + - sgl-project/DeepSeek-V4-Pro-FP8 + vllm_0290_nemoh_profile_oom: + statement: 'vllm 0.29.0/SM90: Nemotron-H mamba profile run OOMs under the generator-rendered default (_state_passing_fwd + allocates 8GiB with ~4GiB free on 143GB). Root knob is max_num_seqs (64 alone passes; batched-tokens reduction + alone does not) but TWO generator walls block the direct fix: the rule engine floors max_batch_size at 512 + (pipeline.py ''512 if x<512 else x*2'') and unknown params.agg keys are SILENTLY IGNORED instead of rejected. + Expressible rescue: gpu-memory-utilization 0.6 via extra_cli_args (now the checkpoint''s cli_extra_args). + Evidence facts/ab_nemoh_{seq,tok,small,mem}_0290.json, 2026-09-19.' + applies_to: + - nvidia/Nemotron-H-56B-Base-8K + - generator:rule-engine + vllm_0290_op_sweep_triage: + statement: 'vllm 0.29.0 op-collector sweep, first pass (H20, quick smokes in the official container, 2026-09-19). + WORK AS-IS on 0.29 (perf rows produced, no code change): gemm (incl. fp8 CutlassFP8ScaledMM kernel_source), + moe (108 cases), encoder_attention (long sweep, no errors), gdn (4k rows), mhc_module. WORK VIA THE 029 LANE''s + generic registry construction: dsa (aligned, done) AND plain mla — DeepSeek-V3 runs unmodified through collect_mla_module_029 + (registry resolves DeepseekV3ForCausalLM -> generic deepseek_v2 model -> serving-same DeepseekV2MLAAttention; + FLASH_ATTN_MLA, 1.81ms ctx b1 s512), i.e. the config-in construction generalizes across architectures with + zero per-model code. FORKED AND CLOSED (second pass, same day): collect_attn_029.py (0.29 replaced the + set/get kv-cache-layout API with get_supported_kv_cache_layouts + resolve_kv_cache_layout and merged the + per-layer cache view to [B,H,N,2*D], flash_attn.py:1013-1014; SM90 smokes green on vllm_flash_attn_fa3; + path_diff aligned ctx+gen vs Meta-Llama-3.1-8B serving) and collect_msa_029.py (0.29 removed + AttentionBackend.get_kv_cache_shape — both caches now allocated via compute_layer_kv_cache_shape_bytes + and bound via each layer''s bind_kv_cache, v1/kv_cache_interface.py:253-269 + v1/worker/utils.py:622; + SM90 smokes green bf16 ctx/gen + fp8 ctx on MiniMaxM3SparseTritonImpl; path_diff aligned ctx+gen vs + MiniMax-M3 serving). SMOKED VIA components/op_smoke.py (generic registry-driven single-case runner — + one standard, no per-op drivers): compute_scale (2/2 cases, dynamic-minus-static fp8 quant rows) and + mla_bmm_gen_pre/post (2/2 each, vllm_torch_bmm) run AS-IS on 0.29. EXPECTED BLOCKED: dsv4_* (0.29 cutedsl + framework regression). OUT OF SCOPE: kda (preview-image runtime lane), xpu variants. Evidence: workspace + facts/opsweep_0290/*.txt.' + applies_to: + - framework:vllm-0.29.0 + - ops:gemm,moe,encoder_attention,gdn,mhc,mla,dsa,attention,msa,compute_scale,mla_bmm + - ops-blocked:dsv4 + vllm_0290_sparse_family_split_needed: + statement: 'UPDATED 2026-09-19 (the forcing condition is GONE): the split was motivated by ''DSA works on 0.29, + DSV4 dead on 0.29'' — but the DSV4 death was retracted (config-triggered alignment crash, not a framework + regression; see vllm_0290_dsv4_cutedsl CORRECTED). With the token-budget fix (round to 64) DSV4 serves on + 0.29, so dsa_* and dsv4_* can move pins TOGETHER and no family split is required for the 0.29 upgrade. + Remaining owner decisions on this thread: (1) approve the generic generator rounding rule (or accept + per-checkpoint cli_extra_args as the interim); (2) dsv4 op-collector 0.29 lane SHIPPED 2026-09-20 + (collect_dsv4_029.py: storage_block_size/get_kv_cache_shape removal -> generic byte-shape pipeline; builder + block_table_width + zero-padded table, the B300 family-100 contract applied preventively; binding via + bind_kv_cache). SM90 smokes green csa/hca x ctx/gen, backend FLASHMLA_SPARSE_DSV4, fp8_ds_mla layout + + FP8 Lightning-Indexer cache engaged. path_diff verdict still pending: it needs an ok DSV4 serving record at + 0.29, which needs the 64-aligned budget in the workspace render chain (the aic rule is unfixed; fix lives in + aisimulate branch fix/generator-round-vllm-batched-tokens). Historical context kept: DSA lane collect_mla_module_029.py validated (FLASH_ATTN_MLA_SPARSE bf16-kv / + FLASHMLA_SPARSE fp8-kv, both keys need collection), first live copy-on-upgrade instance.' + applies_to: + - family:sparse_attention + - deepseek-ai/DeepSeek-V3.2 + - zai-org/GLM-5.x + - deepseek-ai/DeepSeek-V4-* + vllm_0290_upgrades: + statement: 'vllm 0.24.0 -> 0.29.0 improvements confirmed on SM90: Kimi-K3 and Kimi-K3-NVFP4 now pass (arch registered, + as predicted by the 0.27.1 probe); Gemma-4-26B-A4B-NVFP4 passes (0.24''s tied-embedding NotImplementedError + and the NVFP4 MoE path both fixed); Gemma-4-31B-IT-NVFP4 passes after a DUMMY fidelity fix — 0.29''s transformers + loads the multimodal processor, so dummies must carry processor_config.json (fetch it with configs/tokenizers + during onboarding; the 26B dummy already had it, the 31B one did not).' + applies_to: + - moonshotai/Kimi-K3 + - nvidia/Kimi-K3-NVFP4 + - nvidia/Gemma-4-26B-A4B-NVFP4 + - nvidia/Gemma-4-31B-IT-NVFP4 + vllm_fp8kv: + statement: 'vllm 0.24/SM90: head-576 MLA has NO attention backend with fp8 KV; kv-auto(bf16) resolves DeepseekV2MLAAttention+SparseAttnIndexer + and runs. Door 1 engine kv dtype (fp8 ckpts) = config_gap, kv-auto A/B passes; door 2 ckpt hf_quant kv_cache_quant_algo:FP8 + (NVFP4 artifacts) = framework/platform gap, kv-auto still fails. Same two-door structure as trtllm. Evidence + facts/vllm_ab_kvauto_*.json, 2026-08-22. + 0.29.0 ADDENDUM (2026-09-24, GLM-5.3 onboarding): door 2 is now a SPELLING gap, not a missing kernel. With + kv auto the NVFP4 DSA checkpoints (GLM-5/5.1/5.2/5.3-NVFP4, DeepSeek-V3.2-NVFP4) resolve kv_cache_dtype to + the string fp8_e4m3 from the checkpoint kv scheme and the selector rejects every MLA backend + (FLASHMLA_SPARSE: "kv_cache_dtype not supported"); the explicit --kv-cache-dtype fp8 variant of the SAME + run loads FlashMLASparseImpl and generates (705890b31aa0, 4d34afefe419, ...). So on 0.29/SM90 these models + serve only with an explicit fp8 kv dtype — the golden vllm render passes none. Generator-relevant + (aiconfigurator should render --kv-cache-dtype fp8 for NVFP4 DSA on vllm 0.29) and upstream-worthy + (fp8_e4m3 vs fp8 in the sparse-MLA support check). Raw: archive/raw/67b8d5c6ce5e.json (Reasons dict). + GENERATOR FIX LANDED (owner "1，好的", commit ea0244a2): utils.vllm_dsa_kv_cache_dtype prescribes fp8 + keyed on architecture (GlmMoeDsaForCausalLM, DeepseekV32ForCausalLM) + the task''s nvfp4 GEMM quant mode; + module_bridge applies it to the worker payload (naive path has no quant fact). Left open for the SDK + owner: the perf task itself models these checkpoints with bf16 KV while serving runs fp8 KV either way.' + applies_to: + - zai-org/GLM-5-FP8 + - nvidia/GLM-5-NVFP4 + - nvidia/GLM-5.1-NVFP4 + - nvidia/DeepSeek-V3.2-NVFP4 + - zai-org/GLM-5.2-FP8 + - nvidia/GLM-5.2-NVFP4 + vllm_kv_required: + statement: 'INVERTED requirement vs GLM: vllm''s DeepseekV4 fp8_ds_mla layout REQUIRES fp8 kv-cache (attention.py:79 + assert, golden auto crashed 2026-08-23) while GLM head-576 has NO backend with fp8 KV. kv dtype must be rendered + per (model, backend) — neither blanket auto nor blanket profile->fp8 works.' + applies_to: + - family:dsv4 + probe_cudagraph_kernel_blindness_fixed: + statement: 'INSTRUMENT FIX (probe_vllm, 2026-09-19): the whole-run kernel tables were built by walking + torch.profiler CPU function_events — kernels replayed inside CUDA graphs have no CPU launch event and + were INVISIBLE. Consequence: vllm 0.29 MiniMax-M3 serving evidence showed ZERO attend/indexer kernels + (M3''s graph-safe attend runs under full cudagraph), and Llama FA evidence missed the in-graph + FlashAttnFwdCombine + prepare_varlen_num_blocks — path_diff produced FALSE "diverged" verdicts against + collector captures (collectors run eager, so their kernels were all visible). Fix in code, generic + (no per-model case): tables now come from the kineto DEVICE event stream (every executed kernel appears + there, graph-replayed or not), CPU-tree walk kept as fallback. Second generalized knob from the same + episode: probe prompt length is now --isl / AIC_PROBE_ISL (default 4096, owner-set 2026-09-19; the + historical implicit value was 32; recorded per-record as probe_isl) because + shape-conditional serving paths (M3 sparse threshold at topk_blocks*block_size tokens, FA split-KV) + are invisible below their thresholds — coverage is a parameter, never a per-model special case. + MiniMax-M3 and Meta-Llama-3.1-8B 0.29 raws re-probed at isl 4096 and superseded (old raws kept in + workspace facts/raw_superseded/); after the fix all four attn/msa 029-lane path_diff verdicts flipped + to aligned with zero collector changes.' + applies_to: + - instrument:probe_vllm + - framework:vllm-0.29.0 + - MiniMaxAI/MiniMax-M3 + - meta-llama/Meta-Llama-3.1-8B + pathdiff_fp8kv_coverage_complete: + statement: | + HARNESS COMPLETENESS (2026-09-23, driven by "fp8-KV must be probed and + compared, not skipped"; REVISED the same day after a second review found + four defects in the first pass — see probe_prefill_prefix_cache_contamination + for the largest). fp8-KV is a first-class path_diff dimension and the fp8 + attention paths are verified aligned with matched evidence: + (a) probe fp8-KV as a config axis: probe_vllm --kv-cache-dtype + + probe_driver kv_variants=[None,fp8] (vllm), path_diff --kv-dtype + selects the same-kv serving record (commit a2076196). + (b) build_ops orphan keep-rule is STRUCTURAL: every orphan the taxonomy + labels with an identity-bearing backend is kept; only unlabeled/ + noise orphans are capped (24, by device time). The first fix + (top-40 by time) was still a count; the original alphabetical + [:15] had truncated half the records (137/274 raws exceed 15). + (c) taxonomy: vllm kernel_unified_attention / reduce_segments + (TRITON_ATTN, the only fp8 head_dim>256 Gemma-4 path) labeled + `triton` — the shared vocabulary's name for the Triton attention + backend (kernel_source_backends.yaml: sglang `triton`, vllm + `vllm_triton_attn` -> triton). The first pass invented + `triton_attn`, breaking the one-vocabulary contract. Routing + kernels moved triton -> framework_native, so `triton` now means + attention only and left path_diff's infra set. + (d) verdict artifacts are reproducible: reports carry capture_file, + capture_cmd, op_hint, kv_dtype and the serving record (id, tp, kv, + isl, prefix_caching, candidate count); records carry runtime.isl; + path_diff --isl selects by prompt length and --serving-raw diffs + against explicit A/B evidence. The first pass saved verdicts to + results/pathdiff/sm90/ where the gate never reads them (it reads + sm90/-/). + VERDICTS (sm90, vllm 0.29.0, kv=fp8, results/pathdiff/sm90/vllm-0.29.0/): + - Gemma-4 head512 fp8 -> ALIGNED (triton both sides). + - DSV3.2 DSA fp8 GENERATION -> ALIGNED (flashmla sparse_fp8 decode + + dsa_nsa paged_mqa_logits + deepgemm). The first pass called this + capture "ctx": run_mla_module picks the phase from + `"context" in perf_filename` (collect_mla_module.py:920) and the + capture used /tmp/cap_perf.txt. + - DSV3.2 DSA fp8 CONTEXT s=4096 (128 heads, separate prefill path: + cp_gather_and_upconvert -> bf16 workspace -> sparse_attn_fwd_kernel) + -> ALIGNED against cache-cold serving at isl 4096. + - DSV3.2 DSA fp8 CONTEXT s=512 (dense FA3 MLA prefill, seq <= index_topk) + -> ALIGNED against cache-cold serving at isl 512 (--serving-raw + facts/reprobe_nopc/f208a101a9c8_isl512.json). Against the isl-4096 + record it is legitimately "diverged" (serving never ran FA3 at 4096): + prefill dispatch is length-conditional, so the serving evidence must + be isl-matched to the collector cell. + - heads=16 fp8 context (TP8-equivalent, num_heads < 32) follows the + mixed-batch path in the collector too (sparse_decode_fwd), matching + flashmla_sparse.py:307 — capture facts/pathdiff/cap_dsa_ctx_fp8_s4096_h16.json. + Capture precision must match serving: gemm_type=bfloat16 shows a false + fused_a-projection drift (cutlass vs fp8_blockscale); use fp8_block. + FOLLOW-UP RESOLVED 2026-09-24 (measured, not read): the Gemma-4 case + file declares sglang_backends {90: triton, ...} on every attention + profile (head_dim 256 and 512), which overrides collect_attn's default + SM90->fa3 table; op_smoke case 168 (Gemma4ForConditionalGeneration, + head_dim 512, triton) captured _fwd_kernel/_fwd_kernel_stage2 = the + serving record's TritonAttnBackend kernels -> ALIGNED + (attn_ctx_hd512_gemma-4-26B-A4B). Original: (unconfirmed, needs an + sglang capture): sglang 0.5.16 serves Gemma-4 (head512) on Triton + attention (_fwd_grouped_kernel_stage1 in the raws) while + collector/sglang/collect_attn.py pins FlashAttentionBackend on SM90 + unless head_dim==192 — same class as the vllm Gemma-4 override. + applies_to: + - instrument:path_diff + - instrument:kernel_taxonomy + - instrument:probe_driver + - framework:vllm-0.29.0 + - platform:sm90 + - google/gemma-4-26B-A4B + - deepseek-ai/DeepSeek-V3.2 + probe_prefill_prefix_cache_contamination: + statement: | + INSTRUMENT FIX (probe_vllm, 2026-09-23): the profiled request reused the + warmup prompt (list(range(isl))) and vllm enables prefix caching by + default, so every "prefill" step the probe profiled was a near-total + cache hit recomputing only the last block — a query of <= block_size + tokens with a full-length KV. ALL 173 vllm serving records with prefill + tables carried this residual, not an isl-token prefill. Identity verdicts + survived wherever the residual still routes to the same kernel family + (FA3, TRITON_ATTN), which is why it stayed invisible; DSA fp8 exposed it: + FlashMLASparse's builder classifies query_len <= 256 (128 heads) as + decode (flashmla_sparse.py:244 @0.29.0), so serving "prefill" evidence + showed the fp8 DECODE kernel and path_diff flagged the collector's + correct separate-path prefill (sparse_attn_fwd_kernel) as kernel drift. + Cache-cold re-probes confirm: DSV3.2 fp8 prefill 688us decode kernel -> + 26.9ms sparse_attn_fwd (+cp_gather_and_upconvert); Gemma-4 fp8 + kernel_unified_attention 863us -> 10.5ms; Llama-3.1-8B FA3 41us -> + 2.08ms. Fix: ea.enable_prefix_caching=False (same policy as the sglang + probe's --disable-radix-cache); records carry runtime.prefix_caching. + TARGETED RE-PROBE (owner decision 2026-09-23: no full re-sweep; GEMM/MoE + identity is not length-conditional, so only attention classes whose + DISPATCH changes with query length were redone). Triage over the 173 + records by mechanism, then 37 + 3 records re-probed cache-cold (all ok): + - DSA family (DSV3.2 variants, GLM-5/5.1/5.2 + FP8/NVFP4; 18): fp8-KV + prefill evidence flipped from the fp8 DECODE kernel to + sparse_attn_fwd_kernel (the separate-path bf16-workspace prefill), + matching the collector; bf16-KV DSV3.2 stayed FA3. + - MLA + fp8-KV (DeepSeek-R1/V3 fp8, Kimi-K2/K2.5, DSV3.1/K2.5/K2.6/ + K2.7 NVFP4; 12): flipped from flash_fwd_splitkv_mla (FlashMLA + decode) to FA3 prefill — the residual query had been routed through + the MLA decode path, so every MLA fp8 prefill row was wrong. + - MiniMax-M3 (7): prefill still showed NO labeled attention kernel — + that was a taxonomy gap, not the probe: 0.29 renamed the M3 Triton + kernels (_gqa_sparse_fwd/_decode, _index_block_score, + _decode_index_score, _topk_index_{partial,merge}); labeled `triton` + (kernel_source_backends.yaml: MiniMaxM3SparseTritonImpl -> triton), + closing the dsa_nsa-covers-M3 naming debt. msa029_m3_{ctx,gen} were + previously aligned on an EMPTY attention set; now kernel-matched. + - Dense FA3 / TRITON_ATTN / linear-attention (133 records): same + family before and after in the three cache-cold controls (Llama, + Gemma-4, DSV3.2 MoE/quant/routing identical; GEMM differs only in + cuBLAS nvjet tile shape) — identity verdicts are trustworthy, prefill + MAGNITUDES are not. build_records prints the count of such records + on every rebuild ("NOTE: N vllm records still carry cached-residual + prefill evidence"). Superseded raws: facts/raw_superseded/prefixcache_2026-09-23/. + DSV4 on vllm has NO serving evidence at all (36 records, mostly `load` + errors from the --max-num-batched-tokens 6012 trigger; the generator fix + lives on its own branch) — unrelated to this contamination. + The trtllm probe has the same shape (warmup and profiled prompt both + [[1]*16], block reuse on) plus no --isl and no phase tables — a weaker + lane that needs the same treatment. Evidence facts/reprobe_nopc/ + (candidates.json, postprocess.py), facts/pathdiff/cap_dsa_ctx_fp8_s4096_fp8gemm.json. + applies_to: + - instrument:probe_vllm + - instrument:probe_trtllm + - framework:vllm-0.29.0 + - platform:sm90 + op_coverage_vllm_0290: + statement: | + OP-LEVEL ALIGNMENT SWEEP (2026-09-23, after the probes were fixed): every + vllm registry op with a serving reference now has a path_diff verdict in + results/pathdiff/sm90/vllm-0.29.0/ (23 files, gate aligned). Captures are + one representative cell per op at a serving model's real shape, run via + the collector's own run_* entry (op_smoke semantics), diffed against + cache-cold serving records (re-probed for DSV3 bf16 and Qwen3.5-0.8B). + aligned: gemm {fp8_block/deepgemm vs DSV3.2, bf16/cublas vs Llama, + fp8 per-tensor vs Qwen3-32B-FP8}; moe {fp8_block DSV3.2, bf16 Gemma-4, + mxfp4 gpt-oss-120b}; mla_context {bf16 DSV3, fp8 R1}; mla_generation + {fp8 R1}; mla_bmm_gen_pre/post; gdn {ctx: flashinfer GDN + conv1d, + gen: fla_triton fused_recurrent} vs Qwen3.5-0.8B; compute_scale; + plus the earlier attention/dsa/msa set. + REAL DIVERGENCE FOUND AND FIXED — moe w4a16_mxfp4 (gpt-oss): the + collector built a bare Mxfp4Config -> Mxfp4MoEMethod -> MARLIN, while + vllm 0.29 overrides OCP "mxfp4" checkpoints of model_type gpt_oss to + GptOssMxfp4Config/GptOssMxfp4MoEMethod (quantization/mxfp4.py:119-139), + whose gpt-oss priority list (fused_moe/oracle/mxfp4.py:584-602) picks + TRITON matmul_ogs on SM90 — exactly what the gpt-oss-120b serving probe + shows. Same class as the Gemma-4 attention override: the collector + constructed the generic config and lost the model-specific method. + collect_moe.py now mirrors the override (commit in this series). + HARNESS FIXES surfaced by the sweep: (1) path_diff picked the LAST + matching serving record; a decode-only Qwen3.5 record beat the + cache-cold one and produced a false collector-only `flashinfer` — records + are now ranked by evidence quality (prefix_caching False > phase tables > + known isl); (2) Triton autotune tile suffixes (_matmul_ogs_..._16x256x128x1 + vs _128x256x128x1, BLOCK_M from the token count) are stripped by + normalize_kernel so kernel-drift compares kernels, not tiles; + (3) FlashInfer GDN kernels relabeled trtllm_internal -> flashinfer (the + vocabulary name; vllm runs them on SM90 @0.29). + NOT ALIGNED-ABLE TODAY (no serving reference, not a collector defect): + - mla_generation bf16-gemm cells: every DeepSeek/Kimi checkpoint is fp8 + or NVFP4, so the bf16-weight grid has no serving instance; the + collector's fused_a_gemm there is vllm's own small-M dispatch + (deepseek_v2.py:~915-925, num_tokens <= 16). Verdict file removed; + the fp8 cell is the reference comparison. + - dsv4_* (6 ops) and mhc_module: vllm DSV4 serving has no evidence + (load errors from the 6012 token-budget trigger; generator fix on + its own branch). + - kda: Kimi-K3 runs only on the vllm kimi-k3 preview image; its + records carry no kernels. + - encoder_attention: the probe prompt is text-only, so vision-encoder + kernels never execute in the profiled window (Qwen3-VL records have + none) — needs a multimodal probe request. + FOLLOW-UP: sglang (91 records at isl 32) and trtllm (74 at 16 tokens, + block reuse on) serving evidence predates their probe fixes; their + length-conditional classes (DSA/NSA, M3) need the same targeted + re-probe before sglang/trtllm op verdicts are drawn. + applies_to: + - instrument:path_diff + - instrument:probe_driver + - framework:vllm-0.29.0 + - platform:sm90 + - openai/gpt-oss-120b + - Qwen/Qwen3.5-0.8B + - deepseek-ai/DeepSeek-V3 + - deepseek-ai/DeepSeek-R1 + op_coverage_sglang_trtllm_2026_09_24: + statement: | + OP-LEVEL ALIGNMENT SWEEP, sglang 0.5.16 and trtllm 1.3.0rc23 on SM90, after + both probes were brought to the vllm standard (isl 4096, cache-cold, phase + tables) and their length-conditional records re-probed. Captures run the + collector through op_smoke (--case-filter/--case-prefix) or the module + CLI's --quick / in-process entry (the sglang DSA/MSA module collectors run + each case in a SUBPROCESS, which the parent profiler cannot see — a + capture of the worker wrapper is silently empty). + HARNESS DEFECT found first: path_diff graded a crashed or empty capture + "aligned" (empty set is a subset of everything) — six broken sglang + captures passed. Verdicts are now invalid-capture / no-collector-signal. + COLLECTOR DRIFT ON THE PINNED VERSIONS (all fixed, each surfaced as a + zero-kernel capture; 286 sglang + 114 vllm collector unit tests pass): + sglang 0.5.16 — ModelRunner takes ps: ParallelState (mla_module built + it with tp_rank/tp_size...), attention backends read model_runner.ps / + server_args.enable_prefill_cp / enable_dp_attention / + model_config.linear_attn_registry_result / hf_config.get_text_config() + / is_draft_worker (attention + MLA mocks lacked them), + Req.set_extend_input_len -> set_extend_range (mla_module, dsv4), + ForwardBatch.init_new requires return_hidden_states_before_norm + (mla_module context+generation, dsv4). The sglang collector had not + run on its own pin since the bump. + trtllm — gemm/computescale/mla_bmm imported case_generator and helper + as bare modules (only works from collector/ cwd); GDN prefill/decode + imported the vendored FLA Triton kernels directly while serving + dispatches to FlashInfer's cute-DSL GDN chunk + bf16-state decode on + SM90/SM100 (gdn_mixer.py:57-70,670; fused_sigmoid_gating_recurrent.py: + 205-238). The collector now calls the mixer's resolver for prefill + (packed [1,T,H,D] + cu_seqlens, fp32 g — FlashInfer's contract) and + serving's fused_sigmoid_gating_delta_rule_update for decode, labeling + rows by the dispatch that actually ran (new kernel_source values in + kernel_source_backends.yaml). + path_diff/normalize — cute-DSL symbols carry tensor specs + (…_kernel_tensorptrbf16gmemalign32o1291…): identity cut at + _tensorptr/_object; `trtllm::` launchers excluded like vllm/sglang; + collector EXIT_CODE_RESTART (10) is a protocol signal, not a failure. + VERDICTS (results/pathdiff/sm90/{sglang-0.5.16,trtllm-1.3.0rc23}/): + sglang ALIGNED: attn ctx+gen (Llama-3.1-8B shape, fa3), dsa ctx+gen + (DSV3.2 bf16: sparse_attn_fwd/FA3 + indexer), mla ctx+gen (DSV3, fa3), + msa ctx (M3), gemm fp8_block, moe fp8_block, gdn ctx+gen (Qwen3.5). + trtllm ALIGNED: attn ctx (Llama, fmha_v2 causal), dsa ctx+gen (DSV3.2), + mla ctx+gen (DSV3), gemm fp8_block, moe fp8_block, gdn ctx (FlashInfer + chunk + trtllm conv1d). + RESOLVED (2026-09-24, owner asked for these one at a time): + - trtllm attention_generation (Llama-3.1-8B): NOT a collector bug — + a cell mismatch. The first capture took the kv_len=1 cell, where + trtllm's own heuristic declines XQA (TLLM debug: "JIT XQA is not + used: maybe no performance gain" -> MMHA); the serving probe's + warmup does the same at short KV (8 of 424 decisions) and selects + XQA (kernel_mha) for the 4096-token decode. At the grid point that + matches the probe (batch 1, kv 4095) the collector selects XQA too + -> ALIGNED. TRTLLM_ENABLE_XQA_JIT no longer exists in rc23 binaries + (only TRTLLM_FORCE_XQA / TRTLLM_XQA_BLOCKS_PER_SEQUENCE); the + collector's toggle is documented as inert. Rule reinforced: capture + the collector cell at the probe's kv length, never the first cell. + - sglang msa_generation (MiniMax-M3): the PROBE is the unfaithful side. + The gate (flash_with_topk_idx.py:822-826) uses score.shape[2], which + under CUDA-graph capture is ceil(max_context_len / 128) (static + shapes, minimax_sparse_backend.py:175-178) and the live KV length in + eager. Production decodes under graphs; M3's context_len makes the + static block count exceed 4096 -> topk_split, which is what the + collector models (collect_msa_module.py:892-897, use_graph). The + identity probe disables CUDA graphs, so it saw the eager radix path. + probe_sglang gained --cuda-graph (recorded as probe_cuda_graph); + M3 re-probed with graphs on: decode now runs _topk_index_partial / + _topk_index_merge (topk_split), matching the collector -> ALIGNED + (verdict computed with --serving-raw against that record). All + three gates green (vllm 33, sglang 12, trtllm 9). + HARNESS FOLLOW-UP this opens (RESOLVED 2026-09-24, owner decision: + probes now run the framework's own mode for prefill AND decode — + see probe_execution_mode_is_the_framework_default_2026_09_24): + the identity probes ran eager on every backend. Any op whose + kernel choice reads capture-time static shapes (not just M3 top-k: + FlashMLA/FA split-KV planning, XQA multi-block, trtllm-gen decode + scheduling are candidates) is mis-identified by an eager probe. + The device-stream tables already see graph-replayed kernels, so the + probes can run decode under graphs; the question for the owner is + whether to make graph-mode the default decode evidence (production + truth) and keep eager only for prefill. + ROSTER GATE — RESOLVED 2026-09-24 (owner: "fp8 就用 zai-org/GLM-5.3"): + rename landed in ac274b36 — case YAML model_paths + both alias rows, + packaged model_configs (GLM-5.3 = Hub fp8 config, GLM-5.3-BF16 = the + former bf16 file, -FP8 removed), sdk/common.py list, support-matrix + b60.csv rows, two unit tests (178 pass); workspace roster + (configs/repos.txt, targets.yaml overlay) + GLM-5.3-BF16 dummies + generated; probe_driver --plan now enumerates GLM-5.3 (fp8), + GLM-5.3-BF16 (bfloat16) and GLM-5.3-NVFP4 on all three backends, and a + roster repo without dummy variants is skipped loudly instead of + crashing enumeration. Original: (probe_driver --plan refused: "no + fetched config for zai-org/GLM-5.3-FP8"): that repository does not + exist on Hugging Face + (404 with a token; zai-org lists GLM-5.3, GLM-5.3-BF16, GLM-5.3-Flash, + GLM-5.3-Flash-BF16). zai-org/GLM-5.3 itself is the FP8 artifact + (config quantization_config fp8 e4m3 block 128) and GLM-5.3-BF16 the + bf16 one — 5.3 inverted the 5.x naming. The collector case file + (cases/models/GlmMoeDsaForCausalLM_cases.yaml) aliases GLM-5.3 to the + bf16 row and GLM-5.3-FP8 to the fp8 row, and the SDK's supported-model + list (src/aisimulate_core/sdk/common.py:690-692) names GLM-5.3-FP8 + too — a producer+consumer naming fix (collector YAML + tests + SDK + list), so it is an OWNER DECISION: rename (GLM-5.3 -> fp8 row, + GLM-5.3-BF16 -> bf16 row) or sign an exclusion. configs/ already holds + zai-org_GLM-5.3.json and nvidia_GLM-5.3-NVFP4.json; GLM-5.3-BF16's + config fetches fine (200) once the rename lands. + EXPLAINED (kept out of the gate, same class as vllm kda_gen): trtllm + gdn_gen — FlashInfer decode row matches serving kernel-for-kernel; the + FLA fused_recurrent row is the deliberate fallback lane. + NOT RUN: 5 trtllm records probed on trtllm-probe:1.3.0rc20-onnxfix + (image gone); sglang gdn_ctx via op_smoke hits the known int32-offset + FIXME at the largest sweep points (captured via a direct 4096-token + call instead). + applies_to: + - instrument:path_diff + - instrument:op_smoke + - framework:sglang-0.5.16 + - framework:trtllm-1.3.0rc23 + - platform:sm90 + - meta-llama/Meta-Llama-3.1-8B + - deepseek-ai/DeepSeek-V3.2 + - deepseek-ai/DeepSeek-V3 + - MiniMaxAI/MiniMax-M3 + - Qwen/Qwen3.5-0.8B + unalignable_ops_followup_2026_09_24: + statement: | + Follow-up on the "no serving reference" ops (owner 2026-09-24: "if running + it produces evidence, run it; if load crashes, try to fix it"). + - encoder_attention: FIXED without a multimodal prompt. Engine construction + runs vLLM's own profile_run (full-model dummy forward; for VL models it + pushes max-size dummy images through the vision tower). probe_vllm now + records that run as profile_run_kernels and build_ops counts it as + identity evidence. Qwen3-VL-2B ViT: FlashAttnFwdSm90 x24 layers in + serving; collector encoder_attention (FLASH_ATTN for vit attention, + head_dim 64) -> ALIGNED. + - DeepSeek-V4 on vllm: the load crash is the rendered + --max-num-batched-tokens 6012 (cutedsl compress asserts 64-byte + slot-mapping alignment). The harness golden renderer runs the aic/ + checkout of aiconfigurator, not aisimulate, so the fix branch + (fix/generator-round-vllm-batched-tokens) never reached it. Mirrored + the rounding into aic/ (a71f27955: rule_plugin/vllm.rule, + benchmark/vllm.rule, backend_config_mapping.yaml); the 24 planned DSV4 + goldens re-render at 6016 and are being re-probed. Planning via + probe_driver is currently blocked by the roster gate ("no fetched + config for zai-org/GLM-5.3-FP8 — OWNER DECISION NEEDED"); the goldens + were rendered through render_golden() on the existing plan entries. + The product fix still needs the aisimulate branch merged. + - Kimi-K3: vllm 0.29.0 ships K3 natively (KimiK3ForConditionalGeneration, + vllm/models/kimi_k3/nvidia/kda importable) — the collect_kda docstring + ("kimi-k3 preview image") is stale. K3 0.29 records exist with kernel + tables; their kernels were unlabeled (FlashKDA _flash_kda_fwd_*, + kda_decode_fusion_many_heads, attn_res, layer_norm_gated, situ_and_mul) + — taxonomy rules added under the vocabulary names (flashkda, + vllm_kernel). K3 records are being re-probed cache-cold; the kda op + capture follows. + - bf16-gemm MLA cells: CONSUMED. The MLA module tables carry a gemm_quant + axis (crates/core/src/perfmodel/perf_database/mla.rs: query_context_module / + query_generation_module take GemmQuantMode); any aisimulate run that + models an MLA architecture with gemm_quant_mode=bfloat16 reads them + (readiness.rs even tests DeepseekV4 + bfloat16). They measure vLLM's + own bf16-weight dispatch (fused_a_gemm at num_tokens<=16) — faithful + to what vLLM would run, just with no deployed checkpoint to cross-check. + RESULTS (same day): 24 DSV4 vllm records + 6 K3 records re-probed clean + (6016 budget; first DSV4 serving evidence on vllm: sparse_attn_fwd, + cute-DSL compressor/indexer, mHC TileLang). Verdicts: dsv4_csa/hca + context+generation, dsv4_hca_attn, dsv4_paged_mqa_logits -> ALIGNED vs + sgl-project/DeepSeek-V4-Flash-FP8 fp8; kda_ctx -> ALIGNED after fixing + the collector's FlashKDA prefill call (0.29 takes caller-owned + out/final_state/workspace buffers, kda.py:193-215/536-547; the + preview-era call raised TypeError on every context cell). Two + divergences are EXPLAINED and deliberately kept out of the gate: + - mhc_module: serving runs mhc_pre_big_fuse_WITH_NORM (norm_weight + passed, tilelang.py:90) and mhc_fused_post_pre for middle layers; + the collector measures the norm_weight=None pre + separate post + because the SDK bills attn_norm separately (collect_mhc_module.py: + 118-134, measured 1-2% off the fused path). Row-semantics contract; + switching needs a producer+consumer decision. + - kda_gen: the collector measures BOTH the fused decode + (ops.fused_kda_decode = kda_decode_fusion_many_heads, serving's + pure-decode path, kda.py:686-695) and the Triton fallback lane + (causal_conv1d_update + fused_recurrent_kda_packed_decode, taken by + serving only for spec-decode / mixed batches). The fused row matches + serving kernel-for-kernel; the fallback rows are collector-only by + construction. A verdict per kernel_source lane would express this; + today's subset semantics cannot. + sglang/trtllm evidence: 45 length-conditional records re-probed with the + fixed probes (isl 4096, cache-cold, phase tables). sglang DSA prefill at + isl 32 was dense FA3 (SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD: + use_mha when max_kv_len is short, dsa_backend.py:2999-3012 @0.5.16); + at 4096 it is sparse_attn_fwd — the old sglang prefill evidence was + the dense fallback. The old sglang fp8-KV decode evidence (flashmla_kv) + came from a retired probe injection (--kv-dtype fp8_e4m3): the golden + sglang CLI for fp8 profiles renders NO kv dtype, so current serving + truth is bf16 KV (dsa prefill flashmla_sparse, decode fa3). Added the + kv variant to sglang planning (fp8_e4m3) so the fp8-KV DSA/MLA paths are + probed as a separate, labeled config. 5 trtllm records were probed on + trtllm-probe:1.3.0rc20-onnxfix, an image no longer present — not + re-probed. 8 DSV4 sglang runs exhausted the probe's fixed 16384-token + KV pool at isl 4096 (multi-cache-group) — pool now scales with isl; + rerun in progress. + applies_to: + - instrument:probe_vllm + - instrument:probe_sglang + - instrument:render_golden + - framework:vllm-0.29.0 + - framework:sglang-0.5.16 + - platform:sm90 + - family:dsv4 + - moonshotai/Kimi-K3 + - Qwen/Qwen3-VL-2B-Instruct + probe_execution_mode_is_the_framework_default_2026_09_24: + statement: | + OWNER DECISION 2026-09-24: the identity probes run the framework's OWN + execution mode — torch.compile + piecewise/full CUDA graphs exactly as + the rendered config asks — for prefill AND decode; `--eager` is an A/B + switch only (records carry runtime.probe_eager / probe_cuda_graph; + None = a pre-2026-09-24 eager record). Why they ran eager before: a + kernel-visibility shortcut from before the device-stream tables existed + (graph-replayed kernels have no CPU launch event) plus capture time / + capture failures on dummies. It had become a fidelity liability: any + dispatch that reads capture-time static shapes is mis-identified by an + eager probe (sglang MiniMax-M3 top-k was the proven case), and eager + prefill is not serving's prefill. + A/B (9 records re-probed in framework mode, eager raws kept under + facts/raw_superseded/eager_2026-09-24/): vllm Llama-3.1-8B, + DeepSeek-V3.2 bf16-KV + fp8-KV, MiniMax-M3 bf16-KV + fp8-KV; sglang + DeepSeek-V3.2 bf16, MiniMax-M3 fp8-KV; trtllm Llama-3.1-8B, + DeepSeek-V3.2. Attention / GEMM / MoE / indexer kernel sets are + IDENTICAL to the eager records on every one of them (trtllm: same + 14+16 and 36+32 kernels; vllm DSV3.2 prefill 46/46, M3 38/38; sglang + DSV3.2 prefill 61/61). What changes is what the profiler SEES and the + glue: (a) the Python API spans (AIC::attn::*, AIC::quant_apply::*) + that attributed GEMM/quant/attention kernels to ops vanish under graph + replay and under torch.compile (vllm Llama: both phases) — the kernels + arrive as orphans; (b) vLLM compile replaces its custom elementwise + ops (act_and_mul, rms_norm, rotary_embedding, _typeConvert) with + Inductor triton_{poi,red}_fused_* kernels on Llama (not on DSV3.2/M3, + whose rendered configs keep them); (c) sglang M3 fp8-KV decode under + graphs runs _topk_index_partial/_merge (topk_split = the collector's + path) — the eager fp8-KV record had hidden it and diff_sgl picked that + record over the graph-mode bf16 one, so msa_gen read DIVERGED until + the re-probe. sglang graph mode adds _fused_dsa_decode_metadata_kernel + / _fused_metadata_kernel_general (metadata glue, framework_native). + HARNESS CONSEQUENCES (all landed): probe_driver keeps every + taxonomy-labeled orphan (cublas/vllm_kernel/sgl_kernel/torch were + previously capped as noise — under graphs they are the ONLY place + GEMM and quant kernels appear); identity.attn_backend falls back to + the attention kernel family when no span exists; taxonomy gained the + Inductor and sglang-metadata rules. path_diff: the first recompute + after the no-collector-signal rule flipped FOUR gemm-class verdicts + (gemm_bf16 Llama, gemm_fp8 Qwen3-32B, mla_bmm_gen DSV3, compute_scale) + that had been graded "aligned" BEFORE the rule existed — their only + families (cublas, vllm_kernel) are infra by design, so they can never + have a signal family. Rule refined: no signal family -> grade on + kernel-NAME overlap inside the infra families (infra_family_name_matches + in the report). Three re-align on names (the collector's nvjet / + scaled_fp8_quant instantiations are ones serving launched); + compute_scale did not — it had been paired with fp8-BLOCK DeepSeek-V3.2 + (per_token_group_quant), never a valid instance of scaled_fp8_quant; + re-paired with per-tensor Qwen3-32B-FP8 -> aligned. Gates after all of + it: vllm 33 / sglang 15 / trtllm 9 aligned, zero non-aligned files. + sglang DSA prefill is LENGTH-CONDITIONAL and the collector mirrors it: + at isl 512 (framework mode, fp8-KV and bf16-KV alike — both probed) + serving runs dense FA3 with NO indexer logits or top-k (only + _quantize_k_cache); at isl 4096 the + indexer (sm90_fp8_mqa_logits) + fp8 sparse decode kernel. The + whole-sweep fp8 ctx capture unions both regimes and read DIVERGED + (fa3 collector-only vs the 4096 record; mqa_logits collector-only vs + the 512 record — the prefix>0 cells push kv_len over the threshold). + Single-cell captures (frozen sweep spec replaced; AIC_DSA_CONTEXT_* + env filters do the same): s=4096 vs the framework-mode fp8-KV isl-4096 + record -> ALIGNED (dsa_ctx_fp8_DeepSeek-V3.2); s=512 prefix-0 vs the + isl-512 record -> ALIGNED (dsa_ctx_fp8_s512_DeepSeek-V3.2: FA3 only, + no indexer on either side). sglang gate: 15 aligned. + NOT DONE: the other ~600 records are still eager-probed (identity-safe + per the A/B, but their op attribution differs from framework-mode + records); re-probe them when a verdict needs one, not wholesale. + applies_to: + - instrument:probe_vllm + - instrument:probe_sglang + - instrument:probe_trtllm + - instrument:probe_driver + - instrument:path_diff + - instrument:kernel_taxonomy + - framework:vllm-0.29.0 + - framework:sglang-0.5.16 + - framework:trtllm-1.3.0rc23 + - platform:sm90 + harness_hygiene_2026_09_24: + statement: | + Owner asked for the open-issue list after the framework-mode change and + told me to fix the harness items ("别搞复杂问题"). Landed: + - probe_driver --matrix: the moe column read only API spans; framework- + mode records carry MoE kernels as orphans -> orphans are scanned too. + - First unit tests for the harness: tests/unit/collector/opharness/ + (path_diff.grade extracted as a pure function: subset semantics, + within-family drift, invalid-capture, no-collector-signal, gemm-class + infra-name grading; probe_driver: normalize_kernel, taxonomy contract + incl. "only attention rules emit `triton`", orphan keep rule, span/ + launcher exclusion). 34 tests, pytest.mark.unit. + - Taxonomy backlog: 61 -> 2 unmatched kernel names across all 637 + records (sources verified in the pinned images): identity-bearing + additions dsa_nsa (_fused_q_kernel, FilteredTopK, index_buf_accessor, + DeepGEMM smxx paged-logits helpers), fla_triton (Kimi-K3 KDA chunk / + recurrent / gate kernels, fused post-conv), triton_fused_moe + (_reduce_grouped); the rest is allocator / position / DSV4 SWA and + compress metadata / norm-rope / routing / sampler / load-time repack + glue. Left unmatched on purpose: sglang Kimi-K3 _score_kernel and + _combine_kernel (no source hit; 1 record each). + - path_diff --capture --env K=V sets collector cell filters + (AIC_DSA_CONTEXT_{SEQ_LENS,PREFIX_LENS,BATCH_SIZES}) and records + every AIC_*/AIS_* cell variable in the capture; the verdict carries + capture_env. Whole-sweep captures union length/prefix-conditional + paths and cannot match a single serving record. + - components/captures/: the capture scripts and the four verdict + helpers (verdicts_{all,dsv4,sgl,trt}.sh) that produce every file + under results/pathdiff/, parametrized on AIS_PROBE_WORKSPACE, so a + verdict can be re-derived from the repository alone. + Checked and already fixed (stale in my own list): vllm DSA context + mirrors serving's chunked prefill (context_dsa_chunk_prefill_size + new-token budget + context_dsa_prefix_lengths prefix axis, seq cap + 16384) — no monolithic large-isl construction remains. + Gates after all of it: vllm 33 / sglang 16 / trtllm 9 aligned. + applies_to: + - instrument:probe_driver + - instrument:path_diff + - instrument:kernel_taxonomy + - platform:sm90 + trtllm_kv_dtype_variants_2026_09_24: + statement: | + trtllm was the only backend whose fp8-KV path was never compared. Now: + probe_trtllm --kv-dtype overrides the rendered kv_cache_config.dtype + (recorded as probe_kv_cache_dtype + probe_overrides; the KV manager's + resolved dtype is the ground truth), probe_driver plans a `fp8` variant + per trtllm run (rid tag kvfp8), and the verdict helper pins the bf16 + entries to kv auto so a variant record can never be picked for them. + Four fp8-KV records probed (framework mode): Llama-3.1-8B, DeepSeek-V3, + Qwen3.5-0.8B (KV manager DataType.FP8 on all three), DeepSeek-V3.2 — + which FAILS at load: `RuntimeError: Expected kv.dtype() == + torch::kBFloat16` inside flash_mla_interface (mla.py forward_sparse -> + flash_mla). trtllm 1.3.0rc23 sparse MLA (DSA) requires bf16 KV; there is + no fp8-KV DSV3.2 serving instance on this pin, so the collector's DSA + fp8-KV rows (_get_precision_combos adds ('bfloat16','fp8') for dsa on + SM>=89) measure a path serving cannot run. Kept OUT of the gate (no + serving record); the collector rows are not wrong data, they are + unreachable data — flag to the trtllm collector owner, retest on the + next pin. + Single-cell captures at the bf16 gate cells with fp8 KV + (components/captures/trt_kv/): attention context and generation + (Llama, use_fp8_kv_cache=True) ALIGNED — serving keeps the bf16 + context FMHA and writes fp8 KV; the use_fp8_context_fmha=True cell + (e4m3 FMHA) is NOT Llama's serving path but IS DeepSeek-V3's + (fp8 weights + fp8 KV -> fmha_v2_flash_attention_e4m3 in the record); + MLA context + generation (DeepSeek-V3, kv 'fp8' + gemm fp8_block) + ALIGNED. Lesson repeated: a substring --case-filter picked a head_dim + 64 sliding-window cell first; the captures use --case-prefix with the + full leading tuple. + Also fixed on the way: the archived sglang MiniMax-M3 bf16 record was + the eager run (the graph-mode raw had only been used via + --serving-raw); superseded by the graph-mode raw so msa_gen holds + under kv auto. Gates: vllm 33 / sglang 16 / trtllm 13 aligned. + A/B (GLM-5.2-NVFP4, trtllm rc23, KV forced to bfloat16 to escape the + sparse-FlashMLA bf16-KV assert): llmapi REFUSES it — "Overriding KV + cache quantization with an invalid type ... Accepted types are + ('fp8', 'nvfp4', 'auto')" (facts/trt_kv/glm52nvfp4_kvbf16.json). So an + NVFP4 DSA artifact (hf_quant kv_cache_quant_algo FP8) has NO servable + KV dtype on trtllm rc23 / SM90: auto and fp8 hit the kernel assert, + bf16 is rejected at config validation. Every NVFP4 DSA cell in the + trtllm matrix (GLM-5 / 5.1 / 5.2 / 5.3-NVFP4, DeepSeek-V3.2-NVFP4) fails + for this one reason; upstream item 4 in facts/upstream_reports_2026-09-24.md. + applies_to: + - instrument:probe_trtllm + - instrument:probe_driver + - instrument:path_diff + - framework:trtllm-1.3.0rc23 + - platform:sm90 + - deepseek-ai/DeepSeek-V3.2 + - deepseek-ai/DeepSeek-V3 + - meta-llama/Meta-Llama-3.1-8B + glm53_onboarding_2026_09_24: + statement: | + GLM-5.3 trio probed on all three backends after the rename (18 runs: + rendered + kv variant per backend; owner: "5.3 吧"). Matrix cells + (rendered config, framework mode): + vllm 0.29.0 GLM-5.3 (fp8) PASS FlashMLASparseImpl / Fp8MoE->triton_fused_moe; + GLM-5.3-BF16 PASS FlashMLASparseImpl / UnquantizedFusedMoE->triton_fused_moe; + GLM-5.3-NVFP4 FAIL (kv auto -> fp8_e4m3, no MLA backend; the + explicit fp8 variant PASSES with FlashMLASparseImpl — see the + vllm_fp8kv 0.29 addendum; generator should render + --kv-cache-dtype fp8 for NVFP4 DSA on vllm 0.29). + sglang 0.5.16 all three PASS (DeepseekSparseAttnBackend; NVFP4 MoE -> + ModelOptNvFp4FusedMoE->marlin on SM90); fp8_e4m3 variants + pass too. + trtllm rc23 GLM-5.3 PASS (DeepSeekFP8BlockScalesFusedMoE), GLM-5.3-BF16 + PASS; GLM-5.3-NVFP4 FAIL: the artifact is modelopt + MIXED_PRECISION (87 per-layer entries: FP8 dense mlp + projections + NVFP4 experts — GLM-5.2-NVFP4 was pure NVFP4) + and modeling_deepseekv3.py:1298 asserts "MIXED_PRECISION is + ambiguous" for every decoder layer -> unloadable on this + pin (real artifact property, not a dummy artifact; the + dummy keeps the top-level algo). fp8-KV variants of all + three fail at load like every DSA model on trtllm rc23 + (sparse FlashMLA asserts bf16 KV, see trtllm_kv_dtype_variants). + HARNESS GAP found: gen_dummy_models writes config.json only; the + tokenizer files every dummy dir needs were provisioned once by hand + (2026-09-19) and NO script does it for a new roster repo. GLM-5.3-BF16's + six runs first failed on "Couldn't instantiate the backend tokenizer" + (vllm/sglang at load, trtllm at generate: end_id needs a tokenizer); + fixed by copying tokenizer.json / tokenizer_config.json / + generation_config.json / chat_template.jinja from the GLM-5.3 sibling, + then re-probed. workflow_check already flags bare dirs ("variants + missing tokenizer"); onboard_model step 1 should own the copy. + MATRIX BUILDER DEFECT found and fixed the same hour: build_matrix took + ONE plan file per backend (the newest carrying the pin) and the version + from whichever backend's run came last in that file — the 18-run GLM-5.3 + plan produced an 18-cell matrix named *-1.3.0rc23.yaml for all three + backends. Now every plan file carrying (backend, pin) contributes, only + rendered (non-kv-variant) runs define cells, a passed cell is never + overwritten by a later crash/no-raw run, counts are recomputed, and the + cell's identity/kv_allocated comes from the rendered record (variant + records exist for the same repo x backend). Matrix: 98 models per + backend (95 + the trio); vllm 84 pass / 9 pass+custom / 5 fail (DSV4 + cells flipped fail -> pass+custom from the 6016-budget re-render), + sglang 86 / 6 / 6, trtllm 62 / 0 / 36. + applies_to: + - zai-org/GLM-5.3 + - zai-org/GLM-5.3-BF16 + - nvidia/GLM-5.3-NVFP4 + - instrument:probe_driver + - instrument:dummies + - framework:vllm-0.29.0 + - framework:sglang-0.5.16 + - framework:trtllm-1.3.0rc23 + - platform:sm90 + full_reprobe_framework_mode_2026_09_24: + statement: | + FULL RE-PROBE in framework mode (owner "现在跑吧"): every roster model on + the three pins, rendered config + kv-dtype variant — sglang 98+98, + trtllm 98+98, vllm 98+98 (4 Qwen3.8 runs skipped: the golden CLI + rejects the architecture). Old evidence retired under + facts/raw_superseded/ (187 eager vllm raws that already sat under the + current ids — the queue had skipped them as "done"; 291 pre-kv-tag raws + at the pins; 219 other-version records kept). records.jsonl now 713. + OUTCOMES (rendered / fp8-KV variant): sglang 93+91 ok — fails MiniMax- + M3-MXFP8 (mxfp8 unsupported), Kimi-K3 (+NVFP4, model module missing on + 0.5.16), Gemma-4-26B-A4B-NVFP4 (only gated SiLU), Llama-4-Scout (new + under graphs: sglang internal crash "Open an issue"), MiniMax-M2/M2.5 + fp8-KV only (bf16 output dtype assert); trtllm 62+51 ok — the known + SM90 wall (DSV4 pre-Blackwell assert, NVFP4 DSA fp8-KV, MiMo, M3 sparse + needs the kv cache type) plus every DSA fp8-KV variant; vllm 93+96 ok. + GATES on the new records: vllm 32 / sglang 16 / trtllm 13 aligned, + after three evidence corrections that framework mode forced: + - gemm_fp8 was paired with Qwen/Qwen3-32B-FP8-Static-PerTensor, which + vLLM 0.29 loads UNQUANTIZED (UnquantizedLinearMethod on every + linear; its config.json carries no quant_method and the modelopt + hf_quant_config.json in the dir is ignored) — never a valid fp8 + instance; the old "aligned" came from the fp8-KV variant record's + quant kernel. Re-paired with nvidia/Llama-3.1-70B-Instruct-FP8 + (ModelOptFp8LinearMethod, cutlass_3x_gemm_sm90_fp8) -> aligned. + - compute_scale (standalone scaled_fp8_quant kernels) has no serving + instance under torch.compile: the norm_quant / act_quant fusion + passes fold activation quant into Inductor kernels + (triton_*_fused__to_copy_clamp_cutlass_scaled_mm_*). Removed from + the gate; report kept next to the facts. The collector row is + eager-only truth — flag to the SDK owner (the op-level model bills + it separately). + - vllm MLA fp8 (DeepSeek-R1): under CUDA graphs the DeepGEMM + projection kernels sit under quant spans the MLA hint excluded -> + false collector-only deepgemm drift; hint widened (gemm|deepgemm|quant) + like the sglang DSA hint. + MATRIX (results/sm90, 100 models per backend): vllm 85 pass / 9 + pass+custom / 6 fail; sglang 87 / 6 / 7; trtllm 62 / 0 / 38. Flips vs + the eager matrix: GLM-5.1 / 5.2 / 5.3-NVFP4 vllm fail -> pass (the + generator now renders --kv-cache-dtype fp8; GLM-5-NVFP4 and + DeepSeek-V3.2-NVFP4 stay failed: bundled configs without + quantization_config, SDK data gap); Kimi-K3 (+NVFP4) vllm pass -> FAIL: + illegal memory access in kimi_k3 embed_input_ids during + compile_or_warm_up_model (CUDA-graph / compile warm-up) — --eager A/B + loads and serves (facts/full_reprobe/k3_eager.json), so it is a vLLM + 0.29 K3 compile/graph defect on SM90; kda_ctx keeps its verdict against + the eager A/B record (raw_file recorded). sglang MiniMax-M2 / M2.5 + fail -> pass under graphs; Llama-4-Scout pass -> fail (sglang crash). + Framework-mode probes: probe_eager False on all 588 new records. + applies_to: + - instrument:probe_driver + - instrument:path_diff + - framework:vllm-0.29.0 + - framework:sglang-0.5.16 + - framework:trtllm-1.3.0rc23 + - platform:sm90 + - moonshotai/Kimi-K3 + - Qwen/Qwen3-32B-FP8-Static-PerTensor + - nvidia/Llama-3.1-70B-Instruct-FP8 + followups_after_full_reprobe_2026_09_24: + statement: | + Owner directives after the full re-probe report: + (1) "统一所有的都对齐到64整数倍，别case by case了": the 64-alignment of + token budgets is now ONE rule in backend_config_mapping.yaml applied to + every backend (vllm max-num-batched-tokens, trtllm max_num_tokens, + sglang max-prefill-tokens and chunked-prefill-size); the vllm rule + formulas are plain again (commit 2446f6cc, mirrored into the aic + checkout the golden CLI renders with). Verified: sglang 5500 -> 5504, + trtllm 4500 -> 4544, vllm 6012 -> 6016. + (2) "关于hf config的quant，能不能看看其他字段": the KV fact is not in + config.json for the modelopt NVFP4 artifacts at all — the SDK loader + attaches the bundled hf_quant_config.json (kv_cache_quant_algo: FP8) + to the loaded config, so vllm_dsa_kv_cache_dtype now reads that merged + view. All five NVFP4 DSA artifacts (GLM-5 / 5.1 / 5.2 / 5.3, DeepSeek- + V3.2) render --kv-cache-dtype fp8; GLM-5-NVFP4 and DeepSeek-V3.2-NVFP4 + re-probed with the new render (see the matrix). + (3) "关于kimi k3，我感觉可能还有其他构造问题": the construction was + audited twice. Kimi-K3 names its layer kinds by 1-BASED index lists + (configuration_kimi_k3.is_kda_layer: `layer_idx + 1 in kda_layers`). + The depth cut kept the lists verbatim; by luck the 2-layer "rep" dummy + (original layers 1,2 -> 1-based 2,3, both KDA) still resolved to two + KDA layers, so the records' KDA evidence was real. My first remap + assumed 0-based lists and produced an all_kinds dummy whose two layers + both resolved to FULL attention — that variant "passed" the + framework-mode probe while running no KDA kernel at all (false pass, + caught by the record's kernel families). The generator now detects + the base from the config data (commits 85d1dd32 + 51158047): + all_kinds = one KDA (1) + one full-attention layer (2), no stale refs. + The crash itself: eleven A/Bs on the rendered config (max-num-seqs + 512) — every variant that runs a KDA layer fails (rep, 3-layer with + the dense layer, kda+kda, full+kda); CUDA graphs off passes; compile + off (graphs on) fails; piecewise-only graphs fail; text-only + (limit-mm image 0) fails; max-num-seqs 64 and 256 PASS. CUDA_LAUNCH_ + BLOCKING places the illegal memory access after graph capture, in + warmup_kernels -> _run_decode_step -> run_fullgraph — the decode + warm-up step at the generator's max_num_seqs=512 with CUDA graphs and + a KDA layer present. So: a real dummy defect existed and is fixed, and + independently vLLM 0.29's K3 (KDA) decode warm-up faults at batch 512 + on SM90 (256 passes) — the generator's 512 is what real deployments + get. Upstream draft 7 updated. The corrected all_kinds dummy (one KDA + + one full-attention layer) re-probed: the same warm-up crash — + attribution stands; kda_ctx keeps the eager A/B evidence (verdict + helper passes it explicitly). Side fact: the K3 fp8-KV variant is + rejected by vLLM at load — "Kimi-K3 fp8 KV cache requires an fp8 + prefill query; enable --attention-config use_prefill_q..." — an + extra flag the generator would have to render if fp8 KV were ever + chosen for K3 (it is not today). + (4) Dummy dirs: no step provisioned the tokenizer / custom-code files + (Kimi needs tiktoken.model + tokenization_kimi.py + modeling_*.py). + dummies.py now copies every auxiliary artifact file from + configs/aux_files/_/ (exported once from the existing dirs, + 100 repos), else an earlier variant of the repo, else the declared + targets.yaml dummy_overrides.tokenizer_from sibling; otherwise a loud + MISSING caveat. A fresh regeneration of all 196 variants from configs/ + is complete without hand copies (commit a2f34f3c). + (5) NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 on trtllm rc23 ("no raw"): + the engine aborts right after model init — "CUDA runtime error in + ::cudaStreamDestroy(stream): an illegal memory access was encountered" + (tensorrt_llm TllmException, SIGABRT, so the probe cannot dump). + Same SM90 NVFP4 wall class as the other NVFP4 trtllm cells; log + facts/full_reprobe/nemotron_ultra_nvfp4_trt.full.log. + (6) Golden renderer switched from the aic checkout to THIS repository's + generator (venv_ais, `aiconfigurator cli generate` console script, + stamp = this repo's HEAD): vllm/trtllm renders byte-identical to the + aic ones for Llama-3.1-8B, sglang flags identical; Qwen3.8-2.4T-A95B + now renders (was "generator rejects" on aic). No more aic mirroring. + applies_to: + - instrument:dummies + - instrument:probe_vllm + - moonshotai/Kimi-K3 + - nvidia/Kimi-K3-NVFP4 + - framework:vllm-0.29.0 + - platform:sm90 + onboarding_recent_releases_2026_09_25: + statement: | + Owner request (2026-09-25): "再试试 qwen3.8 和 glm5.3-flash … 把这些模型也 + 弄进我们的列表 … inkling 啥的，最近这 1-2 月的模型 … probe 一把，增加对应的 + config". 33 Hub repos were surveyed and fetched with the new + components/fetch_inputs.py (config, hf_quant, every non-weight file into + configs/aux_files/); 31 have a transformers config, 2 do not + (mistralai/Leanstral-1.5-119B-A6B, MiniMaxAI/MiniMax-H3 — no config.json + on the Hub; recorded in targets.yaml roster.excluded, decided_by + pending-owner). The 30 not yet in the roster were added to + configs/repos.txt + roster.extra_repos; dummies for all 30 generated + without hand edits (tokenizer/custom code provisioned from aux_files). + Three gates decide whether a checkpoint yields an identity record, and + each of them rejected part of the list: + (1) THE GENERATOR (this repo's SDK) parses the architecture — rejected + 9 architectures / 19 repos, now recorded as "generator rejects" matrix + cells: Glm5NextForConditionalGeneration (GLM-5.3-Flash, -BF16, nvidia + -NVFP4), Qwen4ExpForConditionalGeneration (Qwen3.8-Flash-Next, -FP8, + -NVFP4), InklingForConditionalGeneration (Inkling, Inkling-Small and the + two baerquants NVFP4 artifacts), MiMoV2ForCausalLM (MiMo-V2.6-Pro-RL, + -Flash-RL; note the SDK knows MiMoV2FlashForCausalLM only), + Xing4_0ForCausalLM (x2), AliceAIForCausalLM, K2HorizonForCausalLM, + Spark2_5ForCausalLM. A tenth, Qwen3_5ForCausalLM (text-only dense + Qwen3.5: Altworld/Hemmingway-1, TokenRhythm/NeoHorse-1-9B), was the + flat-config twin of an already-supported class and is now mapped + (commit 8d8e154d) — both rendered and were probed in pass 2; + Hemmingway-1 was then DROPPED by owner decision (CC-BY-NC-4.0, see + below), its inputs/raws live in facts/dropped/. + (2) THE PINNED FRAMEWORKS know the architecture — checked in the three + images: vllm 0.29.0 lacks Glm5Next and DeepseekV41; sglang 0.5.16 lacks + Glm5Next, Qwen4Exp, Qwen3_5MoeForCausalLM, DeepseekV41, MuseGlimmer; + trtllm 1.3.0rc23 lacks Glm5Next, Qwen4Exp, DeepseekV41, MuseGlimmer, + Inkling, MiMoV2. Consequence for the owner's two named targets: + GLM-5.3-Flash is unservable on EVERY pinned framework AND unparsed by + the SDK; Qwen3.8-Flash-Next (Qwen4Exp) loads only under vllm 0.29. + Both need a framework pin bump before any identity can be recorded. + DeepSeek-V4.1-Flash (SDK-supported) fails on all three pins at the + transformers config stage ("model type deepseek_v41 … Transformers does + not recognize"); Muse-Glimmer-30B (+NVFP4) the same on sglang/trtllm + (`muse_glimmer`) but passes on vllm. + (3) THE PROBE GPU holds the dummy cut — Qwen3.8-2.4T-A95B: the driver + only ever queued dummy variant index 0 (depth8 = 8 x 26B params), a + permanent OOM. New driver rule (commit 4447cf41): a raw whose LOAD + stage failed with CUDA OOM / trtllm "insufficient GPU memory" advances + the representative to the next smaller faithful cut, recorded as + capacity_fallback_from on the run — observed, never predicted; the + smallest cut stays queued when everything OOMs. Outcome: bf16 and FP8 + 2.4T OOM even at depth4 (one full period, 4 x 26B params ~ 194 / 104 + GiB of weights on a 139.8 GiB GPU) on vllm and trtllm — no faithful + single-GPU cut exists, the cells stay honest OOM fails; sglang rejects + Qwen3_5MoeForCausalLM outright. NVFP4 2.4T: depth8 OOMs, depth4 result + in pass 2 (below); trtllm rc23 additionally fails NVFP4 MoE on SM90 + ("No supported MoE GEMM tactic remains after replacing unsupported + NO_SMEM epilogues") — the same NVFP4-on-Hopper wall class as the other + trtllm NVFP4 cells. + Identity records landed (framework mode, tp1, rendered + fp8-KV + variants) for: Qwen3.8-27B / -FP8 / nvidia -NVFP4 (all three backends; + vllm ChunkGatedDeltaRule + fla_triton/causal_conv1d + fa3, sglang + HybridLinearAttnBackend, trtllm fmha), NVIDIA-Nemotron-3.5-Lightning- + 30B-A3B-BF16 (all three), Ornith-1.5-35B-A3B (Qwen3_5MoeForConditional + Generation; all three), openbmb/MiniCPM5-2B (Llama; all three), + Muse-Glimmer-30B / -NVFP4 (vllm), plus the pass-2 additions. Taxonomy: + every vllm kernel of the new models is already labeled; the only + unlabeled sglang orphans were the per-variant metadata builder + (_fused_metadata_kernel_ps1_no_swa) and raw memcpy128/memset32 runtime + copies — both glue, rules added (commit 995c7430). No new op family + appeared: GDN/KDA/mamba/MoE/attention kernel families of the onboarded + models are all in the existing taxonomy. + Configs bundled into the SDK (byte-for-byte at pinned revisions, + THIRD_PARTY_NOTICES updated; commits 97bce88f, 8d8e154d, 895586de): + Qwen3.8-27B / -FP8, Qwen3.8-2.4T-A95B-NVFP4, DeepSeek-V4.1-Flash-NVFP4, + Nemotron-3.5-Lightning-BF16, Ornith-1.5, MiniCPM5-2B, NeoHorse-1-9B. + NOT bundled: every generator-rejected architecture (dead weight until + modeled), and the two modelopt mixed_precision NVFP4 artifacts + (Qwen3.8-27B-NVFP4, Muse-Glimmer-30B-NVFP4) — the SDK quant inference + maps mixed_precision to fp8_static for dense models and the support + matrix then reports FAIL instead of HW_INCOMPATIBLE on non-FP4 + systems (TODO.md); they stay in the probe roster and render from the + Hub. The other eight classify like their bundled siblings in a + tools/support_matrix dry run on h200_sxm. + Side answers: Kimi-K3 max-num-seqs bisection completed — 64, 256, 384, + 448, 480, 496 all pass framework-mode; only the generator's 512 fails, + i.e. the fault is the 512-batch CUDA-graph decode warm-up itself, not a + threshold "above 256" (facts/k3_ab/seqs*.json). The compute_scale + standalone-kernel deviation is now a governed standards exception + (collector/evidence_exceptions.yaml, expires 2026-12-31). + Pass 2 (after the Qwen3_5ForCausalLM alias and the capacity rule): + NeoHorse-1-9B (and Hemmingway-1 before its removal) pass on vllm and + trtllm, sglang 0.5.16 has no Qwen3_5ForCausalLM implementation + (framework gap); + Qwen3.8-2.4T-A95B-NVFP4 passes on vllm at depth4 (capacity_fallback_from + depth8; ModelOptNvFp4FusedMoE -> marlin W4A16 on SM90, as for the other + NVFP4 MoE checkpoints), sglang framework gap, trtllm NVFP4 MoE wall. + Matrix after onboarding (Hemmingway-1 removed): 129 rows per backend + — vllm 97 pass / 9 pass+custom / 23 fail, sglang 93 / 6 / 30, trtllm + 69 / 0 / 60; of the 29 new rows: vllm 10 pass, sglang 6 pass, trtllm + 7 pass, 19 rows + "generator rejects" on every backend, the rest framework gaps / walls + named above. Note (display, pre-existing): NemotronH rows on vllm show + attention "ReLUSquaredActivation" and the text-only Qwen3.5 rows show + the fa3 kernel family instead of ChunkGatedDeltaRule — the identity + fallback reads the first wrapped/attention span; kernel families in the + records are correct. + Evidence: facts/onboard_2026-09-25_repos.txt (33 surveyed repos), + facts/onboard_render/arch_support*.txt (framework registry checks), + facts/onboard_render/{plan.log,plan_onboard_pass1.json,queues*/}. + OWNER DECISIONS (tianhaox, 2026-09-25): (a) Leanstral-1.5 / MiniMax-H3 + dropped (params.json-only; roster scope = language models and models + with a vision module, no diffusion/DiT — none of the 30 was DiT); + (b) models with license problems are dropped: Hemmingway-1 + (CC-BY-NC-4.0) removed from roster, configs, dummies, plan and matrix + (raws in facts/dropped/); every other new repo is Apache-2.0 / MIT / + NVIDIA Open Model / OpenMDW / Qwen-community (Qwen3.8-Flash-Next only, + not bundled anyway); (c) framework gaps and walls stay as recorded — + no pin bumps now; (d) SDK modeling of the 9 rejected architectures + stays open. Structural notes for (d), from the Hub configs: Glm5Next + = GLM-5.3 DSA (kv_lora 512, indexer topk 2048, mHC fields) with 34/45 + layers swapped to KDA linear attention (linear_attn_config as in + Kimi-K3, 0-based lists) + 288-expert MoE + MTP — closest existing + families DEEPSEEKV32/GLM-5.3 + KIMIK3; Qwen4Exp = Qwen3.5 GDN hybrid + (interval 4, 512 experts x 640, MTP) plus new n-gram embedding + (ngram_*, ple_*) and indexer_* fields — extends QWEN35; AliceAI = + Qwen3.5-style GDN/KDA hybrid MoE (512 x 512, interval-4 layer_types) + with block attention residual — QWEN35-like; Inkling = 66-layer + GQA MoE (256 experts top-6, 2 shared) with SWA layers (local_layer_ids, + swa_* heads), short conv, vision hmlp encoder + audio — new; MiMoV2 = + MiMo-V2-Flash lineage (hybrid SWA 128 + full attention, 384-expert + MoE, head_dim 192/v 128, vision + audio configs) — extends HYBRIDMOE; + Xing4_0 = DeepSeek-V3-style MLA (q_lora 768, kv_lora 512, nope 128 / + rope 64) + 64-expert MoE + mHC + MTP — DEEPSEEK-like; K2Horizon = + GQA MoE (100 experts top-8 + MoVA expert set) — MOE-like; Spark2_5 = + dense 36-layer with 27 SWA(512)/9 full layers, per-kind rope — + Gemma-like SWA hybrid. Framework side: vllm 0.29 loads Qwen4Exp, + Inkling, MiMoV2 natively; none of the pins knows Glm5Next, Xing4_0, + AliceAI, K2Horizon or Spark2_5 (custom auto_map code only). + FOLLOW-UP DECISION (tianhaox, 2026-09-25, later the same day): AliceAI, + Xing4.0 (both artifacts), Spark-X2.5 and K2-Horizon are dropped from + the roster altogether (custom-code-only architectures no pinned + framework or the SDK knows); inputs, dummies and plan rows moved to + facts/dropped/. Remaining unmodeled architectures in the roster: 4 + (Glm5Next, Qwen4Exp, Inkling, MiMoV2 — 12 repos). Matrix after the + drops: 124 rows per backend — vllm 97 / 9 / 18, sglang 93 / 6 / 25, + trtllm 69 / 0 / 55. + DECISIONS CLOSED 2026-09-25 (tianhaox): the NVFP4 DSA artifacts on + trtllm rc23 / SM90 stay modeled as bf16-KV (rc23's sparse FlashMLA + asserts bf16 KV; the perf task keeps that dtype) — continue as is. + compute_scale rows are an approximation by design (governed exception + stands, no further follow-up). SDK modeling of the four remaining + unmodeled architectures is out of this work — see TODO.md. + DECOMPOSE (step 5, component landed 2026-09-25): every passing record + on the three pins decomposes with EMPTY residue after six glue rules + (torch xmma f32/tf32 GEMM, fa3 split-kv combine, vision-encoder + attention/conv glue, trtllm MoE tactic-profiler buffers, trtllm GDN + layout transposes). One identity fact surfaced: sglang 0.5.16 serves + MiniMax-M2 / M2.5 (block-fp8 checkpoints) through the Triton + _w8a8_block_fp8_matmul GEMM, while the collector's fp8_block gemm lane + is DeepGEMM (kernel_source_backends gemm_dtype fp8_block) — labeled + triton_block_fp8 so a gemm path_diff on those repos reads diverged + instead of hiding under the DeepGEMM label. No capture exists yet for + that cell; recorded, not graded. + KDA DECODE-BATCH CAP (owner decision 2026-09-25, "带kda的模型覆盖max num + seq到256"): vllm.rule caps max_batch_size at 256 for checkpoints with + linear_attn_config.kda_layers (generator commit a0820789; Kimi-K3 now + renders --max-num-seqs 256 --max-num-batched-tokens 5760). Re-probed + under the capped render in framework mode: moonshotai/Kimi-K3 and + nvidia/Kimi-K3-NVFP4 PASS on vllm 0.29.0 (prefill 63/70, decode 51/55 + kernels; identity KimiK3DeltaAttention + kda_decode_fusion_many_heads + + flash_kda prefill, MoE Mxfp4MoE->marlin / ModelOptNvFp4FusedMoE-> + marlin); the fp8-KV variants still fail at load with the known + "fp8 KV cache requires an fp8 prefill query" requirement (not + rendered). kda_ctx path_diff now grades against the framework-mode + record (default selection, --serving-raw dropped): ALIGNED. The old + 512 raws are kept in facts/k3_ab/pre_cap/. vllm matrix: 99 / 9 / 16. + NORMALIZER FACT (2026-09-25): "(anonymous namespace)::" was never + stripped, so every kernel defined in an anonymous namespace — ALL of + sgl_kernel's C++ kernels (store_kvcache, concat_mla_*, fused_*norm_rope*, + flash_c4/c128 DSV4 attention, topk_* indexer helpers, per-tensor fp8 + quant, act/silu) plus torch's at::native helpers and K3's KDA decode + kernel — collapsed into one bogus name "namespace" and was invisible + to the taxonomy, the identity column and the gates. Stripping it + exposed 26 sglang kernels, now labeled by role (flash_c4_*/flash_c128_* + = dsa_nsa attention; the DSA indexer top-k helpers dispatch by batch + — topk_small_batch_kernel in the probe's decode vs topk_main_kernel in + the collector's case — so they carry the sgl_kernel infra label and + are not name-graded). Every saved gate verdict stays aligned (61), + residue is empty on all pins, no pass row lost its attention identity. + applies_to: [zai-org/GLM-5.3-Flash, Qwen/Qwen3.8-27B, Qwen/Qwen3.8-Flash-Next, nvidia/Qwen3.8-2.4T-A95B-NVFP4, + deepseek-ai/DeepSeek-V4.1-Flash, meta-models/Muse-Glimmer-30B, baerquant/Inkling, Altworld/Hemmingway-1, + TokenRhythm/NeoHorse-1-9B, ornith-ai/Ornith-1.5-35B-A3B, openbmb/MiniCPM5-2B, + 'instrument:fetch_inputs', 'instrument:probe_driver', 'instrument:dummies', 'framework:vllm-0.29.0', + 'framework:sglang-0.5.16', 'framework:trtllm-1.3.0rc23', 'platform:sm90'] + codex_review_evidence_contracts_2026_09_25: + statement: | + Codex review of 17ee75c2 (workspace file "Review Comment Codex.md") + reproduced six ways the harness turned stale, missing or insufficient + evidence into "pass"; all six reproduced on HEAD 9d860284 and are + closed by contract changes, each with a regression test: + (1) TARGET-OP GRADING. path_diff graded a gate with no signal family + on ANY infra-kernel overlap, so a capture sharing only a quant helper + with serving "proved" a GEMM path (the committed fp8 GEMM verdict had + exactly that: only scaled_fp8_quant in infra_family_name_matches; + the real reason was the launcher filter dropping + vllm::cutlass_3x_gemm_sm90_fp8 — a kernel, not a launcher). grade() + now takes the gate's TARGET ROLES (taxonomy role axis; inferred from + the gate name, or --target-roles) and requires every collector kernel + of each target role to name-match a serving kernel of the same role; + a role the collector ran but serving did not, or a collector-only + backend family inside a target role, is diverged; auxiliary roles are + reported, never graded; explicit role-scoped equivalences only (GEMM + tile blobs with layout kept; sgl_kernel top-k small_batch/main batch + dispatch). + (2) PHASE. prefill/decode/profile-run orphans were one set, and span + attribution by NAME across phases hid a kernel from the phase whose + span did not cover it. Records carry kernel_phases (every device-table + kernel by phase) and orphan_phases; select_serving picks serving + evidence by the gate's phase; vllm api_trace ops carry phase None (was + a borrowed "decode"); trtllm's flat table stays unscoped and the + verdict says so (phase_scoped). Re-derivation under (1)+(2): 60 gates + aligned, 1 diverged — trtllm rc23 DeepSeek-V3 MoE routing: the + collector routes with customMoeRoutingKernel, serving with + deepseek_v3_topk_kernel + dsv3MinLatencyKernels::router_gemm_kernel + (a collector serving-parity finding, left red); the sglang DSV3.2 DSA + context gate needed a single-cell capture (the whole-sweep capture + unions short cells that run dense FA3 — opcov_sgl_dsa_ctx_s4096); + compute_scale moved to the explained set (governed approximation); + the two trtllm fp8-KV DSA gate lines are retired (rc23 cannot serve + fp8 KV for DSA — a recorded wall, not an alignment question). + (3) EXECUTION IDENTITY. The run id names the case only; a raw probed + under an older render counted as done. Every plan run now carries an + execution fingerprint (engine tokens from the render — first flag on, + deployment-wrapper values dropped —, dummy config, probe code, image, + kv); the queue writes it as a .fp sidecar next to the raw and skips a + run only when the sidecar matches; records carry evidence_status + current | stale | unverified (sidecar decisive; without one, vllm / + sglang raws are checked on the argv they recorded, trtllm on its + engine yaml semantically); the matrix is derived from the NEWEST plan + per cell, reads the record of ITS run id (never a sibling version's), + and shows stale evidence as a fail cell. First run over the archive: + sglang 216 current / 12 unverified (retired pins), trtllm 222 / 37, + vllm 121 current / 103 stale (mostly fp8-KV variant raws from before + the 64-alignment) / 98 unverified; exactly ONE current-pin matrix + cell rested on stale evidence — MiniMax-M3 on vllm, probed at + max-num-batched-tokens 6012 before the alignment rule, re-probed at + 6016 (the review's own counterexample, found by the mechanism). + (4) PLAN-RELATIVE PROGRESS. matrix_complete now requires every repo + the plan declares for (fw, version) to have a verdict (an empty + matrix is not complete; stale cells are not complete); + path_verdicts_aligned requires every gate the verdict scripts declare + to have a verdict naming this framework/version and reading aligned + (declared_gates parses components/captures/verdicts_*.sh); + component_pending never completes a step (implemented -> todo "no + completion predicate", unimplemented -> blocked); new_op_collector + steps got artifact predicates (family_observed / family_unit_defined / + family_collector_exists / family_gates_aligned) and onboard_model's + path_aligned became model_gates_aligned. A nonexistent op family now + shows four todo steps, not four done ones. + (5) ONE CODE SOURCE. Queues invoke the probes from this checkout + (components/probes, visible under the workspace mount or mounted at + /harness); build_images.sh builds venv_ais from the checkout; the + workspace holds data only (configs, dummies, archive with raw + .fp, + facts, jitcache). The workspace's twin probe scripts are retired. + (6) THIRD_PARTY_NOTICES root copy synced (check_packaged_legal_files + passes). + FOLLOW-THROUGH (same day): the 102 vllm fp8-KV variant raws that read + stale (probed before the 64-alignment) were re-probed through the new + queue lines — every record on the three pinned versions is now + 'current' (remaining 'unverified' = retired pins only); the six sglang + 0.5.16 customizations (DSV4 moe_backend=marlin) were A/B'd bare in a + separate workspace and all still_needed (results/retests/sm90/ + sglang-0.5.16.yaml); the clean-checkout acceptance ran end to end in a + fresh workspace (build_images.sh -> venv_ais -> fetch_inputs -> + dummies -> emit with the /harness read-only mount -> probe -> record + 'current' -> matrix pass for Qwen3-0.6B). Harness unit tests: 73. + EVIDENCE LAYERING (owner decision 2026-09-26, "facts 太多进 repo 不是好 + practice"): the PR had 33k lines of kernel-level decompositions and + 5k lines of verdict kernel lists committed as results. Consumers were + checked: the workflow predicates read role/backend SETS, residue and + verdict/framework/version only; kernel names are debugging evidence. + decompose.py now commits a summary (role -> backend -> count, residue, + ops, record id) and writes kernel names to /archive/evidence/; + path_diff commits a verdict summary (both inputs' identity: record id, + exec fingerprint, capture sha256; roles, phase, per-role counts; the + deciding names only when red) and writes the full report to the + evidence dir; evidence_bundle.py packs a campaign's raw + .fp, + records, plans, rendered engine configs, captures and full reports + into one content-addressed tar.gz (first bundle 2026-09-26_sm90: + 856 raws, 6.2 MB) indexed in results/evidence_index.yaml with sha256 + and location. Committed results shrink from ~44k to ~9k lines; LFS + was rejected for the evidence (it hides content without making it + reviewable); the bundle's storage location is the owner's call and + is recorded per campaign. + applies_to: ['instrument:path_diff', 'instrument:probe_driver', 'instrument:workflow_check', + 'instrument:build_images', 'framework:vllm-0.29.0', 'framework:sglang-0.5.16', + 'framework:trtllm-1.3.0rc23', 'platform:sm90', MiniMaxAI/MiniMax-M3, deepseek-ai/DeepSeek-V3, + deepseek-ai/DeepSeek-V3.2] + vllm_0300_upgrade_2026_09_27: + statement: | + Owner decision 2026-09-27: move the vllm pin from 0.29.0 straight to + 0.30.0 (vllm/vllm-openai:v0.30.0, the newest image; 0.29.x patches do + not exist), collector included — the 0.29 pin upgrade is abandoned and + its records left the PR (kept in the workspace, facts/local_only_results). + The upgrade_op workflow ran end to end as the stability test of the + harness; every step's evidence: + PROBES (224 vllm runs, framework mode, rendered + fp8-KV variants, + ~1.1 min median per run on the dummies, 50 min on six GPUs): 124-cell + final matrix 110 pass / 1 pass+custom / 13 fail (12 generator rejects, + 1 capacity: Qwen3.8-2.4T-A95B bf16 — no faithful cut fits one 139.8 GiB + GPU; its FP8 and NVFP4 artifacts pass at depth4). Two first-pass OOMs + on the fp8/NVFP4 2.4T cuts and on two fp8-KV variants were transient + GPU sharing with the concurrent capture wave and cleared on re-probe — + probes and captures must not share a GPU. Verdict changes vs 0.29: + DeepSeek-V4.1-Flash and its NVFP4 artifact now PASS (0.30 registers + DeepseekV41); Qwen3.8-2.4T-A95B-FP8 capacity -> pass (depth4 fits). IDENTITY: across the 94 repos + probed on both versions the identity-role (backend, role) sets are + IDENTICAL — no attention / gemm / moe / linear-attention / dsa / quant / + mhc kernel family changed between 0.29.0 and 0.30.0; 47 repos differ + only in sampler / scheduler glue kernels (0.30 drops + _convert_req_index_to_global_index / _compute_slot_mappings / + _get_num_sampled_and_rejected, adds cute-DSL cu_blocks conversion and a + symbol literally named "kernel"). New kernels labeled: DSV4 + _token_request_mapping_kernel (infra), Nemotron fused + _relu_squared_static_fp8_quant_kernel (0.30 fuses the activation into + the quant), DSV4.1 _hc_head_reduce_store_kernel (mhc). RESIDUE left on + purpose for the owner's granularity call: DSV4.1's engram module + (_engram_lookup_kernel, _hash_ids_kernel, _fused_engram_post_wkv_kernel) + and a DSV4 cute-DSL kernel_cutlass_device_kernel_tensorptrf32... whose + role is unattributed. The Qwen3.5 rows' attention column reads FA3 + + the GDN decode kernel instead of ChunkGatedDeltaRule + the same kernel: + 0.30 emits no wrapped attention span there, kernels unchanged. + COLLECTOR ON 0.30 (the harness's reason to exist): 21 captures re-taken + in the 0.30 image with versioned names (opcov_@0.30.0.json). The + first wave found the collector broken in two places — exactly the + metadata-parity class layer_permissions.md warns about: (a) + CommonAttentionMetadata lost the private _seq_lens_cpu / + _num_computed_tokens_cpu fields, the seq_lens_cpu property and the + num_computed_tokens_cpu property (10 MLA/DSV4 captures invalid); (b) + is_flashkda_supported() gained recurrent_state_dtype (kda_ctx invalid). + Fixed in collector/vllm/utils.py, collect_kda.py, collect_msa_module.py + with citations to the 0.30 population sites (v1/worker/gpu/attn_utils.py + :456-475, CommonAttentionMetadata.compute_num_computed_tokens, + models/kimi_k3/nvidia/kda.py:455-495 + :665, MambaStateDtypeCalculator. + kda_state_dtype -> float32 for the default mamba_ssm_cache_dtype); + framework_manifest vllm default -> 0.30.0 (digest recorded); the + __compat__ upper bound of all 11 vllm collector modules -> 0.30.0; + manifest tests updated. Re-captured: 19/19 declared gates ALIGNED + (compute_scale stays explained: the quant is still fused on 0.30, the + governed exception now scopes 0.30.0). Citation audit debt: only the + three broken sites were re-verified against 0.30 source; the other + @0.29.0 citations in the collector still hold by behaviour (their + captures align) but were not re-read. + CUSTOMIZATIONS (bare-default A/B at 0.30.0, results/retests/sm90/ + vllm-0.30.0.yaml): Nemotron-H-56B gpu-memory-utilization 0.6 still + needed (bare OOMs in the mamba profile run as on 0.29); the DSV4 + moe_backend=marlin prescription for the four fp4-experts checkpoints + is now DROPPABLE — 0.30's runner auto selects the same + marlin_moe_wna16 kernels the prescription forced (identity identical), + so all eight DSV4 vllm prescriptions were dropped from targets.yaml and + the eight repos re-probed under the bare render (their cells move from + pass+custom to pass). + KDA PIN RETIRED (owner 2026-09-28, "kda可以到0.30吗，不pin了"): the + manifest's kda family pin on the kimi-k3 preview image is gone; the + family resolves to the 0.30.0 default like every other vllm op + (collect_kda __compat__ 0.30.0). SMOKE ON 0.30 (collect.py --backend + vllm --smoke, one vllm/vllm-openai:v0.30.0 container, 4 random cases + per op, evidence facts/collector_smoke_0300/{main,kda0300,mini,full}): + the first passes exposed three more breaks, all fixed: (a) + is_fused_kda_decode_supported() also gained recurrent_state_dtype + (kda.py:665-680@v0.30.0; float32 as for the prefill probe) — kda decode + cases failed with TypeError; (b) finalize rejected the run — "staged + table scale_matrix_perf.txt has no selected producer": collect.py binds + every staged table to the registry entries declaring it and a producer + checkpoint owns exactly one table, while the compute_scale collectors + hard-coded a second table (PerfFile.SCALE_MATRIX registered 2026-09-21 + never fixed that). First fix (863bfc75) split the op into a + `scale_matrix` registry producer; OWNER DECISION 2026-09-28 ("做兼容 + 多表"): the executor supports MULTI-TABLE PRODUCERS instead — + OpEntry.extra_perf_filenames declares every further table one run_func + writes, the executor hands them in (run_computescale(..., + extra_perf_filenames)), provenance collections carry them, and every + finalize/transaction binding (_registered_checkpoint_tables, + producer plan, perf-transaction journal `tables`, sidecar table + ownership, collection_meta) treats a checkpoint as owning a SET of + tables. compute_scale is back to one op writing both tables from one + measurement; the split op and its YAML activations are gone. Verified: + mini smoke — one checkpoint vllm.compute_scale.json, computescale_perf + + scale_matrix_perf finalized with the same case_plan_hash, exit 0; + (c) the vllm image ships no pyarrow — the launchers pip-install it + before finalize. Full smoke (24 ops incl. the interim split op): 95/96 + cases pass (the one failure is the classified kernel-limit raise for + msa_generation_module at 6 indexer heads), 23 tables finalized as + parquet with a provenance sidecar, exit 0. + DATA: the collector default is 0.30.0 now and the collector finalizes + end to end on 0.30, but no 0.30 perf data campaign has run — the SDK + still ships vllm 0.24.0 rows; the campaign is the next (collector-side) + step. + applies_to: ['framework:vllm-0.30.0', 'instrument:probe_driver', 'instrument:path_diff', + 'instrument:workflow_check', 'collector:vllm', 'platform:sm90', deepseek-ai/DeepSeek-V4.1-Flash, + nvidia/Qwen3.8-2.4T-A95B-NVFP4, moonshotai/Kimi-K3, deepseek-ai/DeepSeek-V4-Flash, nvidia/Nemotron-H-56B-Base-8K] + vllm_0300_full_op_sweep_2026_09_28: + statement: | + Owner request 2026-09-28: sweep every vllm op except gemm on 0.30.0 + (H20, SM90) and look for new large-scale failures. Code frozen at + 863bfc75 (workspace staging_0300), one container per GPU, ops split + by --case-filter as GPUs freed; ~22 h wall, ~48 GPU-hours, 23 tables + finalized (evidence: workspace facts/opsweep_full_0300/{TRIAGE.md, + SUMMARY.txt,PARTITION.txt,logs/}, parquet under staging_0300/runs/). + VERDICT: no deterministic 0.29.0 -> 0.30.0 regression. Row parity with + the 0.29 sweep on every table except where 0.30 is BETTER: + context/generation attention +697/+928 rows (the fp8-KV x head512 + Gemma-4 cells now run vllm_triton_attn, the head_dim>256 + fp8 fix + verified at full grid); dsa_generation +406 rows and 0/17664 failures + (the 0.29 sparse-MLA decode IMA race is gone); moe +27 rows (the + Llama-4 fp8 128E power_law_1.01 OOM seen in the sweep reproduced + clean 3/3 on an idle GPU -> worker-history transient, not a + regression). Every other failure cluster matched 0.29 cell by cell: + msa_generation 1472 (heads 48/96 kernel-limit), gdn 5 (grid-y guard + + one OOM cell empty on 0.29 too), moe 40 (fp8_block block_n / K3 mxfp4 + tp32, no 0.29 rows either), dsv4 csa_context 16 + hca_context 4 (the + 0.29 residual IMA family and the same aten::new_empty cells at 262144 + tokens per forward on fp8 DSV4-Pro). NEW on 0.30 with no baseline: + kda full grid 12 tasks, 4 failed on Kimi-K3 TP1 shapes — decode + bs1024 hits the CUDA grid-y limit (fused_recurrent.py:645 grid = + (cdiv(V,BV), B*H); the kda collector lacks gdn's classified guard), + three context OOMs on the batch x heads = 1536 boundary at isl 32768 + (no generation-time memory filter in the kda context grid). + COLLECTOR DEFECT FOUND (mine): the msa index-head kernel-limit guard + of 2026-09-22 keyed on the head count alone and rejected 1808 + msa_context prefill rows the 0.29 sweep had collected — the limit is + the DECODE scorer's (0.29 evidence: context failed only at isl=1, + 904/976 rows per head otherwise). Fixed in c7269391 + (msa_decode_scorer_kernel_limit, query_len-scoped) and re-run: 904 + + 904 rows recovered, exactly 144 isl=1 cells classified. FOLLOW-UP + (owner 2026-09-29): MSA is one fixed geometry (M3 64q/4kv/4idx, + GQA-style sharding, index heads = kv heads per rank), so the shared + 12-value head grid was replaced by the model row's declared shards — + tensor_parallel_sizes [1,2,4,8,16] -> heads 64/32/16/8/4 + (a22e4553; case_generator.get_msa_head_counts). msa_context 11712 -> + 4880 cases, msa_generation 8826 -> 3680; the 128/96/48/24/12-head + rows were unreachable points and, per the owner ("数据别放这种临时 + 的了"), the 0.30 msa tables were re-collected on the new axis: + 4880 + 3680 rows, zero failures; the sweep's shared-grid msa tables + are superseded (workspace staging_0300/superseded/). dsa_context + grew from 22640 to 169512 rows (the 2026-09-22 chunked-prefill prefix + axis: 2 models x 4 combos x 12 heads x 1816 tuples = 174336 cases); + 4827 tasks failed, 2.7% deterministic ModelConfig rejects (prefix + 262144+ over DeepSeek-V3.2's 163840 context) plus ~20 largest-shape + OOMs. OWNER DECISION 2026-09-28: the dsa grid stays as declared + (over-collection + classified failures). Lessons: split large-grid + ops by (model x precision combo) from the start; the case order is + model-major; a producer's checkpoint can be finalized after an early + stop by a zero-work --resume with the slice's own filters. + applies_to: ['framework:vllm-0.30.0', 'collector:vllm', 'platform:sm90', 'instrument:opsweep', + meta-llama/Llama-4-Maverick-17B-128E-Instruct, moonshotai/Kimi-K3, MiniMaxAI/MiniMax-M3, + deepseek-ai/DeepSeek-V3.2, zai-org/GLM-5, sgl-project/DeepSeek-V4-Pro-FP8, google/gemma-4-26B-A4B] + vllm_0300_gate_set_restored_2026_09_29: + statement: | + Owner question 2026-09-29 ("msa 的 kernel 和 probe 的 align 吗") exposed + that the vllm 0.30.0 gate set I rebuilt kept only 19 of the 32 gates + the 0.29.0 set had — attention ctx/gen (Llama-3.1-8B), encoder_attn + (Qwen3-VL-2B), dsa ctx x4 + gen (DeepSeek-V3.2), gemm m=1 x2, msa + ctx/gen (MiniMax-M3) and kda decode were silently missing, and + path_verdicts_aligned reported 19/19. Restored: every family + re-captured in the 0.30 image with the current collectors and graded; + RESULT 32 declared gates, 32 ALIGNED (msa ctx 3/3 and gen 5/5 attention + kernels matched; dsa fp8 isl-512 graded against a dedicated isl-512 + serving raw of the same record as on 0.29, because isl 512 is below the + sparse threshold and runs dense FA3). Two deviations are EXPLAINED, not + gates: mhc (collector measures the no-norm mhc_pre variant by + producer/consumer contract — the SDK bills attn_norm separately, + collect_mhc_module._mhc_pre — while 0.30 serving selects the + with_norm big_fuse variant via MhcPreBigFuseTileLangKernel.CompileKey + .use_norm_weight) and compute_scale (governed approximation). kda + decode: the collector measures both the fused non-spec kernel and the + packed conv-update + recurrence pair (serving's spec-decode path); the + gate captures a non-spec probe, so the capture sets + AIS_KDA_DECODE_PATHS=fused (collect_kda._decode_paths, recorded in + capture_env) and the fused path is ALIGNED with + kda_decode_fusion_many_heads_kernel. MECHANISM: workflow_check gained + gates_cover_registry_ops (upgrade_op step gates_declared before + path_aligned): every backend registry op must map to a declared gate + family (_OP_GATE_FAMILIES) or be listed gate-free with a reason; an + unmapped op is a finding. It flagged mhc_module immediately (now + explained-by-contract). Lesson: a gate set is checked against the + registry, never against what the script happens to declare. + applies_to: ['framework:vllm-0.30.0', 'instrument:workflow_check', 'instrument:path_diff', 'platform:sm90', + MiniMaxAI/MiniMax-M3, deepseek-ai/DeepSeek-V3.2, moonshotai/Kimi-K3, sgl-project/DeepSeek-V4-Flash-FP8, + meta-llama/Meta-Llama-3.1-8B, Qwen/Qwen3-VL-2B-Instruct] + blackwell_handoffs_absorbed_2026_09_30: + statement: | + Three off-box campaigns handed back to the H20 harness owner on + 2026-09-30 — B300/sm103 (0.29.0, 2026-09-20), B200/sm100 (0.30.0, + 2026-09-29, 109/1/14, 32 gates 27 aligned) and two sm120 boxes (RTX PRO + 6000 96GB: 101/7/16, 30 gates 29 aligned; RTX PRO 5000 72GB: 100/1/23, + 31 gates 29 aligned + 2 platform floors). ABSORBED into pr-228 here: + the RTX 5000 bundle as delivered (patch + captures + sm120 results, + taxonomy_sm120, 8 sm120 findings, gemm_fp8 fp8_static governed + exception per the owner's 2026-09-30 decision, verdict script + parametrized by AIS_SM, --matrix single-backend + writes to the harness + results dir, sm120 fail causes, kv 'auto' declared on bf16 gates, + gemvx/cutlass hints); plus, re-implemented from the B200 / RTX 6000 + notes whose patches are not on this box: render_golden runs the + generator with cwd = workspace (aic-core safe_mkdir; a workspace on + another mount rendered 124/124 as generator rejects on B300 and B200), + path_diff prefers the RENDERED (kv None/auto) record when no + --kv-dtype is asked, and per-SM platform-floor gate markers + (FLOOR_SM= / FLOOR_NOTE= on a verdict line -> platform-floor verdict + file, excluded from path_aligned on that SM, still declared coverage): + sm120's two fp8-KV dense-MLA gates are floors (TRITON_MLA smem), so the + sm120 upgrade_op view is all done honestly and the sm90 view stays + 32/32. The 5000 bundle's platform pin (rtx5000_sm120) was NOT taken — + targets.platform stays the H20. NOT YET ABSORBED (need the patches or + real work): B200's collector fix for the block_size = 64 literal in + collect_attn.py / collect_mla_module.py (ask the framework per case; + re-grade attn/mla gates on sm90 AND sm100), kernel_taxonomy_sm100.yaml + seed and the b200-opharness-fixes.patch (on umb-b200-238), RTX 6000's + platform-scoped cli_extra_args and no-baseline fails_root_caused + semantics (sm120-opharness-fixes.patch on the 6000 box), + fetch_inputs gated-repo mirror fallback (both), identity.attention + column via taxonomy backend labels, platform-blind run ids, the sm120 + generator rule (bf16 KV for NVFP4 MLA on CC 12.0) and the silently + ignored --generator-set kv_cache_dtype (generator module, separate PR). + applies_to: ['platform:sm120', 'platform:sm100', 'platform:sm103', 'framework:vllm-0.30.0', + 'instrument:probe_driver', 'instrument:path_diff', 'instrument:workflow_check'] + taxonomy_dsa_nsa_family_covers_m3: + statement: 'RESOLVED 2026-09-23: M3 sparse-attention kernels now label `triton` (the vocabulary name of + MiniMaxM3SparseTritonImpl), dsa_nsa is DeepSeek-DSA only; see probe_prefill_prefix_cache_contamination. + Original: NAMING DEBT: the kernel_taxonomy family "dsa_nsa" matches MiniMax-M3 sparse-attention kernels + too (sparse_attn_fwd/_gqa_share_sparse/_merge_topk_attn_out/_decode_score are models/minimax_m3 Triton + kernels, ops/sparse_attn.py). Verdicts are computed within-family by kernel-name overlap so the lumping + is functionally harmless today, but the family name misleads readers of verdict files. Rename (e.g. + block_sparse_attn) or split when the owner next touches the taxonomy.' + applies_to: + - instrument:kernel_taxonomy + - MiniMaxAI/MiniMax-M3 + - deepseek-ai/DeepSeek-V3.2 + cross_sm_evidence_contamination: + statement: 'HARNESS STRUCTURAL LESSON (B300 first run outside H20, 2026-09-20): gate evidence keyed by + (fw, version) alone let a fresh arch INHERIT another arch''s green checks — with zero B300 jobs run, + workflow_check --param sm=sm103 reported path_aligned (7 sm90 verdicts) and customizations_retested (9 sm90 + records), and saving B300 verdicts would have OVERWRITTEN the sm90 files. Same class: kernel_taxonomy was a + de-facto SM90 vocabulary (Blackwell fmhaSm103a*/fmhaSm100f*/sm100 cutlass/deepgemm-sm100/fp4 kernels all + unmatched -> attn path_diff went "empty vs empty" ALIGNED, a false green; missing serving kernels made dsa + falsely diverged); per-checkpoint customizations keyed (repo, fw) carried a 143GB-H20 memory workaround to a + 275GB B300. Fixes landed on H20: results/pathdiff//-/ + results/retests//-.yaml + (sm90 evidence git-mv''d; sm103 view now shows honest todos), and path_diff emits "inconclusive" when the + collector signal set is empty (B300 patch, pending). Rule going forward: SM is a first-class dimension of + gate evidence, taxonomy rules (arch-anchored + audited against other arches'' records before merge), + customization facts, and case plans; editing a shared file re-opens verdicts for every SM with shipped data + (unverified_sms when no box is available). Collectors do NOT fork per arch — dispatch stays the framework''s.' + applies_to: + - instrument:workflow_check + - instrument:kernel_taxonomy + - instrument:path_diff + - platform:sm90 + - platform:sm103 + vllm_0290_full_op_sweep: + statement: 'FULL-GRID VALIDATION of the in-place 0.29 upgrade (H20, 4 GPUs, 2026-09-20/21, ~290k perf rows, + 15 op tables finalized to parquet; evidence workspace facts/opsweep_full_0290/ incl. TRIAGE.md + staging_patch.diff + — the staging rehearsal that became commit 993f9cdf). Row parity vs 0.24/0.25 baselines: gemm 111444 (exact), + attention ctx/gen 53777/66708 (exact), computescale/scale_matrix/encoder/mla_bmm exact, msa+gdn above baseline, + moe 58k. ALL four failure clusters root-caused, ZERO 0.29 regressions and ZERO upgrade-code bugs: (1) msa + Triton CompilationError 1616x = kernel domain boundary — sparse_num_index_heads shards to non-power-of-2 at + collector heads 48/96, unreachable in serving (real tp gives 4/2/1); (2+3) attention fp8-KV 1625x = HISTORICAL + blank: 0.24/0.25 baselines contain zero fp8 rows (100% bfloat16), the grid always swept an unsupported region, + only the error text changed; (4a) dsa ctx fp8-KV 776x = flashmla int64_stride_to_int input constraint on the + NEW fp8 collection key — FIXME(kernel-limit) class; (4b) dsa gen IMA ~4% = RACE, audit complete + 2026-09-22: bisect exonerated the block-table fix (22 vs 19 fails on identical 300 cases) AND cudagraph + (eager 21/300) AND the topk buffer (allocation parity with serving — both torch.empty; poison prefill 2e9 + and -1 both pass); CUDA_LAUNCH_BLOCKING pins the faulting kernel to vllm-flash-attn Hopper flash_fwd + (flash_fwd_launch_template.h:205); compute-sanitizer memcheck serializes and the fault VANISHES (8/8, 0 + errors) — a timing race in the 0.29 FA3 sparse-MLA decode chain, worth reporting upstream. Recovery is + operational: failures are stochastic with ~75%/round retry recovery, geometric to near-zero + (GLM gen 300->70->17->5->2 of 8832; DeepSeek 346->106->28->8) — --resume-retry-failed sweeps recover the + rows without touching measurement semantics. ADDENDA: (2+3) VERDICT FLIPPED AGAIN 2026-09-22 via live-engine + probe — collector-specific, NOT a framework selector gap. head_dim=512 dense attention comes only from + Gemma-4 (global layers: global_head_dim=512, num_global_key_value_heads=2). Real vllm serving of Gemma-4 with + --kv-cache-dtype fp8 on SM90 GENERATES fine: Gemma4Config.verify_and_update_config + (model_executor/models/config.py:209-256) is a model-level hook that, for heterogeneous head dims (256/512), + forces FA4 for all layers (or TRITON_ATTN fallback) because default FA3 on Hopper cannot handle head_dim>256 + — so serving NEVER routes Gemma-4 head512 to FlashInfer (resolved backend TRITON_ATTN, candidate list + [TRITON_ATTN]). collect_attn constructs the attention in isolation and calls the bare platform selector + (get_attn_backend_cls) WITHOUT running that model hook, so it picks FlashInfer whose SM90 JIT cannot compile + fp8+head512 -> the 1625 dsa... i.e. attention fp8 failures. Same class as DSA 4a / Gemma monolithic: the + collector reproduces a backend serving never selects. Fix: the collector must select the backend serving + actually uses for head_dim>256 (apply the Gemma4 heterogeneous-head override: FA4/Triton, never FlashInfer), + with the config.py:209-256 citation. Evidence facts/gemma4_fp8_probe (GENERATE OK, TRITON_ATTN). (4a) VERDICT FLIPPED 2026-09-22 via live-engine probe: + collector-specific, NOT a framework limit. fp8-KV + num_heads<32 (the collector head-sweep emulating TP>=5) + selects fp8 mixed mode (MIN_HEADS_FOR_BF16_PREFILL=32, flashmla_sparse.py:63/307), which runs the whole batch + as one MQA batch through the fp8 decode kernel (head padding 16->64); the collector builds the prefill as a + SINGLE monolithic forward so num_actual_tokens=isl(32768) and stride = tokens*padded_heads*576 breaks int32. + Real-engine tests (8 GPUs idle): TP=1 native-128-heads isl=32k fp8 runs (separate bf16 splitkv path); TP=8 + (16 heads/rank -> the SAME mixed fp8 decode kernel, padding 16->64) fed a 32768-token prompt -> GENERATE OK. + Serving chunks the prefill (max_num_batched_tokens), keeping per-forward num_actual_tokens well under 29k; + the collector does not chunk. Fix: the collector context construction should mirror serving''s chunked prefill + (split at max_num_batched_tokens) -- this both removes the 776 failures AND makes large-isl context rows + faithful to how serving actually invokes the kernel (the monolithic measurement is itself non-faithful for + large isl; even the passing large-isl context rows warrant review). Evidence + facts/ab_dsa_stride_{oneshot,tp8}.json. + (1) msa kernel limit converted to an invocation-site guard raising a classified error with the framework + citation (commit c0f7f035). dsv4 full-grid rerun on the merged lane, FINAL: + 22589/22604 collected (99.93%) across 6 ops — generation 3088/3088, paged_mqa_logits + hca_attn 3918/3918 + each, hca ctx 5840/5840 after retries; the sole residual is a DETERMINISTIC 15-case subcluster in csa ctx, + all one shape family (seq=16 chunk + batch=64 + fp8-KV + prefix=4096, both FP8 checkpoints, heads 1-8) — + the fp8 csa large-prefix small-tail-chunk path; classified, re-tested free on the next version bump. + Harness lessons banked: PerfFile.SCALE_MATRIX enum gap (fixed), measurement containers must never share host + network (setup_distributed port collisions), collection images need deps pre-baked, executor ignores --ops + CLI order, sharding is manual via --case-filter.' + applies_to: + - framework:vllm-0.29.0 + - platform:sm90 + - instrument:collect.py + sm120_vllm_0300_campaign: + statement: 'FIRST SM120 CAMPAIGN (RTX PRO 5000 Blackwell 72GB x8, compute capability 12.0, 2026-09-30; vllm 0.30.0, + golden renders with --system rtx_pro_6000_server as the 5000 proxy — no SDK entry for the 5000). 124 roster cells: + 100 pass / 1 pass+custom / 23 fail (12 generator rejects = the same SDK-gap repos as sm90; 4 platform limit; + 2 platform gap; 5 capacity). Deployed identity on sm120 differs from sm90 in every family: dense attention + = FA2 (flash::flash_fwd_splitkv_kernel; FA3 is sm90-only) with FlashInfer paged prefill + XQA decode under + fp8 KV; MLA = TRITON_MLA (prefill FA2 hd192, decode _fwd_grouped_kernel_stage1); sparse MLA (DSV3.2/GLM-5.2/DSV4) + = FLASHINFER_MLA_SPARSE_SM120 / DeepseekV4FlashInferSM120Attention (flashinfer::sparse_mla_sm120::*, + sparse_mla_prefill_mg*); fp8 linears AND fp8 MoE = DeepGEMM sm120 lane (deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl + + ep_scatter/gather glue) instead of cutlass/triton_fused_moe; bf16 MoE (GLM-5.2, MiniMax-M3, Qwen1.5-MoE) = + FlashInfer CUTLASS fused MoE (fused_moe::run_global / MoeFCGemm) instead of triton fused_moe_kernel; M=1 + GEMMs = cuBLAS gemvx instead of nvjet; per-tensor fp8 (ModelOpt) decode GEMMs = cuBLAS sm89_xmma_gemm_e4m3. + kernel_taxonomy_sm120.yaml was seeded from sm90 and grew 25 evidence-based rules (every rule names a kernel + recorded on this box); decompose residue = 1 kernel (the DSV4 cute-DSL kernel_cutlass_device_kernel, same + owner-decided residue as sm90). Gates: 29 aligned, 1 explained, 2 ungradable; the 2 DeepSeek-R1 fp8-KV MLA gates have no serving + record (see sm120_triton_mla_fp8kv_smem); gemm_fp8 (dynamic-activation row, no faithful serving reference) is an explained deviation under an owner exception (sm120_collector_gemm_fp8_static_row); + compute_scale and mhc stay explained deviations as on sm90. Evidence bundle 2026-09-30_sm120_vllm-0.30.0 + (results/evidence_index.yaml).' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + sm120_triton_mla_fp8kv_smem: + statement: 'PLATFORM LIMIT (vllm 0.30.0 on sm120): every MLA checkpoint that resolves an fp8 KV cache dies at + CUDA-graph capture with triton OutOfResources "shared memory, Required: 102400, Hardware limit: 101376" in + TRITON_MLA decode (_decode_grouped_att_m_fwd, v1/attention/ops/triton_decode_attention.py:534; BLOCK_DMODEL 512 + + BLOCK_DPE 64, num_stages 2 — the guard at :524-532 only drops num_stages for BLOCK_DMODEL >= 1024). The sm120 + opt-in smem/block is 101376 B (torch.cuda.get_device_properties inside the image), 102400 B/SM. Hit by the fp8-KV + probe sweep of DeepSeek-V3, DeepSeek-R1, Kimi-K2-Instruct, Kimi-K2.5 (bf16-KV cells of the same repos PASS, + identity TRITON_MLA) AND by the RENDERED config of the NVFP4 artifacts whose hf_quant pins fp8 KV + (nvidia/DeepSeek-V3.1-NVFP4, Kimi-K2.5/K2.6/K2.7-Code-NVFP4: matrix fail cells). The collector''s mla_gen_fp8 + capture crashes identically (observe-don''t-predict: same dispatch, same limit), so the mla_ctx_fp8 / mla_gen_fp8 + DeepSeek-R1 gates have no serving record on this SM and cannot be graded. Not a generator defect: `cli generate` + renders no kv dtype; the crash is the framework''s only MLA decode backend on major==12 (platforms/cuda.py:132-135 + lists TRITON_MLA + FLASHINFER_MLA_SPARSE_SM120). A deployment of those NVFP4 checkpoints on RTX PRO 5000/6000 + with vllm 0.30.0 does not boot; upstream-report candidate (reduce num_stages/BLOCK for fp8 K on sm12x). + Evidence: archive/raw/{175d26fcbb87,c78550b69fb3,c73c64978344,f6e846bb5051,f157c1c39890,691d6cce3327, + 9c912acc9024,c91c53ea605c}.json, facts/pathdiff/opcov_mla_gen_fp8@0.30.0.json (RUN ERROR).' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - deepseek-ai/DeepSeek-V3 + - deepseek-ai/DeepSeek-R1 + - moonshotai/Kimi-K2-Instruct + - moonshotai/Kimi-K2.5 + - nvidia/DeepSeek-V3.1-NVFP4 + - nvidia/Kimi-K2.5-NVFP4 + - nvidia/Kimi-K2.6-NVFP4 + - nvidia/Kimi-K2.7-Code-NVFP4 + sm120_dsv41_sparse_mla_no_decode_shape: + statement: 'PLATFORM GAP (vllm 0.30.0 on sm120), re-verified 2026-10-01 on the faithful dsv41 dummy (7-layer source+consumer + cut, upstream a04c849c) after the first verdict ran on the unfaithful 2-layer cut: DeepSeek-V4.1-Flash and + nvidia/DeepSeek-V4.1-Flash-NVFP4 cannot boot on sm120 under ANY kv block size. The KV block size is pinned from two + sides that disagree off Hopper: DeepseekV41IndexerBackend.get_supported_kernel_block_sizes() returns [64] only on + capability family 90 and [128] everywhere else (vllm/v1/attention/backends/mla/indexer.py:262-263), while the sm120 + sparse-MLA decode path (flashinfer 0.6.18 _sparse_mla_sm120.py: _DECODE_DSV4_PAGE_BLOCK_SIZE = 64; 0.7.0.post1 + _sparse_mla_sm120_plan.py: _PAGE_BLOCK_SIZE = 64) accepts page 64 only. Observed: default block -> DeepGEMM + "block_kv == 32 or block_kv == 64" (attention.hpp:409) from the indexer metadata build; --block-size 64 (the V4.1 facts + value that rescues H20/sm90) -> "No common block size for 64" (worker/utils.py:386, select_common_block_size); the + original 2-layer run showed the third face of the same conflict ("no decode kernel for this shape ... page_block_size=32"). + Not config-rescuable; needs a vllm change (indexer block 64 when the sm120 FlashInfer sparse backend is selected). + The latest vllm RELEASE is still 0.30.0 (PyPI 2026-10-01), so no released vllm runs V4.1 on RTX PRO / sm120; vllm main + not probed. DeepSeek-V4-Flash/Pro (indexer block 256, no V4.1 indexer class) PASS on sm120. A/B evidence: + facts/ab_sm120/{32100c75d1df,c78c20c47673}_bs64.json; stale goldens/dummies in archive/superseded_*/2026-10-01_*.' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - model:deepseek-ai/DeepSeek-V4.1-Flash + - model:nvidia/DeepSeek-V4.1-Flash-NVFP4 + rtx_systems_have_no_generator_facts_hardware_profile: + statement: 'GENERATOR GAP (found while re-probing V4.1 on sm120, 2026-10-01): model-default cli args from + generator/facts/models.yaml (e.g. V4.1 block-size 64, trust-remote-code, no-enable-flashinfer-autotune; V4 block-size 256) + are NEVER applied when --system is an RTX PRO system. request_resolution.hardware_key_for_system() has no entry for + rtx_pro_6000_server (map covers h100/h200/h20/b200/b300/gb200/gb300 only) and falls back to the raw name; resolve_facts() + raises KeyError("Unknown hardware profile rtx_pro_6000_server ... [b200, b60, gb200, h100, h20, h200]") and + pipeline.run_pipeline swallows it ("Fact resolution failed; continuing without facts", facts=None), so apply_facts is a + no-op. Every sm120 golden render in this campaign (archive/golden/*/render.log carries the KeyError) therefore ran + without model defaults; the harness has been compensating with --generator-set params.agg.trust_remote_code=true in + targets.yaml. The upstream "drop the harness override" for V4.1 block-size (a04c849c) is correct on H20 and silently + inert on RTX. Candidate fix (generator, owner approval): add an rtx_pro hardware profile to facts/hardware.yaml and map + rtx_pro_6000_server (and the future RTX PRO 5000 system) in _SYSTEM_TO_HW; or make unknown hardware degrade to + model-only facts instead of none.' + applies_to: + - platform:sm120 + - instrument:generator + - instrument:probe_driver + sm120_capacity_72gb: + statement: 'CAPACITY (72GB probe GPU vs 143GB H20): Qwen/Qwen3.8-2.4T-A95B{,-FP8} and nvidia/Qwen3.8-2.4T-A95B-NVFP4 + OOM at load even on the depth4 cut (one MoE layer of the 2.4T model exceeds the card: 32 GiB single allocation + bf16), meta-llama/Llama-4-Maverick-17B-128E-Instruct OOMs on its smallest cut (depth4, 20 GiB expert tensor) — + "no faithful cut fits one probe GPU"; Qwen3.5-397B-A17B needed the depth8 -> depth4 fallback and passes. The + capacity fallback (probe_driver enumerate_runs) worked as designed but needs a second --emit-queues after the + first sweep (the OOM must be observed first) — documented, not automatic. Nemotron-3-Ultra-550B: all_kinds (3 + layers) OOMs; the fallback ranks single-kind cuts ALPHABETICALLY and picked layers_block_type_attention, which + passes for BF16/FP8 (identity: attention + no MoE column) but for the NVFP4 artifact strips every quantized layer + -> vllm rejects MIXED_PRECISION with an empty quantized_layers map (dummy artifact). Owner-side variant_override + to the MoE cut then fails the probe''s attention hook (no attention module in a MoE-only cut). Both are recorded + as capacity cells, not framework facts. Lesson: single-kind cuts are not identity probes; a 72GB box needs a + 2-layer mixed cut (attention+moe) variant for Nemotron-H-class models.' + applies_to: + - platform:sm120 + - Qwen/Qwen3.8-2.4T-A95B + - Qwen/Qwen3.8-2.4T-A95B-FP8 + - nvidia/Qwen3.8-2.4T-A95B-NVFP4 + - meta-llama/Llama-4-Maverick-17B-128E-Instruct + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 + sm120_gemm_fp8_kernel_level_drift: + statement: 'gemm_fp8 gate DIVERGED on sm120 (GEMM-class gates are graded on kernel-name overlap; the collector''s + one GEMM kernel is not among serving''s 14): the collector''s per-tensor fp8 case (Fp8LinearMethod -> CutlassFP8ScaledMMLinearKernel, op_smoke confirms the + selection) runs cutlass MainloopSm120TmaWarpSpecialized, while serving Llama-3.1-70B-Instruct-FP8 + (ModelOptLinearMethod, fp8 per-tensor weights) runs MainloopSm120TmaWarpSpecializedBlockwiseScaling for the + prefill GEMMs and cuBLAS sm89_xmma_gemm_e4m3 (torch._scaled_mm) for the M=1 decode GEMMs. Same sm90 pairing was + kernel-identical. The vllm gemm fp8 perf rows on sm120 therefore measure a different mainloop than the ModelOpt + checkpoints deploy; whether ModelOpt''s sm120 blockwise mainloop is a vllm dispatch quirk or the intended path + for per-tensor scales needs a collector-side serving-parity audit before the sm120 gemm data is trusted. + Evidence facts/pathdiff/opcov_gemm_fp8@0.30.0.json vs archive/raw/da1900656836.json.' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - nvidia/Llama-3.1-70B-Instruct-FP8 + harness_generalization_sm120: + statement: 'HARNESS LESSONS from the first campaign on a fresh SM + fresh workspace (2026-09-30), each fixed in + this checkout unless noted: (1) build_matrix aborted when ANY targets backend lacked a plan (single-backend + campaign impossible) -> now skips backends without a plan; (2) build_matrix wrote results// under the + WORKSPACE, so workflow_check (which reads the checkout) never saw the matrix -> now writes the harness results/; + (3) the taxonomy SM comes from the AIS_SM env var, not targets.platform (forgot once = sm90 vocabulary graded + sm120 records) — NOT fixed, documented; (4) verdicts_vllm_0300.sh hardcoded AIS_SM=sm90, the sm90 output dir + and a checkout-inside-workspace path -> SM-parametric + script-relative; (5) 9 of the 32 vllm gates (attn_ctx/gen, + dsa_ctx_*, dsa_gen_fp8, msa_ctx/gen) had NO committed capture recipe (produced ad hoc via op_smoke on sm90) -> + capture scripts added; kda_gen needed AIS_KDA_DECODE_PATHS=fused that only lived in a shell history -> now set + in the script; compute_scale capture had drifted from the registry signature (extra_perf_filenames) -> fixed; + (6) gates for bf16-KV models declared kv "" and path_diff then admitted the fp8-KV record of the same repo (newest + wins) -> attn/gemm_bf16 gates declare kv auto; op hints lacked gemvx (the M=1 cuBLAS gemv on sm120), which hid + serving GEMM evidence; (7) the matrix attention-identity heuristic matched the kv-cache glue + vllm::concat_and_cache_mla_kernel on TRITON_MLA records -> excluded, _fwd_grouped_kernel_stage1 added; (8) + fetch_inputs hard-stops on gated repos with no mirror fallback: meta-llama x5 and the gated Nemotron-3-Ultra-FP8 + were provisioned by hand from unsloth mirrors / SDK model_configs (configs/GATED_SOURCES.md in the workspace); + (9) the generator vllm.rule has no SM branch — sm90 and sm120 render byte-identical engine args, so every sm120 + fail above is a framework/platform fact, none a rendering fact. Data plane grew: kernel_taxonomy_sm120.yaml, + results/sm120/{vllm-0.30.0.yaml,decompose/}, results/pathdiff/sm120/vllm-0.30.0/ (30 verdicts), + results/retests/sm120/vllm-0.30.0.yaml, _fail_cause vocabulary for the sm120 causes.' + applies_to: + - platform:sm120 + - instrument:probe_driver + - instrument:path_diff + - instrument:workflow_check + sm120_config_ab_2026_09_30: + statement: 'CONFIG-OR-FRAMEWORK A/B for every non-SDK-gap fail cell on sm120 (owner question: is the rendered config + wrong, and does the framework hint at a backend/flag that makes it boot). Method: the rendered run.sh with ONE + extra engine arg appended, same dummy, same image, same probe (facts/ab_sm120/_.{sh,json}). + RESULTS. (a) NVFP4 MLA checkpoints (nvidia/Kimi-K2.5-NVFP4 691d6cce3327, nvidia/DeepSeek-V3.1-NVFP4 f157c1c39890, nvidia/Kimi-K2.6-NVFP4 9c912acc9024, nvidia/Kimi-K2.7-Code-NVFP4 c91c53ea605c): + + --kv-cache-dtype bfloat16 -> PASS on all four (identity TRITON_MLA bf16 KV, DeepGEMM sm120 linears). So the cell IS + config-rescuable: the checkpoint''s hf_quant pins fp8 KV, and on cc 12.0 the only non-sparse MLA backend that + declares support is TRITON_MLA (enumerated: FLASHINFER_MLA / CUTLASS_MLA / FLASHMLA / FLASH_ATTN_MLA / + TOKENSPEED_MLA all report cc12=False), whose fp8-KV decode kernel overflows smem — bf16 KV is the only route. + GENERATOR GAP: params.agg.kv_cache_dtype=bfloat16 renders `--kv-cache-dtype auto` (backend_config_mapping.yaml + kv_cache_dtype guard "vLLM doesn''t accept bfloat16") and auto re-resolves to the checkpoint''s fp8 -> the + rescue cannot be expressed through the generator today; vllm 0.30.0 accepts the literal bfloat16 (this A/B). + Bucket-1 candidate: drop/condition the guard, and an SM120 rule "MLA + quantized KV -> kv_cache_dtype bfloat16". + (b) DeepSeek-V3 / R1 / Kimi-K2 fp8-KV probe variants: + --max-num-seqs 64 --max-num-batched-tokens 4608 -> same + smem failure (kernel tile config, not batch); no alternative backend exists (see a). Not a config issue; the + rendered default (bf16 KV) passes. + (c) DeepSeek-V4.1-Flash (both repos): + --attention-backend FLASHINFER_MLA_SPARSE_DSV41 / TRITON_MLA, + --max-num-seqs + 32, + --block-size 64 / 128 / 256 -> all FAIL with the same "no decode kernel ... page_block_size=32". Root: the DSV41 sm120 + backend pins kernel block size [128] (models/deepseek_v41/nvidia/flashinfer_sparse.py:112) and computes the + compressed-layer page as block_size // compress_ratio = 128 // 4 = 32 (:759, :839), while FlashInfer''s sm120 DSV4 + decode requires page 64 (_DECODE_DSV4_PAGE_BLOCK_SIZE = 64; (64,128) IS in _DECODE_DSV4_DISPATCH). No engine flag + reaches that arithmetic -> framework gap, upstream-report candidate (vllm/flashinfer sm120 DSV4.1 decode). + (d) Kimi-K3 fp8-KV variant: + --attention-config {"use_prefill_query_quantization": true} (the flag vLLM''s own + assertion names) -> same AssertionError: on sm120 the prefill backend is FA2, which cannot consume fp8 queries + (mla_attention.py determine_prefill_query_data_type -> backend_supports_prefill_query_quantization False), so the + flag is silently ignored and the K3 assert fires. fp8 KV for Kimi-K3 is not reachable on sm120 by any flag; bf16 KV + (the rendered default) passes. Arch-neutral half of this is already in findings (0.29 K3 followups). + (e) nvidia/Nemotron-H-56B-Base-8K: bare render OOMs on 72GB too; + --max-num-seqs 64 (no gpu-memory-utilization) + -> PASS, confirming the root knob is max_num_seqs on sm120 as on sm90; the generator still floors max_batch_size + at 512 (vllm.rule) so the expressible rescue stays extra_cli_args gpu-memory-utilization 0.6 (retest: still_needed). + (f) capacity cells (Qwen3.8-2.4T x3, Llama-4-Maverick, Nemotron-Ultra-NVFP4 single-kind cut): dummy weights do not + fit 72GB — not a config question; no A/B.' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - nvidia/Kimi-K2.5-NVFP4 + - nvidia/Kimi-K2.6-NVFP4 + - nvidia/Kimi-K2.7-Code-NVFP4 + - nvidia/DeepSeek-V3.1-NVFP4 + - deepseek-ai/DeepSeek-V4.1-Flash + - nvidia/DeepSeek-V4.1-Flash-NVFP4 + - moonshotai/Kimi-K3 + - nvidia/Nemotron-H-56B-Base-8K + sm120_collector_gemm_fp8_static_row: + statement: 'COLLECTOR-SIDE DISPOSITION of the gemm_fp8 divergence (serving-parity audit + owner decision, 2026-09-30). + Audit: vLLM 0.30.0 selects the fp8 scaled-mm kernel by activation scheme — FlashInferFP8ScaledMMLinearKernel is + first in _POSSIBLE_FP8_KERNELS (kernels/linear/__init__.py:421-430), is_supported needs CC >= 100 + (scaled_mm/flashinfer.py:50-51), can_implement needs per-tensor activation AND weight scales (:56-63). ModelOpt + FP8 checkpoints (quant_algo FP8, static input_scale; modelopt.py:2333) satisfy it -> serving log "Selected + FlashInferFP8ScaledMMLinearKernel for QKVParallelLinear/RowParallelLinear/MergedColumnParallelLinear" + (facts/ab_sm120/da1900656836_baseline_log.log) -> cutlass MainloopSm120TmaWarpSpecializedBlockwiseScaling + + cuBLAS sm89_xmma at M=1. The collector''s `fp8` row is Fp8Config(activation_scheme="dynamic") -> can_implement + False -> CutlassFP8ScaledMMLinearKernel (op_smoke confirms). Reproduced in isolation: static vs dynamic Fp8Config + on one shape selects FlashInfer vs Cutlass (facts/ab_sm120/opcov_gemm_fp8_static_probe.json). fp8_static was NEVER + a collected row: the SDK derives it from the dynamic fp8 row + compute_scale + scale_matrix + (perf_database._gemm_key_names), which presumes one shared GEMM kernel — true below CC 100, false on SM100/103/120. + Every per-tensor FP8 checkpoint in the roster is ModelOpt static, so the gemm_fp8 gate never had a faithful + serving reference (sm90 aligned by coincidence). OWNER DECISION (tianhaox): do NOT add a collected fp8_static row + (a trial implementation was validated — op_smoke kernel_source=FlashInferFP8ScaledMMLinearKernel, gate aligned — + and then reverted); handle it as a governed approximation: collector/evidence_exceptions.yaml standards_exceptions + entry "fp8_static derived ... CC>=100" (scope gemm_perf / vllm 0.30.0 / CC>=100 systems, expires 2026-12-31), + gemm_fp8_Llama-3.1-70B-FP8 stays an explained deviation (facts/pathdiff/explained_0300). MLA fp8-KV on sm120 + (mla_gen_fp8 capture crashes like serving) is a framework-version gap with no YAML home (failure_handling.md): the + case fails classified and re-tests on the next pin bump; no registry unverified_sms mark since the bf16-KV rows + of the same op are validated.' + applies_to: + - platform:sm120 + - platform:sm100 + - platform:sm103 + - framework:vllm-0.30.0 + - instrument:collect_gemm + sm120_dsa_head_axis_failures_accepted: + statement: 'OWNER DECISION (tianhaox, 2026-09-30) on the sm120 vllm 0.30.0 collection: the dsa_context/generation_module + failures for per-rank head counts 96/48/24/12 (and 4/2/1) are ACCEPTED as classified failures — no sm120-specific + axis pruning, no skip. Facts: mla_module_vllm.head_counts is a shared grid [128,96,64,48,32,24,16,12,8,4,2,1]; the + 96/48/24/12 values are Kimi-K3 (96 heads) TP1/2/4/8 shards and are REAL for the dense mla module, but no DSA model + (DeepSeek-V3.2 128 heads, GLM-5 64 heads) produces them at any TP. sm90 FlashMLA-sparse and sm100 FlashInfer-sparse + pad heads, so those rows collect as fabricated configurations there; sm120 FlashInfer sparse-MLA supports only + h_q in {8,16,32,64,128} and refuses ("Unsupported sparse-MLA prefill configuration" / "no decode kernel for this + shape"), so on sm120 the rows fail at metadata build in ~0.3 s each (~37% of dsa_context cases together with the + prefix >= 262144 max_model_len group; ~40 min per 21.8k-case shard). Shipped vllm 0.24.0 dsa tables on every system + carry heads {8,16,32,64,128} (+1/2/4 on Hopper) only, so nothing regresses. Rationale: layer_permissions offers no + head-axis negative gate (max_sm_exclusive is quant-mode only) and failure_handling wants framework gaps to fail and + re-test on the next pin. FOLLOW-UP (low priority, arch-neutral, for the H20 owner at the next bump): derive the + mla/dsa head axis per model row from declared tensor_parallel_sizes like get_msa_head_counts does (mla keeps + {128,96,64,48,32,24,16,12,8} via Kimi-K3, dsa drops to {128,64,32,16,8}), and cap dsa_prefix_lengths by the model + row''s max_position_embeddings (262144/524288/1048575 exceed DeepSeek-V3.2 163840 / GLM-5 202752 on every SM).' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - instrument:collect_mla_module + sm120_vllm_0300_collection_2026_10_01: + statement: 'FULL PERF-DATA COLLECTION on the RTX PRO 5000 (sm120) box, vllm 0.30.0, --model-cases-full --sm 120: + 20 ops / 634,878 planned cases / 41 case-filter shards on 8 GPUs, ~12 h wall time incl. clean-GPU retries; + launcher = tools/perf_database/collect_campaign.py (written during this run, see aic-auto-collect skill). + DATA (rows, after retries): gemm 144,142 (bf16 37,472 / fp8 37,518 / fp8_block 31,598 / nvfp4 37,518); + attention_context bf16 27,218; attention_generation bf16 33,785; dsa_context 106,020 (12 x 8,835); + dsa_generation 10,862; mla_context 8,487; mla_generation 7,059; msa 7,320 + 5,511; dsv4 csa/hca ctx 5,718 / + 5,789, gen 1,544 / 1,544; encoder_attention 7,679; moe 76,947; compute_scale 3,256 (2 tables); mla_bmm 300 + + 336; gdn 10,284; kda 1,247. attention fp8-KV ctx/gen re-collected from scratch after the collector fix below: context 27,144 / generation 32,370 rows (labels fiprefill / trtllmapidecode_xqa / triton_attn), 90 + 1,448 XQA power-of-2 failures left; campaign total 524,972 rows. + FAILURE GROUPS, all classified, none hidden: (1) dsa heads 96/48/24/12/4/2/1 + prefix >= 262144 — owner + decision sm120_dsa_head_axis_failures_accepted (~60% of dsa_context cases, fail at metadata build); + (2) TRITON_MLA fp8-KV smem 102400 > 101376 (mla_generation 6,036; the platform limit from the probe campaign); + (3) fp8 prefill query on FA2 (mla_context compute=fp8 4,392: FA2 cannot take fp8 Q, collector refuses to + mislabel — declared min_sm 100 but SM120 is >= 100 by number while lacking the SM100 path: the combo axis + needs a family gate, not a floor); (4) sparse-MLA sm120 decode shape table (dsa_generation 9,996); + (5) gemm fp8_block: cutlass c3x "Invalid status" 4,491 (documented SM120 gap) + triton_scaled_mm scale-shape + assert 546 at N in {4, 8}; (6) XQA JIT static_assert isPowerOf2 for MQA group size 48/24 (attention fp8-KV, + n_kv=1, ~1,450 cases; the same shapes IMA under bf16 FA2 decode: 3 cases); (7) large-shape CUDA faults — + dsv4_csa_context b*s = 262144 fp8-KV (IMA / capture "unjoined work" / aten::new_empty ~50), mla_generation + b*s = 16.7M tokens (3 IMA per gemm type; int32-index class); (8) moe: NVFP4 w1/w3 intermediate padding + unsupported 69, gate/up not divisible by block 128: 36, inter_size % alignment 3; (9) gdn packed recurrent + grid-y limit 4, kda strict-completeness 1; (10) residual OOMs: 30 -> 19 (attention_generation bf16), 16 -> 5 + (attention_context bf16), 46 -> 10 (gemm bf16 32768 x 65536 x 51200-class, 6 weight copies ~40 GB) after + a clean-GPU --resume-retry-failed pass; the ones that reproduce are genuine 72 GB capacity limits. + COLLECTOR FIXES that came out of it (branch tianhaox/aisimulate:sm120-collect-campaign): collect_attn lays + the paged KV cache out in the framework-resolved kv_cache_layout (NHD on FlashInfer outside SM100; the fixed + HND memory failed every fp8-KV GQA decode on sm120, 20,458 + 1,354 cases, while serving passed) and the + FlashInfer decode kernel_source names the kernel (…trtllmapidecode_xqa / _trtllm_gen; 0.30 merged both into + one metadata class); collector/helper _merge_perf_rows treats string/large_string as one type (the third + finalize of a table used to OVERWRITE it — 27,303 rows silently replaced by 5,067; mechanism change flagged + for owner review). DELIVERY BLOCKER: no SDK system definition for the RTX PRO 5000 (playbook §8 forbids the + rtx_pro_6000_server relabel); data stays in the workspace until systems/.yaml exists.' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - instrument:collect.py + - instrument:collect_campaign + sm120_dsa_kv_dtype_axis_collapses: + statement: 'DSA module rows on sm120 carry kv_cache_dtype=fp8 for BOTH declared KV variants: vLLM 0.30.0''s + FLASHINFER_MLA_SPARSE_SM120 path resolves the cache to fp8_ds_mla regardless of the requested dtype (opharness + serving probes of DeepSeek-V3.2 and GLM-5.2: kv None and kv fp8 both -> cache_config_dtype fp8_ds_mla), and the + collector records the RESOLVED dtype (serving truth). Consequence: the bf16-KV and fp8-KV shards of dsa_context / + dsa_generation produce identical identity keys (8,835 of 8,835 per shard pair, latency ratio 1.01), so half of + the sm120 DSA collection (~4 h x 8 GPUs) re-measured the same physical configuration; the merged tables hold + 53,010 + 5,432 unique rows (106,020 + 10,862 collected). Not a data defect — the rows are truthful — but the + case plan''s kv axis is not a real axis for DSA on this SM. Candidate: dedup DSA cases on the resolved kv dtype at + plan time (the collector already collapses physical keys for mla specs), or declare the DSA kv axis per SM.' + applies_to: + - platform:sm120 + - framework:vllm-0.30.0 + - instrument:collect_mla_module + dsv41_dummy_cut_and_vllm_block_size_2026_10_01: + statement: 'CORRECTION of the DeepSeek-V4.1 rows in vllm_0300_upgrade_2026_09_27 and of the 09-30 support probe: the + V4.1 dummy those verdicts ran on was not V4.1. The generic depth cut kept original layers [0,1] — two sliding-window + layers (compress_ratio 0) — with kv_source_layer_ids / index_source_layer_ids / candidate_source_layer_id / + engram_layer_ids left verbatim (keys end in `_layer_ids`; the stale-ref post-check only knew `layers`), so no + compressor, indexer, compressed-KV attention or candidate path was ever exercised. FIX (owner request 2026-09-30, + "put the engram shrink in the dummy handling, no need for full size"): dummies.py `dsv41` adapter, routed by + targets.yaml dummy_overrides.family. Rep cut = original layers [0,1,2,3,20,21,24] (swa; engram swa; ratio-2 kv/index + source + consumer; candidate/ratio-1 source + consumer; candidate-masked indexer), rep_min = [1,2,3,20,21] as the + capacity fallback. Every pointer is renumbered (kv sources [2,4], index sources [2,4,6], candidate 4, engram [1]) and + the adapter raises if a compressed layer has no source at or below it (the frameworks assert exactly that: + vllm 0.30.0 models/deepseek_v41/attention.py:283-299, sglang main models/deepseek_v4.py:4316-4322). Engram hash + tables shrink by lowering engram_vocab_size 16000000 -> 100000 and recomputing engram_num_embeddings with the + frameworks'' prime-sum rule (vllm common/engram.py:186-199, sglang layers/engram.py:158-175; reproduces the + shipped 384006168/384016682 exactly): 2.4M rows = 0.6 GB per layer instead of 98 GB. Row count is memory only — + hash/gather/gate kernels and per-token work are unchanged; 100000 keeps buckets-per-head above the compressed + vocab (99092). The stale-ref check now covers `*_layer_id(s)` keys and checks nested sub-configs against their own + depth (vision_config, Inkling mtp_config). SIDE EFFECT: Inkling''s local_layer_ids (55 of 66) is a layer-kind + axis, so its generic variants became all_kinds [0,5] + single-kind with the list remapped (Inkling is a + generator reject on every backend, no record changed). RESULTS on the faithful cut (H20/sm90): vllm 0.30.0 + deepseek-ai/DeepSeek-V4.1-Flash FAILS at load on its golden render — the generator''s deepseek_v4 fact + (facts/models.yaml, matched through the `deepseek-ai/DeepSeek-V4*` path glob) renders block-size 256, and V4.1''s + compressed layers use 64-token kernel blocks, so the 256-token manager block must split 4-way, which the BLHNC + layout cannot do once a KV tensor is shared by several layers ("block stride 275904 != page 74752", + kv_cache_interface.py:373). Depth-independent: the real 40-layer deployment fails the same way. Rescue as the + framework''s own message suggests: `--generator-set params.agg.tokens_per_block=64` -> PASS (rendered and fp8-KV; + A/B in facts/../archive/superseded_dummies/2026-10-01_dsv41_inkling/raws_block256). The nvidia NVFP4 sibling is + NOT matched by that fact (arch DeepseekV41 / model_type deepseek_v41 / nvidia path), renders no block-size and + passes at vllm''s default — an inconsistency between two artifacts of one model that is the generator''s to + resolve. Matrix vllm-0.30.0: 110/1/13 -> 109/2/13 (V4.1 fp8 pass -> pass+custom). Identity now includes + DeepseekCompressor, DeepseekV4Indexer (deep_gemm sm90_fp8_mqa_logits / paged + sched metadata, + cp_gather_indexer_k_quant_cache, topKPerRowPrefill, cooperative_topk) and the candidate filter + (_block_scores / _candidate_flags / _mask_candidates / _store_candidates). RESIDUE for the owner''s granularity + call (unlabeled by the sm90 taxonomy on purpose): compressor glue (_fused_save_compress_norm, _rope_quant_insert, + _ring_slot_mapping), indexer-K store (_indexer_k_norm_rope_quant_store), the candidate kernels, engram + (_hash_ids / _engram_lookup / _fused_engram_post_wkv / _gather_lookback) and the cute-DSL LL bf16 router-gate + GEMM. sglang main (nightly-dev-20260930-cae69be5, exploratory, not a pin): all four V4.1 runs PASS (fp8 rendered, + fp8_e4m3 KV, NVFP4 x2 with the dsv4 marlin customization — runner=auto refuses NVFP4 on SM90 as for V4): backend + DeepseekV4AttnBackend, pool DeepSeekV4TokenToKVPool fp8, sparse_prefill_fwd + FlashMLA sparse decode (same sm90 + sparse kernels as vllm), but the indexer runs an fp4 index-K path (_quantize_fp4_indexer / _fp4_index_logits / + _store_fp4_index_k_cache) instead of vllm''s DeepGEMM fp8 mqa_logits, engram as _engram_hash/_gather/_gate. Before + the dummy fix the same run OOMed allocating the full engram table (183 GiB fp16 temp in initialize_dummy_weights). + sglang 0.5.20 (newest release) and trtllm 1.3.0rc29 do not support V4.1 (no model/config class), unchanged. + Probe-side compatibility for sglang main (probe_sglang.py): ServerArgs is a msgspec Struct, bench_one_batch moved + to sglang.benchmark.one_batch, config initializers take no args after publish(server_args); the generator still + renders `--cuda-graph-bs` which main splits into -decode/-prefill (one-off queue patch, argv stale vs plan by + design). COLLECTOR GAPS (unchanged): dsv4 case declarations for compress ratio 2/1 source/consumer layers and the + candidate filter; engram op; SDK composition by source/consumer layer counts.' + applies_to: + - deepseek-ai/DeepSeek-V4.1-Flash + - nvidia/DeepSeek-V4.1-Flash-NVFP4 + - baerquant/Inkling + - framework:vllm-0.30.0 + - framework:sglang-main-2026-09-30 + - instrument:dummies + - platform:sm90 + dsv41_generator_profile_block_size_2026_10_01: + statement: 'FOLLOW-UP to dsv41_dummy_cut_and_vllm_block_size_2026_10_01 (owner request: fix the generator default + instead of carrying a harness override). facts/models.yaml now has a `deepseek-v4.1` profile (match + deepseek-ai/DeepSeek-V4.1*, nvidia/DeepSeek-V4.1*; arch DeepseekV41ForCausalLM; model_type deepseek_v41) whose + vllm defaults are trust-remote-code + block-size 64 + no-enable-flashinfer-autotune and whose sglang defaults + equal V4''s; the `deepseek-v4` globs narrowed from `DeepSeek-V4*` to `DeepSeek-V4-*` so V4.1 no longer inherits + V4''s block-size 256. V4''s gb200-specific defaults (deep_gemm_mega_moe, compilation-config) were NOT copied — + they are validated for V4 only. Re-probed on H20 with the native render (block-size 64 for BOTH V4.1 artifacts, + no harness cli_extra_args): vllm 0.30.0 rendered + fp8-KV PASS x4; sglang main (nightly 0930) PASS x4 (the + nvidia artifact now also renders trust-remote-code / disable-flashinfer-autotune). Matrix vllm-0.30.0 back to + 110/1/13 with deepseek-ai/DeepSeek-V4.1-Flash a plain pass. Unit test + tests/unit/generator/test_model_profile_deepseek_v41.py pins both the glob split and the 64/256 defaults. + Still open on the generator side: nvidia/DeepSeek-V4-*-NVFP4 artifacts of V4 itself are matched by no profile + (deliberately left alone — changing it would alter V4-NVFP4 golden renders that currently pass).' + applies_to: + - deepseek-ai/DeepSeek-V4.1-Flash + - nvidia/DeepSeek-V4.1-Flash-NVFP4 + - module:generator + - framework:vllm-0.30.0 + - platform:sm90 + hopper_trtllm_upgrade_rc23_to_rc29_2026_10_01: + statement: 'HOPPER UPGRADE trtllm 1.3.0rc23 -> 1.3.0rc29 (H20/sm90, 2026-10-01; latest nvcr tag, no 1.3.0 GA / 1.4 rc yet). + IDENTITY: 224 runs (112 repos x rendered + fp8-KV), matrix results/sm90/trtllm-1.3.0rc29.yaml 70 pass / 6 pass+custom / 48 + fail vs rc23 69/0/55. Verdict flips: nvidia/Qwen3.6-27B-NVFP4 and nvidia/Qwen3.6-35B-A3B-NVFP4 fail->pass (rc29 serves them + through the W4A16 NVFP4 lane on SM90: W4A16NVFP4CutlassFusedMoE); nvidia/Qwen3.8-27B-NVFP4 pass->fail ("CutlassFp4GemmRunner + Arch unsupported for CUTLASS FP4 GEMM": rc29 routes its dense FP4 GEMM to the W4A4 runner, SM100+ only — the two Qwen artifacts + swapped lanes between rcs; platform floor on Hopper). DeepSeek-V4 family UNLOCKED on Hopper: rc29 drops the "not supported on + pre-blackwell" assert and instead requires kv_cache_config.dtype=fp8_ds_mla (attention/backends/sparse/deepseek_v4/ + cache_manager.py:346); the generator renders auto, so the dsv4 family trtllm customization gained params.agg.kv_cache_dtype= + fp8_ds_mla next to tokens_per_block=128 (A/B facts/upgrade_rc29/ab_dsv4_kv_*.json) -> 6 pass+custom (V4-Pro/Flash/-0813/-0731, + sgl-project FP8 x2), attention DeepseekV4 sparse (dsa_nsa). The harness fp8-KV variant (--kv-dtype fp8) cannot spell + fp8_ds_mla and stays a recorded fail for DSV4 on trtllm. NVFP4 DSV4 (and every other NVFP4 MoE artifact) now fails uniformly + at MoE resolution ("CutlassFusedMoE NVFP4 only supports SM100/103/107/120/121") — the same Hopper floor rc23 reported as + per-model GEMM/tactic errors. Other cause changes: moonshotai/Kimi-K3 + Kimi-K3-NVFP4 "No module named fla" (the rc29 image + ships kimi_kda importing the external fla package; rc23 failed earlier on quant parsing); google/gemma-4-26B-A4B hard-aborts + before the probe can dump ("Assertion failed: Head size 512 is not supported by MMHA", attentionOp.cpp:2954, + facts/upgrade_rc29/gemma4_bf16_repro.log) — Hopper floor for head_dim 512 decode; MiniMax-M3 x3 now fail with "requires + sparse_attention_config.algorithm=minimax_m3": the generator never rendered that key (rc23 template emits only + sparse_attention_config.implementation from msa_sparse_implementation; the targets customization params.agg. + sparse_attention_config={...} is an unknown params.agg key and is silently dropped — rc23 masked this with an earlier + "M3-sparse not wired" error). A/B with the key hand-added to the engine yaml (facts/upgrade_rc29/ab_m3/) gets past the + KV-cache manager and fails in MiniMaxM3SparseRuntimeBackend.forward ("requires k, v, k_cache, v_cache, idx_k_cache and + out_cache_loc") -> framework gap on Hopper either way; the generator-side key is still a gap to close when M3 works. + Identity on pass->pass rows unchanged except Qwen3-Coder-480B-A35B moe label gaining its full backend set (richer trace). + COLLECTOR ON rc29 (the audit the harness exists for): (1) tensorrt_llm._torch.attention_backend.* moved to + _torch.attention.backends.* and _torch.modules.fused_moe to _torch.moe.fused_moe (old root is a deprecation shim without + submodules) -> seven collectors import-shimmed (path only, same classes/kernels; commit f49e76c3); (2) collect_gdn''s extra FLA + row fused_recurrent_gated_delta_rule no longer exists (fla/fused_recurrent.py keeps only *_update; serving''s decode is + fused_sigmoid_gating_delta_rule_update at rc23:670 / rc29:706, never the removed kernel) -> the row is omitted with a printed + note, serving-dispatch rows unchanged; (3) collect_moe fp8_block passes dtype=torch.float8_e4m3fn to create_moe and rc29''s + MoE resolution now refuses it ("CutlassFusedMoE FP8_BLOCK_SCALES requires torch.bfloat16") — the moe gate was already DIVERGED + at rc23 (routing kernel drift) and is now INVALID-CAPTURE; serving passes the bf16 model dtype with the quant config, so the + collector''s fp8_block construction needs a serving-parity fix (collect_moe.py:313-319) — OPEN. GATES (captures re-taken in the + rc29 image as facts/pathdiff/opcov_trt_*@1.3.0rc29.json, components/captures/verdicts_trt_rc29.sh): 12/13 ALIGNED + (attn ctx/gen bf16+fp8-KV, dsa ctx/gen, mla ctx/gen bf16+fp8-KV, gemm fp8_block, gdn ctx), moe_fp8block invalid-capture. Lesson + recorded in the launcher: re-take captures from the opcov json `cmd` (exact --case-prefix cells); the first pass reused an older + --case-filter launcher and the rc29 plan order handed it head_dim-64 / sliding-window cells that looked like divergences. + 11 gate families remain undeclared for trtllm (8 dsv4_* now gradable since DSV4 loads on Hopper, msa x2, encoder/mla_bmm/mhc/ + compute_scale per workflow_check) — backfill scope is the owner''s call. RETESTS: all 11 trtllm customizations still_needed + (results/retests/sm90/trtllm-1.3.0rc29.yaml; bare DSV4 renders fail on tokens_per_block 32, NVFP4/M3 fail identically with or + without). Collector pins (framework_manifest trtllm default 1.3.0rc20) untouched — owner decision.' + applies_to: + - framework:trtllm-1.3.0rc29 + - platform:sm90 + - deepseek-ai/DeepSeek-V4-Flash + - deepseek-ai/DeepSeek-V4-Pro + - nvidia/Qwen3.8-27B-NVFP4 + - MiniMaxAI/MiniMax-M3 + - google/gemma-4-26B-A4B + - instrument:collect_moe + - instrument:collect_gdn + hopper_sglang_upgrade_0516_to_0521_2026_10_01: + statement: 'HOPPER UPGRADE sglang 0.5.16 -> 0.5.21 (H20/sm90, 2026-10-01; v0.5.21 is the newest release tag). GENERATOR + FIRST: 0.5.20 turned ServerArgs into a msgspec Struct whose argparse flags derive from field names; the 0.5.16-0.5.19 + aliases `--cuda-graph-bs` / `--cuda-graph-max-bs` are gone and 0.5.21 rejects the rendered `--cuda-graph-bs` as ambiguous, + so every golden boot failed in seconds. New versioned template cli_args.0.5.20.j2 renders the *-decode spellings (commit + 4da917b7, boundary verified from the v0.5.20 server_args.py; prior templates untouched). IDENTITY: 224 runs, matrix + results/sm90/sglang-0.5.21.yaml 99 pass / 3 pass+custom / 22 fail vs 0.5.16 93/6/25. fail->pass: moonshotai/Kimi-K3, + meta-models/Muse-Glimmer-30B, nvidia/Muse-Glimmer-30B-NVFP4, TokenRhythm/NeoHorse-1-9B. pass->fail: MiniMaxAI/MiniMax-M3 + ("Mixed chunk prefill requires the breakable prefill CUDA graph backend; got tc_piecewise" — the golden renders + --enable-mixed-chunk; A/B adding `--cuda-graph-backend-prefill breakable` PASSES on 0.5.21 (facts/upgrade_rc29/ab_m3_sgl/ + m3_A.json, MiniMaxHybridAttnBackend), dropping --enable-mixed-chunk does not; the generator has no cuda_graph_backend_prefill + parameter and no sglang extra_cli_args passthrough, so this rescue is not expressible from targets — generator gap); + nvidia/MiniMax-M3-NVFP4 (deep_gemm fp8_fp4_gemm_nt "Unknown SF transformation", layout.hpp:57); nvidia/Qwen3.8-27B-NVFP4 + (sglang::apply_fp4_marlin_linear receives a tuple for `input` — modelopt_quant.py:2055, framework bug on the SM90 NVFP4 + dense path). Identity changes on passing rows: Qwen3-VL family attention label FlashAttentionBackend -> HybridAttnBackend + (fa3 prefill underneath, same role); DSA/GLM/DSV4 kv_allocated column now reads the server arg (auto) instead of the pool + dtype — label change only. CUSTOMIZATIONS: 0.5.21 runner=auto now loads fp4-expert DeepSeek-V4/V4.1 on SM90 and selects + FLASHINFER_MXFP4 (trtllm cutlass MoE kernels, Mxfp4FlashinferCutlassMoE) — a different identity from the marlin_moe the + 0.5.16 rescue forced. Per the retest rule the five deepseek-ai V4/V4.1 marlin customizations are DROPPED and the cells + re-probed bare (results/retests/sm90/sglang-0.5.21.yaml; bare raws facts/upgrade_rc29/retest_bare_sgl_*); the three nvidia + NVFP4 artifacts still refuse auto ("Current platform does not support NVFP4 quantization with the selected MoE backend") + and keep marlin (pass+custom). Consequence for the collector: the sglang DSV4 moe row on SM90 must follow + flashinfer_mxfp4, not marlin, once the collector pin moves. GATES: captures re-taken in the v0.5.21 image + (facts/pathdiff/opcov_sgl_*@0.5.21.json, components/captures/verdicts_sgl_0521.sh) are the compat audit the collector + modules (`__compat__ sglang==0.5.14` / 0.5.16) never had: 13/15 INVALID-CAPTURE — pkg_resources absent from the 0.5.21 + image (collect_attn/mla/mla_bmm/computescale import it for the version string; 8 captures), get_device_memory_capacity + moved out of sglang.srt.server_args (dsv4/dsa context modules), initialize_moe_config() now takes no args after + publish(server_args) (dsa generation module; probe_sglang already carries the publish-then-initialize pattern), + sglang.srt.distributed.parallel_state_wrapper gone (msa, mla, attn); gdn ctx/gen capture but DIVERGE (0.5.21 serving runs + chunk_gated_delta_rule_fwd_kernel_h_blockdim64 / ChunkGatedDeltaRuleFunction for prefill and _causal_conv1d_update_kernel + for decode where the collector lane differs). The short-isl DSA gate (s512) is parked: its serving side is a 0.5.16 raw. + 15 gate families undeclared (dsv4 x5, msa, encoder, mla_bmm, mhc, compute_scale, kda, ...). OWNER DECISIONS: (1) move + the sglang collector pin to 0.5.21 (a full module audit: imports, moe init contract, gdn lanes, dsv4 moe backend) or keep + sglang path gates at 0.5.16; (2) M3 generator parameter for the prefill cuda-graph backend; (3) gate backfill scope. + framework_manifest sglang pins (0.5.14 default) untouched.' + applies_to: + - framework:sglang-0.5.21 + - platform:sm90 + - module:generator + - MiniMaxAI/MiniMax-M3 + - nvidia/Qwen3.8-27B-NVFP4 + - deepseek-ai/DeepSeek-V4-Flash + - deepseek-ai/DeepSeek-V4.1-Flash + - instrument:collect_attn + - instrument:collect_mla_module + hopper_sglang_collector_port_0514_to_0521_2026_10_01: + statement: 'SGLANG COLLECTOR PORT 0.5.14 -> 0.5.21 + GATE BACKFILL (H20/sm90, 2026-10-01; owner decision 2026-10-01 ''跟到0.5.21''). + PIN: framework_manifest sglang default 0.5.14 -> 0.5.21 (lmsysorg/sglang:v0.5.21, index digest b1259f3e..., cu130 tag + same digest); the kda (kimi-k3 branch 0.5.16) and msa (v0.5.16) family pins are retired because 0.5.21 ships kimi_linear.py + / minimax_m3.py and both families smoke+gate clean on it; the three Qwen3.8 model pins stay at 0.5.17 (the 0.5.21 serving + matrix fails Qwen3.8-2.4T-A95B/-FP8 at boot). Every sglang module now declares __compat__ with 0.5.21 (validated set + = {0.5.14, 0.5.21} resp. {0.5.14, 0.5.17, 0.5.21}; intermediate releases excluded, unvalidated). COMPAT LAYER (collector/sglang/runtime_compat.py): + ServerArgs is a msgspec Struct (publish(server_args, role, ranks) then zero-arg initialize_moe/fp8/fp4 config; derived + args behind arg_groups resolving_view -> resolved_arg(); read-only after resolution -> get_context().override(source, + ...) for the CSA-context token cap), bootstrap.init_parallel_runtime + init_layer_runtime before ModelRunner, ensure_offline_runtime_published() + for mock backends, kv_index_translator / decode_num_tokens_per_req / kv_cache_dtype_str on the mock runners, import + relocations (jit_kernel -> kernels.ops, fp8_kernel -> kernels.ops.quantization, flash_mla -> sgl_kernel.flash_mla, communicator + -> layer_boundary.adapters, configurator._profile_available_bytes via model_runner.init_kv_cache_configurator()), native + DeepseekV4Config, FlashMLA sched-meta API detection. KERNEL-LANE FORKS (serving dispatch changed, all probed from the + framework source, not the version string): (1) GDN prefill on SM90/SM100 = FlashInfer chunk_gated_delta_rule (+gdn_prefill_qkv_prepare_fwd) + when gdn_backend.flashinfer_gdn_prefill_default would pick it (fp32 state on SM90, 128/128 head dims; Triton FLA otherwise) + and GDN decode conv = fused_qkvzba_causal_conv1d_update_contiguous (SGLANG_ENABLE_GDN_DECODE_FUSED_PROJ_CONV default + on) with the framework''s own eligibility fallback; gdn_ctx/gdn_gen gates went DIVERGED -> ALIGNED. (2) KDA decode for + lower-bound layers (Kimi-K3) = plain fused_sigmoid_gating_delta_rule_update (kda_backend.py:737-786), not the packed + kernel; kda_gen gate DIVERGED -> ALIGNED. (3) DSV4 paged-MQA schedule = sglang JIT get_paged_mqa_logits_metadata (paged_mqa_metadata_tiny_kernel, + SGLANG_OPT_USE_JIT_INDEXER_METADATA default on) instead of deep_gemm''s sched kernel; (4) DSV4 topk calib entry points + renamed topk_transform_512{,_v2} -> topk_transform_paged{,_v2} (same v1/v2 lanes). MEMORY PLANS: DSV4 HCA on 0.5.21 + prices a ratio-128 full token at ~64 B so the framework-derived pool on an empty H20 reaches ~2e9 tokens and the int64 + full_to_swa_index_mapping (not priced by DSV4PoolConfigurator) OOMs -> HCA now caps max_total_tokens to the planned + shapes (sglang clamps to min(profiled, cap)); MSA plan-time filter now prices every loaded layer (2) -> the three 33.6M-token + decode cells are dropped at plan time instead of failing the worker capacity check. Conv int32 guards in gdn/kda stay: + 0.5.21 casts offsets to int64 yet the >=2**31 cells still fault on H20. SMOKE: op_smoke 30 ops in the v0.5.21 image + -> 28 OK; gdn (106/112) and kda (91/112) fail only their own guard cells (classified raises). GATES sglang 0.5.21: 28/28 + declared gates ALIGNED (15 prior + 13 backfilled: dsv4 csa/hca ctx/gen, csa/hca attn, paged_mqa_logits, glm5 dsa_attn/mqa/topk, + kda ctx/gen, mla_bmm); encoder_attn parked as EXPLAINED (facts/pathdiff/explained_sgl_0521): the sglang plan probe is + text-only, its Qwen3-VL record has no vision-tower kernels (vllm proves this family from its multimodal profile_run) + -> the one remaining undeclared sglang family until the probe exercises the vision tower. GATES trtllm rc29 backfill + (decision 3): dsv4 csa/hca gen ALIGNED after two collector ports — DeepseekV4Attention(aux_stream_dict=...) and the + Hopper fp8_ds_mla pool (KvCacheConfig.dtype + extra_attrs[''kv_cache_dtype''], which also keeps the module off the fused + Q-FP8/KV-norm prologue serving never runs on SM90); dsv4 csa/hca ctx ALIGNED once the module learned the pool dtype + (without extra_attrs[''kv_cache_dtype''] it took the fused prologue and mla_rope_inplace rejected the strided raw latent) + -> trtllm rc29 16/17 gates aligned, 4 DSV4 gates backfilled. Known gaps carried: trtllm moe fp8_block gate still DIVERGED + on routing (RenormalizeMoeRoutingMethod vs DeepseekV3Gate); trtllm collector manifest default still 1.3.0rc20 (only + the harness targets pin moved to rc29); test_fullnode real_checkpoint tests fail on HEAD before this work (logger None + at collect.py:1279), unrelated. Evidence: facts/upgrade_rc29/sgl_smoke_r{1..9}, sgl_gates_0521.txt, trt_gates_rc29_backfill.txt, + launch_sgl_caps_backfill.sh.' + applies_to: + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - platform:sm90 + - module:collector + - instrument:collect_gdn + - instrument:collect_kda + - instrument:collect_dsv4_attn + - instrument:deepseekv4_sparse_modules + - instrument:collect_msa_module + - instrument:collect_attn + - instrument:collect_mla_bmm + - instrument:runtime_compat + - instrument:trtllm.collect_dsv4_attn + - Qwen/Qwen3-VL-2B-Instruct + - moonshotai/Kimi-K3 + - sgl-project/DeepSeek-V4-Flash-FP8 + hopper_trtllm_collector_pin_rc20_to_rc29_2026_10_02: + statement: 'TRTLLM COLLECTOR PIN 1.3.0rc20 -> 1.3.0rc29 + MOE ROUTING PARITY (H20/sm90, 2026-10-02; owner asked whether + the two leftovers of the sglang port could be closed). PIN: framework_manifest trtllm default rc20 -> rc29 (digest 124da3f1...); + the rc23 moe / sparse_attention / msa family pins are retired, their history kept in the manifest comment (rc20 moe + data still flows through the declared reuse.yaml channel; the rc23 mhc evidence exception keeps its own scope). Every + trtllm module already declared __compat__ >= rc19/20/21. SMOKE: op_smoke over all 23 trtllm registry ops in the rc29 + image -> 21 OK; moe fails only on its first plan case by the collector''s own classified raise (fp8_block needs 128-aligned + shards: hidden 1024 / inter 2688 at tp16) and mamba2 only on its bs=64 x 32768 cells (CUDA OOM, memory) — both pre-existing + behaviours, not rc29 drift. GATES rc29: 17/17 ALIGNED. The moe_fp8block DeepSeek-V3 gate had been DIVERGED since the + rc29 audit because collect_moe routed every non-min-latency case through RenormalizeMoeRoutingMethod (customMoeRoutingKernel) + while serving builds DeepSeek-V3-family / GLM / Nemotron-H MoE from DeepseekV3Gate.routing_method (noaux_tc grouped + top-k with the correction bias -> deepseek_v3_topk_kernel; modeling_deepseekv3.py:979-990, modeling_glm.py:273-285, + modeling_nemotron_h.py:236). The collector now reads the model row''s declared routing (cases/models/*_cases.yaml sglang_moe_routing_method_type=DeepSeekV3 + with num_expert_group / topk_group / routed_scaling_factor — framework-neutral checkpoint facts despite the prefix) + and builds the same gate routing with the resolved moe_backend; softmax-top-k models keep Renormalize. The router GEMM + (dsv3_router_gemm_op) stays outside the op as before. OPEN on trtllm: 7 undeclared gate families (encoder_attn, gdn_gen, + mamba2, mla_bmm, msa ctx/gen, ...) — gate_declared todo, same backfill mechanics as the sglang round. TEST HYGIENE: + collector/collect.py''s module logger defaulted to None until main() ran setup_logging, so in-process callers of the + checkpoint/provenance paths (fullnode finalization, unit tests run in isolation) crashed on logger.info; it now defaults + to logging.getLogger(''collector'') with a configured flag — mechanism-adjacent plumbing only, no behaviour change for + the CLI (main still installs the configured logger once). test_fullnode 8/8 standalone; the 30 test_collect_provenance_writer + failures are environment-only (no torch in this venv) and identical on the pre-change HEAD.' + applies_to: + - framework:trtllm-1.3.0rc29 + - platform:sm90 + - module:collector + - instrument:trtllm.collect_moe + - instrument:collect.py + - deepseek-ai/DeepSeek-V3 + - zai-org/GLM-5 + - sgl-project/DeepSeek-V4-Flash-FP8 + hopper_vision_evidence_and_tp4_probe_routes_2026_10_02: + statement: 'VISION-TOWER EVIDENCE + TP>1 PROBE ROUTES (H20/sm90, 2026-10-02; owner: ''3.8这个probe走tp4试试, vision 那个加一下, + trtllm也看一下''). VISION: the plan probes are text-only, so VL checkpoints never ran their encoder on sglang/trtllm and + the encoder_attn gate had no serving evidence there (vLLM gets it from its own multimodal profile_run). Added probes/vision_{sglang,trtllm}.py: + load the dummy like the text probe, push one synthetic 448x448 image through the checkpoint''s own image processor (preprocessor_config.json + travels with the dummy dir) into model.visual (sglang qwen3_vl.py:1459) / model.mm_encoder.visual (trtllm modeling_qwen3vl.py:1206, + encoder owned by the normal worker :1358) under torch.profiler, and write archive/raw/.vision.json; build_records + merges it as the profile_run phase. It is a SEPARATE script on purpose: the probe code hash is part of every run''s + execution fingerprint, so touching probe_.py would have made all 448 text records stale. Qwen3-VL-2B serving + vision attention: sglang = sgl_kernel::fwd (FA3 FlashAttnFwdSm90), trtllm = fmha_v2_flash_attention_bf16 (trtllm_mha). + encoder_attn gates now ALIGNED on both -> sglang 0.5.21 29/29 gates, upgrade_op all_done=True; trtllm rc29 18/18 (6 + families still undeclared: gdn_gen, mamba2, mla_bmm x2, msa ctx/gen). TP>1: targets.topologies had tp8 ''implemented'' + but no record in the archive ever ran at tp>1 and the driver put every run on one GPU. New: topology_policy.tp_required + (repo, backend, tp) replaces tp1 for the listed checkpoints; the driver gives such a run the first tp GPUs of --gpu-list + on one queue (run those plans ALONE, via --only) and hashes the route script in the fingerprint. Routes = the real servers + inside one N-GPU container: probe_sglang_server.py (launch_server with the golden CLI; identity from /get_server_info; + kernels from /start_profile profile_by_stage num_steps=1 activities CPU+GPU -> -TP-0-{EXTEND,DECODE}.trace.json.gz; + never /stop_profile in by-stage mode — manual_stop raises in the scheduler and the gloo barrier kills every rank; with_stack=False + — rank 0 segfaulted in kineto finalizeTrace() with python stacks on the bf16 depth8 run) and probe_trtllm_server.py + (trtllm-serve --tp_size N --extra_llm_api_options + load_format dummy; kernels from /start_profile + num_steps windows 1 and 2 diffed per kernel -> trtllm-trace-*-rank-0.json). What the server routes cannot see is flagged + identity_source: quant_methods / param_dtypes / api span maps (path_diff grades from the device-stream tables regardless). + RESULT Qwen3.8-2.4T-A95B (+FP8) at tp4: sglang 0.5.21 PASS both (server up in ~70 s, attention fa3, GDN prefill = flashinfer + cute-DSL delta rule, fused_moe triton; kv auto and fp8_e4m3 variants), trtllm rc29 PASS both (165-245 s startup, trtllm_mha + fmha_v2 + cutlass GDN decode kernel, fused_moe_oss / deep_gemm fp8 for the FP8 checkpoint). Why tp1 failed: the depth-4 + (trtllm) / depth-8 (sglang) dummies keep all 512 experts (bf16 ~51 GB per MoE layer); 0.5.16 never reached loading (no + model class) and the 0.5.17 manifest pin was a collector-runtime declaration, never a serving validation. Matrices: + sglang 101/4/19, trtllm 72/6/46 (vllm unchanged; vllm tp>1 route not implemented — its probe is the golden run.sh in-process + and would need the same server treatment). Evidence: archive/raw/{b9f90ee85db2,9bce9bf007e4,5f3e9962ae80,6ff23fb0fc31,699fceaf86e2,937771098635,f9b89de0ef48,5d8a431b0ade}.json + (+ .profile/ server logs and traces), archive/raw/{1a0e76dd419d,a1255b13536c}.vision.json, facts/upgrade_rc29/qwen38_tp4*_queue.log. + PITFALL recorded: --emit-queues rewrites archive/queues/gpuN.sh — never emit while a queue is executing from that file + (bash reads it lazily); copy the queue aside first.' + applies_to: + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - platform:sm90 + - module:opharness + - instrument:probe_sglang_server + - instrument:probe_trtllm_server + - instrument:vision_sglang + - instrument:vision_trtllm + - Qwen/Qwen3.8-2.4T-A95B + - Qwen/Qwen3.8-2.4T-A95B-FP8 + - Qwen/Qwen3-VL-2B-Instruct + hopper_trtllm_gate_backfill_complete_and_full_campaign_2026_10_02: + statement: 'TRTLLM GATE BACKFILL COMPLETE + FULL H20 CAMPAIGN STARTED (2026-10-02; owner: ''先把 trtllm 那 3 个 gate 补了，然后开一轮完整采集''). + GATES rc29 now 22/22 ALIGNED: gdn_gen (the earlier verdict script had it parked as ''explained'' because the collector''s + FLA fused_recurrent row was a deliberate fallback lane; on rc29 collect_gdn''s decode lane matches serving''s FlashInfer + decode, so it is a gate again), mla_bmm (cublas bmm pre/post), mamba2 ctx+gen (Nemotron-H-56B). mamba2_ctx first came + back DIVERGED: rc29 Mamba2Mixer defaults TRTLLM_MAMBA_TOKEN_MAJOR_CONV=1 (mamba2_mixer.py:319) and hands causal_conv1d_fn + a channel-last VIEW of the token-major projection plus a separate token-major out (:541-569), which selects causal_conv1d_channellast_fwd_kernel + (causal_conv1d.py:26-60), while the collector ran the channel-major kernel on (batch, dim, seq) inputs. collect_mamba2 + now builds the serving varlen token-major buffer with cu_seqlens / cache_indices / has_initial_state=False whenever + the runtime''s causal_conv1d_fn has the `out` parameter and the env default is on, records kernel_source=causal_conv1d_fn_token_major + (label added to kernel_source_backends.yaml), and keeps the old call for older runtimes; gate ALIGNED after the change. + The 6 msa gate families on trtllm remain undeclared: MiniMax-M3 does not boot on rc29 (MiniMaxM3SparseRuntimeBackend.forward), + so there is no serving record to grade against — framework gap, recorded only. CAMPAIGN: the first full collection on + the new pins runs from a frozen snapshot staging_1002 (ea76d9ab + the ba56d55d collect_mamba2/kernel_source_backends + overlay) via staging_1002/run_op.sh ( [filters]: --model-cases-full, own run + checkpoint dir, --resume + --resume-retry-failed, images = manifest defaults, HF offline + shared jitcache). Partition facts/campaign_1002/PARTITION.txt: + sglang on GPUs 2-4, trtllm on GPUs 5-7, three op chains per backend (light ops first; dsa/dsv4 modules on their own + GPU). Excluded: trtllm moe_ep (WideEP runtime, separate container), DSV4.1 (no collector ops), sglang dsv4 *_attn/paged_mqa + (not in the sm90 plan). First plan sizes: gemm 104636 (sglang) / 104294 (trtllm) cases, mla_context 3080, sglang dsa_context_module + 264 (plan-time memory filter) vs trtllm 7808. Attempt 1 of the launcher died on an empty --case-filter argument (printf + ''%q'' on an empty array yields ''''); fixed before any case ran. Logs facts/campaign_1002/logs/, run dirs staging_1002/runs/_/ + (perf evidence stays in the workspace, never committed).' + applies_to: + - framework:trtllm-1.3.0rc29 + - framework:sglang-0.5.21 + - platform:sm90 + - module:collector + - instrument:trtllm.collect_mamba2 + - nvidia/Nemotron-H-56B-Base-8K + - Qwen/Qwen3.5-0.8B + - deepseek-ai/DeepSeek-V3 + hopper_full_campaign_1002_complete_2026_10_04: + statement: 'FULL H20 CAMPAIGN COMPLETE (2026-10-02 14:3x -> 2026-10-04 04:54, ~37.5 h wall on 6 GPUs; sglang 0.5.21 + trtllm 1.3.0rc29; + everything except DSV4.1 and trtllm msa, which the owner skipped this round). 967,892 perf rows / 925,704 unique keys across 48 + (backend, op) tables; 65,405 failed tasks, all recorded. Duplicate keys (sglang moe 22,627, trtllm moe 8,991) come from two mid-run + splits done without seeding the new run dir''s checkpoint — the re-measurements agree (median 0.3%, p90 3.6%) and dedupe at merge; + lesson: seed ckpt//..json with every prior dir''s done+failed before splitting (case strings carry quant and model but no + single substring spans both, so model/dist filters only work on seeded checkpoints). Failure families: (a) classified grid/gate + raises — sglang attention_context 17,239 fa3 FP8-KV x bf16 guard; sglang moe 17,577 Triton fp8_block divisibility, 3,726 w4a16_mxfp4 + version gate; sglang/trtllm gemm 666/1,110 fp8 16-byte alignment on tiny N; trtllm attention 4,182+4,148 Gemma no-SM90-kernel; + trtllm moe 658 TP-shard alignment; gdn/kda own >=2**31 conv guard cells. (b) collector gaps found by this run — sglang moe + "bias is required for non-softmax routing" 7,533 = Llama-4 Maverick/Scout every quant mode; sglang moe "parallel name tp_group + has not been written" 3,807 = Nemotron-3 Ultra-550B/Nano-30B BF16 on 0.5.21. (c) real crashes awaiting the serving-parity audit — + trtllm attention_context 500 IMA (96 heads x kv 1/2/4/8 x head_dim 256 and 96/96/128, seq 16384, TRTLLM backend), trtllm + attention_generation 512 launch failure + 162 IMA (96 heads, kv 4, head_dim 64, fp8 KV), trtllm mla_context_module 41 IMA + (DeepSeek-V3). (d) trtllm dsv4 csa/hca ctx 47+47 OOM on DeepSeek-V4-Pro largest-prefix cells with the snapshot''s free*0.5 pool — + re-run with the max_tokens cap (d8d9aafb). Evidence: workspace facts/campaign_1002/{REPORT.md,SUMMARY.txt,PARTITION.txt,logs/}, + perf rows in staging_1002/runs/ (never committed).' + applies_to: + - framework:trtllm-1.3.0rc29 + - framework:sglang-0.5.21 + - platform:sm90 + - module:collector + - meta-llama/Llama-4-Maverick-17B-128E-Instruct + - meta-llama/Llama-4-Scout-17B-16E-Instruct + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 + - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + - deepseek-ai/DeepSeek-V3 + - sgl-project/DeepSeek-V4-Pro-FP8 + hopper_campaign_1002_followup_fixes_2026_10_04: + statement: 'CAMPAIGN FOLLOW-UP CLEARED (2026-10-04, owner: ''一步步清掉剩余的问题''; WideEP explicitly not upgraded). (1) sglang moe: + Llama-4 Maverick/Scout now build the router like serving (Llama4MoE.custom_routing_function) and the 0.5.21 flashinfer_cutlass + runner gets the single-process get_parallel stand-in; re-collected 10,530 Llama-4 rows (0 failures) and 8,531 Nemotron Ultra/Nano + BF16 rows (243 remaining = the collector''s own high-TP alignment guard). (2) weight-only lanes re-verified on 0.5.21 for + SM89/90 only (int4_wo -> Marlin, compressed_tensors.py:838-877; w4a16_mxfp4 -> Triton, mxfp4.py:1423-1430, the 0.5.17 gpt-oss + override is gone from arg_groups/overrides.py); 4,455 Kimi int4_wo + 5,751 gpt-oss mxfp4 rows re-collected, SM100/103 + declarations stay closed. (3) trtllm attention_context 500 IMA / attention_generation 512+162 / mla_context_module 41: + the context FMHA faults exactly when (batch-1)*seq*heads*head_dim*2 >= 2**31 (fitted cell-by-cell on three head grids, + reproduced b16 s3072 h96 d256 vs b8 s6144 passing), decode fails for fp8 KV at GQA ratio 24/32 — FIXME(kernel-limit) at + both invocation sites, cases remain classified. (4) trtllm dsv4 Pro OOM cells all pass with the max_tokens cap (floor + lowered 4M -> 2M after Pro reproduced the free*0.5 pool). (5) WideEP same-pin assumption dropped: moe_ep leaves the default + trtllm plans and resolves on wideep_trtllm 1.3.0rc20 alone; mixed stock+moe_ep requests fail-close like sglang. New + classified family: Nemotron-3-Super-120B bf16 tp32 (81 cells) asserts in sglang''s JIT activation.cuh:186 on the shard width. + Evidence: workspace facts/campaign_1002/REPORT.md, merged tables facts/campaign_1002/merged/ (sglang moe 109,781 unique rows).' + applies_to: + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - platform:sm90 + - module:collector + - meta-llama/Llama-4-Maverick-17B-128E-Instruct + - meta-llama/Llama-4-Scout-17B-16E-Instruct + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 + - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 + - openai/gpt-oss-120b + - moonshotai/Kimi-K2.5 + - sgl-project/DeepSeek-V4-Pro-FP8 + + sm100_vllm_0300_collector_verdict_2026_10_03: + statement: 'COLLECTOR VERDICT FOR THE vllm 0.30.0 UPGRADE ON sm100 (B200 umbriel-b200-049, 8 GPUs, PR head d8d9aafb, 2026-10-03; + the Hopper port of the same pin was already graded on H20). Question: do the 0.30 collectors need sm100-specific fixes? + Answer: mostly NO, two YES. All 35 vllm gate captures were re-taken in the 0.30.0 image on B200 (32 from components/captures + + mla_ctx_fp8/mla_gen_fp8, which are sm120 FLOOR lines but real gates on sm100) and graded against B200 serving records + (224 runs = 112 rendered + 112 fp8-KV): 26 ALIGNED — gemm bf16/fp8_block (+m1), moe fp8_block/bf16/mxfp4 (all on trtllm_gen_moe), + mla_ctx bf16/fp8, mla_bmm, dsa_ctx_fp8 s512 (vs a B200 isl-512 raw) and s4096, dsa_gen_fp8, msa ctx/gen, dsv4 x6, gdn + x2, kda x2, attn_ctx. NEEDS FIX (1): utils.kv_block_size branches by SM (sm100: attention 16 / mla 32, owner decision + 2026-09-30) — it fixed attn_ctx (P64->P16, aligned) but REGRESSED dsa_ctx_bf16 and dsa_ctx_bf16_s4096 (ALIGNED on 2026-09-29 + with page 64): with page 32 the collector''s bf16 DSA cell selects the TRT-LLM FMHA sparse path (fmhaSm100fKernel_QkvBfloat16...HQk576HV512HVPerCta256PagedKvDenseStaticTok...) + while serving runs vLLM cute-DSL FA sm100 + the indexer. Serving picks the page per (backend, kv dtype, head shape), not + per SM — ask the framework per case (get_supported_kernel_block_sizes / cache_config.block_size) as the B200 handoff proposed; + keep AIS_KV_BLOCK_SIZE as the A/B hook. NEEDS DECISION (2): the remaining attention drifts are tile variants of the same + cubin family — attn_gen MultiCtasKvCga vs MultiCtasKv, mla_gen_fp8/bf16 HVPerCta128 vs 256 — chosen by the FMHA''s kv-length/batch + heuristic; whether perf rows must match at tile level is a granularity call. gemm_fp8 is the sm120 governed exception + (cutlass sm100 GEMM template instance); encoder_attn needs an encoder-scoped op-hint (the decoder''s paged-causal FMHA + spans are swept in; the ViT path is fa_cute_sm100 on both sides); compute_scale/mhc are the known explained deviations. + No 0.30 API drift showed up on sm100 beyond what the Hopper port already fixed.' + applies_to: + - platform:sm100 + - framework:vllm-0.30.0 + - module:collector + - instrument:vllm.collect_attn + - instrument:vllm.collect_mla_module + - instrument:path_diff + + sm100_vllm_0300_identity_2026_10_03: + statement: 'IDENTITY MATRIX sm100 / vllm 0.30.0 on PR head d8d9aafb (results/sm100/vllm-0.30.0.yaml): 106 pass / 1 pass+custom + / 17 fail over 124 rows. 12 fails share sm90''s causes (11 generator rejects: Qwen3.8-Flash-Next x3, MiMo-V2.6 x2, Inkling + x4, GLM-5.3-Flash x2; Qwen3.8-2.4T-A95B capacity). sm100-specific: (a) deepseek-ai/DeepSeek-V4.1-Flash and nvidia/DeepSeek-V4.1-Flash-NVFP4 + ''ValueError: No common block size for 64'' — the generator fact facts/models.yaml V4.1 block-size 64 (added 2026-10-01 + for sm90/sm120) has no accepting backend on sm100; A/B: the same rendered run.sh without --block-size 64 boots and runs + (DeepseekV41ForCausalLM) — a platform-blind generator fact; (b) nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 ''no attention-ish + module classes'' — targets.yaml declares variant_overrides vllm: layers_block_type_moe for it, so the probe ran a moe-only + cut; (c) Qwen/Qwen3.8-2.4T-A95B-FP8 OOM on 178 GiB (passes on the 143 GB H20 — dummy residency differs); (d) fp8-KV variant: + Kimi-K3 and Kimi-K3-NVFP4 require --attention-config ''{"use_prefill_query_quantization": true}'' (framework assertion). + DSV4 family all pass (64-aligned budgets). Identity flips vs sm90 at the same pin: dense attention FA3 -> flashinfer TRT-LLM + FMHA cubins (prefill fmhaSm100a*, decode fmhaSm100f*; page 16 Llama/Qwen, 32 Nemotron and MLA fp8, 64 Gemma-4), DSV3.2 + FlashAttnMLASparse -> FlashMLASparse, NVFP4 DSA FlashMLASparse -> FlashInferMLASparse, M3 SparseTritonImpl -> SparseMSAImpl, + every NVFP4/MXFP4/FP8 MoE marlin/triton_fused_moe -> trtllm_gen_moe. Customization retest (results/retests/sm100/vllm-0.30.0.yaml): + Nemotron-H gpu-memory-utilization 0.6 still_needed (bare default OOMs in the profile run on a clean 178 GiB B200, 2026-09-29 + evidence). Perf-data collection was started on the same box (26 shards, gemm bf16 37,518 / gemm fp8 37,518 / attention_context + 54,468 rows finished with 0 failed at the 2.6 h mark; dsa_context_module plans 261,504 cases and is the long tail) — data + stays in the workspace per the owner''s 2026-10-04 rule, never in this PR.' + applies_to: + - platform:sm100 + - framework:vllm-0.30.0 + - deepseek-ai/DeepSeek-V4.1-Flash + - nvidia/DeepSeek-V4.1-Flash-NVFP4 + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 + - moonshotai/Kimi-K3 + - nvidia/Nemotron-H-56B-Base-8K + + harness_issues_sm100_third_run_2026_10_03: + statement: 'HARNESS ISSUES FROM THE THIRD BLACKWELL RUN (sm100, 2026-10-03), each reproduced: (1) tools/perf_database/collect_campaign.py + image_digest() uses ''{{index .RepoDigests 0}}'' — a locally built collect image has no RepoDigests, so `run` refuses + the image `build-image` just produced (''image not present''); fixed here with ''{{if .RepoDigests}}...{{end}}|{{.Id}}''. + (2) kernel_taxonomy_sm100.yaml did not exist — a new SM starts with an empty vocabulary and every gate reads no-collector-signal; + seeded here from the sm90 file + the B300-validated Blackwell block + fa_cute_sm100 + the cutlass sm100 GEMM symbol (enable_sm100_to_sm120 + guard). (3) dummy_overrides.tokenizer_from only resolves to a ROSTER repo''s dummy dir, not to configs/aux_files/; + gated meta-llama repos on a token-less box (bundled config fallback ships no tokenizer) need a public sibling IN the roster, + and Llama-4''s unsloth mirrors had to be copied into the meta-llama aux dirs by hand (the bundled Llama-4 configs also + lack vision_config). (4) verdicts_vllm_0300.sh is AIS_SM-parametric but the dsa_ctx_fp8 SERVING_RAW is the H20 path facts/reprobe_nopc/04cea4da0eff_isl512.json + (pre-platform run id); on another SM it needs a symlink or a derived path. (5) FLOOR_SM=sm120 lines are real gates on + sm100 — a capture list derived from the `run` lines alone misses mla_ctx_fp8/mla_gen_fp8. (6) path_diff --capture only + fails at the final open() when the output dir is missing — the whole capture runs for nothing; mkdir -p first. (7) the + collect image has no git, so collection_meta.collector_ref records ''unknown''. (8) a generator fact (V4.1 block-size + 64) and a targets variant_override (Nemotron-Ultra-NVFP4 moe-only cut) are platform-blind inputs that turn pass cells + into fails on a new SM — same class as the cli_extra_args lesson from B300. (9) throughput facts for planning: identity + 224 runs on 7 B200 ~2 h (first wave JIT-heavy); collection shard plan must split dsa_context_module (261,504 cases) and + gemm fp8_block (DeepGEMM JIT) across GPUs — one shard per op leaves 7 GPUs idle behind a 20 h tail.' + applies_to: + - platform:sm100 + - instrument:collect_campaign + - instrument:path_diff + - instrument:dummies + - instrument:fetch_inputs + - instrument:probe_driver + + + sm100_kv_block_size_framework_query_2026_10_04: + statement: 'COLLECTOR FIX (owner go-ahead 2026-10-04 ''如果有好的机制，可以实现一下''): the KV page is now ASKED from the framework per + case instead of tabled per SM. Root cause found while implementing: collector/vllm/utils.create_vllm_config built CacheConfig(block_size=16|64), + and pydantic marks any explicit value user_specified_block_size=True — the selector (v1/attention/selector.py:131-135) + then filters backends by that page exactly as --block-size would, which serving never does unless the user asked. On sm100 + that constraint is what moved the collector onto the wrong backends: at page 16 FLASHMLA_SPARSE/FLASHINFER_MLA_SPARSE + are rejected (''No valid attention backend''), at page 32 the bf16 DSA cell landed on the TRT-LLM FMHA sparse cubin. Mechanism, + mirroring serving: (1) CacheConfig(block_size=None) -> default 16, NOT user-specified, selection unconstrained (AttentionSelectorConfig.block_size=None + in collect_attn); (2) after the backend is known (selector result in collect_attn; the built module''s layer in collect_mla_module._create_kv_cache_and_metadata), + utils.framework_kv_block_size(backend_cls) = select_common_block_size(backend.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE), + [backend]) — the two calls platforms/interface.py update_block_size_for_backend and v1/worker/utils.prepare_kernel_block_sizes + make; cache_config.block_size is set to it before the KV cache/metadata are built. AIS_KV_BLOCK_SIZE stays the A/B hook; + utils.kv_block_size (the per-SM table) is no longer called. VALIDATION on umbriel-b200-040 (B200, vllm 0.30.0 image, 10 + captures, families compared with the new attention tile equivalence against the 2026-10-03 sm100 serving kernels): attn_ctx/attn_gen + P16 (FLASHINFER) = serving; dsa_ctx_bf16_s4096 back on FLASHMLA_SPARSE with the FA-cute prefill kernel = serving (DIVERGED + under the table); mla_gen_fp8 FLASHINFER_MLA P32 = serving; mla_gen_bf16 P32; dsa fp8 ctx/gen FLASHINFER_MLA_SPARSE = + serving identity. Open: dsa_ctx_bf16 at s=512 runs the sparse kernel on sm100 while the isl-4096 serving record shows + the FA-cute prefill — a length-conditional path, so on sm100 that gate needs an isl-512 serving raw like dsa_ctx_fp8 already + has. Also in this change: path_diff _EQUIV_RULES gains an ''attention'' rule that keys TRT-LLM FMHA cubins by (sm, qkv + dtypes, head dims, paged-kv kind, PAGE) and drops the tile/CTA tail (HVPerCta, MultiCtasKv[Cga], VarSeqQKv, Persistent/Static/Grouped) + — owner decision 2026-10-04 to relax tile variants; a page mismatch or a backend mismatch still reads diverged.' + applies_to: + - platform:sm100 + - platform:sm90 + - framework:vllm-0.30.0 + - instrument:vllm.collect_attn + - instrument:vllm.collect_mla_module + - instrument:vllm.utils + - instrument:path_diff + + sm100_moe_backend_fact_rejected_2026_10_04: + statement: 'GENERATOR FACT BUG (B200 / sm100, 2026-10-04, first sglang 0.5.21 + trtllm 1.3.0rc29 identity probes on Blackwell; + Hopper never saw it): generator/facts/hardware.yaml b200 and gb200 profiles carry `moe_backend: {trtllm: WIDEEP, sglang: + deepep_moe}` and facts/apply.py apply_moe_backend fills it on EVERY MoE deployment (fill-if-absent, keyed on hardware only + — no EP size, node count or framework version). Neither value exists at the pinned versions: trtllm rc29 MoeConfig.backend + is Literal[AUTO, CUTLASS, CUTEDSL, CUTEDSL_FC12, TRTLLM, DEEPGEMM, DENSEGEMM, VANILLA, TRITON, MARLIN, MEGAMOE_*] (WIDEEP is a + 1.0-era literal; the rc29 template comment still lists it) -> pydantic literal_error before boot; sglang 0.5.21 + --moe-runner-backend rejects deepep_moe (argparse invalid choice). 156 trtllm + 150 sglang MoE renders (every MoE cell x kv + variant) died before any kernel ran; the as-rendered matrices read 29/0/95 (trtllm) and 32/3/89 (sglang). The h100/h200/h20 + profiles say `trtllm: CUTLASS` and leave sglang unset, which rc29/0.5.21 accept — that is why the sm90 campaign passed and the + fact was never exercised. The fact''s premise ("wrong MoE backend on Blackwell is a startup crash") is not what the A/B shows + (facts/ab_trt_moe_backend/, 12 probes on an empty B200): with moe_config dropped (rc29 AUTO) DeepSeek-R1, Qwen3-30B-A3B, + gpt-oss-120b and Qwen3-235B-A22B-NVFP4 all boot and run MoE kernels; explicit CUTLASS fails DeepSeek-R1 (CutlassFusedMoE + FP8_BLOCK_SCALES only supports SM90/SM120, got SM100); DEEPGEMM passes. Wide-EP in current TRT-LLM is an EP-size / + load_balancer / attention-dp deployment choice layered on CUTLASS|TRTLLM|DEEPGEMM, not a per-layer MoE backend literal. + FIX (local commit 8b96aa45b, owner integrates): the two Blackwell values are commented out with this rationale; profiles + keep `moe_backend:` so apply_moe_backend sees None. IDENTITY UNDER THE FRAMEWORKS'' OWN CHOICE: the same 306 cells were + re-probed with the fact removed from the rendered config (B200 workspace archive/retests/moe_auto/{raw,*.engine.yaml}; + table results/retests/sm100/.moe_auto.yaml): sglang 72 pass / 3 fail (Kimi-K3 bf16 `multi_ctas_kv_counter_buffer is + only supported when a trtllm-gen runner is selected`; Kimi-K3-NVFP4 and Qwen3.8-2.4T-NVFP4 `FusedMoE has no attribute + g1_scale_c`), runners FLASHINFER_TRTLLM 47 / AUTO 11 / FLASHINFER_MXFP4 7 / FLASHINFER_TRTLLM_ROUTED 3 / MARLIN 2 / TRITON 1 + / DEEP_GEMM 1, kernel families trtllm_gen_moe 57 / triton_fused_moe 12 / marlin 2; trtllm 61 pass / 17 fail (8 DSV4 — see + sm100_trtllm_dsv4_facts_unservable_2026_10_04; MiniMax-M3 x3 and Kimi-K2.5 quant parser gaps; MiMo-V2-Flash arch; + V4.1-Flash x2 transformers too old in the rc29 image; Kimi-K3 x2 no `fla` module), modules UnquantizedFusedMoE 23 / + NVFP4CutlassFusedMoE 20 / DeepSeekFP8BlockScalesFusedMoE 13 / FP8QDQ 2 / W4A8MXFP4MXFP8TRTLLMGen 2 / W4A16NVFP4Cutlass 1. + For sglang the fixed-fact render equals the override command byte-for-byte (150/150), so those raws ARE the cells'' evidence + (installed with the v2 fingerprint sidecars); for trtllm the fixed-fact render is NOT the override — see the next finding. + HARNESS DEFECTS SURFACED: (1) probe_sglang.py dies in argparse and writes no raw, so 150 cells graded "no raw (crashed before + dump)" with no error text (probe_trtllm records the pydantic error); the probe should dump an error record on argparse + failure. (2) _fail_cause had no rule for a rendered value the framework refuses — the trtllm cells read "framework gap"; + rule added: "generator fact rejected by framework" (literal_error | invalid choice). (3) render_golden''s cache key is + `git rev-parse HEAD`: an UNCOMMITTED generator edit re-renders nothing (the first fixed-fact emit silently served the WIDEEP + goldens) — key on a dirty-tree hash or on the facts files'' content. (4) tp4 jobs are emitted into the first GPU''s queue + (comment says run them alone) and were twice run from a stale script here; sglang''s tp memory-balance check also aborts a + tp4 launch when ANY of its GPUs is busy (Qwen3.8-2.4T-FP8: "memory capacity is unbalanced") — tp>1 cells need a + whole-group reservation in the queue plan. Re-run on umbriel-b200-021 with the group idle: sglang Qwen3.8-2.4T-A95B-FP8 tp4 PASSES (both kv variants; the results/sm100 matrix cell ''framework gap'' is that busy-GPU artifact), sglang Qwen3.8-2.4T-A95B bf16 tp4 flaked once (RemoteDisconnected) and passed on retry, trtllm Qwen3.8-2.4T-A95B-FP8 tp4 fails in the executor worker (CUTLASS fp8-block class; the matrix cell ''no raw'' is this).' + applies_to: + - platform:sm100 + - framework:trtllm-1.3.0rc29 + - framework:sglang-0.5.21 + - instrument:generator.facts.hardware + - instrument:generator.facts.apply + - instrument:probe_driver + - instrument:probes.probe_sglang + collector_impact: + - {op: moe, action: guard, ref: '95085922 collect_moe._raise_if_unverified_moe_lane: int4_wo opened on SM100/103 for 0.5.21 — the moe_auto retest here (Kimi-K2.5 FLASHINFER_TRTLLM / trtllm_gen_moe) is the evidence; lane_evidence.yaml now grades it'} + - {op: none, action: pipeline, ref: 'generator hardware fact (moe_backend deepep_moe) is the generator plane; collector cases unaffected'} + sm100_trtllm_moe_default_cutlass_2026_10_04: + statement: 'GENERATOR DEFAULT, PLATFORM-BLIND (B200, trtllm 1.3.0rc29, 2026-10-04): with the Blackwell moe_backend fact gone, + the rc29 engine template (config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2:16) still renders + `moe_config.backend: {{ moe_config.backend | default(''CUTLASS'') }}` unconditionally, so every sm100 trtllm MoE cell is + served with an explicit CUTLASS — the same literal the Hopper profiles pin. On SM100 rc29''s CutlassFusedMoE refuses + FP8_BLOCK_SCALES ("only supports SM90/SM120, got SM100"), so all fp8-block MoE checkpoints fail under the generator''s + render while AUTO serves them (A/B above): DeepSeek-R1, DeepSeek-V3/V3.1/V3.2, DeepSeek-V4 x6 (also page-blocked), + GLM-5/5.1/5.2/5.3 FP8, MiniMax-M2/M2.5/M2.7, Qwen3-30B-A3B-FP8, Qwen3-235B-A22B-FP8, Kimi-K2.x FP8 — 19 cells in + results/sm100/trtllm-1.3.0rc29.yaml carry the new cause "rendered MoE backend cannot serve this layer on this SM" (the + rule also covers the sm90 "no MoE implementation can serve this layer" NVFP4-on-Hopper cells, previously "framework gap"). + The matrix is the as-rendered truth: 78 pass / 0 custom / 46 fail; the AUTO table (results/retests/sm100/ + trtllm-1.3.0rc29.moe_auto.yaml) is what the framework picks when allowed. OWNER DECISION: the trtllm MoE default must be + platform-aware — AUTO on sm100 (rc29 resolves fp8-block to TrtllmGen/DeepGEMM, NVFP4 to NVFP4Cutlass/TrtllmGen, bf16 to + Cutlass fused_moe_oss), CUTLASS stays correct on sm90. Collector note: label_kernels files rc29''s cutlass-based + `fused_moe_oss::run_global` under triton_fused_moe (taxonomy rule shared with sm90) — a vocabulary nit, not a verdict.' + applies_to: + - platform:sm100 + - framework:trtllm-1.3.0rc29 + - instrument:generator.backend_templates.trtllm + - instrument:probe_driver + sm100_trtllm_dsv4_facts_unservable_2026_10_04: + statement: 'GENERATOR DSV4 FACTS ARE A HOPPER RECIPE (B200, trtllm 1.3.0rc29, 2026-10-04): the 8 DeepSeek-V4 trtllm cells + (Flash/Pro x {bf16, FP8, NVFP4, 0731/0813}) render `kv_cache_config: {dtype: fp8_ds_mla, tokens_per_block: 128}`. On SM100 + rc29 first rejects the page ("DeepSeek-V4 fp8_ds_mla KV cache requires tokens_per_block=256 on SM100, got 128"; cause + "generator DSV4 kv facts unservable on this SM"); with page 256 (archive/retests/tpb256{,_auto}/) the fp8_ds_mla SPARSE + GENERATION step has no FMHA library at all — TLLM_LOG_LEVEL=DEBUG (facts/dsv4_fmha_debug.log): CuTe DSL MLA "sparse + attention (DeepSeekV4Params) is not supported", FlashInfer trtllm-gen "sparse attention", Triton custom-mask "only context", + PrimTS "block-sparse inputs required" -> "No TRT-LLM attention FMHA library supports this request" (attention/backends/ + trtllm.py:2323). The servable sm100 configuration is kv dtype auto (bf16, DeepseekV4CacheManager) + tokens_per_block 256 + + MoE AUTO (facts/ab_dsv4_kvauto/, archive/retests/dsv4_sm100/): DeepSeek-V4-Flash-FP8 runs fmhaSm100f + QkvBfloat16 H512 PagedKvDense DynamicTokenSparse cubins, applyMLARopeAndAssignQKV, topKPerRowDecode, trtllm-gen routing. + GATES: the four trtllm dsv4 gates graded against that raw (--serving-raw) are DIVERGED by KV dtype — the collector''s DSV4 + lane follows the generator fact (E4m3 cubins, dsa_nsa indexer) while the only configuration rc29 can serve on sm100 is bf16 + — so on sm100 either the generator DSV4 facts move to {auto, 256} and the collector grows a bf16 DSV4 lane, or rc29 ships the + fp8_ds_mla sparse-gen kernel; until then DSV4/trtllm on sm100 is a platform gap, not a collector verdict. sglang''s DSV4 + cells and all 7 sglang dsv4 gates are ALIGNED on sm100 (0.5.21 defaults DSV4 kv to fp8_e4m3 itself).' + applies_to: + - platform:sm100 + - framework:trtllm-1.3.0rc29 + - instrument:generator.facts.models + - instrument:trtllm.dsv4 + - instrument:path_diff + sm100_sglang_trtllm_gate_verdicts_2026_10_04: + statement: 'COLLECTOR VERDICT, sglang 0.5.21 + trtllm 1.3.0rc29 on sm100 (B200, 2026-10-04; captures facts/pathdiff/ + opcov_@.json, verdicts results/pathdiff/sm100/-/). sglang: 14 ALIGNED (dsa fp8 ctx s4096 / gen, dsv4 + csa|hca ctx|gen|attn + paged_mqa_logits, glm5 mqa_logits + topk, gdn_ctx, mla_bmm_gen), 3 DIVERGED, 11 MISSING CAPTURE, 2 no + serving record (kda: Kimi-K3 does not boot on 0.5.21/sm100). trtllm: 7 ALIGNED (gdn ctx/gen, mamba2 ctx/gen, mla_ctx_fp8, + mla_gen_fp8, mla_bmm_gen — the three DeepSeek-V3 ones against the AUTO override raw, the render itself cannot serve + DeepSeek-V3 on sm100), 7 DIVERGED (attn fp8 ctx/gen, dsv4 x4, encoder), 8 MISSING CAPTURE. All 32 captures with a checked-in + recipe were taken (18 + 14); the trt_kv recipes hardcode the H20 layout `/work/ais/...` and needed the checkout mounted + there. BLACKWELL-SPECIFIC DIVERGENCES (all ALIGNED on sm90): (a) trtllm attn_ctx_fp8 / attn_gen_fp8 — collector/trtllm/ + collect_attn.py:171 hardcodes `tokens_per_block = 64` while the generator renders 32 for every trtllm deployment + (kv_cache_config.tokens_per_block: 32 in 208/224 sm100 renders); the sm100 cubin names carry the page (fmhaSm100a…P64 vs + …P32) so the mismatch is visible here and invisible in sm90 names — same class as the vllm block_size finding: the + collector should take the page from the deployment facts, not a constant. FIXED + VALIDATED (umbriel-b200-021, 2026-10-04): collect_attn now asks tensorrt_llm.llmapi.KvCacheConfig() for its default page (32; AIS_TRTLLM_TOKENS_PER_BLOCK is the A/B hook); re-captured in the rc29 image the collector runs fmhaSm100aKernel_QkvE4m3OBfloat16H128PagedKvCausalP32VarSeqQ256Kv128PersistentContext (ctx) and fmhaSm100fKernel_QkvE4m3OBfloat16H128PagedKvCausalP32MultiCtasKvCgaVarSeqQ8Kv256 (gen) = the serving cubins; the AIS_TRTLLM_TOKENS_PER_BLOCK=64 A/B reproduces the P64 divergence (facts/pathdiff/opcov_trt_attn_{ctx,gen}_fp8kv@1.3.0rc29{,.p64}.json in the 021 workspace). Re-graded on umbriel-b200-021 against fresh Llama-3.1-8B fp8-KV serving records (scoped re-probe, plan_*_scoped.json): attn_ctx_fp8 and attn_gen_fp8 are now ALIGNED (results/pathdiff/sm100/trtllm-1.3.0rc29/attn_*_fp8_Llama-3.1-8B.json), so trtllm reads 9 ALIGNED / 5 DIVERGED (dsv4 x4 by KV dtype, encoder). (b) sglang dsa_gen_bf16 — 0.5.21 + arg_groups/overrides.py resolves DSA kv auto to fp8_e4m3 when SM major >= 10 (bfloat16 on Hopper), so serving runs + QkvE4m3 cubins and the bf16 DSA lane is a non-case under the default deployment on Blackwell (dsa fp8 gates ALIGNED); + grade bf16 DSA gates on sm100 only against an explicit --kv-cache-dtype bf16 record — DONE on umbriel-b200-021 (archive/retests/dsa_bf16/, verdicts results/pathdiff/sm100/sglang-0.5.21/*.kvbf16.json): with --kv-cache-dtype bfloat16 both dsa_gen_bf16_DeepSeek-V3.2 and glm5_dsa_attn_GLM-5 are ALIGNED, so the collector''s bf16 lanes are right and the default-deployment divergence is purely the Blackwell kv-auto->fp8 rule. (c) sglang glm5_dsa_attn — + collector/sglang/glm5_dsa_sparse_modules.py runs the dsa_attn kernel as bf16 FlashMLA sm100 sparse_attn_fwd while serving + (kv auto -> fp8) runs TRT-LLM FMHA token-sparse cubins (fmhaSm100f QkvE4m3 HQk576HV512 …TokenSparse); the glm5 module + needs the fp8-KV lane the dsv32 module already has (the bf16 lane itself is ALIGNED under explicit bf16 KV). (d) encoder_attn both frameworks — op-hint issue as on vllm (owner + decision pending). OTHER REAL sm100 FAILURES in the matrices: MiMo-7B / MiMo-V2-Flash arch not registered (rc29), Muse- + Glimmer x2 and DeepSeek-V4.1-Flash x2 transformers too old in the rc29 image, MiniMax-M3 x3 sparse cache None, Kimi-K2.5 + quant parser, Kimi-K3 x2 no `fla` (trtllm); Kimi-K3 bf16 multi_ctas_kv_counter_buffer, Kimi-K3-NVFP4 / Qwen3.8-2.4T-NVFP4 + g1_scale_c (sglang). The 12 generator-rejects are identical to sm90 (roster repos aiconfigurator does not know).' + applies_to: + - platform:sm100 + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - instrument:trtllm.collect_attn + - instrument:sglang.glm5_dsa_sparse_modules + - instrument:captures.trt_kv + - instrument:path_diff + collector_impact: + - {op: none, action: none, ref: 'verdict plane only (gates/captures); collector grids unchanged'} + sm100_precision_smoke_2026_10_04: + statement: 'PRECISION-COVERING OP SMOKE on B200 (umbriel-b200-021, 2026-10-04; driver facts/smoke/op_smoke_sweep.py in the + workspace: every registered sglang (30) and trtllm (23) collector op, cases grouped by the precision tokens in the case text, + ONE case per group, op_smoke''s executor contract, one container process per op — a shared process poisons later ops through + torch default-device state and produced false "can''t convert cuda:0 tensor to numpy" errors; collect_moe / collect_mamba2 + exit with EXIT_CODE_RESTART by design after writing their rows, which a sweep must not read as failure). FOUND AND FIXED + (sm100 lanes Hopper never exercised): (1) trtllm collect_gemm and collect_mla_module declared NVFP4 with group_size 128; + 1.3.0rc29 NVFP4LinearMethod.resolve_scaling_vector_size requires 16 — gemm nvfp4 and every nvfp4 group of the mla/dsa module + collectors failed to construct (nvfp4 is a platform floor on sm90, so the lane never ran there); now 16, smoke OK. + (2) sglang collect_mla and collect_attn MockModelRunner lacked max_running_requests / is_draft_worker / + prefill_attention_backend_str / decode_attention_backend_str / model — attributes the 0.5.21 TRTLLMMLABackend and + TRTLLMHAAttnBackend read, and those are the backends sglang picks on SM100 (fa3/flashinfer on Hopper never asked); added, + mla_generation and attention_* smoke OK. (3) trtllm collect_dsv4_attn pinned the fp8 KV pool + kv_cache_quant_algo FP8 on + every SM; on SM100 rc29 cannot serve fp8_ds_mla (finding sm100_trtllm_dsv4_facts_unservable_2026_10_04) and kv auto resolves + to the bf16 DeepseekV4CacheManager at the generator''s page 128 (facts/ab_dsv4_p128: boots, fmhaSm100f QkvBfloat16 + DynamicTokenSparse) — the collector lane is now bf16-KV on SM100 (_serving_kv_cache_dtype; AIS_DSV4_KV_CACHE_DTYPE hook) + and all four trtllm dsv4 gates read ALIGNED against that serving raw. (4) GLM-5 DSA attention on sglang: the fp8-KV lane is + NOT a new code path — collect_mla_module ''dsa'' with kv fp8 already serves GLM-5 (GlmMoeDsaForCausalLM); recipes + sgl_glm5_dsa_attn_fp8_{ctx,gen} capture it: attention family == sm100 serving (fmhaSm100f QkvE4m3 HQk576HV512 TokenSparse), + residual drift is one tiny cutlass_80 fp16 wmma 32x32 GEMM the module lane issues and serving does not (gemm family) — + owner call whether the glm5 gate should move to the module lane on Blackwell. FIXED LATER (umbriel-b200-028, 2026-10-05): (5) sglang mla_context on + SM100: the collector times the prefill kernel under CUDA-graph capture; flashinfer.prefill.trtllm_ragged_attention_deepseek + (prefill.py:5088) refuses capture without q_seq_lens_cpu/kv_seq_lens_cpu and sglang''s trtllm_mla _run_prefill_kernel does + not pass them (serving never graph-captures prefill) — collect_mla.benchmark_layer now times the SM100 trtllm_mla PREFILL lane eagerly (use_cuda_graph=False; decode and + the Hopper backends keep CUDA-graph timing); mla_context bf16/fp8 smoke OK on B200. (6) sglang moe mxfp4 lanes: the guard is + extended to 0.5.21 on SM100/103 for w4a16_mxfp4 AND w4a8_mxfp4_mxfp8, verified from the identity records (gpt-oss-120b/20b + and the five deepseek-ai/DeepSeek-V4* fp4-experts checkpoints serve with Mxfp4MoEMethod -> FLASHINFER_MXFP4, kernel family + trtllm_gen_moe, under the default render and under AUTO; the marlin prescription is the SM90 workaround); the leaf check + that only knew Hopper''s cutlass_sm90 now maps the SM100 leaf trtllm_sm100 -> kernel_source sglang_mxfp4_flashinfer_trtllm_moe + (and SM120 flashinfer_cutlass_sm120). Smoke on B200: gpt-oss-120b w4a16_mxfp4 and DeepSeek-V4-Flash w4a8_mxfp4_mxfp8 both run + and record sglang_mxfp4_flashinfer_trtllm_moe = the serving lane. Still open: (7) the kda/gdn int32 guard at very large + shapes (owner: leave it; only batch>=16 x seq>=8k is affected). (7) sglang kda: the collector''s own int32-overflow guard + (causal_conv1d_triton token-offset) rejects every Kimi-K3 case at the sweep''s shapes — not sm100-specific; the same guard + trips 6/112 sglang gdn context sub-cases (batch 16-64 x seq 8192-32768, total_tokens*6144 >= 2**31) and the op''s + strict-completeness check then fails the whole sweep (gate cell gdn_ctx Qwen3.5-0.8B itself ALIGNED). FINAL TABLE: + trtllm 58/59 cells pass (9 = designed restart exits; 1 stale shared-process entry for a group that no longer exists on + sm100), sglang 47/52 (open: mla_context x2, moe mxfp4 guard, kda guard, gdn guard); vLLM 0.30.0 (umbriel-b200-037, 23 ops, 55 cells across bf16/fp8/fp8_block/nvfp4 groups) 54/55 — the one failure is kda: the Kimi-K3 KDA sweep OOMs at batch 64 x seq 16384/32768 (24-144 GiB allocations on a 178 GiB B200; no shape budget filter in that collector) and the strict-completeness check fails the whole op; the gate cell is unaffected. BOOTSTRAP ROBUSTNESS (037): the sequential roster fetch hit Hub HTTPErrors on 30 repos and fell back to the bundled SDK copies — for MiniMaxAI/MiniMax-M3 the bundled copy is the FLAT text-only config (model_type minimax_m3) while the Hub repo is the VL wrapper (minimax_m3_vl, layers under text_config), so dummies.py''s M3 adapter died with KeyError text_config and took the whole 256-variant run with it; a sibling hf_quant fallback without exclude_modules did the same (KeyError). Both adapters now tolerate the other layout / log the gap instead of aborting; the fetch should retry on 429/HTTPError before falling back, and dummies.py should isolate per-repo failures. Everything else + (attention, gemm bf16/fp8/fp8_block, mla ctx/gen, mla_bmm, dsa/dsv4 modules across bf16/fp8/fp8_block/nvfp4, gdn, mamba2, + mhc, compute_scale, encoder, moe fp8/fp8_block/nvfp4/bf16, msa) runs on B200. Per-cell table: facts/smoke/sweep_*.json + + summarize_smoke.py in the 021 workspace.' + applies_to: + - platform:sm100 + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - instrument:trtllm.collect_gemm + - instrument:trtllm.collect_mla_module + - instrument:trtllm.collect_dsv4_attn + - instrument:sglang.collect_mla + - instrument:sglang.collect_attn + - instrument:sglang.collect_mla_module + - instrument:op_smoke + collector_impact: + - {op: mla_context, action: collector_fix, ref: '8f4f36c (sm100 branch): trtllm_mla prefill timed eagerly on SM100'} + - {op: moe, action: guard, ref: '8f4f36c: w4a16_mxfp4 + w4a8_mxfp4_mxfp8 opened on SM100/103 from these identity records'} + - {op: gemm, action: collector_fix, ref: '8f4f36c: trtllm nvfp4 group_size 16; mock runner attributes for 0.5.21 TRTLLM backends'} + sm100_probe_failures_recheck_2026_10_04: + statement: 'RE-CHECK OF EVERY sm100 PROBE FAILURE / NON-ALIGNED GATE FOR A CONFIG CAUSE (owner ask; umbriel-b200-037, fresh + workspace, Hub configs re-fetched). OURS (generator / harness / dummy): (a) 12 generator rejects per framework — roster repos + aiconfigurator does not know (identical on sm90); (b) trtllm 19 fp8-block MoE cells — rc29 template MoE default CUTLASS + (sm100_trtllm_moe_default_cutlass_2026_10_04); (c) trtllm DSV4 x8 — fp8_ds_mla/page facts (sm100_trtllm_dsv4_facts_unservable_2026_10_04); + (d) trtllm Qwen3.8-2.4T-FP8 tp4 — same CUTLASS class (sm90 pass); (e) encoder_attn DIVERGED on sglang AND trtllm — the serving + record carries 0 ops for Qwen3-VL-2B because the identity probe is text-only and the vision sidecar (probes/vision_.py) + exists for vllm only: harness gap, not a kernel finding; (f) **MiniMax-M3 x3 on trtllm (also failed on sm90, graded "quant parser + gap")** — DUMMY CUT BUG: 1.3.0rc29''s MSA cache manager does not read sparse_attention_freq; without explicit sparse_layer_ids it + allocates INDEX_KEY buffers for range(3, num_layers) (attention/backends/sparse/minimax_m3/cache_manager.py:303) while the model + builds sparse layers from the freq list (modeling_minimaxm3.py:363); our 2-layer "moe_sparse_attn" cut made both layers sparse at + positions the manager treats as dense -> idx_cache None -> "''NoneType'' object has no attribute ''shape''" in write_layer_caches. + The real 60-layer model has exactly 3 dense layers first, so serving never sees it; sglang reads the freq list and passed with the + same dummy. dummies.py variants_m3 now cuts the first three dense layers + the first sparse layer (4 layers): all six MiniMax-M3 trtllm cells (M3 / M3-MXFP8 / M3-NVFP4 x bf16 + fp8 KV) PASS on B200 with the MSA kernels visible + (SparseAttentionForwardSm100, _gqa_sparse_decode_kernel, fmhaSm100f cubins) — the three results/sm100/trtllm-1.3.0rc29.yaml + cells graded ''quant parser gap'' become pass once the matrix is rebuilt with the new dummy (same on sm90). FRAMEWORK (same on sm90 unless noted): (g) Kimi-K2.5 + trtllm — compressed-tensors W4A16 config (input_activations: None, mirrors the Hub config) dereferenced unguarded + (inputs_quant_config["strategy"]); (h) MiMo-7B / MiMo-V2-Flash arch not registered; Muse-Glimmer x2 + DeepSeek-V4.1-Flash x2 need a + newer transformers than the rc29 image; Kimi-K3 x2 trtllm image lacks `fla`; (i) **Kimi-K3 sglang bf16 (sm100 ONLY; sm90 pass)**: + Kimi-K3 has 96 attention heads; flashinfer''s trtllm-gen MLA decode refuses 64 < num_heads_q < 128 (mla/_core.py:3948), the runner + falls to cute-dsl, but sglang 0.5.21''s trtllm_mla backend passes multi_ctas_kv_counter_buffer unconditionally on Blackwell -> + "multi_ctas_kv_counter_buffer is only supported when a trtllm-gen runner is selected" (_core.py:4278). tp1 only: at tp>=2 the + per-rank head count is <=48 and supported — the probe''s single-GPU tp1 geometry is half the story; probe 96-head MLA models at tp2 + on sm100. (j) Kimi-K3-NVFP4 / Qwen3.8-2.4T-NVFP4 sglang "FusedMoE has no attribute g1_scale_c": modelopt_quant''s NVFP4 MoE weight + prep gates the flashinfer-trtllm branch on the GLOBAL get_moe_runner_backend() (modelopt_quant.py:2837) while apply() uses the + layer-cached resolved backend (:3096-3129) — under AUTO the two disagree and g1_scale_c is never created. A/B with an explicit + --moe-runner-backend flashinfer_trtllm clears it (facts/ab_k3nvfp4_runner: the cell then proceeds to the 96-head MLA error above), + so that flag is a valid customization once tp2 is used. BOOTSTRAP: the sequential roster fetch hit Hub HTTPErrors on 30 repos and + fell back to bundled copies (MiniMax-M3 flat vs VL layout; a sibling hf_quant without exclude_modules) — dummies.py now tolerates + both; fetch_inputs should retry before falling back.' + applies_to: + - platform:sm100 + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - instrument:dummies + - instrument:fetch_inputs + - instrument:probes.vision + collector_impact: + - {op: mla_generation, action: guard, ref: '95085922 collect_mla.run_mla: FIXME(kernel-limit) classified raise for trtllm_mla decode with 64 < local heads < 128 on SM100/103 (item (i)); job 469988017 failed exactly the 362 num_heads=96 cases before this landed'} + - {op: none, action: none, ref: 'item (j) g1_scale_c: collector passes the runner explicitly, not reachable from the grid; dummy/fetch fixes are the probe plane'} + sm100_smoke_vs_perfdb_regressions_2026_10_04: + statement: 'REGRESSION CHECK, new pins vs the shipped B200 perf database (umbriel-b200-037, 2026-10-04; every smoke perf row + joined on the full case key — and on the kernel lane where an op writes one row per kernel — against the newest b200_sxm + parquet of the same framework: vllm 0.25.0/0.27.1, sglang 0.5.14/0.5.16/0.5.17, trtllm 1.3.0rc20/rc23; compare_perfdb.py + + compare_perfdb_report.txt in the handoff dir). Calibration: this box runs compute-bound GEMMs 5-10% slower than the database + box (40 bf16 gemm cases, ratio 1.05-1.10), and sub-10 us kernels carry a ~2 us launch offset here (mla_bmm 1-token, dsv4 + paged_mqa_logits, topk calib) — ratios <= 1.15 and tiny-kernel deltas are environment, not framework. FUNCTIONAL: every lane + the database has rows for runs on sm100 at the new pins (final per-cell smoke on 037: vllm 54/55, trtllm 58/58, sglang 51/56 — + the sglang misses are the collector gaps of sm100_precision_smoke_2026_10_04 plus kda/gdn guards; vllm kda OOM). IDENTITY + CHANGES: vllm attention decode kernel_source -> flashinfertrtllmapidecode_trtllm_gen (trtllm-gen decode, x1.22 single cell); + trtllm mamba2 conv -> causal_conv1d_fn_token_major (rc29; collector mirrors it, mamba2 gates ALIGNED; per-case ratios 0.03-2.5 + = a different kernel, not comparable). LATENCY REGRESSIONS (same kernel_source): (1) vllm 0.30 moe nvfp4, DeepSeek shape + (hidden 7168, inter 2048, top-8, 256 experts, tp1 ep1, flashinfer trtllm nvfp4 experts): 8192 tokens 1.86 -> 4.05/4.15/4.13 ms + (x2.2, three repeats) while 6144 (1.57 -> 1.46) and 12288 (2.75 -> 2.74) are unchanged and 16384 is x1.14 — an isolated, + reproducible spike at exactly 8192 tokens (8192 slower than 12288): a bad tactic/tile pick at that token count. (2) vllm 0.30 gdn + context, chunk_gated_delta_rule_flashinfer lane, every small shape (seq <= 256, d_model 1024): ~20 us -> ~40 us (x2.0 across 21 + cases), large shapes x0.9 — a fixed per-call overhead added in the 0.30 lane. (3) sglang 0.5.21 vs 0.5.14 DSA (DeepSeek-V3.2, + bf16 KV): context dense_mha_trtllm_ragged lane median x1.23, up to x2.07 (bs 1, isl 16-64, past_kv 1-1.5k); generation + dsa_indexer_trtllm at bs 1024: 1.3 -> 1.8-2.3 ms (x1.4-1.8); sparse/indexer context lanes x1.04-1.13; GLM-5.2 lanes x1.0-1.12 + — REPEATED (two more full sweeps each): generation bs 1024 is x1.37 / x1.39 / x1.39 (systematic ~x1.4; the x1.77 was a single-shot + outlier); context dense_mha_trtllm_ragged lane median x1.21 / x1.23 / x1.23 with max x1.35-1.47 (systematic ~x1.2, i.e. ~x1.1 + after the 5-10% box offset — borderline); sparse/indexer context lanes x1.02-1.05 (environment). NOT regressions: trtllm bf16 512-expert moe_torch_flow_nongated x1.3-1.7 + at 4-16 tokens (a torch reference flow, noisy), sglang dsv4 paged_mqa_logits / topk_calib x1.1-1.2 (3-60 us kernels). + FASTER: vllm dsa modules x0.57 (0.30 vs 0.24), trtllm msa context x0.30 (rc29 vs rc23), sglang msa context x0.71, sglang + mla_bmm / mla gen x0.7, vllm gemm fp8_block 128 -> 111 ms. Everything else within +-25%.' + applies_to: + - platform:sm100 + - framework:vllm-0.30.0 + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - instrument:op_smoke + - instrument:perf_database + collector_impact: + - {op: none, action: none, ref: 'latency regressions are data for owner review, no collector grid change'} + sm89_vllm_0300_probe_2026_10_04: + statement: 'FIRST sm89 (L40, 8 x 44.4 GiB usable, CC 8.9, driver 595.58) identity campaign, vllm 0.30.0 (image digest + 8a69ffad015f), generator = this branch, system l40s (needs the new l40s hardware profile: without it resolve_facts raised + ''Unknown hardware profile l40s'' and the pipeline swallowed it, the sm120 failure class). Matrix results/sm89/vllm-0.30.0.yaml: + 67 pass / 4 pass+custom / 53 fail (sm90: 110/1/13, sm120: 98/7/19). Fails, all caused: 17 capacity (44 GiB: Qwen3.5-122B/397B + depth-4 cuts, Nemotron-3-Ultra, Llama-4 Scout/Maverick - the Llama-4 failure was a dummy-staging gap (no preprocessor_config.json, + gated repo) fixed by the mirror files and is now capacity), 14 PLATFORM GAP no sparse-MLA backend: every DSA checkpoint + (DeepSeek-V3.2, GLM-5/5.1/5.2/5.3 incl. NVFP4) dies at attention-backend selection with use_sparse=True for kv auto + AND fp8 (the catch-all used to file these as the config cause ''ckpt-forced fp8-KV'': the exception line was truncated + before use_sparse), 12 generator rejects (same SDK gaps as sm90/sm120), 7 DeepSeek-V4: DeepGEMM hyperconnection.hpp:56 + ''Unsupported architecture'' at model init, 1 Kimi-K3-NVFP4 (_KimiGDNMergedColumnParallelLinear has no orig_dtype, same + on sm120). The 4 NVFP4 MLA checkpoints (DeepSeek-V3.1 / Kimi-K2.5 / K2.6 / K2.7-Code NVFP4) fail the SAME way as sm120 + - vLLM resolves modelopt NVFP4 to fp8 KV and TRITON_MLA fp8 decode needs 102400 B smem vs the 101376 B opt-in limit + that Ada shares with consumer Blackwell (Required 102400 / Hardware limit 101376); A/B: --kv-cache-dtype bfloat16 boots + (raws 6da3d656dea1, ee9f7f075eb4, dc45ccaead4c, 3924e9cb23ea), the fp8-KV variants stay red (platform floor). The sm120 + customization was extended to sms [sm120, sm89] on that evidence (results/retests/sm89/vllm-0.30.0.yaml). Identity on + Ada: dense attention FlashAttention-2 / FlashInfer (no FA3), MLA TRITON_MLA, MoE marlin/triton_fused_moe, bf16 GEMM + cuBLAS ampere_* kernels. Gates: 13 real, ALL ALIGNED (gemm bf16 + m1, fp8, moe bf16/mxfp4, mla ctx/gen/bmm, gdn ctx/gen, + attn ctx/gen, encoder, msa ctx/gen); 18 declared as sm89 platform floors with the cause in the verdict file; compute_scale + stays the explained governed approximation as on sm90. workflow_check upgrade_op vllm 0.30.0 sm89: 6/7 steps green, + failures_root_caused needs this entry.' + applies_to: + - platform:sm89 + - framework:vllm-0.30.0 + - nvidia/DeepSeek-V3.1-NVFP4 + - nvidia/Kimi-K2.5-NVFP4 + - nvidia/Kimi-K2.6-NVFP4 + - nvidia/Kimi-K2.7-Code-NVFP4 + - deepseek-ai/DeepSeek-V3.2 + - zai-org/GLM-5 + - nvidia/Kimi-K3-NVFP4 + - Qwen/Qwen3.5-122B-A10B + - Qwen/Qwen3.5-397B-A17B + - nvidia/Qwen3.5-122B-A10B-NVFP4 + - nvidia/Qwen3.5-397B-A17B-NVFP4 + - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8 + - meta-llama/Llama-4-Scout-17B-16E-Instruct + - meta-llama/Llama-4-Maverick-17B-128E-Instruct + - instrument:probe_driver + sm89_sglang_0521_probe_2026_10_04: + statement: 'sm89 (L40) identity campaign, sglang 0.5.21 (digest b1259f3ea3). results/sm89/sglang-0.5.21.yaml: 65 pass + / 0 custom / 59 fail (sm90: 101/4/19). Fails: 14+7 capacity (7 = ''Loaded weights leave no GPU memory for the KV cache'' + at the planned mem-fraction: MiniMax-M3, GLM-5, Qwen3.5-122B, nvidia/Qwen3.5-397B-A17B-NVFP4 ...), 12 generator rejects, + 14+3 platform floors (sparse attention ''Sparse Attention Forward Kernel is only supported on SM90/SM100''; DeepSeek-V4 + sparse decode: DeepGEMM attention.hpp:496 asserts arch_major in {9,10,12} at CUDA-graph capture; MXFP dense GEMM / FP4 + quantization have no SM89 backend: flashinfer ''Invalid backend: 89''), 5 FRAMEWORK DEFECT: flashinfer 0.6.18''s cutlass + fused-MoE JIT does not compile for sm_89 (''tensorrt_llm::kernels::cutlass_kernels has no member Fp4Type'', ninja build + of fused_moe_89) - the Nemotron-3 family (Nano/Super/Ultra/Lightning), reproduced from the raw; 2 Qwen3.8-2.4T cells + died in sglang''s tp memory-balance check because the queue puts tp>1 cells on ONE GPU''s queue without reserving the + group (harness defect, rerun alone). Gates: 5 ALIGNED of the 6 real ones (attn ctx/gen, attn_ctx_hd512_gemma-4, gdn + ctx/gen), 23 declared sm89 floors (sparse, capacity for the V3-bound MLA/MSA/KDA/GLM-5 gates, DeepGEMM arch). encoder_attn_Qwen3-VL-2B + was DIVERGED for lack of evidence (the text-only probe never ran the vision tower, so the profile_run phase the gate + asks for was empty); CLOSED in the review pass: probe_sglang now runs one synthetic image (grid 1x32x32 patches, HF + pixel_values layout) through model.visual under the device profiler and records it as the profile_run_kernels table + (same table vLLM''s own profile_run yields) - serving shows Triton `_fwd_kernel` x24 (= depth) for the vision attention, + the collector''s triton lane matches, gate ALIGNED. Every sglang/trtllm raw was re-probed afterwards (the probe file + is part of the execution fingerprint). sm90''s earlier aligned verdict for this gate rests on a record whose provenance + this checkout cannot establish; re-grade it with the new probe when the H20 box is next used. COLLECTOR defects found + by the smoke/gates that Hopper never reached (fa3/trtllm_mha paths don''t read them): MockModelRunner lacked prefill/decode_attention_backend_str + (flashinfer lane), the vision Triton kernel moved to sglang.kernels.ops.attention.prefill_attention, TritonAttnBackend + probes mambaish_config(model_config) so the MLA mock needs hf_config / linear_attn_registry_result, and the Gemma4 / + Llama4 case maps had no sm89 entry so the collector measured flashinfer while serving runs triton (arg_groups/model_overrides/gemma4.py, + llama4.py). All fixed with source citations and re-verified on hardware.' + applies_to: + - platform:sm89 + - framework:sglang-0.5.21 + - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 + - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 + - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 + - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16 + - Qwen/Qwen3.8-2.4T-A95B + - Qwen/Qwen3.8-2.4T-A95B-FP8 + - MiniMaxAI/MiniMax-M3 + - zai-org/GLM-5 + - instrument:collect_attn + - instrument:collect_mla + - instrument:collect_attn_encoder + sm89_trtllm_rc29_probe_2026_10_04: + statement: 'sm89 (L40) identity campaign, trtllm 1.3.0rc29 (digest 124da3f10b73). results/sm89/trtllm-1.3.0rc29.yaml: + 47 pass / 7 pass+custom / 70 fail (sm90: see results/sm90). HEADLINE: 26 cells (fp8-block 13, nvfp4 20 minus the 7 customized, + mxfp4 2, mxfp8 1) die at model init with ''no MoE implementation can serve this layer'': the rc29 template and typed + builder default moe_config.backend to CUTLASS and CutlassFusedMoE refuses FP8_BLOCK_SCALES below SM90/SM120 and NVFP4 + below SM100 - the same generator defect the B200 session found on sm100 (platform-blind CUTLASS default; hardware profile + `moe_backend` is keyed on hardware only). A/B on an empty L40 across the explicit backends (facts/ab_trt_moe_sm89, 12+ + probes): AUTO resolves to CUTLASS (same error, so a hardware-wide AUTO fact - tried and reverted - changes nothing); + TritonFusedMoE is SM90-only; Marlin implements nvfp4/w4a16_nvfp4 only (quant_unsupported for fp8_block, bf16, mxfp4); + VanillaMoE gets past selection and dies in CUDA-graph capture (cudaErrorStreamCaptureInvalidated). Re-verified in the + review pass (clean box, same evening): AUTO raws f883c2769dd6_AUTO (Qwen3-30B-A3B-FP8), 8610141a333f_AUTO (DeepSeek-R1), + 1ae319d4353f_AUTO (Nemotron-3-Nano-NVFP4), 56890dc997ec_AUTO (gpt-oss-20b MXFP4) all report ''requested CUTLASS'' and + the CUTLASS SM reason; DEEPGEMM, which serves fp8-block MoE on sm100, is turned down on Ada with ''DeepGemmFusedMoE + requires SM100, SM103, or SM107, got SM89'' (raws f883c2769dd6_DEEPGEMM, 8610141a333f_DEEPGEMM). So FP8-block / MXFP4 + MoE on rc29 is a PLATFORM FLOOR of the framework on Ada, while NVFP4 MoE is servable through MARLIN (weight-only). FIXED + IN THE GENERATOR (review pass): hardware.yaml l40s carries `moe_backend: {trtllm: {nvfp4: MARLIN}}` keyed by the artifact''s + quantization family (facts/apply.quant_family_of over utils._bundled_quantization), and the trtllm template now honours + the generic `moe_backend` param (it was silently dropped: the hardware fill ignored it and the template never read it; + a user value now beats the fill). The 10 roster NVFP4 MoE checkpoints re-probed under the plain render: 7 pass (MiniMax-M2.5/M2.7, + Nemotron-3 Nano/Super/Ultra, Qwen3.5-122B/397B), 3 reach later floors (Gemma-4: attentionOp ''Head size 512 is not supported + by MMHA'' native abort; Qwen3-235B: CutlassFp4GemmRunner needs Blackwell; Qwen3.8-2.4T capacity). MLA: the documented + ''no MLA FMHA below Hopper'' wall (collect_mla*.py FIXME(kernel-limit), ''Re-verify on the next framework version bump'') + was RE-VERIFIED on rc29 - with only the SM guard patched out a bf16 DeepSeek-V3 MLA module cell aborts in AttentionOp + ''Deepseek should be supported by fmha in generation part'' (attentionOp.cpp:3234, was :3091 on rc20; SIGSEGV in the + worker); guards kept, comments updated. Other fails: 8 DeepSeek-V4 ''requires Hopper or newer GPUs, got SM89'', 4 DeepGEMM + arch, 6 image gaps (transformers too old for DeepSeek-V4.1 / Muse-Glimmer; no `fla` for Kimi-K3 - same as sm100), 12 + generator rejects, 3 capacity, Gemma-4-31B-NVFP4 (flashinfer cute-DSL RMSNormFP4Quant ptxas needs .target sm_90), MiMo + arch not registered, Kimi-K2.5 quant parser (same as sm100). Gates: 7 of 9 real gates ALIGNED (attn ctx/gen + fp8-KV + gen, gdn ctx/gen, mamba2 ctx/gen), 13 declared sm89 floors (V3/R1 cannot boot: MoE + MLA; V3.2; V4). attn_ctx_fp8: serving''s + fp8-KV prefill runs fmha_v2 bf16 compute over the fp8 cache while the collector''s fp8-KV+fp8-context-FMHA case runs + the e4m3 kernel (fmha_v2_flash_attention_e4m3_..._sm89) - the gate recipe was moved to the bf16-compute combo that serving + selects; whether the SDK should bill the e4m3 FMHA on sm89 is an owner decision. encoder_attn was DIVERGED for lack + of evidence; CLOSED the same way as sglang (probe_trtllm runs the synthetic image through Qwen3VisionModel = mm_encoder.visual): + serving vision attention is `fmha_v2_flash_attention_bf16_64_32_S_qkv_64_sm89_kernel_nl` x24, the collector''s trtllm_mha + capture matches, ALIGNED. The two msa gate families the sm90 session left undeclared are declared with capture recipes + (trt_msa_ctx / trt_msa_gen) and GRADE FOR REAL on sm89 (ALIGNED) after the m3 dummy fix - see minimax_m3_dummy_dense_head_2026_10_05: + ''MiniMax-M3 does not boot on rc29'' was a dummy artifact, not a framework gap. On the other SMs the gate stays a FLOOR + whose note asks for a re-probe with the fixed dummy.' + applies_to: + - platform:sm89 + - framework:trtllm-1.3.0rc29 + - nvidia/DeepSeek-V3.1-NVFP4 + - deepseek-ai/DeepSeek-R1 + - deepseek-ai/DeepSeek-V3 + - Qwen/Qwen3-30B-A3B-FP8 + - openai/gpt-oss-120b + - openai/gpt-oss-20b + - nvidia/Gemma-4-26B-A4B-NVFP4 + - nvidia/Gemma-4-31B-IT-NVFP4 + - nvidia/Qwen3-235B-A22B-NVFP4 + - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4 + - nvidia/MiniMax-M2.5-NVFP4 + - instrument:probe_driver + - instrument:collect_mla_module + - component:generator_facts + - MiniMaxAI/MiniMax-M2.7 + - Qwen/Qwen3-235B-A22B-FP8 + - XiaomiMiMo/MiMo-7B-Base + - XiaomiMiMo/MiMo-V2-Flash + sm89_collector_upgrade_vllm0300_sglang0521_trtllm_rc29_2026_10_04: + statement: 'COLLECTOR UPGRADE ON sm89 (L40; manifest pins vllm 0.30.0 / sglang 0.5.21 / trtllm 1.3.0rc29, images digest-pinned, + collect_campaign.build-image adds pyarrow/pandas): NOT a full data collection - plan audit, per-op --smoke (4 cases), + a 40-case --shuffle sample per op on all three frameworks (8 GPUs, one container per op, outputs under the probe workspace + collect/sample40), backend-truth gates (path_diff) and targeted reproductions. Plans (collect.py --model-cases-full + --sm 89 --plan-only): vllm 20 ops, sglang 27 ops, trtllm 23 ops; the DSA / DSV4-attention / GLM-5 sparse module ops + are floored at SM90 by cases/capabilities.yaml and plan to 0 cases on sm89. FIXES MADE (all source-cited, re-verified + on the L40): (1) capabilities.yaml op_min_sm 90 for dsv4_{csa,hca}_{context,generation}_module (was the only DSV4/GLM-5 + sparse family without a floor: every case queued and failed ''unexpected''); (2) sglang collect_attn MockModelRunner.prefill/decode_attention_backend_str + (flashinfer lane, sglang>=0.5.21); (3) sglang collect_attn_encoder: context_attention_fwd moved to sglang.kernels.ops.attention.prefill_attention; + (4) sglang collect_mla MockModelConfig.hf_config / linear_attn_registry_result / is_draft_model (TritonAttnBackend probes + mambaish_config); (5) Gemma4 and Llama4 case maps gain sm89: triton (arg_groups/model_overrides/gemma4.py, llama4.py) + - the collector measured flashinfer while serving runs triton; (6) trtllm MLA guards re-verified on rc29 (attentionOp.cpp:3234). + Items 2-4 are lanes Hopper (fa3) and Blackwell (trtllm_mha / trtllm_mla) never reach, so the sm90 0.5.21 port could + not have caught them; they will bite sm120 too (same flashinfer / triton lanes). SMOKE (4 cases/op): vllm 14 ops with + data, 6 floored; sglang 14 ops with data (mhc / dsv4 are deliberate UnverifiedCollector refusals); trtllm 13 ops with + data (MLA modules refuse by the verified guard). 40-CASE SAMPLE, failure classes (full table in HANDOFF_sm89.md): vllm + 0 failures in 11 of 14 ops, 1 capacity in attention_generation (96-head MHA, 4x4095 tokens = 24 GiB KV), moe 2 (1 capacity, + 1 block_n alignment), kda 5/12 (4 capacity, 1 partial grid); sglang: attention_context 13/40 = ONE deterministic class + - the plan schedules fp8_kv_cache_and_context_fmha for a FlashInfer SM (sm89/sm120 resolve flashinfer) and the collector + refuses it on purpose (''flashinfer has no FP8 prefill compute path'', a stale ''0.5.14'' in the message) - FIXED in + the review pass: get_context_attention_test_cases resolves the lane the same way run_attention_torch does (per-model + map, else the SM default table) and no longer plans fp8_context_fmha on flashinfer; sm89 context plan now 43,114 cases: + 18,752 bf16 + 18,752 fp8_kv_cache + 5,610 fp8_context_fmha on the triton/sink lanes (the 13,142 flashinfer-lane fp8_context_fmha + cases are gone), 4-case smoke clean; gdn 21/40 and kda 5/12 = the causal_conv1d Triton int32 token-offset overflow (total_tokens*conv_channels + >= 2**31, causal_conv1d_triton.py:373-379) at the largest grid points (reproduced: batch 32-64 x seq 16384-32768, 3 + of 112 points; the other 109 rows persist) - framework limit, not Ada; mla_generation 3 capacity + 1 IMA; trtllm: attention_context + 6 (4 CUDA faults + 2 Gemma4 trtllm-gen ''Blackwell-only'' guards), moe 8 (7 TP-shard alignment guards - the collector''s + own, known family - 1 capacity), gemm 2 stride-alignment RuntimeErrors, mhc 2 capacity. Capacity is the dominant Ada + difference: 44 GiB vs 80-141 GiB on Hopper moves the batch x kv corner of every attention / mla / msa / kda grid into + OOM, none of it a collector bug. NOT DONE: the full retained plan (hundreds of thousands of cases per framework - sm120''s + vllm run was ~12 h on 8 GPUs), perf-data finalize/publication, SDK consumer tests against new l40s data. DELIVERY BLOCKER + for any perf data from this box: the SDK system is l40s and its existing tables carry device ''NVIDIA L40S'' (checked + in the parquet device column), while this box is an 8x L40 (same AD102 die, lower TDP/clocks). Playbook section 8 forbids + relabelling a nearby product with the same SM, so L40 measurements must not be published under l40s; an `l40` system + definition would be needed first (the sm120 RTX PRO 5000 campaign hit the same wall).' + applies_to: + - platform:sm89 + - framework:vllm-0.30.0 + - framework:sglang-0.5.21 + - framework:trtllm-1.3.0rc29 + - instrument:collect.py + - instrument:collect_attn + - instrument:collect_mla + - instrument:collect_gdn + - instrument:collect_kda + - cases:capabilities.yaml + sm89_harness_lessons_2026_10_04: + statement: 'HARNESS LESSONS from the first Ada campaign (each fixed in code unless marked OPEN): (1) the SDK system has + no generator hardware profile -> resolve_facts raised and the pipeline swallowed it (l40s profile added, product-keyed + like h100/h200; same class as sm120); (2) golden-render cache keyed on git HEAD only: an uncommitted generator edit + re-rendered nothing (_generator_stamp = HEAD + hash of the dirty generator diff; B200 defect 3); (3) probes that die + by a native abort (trtllm attentionOp asserts, sglang argparse) wrote no raw and the matrix said only ''crashed before + dump'': the queue now keeps each probe''s console tail (archive/logs/.tail, 400 lines) and build_matrix classifies + from it (B200 defect 1); (4) tp>1 cells are emitted with the first N GPUs hard-coded (device=0,1,2,3) while other queues + run on those GPUs: sglang''s tp memory-balance check and trtllm''s executor then fail for reasons that are not the model''s. + Rerun alone on an idle group, the sglang Qwen3.8-2.4T cells show their real cause (CUDA OOM, capacity) and trtllm''s + ''Executor worker returned error'' still carries no cause in the raw (OPEN: the server probe does not surface the worker''s + message); a whole-group reservation in the queue plan is still needed (B200 defect 5, OPEN); (5) _fail_cause classified + on a 200-char slice of the last line: vllm''s selector error is cut before use_sparse=True and sglang/trtllm wrap the + real cause in epilogues, so the matrix filed DSA cells under a config cause and Nemotron under ''framework gap''. Now: + full last line first, then a deep pass over the whole traceback ONLY for unmatched causes; (6) FLOOR_SM took one SM, + and only the vllm verdict script had the floor mechanism (sglang / trtllm scripts now have it; comma lists everywhere, + workflow_check understands them); (7) 4 sglang and 4 trtllm gate recipes were never checked in (sgl_attn_ctx/gen/gemma4_hd512, + sgl_gdn_gen, trt_attn ctx/gen + fp8-KV) - written; (8) taxonomy: seeding sm89 from sm120 left 100+ kernel names unlabelled + on Ada (cuBLAS ampere_* GEMMs, flashinfer merge-states, FLA chunk kernels): collect/unlabeled.py style scan over raws + + captures is how the vocabulary was built and is worth promoting to a component option; (9) encoder_attn gates ask + phase profile_run, which only probe_vllm recorded: FIXED - probe_sglang / probe_trtllm run a synthetic image through + the vision module and record profile_run_kernels (sglang: inside the `trace` dict, where the record builder reads kernel + tables from; the first attempt wrote it top-level and was invisible); the probe file is part of the fingerprint, so + this costs a full sglang + trtllm re-probe (~2 h on 8 GPUs) and will on every SM that adopts it; (10) dummy staging + for gated repos needs preprocessor_config.json for Llama-4 (vllm builds the vision tower in its profile run) and the + tokenizer files: stage_tok.py in the workspace pulls them from unsloth / sibling mirrors - fetch_inputs.py''s bundled-config + fallback still stops short of that (OPEN); (11) workflow_check gates_declared is red for trtllm rc29 on EVERY SM (msa_context_module + / msa_generation_module have no trtllm gate; pre-existing on this branch, sm90 shows the same, OPEN); (12) tooling: + pkill -f / pgrep -f patterns match the invoking shell''s own command line - kill queue wrappers by pid. (13) `--emit-queues + --only X --plan-name P` REPLACES plan P with X''s runs only; build_records/build_matrix then resolve the dropped runs + from older plan files whose fingerprints no longer match the raws, and whole cells turn ''stale evidence'' (hit twice + on sm89). Either keep one plan file per --only set or re-emit the full set into it.' + applies_to: + - platform:sm89 + - instrument:probe_driver + - instrument:workflow_check + - instrument:fetch_inputs + - instrument:path_diff + - component:captures + minimax_m3_dummy_dense_head_2026_10_05: + statement: 'MINIMAX-M3 ON TRTLLM WAS NEVER A FRAMEWORK GAP - the harness dummy was. Diagnosis (L40, trtllm 1.3.0rc29, + 2026-10-05): a sitecustomize shim around MiniMaxM3SparseRuntimeBackend.forward showed the ONLY None argument is idx_k_cache; + MiniMaxM3KVCacheManagerV2.__init__ derives its sparse layer ids from explicit kwargs (no call site passes them), else + the executor''s sparse_attention_config (has no layer-id field), else the checkpoint convention `range(3, num_layers)` + (sparse/minimax_m3/cache_manager.py:275-303), while the model layer follows the config''s sparse_attention_freq. The + old m3 dummy cut kept two sparse/MoE layers renumbered 0..1 (freq [1, 1]) -> every layer sparse for the model, index-K + cache for no layer -> the ValueError the sm90 session (findings trtllm_sparse_config, hopper_trtllm_collector_pin_rc20_to_rc29_2026_10_02) + and the first sm89 pass both filed as a framework gap. Six config variants (no CUDA graph, no overlap scheduler, attn_backend + FLASHINFER, tokens_per_block 64/128) fail identically, because the cut, not the config, is wrong. FIX (dummies.variants_m3): + the sparse variant keeps the checkpoint''s 3-layer dense head in front of the first sparse layer (4 layers, freq [0, + 0, 0, 1]; same class as _ARCH_IMPLICIT_PERIODS - a framework constant the config cannot express). RESULT: MiniMaxAI/MiniMax-M3 + boots on trtllm rc29 on sm89 (kv auto and fp8), serving sparse attention = Triton `_block_max_score_kernel` + `_sparse_softmax_kernel` + (the MSA fmha_sm100 backend is SM100-only), collector trt_msa_ctx/gen captures match: msa_ctx/gen_MiniMax-M3 gates ALIGNED + on sm89; the msa FLOOR on the trtllm verdict script is now a re-probe request for sm90/sm100/sm103/sm120, not a framework + fact. SIDE EFFECT, vLLM: the all-sparse dummy had also been a FALSE PASS there - with a dense head present vllm 0.30.0 + dies at ''No common block size for 16'' (v1/worker/utils.py:386): the KV manager defaults to 16, FlashAttention accepts + MultipleOf(16), MiniMaxM3SparseBackend / MiniMaxM3IndexerBackend accept exactly [128] (models/minimax_m3/common/sparse_attention.py). + A/B on the L40: --block-size 128 boots (facts/ab_vllm_m3/bs128), 64 fails the same way. Generator fact added: models.yaml + profile `minimax-m3` with vllm block-size 128 (platform-neutral; the sm90 M3 vllm pass rests on the same all-sparse + dummy and the real 60-layer checkpoint has the dense head). sglang: with the 4-layer dummy MiniMax-M3 boots on sm89 + as well (pass+custom with the sm90 `cuda_graph_backend_prefill=breakable` customization; the bare default fails the + same mixed-chunk-prefill assertion as on sm90, facts/ab_sgl_m3/bare.json) - the earlier sm89 ''capacity'' verdict was + the 2-layer dummy''s. NVFP4 / MXFP8 M3 variants: platform floors on sm89 as before (trtllm Marlin: ''does not support + swiglu_gptoss_style''; sglang MXFP8 dense GEMM needs Blackwell).' + applies_to: + - platform:sm89 + - platform:sm90 + - framework:trtllm-1.3.0rc29 + - framework:vllm-0.30.0 + - MiniMaxAI/MiniMax-M3 + - MiniMaxAI/MiniMax-M3-MXFP8 + - nvidia/MiniMax-M3-NVFP4 + - instrument:dummies + - component:generator_facts + - family:m3 + sm103_prepared_from_sm100_best_effort_2026_10_05: + statement: 'SM103 (B300) PREPARED FROM THE SM100 ROUND, BEST EFFORT WITHOUT A B300 BOX (2026-10-05; owner: the sglang encoder + gate passed on sm100 and failed on sm103). Root cause found by reading: collector/sglang/collect_attn_encoder.py picked + FA4 with `sm == 100` and sent SM103 down the Triton lane, while serving selects by capability MAJOR (vision.py:1298-1303 + @0.5.21: major 9 -> fa3, major 10 -> fa4, else triton_attn) — a wrong-kernel measurement, not a crash; the collector now + calls serving_vision_backend(sm) (unit-tested for 89/90/100/103/120/121) and the H20 smoke still records flash_attention_v3. + Audit of every exact-SM test in the collectors: the only other `== 100` is trtllm collect_moe''s nvfp4 min-latency option + (opened to SM103 under the owner''s 2026-10-05 decision that sm103 is treated as sm100; re-verify on a B300 box); sglang gemm / + gdn / mla_module already use {100, 103}. Harness: kernel_taxonomy_sm103.yaml seeded verbatim from sm100 (its Blackwell + block was validated on B300 on 2026-09-20; probe_driver refuses to run without a per-SM file), generator facts already map + b300_sxm / gb300 onto the b200 / gb200 profiles, the Blackwell fail-cause rule matches SM1xx, the trtllm msa floors list + sm103. NOT done without the box: labelling the sm103 vocabulary from B300 raws, results/sm103 matrices and gate verdicts, + re-running the sm103 encoder gate. On the B300 box: set targets.platform (b300_sm103 / sm 103 / b300_sxm; facts_systems + b300_sxm, gb300) and run the workflow as on sm100.' + applies_to: + - platform:sm103 + - platform:sm100 + - framework:sglang-0.5.21 + - module:collector + - instrument:sglang.collect_attn_encoder + - instrument:probe_driver + - Qwen/Qwen3-VL-2B-Instruct + collector_impact: + - {op: encoder_attention, action: collector_fix, ref: '14111be3: encoder collector picks the vision lane by capability family on sm103'} + - {op: moe, action: guard, ref: '26d09519: sm103 treated as sm100 — nvfp4 min-latency lane opens on SM103'} diff --git a/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml b/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml new file mode 100644 index 0000000000..3da6d96b99 --- /dev/null +++ b/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml @@ -0,0 +1,602 @@ +_meta: + platform: b200_sm100 + framework: sglang + version: 0.5.21 + what: the MoE cells of results/sm100/-.yaml re-probed with the generator's hardware moe_backend fact removed from the rendered config; the as-rendered cells fail with cause 'generator fact rejected + by framework'. This file is the identity the framework picks on sm100 when it is allowed to choose. + override: --moe-runner-backend deepep_moe (hardware.yaml b200) dropped -> 0.5.21 AUTO + summary: + pass+override: 72 + fail: 3 + evidence: archive/retests/moe_auto/{raw,*.engine.yaml} in the B200 workspace (umbriel-b200-040, 2026-10-04) +results: + MiniMaxAI/MiniMax-M2: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + MiniMaxAI/MiniMax-M2.5: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + MiniMaxAI/MiniMax-M2.7: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + MiniMaxAI/MiniMax-M3: + verdict: pass+override + attention: MiniMaxHybridAttnBackend + moe_runner: TRITON + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + MiniMaxAI/MiniMax-M3-MXFP8: + verdict: pass+override + attention: MiniMaxHybridAttnBackend + moe_runner: DEEP_GEMM + moe_kernel_families: null + moe_kernels_seen: 1 + topology: tp1 + Qwen/Qwen1.5-MoE-A2.7B: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + Qwen/Qwen3-235B-A22B: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3-235B-A22B-FP8: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + Qwen/Qwen3-235B-A22B-Instruct-2507: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3-30B-A3B: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3-30B-A3B-FP8: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + Qwen/Qwen3-Coder-480B-A35B-Instruct: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3-VL-235B-A22B-Instruct: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3-VL-30B-A3B-Instruct: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3.5-122B-A10B: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3.5-35B-A3B: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + Qwen/Qwen3.5-397B-A17B: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + XiaomiMiMo/MiMo-V2-Flash: + verdict: pass+override + attention: FlashAttentionBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + deepseek-ai/DeepSeek-R1: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + deepseek-ai/DeepSeek-V3: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + deepseek-ai/DeepSeek-V3.2: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + deepseek-ai/DeepSeek-V4-Flash: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + deepseek-ai/DeepSeek-V4-Flash-0731: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + deepseek-ai/DeepSeek-V4-Pro: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + deepseek-ai/DeepSeek-V4-Pro-0813: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + deepseek-ai/DeepSeek-V4.1-Flash: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + google/gemma-4-26B-A4B: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + meta-llama/Llama-4-Maverick-17B-128E-Instruct: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 4 + topology: tp1 + meta-llama/Llama-4-Scout-17B-16E-Instruct: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 4 + topology: tp1 + mistralai/Mixtral-8x22B-v0.1: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 4 + topology: tp1 + mistralai/Mixtral-8x7B-v0.1: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 4 + topology: tp1 + moonshotai/Kimi-K2-Instruct: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + moonshotai/Kimi-K2.5: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + moonshotai/Kimi-K3: + verdict: fail + cause: framework gap + error: 'ValueError: multi_ctas_kv_counter_buffer is only supported when a trtllm-gen runner is selected' + topology: tp1 + nvidia/DeepSeek-V3.1-NVFP4: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/DeepSeek-V3.2-NVFP4: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/GLM-5-NVFP4: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/GLM-5.1-NVFP4: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/GLM-5.2-NVFP4: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/GLM-5.3-NVFP4: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Gemma-4-26B-A4B-NVFP4: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Kimi-K2.5-NVFP4: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Kimi-K2.6-NVFP4: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Kimi-K2.7-Code-NVFP4: + verdict: pass+override + attention: TRTLLMMLABackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Kimi-K3-NVFP4: + verdict: fail + cause: framework gap + error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' + topology: tp1 + nvidia/MiniMax-M2.5-NVFP4: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM_ROUTED + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + nvidia/MiniMax-M2.7-NVFP4: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM_ROUTED + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + nvidia/MiniMax-M3-NVFP4: + verdict: pass+override + attention: MiniMaxHybridAttnBackend + moe_runner: FLASHINFER_TRTLLM_ROUTED + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 4 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: MARLIN + moe_kernel_families: + - marlin + moe_kernels_seen: 6 + topology: tp1 + nvidia/Qwen3-235B-A22B-NVFP4: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Qwen3.5-122B-A10B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Qwen3.5-397B-A17B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + nvidia/Qwen3.6-35B-A3B-NVFP4: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: MARLIN + moe_kernel_families: + - marlin + moe_kernels_seen: 6 + topology: tp1 + nvidia/Qwen3.8-2.4T-A95B-NVFP4: + verdict: fail + cause: framework gap + error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' + topology: tp1 + openai/gpt-oss-120b: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + openai/gpt-oss-20b: + verdict: pass+override + attention: TRTLLMHAAttnBackend + moe_runner: FLASHINFER_MXFP4 + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + ornith-ai/Ornith-1.5-35B-A3B: + verdict: pass+override + attention: HybridLinearAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + sgl-project/DeepSeek-V4-Flash-FP8: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + sgl-project/DeepSeek-V4-Pro-FP8: + verdict: pass+override + attention: DeepseekV4AttnBackend + moe_runner: AUTO + moe_kernel_families: + - triton_fused_moe + moe_kernels_seen: 6 + topology: tp1 + zai-org/GLM-5: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + zai-org/GLM-5-FP8: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + zai-org/GLM-5.1: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + zai-org/GLM-5.1-FP8: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + zai-org/GLM-5.2: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 + zai-org/GLM-5.2-FP8: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + zai-org/GLM-5.3: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 6 + topology: tp1 + zai-org/GLM-5.3-BF16: + verdict: pass+override + attention: DeepseekSparseAttnBackend + moe_runner: FLASHINFER_TRTLLM + moe_kernel_families: + - trtllm_gen_moe + moe_kernels_seen: 5 + topology: tp1 diff --git a/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml new file mode 100644 index 0000000000..84a58280ba --- /dev/null +++ b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml @@ -0,0 +1,803 @@ +_meta: + platform: b200_sm100 + framework: sglang + version: 0.5.21 + verdicts: pass = plain `cli generate` output boots+runs; pass+custom = boots with facts-derived extra generate args; fail = root-caused (see results/findings.yaml) + summary: + pass: 102 + pass+custom: 6 + fail: 16 + golden_facts_not_applied: 0 +results: + MiniMaxAI/MiniMax-M2: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Fp8MoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + MiniMaxAI/MiniMax-M2.5: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Fp8MoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + MiniMaxAI/MiniMax-M2.7: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Fp8MoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + MiniMaxAI/MiniMax-M3: + verdict: pass+custom + extra_args: --generator-set params.agg.cuda_graph_backend_prefill=breakable + evidence: current + attention: MiniMaxHybridAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + MiniMaxAI/MiniMax-M3-MXFP8: + verdict: pass+custom + extra_args: --generator-set params.agg.cuda_graph_backend_prefill=breakable + evidence: current + attention: MiniMaxHybridAttnBackend + moe: Fp8MoE + kv_allocated: auto + topology: tp1 + Qwen/Qwen1.5-MoE-A2.7B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-0.6B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-1.7B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-235B-A22B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-235B-A22B-FP8: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-235B-A22B-Instruct-2507: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-30B-A3B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-30B-A3B-FP8: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-32B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-32B-FP8: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-32B-FP8-Static-PerTensor: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-8B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-Coder-480B-A35B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-235B-A22B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-2B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-30B-A3B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-32B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-32B-Thinking: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-4B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3-VL-8B-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-0.8B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-122B-A10B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-27B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-2B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-35B-A3B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-397B-A17B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-4B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.5-9B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.8-2.4T-A95B: + verdict: pass + evidence: current + attention: trtllm_mha + kv_allocated: auto + topology: tp4 + Qwen/Qwen3.8-2.4T-A95B-FP8: + verdict: fail + cause: no raw (crashed before dump) + Qwen/Qwen3.8-27B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.8-27B-FP8: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + Qwen/Qwen3.8-Flash-Next: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next'' is not a valid HuggingFace model path or local pa' + Qwen/Qwen3.8-Flash-Next-FP8: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next-FP8'' is not a valid HuggingFace model path or loca' + TokenRhythm/NeoHorse-1-9B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + XiaomiMiMo/MiMo-7B-Base: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + XiaomiMiMo/MiMo-V2-Flash: + verdict: pass + evidence: current + attention: FlashAttentionBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + XiaomiMiMo/MiMo-V2.6-Flash-RL: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Flash-RL'' is not a valid HuggingFace model path or lo' + XiaomiMiMo/MiMo-V2.6-Pro-RL: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Pro-RL'' is not a valid HuggingFace model path or loca' + baerquant/Inkling: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling'' is not a valid HuggingFace model path or local path wit' + baerquant/Inkling-Small: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling-Small'' is not a valid HuggingFace model path or local pa' + baerquants/Inkling-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-NVFP4'' is not a valid HuggingFace model path or local p' + baerquants/Inkling-Small-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-Small-NVFP4'' is not a valid HuggingFace model path or l' + deepseek-ai/DeepSeek-R1: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V3: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V3.2: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V4-Flash: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V4-Flash-0731: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V4-Pro: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V4-Pro-0813: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + deepseek-ai/DeepSeek-V4.1-Flash: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + google/gemma-4-26B-A4B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + meta-llama/Llama-4-Maverick-17B-128E-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + meta-llama/Llama-4-Scout-17B-16E-Instruct: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + meta-llama/Meta-Llama-3.1-405B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + meta-llama/Meta-Llama-3.1-70B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + meta-llama/Meta-Llama-3.1-8B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + meta-models/Muse-Glimmer-30B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + mistralai/Mixtral-8x22B-v0.1: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + mistralai/Mixtral-8x7B-v0.1: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: UnquantizedFusedMoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + moonshotai/Kimi-K2-Instruct: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + moonshotai/Kimi-K2.5: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: CompressedTensorsFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + moonshotai/Kimi-K3: + verdict: fail + cause: framework gap + error: 'ValueError: multi_ctas_kv_counter_buffer is only supported when a trtllm-gen runner is selected' + nvidia/DeepSeek-V3.1-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/DeepSeek-V3.2-NVFP4: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/DeepSeek-V4-Flash-NVFP4: + verdict: pass+custom + extra_args: --generator-set params.agg.moe_backend=marlin + evidence: current + attention: DeepseekV4AttnBackend + moe: ModelOptNvFp4FusedMoE->marlin + kv_allocated: auto + topology: tp1 + nvidia/DeepSeek-V4-Pro-NVFP4: + verdict: pass+custom + extra_args: --generator-set params.agg.moe_backend=marlin + evidence: current + attention: DeepseekV4AttnBackend + moe: ModelOptNvFp4FusedMoE->marlin + kv_allocated: auto + topology: tp1 + nvidia/DeepSeek-V4.1-Flash-NVFP4: + verdict: pass+custom + extra_args: --generator-set params.agg.moe_backend=marlin + evidence: current + attention: DeepseekV4AttnBackend + moe: ModelOptNvFp4FusedMoE->marlin + kv_allocated: auto + topology: tp1 + nvidia/GLM-5-NVFP4: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/GLM-5.1-NVFP4: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/GLM-5.2-NVFP4: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/GLM-5.3-Flash-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/GLM-5.3-Flash-NVFP4'' is not a valid HuggingFace model path or local' + nvidia/GLM-5.3-NVFP4: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Gemma-4-26B-A4B-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Gemma-4-31B-IT-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/Kimi-K2.5-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Kimi-K2.6-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Kimi-K2.7-Code-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMMLABackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Kimi-K3-NVFP4: + verdict: fail + cause: framework gap + error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' + nvidia/Llama-3.1-70B-Instruct-FP8: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/Llama-3_3-Nemotron-Super-49B-v1: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/MiniMax-M2.5-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/MiniMax-M2.7-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/MiniMax-M3-NVFP4: + verdict: pass+custom + extra_args: --generator-set params.agg.cuda_graph_backend_prefill=breakable + evidence: current + attention: MiniMaxHybridAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Muse-Glimmer-30B-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptFp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptFp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->marlin + kv_allocated: auto + topology: tp1 + nvidia/Nemotron-H-56B-Base-8K: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/Qwen3-235B-A22B-NVFP4: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Qwen3.5-122B-A10B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Qwen3.5-397B-A17B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + nvidia/Qwen3.6-27B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/Qwen3.6-35B-A3B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: ModelOptNvFp4FusedMoE->marlin + kv_allocated: auto + topology: tp1 + nvidia/Qwen3.8-2.4T-A95B-NVFP4: + verdict: fail + cause: framework gap + error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' + nvidia/Qwen3.8-27B-NVFP4: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + kv_allocated: auto + topology: tp1 + nvidia/Qwen3.8-Flash-Next-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/Qwen3.8-Flash-Next-NVFP4'' is not a valid HuggingFace model path or ' + openai/gpt-oss-120b: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Mxfp4MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + openai/gpt-oss-20b: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + moe: Mxfp4MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + openbmb/MiniCPM5-2B: + verdict: pass + evidence: current + attention: TRTLLMHAAttnBackend + kv_allocated: auto + topology: tp1 + ornith-ai/Ornith-1.5-35B-A3B: + verdict: pass + evidence: current + attention: HybridLinearAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + sgl-project/DeepSeek-V4-Flash-FP8: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Fp8MoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + sgl-project/DeepSeek-V4-Pro-FP8: + verdict: pass + evidence: current + attention: DeepseekV4AttnBackend + moe: Fp8MoE->triton_fused_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5-FP8: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.1: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.1-FP8: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.2: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.2-FP8: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.3: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: Fp8MoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.3-BF16: + verdict: pass + evidence: current + attention: DeepseekSparseAttnBackend + moe: UnquantizedFusedMoE->trtllm_gen_moe + kv_allocated: auto + topology: tp1 + zai-org/GLM-5.3-Flash: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash'' is not a valid HuggingFace model path or local path' + zai-org/GLM-5.3-Flash-BF16: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash-BF16'' is not a valid HuggingFace model path or local' diff --git a/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml new file mode 100644 index 0000000000..b390b4482f --- /dev/null +++ b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml @@ -0,0 +1,734 @@ +_meta: + platform: b200_sm100 + framework: trtllm + version: 1.3.0rc29 + verdicts: pass = plain `cli generate` output boots+runs; pass+custom = boots with facts-derived extra generate args; fail = root-caused (see results/findings.yaml) + summary: + pass: 78 + pass+custom: 0 + fail: 46 + golden_facts_not_applied: 0 +results: + MiniMaxAI/MiniMax-M2: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + MiniMaxAI/MiniMax-M2.5: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + MiniMaxAI/MiniMax-M2.7: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + MiniMaxAI/MiniMax-M3: + verdict: fail + cause: quant parser gap + error: 'AttributeError: ''NoneType'' object has no attribute ''shape''' + MiniMaxAI/MiniMax-M3-MXFP8: + verdict: fail + cause: quant parser gap + error: 'AttributeError: ''NoneType'' object has no attribute ''shape''' + Qwen/Qwen1.5-MoE-A2.7B: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3-0.6B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-1.7B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-235B-A22B: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3-235B-A22B-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + Qwen/Qwen3-235B-A22B-Instruct-2507: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3-30B-A3B: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3-30B-A3B-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + Qwen/Qwen3-32B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-32B-FP8: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-32B-FP8-Static-PerTensor: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-8B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-Coder-480B-A35B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-235B-A22B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-2B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-30B-A3B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-32B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-32B-Thinking: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-4B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3-VL-8B-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-0.8B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-122B-A10B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-27B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-2B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-35B-A3B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-397B-A17B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-4B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.5-9B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.8-2.4T-A95B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp4 + Qwen/Qwen3.8-2.4T-A95B-FP8: + verdict: fail + cause: framework gap + error: '[10/04/2026-11:07:37] [TRT-LLM] [W] [_torch][RANK 0] PyTorchModelEngine cleanup failed during destruction: ''PyTorchModelEngine'' object has no attribute ''_torch_compile_backend''' + Qwen/Qwen3.8-27B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.8-27B-FP8: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + Qwen/Qwen3.8-Flash-Next: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next'' is not a valid HuggingFace model path or local pa' + Qwen/Qwen3.8-Flash-Next-FP8: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next-FP8'' is not a valid HuggingFace model path or loca' + TokenRhythm/NeoHorse-1-9B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + XiaomiMiMo/MiMo-7B-Base: + verdict: fail + cause: arch not registered + error: 'ValueError: Unknown architecture for AutoModelForCausalLM: MiMoForCausalLM' + XiaomiMiMo/MiMo-V2-Flash: + verdict: fail + cause: arch not registered + error: 'ValueError: Unknown architecture for AutoModelForCausalLM: MiMoV2FlashForCausalLM' + XiaomiMiMo/MiMo-V2.6-Flash-RL: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Flash-RL'' is not a valid HuggingFace model path or lo' + XiaomiMiMo/MiMo-V2.6-Pro-RL: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Pro-RL'' is not a valid HuggingFace model path or loca' + baerquant/Inkling: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling'' is not a valid HuggingFace model path or local path wit' + baerquant/Inkling-Small: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling-Small'' is not a valid HuggingFace model path or local pa' + baerquants/Inkling-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-NVFP4'' is not a valid HuggingFace model path or local p' + baerquants/Inkling-Small-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-Small-NVFP4'' is not a valid HuggingFace model path or l' + deepseek-ai/DeepSeek-R1: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V3: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V3.2: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V4-Flash: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V4-Flash-0731: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V4-Pro: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V4-Pro-0813: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + deepseek-ai/DeepSeek-V4.1-Flash: + verdict: fail + cause: framework gap + error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports + this mod + google/gemma-4-26B-A4B: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + meta-llama/Llama-4-Maverick-17B-128E-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + meta-llama/Llama-4-Scout-17B-16E-Instruct: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + meta-llama/Meta-Llama-3.1-405B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + meta-llama/Meta-Llama-3.1-70B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + meta-llama/Meta-Llama-3.1-8B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + meta-models/Muse-Glimmer-30B: + verdict: fail + cause: framework gap + error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports + this mod + mistralai/Mixtral-8x22B-v0.1: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + mistralai/Mixtral-8x7B-v0.1: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + moonshotai/Kimi-K2-Instruct: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + moonshotai/Kimi-K2.5: + verdict: fail + cause: quant parser gap + error: 'TypeError: ''NoneType'' object is not subscriptable' + moonshotai/Kimi-K3: + verdict: fail + cause: framework gap + error: 'ModuleNotFoundError: No module named ''fla''' + nvidia/DeepSeek-V3.1-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/DeepSeek-V3.2-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/DeepSeek-V4-Flash-NVFP4: + verdict: fail + cause: generator DSV4 kv facts unservable on this SM + error: 'ValueError: DeepSeek-V4 fp8_ds_mla KV cache requires tokens_per_block=256 on SM100, got 128.' + nvidia/DeepSeek-V4-Pro-NVFP4: + verdict: fail + cause: generator DSV4 kv facts unservable on this SM + error: 'ValueError: DeepSeek-V4 fp8_ds_mla KV cache requires tokens_per_block=256 on SM100, got 128.' + nvidia/DeepSeek-V4.1-Flash-NVFP4: + verdict: fail + cause: framework gap + error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports + this mod + nvidia/GLM-5-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/GLM-5.1-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/GLM-5.2-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/GLM-5.3-Flash-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/GLM-5.3-Flash-NVFP4'' is not a valid HuggingFace model path or local' + nvidia/GLM-5.3-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Gemma-4-26B-A4B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Gemma-4-31B-IT-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + nvidia/Kimi-K2.5-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Kimi-K2.6-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Kimi-K2.7-Code-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Kimi-K3-NVFP4: + verdict: fail + cause: framework gap + error: 'ModuleNotFoundError: No module named ''fla''' + nvidia/Llama-3.1-70B-Instruct-FP8: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + nvidia/Llama-3_3-Nemotron-Super-49B-v1: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + nvidia/MiniMax-M2.5-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/MiniMax-M2.7-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/MiniMax-M3-NVFP4: + verdict: fail + cause: quant parser gap + error: 'AttributeError: ''NoneType'' object has no attribute ''shape''' + nvidia/Muse-Glimmer-30B-NVFP4: + verdict: fail + cause: framework gap + error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports + this mod + nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: FP8QDQFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: FP8QDQFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: UnquantizedFusedMoE + kv_allocated: null + topology: tp1 + nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + moe: W4A16NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Nemotron-H-56B-Base-8K: + verdict: pass + evidence: current + attention: trtllm_fmha + _chunk_scan_fwd_kernel + kv_allocated: null + topology: tp1 + nvidia/Qwen3-235B-A22B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Qwen3.5-122B-A10B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Qwen3.5-397B-A17B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Qwen3.6-27B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + nvidia/Qwen3.6-35B-A3B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Qwen3.8-2.4T-A95B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: NVFP4CutlassFusedMoE + kv_allocated: null + topology: tp1 + nvidia/Qwen3.8-27B-NVFP4: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + kv_allocated: null + topology: tp1 + nvidia/Qwen3.8-Flash-Next-NVFP4: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/Qwen3.8-Flash-Next-NVFP4'' is not a valid HuggingFace model path or ' + openai/gpt-oss-120b: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: W4A8MXFP4MXFP8CutlassFusedMoE + kv_allocated: null + topology: tp1 + openai/gpt-oss-20b: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: W4A8MXFP4MXFP8CutlassFusedMoE + kv_allocated: null + topology: tp1 + openbmb/MiniCPM5-2B: + verdict: pass + evidence: current + attention: trtllm_fmha + kv_allocated: null + topology: tp1 + ornith-ai/Ornith-1.5-35B-A3B: + verdict: pass + evidence: current + attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel + moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + sgl-project/DeepSeek-V4-Flash-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + sgl-project/DeepSeek-V4-Pro-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + zai-org/GLM-5: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + zai-org/GLM-5-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + zai-org/GLM-5.1: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + zai-org/GLM-5.1-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + zai-org/GLM-5.2: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + zai-org/GLM-5.2-FP8: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + zai-org/GLM-5.3: + verdict: fail + cause: rendered MoE backend cannot serve this layer on this SM + error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s + reason:' + zai-org/GLM-5.3-BF16: + verdict: pass + evidence: current + attention: trtllm_fmha + moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal + kv_allocated: null + topology: tp1 + zai-org/GLM-5.3-Flash: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash'' is not a valid HuggingFace model path or local path' + zai-org/GLM-5.3-Flash-BF16: + verdict: fail + cause: generator rejects + error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash-BF16'' is not a valid HuggingFace model path or local' diff --git a/benchmarks/reports/perf-data/perf-data-reuse-analysis.md b/benchmarks/reports/perf-data/perf-data-reuse-analysis.md index f907b2d169..ae08b65e75 100644 --- a/benchmarks/reports/perf-data/perf-data-reuse-analysis.md +++ b/benchmarks/reports/perf-data/perf-data-reuse-analysis.md @@ -9,41 +9,41 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur ## Headline numbers -- Total rows scanned: **15,891,586** -- Within-framework cross-version dedup-able rows: **8,584** (~0.1%) +- Total rows scanned: **31,755,054** +- Within-framework cross-version dedup-able rows: **47,131** (~0.1%) - Tier distribution (groups / rows): - - `shared`: 1084 groups · 15,585,305 rows - - `shared_fallback`: 52 groups · 306,281 rows + - `shared`: 1299 groups · 31,156,118 rows + - `shared_fallback`: 54 groups · 598,936 rows ## `computescale_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1620 | 0 / 1620 | 0 | — | — | — | -| b200_sxm | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | -| b200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| b300_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| b300_sxm | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | -| b300_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| gb200 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1625 | 0 / 1625 | 0 | — | — | — | -| gb200 | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | -| gb200 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| gb300 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1626 | 0 / 1626 | 0 | — | — | — | -| gb300 | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | -| gb300 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `sglang` | shared | sglang | sglang:3 | 0 / 3 | 0 | — | — | — | -| h100_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `sglang` | shared | sglang | sglang:6 | 0 / 6 | 0 | — | — | — | -| h200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| l40s | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1575 | 0 / 1575 | 0 | — | — | — | -| l40s | `sglang` | shared | sglang | sglang:18 | 0 / 18 | 0 | — | — | — | -| l40s | `torch_ops` | shared | trtllm | trtllm:1607 | 0 / 1607 | 0 | — | — | — | -| rtx_pro_6000_server | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1611 | 0 / 1611 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:10 | 0 / 10 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:1613 | 0 / 1613 | 0 | — | — | — | +| b200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3242 | 0 / 1623 | 1 | — | — | — | +| b200_sxm | `sglang` | shared | sglang | sglang:2 | 0 / 1 | 0 | — | — | — | +| b200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 3256 | 0 | — | — | — | +| b300_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3255 | 0 / 1628 | 10 | — | — | — | +| b300_sxm | `sglang` | shared | sglang | sglang:2 | 0 / 1 | 0 | — | — | — | +| b300_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | +| gb200 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3253 | 0 / 1628 | 6 | — | — | — | +| gb200 | `sglang` | shared | sglang | sglang:5 | 0 / 4 | 0 | — | — | — | +| gb200 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | +| gb300 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:4880 | 0 / 1628 | 22 | — | — | — | +| gb300 | `sglang` | shared | sglang | sglang:16 | 0 / 10 | 0 | — | — | — | +| gb300 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 1 | — | — | — | +| h100_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 15 | — | — | — | +| h100_sxm | `sglang` | shared | sglang | sglang:11 | 0 / 8 | 0 | — | — | — | +| h100_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 10 | — | — | — | +| h200_sxm | `sglang` | shared | sglang | sglang:17 | 0 / 11 | 0 | — | — | — | +| h200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | +| l40s | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3155 | 0 / 1584 | 182 | — | — | — | +| l40s | `sglang` | shared | sglang | sglang:35 | 0 / 18 | 1 | — | — | — | +| l40s | `torch_ops` | shared | trtllm | trtllm:3234 | 0 / 1627 | 1 | — | — | — | +| rtx_pro_6000_server | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3216 | 0 / 1615 | 7 | — | — | — | +| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:20 | 0 / 10 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:3237 | 0 / 1625 | 0 | — | — | — | ## `context_attention_perf.parquet` @@ -52,90 +52,101 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `flash_attention` | shared | sglang | sglang:5627 | 0 / 5627 | 0 | — | — | — | | a100_sxm | `torch_flow` | shared | trtllm | trtllm:4864 | 0 / 4864 | 0 | — | — | — | | a100_sxm | `vllm_flash_attn` | shared | vllm | vllm:5457 | 0 / 5049 | 0 | — | — | — | -| b200_sxm | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:68535 | 0 / 68535 | 0 | — | — | — | -| b200_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | -| b200_sxm | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | -| b200_sxm | `trtllm_mha` | shared | sglang | sglang:32093 | 0 / 32093 | 0 | — | — | — | +| b200_sxm | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 4 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:139563 | 0 / 139563 | 0 | — | — | — | +| b200_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 5100 | 0 | — | — | — | +| b200_sxm | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 15 | — | — | — | +| b200_sxm | `trtllm_mha` | shared | sglang | sglang:65618 | 0 / 33526 | 32 | — | — | — | +| b200_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer` | shared | vllm | vllm:35360 | 0 / 35360 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | +| b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3756 | 0 / 3756 | 0 | — | — | — | -| b200_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:92848 | 0 / 47304 | 183 | — | — | — | -| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:4896 | 0 / 3264 | 3 | — | — | — | -| b300_sxm | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:67106 | 0 / 67106 | 0 | — | — | — | -| b300_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | -| b300_sxm | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | -| b300_sxm | `trtllm_mha` | shared | sglang | sglang:32097 | 0 / 32097 | 0 | — | — | — | +| b200_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:139508 | 0 / 47304 | 243 | — | — | — | +| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:7222 | 0 / 3961 | 6 | — | — | — | +| b300_sxm | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 5 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:138136 | 0 / 71030 | 53 | — | — | — | +| b300_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 2788 | 6 | — | — | — | +| b300_sxm | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 14 | — | — | — | +| b300_sxm | `trtllm_mha` | shared | sglang | sglang:65619 | 0 / 33526 | 30 | — | — | — | +| b300_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:1626 | 0 / 1626 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | +| b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3668 | 0 / 3668 | 0 | — | — | — | -| b300_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:91780 | 0 / 47304 | 178 | — | — | — | -| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:3264 | 0 / 1632 | 7 | — | — | — | -| b60 | `vllm_flash_attn` | shared | vllm | vllm:34118 | 0 / 17930 | 1 | — | — | — | -| gb200 | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:68535 | 0 / 68535 | 0 | — | — | — | -| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | -| gb200 | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | -| gb200 | `trtllm_mha` | shared | sglang | sglang:31138 | 0 / 31138 | 0 | — | — | — | +| b300_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:138440 | 0 / 47304 | 303 | — | — | — | +| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:5588 | 0 / 2329 | 15 | — | — | — | +| b60 | `vllm_flash_attn` | shared | vllm | vllm:31688 | 0 / 17930 | 1 | — | — | — | +| gb200 | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 5 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:139562 | 0 / 71027 | 53 | — | — | — | +| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 2788 | 2 | — | — | — | +| gb200 | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 20 | — | — | — | +| gb200 | `trtllm_mha` | shared | sglang | sglang:64661 | 0 / 33525 | 28 | — | — | — | +| gb200 | `vllm_flash_attn_fa2` | shared | vllm | vllm:1630 | 0 / 1630 | 0 | — | — | — | | gb200 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | +| gb200 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | gb200 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3740 | 0 / 3740 | 0 | — | — | — | -| gb200 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:92660 | 0 / 47304 | 124 | — | — | — | -| gb200 | `vllm_triton_attn` | shared | vllm | vllm:3264 | 0 / 1632 | 3 | — | — | — | -| gb300 | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:68533 | 0 / 68533 | 0 | — | — | — | -| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | -| gb300 | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | -| gb300 | `trtllm_mha` | shared | sglang | sglang:31144 | 0 / 31144 | 0 | — | — | — | +| gb200 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:139320 | 0 / 47304 | 232 | — | — | — | +| gb200 | `vllm_triton_attn` | shared | vllm | vllm:5591 | 0 / 2329 | 6 | — | — | — | +| gb300 | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 3 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:139563 | 0 / 71030 | 49 | — | — | — | +| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 2788 | 5 | — | — | — | +| gb300 | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 4 | — | — | — | +| gb300 | `trtllm_mha` | shared | sglang | sglang:64667 | 0 / 33525 | 18 | — | — | — | +| gb300 | `vllm_flash_attn_fa2` | shared | vllm | vllm:1629 | 0 / 1629 | 0 | — | — | — | | gb300 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | +| gb300 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | gb300 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3740 | 0 / 3740 | 0 | — | — | — | -| gb300 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:92660 | 0 / 47304 | 207 | — | — | — | -| gb300 | `vllm_triton_attn` | shared | vllm | vllm:3264 | 0 / 1632 | 4 | — | — | — | -| h100_sxm | `fa3` | shared | sglang | sglang:31622 | 0 / 31622 | 0 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:75840 | 0 / 75840 | 0 | — | — | — | -| h100_sxm | `triton` | shared | sglang | sglang:3468 | 0 / 3468 | 0 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:103326 | 0 / 53074 | 12 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1275 | 0 / 697 | 1 | — | — | — | -| h200_sxm | `fa3` | shared | sglang | sglang:32574 | 0 / 32574 | 0 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:77265 | 0 / 77265 | 0 | — | — | — | -| h200_sxm | `triton` | shared | sglang | sglang:3468 | 0 / 3468 | 0 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:104278 | 0 / 53074 | 155 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1275 | 0 / 697 | 1 | — | — | — | -| l40s | `flashinfer` | shared | sglang | sglang:26236 | 0 / 26236 | 0 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:75768 | 0 / 75768 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:9792 | 0 / 9792 | 0 | — | — | — | -| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:25092 | 0 / 25092 | 0 | — | — | — | -| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:1744 | 0 / 1744 | 0 | — | — | — | -| l40s | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:21690 | 0 / 21690 | 0 | — | — | — | -| l40s | `vllm_triton_attn` | shared | vllm | vllm:1632 | 0 / 1632 | 0 | — | — | — | -| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:23410 | 0 / 23410 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:71929 | 0 / 71929 | 0 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:16116 | 0 / 16116 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:25126 | 0 / 25126 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:1728 | 0 / 1728 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:21677 | 0 / 21677 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:1632 | 0 / 1632 | 0 | — | — | — | +| gb300 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:139320 | 0 / 47304 | 301 | — | — | — | +| gb300 | `vllm_triton_attn` | shared | vllm | vllm:5587 | 0 / 2329 | 5 | — | — | — | +| h100_sxm | `fa3` | shared | sglang | sglang:66100 | 0 / 34478 | 28 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:154782 | 0 / 79050 | 57 | — | — | — | +| h100_sxm | `triton` | shared | sglang | sglang:7650 | 0 / 4182 | 0 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:156400 | 0 / 53074 | 93 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1972 | 0 / 697 | 3 | — | — | — | +| h100_sxm | `vllm_triton_attn` | shared | vllm | vllm:697 | 0 / 697 | 0 | — | — | — | +| h200_sxm | `fa3` | shared | sglang | sglang:67052 | 0 / 34478 | 31 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:156199 | 0 / 79038 | 69 | — | — | — | +| h200_sxm | `triton` | shared | sglang | sglang:7650 | 0 / 4182 | 1 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:157352 | 0 / 53074 | 320 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1972 | 0 / 697 | 3 | — | — | — | +| h200_sxm | `vllm_triton_attn` | shared | vllm | vllm:697 | 0 / 697 | 0 | — | — | — | +| l40s | `flashinfer` | shared | sglang | sglang:52506 | 0 / 29092 | 477 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:154712 | 0 / 78973 | 1671 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:26622 | 0 / 16830 | 39 | — | — | — | +| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:51624 | 0 / 26537 | 1065 | — | — | — | +| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:3690 | 0 / 1946 | 90 | — | — | — | +| l40s | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:45334 | 0 / 23644 | 294 | — | — | — | +| l40s | `vllm_triton_attn` | shared | vllm | vllm:3961 | 0 / 2329 | 35 | — | — | — | +| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:49692 | 0 / 26282 | 50 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:147105 | 0 / 75176 | 179 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:32946 | 0 / 16830 | 15 | — | — | — | +| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:51662 | 0 / 26537 | 71 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:1781 | 0 / 1781 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:45312 | 0 / 23635 | 29 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_flashinfertrtllmapidecode_xqa` | shared | vllm | vllm:1807 | 0 / 1807 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:3961 | 0 / 2329 | 1 | — | — | — | ## `context_mla_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | a100_sxm | `default` | shared_fallback | trtllm | trtllm:2436 | 0 / 2436 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:3080 | 0 / 3080 | 0 | — | — | — | -| b200_sxm | `trtllm_mla` | shared | sglang | sglang:3080 | 0 / 3080 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | -| b300_sxm | `trtllm_mla` | shared | sglang | sglang:3080 | 0 / 3080 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | -| gb200 | `trtllm_mla` | shared | sglang | sglang:3520 | 0 / 1760 | 0 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | -| gb300 | `trtllm_mla` | shared | sglang | sglang:3520 | 0 / 1760 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | -| h100_sxm | `flash_attention` | shared | sglang | sglang:3520 | 0 / 1760 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:3080 | 0 / 3080 | 0 | — | — | — | -| h200_sxm | `flash_attention` | shared | sglang | sglang:3080 | 0 / 3080 | 0 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:6160 | 0 / 4620 | 1 | — | — | — | +| b200_sxm | `trtllm_mla` | shared | sglang | sglang:6160 | 0 / 3080 | 4 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 3 | — | — | — | +| b300_sxm | `trtllm_mla` | shared | sglang | sglang:6160 | 0 / 3080 | 7 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 4 | — | — | — | +| gb200 | `trtllm_mla` | shared | sglang | sglang:6600 | 0 / 3960 | 0 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 4 | — | — | — | +| gb300 | `trtllm_mla` | shared | sglang | sglang:6600 | 0 / 3960 | 3 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 3 | — | — | — | +| h100_sxm | `flash_attention` | shared | sglang | sglang:6600 | 0 / 3080 | 1 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:6160 | 0 / 4620 | 4 | — | — | — | +| h200_sxm | `flash_attention` | shared | sglang | sglang:6160 | 0 / 3080 | 2 | — | — | — | | l40s | `default` | shared_fallback | trtllm | trtllm:2436 | 0 / 2436 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:880 | 0 / 880 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:1760 | 0 / 880 | 0 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:2420 | 0 / 1540 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 4 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:3300 | 0 / 1540 | 2 | — | — | — | ## `custom_allreduce_perf.parquet` @@ -151,34 +162,35 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | b200_sxm | `TRTLLM` | shared | trtllm | trtllm:207 | 0 / 138 | 0 | — | — | — | | b200_sxm | `vLLM_custom_eager` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | | b200_sxm | `vLLM_custom_graph` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | -| b300_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | -| b300_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | +| b300_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | +| b300_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | | b300_sxm | `TRTLLM` | shared | trtllm | trtllm:138 | 0 / 69 | 1 | — | — | — | -| b300_sxm | `vLLM_custom_eager` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | -| b300_sxm | `vLLM_custom_graph` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | -| b60 | `vLLM_custom_eager` | shared | vllm | vllm:207 | 0 / 69 | 23 | — | — | — | -| gb200 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | -| gb200 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | +| b300_sxm | `vLLM_custom_eager` | shared | vllm | vllm:207 | 0 / 69 | 0 | — | — | — | +| b300_sxm | `vLLM_custom_graph` | shared | vllm | vllm:207 | 0 / 69 | 0 | — | — | — | +| b60 | `vLLM_custom_eager` | shared | vllm | vllm:115 | 0 / 69 | 0 | — | — | — | +| b60 | `vLLM_custom_graph` | shared | vllm | vllm:46 | 0 / 46 | 0 | — | — | — | +| gb200 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | +| gb200 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | | gb200 | `TRTLLM` | shared | trtllm | trtllm:92 | 0 / 46 | 0 | — | — | — | -| gb200 | `vLLM_custom_eager` | shared | vllm | vllm:138 | 0 / 46 | 0 | — | — | — | -| gb200 | `vLLM_custom_graph` | shared | vllm | vllm:138 | 0 / 46 | 0 | — | — | — | -| gb300 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | -| gb300 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | +| gb200 | `vLLM_custom_eager` | shared | vllm | vllm:184 | 0 / 46 | 0 | — | — | — | +| gb200 | `vLLM_custom_graph` | shared | vllm | vllm:184 | 0 / 46 | 0 | — | — | — | +| gb300 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | +| gb300 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | | gb300 | `TRTLLM` | shared | trtllm | trtllm:92 | 0 / 46 | 0 | — | — | — | | gb300 | `TRTLLM_MNNVL_oneshot` | shared | trtllm | trtllm:19 | 0 / 19 | 0 | — | — | — | | gb300 | `TRTLLM_MNNVL_twoshot` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | -| gb300 | `vLLM_custom_eager` | shared | vllm | vllm:230 | 0 / 92 | 0 | — | — | — | -| gb300 | `vLLM_custom_graph` | shared | vllm | vllm:230 | 0 / 92 | 0 | — | — | — | -| h100_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | +| gb300 | `vLLM_custom_eager` | shared | vllm | vllm:276 | 0 / 92 | 0 | — | — | — | +| gb300 | `vLLM_custom_graph` | shared | vllm | vllm:276 | 0 / 92 | 0 | — | — | — | +| h100_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | | h100_sxm | `TRTLLM` | shared | trtllm | trtllm:276 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `vLLM_custom_eager` | shared | vllm | vllm:207 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `vLLM_custom_graph` | shared | vllm | vllm:207 | 0 / 69 | 1 | — | — | — | -| h200_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | -| h200_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `vLLM_custom_eager` | shared | vllm | vllm:276 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `vLLM_custom_graph` | shared | vllm | vllm:276 | 0 / 69 | 1 | — | — | — | +| h200_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | +| h200_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | | h200_sxm | `TRTLLM` | shared | trtllm | trtllm:276 | 0 / 69 | 0 | — | — | — | -| h200_sxm | `vLLM_custom_eager` | shared | vllm | vllm:276 | 0 / 69 | 0 | — | — | — | -| h200_sxm | `vLLM_custom_graph` | shared | vllm | vllm:276 | 0 / 69 | 1 | — | — | — | +| h200_sxm | `vLLM_custom_eager` | shared | vllm | vllm:345 | 0 / 69 | 0 | — | — | — | +| h200_sxm | `vLLM_custom_graph` | shared | vllm | vllm:345 | 0 / 69 | 1 | — | — | — | | l40s | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | | l40s | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 1 | — | — | — | | l40s | `TRTLLM` | shared | trtllm | trtllm:276 | 0 / 69 | 0 | — | — | — | @@ -194,89 +206,95 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:4392 | 0 / 4392 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:35080 | 0 / 35080 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:17096 | 0 / 17096 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:17104 | 0 / 17104 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:12963 | 0 / 12963 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:12960 | 0 / 12960 | 0 | — | — | — | -| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:4392 | 0 / 4392 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:27896 | 0 / 20940 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:16918 | 0 / 16918 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:17096 | 0 / 17096 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:6392 | 0 / 6392 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:6480 | 0 / 6480 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:4391 | 0 / 4391 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | -| gb200 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:46371 | 0 / 21048 | 0 | — | — | — | -| gb200 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:31196 | 0 / 16984 | 0 | — | — | — | -| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:25613 | 0 / 16984 | 0 | — | — | — | -| gb200 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:8272 | 0 / 6368 | 0 | — | — | — | -| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:7878 | 0 / 6368 | 0 | — | — | — | -| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:4392 | 0 / 4392 | 0 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | -| gb300 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:28064 | 0 / 21048 | 0 | — | — | — | -| gb300 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:16984 | 0 / 16984 | 0 | — | — | — | -| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:16984 | 0 / 16984 | 0 | — | — | — | -| gb300 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:6368 | 0 / 6368 | 0 | — | — | — | -| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:6368 | 0 / 6368 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:14668 | 0 / 14668 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:11654 | 0 / 7770 | 1 | — | — | — | -| h100_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:50621 | 0 / 31203 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:27320 | 0 / 13660 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:27312 | 0 / 13619 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:11037 | 0 / 11037 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:9512 | 0 / 9512 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:14816 | 0 / 14816 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 7808 | 10 | — | — | — | -| h200_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:32280 | 0 / 32280 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:13660 | 0 / 13660 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:13660 | 0 / 13660 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:11125 | 0 / 11125 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:11111 | 0 / 11111 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254766 | 0 / 159801 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:138371 | 0 / 87864 | 4 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:29274 | 0 / 29274 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:69992 | 0 / 35080 | 14 | — | — | — | +| b200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:34946 | 0 / 29790 | 3 | — | — | — | +| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:34716 | 0 / 29702 | 1 | — | — | — | +| b200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:25657 | 0 / 12963 | 5 | — | — | — | +| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:25654 | 0 / 12960 | 3 | — | — | — | +| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254692 | 0 / 159780 | 5 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:138360 | 0 / 87864 | 4 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:29277 | 0 / 14640 | 19 | — | — | — | +| b300_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:62869 | 0 / 34972 | 1 | — | — | — | +| b300_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:34605 | 0 / 29449 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:34946 | 0 / 29790 | 2 | — | — | — | +| b300_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:19086 | 0 / 19086 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:19174 | 0 / 19174 | 0 | — | — | — | +| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254692 | 0 / 159780 | 2 | — | — | — | +| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:138360 | 0 / 87864 | 7 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:29276 | 0 / 14640 | 18 | — | — | — | +| gb200 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:81451 | 0 / 35080 | 0 | — | — | — | +| gb200 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:49046 | 0 / 29678 | 0 | — | — | — | +| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:43463 | 0 / 29678 | 2 | — | — | — | +| gb200 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:20966 | 0 / 19062 | 0 | — | — | — | +| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:20572 | 0 / 19062 | 0 | — | — | — | +| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254692 | 0 / 159780 | 2 | — | — | — | +| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:138360 | 0 / 87864 | 3 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:29275 | 0 / 14640 | 19 | — | — | — | +| gb300 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:63144 | 0 / 35080 | 1 | — | — | — | +| gb300 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:34834 | 0 / 29678 | 0 | — | — | — | +| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:34834 | 0 / 29678 | 1 | — | — | — | +| gb300 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:19062 | 0 / 19062 | 0 | — | — | — | +| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:19062 | 0 / 19062 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:227400 | 0 / 143058 | 15 | — | — | — | +| h100_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:194364 | 0 / 128988 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:19444 | 0 / 7790 | 1 | — | — | — | +| h100_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:80601 | 0 / 31203 | 5 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:42717 | 0 / 24685 | 3 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:41017 | 0 / 23131 | 1 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:22083 | 0 / 11047 | 11 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:19024 | 0 / 9512 | 2 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:227700 | 0 / 143244 | 12 | — | — | — | +| h200_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:194364 | 0 / 128988 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:19520 | 0 / 7808 | 10 | — | — | — | +| h200_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:64510 | 0 / 32280 | 4 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:29171 | 0 / 24799 | 3 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:29157 | 0 / 24785 | 2 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:22146 | 0 / 11126 | 4 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:22216 | 0 / 11112 | 2 | — | — | — | +| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_SM120` | shared | vllm | vllm:107980 | 0 / 71660 | 0 | — | — | — | ## `dsa_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7705 | 0 / 7705 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:3294 | 0 / 3294 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:11040 | 0 / 11040 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:2448 | 0 / 2448 | 0 | — | — | — | -| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7576 | 0 / 7576 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:3277 | 0 / 3277 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:11037 | 0 / 11037 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7728 | 0 / 7728 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:3311 | 0 / 3311 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:11036 | 0 / 11036 | 0 | — | — | — | -| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | -| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | -| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7727 | 0 / 7727 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:3312 | 0 / 3312 | 0 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:11025 | 0 / 11025 | 0 | — | — | — | -| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | -| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:11679 | 0 / 11679 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 5888 | 2 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:3568 | 0 / 1784 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:3568 | 0 / 1784 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:11776 | 0 / 11776 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 5888 | 7 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:1784 | 0 / 1784 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:1784 | 0 / 1784 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40222 | 0 / 26706 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:19894 | 0 / 13248 | 0 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:22077 | 0 / 22077 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 2 | — | — | — | +| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:4912 | 0 / 2464 | 6 | — | — | — | +| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40584 | 0 / 26916 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:22070 | 0 / 11039 | 120 | — | — | — | +| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 3 | — | — | — | +| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:3688 | 0 / 3688 | 0 | — | — | — | +| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40584 | 0 / 26916 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:22070 | 0 / 11040 | 60 | — | — | — | +| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 2 | — | — | — | +| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:3688 | 0 / 3688 | 0 | — | — | — | +| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40584 | 0 / 26916 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:22062 | 0 / 11040 | 78 | — | — | — | +| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 3 | — | — | — | +| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:3688 | 0 / 3688 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:46782 | 0 / 31083 | 0 | — | — | — | +| h100_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:19548 | 0 / 13032 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:14720 | 0 / 5888 | 10 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:5396 | 0 / 3020 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:5396 | 0 / 3020 | 1 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:2456 | 0 / 1232 | 8 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:2456 | 0 / 1232 | 11 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:46998 | 0 / 31227 | 0 | — | — | — | +| h200_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:14720 | 0 / 5888 | 16 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:3612 | 0 / 3020 | 3 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:3612 | 0 / 3020 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:2456 | 0 / 1232 | 7 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:2456 | 0 / 1232 | 1 | — | — | — | +| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_SM120` | shared | vllm | vllm:11040 | 0 / 7360 | 0 | — | — | — | ## `dsv41_module_perf.parquet` @@ -307,116 +325,117 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5587 | 0 / 5587 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 20 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:43804 | 0 / 43804 | 0 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11544 | 0 / 8624 | 22 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:89689 | 0 / 45449 | 0 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5584 | 0 / 5584 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 9 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:53376 | 0 / 42610 | 0 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11544 | 0 / 8624 | 10 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:94650 | 0 / 45424 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11565 | 0 / 5790 | 17 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:52902 | 0 / 40384 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 5840 | 45 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:40592 | 0 / 40592 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11421 | 0 / 11421 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17510 | 0 / 8760 | 39 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:89024 | 0 / 67120 | 12 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17373 | 0 / 8624 | 35 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:137145 | 0 / 70160 | 6 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11419 | 0 / 11419 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17507 | 0 / 8760 | 27 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:100528 | 0 / 68354 | 11 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17371 | 0 / 8624 | 28 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:142106 | 0 / 70152 | 8 | — | — | — | +| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5739 | 0 / 5739 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17325 | 0 / 5790 | 47 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:92982 | 0 / 40384 | 9 | — | — | — | +| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5840 | 0 / 5840 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17510 | 0 / 5840 | 62 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:80976 | 0 / 40592 | 11 | — | — | — | +| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:5780 | 0 / 5780 | 0 | — | — | — | ## `dsv4_csa_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 9 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:2592 | 0 / 1820 | 12 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 6 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:2592 | 0 / 1820 | 6 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 10 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 29 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 17 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 4 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4136 | 0 / 1820 | 22 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 1 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 13 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 2 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4136 | 0 / 1820 | 17 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 3 | — | — | — | +| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 43 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:6396 | 0 / 3200 | 4 | — | — | — | +| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 91 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 3200 | 10 | — | — | — | +| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:1544 | 0 / 1544 | 0 | — | — | — | | rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:909 | 0 / 868 | 0 | — | — | — | ## `dsv4_csa_topk_calib_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | -| b200_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | -| b300_sxm | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | -| b300_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | -| gb200 | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | -| gb200 | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | -| gb300 | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | -| gb300 | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | -| h100_sxm | `topk_transform_v1` | shared | sglang | sglang:3686 | 0 / 3686 | 0 | — | — | — | -| h100_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | -| h200_sxm | `topk_transform_v1` | shared | sglang | sglang:3686 | 0 / 3686 | 0 | — | — | — | -| h200_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | - -## `dsv4_hca_attn_module_perf.parquet` - -| system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | -|---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 155 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 112 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 116 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 90 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 42 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 128 | — | — | — | +| b200_sxm | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | +| b200_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | +| b300_sxm | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | +| b300_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | +| gb200 | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | +| gb200 | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | +| gb300 | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | +| gb300 | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | +| h100_sxm | `topk_transform_v1` | shared | sglang | sglang:5750 | 0 / 3686 | 0 | — | — | — | +| h100_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | +| h200_sxm | `topk_transform_v1` | shared | sglang | sglang:5750 | 0 / 3686 | 0 | — | — | — | +| h200_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | ## `dsv4_hca_context_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5589 | 0 / 5589 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 17 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:47456 | 0 / 47456 | 0 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 43 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:61527 | 0 / 47550 | 0 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5584 | 0 / 5584 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 15 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:47456 | 0 / 47456 | 0 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 10 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:54924 | 0 / 47552 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11586 | 0 / 5793 | 42 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:40383 | 0 / 40383 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 5840 | 65 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:40592 | 0 / 40592 | 0 | — | — | — | -| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:40384 | 0 / 20192 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11429 | 0 / 11429 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 32 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:95152 | 0 / 71424 | 3 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 51 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:109223 | 0 / 71470 | 11 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11424 | 0 / 11424 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 19 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:95152 | 0 / 71424 | 9 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 37 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:102620 | 0 / 71472 | 11 | — | — | — | +| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5793 | 0 / 5793 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17379 | 0 / 5793 | 65 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:79855 | 0 / 40624 | 14 | — | — | — | +| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5840 | 0 / 5840 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 5840 | 118 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:81376 | 0 / 40784 | 8 | — | — | — | +| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:5810 | 0 / 5810 | 0 | — | — | — | +| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:79856 | 0 / 40048 | 0 | — | — | — | ## `dsv4_hca_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 10 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 20 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 6 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 7 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 10 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 47 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | -| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:1600 | 0 / 1600 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 17 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 10 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 24 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 4 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 12 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 11 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 29 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 14 | — | — | — | +| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 56 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 3200 | 5 | — | — | — | +| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 123 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 3200 | 2 | — | — | — | +| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:1544 | 0 / 1544 | 0 | — | — | — | +| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:4800 | 0 / 3200 | 0 | — | — | — | ## `dsv4_megamoe_module_perf.parquet` @@ -430,47 +449,50 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:2181 | 0 / 2181 | 0 | — | — | — | -| b200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 226 | — | — | — | -| b300_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:4362 | 0 / 2181 | 0 | — | — | — | -| b300_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 263 | — | — | — | -| gb200 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:4261 | 0 / 2181 | 0 | — | — | — | -| gb200 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 86 | — | — | — | -| gb300 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:4362 | 0 / 2181 | 0 | — | — | — | -| gb300 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 149 | — | — | — | -| h100_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:2055 | 0 / 2055 | 0 | — | — | — | -| h100_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 77 | — | — | — | -| h200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:2055 | 0 / 2055 | 0 | — | — | — | -| h200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 192 | — | — | — | +| b200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:6971 | 0 / 4790 | 0 | — | — | — | +| b200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 718 | — | — | — | +| b300_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:9152 | 0 / 4790 | 0 | — | — | — | +| b300_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 879 | — | — | — | +| gb200 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:9051 | 0 / 4790 | 0 | — | — | — | +| gb200 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 398 | — | — | — | +| gb300 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:9152 | 0 / 4790 | 0 | — | — | — | +| gb300 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 471 | — | — | — | +| h100_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:6165 | 0 / 4110 | 0 | — | — | — | +| h100_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 351 | — | — | — | +| h200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:6165 | 0 / 4110 | 0 | — | — | — | +| h200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 432 | — | — | — | +| rtx_pro_6000_server | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:3918 | 0 / 3918 | 0 | — | — | — | ## `encoder_attention_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | -| b200_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 32 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| b200_sxm | `flash_attention_v4` | shared | sglang | sglang:15358 | 0 / 7679 | 12 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 15358 | 0 | — | — | — | +| b200_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 62 | — | — | — | +| b300_sxm | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 21 | — | — | — | | b300_sxm | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| b300_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 28 | — | — | — | -| gb200 | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | -| gb200 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 26 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| b300_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 53 | — | — | — | +| gb200 | `flash_attention_v4` | shared | sglang | sglang:15358 | 0 / 7679 | 14 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 14 | — | — | — | +| gb200 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 49 | — | — | — | +| gb300 | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 17 | — | — | — | | gb300 | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| gb300 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 19 | — | — | — | -| h100_sxm | `flash_attention_v3` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | -| h100_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:15358 | 0 / 7679 | 5 | — | — | — | -| h200_sxm | `flash_attention_v3` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | -| h200_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:15358 | 0 / 7679 | 17 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| l40s | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:7679 | 0 / 7679 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:7679 | 0 / 7679 | 0 | — | — | — | +| gb300 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 45 | — | — | — | +| h100_sxm | `flash_attention_v3` | shared | sglang | sglang:15358 | 0 / 7679 | 3 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 16 | — | — | — | +| h100_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:23037 | 0 / 7679 | 21 | — | — | — | +| h200_sxm | `flash_attention_v3` | shared | sglang | sglang:16458 | 0 / 8779 | 6 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 16 | — | — | — | +| h200_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:24137 | 0 / 8779 | 60 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 96 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:15358 | 0 / 7679 | 121 | — | — | — | +| l40s | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:15358 | 0 / 7679 | 228 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 27 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:15358 | 0 / 7679 | 15 | — | — | — | +| rtx_pro_6000_server | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:15358 | 0 / 7679 | 16 | — | — | — | ## `gdn_perf.parquet` @@ -479,58 +501,77 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `causal_conv1d_fn` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | | a100_sxm | `causal_conv1d_update` | shared | sglang | sglang:88 | 0 / 88 | 0 | — | — | — | | a100_sxm | `fused_recurrent_gated_delta_rule` | shared | sglang | sglang:86 | 0 / 86 | 0 | — | — | — | -| b200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:854, vllm:9408 | 4529 / 5558 | 8 | 6.1 | 12.0 | 28.2 | -| b200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 550 | 2 | 9.8 | 26.8 | 33.8 | -| b200_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:854 | 0 / 5383 | 0 | — | — | — | -| b200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 0 | — | — | — | +| b200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5470, vllm:14112 | 4616 / 5558 | 38 | 89.8 | 208.1 | 277.9 | +| b200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 550 | 2 | 65.6 | 101.6 | 161.9 | +| b200_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:854 | 0 / 5383 | 1 | — | — | — | +| b200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4616, vllm:14112 | 4616 / 4704 | 0 | 9.9 | 23.1 | 30.6 | | b200_sxm | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | +| b200_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | b200_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| b200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 3 | 0.5 | 4.5 | 9.3 | -| b300_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:837, vllm:9408 | 4567 / 4704 | 3 | 8.1 | 138.5 | 277.1 | -| b300_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 462 | 0 | 10.2 | 70.9 | 129.4 | -| b300_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:836 | 799 / 4566 | 0 | 31.2 | 46.0 | 50.6 | -| b300_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 1 | — | — | — | +| b200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 7 | 0.4 | 3.4 | 9.3 | +| b200_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5487, vllm:14112 | 4650 / 4704 | 18 | 92.8 | 208.2 | 278.3 | +| b300_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 462 | 0 | 62.9 | 98.4 | 160.3 | +| b300_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:836 | 799 / 4566 | 0 | 39.6 | 56.6 | 61.1 | +| b300_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4650, vllm:14112 | 4650 / 4704 | 1 | 8.4 | 22.2 | 30.1 | | b300_sxm | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | +| b300_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | b300_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| b300_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 0 | 1.2 | 6.3 | 9.3 | -| gb200 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:854, vllm:9408 | 4559 / 4704 | 5 | 3.5 | 137.4 | 276.6 | -| gb200 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 462 | 2 | 9.8 | 112.0 | 134.2 | -| gb200 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:854 | 824 / 4559 | 0 | 22.8 | 54.0 | 68.0 | -| gb200 | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 0 | — | — | — | +| b300_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 2 | 0.3 | 3.6 | 5.3 | +| b300_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| gb200 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5470, vllm:14112 | 4616 / 4704 | 11 | 97.8 | 210.7 | 278.2 | +| gb200 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 462 | 2 | 83.7 | 135.7 | 163.8 | +| gb200 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:854 | 824 / 4559 | 1 | 13.5 | 49.1 | 70.1 | +| gb200 | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4616, vllm:14112 | 4616 / 4704 | 1 | 11.4 | 27.0 | 33.3 | | gb200 | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | +| gb200 | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | gb200 | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| gb200 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 0 | 1.0 | 4.4 | 6.1 | -| gb300 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:837, vllm:9408 | 4567 / 4704 | 7 | 5.6 | 140.3 | 276.0 | -| gb300 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 462 | 1 | 10.2 | 143.1 | 182.1 | -| gb300 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:836 | 799 / 4566 | 0 | 20.3 | 59.3 | 67.4 | -| gb300 | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 1 | — | — | — | +| gb200 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 0 | 2.7 | 4.7 | 5.9 | +| gb200 | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| gb300 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5487, vllm:14112 | 4650 / 4704 | 12 | 101.6 | 208.0 | 277.4 | +| gb300 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 462 | 1 | 82.1 | 153.6 | 198.4 | +| gb300 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:836 | 799 / 4566 | 0 | 17.6 | 41.9 | 55.4 | +| gb300 | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4650, vllm:14112 | 4650 / 4704 | 1 | 10.2 | 25.7 | 32.5 | | gb300 | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | +| gb300 | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | gb300 | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| gb300 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 1 | 1.4 | 7.2 | 18.1 | -| h100_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:816, vllm:8824 | 4004 / 4683 | 0 | 9.0 | 134.5 | 278.5 | -| h100_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:868 | 407 / 461 | 0 | 17.1 | 100.9 | 135.4 | +| gb300 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 2 | 0.4 | 3.6 | 5.3 | +| gb300 | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5325, vllm:13507 | 4529 / 4694 | 5 | 92.8 | 220.5 | 281.1 | +| h100_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:1329 | 461 / 462 | 1 | 79.4 | 115.8 | 160.1 | | h100_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:806 | 806 / 4004 | 0 | 42.2 | 59.4 | 69.5 | -| h100_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:8824 | 0 / 4683 | 0 | — | — | — | +| h100_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4509, vllm:13507 | 4498 / 4694 | 0 | 18.5 | 65.1 | 83.8 | +| h100_sxm | `flashinfer_chunk_gated_delta_rule` | shared | sglang | sglang:4529 | 0 / 4529 | 0 | — | — | — | +| h100_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | h100_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| h100_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:862 | 405 / 457 | 0 | 4.3 | 11.0 | 13.4 | -| h200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:798, vllm:8847 | 4017 / 4703 | 8 | 5.2 | 194.5 | 279.0 | -| h200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:868 | 407 / 461 | 2 | 14.7 | 100.8 | 133.3 | +| h100_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:1319 | 457 / 457 | 0 | 1.0 | 3.0 | 5.7 | +| h100_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5400, vllm:13550 | 4602 / 4703 | 14 | 93.7 | 220.8 | 281.1 | +| h200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:1329 | 461 / 462 | 2 | 82.2 | 116.6 | 151.5 | | h200_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:797 | 785 / 4016 | 0 | 44.5 | 130.0 | 145.6 | -| h200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:8847 | 0 / 4703 | 1 | — | — | — | +| h200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4584, vllm:13550 | 4584 / 4703 | 1 | 18.1 | 60.7 | 67.5 | +| h200_sxm | `flashinfer_chunk_gated_delta_rule` | shared | sglang | sglang:4529 | 0 / 4529 | 0 | — | — | — | +| h200_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | h200_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| h200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:862 | 405 / 457 | 2 | 1.9 | 71.1 | 104.6 | -| l40s | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:786, vllm:3950 | 3956 / 4100 | 0 | 2.4 | 147.1 | 276.1 | -| l40s | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:407 | 407 / 407 | 0 | 14.1 | 137.2 | 168.2 | -| l40s | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:786 | 786 / 4004 | 0 | 21.5 | 104.5 | 118.2 | -| l40s | `chunk_gated_delta_rule_triton` | shared | vllm | vllm:3935 | 0 / 3935 | 0 | — | — | — | +| h200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:1319 | 457 / 457 | 5 | 0.7 | 3.1 | 6.6 | +| h200_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| l40s | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5186, vllm:8364 | 4511 / 4638 | 74 | 100.5 | 197.6 | 275.8 | +| l40s | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:868 | 461 / 462 | 3 | 109.5 | 151.3 | 181.4 | +| l40s | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:8533, trtllm:5186 | 4400 / 4529 | 2 | 7.5 | 33.9 | 65.3 | +| l40s | `chunk_gated_delta_rule_triton` | shared | vllm | vllm:8330 | 0 / 4395 | 0 | — | — | — | +| l40s | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | l40s | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| l40s | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:405 | 405 / 405 | 0 | 0.4 | 4.3 | 11.7 | -| rtx_pro_6000_server | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:824, vllm:3952 | 3965 / 4102 | 0 | 3.1 | 137.7 | 272.9 | -| rtx_pro_6000_server | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:407 | 407 / 407 | 0 | 9.2 | 105.0 | 139.6 | -| rtx_pro_6000_server | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:824 | 824 / 4004 | 0 | 17.7 | 84.7 | 97.0 | +| l40s | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:862 | 457 / 457 | 3 | 0.4 | 3.0 | 24.3 | +| l40s | `fused_sigmoid_gating_delta_rule_update_triton` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| rtx_pro_6000_server | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5353, vllm:8653 | 4529 / 4701 | 29 | 86.9 | 199.3 | 274.0 | +| rtx_pro_6000_server | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:868 | 461 / 462 | 1 | 70.8 | 94.0 | 146.9 | +| rtx_pro_6000_server | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:8533, trtllm:824 | 824 / 4529 | 4 | 21.2 | 87.6 | 99.6 | +| rtx_pro_6000_server | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4529, vllm:4701 | 4529 / 4701 | 0 | 21.8 | 53.8 | 103.1 | | rtx_pro_6000_server | `chunk_gated_delta_rule_triton` | shared | vllm | vllm:3935 | 0 / 3935 | 0 | — | — | — | +| rtx_pro_6000_server | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | rtx_pro_6000_server | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| rtx_pro_6000_server | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:405 | 405 / 405 | 0 | 0.3 | 8.2 | 23.2 | +| rtx_pro_6000_server | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:862 | 457 / 457 | 0 | 0.3 | 5.7 | 24.9 | +| rtx_pro_6000_server | `fused_sigmoid_gating_delta_rule_update_triton` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | ## `gemm_perf.parquet` @@ -541,92 +582,108 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0` | shared | trtllm | trtllm:6048 | 0 / 6048 | 0 | — | — | — | | a100_sxm | `trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0` | shared | trtllm | trtllm:12096 | 0 / 12096 | 0 | — | — | — | | a100_sxm | `vllm_default` | shared | vllm | vllm:9240 | 0 / 9240 | 0 | — | — | — | -| b200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 81 | — | — | — | -| b200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | -| b200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | -| b200_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 482 | — | — | — | -| b200_sxm | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | +| b200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:146668 | 0 / 37518 | 222 | — | — | — | +| b200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:15466 | 0 / 5328 | 14 | — | — | — | +| b200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:93166 | 0 / 31080 | 38 | — | — | — | +| b200_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:146668 | 0 / 37518 | 695 | — | — | — | +| b200_sxm | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 58610 | 0 | — | — | — | | b200_sxm | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | b200_sxm | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| b200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 15 | — | — | — | -| b200_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 26 | — | — | — | -| b200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71558 | 0 / 35816 | 85 | — | — | — | -| b200_sxm | `sglang_torch_linear` | shared | sglang | sglang:71632 | 0 / 35890 | 86 | — | — | — | -| b200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 214 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | -| b300_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 52 | — | — | — | -| b300_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | -| b300_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | -| b300_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 387 | — | — | — | -| b300_sxm | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | -| b300_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 10 | — | — | — | -| b300_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 30 | — | — | — | -| b300_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71706 | 0 / 35964 | 49 | — | — | — | -| b300_sxm | `sglang_torch_linear` | shared | sglang | sglang:71854 | 0 / 36112 | 104 | — | — | — | -| b300_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 306 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | -| b60 | `vllm_default` | shared | vllm | vllm:40530 | 0 / 22176 | 1 | — | — | — | -| gb200 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 67 | — | — | — | -| gb200 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | -| gb200 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | -| gb200 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 281 | — | — | — | -| gb200 | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | +| b200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:88652 | 0 / 29600 | 36 | — | — | — | +| b200_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:88652 | 0 / 29600 | 70 | — | — | — | +| b200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:108410 | 0 / 36852 | 175 | — | — | — | +| b200_sxm | `sglang_torch_linear` | shared | sglang | sglang:109150 | 0 / 37518 | 190 | — | — | — | +| b200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:146668 | 0 / 37518 | 489 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 204292 | 0 | — | — | — | +| b300_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 298 | — | — | — | +| b300_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:20572 | 0 / 5624 | 28 | — | — | — | +| b300_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:124542 | 0 / 31450 | 213 | — | — | — | +| b300_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 929 | — | — | — | +| b300_sxm | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 29342 | 23 | — | — | — | +| b300_sxm | `sglang_cutedsl_tgv_bf16` | shared | sglang | sglang:1317 | 0 / 1317 | 0 | — | — | — | +| b300_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:120028 | 0 / 31450 | 64 | — | — | — | +| b300_sxm | `sglang_flashinfer_bf16_direct` | shared | sglang | sglang:8 | 0 / 8 | 0 | — | — | — | +| b300_sxm | `sglang_flashinfer_bf16_splitk` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:125282 | 0 / 36704 | 122 | — | — | — | +| b300_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:108558 | 0 / 36852 | 73 | — | — | — | +| b300_sxm | `sglang_torch_linear` | shared | sglang | sglang:144642 | 0 / 38215 | 330 | — | — | — | +| b300_sxm | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | +| b300_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:183298 | 0 / 38258 | 848 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 103404 | 72 | — | — | — | +| b60 | `vllm_default` | shared | vllm | vllm:61698 | 0 / 23058 | 1 | — | — | — | +| gb200 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 357 | — | — | — | +| gb200 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:20572 | 0 / 5624 | 46 | — | — | — | +| gb200 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:124542 | 0 / 31450 | 173 | — | — | — | +| gb200 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 812 | — | — | — | +| gb200 | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 29342 | 18 | — | — | — | | gb200 | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | gb200 | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| gb200 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 14 | — | — | — | -| gb200 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 21 | — | — | — | -| gb200 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71558 | 0 / 35816 | 43 | — | — | — | -| gb200 | `sglang_torch_linear` | shared | sglang | sglang:71632 | 0 / 35890 | 61 | — | — | — | -| gb200 | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 266 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | -| gb300 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 98 | — | — | — | -| gb300 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | -| gb300 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | -| gb300 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 247 | — | — | — | -| gb300 | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | -| gb300 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 4 | — | — | — | -| gb300 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 46 | — | — | — | -| gb300 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71558 | 0 / 35816 | 69 | — | — | — | -| gb300 | `sglang_torch_linear` | shared | sglang | sglang:71632 | 0 / 35890 | 85 | — | — | — | -| gb300 | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 228 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | -| h100_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 1 | — | — | — | -| h100_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:5328 | 0 / 5328 | 0 | — | — | — | -| h100_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:23940 | 0 / 23940 | 0 | — | — | — | -| h100_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:7140 | 0 / 7140 | 0 | — | — | — | +| gb200 | `sglang_cutedsl_tgv_bf16` | shared | sglang | sglang:1317 | 0 / 1317 | 0 | — | — | — | +| gb200 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:120028 | 0 / 31450 | 78 | — | — | — | +| gb200 | `sglang_flashinfer_bf16_direct` | shared | sglang | sglang:8 | 0 / 8 | 0 | — | — | — | +| gb200 | `sglang_flashinfer_bf16_splitk` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:125282 | 0 / 36704 | 134 | — | — | — | +| gb200 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:108410 | 0 / 36852 | 56 | — | — | — | +| gb200 | `sglang_torch_linear` | shared | sglang | sglang:144420 | 0 / 38215 | 347 | — | — | — | +| gb200 | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | +| gb200 | `torch.nn.functional.linear` | shared | vllm | vllm:183298 | 0 / 38258 | 850 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 103404 | 108 | — | — | — | +| gb300 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:183150 | 0 / 38184 | 437 | — | — | — | +| gb300 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:20572 | 0 / 5624 | 41 | — | — | — | +| gb300 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:124542 | 0 / 31450 | 118 | — | — | — | +| gb300 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:183150 | 0 / 38184 | 723 | — | — | — | +| gb300 | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 29342 | 16 | — | — | — | +| gb300 | `sglang_cutedsl_tgv_bf16` | shared | sglang | sglang:1295 | 0 / 1295 | 0 | — | — | — | +| gb300 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:120028 | 0 / 31450 | 67 | — | — | — | +| gb300 | `sglang_flashinfer_bf16_direct` | shared | sglang | sglang:8 | 0 / 8 | 0 | — | — | — | +| gb300 | `sglang_flashinfer_bf16_splitk` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:125134 | 0 / 36556 | 145 | — | — | — | +| gb300 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:144892 | 0 / 37518 | 126 | — | — | — | +| gb300 | `sglang_torch_linear` | shared | sglang | sglang:144294 | 0 / 38163 | 428 | — | — | — | +| gb300 | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1850 | 0 / 1850 | 0 | — | — | — | +| gb300 | `torch.nn.functional.linear` | shared | vllm | vllm:183150 | 0 / 38184 | 888 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 103404 | 97 | — | — | — | +| h100_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:147407 | 0 / 38258 | 97 | — | — | — | +| h100_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:15762 | 0 / 5624 | 25 | — | — | — | +| h100_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:72018 | 0 / 24225 | 90 | — | — | — | +| h100_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:21484 | 0 / 7225 | 60 | — | — | — | | h100_sxm | `sglang` | shared | sglang | sglang:35280 | 0 / 35280 | 0 | — | — | — | | h100_sxm | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | h100_sxm | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h100_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:29526 | 0 / 29526 | 0 | — | — | — | -| h100_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| h100_sxm | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| h100_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:73260 | 0 / 37518 | 23 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:100668 | 0 / 100668 | 0 | — | — | — | -| h200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 48 | — | — | — | -| h200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:5328 | 0 / 5328 | 0 | — | — | — | -| h200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:23940 | 0 / 23940 | 0 | — | — | — | -| h200_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:7140 | 0 / 7140 | 0 | — | — | — | +| h100_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:90501 | 0 / 31450 | 43 | — | — | — | +| h100_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 8 | — | — | — | +| h100_sxm | `sglang_torch_linear` | shared | sglang | sglang:109878 | 0 / 38258 | 42 | — | — | — | +| h100_sxm | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | +| h100_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:147408 | 0 / 38258 | 147 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:203852 | 0 / 103184 | 30 | — | — | — | +| h200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:147400 | 0 / 38258 | 193 | — | — | — | +| h200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:15762 | 0 / 5624 | 44 | — | — | — | +| h200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:72048 | 0 / 24225 | 128 | — | — | — | +| h200_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:21488 | 0 / 7225 | 81 | — | — | — | | h200_sxm | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | h200_sxm | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:29526 | 0 / 29526 | 0 | — | — | — | -| h200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| h200_sxm | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| h200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:73260 | 0 / 37518 | 67 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:100668 | 0 / 100668 | 0 | — | — | — | -| l40s | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:35718 | 0 / 35718 | 0 | — | — | — | +| h200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:90502 | 0 / 31450 | 36 | — | — | — | +| h200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 4 | — | — | — | +| h200_sxm | `sglang_torch_linear` | shared | sglang | sglang:109890 | 0 / 38258 | 53 | — | — | — | +| h200_sxm | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | +| h200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:147408 | 0 / 38258 | 266 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:203852 | 0 / 103184 | 49 | — | — | — | +| l40s | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73217 | 0 / 37516 | 34 | — | — | — | +| l40s | `MarlinFP8ScaledMMLinearKernel` | shared | vllm | vllm:37518 | 0 / 37518 | 0 | — | — | — | | l40s | `TritonFp8BlockScaledMMKernel` | shared | vllm | vllm:35676 | 0 / 35676 | 0 | — | — | — | -| l40s | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| l40s | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| l40s | `torch.nn.functional.linear` | shared | vllm | vllm:35672 | 0 / 35672 | 0 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:100668 | 0 / 100668 | 0 | — | — | — | -| rtx_pro_6000_server | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:35742 | 0 / 35742 | 0 | — | — | — | -| rtx_pro_6000_server | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:518 | 0 / 518 | 0 | — | — | — | -| rtx_pro_6000_server | `FlashInferCutlassNvFp4LinearKernel` | shared | vllm | vllm:35742 | 0 / 35742 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_flashinfer_cutlass_nvfp4` | shared | sglang | sglang:29526 | 0 / 29526 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | -| rtx_pro_6000_server | `torch.nn.functional.linear` | shared | vllm | vllm:35742 | 0 / 35742 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:130156 | 0 / 130156 | 0 | — | — | — | +| l40s | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 469 | — | — | — | +| l40s | `sglang_torch_linear` | shared | sglang | sglang:73260 | 0 / 37518 | 371 | — | — | — | +| l40s | `torch.nn.functional.linear` | shared | vllm | vllm:73168 | 0 / 37505 | 1491 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:203964 | 0 / 103304 | 917 | — | — | — | +| rtx_pro_6000_server | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 39 | — | — | — | +| rtx_pro_6000_server | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:1036 | 0 / 518 | 0 | — | — | — | +| rtx_pro_6000_server | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:31080 | 0 / 31080 | 0 | — | — | — | +| rtx_pro_6000_server | `FlashInferCutlassNvFp4LinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 123 | — | — | — | +| rtx_pro_6000_server | `sglang_flashinfer_cutlass_nvfp4` | shared | sglang | sglang:59126 | 0 / 29600 | 77 | — | — | — | +| rtx_pro_6000_server | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 113 | — | — | — | +| rtx_pro_6000_server | `sglang_torch_linear` | shared | sglang | sglang:73260 | 0 / 37518 | 148 | — | — | — | +| rtx_pro_6000_server | `torch.nn.functional.linear` | shared | vllm | vllm:73260 | 0 / 37518 | 103 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:263014 | 0 / 132866 | 100 | — | — | — | ## `generation_attention_perf.parquet` @@ -635,117 +692,146 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `flash_attention` | shared | sglang | sglang:5093 | 0 / 5093 | 0 | — | — | — | | a100_sxm | `torch_flow` | shared | trtllm | trtllm:5026 | 0 / 5026 | 0 | — | — | — | | a100_sxm | `vllm_flash_attn` | shared | vllm | vllm:5431 | 0 / 5431 | 0 | — | — | — | -| b200_sxm | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:48314 | 0 / 48314 | 0 | — | — | — | +| b200_sxm | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 11 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:98310 | 0 / 98310 | 0 | — | — | — | | b200_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | -| b200_sxm | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | -| b200_sxm | `trtllm_mha` | shared | sglang | sglang:37643 | 0 / 37643 | 0 | — | — | — | +| b200_sxm | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 19 | — | — | — | +| b200_sxm | `trtllm_mha` | shared | sglang | sglang:76995 | 0 / 39358 | 78 | — | — | — | +| b200_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:2224 | 0 / 2224 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer` | shared | vllm | vllm:36240 | 0 / 36240 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | +| b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:60916 | 0 / 60916 | 0 | — | — | — | -| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:6684 | 0 / 4456 | 17 | — | — | — | -| b300_sxm | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:46980 | 0 / 46980 | 0 | — | — | — | +| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:9835 | 0 / 5384 | 30 | — | — | — | +| b300_sxm | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 16 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:96976 | 0 / 49996 | 75 | — | — | — | | b300_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | -| b300_sxm | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | -| b300_sxm | `trtllm_mha` | shared | sglang | sglang:37490 | 0 / 37490 | 0 | — | — | — | +| b300_sxm | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 18 | — | — | — | +| b300_sxm | `trtllm_mha` | shared | sglang | sglang:76840 | 0 / 39360 | 68 | — | — | — | +| b300_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:2227 | 0 / 2227 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | +| b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:58510 | 0 / 58510 | 0 | — | — | — | -| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:4456 | 0 / 2228 | 7 | — | — | — | -| b60 | `vllm_flash_attn` | shared | vllm | vllm:57209 | 0 / 30948 | 3 | — | — | — | -| gb200 | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:48316 | 0 / 48316 | 0 | — | — | — | -| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | -| gb200 | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | -| gb200 | `trtllm_mha` | shared | sglang | sglang:36082 | 0 / 36082 | 0 | — | — | — | +| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:7600 | 0 / 3156 | 14 | — | — | — | +| b60 | `vllm_flash_attn` | shared | vllm | vllm:54675 | 0 / 31096 | 0 | — | — | — | +| gb200 | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 4 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:98312 | 0 / 49996 | 123 | — | — | — | +| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:7688 | 0 / 4148 | 7 | — | — | — | +| gb200 | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 10 | — | — | — | +| gb200 | `trtllm_mha` | shared | sglang | sglang:75434 | 0 / 39360 | 54 | — | — | — | +| gb200 | `vllm_flash_attn_fa2` | shared | vllm | vllm:2225 | 0 / 2225 | 0 | — | — | — | | gb200 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | +| gb200 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | gb200 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:59844 | 0 / 59844 | 0 | — | — | — | -| gb200 | `vllm_triton_attn` | shared | vllm | vllm:4456 | 0 / 2228 | 7 | — | — | — | -| gb300 | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:48316 | 0 / 48316 | 0 | — | — | — | -| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | -| gb300 | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | -| gb300 | `trtllm_mha` | shared | sglang | sglang:36078 | 0 / 36078 | 0 | — | — | — | +| gb200 | `vllm_triton_attn` | shared | vllm | vllm:7601 | 0 / 3156 | 15 | — | — | — | +| gb300 | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 4 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:98312 | 0 / 49996 | 88 | — | — | — | +| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:7688 | 0 / 4148 | 12 | — | — | — | +| gb300 | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 8 | — | — | — | +| gb300 | `trtllm_mha` | shared | sglang | sglang:75428 | 0 / 39354 | 58 | — | — | — | +| gb300 | `vllm_flash_attn_fa2` | shared | vllm | vllm:2226 | 0 / 2226 | 0 | — | — | — | | gb300 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | +| gb300 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | gb300 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:59844 | 0 / 59844 | 0 | — | — | — | -| gb300 | `vllm_triton_attn` | shared | vllm | vllm:4456 | 0 / 2228 | 7 | — | — | — | -| h100_sxm | `fa3` | shared | sglang | sglang:37176 | 0 / 37176 | 0 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:52033 | 0 / 52033 | 0 | — | — | — | -| h100_sxm | `triton` | shared | sglang | sglang:3292 | 0 / 3292 | 0 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:127146 | 0 / 65780 | 23 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1728 | 0 / 928 | 1 | — | — | — | -| h200_sxm | `fa3` | shared | sglang | sglang:38510 | 0 / 38510 | 0 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:53367 | 0 / 53367 | 0 | — | — | — | -| h200_sxm | `triton` | shared | sglang | sglang:3292 | 0 / 3292 | 0 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:128480 | 0 / 65780 | 485 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1728 | 0 / 928 | 4 | — | — | — | -| l40s | `flashinfer` | shared | sglang | sglang:29158 | 0 / 29158 | 0 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:52714 | 0 / 52714 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:9384 | 0 / 9384 | 0 | — | — | — | -| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:30679 | 0 / 30679 | 0 | — | — | — | -| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:27616 | 0 / 27616 | 0 | — | — | — | -| l40s | `vllm_triton_attn` | shared | vllm | vllm:2228 | 0 / 2228 | 0 | — | — | — | -| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:24694 | 0 / 24694 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:51690 | 0 / 51690 | 0 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:15188 | 0 / 15188 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:30677 | 0 / 30677 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:27396 | 0 / 27396 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:2228 | 0 / 2228 | 0 | — | — | — | +| gb300 | `vllm_triton_attn` | shared | vllm | vllm:7603 | 0 / 3156 | 11 | — | — | — | +| h100_sxm | `fa3` | shared | sglang | sglang:78420 | 0 / 41244 | 20 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:107080 | 0 / 55047 | 170 | — | — | — | +| h100_sxm | `triton` | shared | sglang | sglang:7120 | 0 / 3828 | 3 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:192926 | 0 / 65780 | 416 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:2656 | 0 / 928 | 1 | — | — | — | +| h100_sxm | `vllm_triton_attn` | shared | vllm | vllm:928 | 0 / 928 | 0 | — | — | — | +| h200_sxm | `fa3` | shared | sglang | sglang:79754 | 0 / 41244 | 29 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:108414 | 0 / 55047 | 219 | — | — | — | +| h200_sxm | `triton` | shared | sglang | sglang:7120 | 0 / 3828 | 7 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:194260 | 0 / 65780 | 1415 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:2656 | 0 / 928 | 5 | — | — | — | +| h200_sxm | `vllm_triton_attn` | shared | vllm | vllm:928 | 0 / 928 | 0 | — | — | — | +| l40s | `flashinfer` | shared | sglang | sglang:57836 | 0 / 33226 | 624 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:108442 | 0 / 55728 | 1314 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:25108 | 0 / 15724 | 290 | — | — | — | +| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:63551 | 0 / 32887 | 1907 | — | — | — | +| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:58928 | 0 / 31312 | 1428 | — | — | — | +| l40s | `vllm_triton_attn` | shared | vllm | vllm:5384 | 0 / 3156 | 122 | — | — | — | +| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:53808 | 0 / 29114 | 77 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:106394 | 0 / 54704 | 160 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:30912 | 0 / 15724 | 45 | — | — | — | +| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:63543 | 0 / 32888 | 81 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:28324 | 0 / 28324 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_flashinfertrtllmapidecode_xqa` | shared | vllm | vllm:29214 | 0 / 29214 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:5376 | 0 / 3153 | 8 | — | — | — | ## `generation_mla_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | a100_sxm | `default` | shared_fallback | trtllm | trtllm:5068 | 0 / 5068 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| b200_sxm | `trtllm_mla` | shared | sglang | sglang:2896 | 0 / 2896 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| b300_sxm | `trtllm_mla` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| gb200 | `trtllm_mla` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| gb300 | `trtllm_mla` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| h100_sxm | `flash_attention` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| h200_sxm | `flash_attention` | shared | sglang | sglang:2896 | 0 / 2896 | 0 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 7602 | 0 | — | — | — | +| b200_sxm | `trtllm_mla` | shared | sglang | sglang:7602 | 0 / 6154 | 2 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 4706 | 12 | — | — | — | +| b300_sxm | `trtllm_mla` | shared | sglang | sglang:10498 | 0 / 6154 | 1 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 4706 | 8 | — | — | — | +| gb200 | `trtllm_mla` | shared | sglang | sglang:10498 | 0 / 6154 | 4 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 4706 | 9 | — | — | — | +| gb300 | `trtllm_mla` | shared | sglang | sglang:10498 | 0 / 6154 | 2 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:7964 | 0 / 5068 | 16 | — | — | — | +| h100_sxm | `flash_attention` | shared | sglang | sglang:10860 | 0 / 5068 | 4 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:7964 | 0 / 5068 | 12 | — | — | — | +| h200_sxm | `flash_attention` | shared | sglang | sglang:7964 | 0 / 5068 | 1 | — | — | — | | l40s | `default` | shared_fallback | trtllm | trtllm:5068 | 0 / 5068 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:1162 | 0 / 1162 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:2400 | 0 / 1200 | 0 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:3470 | 0 / 2308 | 10 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:7964 | 0 / 5068 | 5 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:4928 | 0 / 2528 | 5 | — | — | — | + +## `glm53_attention_module_perf.parquet` + +| system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | +|---|---|---|---|---|---|---|---|---|---| +| b300_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:926 | 0 / 926 | 0 | — | — | — | +| b300_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:926 | 0 / 926 | 0 | — | — | — | +| b300_sxm | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | shared | vllm | vllm:2315 | 0 / 2315 | 0 | — | — | — | +| gb200 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:463 | 0 / 463 | 0 | — | — | — | +| gb200 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:925 | 0 / 925 | 0 | — | — | — | +| gb200 | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | shared | vllm | vllm:1389 | 0 / 1389 | 0 | — | — | — | +| gb300 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:926 | 0 / 926 | 0 | — | — | — | +| gb300 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:1366 | 0 / 1366 | 0 | — | — | — | +| gb300 | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | shared | vllm | vllm:2315 | 0 / 2315 | 0 | — | — | — | +| h100_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:433 | 0 / 433 | 0 | — | — | — | +| h100_sxm | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseSM90Impl` | shared | vllm | vllm:449 | 0 / 449 | 0 | — | — | — | +| h200_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:911 | 0 / 911 | 0 | — | — | — | +| h200_sxm | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseSM90Impl` | shared | vllm | vllm:925 | 0 / 925 | 0 | — | — | — | ## `glm5_dsa_attn_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | -| b300_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3577 | 0 / 3332 | 0 | — | — | — | -| gb200 | `flash_mla_sparse_fwd` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | -| gb300 | `flash_mla_sparse_fwd` | shared | sglang | sglang:3550 | 0 / 3332 | 0 | — | — | — | -| h100_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | -| h200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | +| b200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | +| b300_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6910 | 0 / 3333 | 0 | — | — | — | +| gb200 | `flash_mla_sparse_fwd` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | +| gb300 | `flash_mla_sparse_fwd` | shared | sglang | sglang:6883 | 0 / 3333 | 0 | — | — | — | +| h100_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | +| h200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | ## `glm5_mqa_logits_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | -| b300_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | -| gb200 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | -| gb300 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | -| h100_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | -| h200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | +| b200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | +| b300_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | +| gb200 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | +| gb300 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | +| h100_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | +| h200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | ## `glm5_topk_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | -| b300_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | -| gb200 | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | -| gb300 | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | -| h100_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:3606 | 0 / 3606 | 0 | — | — | — | -| h200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:3606 | 0 / 3606 | 0 | — | — | — | +| b200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | +| b300_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | +| gb200 | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | +| gb300 | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | +| h100_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:7214 | 0 / 3608 | 0 | — | — | — | +| h200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:7214 | 0 / 3608 | 0 | — | — | — | ## `kda_perf.parquet` @@ -760,50 +846,58 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | b200_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | | b200_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 13.5 | 68.4 | 68.9 | | b200_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_fn` | shared | sglang, vllm | sglang:448, vllm:420 | 420 / 448 | 0 | 19.6 | 122.7 | 142.1 | | b300_sxm | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 184.0 | 193.6 | 199.4 | -| b300_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:33, vllm:152 | 33 / 152 | 0 | 16.4 | 37.4 | 52.4 | -| b300_sxm | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | -| b300_sxm | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:77, vllm:224 | 77 / 224 | 0 | 6.0 | 52.5 | 57.5 | +| b300_sxm | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | +| b300_sxm | `flashkda_fwd` | shared | vllm | vllm:847 | 0 / 847 | 0 | — | — | — | | b300_sxm | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | | b300_sxm | `fused_kda_decode_mtp_dspark` | shared | sglang | sglang:107 | 0 / 107 | 0 | — | — | — | -| b300_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | +| b300_sxm | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | | b300_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 12.7 | 62.7 | 69.6 | +| b300_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:44 | 0 / 44 | 0 | — | — | — | | b300_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | +| gb200 | `causal_conv1d_fn` | shared | sglang, vllm | sglang:447, vllm:419 | 419 / 447 | 0 | 30.3 | 129.5 | 139.1 | | gb200 | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 193.3 | 197.1 | 197.6 | -| gb200 | `causal_conv1d_update` | shared | sglang, vllm | sglang:33, vllm:152 | 33 / 152 | 0 | 13.1 | 27.5 | 28.5 | -| gb200 | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | -| gb200 | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | +| gb200 | `causal_conv1d_update` | shared | sglang, vllm | sglang:77, vllm:224 | 77 / 224 | 0 | 3.6 | 49.0 | 54.0 | +| gb200 | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | +| gb200 | `flashkda_fwd` | shared | vllm | vllm:844 | 0 / 844 | 0 | — | — | — | | gb200 | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | | gb200 | `fused_kda_decode_mtp_dspark` | shared | sglang | sglang:107 | 0 / 107 | 0 | — | — | — | -| gb200 | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | +| gb200 | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | | gb200 | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 6.7 | 69.5 | 77.5 | +| gb200 | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:44 | 0 / 44 | 0 | — | — | — | | gb200 | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | +| gb300 | `causal_conv1d_fn` | shared | sglang, vllm | sglang:232, vllm:327 | 218 / 341 | 0 | 30.3 | 127.7 | 140.0 | | gb300 | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:848 | 396 / 448 | 0 | 24.8 | 196.3 | 197.7 | -| gb300 | `causal_conv1d_update` | shared | sglang, vllm | sglang:33, vllm:304 | 33 / 152 | 0 | 18.5 | 34.4 | 39.6 | -| gb300 | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | -| gb300 | `flashkda_fwd` | shared | vllm | vllm:847 | 0 / 447 | 0 | — | — | — | +| gb300 | `causal_conv1d_update` | shared | sglang, vllm | sglang:55, vllm:358 | 55 / 206 | 0 | 9.5 | 41.5 | 52.2 | +| gb300 | `chunk_kda` | shared | sglang | sglang:619 | 0 / 619 | 0 | — | — | — | +| gb300 | `flashkda_fwd` | shared | vllm | vllm:1173 | 0 / 773 | 0 | — | — | — | | gb300 | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | | gb300 | `fused_kda_decode_mtp_dspark` | shared | sglang | sglang:107 | 0 / 107 | 0 | — | — | — | -| gb300 | `fused_recurrent_kda` | shared | vllm | vllm:216 | 0 / 108 | 0 | — | — | — | +| gb300 | `fused_recurrent_kda` | shared | vllm | vllm:269 | 0 / 161 | 0 | — | — | — | | gb300 | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:86 | 32 / 44 | 1 | 11.8 | 61.4 | 65.1 | +| gb300 | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:22 | 0 / 22 | 0 | — | — | — | | gb300 | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_fn` | shared | sglang, vllm | sglang:444, vllm:416 | 416 / 444 | 0 | 18.2 | 122.7 | 176.4 | | h100_sxm | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 181.9 | 193.1 | 194.3 | -| h100_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:141, vllm:152 | 141 / 152 | 0 | 19.2 | 60.3 | 75.9 | -| h100_sxm | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | -| h100_sxm | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:185, vllm:224 | 185 / 224 | 0 | 13.4 | 57.9 | 75.9 | +| h100_sxm | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | +| h100_sxm | `flashkda_fwd` | shared | vllm | vllm:838 | 0 / 838 | 0 | — | — | — | | h100_sxm | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | -| h100_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | +| h100_sxm | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | | h100_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 16.7 | 66.6 | 72.7 | -| h100_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:108 | 0 / 108 | 0 | — | — | — | +| h100_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:152 | 0 / 152 | 0 | — | — | — | | h100_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_fn` | shared | sglang, vllm | sglang:447, vllm:419 | 419 / 447 | 0 | 17.7 | 119.3 | 179.3 | | h200_sxm | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 189.5 | 196.1 | 196.6 | -| h200_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:141, vllm:152 | 141 / 152 | 0 | 21.0 | 70.4 | 79.3 | -| h200_sxm | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | -| h200_sxm | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:185, vllm:224 | 185 / 224 | 0 | 15.7 | 67.0 | 79.3 | +| h200_sxm | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | +| h200_sxm | `flashkda_fwd` | shared | vllm | vllm:844 | 0 / 844 | 0 | — | — | — | | h200_sxm | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | -| h200_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | +| h200_sxm | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | | h200_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 19.4 | 49.9 | 52.6 | -| h200_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:108 | 0 / 108 | 0 | — | — | — | +| h200_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:152 | 0 / 152 | 0 | — | — | — | | h200_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | | l40s | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 190.5 | 196.8 | 198.0 | | l40s | `causal_conv1d_update` | shared | sglang, vllm | sglang:141, vllm:152 | 141 / 152 | 0 | 17.0 | 53.6 | 102.9 | @@ -827,185 +921,274 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | b200_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| b200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| b200_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | +| b200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 132 | 0 | — | — | — | +| b200_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | +| b200_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | b300_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:586 | 0 / 586 | 0 | — | — | — | -| b300_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:624 | 0 / 624 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | +| b300_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:624 | 0 / 624 | 0 | — | — | — | +| b300_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | gb200 | `causal_conv1d_fn` | shared | trtllm | trtllm:624 | 0 / 624 | 0 | — | — | — | -| gb200 | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| gb200 | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | +| gb200 | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | +| gb200 | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | +| gb200 | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | gb300 | `causal_conv1d_fn` | shared | trtllm | trtllm:586 | 0 / 586 | 0 | — | — | — | -| gb300 | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| gb300 | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:577 | 0 / 577 | 0 | — | — | — | +| gb300 | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | +| gb300 | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:576 | 0 / 576 | 0 | — | — | — | +| gb300 | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | h100_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:592 | 0 / 592 | 0 | — | — | — | -| h100_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:579 | 0 / 579 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 1 | — | — | — | +| h100_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:579 | 0 / 579 | 0 | — | — | — | +| h100_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | h200_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| h200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:609 | 0 / 609 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 1 | — | — | — | +| h200_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:609 | 0 / 609 | 0 | — | — | — | +| h200_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | l40s | `causal_conv1d_fn` | shared | trtllm | trtllm:559 | 0 / 559 | 0 | — | — | — | -| l40s | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| l40s | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:559 | 0 / 559 | 0 | — | — | — | +| l40s | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 2 | — | — | — | +| l40s | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:559 | 0 / 559 | 0 | — | — | — | +| l40s | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | rtx_pro_6000_server | `causal_conv1d_fn` | shared | trtllm | trtllm:598 | 0 / 598 | 0 | — | — | — | -| rtx_pro_6000_server | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| rtx_pro_6000_server | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:592 | 0 / 592 | 0 | — | — | — | +| rtx_pro_6000_server | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | +| rtx_pro_6000_server | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:592 | 0 / 592 | 0 | — | — | — | +| rtx_pro_6000_server | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | ## `mhc_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| b200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | -| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 8 | — | — | — | -| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:134 | 0 / 70 | 2 | — | — | — | -| b300_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| b300_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| b300_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | -| b300_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:28 | 0 / 28 | 0 | — | — | — | -| b300_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | -| b300_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | -| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 9 | — | — | — | -| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:133 | 0 / 70 | 5 | — | — | — | -| gb200 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| gb200 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| gb200 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | -| gb200 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | -| gb200 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | -| gb200 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | -| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 6 | — | — | — | -| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 7 | — | — | — | -| gb300 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| gb300 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| gb300 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | -| gb300 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:28 | 0 / 28 | 0 | — | — | — | -| gb300 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | -| gb300 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | -| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 10 | — | — | — | -| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 13 | — | — | — | -| h100_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:68 | 0 / 68 | 0 | — | — | — | -| h100_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:69 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:24 | 0 / 24 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | -| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:138 | 0 / 69 | 5 | — | — | — | -| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:24 | 0 / 24 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:19 | 0 / 19 | 0 | — | — | — | -| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 5 | — | — | — | -| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 8 | — | — | — | -| l40s | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:67 | 0 / 67 | 0 | — | — | — | -| l40s | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:65 | 0 / 65 | 0 | — | — | — | -| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:67 | 0 / 67 | 0 | — | — | — | -| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:69 | 0 / 69 | 0 | — | — | — | +| b200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 8 | — | — | — | +| b200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 6 | — | — | — | +| b200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 140 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:50 | 0 / 50 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:58 | 0 / 58 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 30 | 0 | — | — | — | +| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 29 | — | — | — | +| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:204 | 0 / 70 | 15 | — | — | — | +| b300_sxm | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 12 | — | — | — | +| b300_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 4 | — | — | — | +| b300_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 3 | — | — | — | +| b300_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:52 | 0 / 28 | 0 | — | — | — | +| b300_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:58 | 0 / 32 | 0 | — | — | — | +| b300_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 16 | 0 | — | — | — | +| b300_sxm | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | +| b300_sxm | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | +| b300_sxm | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 28 | — | — | — | +| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:203 | 0 / 70 | 15 | — | — | — | +| b300_sxm | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| b300_sxm | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 3 | — | — | — | +| gb200 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 4 | — | — | — | +| gb200 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 4 | — | — | — | +| gb200 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:48 | 0 / 26 | 0 | — | — | — | +| gb200 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:60 | 0 / 33 | 0 | — | — | — | +| gb200 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 16 | 0 | — | — | — | +| gb200 | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | +| gb200 | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | +| gb200 | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 21 | — | — | — | +| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 17 | — | — | — | +| gb200 | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb200 | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 2 | — | — | — | +| gb300 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 4 | — | — | — | +| gb300 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 0 | — | — | — | +| gb300 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:51 | 0 / 28 | 0 | — | — | — | +| gb300 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:59 | 0 / 33 | 0 | — | — | — | +| gb300 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 16 | 3 | — | — | — | +| gb300 | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | +| gb300 | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | +| gb300 | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 20 | — | — | — | +| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 26 | — | — | — | +| gb300 | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| gb300 | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:136 | 0 / 68 | 0 | — | — | — | +| h100_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:139 | 0 / 70 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:138 | 0 / 69 | 4 | — | — | — | +| h100_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:45 | 0 / 24 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:61 | 0 / 34 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:28 | 0 / 16 | 0 | — | — | — | +| h100_sxm | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | +| h100_sxm | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:28 | 0 / 28 | 0 | — | — | — | +| h100_sxm | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:207 | 0 / 69 | 11 | — | — | — | +| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 2 | — | — | — | +| h100_sxm | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h100_sxm | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:48 | 0 / 26 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:59 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:31 | 0 / 19 | 0 | — | — | — | +| h200_sxm | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | +| h200_sxm | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | +| h200_sxm | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 21 | — | — | — | +| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 13 | — | — | — | +| h200_sxm | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| h200_sxm | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | +| l40s | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:134 | 0 / 67 | 6 | — | — | — | +| l40s | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:130 | 0 / 65 | 1 | — | — | — | +| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:134 | 0 / 67 | 6 | — | — | — | +| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:138 | 0 / 69 | 2 | — | — | — | | rtx_pro_6000_server | `sglang_tilelang_mhc_post` | shared | sglang | sglang:67 | 0 / 67 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_torch_mhc_pre` | shared | sglang | sglang:67 | 0 / 67 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:69 | 0 / 69 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:67 | 0 / 67 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:138 | 0 / 69 | 13 | — | — | — | +| rtx_pro_6000_server | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:18 | 0 / 18 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:37 | 0 / 37 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:79 | 0 / 67 | 3 | — | — | — | +| rtx_pro_6000_server | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:69 | 0 / 69 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:70 | 0 / 70 | 0 | — | — | — | ## `mla_bmm_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | a100_sxm | `default` | shared_fallback | sglang, trtllm | sglang:424, trtllm:424 | 224 / 224 | 0 | 1.4 | 18.4 | 115.9 | +| b200_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | b200_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | -| b200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | -| b200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 424 | 0 | — | — | — | -| b200_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 424 | 0 | — | — | — | -| b200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 1 | — | — | — | +| b200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1272 | 0 / 336 | 1 | — | — | — | +| b200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 760 | 0 | — | — | — | +| b200_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 760 | 0 | — | — | — | +| b200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 5 | — | — | — | +| b300_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | b300_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| b300_sxm | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| b300_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| b300_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| b300_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 3 | — | — | — | +| b300_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 1 | — | — | — | +| b300_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 2 | — | — | — | +| b300_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| b300_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 9 | — | — | — | +| gb200 | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | gb200 | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| gb200 | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| gb200 | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| gb200 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| gb200 | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 1 | — | — | — | +| gb200 | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | +| gb200 | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| gb200 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 336 | 1 | — | — | — | +| gb200 | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 2 | — | — | — | +| gb300 | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | gb300 | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| gb300 | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| gb300 | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| gb300 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| gb300 | `vllm_torch_bmm` | shared | vllm | vllm:1708 | 0 / 336 | 4 | — | — | — | +| gb300 | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | +| gb300 | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| gb300 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 336 | 2 | — | — | — | +| gb300 | `vllm_torch_bmm` | shared | vllm | vllm:2344 | 0 / 336 | 5 | — | — | — | +| h100_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | h100_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| h100_sxm | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| h100_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| h100_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| h100_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 1 | — | — | — | +| h100_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | +| h100_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 1 | — | — | — | +| h100_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| h100_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 2 | — | — | — | +| h200_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | h200_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | -| h200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | -| h200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| h200_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| h200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 2 | — | — | — | +| h200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1060 | 0 / 336 | 0 | — | — | — | +| h200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| h200_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| h200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 5 | — | — | — | | l40s | `default` | shared_fallback | trtllm | trtllm:848 | 0 / 448 | 0 | — | — | — | +| l40s | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | l40s | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | -| l40s | `sglang_torch_bmm` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | -| l40s | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| l40s | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| l40s | `sglang_torch_bmm` | shared | sglang | sglang:1060 | 0 / 336 | 2 | — | — | — | +| l40s | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| l40s | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| l40s | `vllm_torch_bmm` | shared | vllm | vllm:636 | 0 / 336 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 1 | — | — | — | +| rtx_pro_6000_server | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_torch_bmm` | shared | vllm | vllm:636 | 0 / 336 | 0 | — | — | — | ## `mla_context_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:1503 | 0 / 855 | 5 | — | — | — | -| b200_sxm | `FLASH_ATTN` | shared | vllm | vllm:13560 | 0 / 8136 | 58 | — | — | — | -| b200_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:6780 | 0 / 4068 | 27 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:2475 | 0 / 972 | 35 | — | — | — | +| b200_sxm | `FLASH_ATTN` | shared | vllm | vllm:21696 | 0 / 8136 | 118 | — | — | — | +| b200_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:10848 | 0 / 4068 | 58 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 11712 | 0 | — | — | — | | b200_sxm | `mla_trtllm_mla` | shared | sglang | sglang:218 | 0 / 218 | 0 | — | — | — | -| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:1503 | 0 / 855 | 0 | — | — | — | -| b300_sxm | `FLASH_ATTN` | shared | vllm | vllm:13560 | 0 / 8136 | 0 | — | — | — | -| b300_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:6780 | 0 / 4068 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | +| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:2475 | 0 / 972 | 17 | — | — | — | +| b300_sxm | `FLASH_ATTN` | shared | vllm | vllm:21696 | 0 / 8136 | 27 | — | — | — | +| b300_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:10848 | 0 / 4068 | 8 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 5856 | 44 | — | — | — | | b300_sxm | `mla_trtllm_mla` | shared | sglang | sglang:218 | 0 / 218 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:1503 | 0 / 855 | 0 | — | — | — | -| gb200 | `FLASH_ATTN` | shared | vllm | vllm:13560 | 0 / 8136 | 0 | — | — | — | -| gb200 | `TRTLLM_RAGGED` | shared | vllm | vllm:6780 | 0 / 4068 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | -| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:3119 | 0 / 1827 | 6 | — | — | — | -| gb300 | `FLASH_ATTN` | shared | vllm | vllm:27061 | 0 / 16272 | 49 | — | — | — | -| gb300 | `TRTLLM_RAGGED` | shared | vllm | vllm:13523 | 0 / 8136 | 15 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | -| h100_sxm | `FLASHMLA` | shared | vllm | vllm:1746 | 0 / 1044 | 0 | — | — | — | -| h100_sxm | `FLASH_ATTN` | shared | vllm | vllm:5440 | 0 / 3264 | 0 | — | — | — | -| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:2520 | 0 / 1512 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:3866 | 0 / 3866 | 0 | — | — | — | -| h200_sxm | `FLASHMLA` | shared | vllm | vllm:1746 | 0 / 1044 | 0 | — | — | — | -| h200_sxm | `FLASH_ATTN` | shared | vllm | vllm:5440 | 0 / 3264 | 0 | — | — | — | -| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:2520 | 0 / 1512 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:3869 | 0 / 3869 | 0 | — | — | — | -| l40s | `FLASH_ATTN` | shared | vllm | vllm:1790 | 0 / 1790 | 0 | — | — | — | -| l40s | `TRITON_MLA` | shared | vllm | vllm:144 | 0 / 144 | 0 | — | — | — | -| rtx_pro_6000_server | `FLASH_ATTN` | shared | vllm | vllm:3616 | 0 / 3616 | 0 | — | — | — | -| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:162 | 0 / 162 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:5828 | 0 / 5828 | 0 | — | — | — | +| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:2475 | 0 / 972 | 16 | — | — | — | +| gb200 | `FLASH_ATTN` | shared | vllm | vllm:21696 | 0 / 8136 | 26 | — | — | — | +| gb200 | `TRTLLM_RAGGED` | shared | vllm | vllm:10848 | 0 / 4068 | 13 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 5856 | 46 | — | — | — | +| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:4091 | 0 / 1944 | 15 | — | — | — | +| gb300 | `FLASH_ATTN` | shared | vllm | vllm:35197 | 0 / 16272 | 98 | — | — | — | +| gb300 | `TRTLLM_RAGGED` | shared | vllm | vllm:17591 | 0 / 8136 | 30 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 5856 | 26 | — | — | — | +| h100_sxm | `FLASHMLA` | shared | vllm | vllm:2826 | 0 / 1080 | 11 | — | — | — | +| h100_sxm | `FLASH_ATTN` | shared | vllm | vllm:8704 | 0 / 3264 | 9 | — | — | — | +| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:4032 | 0 / 1512 | 20 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:7740 | 0 / 3880 | 14 | — | — | — | +| h200_sxm | `FLASHMLA` | shared | vllm | vllm:2826 | 0 / 1080 | 4 | — | — | — | +| h200_sxm | `FLASH_ATTN` | shared | vllm | vllm:8704 | 0 / 3264 | 0 | — | — | — | +| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:4032 | 0 / 1512 | 5 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:7739 | 0 / 3879 | 7 | — | — | — | +| l40s | `FLASH_ATTN` | shared | vllm | vllm:4502 | 0 / 2712 | 0 | — | — | — | +| l40s | `TRITON_MLA` | shared | vllm | vllm:360 | 0 / 216 | 0 | — | — | — | +| rtx_pro_6000_server | `FLASH_ATTN` | shared | vllm | vllm:11752 | 0 / 8136 | 0 | — | — | — | +| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:513 | 0 / 351 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:9719 | 0 / 5837 | 24 | — | — | — | ## `mla_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:20232 | 0 / 11400 | 70 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:8831 | 0 / 8831 | 0 | — | — | — | -| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:20232 | 0 / 11400 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:20232 | 0 / 11400 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | -| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:42212 | 0 / 24648 | 30 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | -| h100_sxm | `FLASHMLA` | shared | vllm | vllm:7360 | 0 / 4416 | 0 | — | — | — | -| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:7240 | 0 / 4344 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:5888 | 0 / 5888 | 0 | — | — | — | -| h200_sxm | `FLASHMLA` | shared | vllm | vllm:7360 | 0 / 4416 | 0 | — | — | — | -| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:7360 | 0 / 4416 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:5888 | 0 / 5888 | 0 | — | — | — | -| l40s | `TRITON_MLA` | shared | vllm | vllm:2656 | 0 / 2656 | 0 | — | — | — | -| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:2990 | 0 / 2990 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:33480 | 0 / 13248 | 359 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:17662 | 0 / 17662 | 0 | — | — | — | +| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:33480 | 0 / 13248 | 196 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:17663 | 0 / 8832 | 256 | — | — | — | +| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:33480 | 0 / 13248 | 161 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:17663 | 0 / 8832 | 204 | — | — | — | +| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:55460 | 0 / 26496 | 252 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:17664 | 0 / 8832 | 182 | — | — | — | +| h100_sxm | `FLASHMLA` | shared | vllm | vllm:11776 | 0 / 4416 | 25 | — | — | — | +| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:11584 | 0 / 4344 | 99 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:11776 | 0 / 5888 | 199 | — | — | — | +| h200_sxm | `FLASHMLA` | shared | vllm | vllm:11776 | 0 / 4416 | 38 | — | — | — | +| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:11776 | 0 / 4416 | 39 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:11776 | 0 / 5888 | 244 | — | — | — | +| l40s | `TRITON_MLA` | shared | vllm | vllm:6977 | 0 / 4321 | 0 | — | — | — | +| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:10122 | 0 / 7132 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:14719 | 0 / 8832 | 31 | — | — | — | ## `moe_a2a_perf.parquet` @@ -1026,218 +1209,252 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:36531 | 0 / 33453 | 0 | — | — | — | | a100_sxm | `sglang_marlin_moe` | shared | sglang | sglang:19035 | 0 / 16686 | 0 | — | — | — | | a100_sxm | `vllm_fused_moe` | shared | vllm | vllm:6360 | 0 / 6360 | 0 | — | — | — | -| b200_sxm | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:10854 | 0 / 10854 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:36045 | 0 / 36045 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | -| b200_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:124821 | 0 / 124821 | 0 | — | — | — | -| b200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:36936 | 0 / 36936 | 0 | — | — | — | -| b200_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:5214 | 0 / 5214 | 0 | — | — | — | +| b200_sxm | `deepgemm` | shared | trtllm | trtllm:79080 | 0 / 79080 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:17334 | 0 / 13122 | 5 | — | — | — | +| b200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 239760 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:75510 | 0 / 75510 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 50868 | 0 | — | — | — | +| b200_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:241292 | 0 / 137294 | 5 | — | — | — | +| b200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:80352 | 0 / 43416 | 4 | — | — | — | +| b200_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:28056 | 0 / 22872 | 0 | — | — | — | | b200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:6156 | 0 / 3402 | 4 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:29786 | 0 / 15101 | 13 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:10287 | 0 / 4131 | 7 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:45318 | 0 / 16154 | 34 | — | — | — | | b200_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | +| b200_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_trtllm_moe_fp4` | shared | vllm | vllm:10287 | 0 / 9234 | 0 | — | — | — | -| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 1 | — | — | — | -| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 12 | — | — | — | +| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 3 | — | — | — | +| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:37422 | 0 / 12474 | 21 | — | — | — | +| b200_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2754 | 0 / 2754 | 0 | — | — | — | | b200_sxm | `vllm_fused_moe` | shared | vllm | vllm:83407 | 0 / 80329 | 0 | — | — | — | -| b200_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 6 | — | — | — | +| b200_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 8 | — | — | — | | b200_sxm | `vllm_marlin_int4_moe` | shared | vllm | vllm:10209 | 0 / 9237 | 0 | — | — | — | -| b200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 1 | — | — | — | -| b200_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24573 | 0 / 12503 | 17 | — | — | — | +| b200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 5 | — | — | — | +| b200_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:36738 | 0 / 12503 | 39 | — | — | — | | b200_sxm | `vllm_mxfp4_moe` | shared | vllm | vllm:1138 | 0 / 1138 | 0 | — | — | — | -| b200_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 3 | — | — | — | -| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 0 | — | — | — | +| b200_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 4 | — | — | — | +| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 5 | — | — | — | | b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25720 | 0 / 13556 | 9 | — | — | — | -| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| b200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| b300_sxm | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | -| b300_sxm | `moe_torch_flow` | shared | trtllm | trtllm:12472 | 0 / 7371 | 9 | — | — | — | -| b300_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | -| b300_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | -| b300_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:198607 | 0 / 118584 | 0 | — | — | — | -| b300_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:61582 | 0 / 36936 | 0 | — | — | — | -| b300_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:9720 | 0 / 5184 | 0 | — | — | — | +| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:16180 | 0 / 15127 | 4 | — | — | — | +| b200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | +| b300_sxm | `deepgemm` | shared | trtllm | trtllm:79212 | 0 / 40497 | 57 | — | — | — | +| b300_sxm | `moe_torch_flow` | shared | trtllm | trtllm:19843 | 0 / 7371 | 18 | — | — | — | +| b300_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 125712 | 123 | — | — | — | +| b300_sxm | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:39465 | 0 / 39465 | 0 | — | — | — | +| b300_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 27945 | 1 | — | — | — | +| b300_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:320425 | 0 / 136404 | 15 | — | — | — | +| b300_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:104998 | 0 / 43416 | 5 | — | — | — | +| b300_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:32562 | 0 / 22842 | 0 | — | — | — | | b300_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:4050 | 0 / 3402 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:28785 | 0 / 15127 | 23 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:8181 | 0 / 4131 | 1 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:44266 | 0 / 16180 | 38 | — | — | — | | b300_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 5 | — | — | — | -| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 21 | — | — | — | -| b300_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 5 | — | — | — | -| b300_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 1 | — | — | — | -| b300_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:23821 | 0 / 12555 | 13 | — | — | — | -| b300_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 3 | — | — | — | -| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 1 | — | — | — | +| b300_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 7 | — | — | — | +| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:38232 | 0 / 13284 | 33 | — | — | — | +| b300_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2835 | 0 / 2835 | 0 | — | — | — | +| b300_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 7 | — | — | — | +| b300_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 2 | — | — | — | +| b300_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:36116 | 0 / 12555 | 26 | — | — | — | +| b300_sxm | `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:891 | 0 / 891 | 0 | — | — | — | +| b300_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 3 | — | — | — | +| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 7 | — | — | — | | b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25728 | 0 / 13582 | 15 | — | — | — | -| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| b300_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| b60 | `vllm_xpu_moe` | shared | vllm | vllm:1836 | 0 / 1404 | 0 | — | — | — | +| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:16102 | 0 / 15049 | 0 | — | — | — | +| b300_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | +| b60 | `vllm_xpu_moe` | shared | vllm | vllm:3996 | 0 / 2808 | 0 | — | — | — | | b60 | `vllm_xpu_moe_mxfp4` | shared | vllm | vllm:2592 | 0 / 1296 | 0 | — | — | — | -| gb200 | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | -| gb200 | `moe_torch_flow` | shared | trtllm | trtllm:12312 | 0 / 7371 | 4 | — | — | — | -| gb200 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | -| gb200 | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:36045 | 0 / 36045 | 0 | — | — | — | -| gb200 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | -| gb200 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:159372 | 0 / 124818 | 0 | — | — | — | -| gb200 | `sglang_fused_moe_triton` | shared | sglang | sglang:50625 | 0 / 36936 | 0 | — | — | — | -| gb200 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:7563 | 0 / 5214 | 0 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | +| gb200 | `deepgemm` | shared | trtllm | trtllm:79299 | 0 / 40500 | 60 | — | — | — | +| gb200 | `moe_torch_flow` | shared | trtllm | trtllm:19683 | 0 / 7371 | 10 | — | — | — | +| gb200 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 125712 | 124 | — | — | — | +| gb200 | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:75510 | 0 / 39465 | 48 | — | — | — | +| gb200 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 27945 | 1 | — | — | — | +| gb200 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:281187 | 0 / 142638 | 3 | — | — | — | +| gb200 | `sglang_fused_moe_triton` | shared | sglang | sglang:94041 | 0 / 43416 | 2 | — | — | — | +| gb200 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:30405 | 0 / 22872 | 0 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 1 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:6156 | 0 / 3402 | 5 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:29864 | 0 / 15179 | 23 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:10287 | 0 / 4131 | 5 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:45343 | 0 / 16232 | 40 | — | — | — | | gb200 | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 2 | — | — | — | -| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 13 | — | — | — | -| gb200 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 1 | — | — | — | -| gb200 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 1 | — | — | — | -| gb200 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:23693 | 0 / 12555 | 21 | — | — | — | -| gb200 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 1 | — | — | — | -| gb200 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 3 | — | — | — | +| gb200 | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 3 | — | — | — | +| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:38232 | 0 / 13284 | 35 | — | — | — | +| gb200 | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2835 | 0 / 2835 | 0 | — | — | — | +| gb200 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 5 | — | — | — | +| gb200 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 2 | — | — | — | +| gb200 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:35988 | 0 / 12555 | 41 | — | — | — | +| gb200 | `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:891 | 0 / 891 | 0 | — | — | — | +| gb200 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 1 | — | — | — | +| gb200 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 5 | — | — | — | | gb200 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25876 | 0 / 13556 | 12 | — | — | — | -| gb200 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1026 | 0 / 1026 | 0 | — | — | — | -| gb200 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| gb300 | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | -| gb300 | `moe_torch_flow` | shared | trtllm | trtllm:12474 | 0 / 7371 | 6 | — | — | — | -| gb300 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | -| gb300 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | -| gb300 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:162504 | 0 / 124818 | 0 | — | — | — | -| gb300 | `sglang_fused_moe_triton` | shared | sglang | sglang:50627 | 0 / 36936 | 0 | — | — | — | -| gb300 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:7533 | 0 / 5184 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | +| gb200 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:15945 | 0 / 14919 | 1 | — | — | — | +| gb200 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | +| gb300 | `deepgemm` | shared | trtllm | trtllm:79299 | 0 / 40500 | 66 | — | — | — | +| gb300 | `moe_torch_flow` | shared | trtllm | trtllm:19845 | 0 / 7371 | 17 | — | — | — | +| gb300 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 125712 | 152 | — | — | — | +| gb300 | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:39465 | 0 / 39465 | 0 | — | — | — | +| gb300 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 27945 | 1 | — | — | — | +| gb300 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:284303 | 0 / 142619 | 3 | — | — | — | +| gb300 | `sglang_fused_moe_triton` | shared | sglang | sglang:94043 | 0 / 43416 | 1 | — | — | — | +| gb300 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:30375 | 0 / 22842 | 0 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 0 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:4050 | 0 / 3402 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:28785 | 0 / 15127 | 23 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:8181 | 0 / 4131 | 0 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:44269 | 0 / 16180 | 40 | — | — | — | | gb300 | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 4 | — | — | — | -| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 9 | — | — | — | -| gb300 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 4 | — | — | — | -| gb300 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 2 | — | — | — | -| gb300 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:23850 | 0 / 12555 | 20 | — | — | — | -| gb300 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 4 | — | — | — | -| gb300 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 3 | — | — | — | +| gb300 | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 5 | — | — | — | +| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:38232 | 0 / 13284 | 34 | — | — | — | +| gb300 | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2835 | 0 / 2835 | 0 | — | — | — | +| gb300 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 7 | — | — | — | +| gb300 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 5 | — | — | — | +| gb300 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:36327 | 0 / 12555 | 46 | — | — | — | +| gb300 | `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:891 | 0 / 891 | 0 | — | — | — | +| gb300 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 4 | — | — | — | +| gb300 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 5 | — | — | — | | gb300 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25849 | 0 / 13608 | 15 | — | — | — | -| gb300 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1026 | 0 / 1026 | 0 | — | — | — | -| gb300 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| h100_sxm | `moe_torch_flow` | shared | trtllm | trtllm:1539 | 0 / 1053 | 1 | — | — | — | -| h100_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:185328 | 0 / 129357 | 16 | — | — | — | -| h100_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:32643 | 0 / 24462 | 9 | — | — | — | -| h100_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:8601 | 0 / 8601 | 0 | — | — | — | -| h100_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:157464 | 0 / 112509 | 1 | — | — | — | -| h100_sxm | `sglang_marlin_moe` | shared | sglang | sglang:22113 | 0 / 17415 | 0 | — | — | — | +| gb300 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:16049 | 0 / 15023 | 0 | — | — | — | +| gb300 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | +| h100_sxm | `moe_torch_flow` | shared | trtllm | trtllm:2592 | 0 / 1053 | 1 | — | — | — | +| h100_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:301482 | 0 / 134622 | 95 | — | — | — | +| h100_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:52164 | 0 / 25191 | 9 | — | — | — | +| h100_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:17187 | 0 / 8601 | 0 | — | — | — | +| h100_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:256932 | 0 / 124497 | 10 | — | — | — | +| h100_sxm | `sglang_marlin_moe` | shared | sglang | sglang:28269 | 0 / 20493 | 0 | — | — | — | | h100_sxm | `sglang_marlin_moe_situ_as_silu` | shared | sglang | sglang:541 | 0 / 541 | 0 | — | — | — | | h100_sxm | `sglang_mxfp4_humming_moe` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h100_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | +| h100_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 0 | — | — | — | | h100_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| h100_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:19521 | 0 / 10287 | 3 | — | — | — | -| h100_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:12549 | 0 / 6561 | 3 | — | — | — | -| h100_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 4 | — | — | — | -| h100_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:29727 | 0 / 15309 | 4 | — | — | — | -| h100_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:34559 | 0 / 18306 | 11 | — | — | — | -| h200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:2268 | 0 / 1134 | 1 | — | — | — | -| h200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:176580 | 0 / 129357 | 12 | — | — | — | -| h200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:32724 | 0 / 24543 | 14 | — | — | — | -| h200_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:8682 | 0 / 8682 | 0 | — | — | — | -| h200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:157464 | 0 / 112509 | 1 | — | — | — | -| h200_sxm | `sglang_marlin_moe` | shared | sglang | sglang:22113 | 0 / 17415 | 0 | — | — | — | +| h100_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| h100_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:30213 | 0 / 10692 | 4 | — | — | — | +| h100_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:22323 | 0 / 9801 | 5 | — | — | — | +| h100_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 5 | — | — | — | +| h100_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:45927 | 0 / 16200 | 9 | — | — | — | +| h100_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:52753 | 0 / 18306 | 11 | — | — | — | +| h200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:3402 | 0 / 1134 | 1 | — | — | — | +| h200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:292734 | 0 / 134622 | 125 | — | — | — | +| h200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:52320 | 0 / 25272 | 14 | — | — | — | +| h200_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:17349 | 0 / 8682 | 0 | — | — | — | +| h200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:256932 | 0 / 124497 | 7 | — | — | — | +| h200_sxm | `sglang_marlin_moe` | shared | sglang | sglang:28269 | 0 / 20493 | 1 | — | — | — | | h200_sxm | `sglang_marlin_moe_situ_as_silu` | shared | sglang | sglang:488 | 0 / 488 | 0 | — | — | — | | h200_sxm | `sglang_mxfp4_humming_moe` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | +| h200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 0 | — | — | — | | h200_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| h200_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:19521 | 0 / 10287 | 14 | — | — | — | -| h200_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:12555 | 0 / 6561 | 0 | — | — | — | +| h200_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| h200_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:30213 | 0 / 10692 | 17 | — | — | — | +| h200_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:22356 | 0 / 9801 | 4 | — | — | — | | h200_sxm | `vllm_fused_moe` | shared | vllm | vllm:36288 | 0 / 33210 | 0 | — | — | — | -| h200_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 3 | — | — | — | +| h200_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 4 | — | — | — | | h200_sxm | `vllm_marlin_int4_moe` | shared | vllm | vllm:10206 | 0 / 9234 | 0 | — | — | — | -| h200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:29727 | 0 / 15309 | 5 | — | — | — | +| h200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:45927 | 0 / 16200 | 11 | — | — | — | | h200_sxm | `vllm_mxfp4_moe` | shared | vllm | vllm:1701 | 0 / 1296 | 0 | — | — | — | -| h200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:34587 | 0 / 18306 | 1 | — | — | — | +| h200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:52893 | 0 / 18306 | 4 | — | — | — | | l40s | `moe_torch_flow` | shared | trtllm | trtllm:1960 | 0 / 1960 | 0 | — | — | — | -| l40s | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:77077 | 0 / 72092 | 7 | — | — | — | -| l40s | `moe_torch_flow_nongated` | shared | trtllm | trtllm:14372 | 0 / 14372 | 0 | — | — | — | -| l40s | `sglang_fused_moe_triton` | shared | sglang | sglang:54402 | 0 / 54402 | 0 | — | — | — | +| l40s | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:153460 | 0 / 76385 | 2095 | — | — | — | +| l40s | `moe_torch_flow_nongated` | shared | trtllm | trtllm:32919 | 0 / 18746 | 15 | — | — | — | +| l40s | `sglang_fused_moe_triton` | shared | sglang | sglang:120390 | 0 / 65988 | 77 | — | — | — | | l40s | `sglang_fused_moe_triton_situ_as_silu` | shared | sglang | sglang:2997 | 0 / 2997 | 0 | — | — | — | -| l40s | `sglang_marlin_moe` | shared | sglang | sglang:3078 | 0 / 3078 | 0 | — | — | — | +| l40s | `sglang_marlin_moe` | shared | sglang | sglang:6156 | 0 / 3078 | 2 | — | — | — | +| l40s | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | l40s | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:1026 | 0 / 1026 | 0 | — | — | — | -| l40s | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:15118 | 0 / 15118 | 0 | — | — | — | -| l40s | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:1944 | 0 / 1944 | 0 | — | — | — | -| l40s | `vllm_modeloptfp8moe_triton_tritonexperts` | shared | vllm | vllm:14174 | 0 / 14174 | 0 | — | — | — | -| l40s | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:16055 | 0 / 16055 | 0 | — | — | — | -| rtx_pro_6000_server | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:119394 | 0 / 119394 | 0 | — | — | — | -| rtx_pro_6000_server | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22921 | 0 / 22921 | 0 | — | — | — | -| rtx_pro_6000_server | `moe_torch_flow_triton_fp8_block` | shared | trtllm | trtllm:32238 | 0 / 32238 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:35478 | 0 / 35478 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_fused_moe_triton` | shared | sglang | sglang:77355 | 0 / 77355 | 0 | — | — | — | +| l40s | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| l40s | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:34573 | 0 / 19515 | 2 | — | — | — | +| l40s | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:3888 | 0 / 1944 | 1 | — | — | — | +| l40s | `vllm_modeloptfp8moe_triton_tritonexperts` | shared | vllm | vllm:30345 | 0 / 16200 | 2 | — | — | — | +| l40s | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:34042 | 0 / 18069 | 3 | — | — | — | +| rtx_pro_6000_server | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:250452 | 0 / 131058 | 102 | — | — | — | +| rtx_pro_6000_server | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50785 | 0 / 27943 | 0 | — | — | — | +| rtx_pro_6000_server | `moe_torch_flow_triton_fp8_block` | shared | trtllm | trtllm:65205 | 0 / 32967 | 25 | — | — | — | +| rtx_pro_6000_server | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:91530 | 0 / 56052 | 2 | — | — | — | +| rtx_pro_6000_server | `sglang_fused_moe_triton` | shared | sglang | sglang:167184 | 0 / 89829 | 6 | — | — | — | | rtx_pro_6000_server | `sglang_marlin_moe` | shared | sglang | sglang:4456 | 0 / 4456 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_marlin_moe_situ_as_silu` | shared | sglang | sglang:551 | 0 / 551 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:15822 | 0 / 15822 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:35505 | 0 / 19683 | 5 | — | — | — | | rtx_pro_6000_server | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:1944 | 0 / 1944 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:13284 | 0 / 13284 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:15903 | 0 / 15903 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_fp8moe_deepgemm_deepgemmexperts` | shared | vllm | vllm:16821 | 0 / 16821 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2754 | 0 / 2754 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:3888 | 0 / 1944 | 1 | — | — | — | +| rtx_pro_6000_server | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:29484 | 0 / 16200 | 6 | — | — | — | +| rtx_pro_6000_server | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:32670 | 0 / 17901 | 9 | — | — | — | +| rtx_pro_6000_server | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | ## `msa_context_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25822 | 0 / 14396 | 47 | — | — | — | -| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | -| b200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26933 | 0 / 26933 | 0 | — | — | — | -| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25849 | 0 / 14431 | 73 | — | — | — | -| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | -| b300_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26850 | 0 / 26850 | 0 | — | — | — | -| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25820 | 0 / 14388 | 57 | — | — | — | -| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | -| gb200 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26608 | 0 / 26608 | 0 | — | — | — | -| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25886 | 0 / 14403 | 26 | — | — | — | -| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | -| gb300 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:27242 | 0 / 27242 | 0 | — | — | — | -| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:19078 | 0 / 11567 | 30 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:3382 | 0 / 3382 | 0 | — | — | — | -| h100_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26348 | 0 / 26348 | 0 | — | — | — | -| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:19076 | 0 / 11567 | 144 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:3388 | 0 / 3388 | 0 | — | — | — | -| h200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26845 | 0 / 26845 | 0 | — | — | — | -| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:7808 | 0 / 7808 | 0 | — | — | — | -| l40s | `default` | shared_fallback | trtllm | trtllm:3313 | 0 / 3313 | 0 | — | — | — | -| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:7808 | 0 / 7808 | 0 | — | — | — | +| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33079 | 0 / 14396 | 122 | — | — | — | +| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | +| b200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60326 | 0 / 33649 | 0 | — | — | — | +| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33095 | 0 / 14431 | 147 | — | — | — | +| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | +| b300_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60534 | 0 / 34014 | 0 | — | — | — | +| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33080 | 0 / 14388 | 99 | — | — | — | +| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | +| gb200 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60001 | 0 / 33660 | 0 | — | — | — | +| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33160 | 0 / 14403 | 95 | — | — | — | +| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | +| gb300 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60947 | 0 / 34014 | 0 | — | — | — | +| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:23958 | 0 / 11567 | 103 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:5791 | 0 / 3382 | 12 | — | — | — | +| h100_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:58866 | 0 / 32870 | 0 | — | — | — | +| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:23956 | 0 / 11567 | 268 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:5808 | 0 / 3388 | 8 | — | — | — | +| h200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:59957 | 0 / 33505 | 0 | — | — | — | +| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:12688 | 0 / 7808 | 35 | — | — | — | +| l40s | `default` | shared_fallback | trtllm | trtllm:5671 | 0 / 3313 | 11 | — | — | — | +| l40s | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:15731 | 0 / 15731 | 0 | — | — | — | +| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:15128 | 0 / 10248 | 77 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:3630 | 0 / 3630 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:32732 | 0 / 32732 | 0 | — | — | — | ## `msa_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | -| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | -| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:119 | 0 / 119 | 0 | — | — | — | -| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1358 | 0 / 1358 | 0 | — | — | — | -| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | -| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | -| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:120 | 0 / 120 | 0 | — | — | — | -| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1360 | 0 / 1360 | 0 | — | — | — | -| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | -| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | -| gb200 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:119 | 0 / 119 | 0 | — | — | — | -| gb200 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1358 | 0 / 1358 | 0 | — | — | — | -| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | -| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | -| gb300 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:120 | 0 / 120 | 0 | — | — | — | -| gb300 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1360 | 0 / 1360 | 0 | — | — | — | -| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5870 | 0 / 5870 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:2548 | 0 / 2548 | 0 | — | — | — | -| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:240 | 0 / 240 | 0 | — | — | — | -| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1227 | 0 / 1227 | 0 | — | — | — | -| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5882 | 0 / 5882 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:2556 | 0 / 2556 | 0 | — | — | — | -| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:246 | 0 / 246 | 0 | — | — | — | -| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1231 | 0 / 1231 | 0 | — | — | — | -| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5850 | 0 / 5850 | 0 | — | — | — | -| l40s | `default` | shared_fallback | trtllm | trtllm:2508 | 0 / 2508 | 0 | — | — | — | -| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5882 | 0 / 5882 | 0 | — | — | — | +| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | +| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | +| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:268 | 0 / 149 | 0 | — | — | — | +| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3056 | 0 / 1698 | 69 | — | — | — | +| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | +| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | +| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:270 | 0 / 150 | 0 | — | — | — | +| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3060 | 0 / 1700 | 87 | — | — | — | +| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | +| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | +| gb200 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:268 | 0 / 149 | 0 | — | — | — | +| gb200 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3056 | 0 / 1698 | 17 | — | — | — | +| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | +| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | +| gb300 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:270 | 0 / 150 | 0 | — | — | — | +| gb300 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3060 | 0 / 1700 | 67 | — | — | — | +| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:9544 | 0 / 9544 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:4360 | 0 / 4360 | 0 | — | — | — | +| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:528 | 0 / 298 | 0 | — | — | — | +| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:2755 | 0 / 1533 | 16 | — | — | — | +| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:9562 | 0 / 9562 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:4376 | 0 / 4376 | 0 | — | — | — | +| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:550 | 0 / 308 | 1 | — | — | — | +| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:2768 | 0 / 1539 | 19 | — | — | — | +| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:9512 | 0 / 9512 | 0 | — | — | — | +| l40s | `default` | shared_fallback | trtllm | trtllm:4292 | 0 / 4292 | 0 | — | — | — | +| l40s | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:370 | 0 / 370 | 0 | — | — | — | +| l40s | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:515 | 0 / 515 | 0 | — | — | — | +| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:11402 | 0 / 11402 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:2730 | 0 / 2730 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:294 | 0 / 294 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1531 | 0 / 1531 | 0 | — | — | — | ## `nccl_perf.parquet` @@ -1249,30 +1466,30 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| b200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| b200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| b300_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| b300_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| b300_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| gb200 | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| gb200 | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| gb200 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| gb300 | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| gb300 | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| gb300 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| l40s | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| l40s | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| l40s | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | -| rtx_pro_6000_server | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| b200_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 7 | — | — | — | +| b200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 11 | — | — | — | +| b200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 3256 | 0 | — | — | — | +| b300_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | +| b300_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 14 | — | — | — | +| b300_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 15 | — | — | — | +| gb200 | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | +| gb200 | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 9 | — | — | — | +| gb200 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 11 | — | — | — | +| gb300 | `sglang` | shared | sglang | sglang:4884 | 0 / 1628 | 1 | — | — | — | +| gb300 | `static_scaled_fp8_quant` | shared | vllm | vllm:4884 | 0 / 1628 | 25 | — | — | — | +| gb300 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 7 | — | — | — | +| h100_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 17 | — | — | — | +| h200_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 2 | — | — | — | +| h200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 24 | — | — | — | +| l40s | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 66 | — | — | — | +| l40s | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 272 | — | — | — | +| l40s | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 1 | — | — | — | +| rtx_pro_6000_server | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 7 | — | — | — | +| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 2 | — | — | — | ## `trtllm_alltoall_perf.parquet` @@ -1382,71 +1599,86 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur Each row is one distinct `kernel_source` value seen in the corpus, with the union of frameworks, op files, and systems it appears in. Tier is determined by the kernel_source name alone, so a single kernel_source has one tier across the whole corpus. -### `shared` (175 kernel sources) +### `shared` (211 kernel sources) | kernel_source | frameworks | op files | systems | rows | |---|---|---|---|---| -| `causal_conv1d_fn` | sglang, trtllm, vllm | gdn_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 109,550 | +| `causal_conv1d_fn` | sglang, trtllm, vllm | gdn_perf.parquet, kda_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 223,690 | | `causal_conv1d_fn_qkv3` | sglang, vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 7,012 | -| `causal_conv1d_update` | sglang, trtllm, vllm | gdn_perf.parquet, kda_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 13,098 | -| `chunk_gated_delta_rule` | sglang, trtllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 40,725 | -| `chunk_gated_delta_rule_flashinfer` | vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 55,303 | -| `chunk_gated_delta_rule_triton` | vllm | gdn_perf.parquet | l40s, rtx_pro_6000_server | 7,870 | -| `chunk_kda` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,168 | +| `causal_conv1d_fn_token_major` | trtllm | mamba2_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 4,778 | +| `causal_conv1d_update` | sglang, trtllm, vllm | gdn_perf.parquet, kda_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 21,550 | +| `chunk_gated_delta_rule` | sglang, trtllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 72,299 | +| `chunk_gated_delta_rule_flashinfer` | trtllm, vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 120,360 | +| `chunk_gated_delta_rule_triton` | vllm | gdn_perf.parquet | l40s, rtx_pro_6000_server | 12,265 | +| `chunk_kda` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,103 | | `chunk_kda_with_fused_gate` | vllm | kda_perf.parquet | l40s | 414 | -| `compressed_flashmla` | sglang | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 748,644 | -| `CutlassFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 51,726 | -| `CutlassFP8ScaledMMLinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 654,580 | -| `deep_gemm.fp8_mqa_logits` | sglang | glm5_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 19,378 | -| `deep_gemm.fp8_paged_mqa_logits` | sglang | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 19,276 | +| `compressed_flashmla` | sglang | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 1,368,500 | +| `CutlassFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 109,742 | +| `CutlassFP8ScaledMMLinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,137,698 | +| `deep_gemm.fp8_mqa_logits` | sglang | glm5_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 38,762 | +| `deep_gemm.fp8_paged_mqa_logits` | sglang | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 46,656 | | `deepep` | sglang, trtllm, vllm | moe_a2a_perf.parquet, wideep_deepep_ll_perf.parquet, wideep_deepep_normal_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 23,512 | | `deepepmoe` | sglang | wideep_context_moe_perf.parquet, wideep_generation_moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 6,333 | -| `deepgemm` | trtllm | gemm_perf.parquet, moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 272,268 | +| `deepgemm` | trtllm | gemm_perf.parquet, moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 551,330 | | `deepgemm_megamoe` | sglang | dsv4_megamoe_module_perf.parquet | b200_sxm, gb200, gb300 | 776 | -| `DeepGemmFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 296,224 | +| `DeepGemmFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 641,938 | | `deepseek_v3` | sglang | wideep_context_mlp_perf.parquet, wideep_generation_mlp_perf.parquet | h100_sxm, h200_sxm | 72 | -| `DeepseekV4TrtllmAttention` | trtllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 57,096 | -| `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | vllm | computescale_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 12,941 | -| `fa3` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | h100_sxm, h200_sxm | 145,130 | -| `fast_topk_transform_fused` | sglang | glm5_topk_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 24,092 | -| `flash_attention` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | a100_sxm, h100_sxm, h200_sxm | 26,008 | -| `flash_attention_v3` | sglang | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 15,358 | -| `flash_attention_v4` | sglang | encoder_attention_perf.parquet | b200_sxm, gb200 | 15,358 | -| `FLASH_ATTN` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 84,027 | -| `FLASH_ATTN_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 19,640 | -| `flash_mla_sparse_fwd` | sglang | glm5_dsa_attn_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 19,841 | -| `flashinfer` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 133,439 | +| `DeepseekV4TrtllmAttention` | trtllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 145,545 | +| `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | vllm | computescale_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 27,513 | +| `fa3` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | h100_sxm, h200_sxm | 296,574 | +| `fast_topk_transform_fused` | sglang | glm5_topk_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 48,196 | +| `flash_attention` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | a100_sxm, h100_sxm, h200_sxm | 42,304 | +| `flash_attention_v3` | sglang | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 31,816 | +| `flash_attention_v4` | sglang | encoder_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 46,074 | +| `FLASH_ATTN` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 133,947 | +| `FLASH_ATTN_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 31,424 | +| `FLASH_ATTN_MLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 428,148 | +| `flash_mla_sparse_fwd` | sglang | glm5_dsa_attn_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 39,225 | +| `flashinfer` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 268,399 | +| `flashinfer_chunk_gated_delta_rule` | sglang | gdn_perf.parquet | h100_sxm, h200_sxm | 9,058 | | `flashinfer_gated_delta_rule_decode` | sglang | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 208 | -| `FLASHINFER_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 110,536 | -| `FLASHINFER_MLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 71,728 | -| `FlashInferCuteDslNvFp4LinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 436,600 | -| `FlashInferCutlassNvFp4LinearKernel` | vllm | gemm_perf.parquet | rtx_pro_6000_server | 35,742 | -| `FlashInferFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | h100_sxm, h200_sxm | 14,280 | -| `flashkda_fwd` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 3,415 | -| `FLASHMLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 18,212 | -| `FLASHMLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 83,700 | -| `FLASHMLA_SPARSE_DSV4` | vllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_attn_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 211,005 | +| `FLASHINFER_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 167,416 | +| `FLASHINFER_MLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 1,180,816 | +| `FLASHINFER_MLA_SPARSE_DSV4` | vllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | rtx_pro_6000_server | 14,678 | +| `FLASHINFER_MLA_SPARSE_SM120` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | rtx_pro_6000_server | 119,020 | +| `FlashInferCuteDslNvFp4LinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 696,414 | +| `FlashInferCutlassNvFp4LinearKernel` | vllm | gemm_perf.parquet | rtx_pro_6000_server | 73,260 | +| `FlashInferFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | h100_sxm, h200_sxm | 42,972 | +| `flashkda_fwd` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 5,402 | +| `FLASHMLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 29,204 | +| `FLASHMLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 1,181,841 | +| `FLASHMLA_SPARSE_DSV4` | vllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 264,167 | | `fused_kda_decode` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 308 | | `fused_kda_decode_mtp_dspark` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 428 | +| `fused_qkvzba_causal_conv1d_update_contiguous` | sglang | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,696 | | `fused_recurrent_gated_delta_rule` | sglang, trtllm | gdn_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 774 | -| `fused_recurrent_gated_delta_rule_packed_decode` | sglang, vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 9,638 | -| `fused_recurrent_kda` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 972 | +| `fused_recurrent_gated_delta_rule_packed_decode` | sglang, vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 16,950 | +| `fused_recurrent_kda` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,309 | | `fused_recurrent_kda_packed_decode` | sglang, vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 651 | -| `fused_sigmoid_gating_delta_rule_update` | sglang | kda_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 432 | +| `fused_sigmoid_gating_delta_rule_update` | sglang | kda_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 630 | +| `fused_sigmoid_gating_delta_rule_update_flashinfer` | trtllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 2,772 | +| `fused_sigmoid_gating_delta_rule_update_triton` | trtllm | gdn_perf.parquet | l40s, rtx_pro_6000_server | 924 | | `kda_fused_decode` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 77 | -| `MiniMaxM3SparseMSAImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 138,705 | -| `MiniMaxM3SparseTritonImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 77,254 | +| `mamba_chunk_scan_combined` | trtllm | mamba2_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 4,777 | +| `MarlinFP8ScaledMMLinearKernel` | vllm | gemm_perf.parquet | l40s | 37,518 | +| `MiniMaxM3SparseMSAImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 189,822 | +| `MiniMaxM3SparseTritonImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 115,750 | | `mla_trtllm_mla` | sglang | mla_context_module_perf.parquet | b200_sxm, b300_sxm | 436 | -| `moe_torch_flow` | trtllm | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 62,819 | -| `moe_torch_flow_cutlass` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,014,571 | -| `moe_torch_flow_min_latency` | trtllm | moe_perf.parquet | b200_sxm, gb200 | 72,090 | -| `moe_torch_flow_nongated` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 194,352 | -| `moe_torch_flow_triton_fp8_block` | trtllm | moe_perf.parquet | rtx_pro_6000_server | 32,238 | -| `msa_fmha_sm100` | trtllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 35,736 | +| `moe_torch_flow` | trtllm | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 93,599 | +| `moe_torch_flow_cutlass` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,957,168 | +| `moe_torch_flow_min_latency` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 229,950 | +| `moe_torch_flow_nongated` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 391,660 | +| `moe_torch_flow_triton_fp8_block` | trtllm | moe_perf.parquet | rtx_pro_6000_server | 65,205 | +| `msa_fmha_sm100` | trtllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 58,920 | | `NCCL` | trtllm | nccl_perf.parquet | b300_sxm | 1,512 | | `NVLinkOneSided` | trtllm | trtllm_alltoall_perf.parquet | gb200 | 296 | | `NVLinkTwoSided` | trtllm | trtllm_alltoall_perf.parquet | gb200 | 1,800 | -| `sglang` | sglang | computescale_perf.parquet, gemm_perf.parquet, scale_matrix_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 84,087 | +| `selective_state_update_flashinfer` | trtllm | mamba2_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 528 | +| `sglang` | sglang | computescale_perf.parquet, gemm_perf.parquet, scale_matrix_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 98,806 | +| `sglang.hc_post[tilelang]` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `sglang.kernels.ops.layernorm.mhc.hc_contract` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `sglang.kernels.ops.layernorm.mhc.hc_expand` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | | `sglang.srt.layers.engram.Engram.apply_gate+sglang.srt.layers.engram.EngramEmbedding._owned_rows+sglang.srt.layers.linear.ReplicatedLinear.forward/sglang.srt.layers.quantization.fp8.Fp8LinearMethod` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 240 | | `sglang.srt.layers.linear.MergedColumnParallelLinear.forward/sglang.srt.layers.quantization.fp8.Fp8LinearMethod` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 120 | | `sglang.srt.layers.linear.RowParallelLinear.forward/sglang.srt.layers.quantization.fp8.Fp8LinearMethod` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 120 | @@ -1454,111 +1686,132 @@ Each row is one distinct `kernel_source` value seen in the corpus, with the unio | `sglang.srt.models.deepseek_v2.MoEGate.forward` | sglang | gemm_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 60 | | `sglang.srt.models.deepseek_v4.DeepseekV4DecoderLayer._hc_mix_and_combine+sglang.srt.models.deepseek_v4.DeepseekV4DecoderLayer.hc_post` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 60 | | `sglang.srt.models.deepseek_v4.MQALayer.forward` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 9,040 | -| `SGLang_CustomAllReduce_eager` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,725 | -| `SGLang_CustomAllReduce_graph` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,725 | -| `sglang_deepgemm_gemm_nt_f8f8bf16` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 295,260 | -| `sglang_dsa_dense_mha_fa3` | sglang | dsa_context_module_perf.parquet | h100_sxm, h200_sxm | 82,901 | -| `sglang_dsa_dense_mha_trtllm_ragged` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 137,411 | -| `sglang_dsa_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 5,352 | -| `sglang_dsa_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 46,332 | -| `sglang_dsa_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 123,166 | -| `sglang_dsa_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 91,197 | -| `sglang_dsa_skip_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 2,448 | -| `sglang_dsa_skip_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 24,610 | -| `sglang_dsa_skip_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 54,618 | -| `sglang_dsa_skip_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 39,806 | -| `sglang_flashinfer_cutedsl_nvfp4` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 236,208 | -| `sglang_flashinfer_cutlass_moe` | sglang | moe_perf.parquet | h100_sxm, h200_sxm, rtx_pro_6000_server | 52,761 | -| `sglang_flashinfer_cutlass_nvfp4` | sglang | gemm_perf.parquet | rtx_pro_6000_server | 29,526 | -| `sglang_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 645,304 | -| `sglang_fused_moe_triton` | sglang | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 682,986 | +| `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | sglang | glm53_attention_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 3,659 | +| `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | sglang | glm53_attention_module_perf.parquet | b300_sxm, gb200, gb300 | 3,217 | +| `SGLang_CustomAllReduce_eager` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 2,024 | +| `SGLang_CustomAllReduce_graph` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 2,024 | +| `sglang_cutedsl_tgv_bf16` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300 | 3,929 | +| `sglang_deepgemm_gemm_nt_f8f8bf16` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 629,739 | +| `sglang_dsa_dense_mha_fa3` | sglang | dsa_context_module_perf.parquet | h100_sxm, h200_sxm | 145,111 | +| `sglang_dsa_dense_mha_trtllm_ragged` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 277,456 | +| `sglang_dsa_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 9,008 | +| `sglang_dsa_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 80,896 | +| `sglang_dsa_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 223,605 | +| `sglang_dsa_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 176,983 | +| `sglang_dsa_skip_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 4,912 | +| `sglang_dsa_skip_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 49,141 | +| `sglang_dsa_skip_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 126,011 | +| `sglang_dsa_skip_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 100,438 | +| `sglang_flashinfer_bf16_direct` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300 | 24 | +| `sglang_flashinfer_bf16_splitk` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300 | 105 | +| `sglang_flashinfer_bmm_fp8` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,088 | +| `sglang_flashinfer_cutedsl_nvfp4` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 464,350 | +| `sglang_flashinfer_cutlass_moe` | sglang | moe_perf.parquet | h100_sxm, h200_sxm, rtx_pro_6000_server | 126,066 | +| `sglang_flashinfer_cutlass_nvfp4` | sglang | gemm_perf.parquet | rtx_pro_6000_server | 59,126 | +| `sglang_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 1,127,207 | +| `sglang_fused_moe_triton` | sglang | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,211,403 | | `sglang_fused_moe_triton_situ_as_silu` | sglang | moe_perf.parquet | l40s | 2,997 | -| `sglang_marlin_moe` | sglang | moe_perf.parquet | a100_sxm, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 70,795 | +| `sglang_marlin_moe` | sglang | moe_perf.parquet | a100_sxm, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 86,185 | | `sglang_marlin_moe_situ_as_silu` | sglang | moe_perf.parquet | h100_sxm, h200_sxm, rtx_pro_6000_server | 1,580 | -| `sglang_minimax_decode_triton_sparse_topk_radix` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 964 | -| `sglang_minimax_decode_triton_sparse_topk_split` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 7,894 | -| `sglang_minimax_prefill_triton_sparse` | sglang | msa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 160,826 | -| `sglang_mxfp4_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 30,030 | +| `sglang_minimax_decode_triton_sparse_topk_radix` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 2,818 | +| `sglang_minimax_decode_triton_sparse_topk_split` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 19,801 | +| `sglang_minimax_prefill_triton_sparse` | sglang | msa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 409,094 | +| `sglang_mxfp4_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 121,398 | | `sglang_mxfp4_humming_moe` | sglang | moe_perf.parquet | h100_sxm, h200_sxm | 30 | | `sglang_sgl_kernel_bmm_fp8` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,724 | -| `sglang_sgl_kernel_fp8_scaled_mm` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 429,348 | -| `sglang_tilelang_mhc_post` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 485 | -| `sglang_tilelang_mhc_pre` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 420 | -| `sglang_torch_bmm` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,724 | -| `sglang_torch_linear` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 429,718 | +| `sglang_sgl_kernel_fp8_scaled_mm` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 760,646 | +| `sglang_tilelang_mhc_post` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 903 | +| `sglang_tilelang_mhc_pre` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 874 | +| `sglang_torch_bmm` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 10,812 | +| `sglang_torch_linear` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 908,794 | | `sglang_torch_mhc_pre` | sglang | mhc_module_perf.parquet | rtx_pro_6000_server | 67 | -| `static_scaled_fp8_quant` | vllm | scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 13,024 | -| `topk_transform_v1` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 25,764 | -| `topk_transform_v2` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 1,464 | -| `torch.nn.functional.linear` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 654,534 | -| `torch_flow` | trtllm | context_attention_perf.parquet, encoder_attention_perf.parquet, gemm_perf.parquet, generation_attention_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,891,515 | -| `torch_flow_flashinfer` | trtllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 23,408 | -| `torch_ops` | trtllm | computescale_perf.parquet, scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 26,012 | -| `triton` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, encoder_attention_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 173,342 | -| `TRITON_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | l40s, rtx_pro_6000_server | 5,952 | +| `sglang_triton_w8a8_block_fp8` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 9,842 | +| `static_scaled_fp8_quant` | vllm | scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 27,676 | +| `topk_transform_v1` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 40,124 | +| `topk_transform_v2` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 2,304 | +| `torch.nn.functional.linear` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,137,658 | +| `torch_flow` | trtllm | context_attention_perf.parquet, encoder_attention_perf.parquet, gemm_perf.parquet, generation_attention_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,825,706 | +| `torch_flow_flashinfer` | trtllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 42,856 | +| `torch_ops` | trtllm | computescale_perf.parquet, scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 52,055 | +| `triton` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, encoder_attention_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 350,168 | +| `TRITON_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | l40s, rtx_pro_6000_server | 17,972 | | `TritonFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | l40s | 35,676 | | `trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0` | trtllm | gemm_perf.parquet | a100_sxm | 6,048 | | `trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0` | trtllm | gemm_perf.parquet | a100_sxm | 12,096 | | `TRTLLM` | trtllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,518 | -| `trtllm_bmm_out` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,392 | -| `trtllm_bmm_out_dequant_bf16` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 1,696 | -| `trtllm_fp8_block_scaling_bmm_out` | trtllm | mla_bmm_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,696 | -| `trtllm_mha` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 273,765 | -| `trtllm_mhc_post_mapping` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 555 | -| `trtllm_mhc_pre_dg_nosplit` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 159 | -| `trtllm_mhc_pre_dg_splitk` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 156 | -| `trtllm_mhc_pre_fma` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 231 | -| `trtllm_mla` | sglang | context_mla_perf.parquet, generation_mla_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 39,383 | +| `trtllm_bmm_out` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 8,480 | +| `trtllm_bmm_out_dequant_bf16` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 4,240 | +| `trtllm_fp8_block_scaling_bmm_out` | trtllm | mla_bmm_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 4,240 | +| `trtllm_mha` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 565,262 | +| `trtllm_mhc_post_mapping` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,110 | +| `trtllm_mhc_pre_dg_nosplit` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 312 | +| `trtllm_mhc_pre_dg_splitk` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 392 | +| `trtllm_mhc_pre_fma` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 388 | +| `trtllm_mla` | sglang | context_mla_perf.parquet, generation_mla_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 70,527 | | `TRTLLM_MNNVL_oneshot` | trtllm | custom_allreduce_perf.parquet | gb300 | 19 | | `TRTLLM_MNNVL_twoshot` | trtllm | custom_allreduce_perf.parquet | gb300 | 27 | -| `TRTLLM_RAGGED` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 33,863 | -| `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 905 | -| `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 896 | -| `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | vllm | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 35,262 | +| `TRTLLM_RAGGED` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 50,135 | +| `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 30 | +| `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 144 | +| `vllm.mhc_post_tilelang` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,460 | +| `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,455 | +| `vllm.model_executor.layers.mhc.hc_contract` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `vllm.model_executor.layers.mhc.hc_expand` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | vllm | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 62,688 | +| `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | vllm | glm53_attention_module_perf.parquet | b300_sxm, gb200, gb300 | 6,019 | +| `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseSM90Impl` | vllm | glm53_attention_module_perf.parquet | h100_sxm, h200_sxm | 1,374 | | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm | 1,944 | -| `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 9,720 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 23,598 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 20,412 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 117,220 | +| `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 17,496 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 3,888 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 43,281 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 36,936 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 179,196 | | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 14,715 | -| `vLLM_custom_eager` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, b60, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,633 | -| `vLLM_custom_graph` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,426 | -| `vllm_default` | vllm | gemm_perf.parquet | a100_sxm, b60 | 49,770 | -| `vllm_flash_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | a100_sxm, b60 | 102,215 | -| `vllm_flash_attn_fa2` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | l40s, rtx_pro_6000_server | 111,574 | -| `vllm_flash_attn_fa3` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 463,230 | -| `vllm_flash_attn_fa4` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 6,006 | +| `vllm_compressedtensorswna16moe_marlin_marlinexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 8,424 | +| `vLLM_custom_eager` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, b60, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,840 | +| `vLLM_custom_graph` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, b60, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,771 | +| `vllm_default` | vllm | gemm_perf.parquet | a100_sxm, b60 | 70,938 | +| `vllm_flash_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | a100_sxm, b60 | 97,251 | +| `vllm_flash_attn_fa2` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, l40s, rtx_pro_6000_server | 245,795 | +| `vllm_flash_attn_fa3` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 700,938 | +| `vllm_flash_attn_fa4` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 9,256 | | `vllm_flashinfer` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm | 71,600 | -| `vllm_flashinfer_fidecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | l40s, rtx_pro_6000_server | 58,484 | -| `vllm_flashinfer_fiprefill` | vllm | context_attention_perf.parquet | l40s, rtx_pro_6000_server | 43,367 | +| `vllm_flashinfer_fidecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | l40s, rtx_pro_6000_server | 92,723 | +| `vllm_flashinfer_fiprefill` | vllm | context_attention_perf.parquet | l40s, rtx_pro_6000_server | 90,646 | | `vllm_flashinfer_flashinfertrtllmapidecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 268,320 | +| `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 264,396 | +| `vllm_flashinfer_flashinfertrtllmapidecode_xqa` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | rtx_pro_6000_server | 31,021 | | `vllm_flashinfer_trtllm_moe_fp4` | vllm | moe_perf.parquet | b200_sxm | 10,287 | | `vllm_flashinfer_trtllmdecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 254,018 | -| `vllm_flashinfer_trtllmprefill` | vllm | context_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 369,948 | -| `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 39,042 | -| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 32,724 | -| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 99,792 | -| `vllm_fp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | h100_sxm, h200_sxm, l40s | 40,222 | +| `vllm_flashinfer_trtllmprefill` | vllm | context_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 556,588 | +| `vllm_fp8moe_deepgemm_deepgemmexperts` | vllm | moe_perf.parquet | rtx_pro_6000_server | 16,821 | +| `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 60,426 | +| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 50,220 | +| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 152,118 | +| `vllm_fp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 93,265 | | `vllm_fused_moe` | vllm | moe_perf.parquet | a100_sxm, b200_sxm, h200_sxm | 126,055 | -| `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 15,552 | -| `vllm_gptossmxfp4moe_marlin_marlinexperts` | vllm | moe_perf.parquet | l40s, rtx_pro_6000_server | 3,888 | -| `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 7,776 | +| `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 23,328 | +| `vllm_gptossmxfp4moe_marlin_marlinexperts` | vllm | moe_perf.parquet | l40s, rtx_pro_6000_server | 7,776 | +| `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 11,664 | | `vllm_marlin_int4_moe` | vllm | moe_perf.parquet | b200_sxm, h200_sxm | 20,415 | -| `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 84,078 | -| `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 95,937 | -| `vllm_modeloptfp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | l40s | 14,174 | +| `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 140,454 | +| `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 145,169 | +| `vllm_modeloptfp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | l40s | 30,345 | +| `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | vllm | moe_perf.parquet | b300_sxm, gb200, gb300 | 2,673 | | `vllm_mxfp4_moe` | vllm | moe_perf.parquet | b200_sxm, h200_sxm | 2,839 | -| `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 17,496 | -| `vllm_torch_bmm` | vllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 8,068 | -| `vllm_triton_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, l40s, rtx_pro_6000_server | 42,460 | -| `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 30,807 | +| `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 26,244 | +| `vllm_torch_bmm` | vllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 13,156 | +| `vllm_triton_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 78,559 | +| `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 54,702 | | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 103,173 | -| `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 4,158 | -| `vllm_unquantizedfusedmoe_triton_tritonexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 89,413 | -| `vllm_vit_flash_attn_fa2` | vllm | encoder_attention_perf.parquet | l40s, rtx_pro_6000_server | 15,358 | -| `vllm_vit_flash_attn_fa3` | vllm | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 30,716 | -| `vllm_vit_flash_attn_fa4` | vllm | encoder_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 61,432 | -| `vllm_xpu_moe` | vllm | moe_perf.parquet | b60 | 1,836 | +| `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 64,276 | +| `vllm_unquantizedfusedmoe_triton_tritonexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 149,165 | +| `vllm_vit_flash_attn_fa2` | vllm | encoder_attention_perf.parquet | l40s, rtx_pro_6000_server | 30,716 | +| `vllm_vit_flash_attn_fa3` | vllm | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 47,174 | +| `vllm_vit_flash_attn_fa4` | vllm | encoder_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 92,148 | +| `vllm_xpu_moe` | vllm | moe_perf.parquet | b60 | 3,996 | | `vllm_xpu_moe_mxfp4` | vllm | moe_perf.parquet | b60 | 2,592 | | `wideep_compute_cutlass` | trtllm | wideep_moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 26,298 | @@ -1566,4 +1819,4 @@ Each row is one distinct `kernel_source` value seen in the corpus, with the unio | kernel_source | frameworks | op files | systems | rows | |---|---|---|---|---| -| `default` | sglang, trtllm | context_mla_perf.parquet, dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet, generation_mla_perf.parquet, mla_bmm_perf.parquet, mla_context_module_perf.parquet, mla_generation_module_perf.parquet, msa_context_module_perf.parquet, msa_generation_module_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 306,281 | +| `default` | sglang, trtllm | context_mla_perf.parquet, dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet, generation_mla_perf.parquet, mla_bmm_perf.parquet, mla_context_module_perf.parquet, mla_generation_module_perf.parquet, msa_context_module_perf.parquet, msa_generation_module_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 598,936 | diff --git a/crates/core/src/perfmodel/common/enums.rs b/crates/core/src/perfmodel/common/enums.rs index 9e024319fa..78a0f5c952 100644 --- a/crates/core/src/perfmodel/common/enums.rs +++ b/crates/core/src/perfmodel/common/enums.rs @@ -641,8 +641,6 @@ pub enum PerfDataFilename { Dsv4CsaGenerationModule, Dsv4HcaGenerationModule, Dsv4PagedMqaLogitsModule, - Dsv4HcaAttnModule, - Dsv4CsaAttnModule, Dsv4CsaTopkCalib, Dsv4MegamoeModule, MoeA2a, @@ -682,8 +680,6 @@ impl PerfDataFilename { Self::Dsv4CsaGenerationModule => "dsv4_csa_generation_module_perf.parquet", Self::Dsv4HcaGenerationModule => "dsv4_hca_generation_module_perf.parquet", Self::Dsv4PagedMqaLogitsModule => "dsv4_paged_mqa_logits_module_perf.parquet", - Self::Dsv4HcaAttnModule => "dsv4_hca_attn_module_perf.parquet", - Self::Dsv4CsaAttnModule => "dsv4_csa_attn_module_perf.parquet", Self::Dsv4CsaTopkCalib => "dsv4_csa_topk_calib_perf.parquet", Self::Dsv4MegamoeModule => "dsv4_megamoe_module_perf.parquet", Self::MoeA2a => "moe_a2a_perf.parquet", diff --git a/docs/perf-model/collector/upgrade.md b/docs/perf-model/collector/upgrade.md index 1c3f2916e9..eaa3523180 100644 --- a/docs/perf-model/collector/upgrade.md +++ b/docs/perf-model/collector/upgrade.md @@ -163,7 +163,11 @@ on its observed speed. After the operation finishes, verify its checkpoint, summary, output shape, errors, container exit, and GPU state, then continue to the next operation without waiting for approval unless a real anomaly appears. -Use one output/checkpoint namespace per framework, version, platform, and +`tools/perf_database/collect_campaign.py` is the checked-in way to run this +stage: manifest-pinned image plus finalize deps, one namespace per declared +shard, `--resume` always and `--keep-csv` never (that flag skips finalization +and leaves no parquet or provenance), a GPU pool, and a `campaign.jsonl` work +log. Use one output/checkpoint namespace per framework, version, platform, and operation. Resume only when the current plan still contains the completed task IDs. If enumeration changes, prove which completed IDs remain valid and keep a hash-verified checkpoint backup before migration. diff --git a/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md b/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md index 23ff177e6c..7c8dff7415 100644 --- a/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md +++ b/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md @@ -239,6 +239,37 @@ python3 collector/collect.py --backend --ops \ --checkpoint-dir "$COLLECTOR_CHECKPOINT_DIR///" --resume ``` +### Campaign mode (full plan, several GPUs) — use the checked-in launcher, do not hand-roll one + +Once an op's smoke/limited runs are clean, run the full retained plan through +`tools/perf_database/collect_campaign.py`. It fixes the parameters that every +hand-written launcher has drifted on (sm120 campaign 2026-09-30: `--keep-csv` +skipped finalize for 13 shards; the official image lacked pyarrow): + +```bash +# 1. image = the framework_manifest pin for (backend, op) + the finalize deps (pyarrow, pandas) +python3 tools/perf_database/collect_campaign.py build-image --backend --op [--build-arg http_proxy=...] +# 2. declare the shards once (YAML: backend, sm, shards[{op, name, case_filter}]) — one shard = one +# output/checkpoint namespace ///; case_filter is collect.py's substring filter +# 3. run the GPU pool; re-run the same command after any interruption (DONE shards skip, others --resume) +python3 tools/perf_database/collect_campaign.py run --plan shards.yaml --root /collect --gpus 0,1,2,3 +python3 tools/perf_database/collect_campaign.py status --plan shards.yaml --root /collect +# 4. shards that were produced by hand with --keep-csv have no parquet: finalize them (resume-only re-entry) +python3 tools/perf_database/collect_campaign.py finalize --plan shards.yaml --root /collect --gpus 0 +``` + +Parameter rules the tool enforces (and that apply to any manual `collect.py` run too): + +- `--resume` always, `--keep-csv` never for data you intend to deliver: `--keep-csv` means + "keep CSV *instead of* finalizing" — no parquet, no `collection_meta.yaml`, no provenance. + Its only use is inspecting raw staging rows. +- `--sm ` explicit on every run (never rely on auto-detect inside a container). +- Never point two concurrent runs at one cwd: staging files are named per table + (`gemm_perf.txt`) and would interleave. Shards of one op are merged at delivery + (playbook §8), not by sharing a directory. +- The image is the manifest pin for the op (family pins can differ per op) — a + hand-typed tag is a review finding. + Use a separate log directory per op when iterating: ```bash diff --git a/python/aisimulate/THIRD_PARTY_NOTICES.md b/python/aisimulate/THIRD_PARTY_NOTICES.md index 2fd2b58b6b..1f2c508bd2 100644 --- a/python/aisimulate/THIRD_PARTY_NOTICES.md +++ b/python/aisimulate/THIRD_PARTY_NOTICES.md @@ -851,6 +851,49 @@ THE SOFTWARE AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED "AS IS", WITHOUT For any questions regarding this license, please contact model-business@notice.qwencloud.com. ``` +## Qwen3.8-27B model configuration files + +The following model configuration files are copied byte-for-byte from the +named repositories at the immutable revisions shown (onboarding 2026-09-25): + +| Packaged file | Upstream revision | +| --- | --- | +| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json` | `Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | +| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json` | `Qwen/Qwen3.8-27B-FP8@017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` | + +Copyright owner: Qwen. Both upstream revisions are licensed under the Apache +License 2.0 (`LICENSE` in each repository); the Apache License 2.0 terms are +reproduced in this distribution's `LICENSE` file. + +## Ornith 1.5 model configuration + +`aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json` is +an unmodified copy of `config.json` from +https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B at immutable revision +`10fbf86fed7ecee4a061f8b499a618f46001cac1`. The repository declares the MIT +License in its model card (no separate LICENSE file is shipped); copyright +owner: ornith-ai. The MIT License terms are reproduced in the DeepSeek +section above. + +## MiniCPM5 model configuration + +`aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json` is an +unmodified copy of `config.json` from +https://huggingface.co/openbmb/MiniCPM5-2B at immutable revision +`12a3808a956f869c767195e9266b59c4d21d92e2`. The repository declares the +Apache License 2.0 in its model card (no separate LICENSE file is shipped); +copyright owner: OpenBMB. The Apache License 2.0 terms are reproduced in this +distribution's `LICENSE` file. + +## NeoHorse 1 model configuration + +`aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json` is an +unmodified copy of `config.json` from +https://huggingface.co/TokenRhythm/NeoHorse-1-9B at immutable revision +`ba5b6e40d88a6ddf4591e176738254a3bc715765`. Copyright owner: TokenRhythm. +License: Apache License 2.0 (`LICENSE` in the repository), reproduced in this +distribution's `LICENSE` file. + ## Meta Llama 4 The following bundled model configs are modified copies of Meta Llama 4 diff --git a/python/aisimulate/collector/README.md b/python/aisimulate/collector/README.md index 96834013c9..a7e8d0ffae 100644 --- a/python/aisimulate/collector/README.md +++ b/python/aisimulate/collector/README.md @@ -451,15 +451,28 @@ If unavailable, a warning is logged and execution continues without power data. - Kernel iterations are automatically adjusted to meet minimum duration for accurate measurements - Backward compatible: without `--measure_power`, CSVs remain unchanged -## CUDA Graph Fallback Support - -The `benchmark_with_power` helper function now supports graceful fallback to eager execution when CUDA graph capture fails. This is particularly useful for complex operations like MOE (Mixture of Experts) with large batch sizes. - -### Features -- **Automatic fallback**: When `allow_graph_fail=True`, CUDA graph capture failures trigger eager execution instead of raising exceptions -- **Power measurement in both paths**: Power monitoring works correctly in both graph replay and eager execution modes -- **Memory safety**: Automatic `torch.cuda.empty_cache()` call on graph capture failure to prevent memory fragmentation -- **Transparency**: Results include `used_cuda_graph` flag to indicate which execution path was used +## CUDA Graph Timing Policy + +`benchmark_with_power` times kernels under CUDA-graph replay. A failed capture +raises (`allow_graph_fail=False`, the default) and the case fails classified. + +- **No collector that writes a perf table may pass `allow_graph_fail=True`.** + The eager fallback times the Python launch loop and ships that floor as the + kernel latency, and no perf table records the helper's `used_cuda_graph` + flag — `.claude/rules/collector/layer_permissions.md` allows a + measurement-method degradation only when it is recorded in the output row. + `tests/unit/collector/test_no_eager_graph_fallback_sites.py` pins this for + every file under `collector/`. The last fallback sites (trtllm mamba2 and + moe, wideep moe_compute, network trtllm alltoall) were closed on 2026-10-12; + the sglang trtllm_mla prefill (B1) and the trtllm GDN lanes the day before. +- A collector that sweeps several cells inside one task counts failed cells + and raises a strict-completeness error at the end (GDN, mamba2), so the + other cells still ship rows and the failure lands in `errors_.json`. +- Multi-rank collectors agree on the capture outcome before any rank replays + (`collect_trtllm_alltoall._agree_graph_capturable`); a one-sided capture + failure would otherwise hang the peers' first replay. +- `allow_graph_fail=True` remains available for ad-hoc diagnosis only; the + result then carries `used_cuda_graph=False` and must not be persisted. ### Usage Example ```python @@ -469,33 +482,17 @@ def my_kernel(): # Your kernel code here moe.forward(hidden_states, logits) -# Use benchmark_with_power with fallback support with benchmark_with_power( device=device, kernel_func=my_kernel, num_warmups=3, num_runs=6, repeat_n=1, - allow_graph_fail=True, # Enable graceful fallback ) as results: latency = results["latency_ms"] - power_stats = results["power_stats"] # Available in both paths - - # Check which execution path was used - if not results["used_cuda_graph"]: - print("CUDA graph capture failed, used eager execution") + power_stats = results["power_stats"] ``` -### When to Use -- **Complex operations**: MOE, dynamic memory patterns, or operations that may not be graph-compatible -- **Large batch sizes**: When graph capture may fail due to memory constraints -- **Development/debugging**: To ensure collection continues even if graph capture fails - -### Backward Compatibility -- Default behavior unchanged: `allow_graph_fail=False` maintains existing behavior -- Existing collectors work without modifications -- Only opt-in when needed for specific use cases - ## For TensorRT-LLM ### Optional( Read if collecting for mxfp4 kernels) To collect performance data for mxfp4 kernels used in GPTOSS models, depending on the version of TensorRT-LLM, you might need to manually install triton-kernels. If your version is **>= 1.3.0rc2**, nothing needs to be done and you can run the collection process directly. For any version before **1.3.0rc2**. Follow the below instructions to install triton-kernels and expose them to TensorRT-LLM diff --git a/python/aisimulate/collector/case_generator.py b/python/aisimulate/collector/case_generator.py index e030be2ab9..ce931dd199 100644 --- a/python/aisimulate/collector/case_generator.py +++ b/python/aisimulate/collector/case_generator.py @@ -715,6 +715,12 @@ class MLAModuleModelSpec: architecture: str native_num_heads: int wideep_mla: bool + # Declared real tensor-parallel shards of this model (case_authoring's + # model-correlated narrowing). When set, the MSA collectors derive their + # per-rank head axis as native_num_heads // tp instead of sweeping the + # shared inner_sweep_head_counts grid (owner decision 2026-09-29: MSA is + # one fixed geometry, MiniMax-M3 64q/4kv/4idx; TP 16 is the ceiling). + tensor_parallel_sizes: tuple[int, ...] | None = None @dataclasses.dataclass(frozen=True) @@ -745,6 +751,19 @@ class MLAModuleSweepSpec: # memory contract (their guard tests pin it). None -> fall back to # generation_max_tokens. generation_msa_max_tokens: int | None = None + # Optional MSA context budget for the TRT-LLM Triton reference + # implementation (cases/base_ops/mla_module.yaml mla_module_trtllm.context + # .msa_triton_max_tokens): the collector applies it only where it resolves + # implementation="triton" (off SM100/103); fmha_sm100 systems keep + # context_max_tokens. None -> no extra cap. + context_msa_triton_max_tokens: int | None = None + # Optional DSA-specific context budget: DSA context follows serving's + # chunked-prefill shape (new-token budget = chunk size, long context via + # prefix). None on all three -> the DSA branch falls back to the shared + # context axes (context_max_tokens / context_prefix_lengths). + context_dsa_chunk_prefill_size: int | None = None + context_dsa_max_full_sequence_length: int | None = None + context_dsa_prefix_lengths: list[int] | None = None @dataclasses.dataclass(frozen=True) @@ -790,13 +809,28 @@ def get_mla_module_model_specs( continue if wideep_mla is not None and bool(value.get("wideep_mla", False)) != wideep_mla: continue + native_num_heads = int(value["native_num_heads"]) + tp_sizes = None + if value.get("tensor_parallel_sizes") is not None: + tp_list = _as_int_list( + value.get("tensor_parallel_sizes"), + field_name="model_case_values.mla_module.tensor_parallel_sizes", + ) + for tp in tp_list: + if tp <= 0 or native_num_heads % tp != 0: + raise ValueError( + "model_case_values.mla_module.tensor_parallel_sizes must be positive divisors of " + f"native_num_heads: model={value['model_path']}, native_num_heads={native_num_heads}, tp={tp}" + ) + tp_sizes = tuple(tp_list) specs.append( MLAModuleModelSpec( model_path=str(value["model_path"]), attention_type=str(value["attention_type"]), architecture=str(value["architecture"]), - native_num_heads=int(value["native_num_heads"]), + native_num_heads=native_num_heads, wideep_mla=bool(value.get("wideep_mla", False)), + tensor_parallel_sizes=tp_sizes, ) ) @@ -847,6 +881,28 @@ def get_mla_module_model_specs( return specs +def get_msa_head_counts(backend: str | None = None) -> list[int]: + """Per-rank head axis for the MSA module collectors, descending. + + Derived from the declared MSA model rows: ``native_num_heads // tp`` for + each ``tensor_parallel_sizes`` entry (falling back to the sweep's + ``module_tp_sizes`` when a row declares none). MSA serves one fixed + geometry, so unlike GQA attention there is no shared head grid to sweep; + head counts no real shard produces (48/96 -> 3/6 index heads) are never + generated. + """ + sweep = get_mla_module_sweep_spec(backend) + heads: set[int] = set() + for spec in get_mla_module_model_specs(attention_type="msa", backend=backend): + tp_sizes = spec.tensor_parallel_sizes or tuple(sweep.module_tp_sizes) + for tp in tp_sizes: + if spec.native_num_heads % tp == 0: + heads.add(spec.native_num_heads // tp) + if not heads: + raise ValueError("mla_module: no MSA model row yields a per-rank head count (check tensor_parallel_sizes)") + return sorted(heads, reverse=True) + + def _required_mapping(value: object, *, field_name: str) -> dict[str, object]: if not isinstance(value, dict): raise TypeError(f"{field_name} must be a mapping") @@ -1013,6 +1069,7 @@ def get_mla_module_sweep_spec(backend: str | None = None) -> MLAModuleSweepSpec: context_max_tokens=int(context["max_tokens"]), context_large_sequence_min=_optional_int(context.get("large_sequence_min")), context_large_sequence_max_batch_size=_optional_int(context.get("large_sequence_max_batch_size")), + context_msa_triton_max_tokens=_optional_int(context.get("msa_triton_max_tokens")), generation_max_tokens=int(generation["max_tokens"]), generation_msa_max_tokens=_optional_int(generation.get("msa_max_tokens")), generation_large_sequence_min=_optional_int(generation.get("large_sequence_min")), @@ -1023,6 +1080,16 @@ def get_mla_module_sweep_spec(backend: str | None = None) -> MLAModuleSweepSpec: field_name="mla_module.context.prefix_lengths", default=[0], ), + context_dsa_chunk_prefill_size=_optional_int(context.get("dsa_chunk_prefill_size")), + context_dsa_max_full_sequence_length=_optional_int(context.get("dsa_max_full_sequence_length")), + context_dsa_prefix_lengths=( + _optional_int_list( + context.get("dsa_prefix_lengths"), + field_name="mla_module.context.dsa_prefix_lengths", + default=[], + ) + or None + ), ) @@ -2475,10 +2542,6 @@ def _dsv4_module_budgets() -> dict: _DSV4_SPARSE_CHUNK_PREFILL_SIZE = int(_DSV4_CONFIG["sparse_chunk_prefill_size"]) _DSV4_SPARSE_MAX_FULL_S = int(_DSV4_CONFIG["sparse_max_full_sequence_length"]) _DSV4_SPARSE_TP_SIZES = _required_mapping(_DSV4_CONFIG["sparse_tp_sizes"], field_name="dsv4.sparse_tp_sizes") -_DSV4_SPARSE_TP_LIST_ATTN = _as_int_list( - _DSV4_SPARSE_TP_SIZES["hca_attn"], - field_name="dsv4.sparse_tp_sizes.hca_attn", -) _DSV4_SPARSE_TP_LIST_INDEXER = _as_int_list( _DSV4_SPARSE_TP_SIZES["paged_mqa_logits"], field_name="dsv4.sparse_tp_sizes.paged_mqa_logits", @@ -2695,8 +2758,8 @@ def get_dsv4_hca_generation_test_cases(): def get_dsv4_topk_calib_test_cases(): """One case per model (the topk DELTA calibration is just another member of - the sparse-op family — same ``[model_path, kernel]`` shape as paged_mqa / - csa_attn / hca_attn, dispatched through ``run_dsv4_sparse_kernel_worker``). + the sparse-op family — same ``[model_path, kernel]`` shape as + paged_mqa_logits, dispatched through ``run_dsv4_sparse_kernel_worker``). The grid matches the CSA module data 1:1: the worker reads the already-collected ``dsv4_csa_*_module_perf.txt`` and benches exactly those @@ -2709,7 +2772,11 @@ def get_dsv4_topk_calib_test_cases(): return [[model_path, "topk"] for model_path in _selected_dsv4_calib_models()] -DSV4_SPARSE_KERNELS = ("paged_mqa_logits", "hca_attn") +# The CSA indexer scoring GEMM is the only sparse sub-kernel collected standalone. +# hca_attn / csa_attn (FMLA sidecars) were removed 2026-10-15: retired SDK +# placeholders with no query path, and registry-only ops that failed every +# shard plan generated from the registry. +DSV4_SPARSE_KERNELS = ("paged_mqa_logits",) def _build_dsv4_sparse_test_cases( @@ -2717,7 +2784,6 @@ def _build_dsv4_sparse_test_cases( bs_list=None, isl_list=None, past_kv_list=None, - tp_list_attn=None, tp_list_indexer=None, ): """Generate ``(bs, isl, past_kv, tp_size, kernel, model)`` tuples. @@ -2729,14 +2795,12 @@ def _build_dsv4_sparse_test_cases( bs_list = list(bs_list) if bs_list is not None else list(_DSV4_SPARSE_BS_LIST) isl_list = list(isl_list) if isl_list is not None else list(_DSV4_SPARSE_ISL_LIST) past_kv_list = list(past_kv_list) if past_kv_list is not None else list(_DSV4_SPARSE_PAST_KV_LIST) - tp_list_attn = list(tp_list_attn) if tp_list_attn is not None else list(_DSV4_SPARSE_TP_LIST_ATTN) tp_list_indexer = list(tp_list_indexer) if tp_list_indexer is not None else list(_DSV4_SPARSE_TP_LIST_INDEXER) cases = [] for model_path in _selected_dsv4_models(): for kernel in kernels: - tp_list = tp_list_attn if kernel == "hca_attn" else tp_list_indexer - for tp_size in tp_list: + for tp_size in tp_list_indexer: for bs in bs_list: for isl in isl_list: if bs * isl > _DSV4_SPARSE_CHUNK_PREFILL_SIZE: @@ -2747,8 +2811,6 @@ def _build_dsv4_sparse_test_cases( full_s = isl + past_kv if kernel == "paged_mqa_logits" and full_s < 4: continue - if kernel == "hca_attn" and full_s < 64: - continue cases.append( [ bs, @@ -2769,19 +2831,6 @@ def get_dsv4_paged_mqa_logits_test_cases(): return [[model_path, "paged_mqa_logits"] for model_path in _selected_dsv4_models()] -def get_dsv4_hca_attn_test_cases(): - """One case per model. The worker derives shapes from the HCA module CSV - (hca_attn is the HCA c128 FMLA sub-kernel), 1:1 with the hca-module rows.""" - return [[model_path, "hca_attn"] for model_path in _selected_dsv4_models()] - - -def get_dsv4_csa_attn_test_cases(): - """One case per model. The worker derives shapes from the CSA module CSV - (csa_attn is the CSA sparse FMLA over the topk-selected c4 positions), - 1:1 with the csa-module rows — same source as paged_mqa_logits/topk.""" - return [[model_path, "csa_attn"] for model_path in _selected_dsv4_models()] - - # Backward-compatible names for older PR comments/tests while the registry and # persisted perf files use the upstream DeepSeek-V4 op names. _DSV4_FLASH_CONFIG = _DSV4_CONFIG @@ -2796,7 +2845,6 @@ def get_dsv4_csa_attn_test_cases(): _DSV4_FLASH_SPARSE_PAST_KV_LIST = _DSV4_SPARSE_PAST_KV_LIST _DSV4_FLASH_SPARSE_CHUNK_PREFILL_SIZE = _DSV4_SPARSE_CHUNK_PREFILL_SIZE _DSV4_FLASH_SPARSE_MAX_FULL_S = _DSV4_SPARSE_MAX_FULL_S -_DSV4_FLASH_SPARSE_TP_LIST_ATTN = _DSV4_SPARSE_TP_LIST_ATTN _DSV4_FLASH_SPARSE_TP_LIST_INDEXER = _DSV4_SPARSE_TP_LIST_INDEXER DSV4_FLASH_SPARSE_KERNELS = DSV4_SPARSE_KERNELS _dsv4_flash_config = _dsv4_config @@ -2819,4 +2867,3 @@ def _dsv4_flash_model_path() -> str: get_dsv4_flash_csa_generation_test_cases = get_dsv4_csa_generation_test_cases get_dsv4_flash_hca_generation_test_cases = get_dsv4_hca_generation_test_cases get_dsv4_flash_paged_mqa_logits_test_cases = get_dsv4_paged_mqa_logits_test_cases -get_dsv4_flash_hca_attn_test_cases = get_dsv4_hca_attn_test_cases diff --git a/python/aisimulate/collector/cases/base_ops/mla_module.yaml b/python/aisimulate/collector/cases/base_ops/mla_module.yaml index b17830b580..4777ac4821 100644 --- a/python/aisimulate/collector/cases/base_ops/mla_module.yaml +++ b/python/aisimulate/collector/cases/base_ops/mla_module.yaml @@ -266,6 +266,23 @@ common_case_values: - 1 context: max_tokens: 131072 + # MSA (MiniMax-M3) on TRT-LLM has two implementations: the fmha_sm100 + # kernels (implementation="msa", SM100/103 only — msa_availability.py + # ensure_msa_available@1.3.0rc29 raises everywhere else, SM120 included) + # and the Triton reference (implementation="triton", the only path on + # SM89/SM90/SM120). The reference computes the block-sparse GQA as a + # masked DENSE FP32 attention with per-query K/V expansion + # (triton_backend.py:_sparse_gqa_masked@1.3.0rc29), so its context cost + # is quadratic per sequence: 64 heads, isl 32768, b=1 takes 24.5 s per + # forward on rtxpro6000 and 18 s on h200_sxm (fmha_sm100 on gb200: 9 ms), + # and b*isl=131072 OOMs on the unchunked FP32 index-score slab (64 GiB). + # Owner decision 2026-10-11: on the Triton reference path cap context at + # batch*isl <= 32768 (isl 32768 itself stays, at b=1); the fmha_sm100 + # systems keep the shared 131072. Applied by collect_msa_module's + # context getter where the collector resolves implementation="triton"; + # re-verify on every TRT-LLM version bump (a real sparse prefill kernel + # for these SMs lifts it). + msa_triton_max_tokens: 32768 prefix_lengths: - 0 - 128 @@ -464,6 +481,47 @@ common_case_values: prefix_lengths: - 0 - 128 + # DSA context follows serving's chunked-prefill shape, matching the + # DeepSeek-V4 sparse + sglang-DSA convention (DeepseekV4ForCausalLM_cases + # sparse_chunk_prefill_size / sparse_past_kv_lengths; mla_module_sglang + # context.prefix_lengths). The per-forward NEW-token budget is the chunk + # size and the long context enters through prefix, so (a) each row is + # faithful to what the engine actually queries — context_attention at + # isl=chunk, prefix=context, engine/runtime.rs mixed_step Pass 2, NOT a + # monolithic isl forward serving never runs — and (b) the fp8 sparse + # decode kernel's q stride (bs*isl*padded_heads*576, mixed mode at + # num_heads<32) stays under the int32 limit (overflow at bs*isl>~58k; + # the 776 dsa_context fp8 failures, 2026-09-22). MLA/MSA are unaffected: + # only the collector's DSA branch sweeps prefix. + dsa_chunk_prefill_size: 16384 + dsa_max_full_sequence_length: 1048576 + dsa_prefix_lengths: + - 0 + - 1 + - 4 + - 8 + - 16 + - 32 + - 64 + - 128 + - 256 + - 512 + - 1024 + - 1536 + - 2048 + - 3072 + - 4096 + - 6144 + - 8192 + - 10240 + - 12288 + - 16384 + - 32768 + - 65536 + - 131072 + - 262144 + - 524288 + - 1048575 generation: max_tokens: 33554432 large_cache_tokens: 16777216 diff --git a/python/aisimulate/collector/cases/capabilities.yaml b/python/aisimulate/collector/cases/capabilities.yaml index 68b0589811..677fb7dc83 100644 --- a/python/aisimulate/collector/cases/capabilities.yaml +++ b/python/aisimulate/collector/cases/capabilities.yaml @@ -47,11 +47,19 @@ op_min_sm: # The DSV4/GLM-5 sparse sub-kernel collectors depend on the same SM90+ # kernel libraries (DeepGEMM fp8_mqa_logits, FlashMLA sparse, jit topk). dsv4_paged_mqa_logits_module: 90 - dsv4_hca_attn_module: 90 - dsv4_csa_attn_module: 90 dsv4_csa_topk_calib: 90 glm5_mqa_logits_module: 90 glm5_topk_module: 90 glm5_dsa_attn_module: 90 + # The DSV4 attention module collectors run the production DSV4 attention class, whose + # only backends are FlashMLA (SM90 / SM10x) and the SM100+ paths: on SM89 the selector + # falls back to a FlashMLA that does not support the arch, and serving fails the same way + # (vLLM 0.30.0 probe on L40, DeepGEMM hyperconnection "Unsupported architecture", + # collector/opharness findings sm89_*). Before this entry every sm89 case was queued and + # failed one by one as an `unexpected` RuntimeError. + dsv4_csa_context_module: 90 + dsv4_csa_generation_module: 90 + dsv4_hca_context_module: 90 + dsv4_hca_generation_module: 90 # compute_scale measures FP8 quantization overhead. compute_scale: 89 diff --git a/python/aisimulate/collector/cases/models/DeepseekV32ForCausalLM_cases.yaml b/python/aisimulate/collector/cases/models/DeepseekV32ForCausalLM_cases.yaml index f7e3b461c7..dedd779d84 100644 --- a/python/aisimulate/collector/cases/models/DeepseekV32ForCausalLM_cases.yaml +++ b/python/aisimulate/collector/cases/models/DeepseekV32ForCausalLM_cases.yaml @@ -20,6 +20,11 @@ model_case_values: - model_path: deepseek-ai/DeepSeek-V3.2 attention_type: dsa native_num_heads: 128 + # Real tensor-parallel shards (case_authoring model-correlated narrowing): + # module collectors that honour it sweep per-rank heads = 128 // tp + # (128/64/32/16/8) instead of the shared inner head grid, whose + # 96/48/24/12 entries belong to the 96-head Kimi-K3 MLA module. + tensor_parallel_sizes: [1, 2, 4, 8, 16] all_frameworks_op_cases: moe: @@ -31,8 +36,6 @@ framework_specific_op_cases: cases: all dsa_generation_module: cases: all - moe_ep: - cases: all vllm: dsa_context_module: cases: all diff --git a/python/aisimulate/collector/cases/models/DeepseekV3ForCausalLM_cases.yaml b/python/aisimulate/collector/cases/models/DeepseekV3ForCausalLM_cases.yaml index 515b85edbd..d1671115a1 100644 --- a/python/aisimulate/collector/cases/models/DeepseekV3ForCausalLM_cases.yaml +++ b/python/aisimulate/collector/cases/models/DeepseekV3ForCausalLM_cases.yaml @@ -13,9 +13,10 @@ base_ops: - gemm description: > DeepSeek-V3-family model-centric collection plan. Includes shared base op cases - plus stock MoE and MLA cases. SGLang WideEP ops (moe_ep) require a separate - runtime and must be requested independently; trtllm moe_ep pins the same image - as stock trtllm and is activated below. + plus stock MoE and MLA cases. WideEP ops (moe_ep) live on their own runtime + pins (wideep_sglang 0.5.10, wideep_trtllm 1.3.0rc20 vs stock trtllm 1.3.0rc29) + and are requested independently of this plan on both backends (2026-10-04: + the trtllm same-pin assumption was dropped when stock trtllm moved to rc29). model_case_values: moe: @@ -108,8 +109,6 @@ framework_specific_op_cases: cases: all mla_generation_module: cases: all - moe_ep: - cases: all vllm: mla_context_module: cases: all diff --git a/python/aisimulate/collector/cases/models/DeepseekV4ForCausalLM_cases.yaml b/python/aisimulate/collector/cases/models/DeepseekV4ForCausalLM_cases.yaml index 56c2e2ff86..f659854c6a 100644 --- a/python/aisimulate/collector/cases/models/DeepseekV4ForCausalLM_cases.yaml +++ b/python/aisimulate/collector/cases/models/DeepseekV4ForCausalLM_cases.yaml @@ -393,8 +393,6 @@ model_case_values: sparse_chunk_prefill_size: 8192 sparse_max_full_sequence_length: 1048576 sparse_tp_sizes: - hca_attn: - - 1 paged_mqa_logits: - 1 @@ -419,17 +417,31 @@ framework_specific_op_cases: # after the csa module entries). dsv4_csa_topk_calib: cases: all - # Standalone paged_mqa/hca_attn/csa_attn rows are experimental-only: the - # production SDK has no query call-site for them. Keep their registry - # entries available for explicit research runs, but do not schedule them in - # the default model plan. + # paged_mqa_logits IS consumed: it is the only sparse-kernel sidecar with a + # live SDK query path (sdk/operations/dsv4.py: the CP chunk walk loads + # PerfDataFilename.dsv4_paged_mqa_logits_module) and every shipped platform + # carries the table. The 2026-10-05 b200 shard run lost it because this + # file still called it experimental (job 469988017). The hca_attn / + # csa_attn sidecars were removed from the collector and SDK on 2026-10-15: + # their SDK readers had been retired (csa_attn never shipped data) and the + # registry-only ops failed every sm100 shard plan generated from the + # registry since 2026-10-05. + dsv4_paged_mqa_logits_module: + cases: all mhc_module: cases: all vllm: # These four full-module paths are consumed by the production DSV4 model. - # Standalone sparse kernels and mHC remain registry-only because their - # current consumer contract/schedule does not represent vLLM 0.24.0's - # fused TileLang post->pre+norm and boundary/head calls. + # hca_attn / csa_attn were removed 2026-10-15 (see the sglang note above); + # paged_mqa_logits is consumed (see the sglang note above) and shipped for + # vllm 0.24/0.25. mHC is scheduled again (2026-10-11): the SDK queries + # mhc pre/post rows on every backend, collector/vllm/collect_mhc_module.py + # measures vLLM's own mhc_pre_tilelang / mhc_post_tilelang (qualified on + # B200 0.25, shipped 0.22/0.24/0.25) and documents why pre(no-norm)+post + # tracks serving's fused post+pre+norm boundary within 1-2%; the earlier + # "contract does not represent the fused TileLang path" note predated that + # measurement. Without this row vLLM 0.30 priced mHC from the 0.25 table + # through the nearest-earlier fallback. dsv4_csa_context_module: cases: all dsv4_hca_context_module: @@ -438,6 +450,10 @@ framework_specific_op_cases: cases: all dsv4_hca_generation_module: cases: all + dsv4_paged_mqa_logits_module: + cases: all + mhc_module: + cases: all trtllm: # TRT-LLM 1.3.0rc20 ships the mHC module + torch.ops.trtllm.mhc_* kernels # (tensorrt_llm/_torch/modules/mhc/) but no DeepseekV4 model class yet, so diff --git a/python/aisimulate/collector/cases/models/Gemma4ForConditionalGeneration_cases.yaml b/python/aisimulate/collector/cases/models/Gemma4ForConditionalGeneration_cases.yaml index 8af12e525e..7c6312c19d 100644 --- a/python/aisimulate/collector/cases/models/Gemma4ForConditionalGeneration_cases.yaml +++ b/python/aisimulate/collector/cases/models/Gemma4ForConditionalGeneration_cases.yaml @@ -25,6 +25,10 @@ description: > model_case_values: attention: + # sglang_backends: sglang 0.5.21 arg_groups/model_overrides/gemma4.py:_gemma4_overrides forces + # triton on every SM except SM100 (trtllm_mha); a missing SM key falls through to the generic + # table (flashinfer on sm89/sm120), which serving never runs for Gemma4. sm89 added 2026-10-04 + # (L40: gate attn_ctx_hd512_gemma-4-26B-A4B diverged — collector FlashInfer prefill vs serving Triton). # trtllm_attn_backend: TRT-LLM 1.3.0rc20 serving forces # attn_backend=FLASHINFER for Gemma4 on every SM # (Gemma4ForConditionalGeneration.get_model_defaults, @@ -42,7 +46,7 @@ model_case_values: window_size: 1024 sglang_runtime_window_size: 1023 sglang_scaling: 1.0 - sglang_backends: {90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} + sglang_backends: {89: triton, 90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} trtllm_attn_backend: FLASHINFER tensor_parallel_sizes: [1, 2, 4, 8, 16] - id: gemma4_global_attention @@ -54,7 +58,7 @@ model_case_values: head_dim: 512 window_size: 0 sglang_scaling: 1.0 - sglang_backends: {90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} + sglang_backends: {89: triton, 90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} trtllm_attn_backend: FLASHINFER tensor_parallel_sizes: [1, 2, 4, 8, 16] - id: gemma4_31b_swa_attention @@ -65,7 +69,7 @@ model_case_values: window_size: 1024 sglang_runtime_window_size: 1023 sglang_scaling: 1.0 - sglang_backends: {90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} + sglang_backends: {89: triton, 90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} trtllm_attn_backend: FLASHINFER tensor_parallel_sizes: [1, 2, 4, 8, 16, 32] - id: gemma4_31b_global_attention @@ -75,7 +79,7 @@ model_case_values: head_dim: 512 window_size: 0 sglang_scaling: 1.0 - sglang_backends: {90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} + sglang_backends: {89: triton, 90: triton, 100: trtllm_mha, 103: trtllm_mha, 120: triton} trtllm_attn_backend: FLASHINFER tensor_parallel_sizes: [1, 2, 4, 8, 16, 32] moe: diff --git a/python/aisimulate/collector/cases/models/GlmMoeDsaForCausalLM_cases.yaml b/python/aisimulate/collector/cases/models/GlmMoeDsaForCausalLM_cases.yaml index dca784c116..00ac121374 100644 --- a/python/aisimulate/collector/cases/models/GlmMoeDsaForCausalLM_cases.yaml +++ b/python/aisimulate/collector/cases/models/GlmMoeDsaForCausalLM_cases.yaml @@ -16,6 +16,8 @@ model_paths: - nvidia/GLM-5.2-NVFP4 # GLM-5.3: the plain zai-org repository is the FP8-block release and the # BF16 weights carry the -BF16 suffix (no zai-org/GLM-5.3-FP8 exists). + # Do not spell the non-existent name here: the roster derives the coverage + # floor from repo mentions in these files. - zai-org/GLM-5.3 - zai-org/GLM-5.3-BF16 - nvidia/GLM-5.3-NVFP4 @@ -207,11 +209,15 @@ model_case_values: - zai-org/GLM-5.2 - zai-org/GLM-5.2-FP8 - nvidia/GLM-5.2-NVFP4 - - zai-org/GLM-5.3 - zai-org/GLM-5.3-BF16 + - zai-org/GLM-5.3 - nvidia/GLM-5.3-NVFP4 attention_type: dsa native_num_heads: 64 + # Real tensor-parallel shards (model-correlated narrowing): per-rank + # heads = 64 // tp (64/32/16/8/4); the shared inner head grid's 128 and + # 96/48/24/12 entries are not GLM-5.x shards. + tensor_parallel_sizes: [1, 2, 4, 8, 16] framework_specific_model_case_values: vllm: @@ -226,6 +232,7 @@ framework_specific_model_case_values: architecture: GlmMoeDsaForCausalLM attention_type: dsa native_num_heads: 64 + tensor_parallel_sizes: [1, 2, 4, 8, 16] all_frameworks_op_cases: gemm: @@ -239,8 +246,6 @@ framework_specific_op_cases: cases: all dsa_generation_module: cases: all - moe_ep: - cases: all vllm: dsa_context_module: cases: all diff --git a/python/aisimulate/collector/cases/models/KimiK25ForConditionalGeneration_cases.yaml b/python/aisimulate/collector/cases/models/KimiK25ForConditionalGeneration_cases.yaml index c0a920a7b4..fcae8120c7 100644 --- a/python/aisimulate/collector/cases/models/KimiK25ForConditionalGeneration_cases.yaml +++ b/python/aisimulate/collector/cases/models/KimiK25ForConditionalGeneration_cases.yaml @@ -213,8 +213,6 @@ framework_specific_op_cases: cases: all mla_bmm_gen_post: cases: all - moe_ep: - cases: all sglang: mla_context: cases: all diff --git a/python/aisimulate/collector/cases/models/Llama4ForConditionalGeneration_cases.yaml b/python/aisimulate/collector/cases/models/Llama4ForConditionalGeneration_cases.yaml index 0292d8963b..d0c760befd 100644 --- a/python/aisimulate/collector/cases/models/Llama4ForConditionalGeneration_cases.yaml +++ b/python/aisimulate/collector/cases/models/Llama4ForConditionalGeneration_cases.yaml @@ -39,7 +39,7 @@ model_case_values: window_sizes: [0, 8192] sglang_runtime_window_size: -1 sglang_attention_chunk_size: 8192 - sglang_backends: {90: fa3, 100: trtllm_mha, 103: trtllm_mha, 120: triton} + sglang_backends: {89: triton, 90: fa3, 100: trtllm_mha, 103: trtllm_mha, 120: triton} # sm89: arg_groups/model_overrides/llama4.py "other platforms" -> triton tensor_parallel_sizes: [1, 2, 4, 8] moe: - model_path: meta-llama/Llama-4-Scout-17B-16E-Instruct diff --git a/python/aisimulate/collector/cases/models/MiniMaxM2ForCausalLM_cases.yaml b/python/aisimulate/collector/cases/models/MiniMaxM2ForCausalLM_cases.yaml index 2ca036300a..ff7d15da02 100644 --- a/python/aisimulate/collector/cases/models/MiniMaxM2ForCausalLM_cases.yaml +++ b/python/aisimulate/collector/cases/models/MiniMaxM2ForCausalLM_cases.yaml @@ -74,6 +74,3 @@ all_frameworks_op_cases: cases: all framework_specific_op_cases: - trtllm: - moe_ep: - cases: all diff --git a/python/aisimulate/collector/cases/models/MiniMaxM3ForCausalLM_cases.yaml b/python/aisimulate/collector/cases/models/MiniMaxM3ForCausalLM_cases.yaml index 546076296a..e7e852a0cd 100644 --- a/python/aisimulate/collector/cases/models/MiniMaxM3ForCausalLM_cases.yaml +++ b/python/aisimulate/collector/cases/models/MiniMaxM3ForCausalLM_cases.yaml @@ -87,6 +87,13 @@ model_case_values: - nvidia/MiniMax-M3-NVFP4 attention_type: msa native_num_heads: 64 + # Real shards only (owner decision 2026-09-29): the MSA collectors + # derive per-rank heads as 64 // tp -> 64/32/16/8/4 (kv/index heads + # 4/2/1/1/1). The shared 12-value head grid also produced 128/96/48/ + # 24/12, which no M3 deployment reaches; 48 and 96 shard to 3/6 index + # heads and only ever failed the decode scorer's power-of-2 tile. + # TP 16 is the ceiling anyone runs this model at. + tensor_parallel_sizes: [1, 2, 4, 8, 16] all_frameworks_op_cases: gemm: diff --git a/python/aisimulate/collector/collect.py b/python/aisimulate/collector/collect.py index cdcd958c04..f5b02e18cd 100644 --- a/python/aisimulate/collector/collect.py +++ b/python/aisimulate/collector/collect.py @@ -11,10 +11,11 @@ import contextlib import functools +import logging import os import warnings -from helper import get_device_module, get_device_str +from helper import cuda_context_error, get_device_module, get_device_str def setup_warning_filters(): @@ -89,6 +90,7 @@ def setup_warning_filters(): from pathlib import Path from helper import ( + EXIT_CODE_OPS_NOT_IN_PLAN, EXIT_CODE_RESTART, PERF_TRANSACTION_FILENAME, SIDECAR_STAGING_FILENAME, @@ -115,14 +117,45 @@ def setup_warning_filters(): perf_transaction_artifact_paths, restore_perf_publications, save_error_report, + set_worker_heartbeat, setup_logging, setup_signal_handlers, validate_restored_perf_publications, ) -logger = None +# Module logger. ``setup_logging`` (main / workers) installs the configured +# collector logger; until then this is the bare library logger so in-process +# callers (fullnode finalization, unit tests, tooling) that reach the checkpoint +# and provenance paths never trip on a None logger. +logger = logging.getLogger("collector") +_LOGGER_CONFIGURED = False RESUME_SCHEMA_VERSION = "collector-resume-v2" STALL_THRESHOLD = 30 # iterations (x 0.5 s sleep = 15 s) before logging a stall warning +# Seconds of ZERO progress with live workers before the stall watchdog acts. +# Progress = a task finished, or a worker heartbeat (helper.worker_heartbeat, +# one per benchmark_with_power call): one task measuring hundreds of points +# stays alive, a point wedged inside a kernel does not. +# dump every worker's thread stacks (SIGUSR1 -> faulthandler), kill the workers, +# record their in-flight tasks as WorkerStallTimeout and restart the workers so +# the queue keeps draining. 0 disables. sglang 0.5.21 moe/bf16 on l40s/b300 +# (2026-10-06) sat 76 min at 0/58482 with no error until the pipeline watchdog +# cancelled the whole job; a finite, diagnosable failure is worth more than an +# open-ended wait. +STALL_TIMEOUT_SEC = int(os.environ.get("AISIM_STALL_TIMEOUT_SEC") or os.environ.get("AIC_STALL_TIMEOUT_SEC") or 1800) +# Consecutive stall rounds (kill + restart) with NO task finished by a worker in +# between before the executor closes the op (abandons the worker slots; the +# never-dispatched tasks are recorded as WorkerOrphanedTask, not as failed). +# 1 = close on the first stall. b200 vllm dsa_context_module s256 (job +# 481670367, 2026-10-11): the first stall at 13,002/21,960 — eight workers each +# wedged on a poison shape — closed the op and wrote the 8,950 undispatched +# tasks into the failed ledger, so the retry-failed resume re-ran all of them +# behind the same poison shapes and stalled again; two full loops were burnt. +STALL_MAX_ROUNDS = max(1, int(os.environ.get("AISIM_STALL_MAX_ROUNDS") or 3)) +# Seconds of zero progress after which the executor asks every live worker for a +# stack dump (SIGUSR1 -> faulthandler) ONCE, without killing: the l40s sglang +# moe/bf16 first case legitimately took minutes (JIT) and then the shard ran 2h23m +# to completion — the diagnosis must not cost the run. +STALL_DIAG_SEC = int(os.environ.get("AISIM_STALL_DIAG_SEC") or 600) # Failures of one (model, dtype) group within an op before the summary flags # it as systemic (a fix-me warning; nothing is skipped). SYSTEMIC_GROUP_THRESHOLD = 5 @@ -165,7 +198,7 @@ def setup_warning_filters(): @dataclass(frozen=True) class _ProducerCheckpointPlan: attestation: "_FileAttestation" - table: str + tables: tuple[str, ...] # every registry-declared table this producer owns (sorted) identity: tuple[tuple[str, object], ...] done: frozenset[str] attempted: frozenset[str] @@ -194,7 +227,7 @@ def identity(self) -> tuple[int, int]: @dataclass(frozen=True) class _ValidatedCheckpoint: attestation: _FileAttestation - table: str + tables: tuple[str, ...] attempted: frozenset[str] document: dict @@ -1111,6 +1144,115 @@ def _get_test_cases_for_model(get_func, model_path: str | None): return get_func() +class ReplaySpec: + """Failure replay (``--cases-from``): run exactly the cases an earlier run's failure + records name, plus an optional random control sample per op. + + Selection is by exact case string — the same ``str(task)`` the failure records store + as ``task_params`` — never by substring and never by reconstructing tuples, so the + requested set is unambiguous. Nothing else about the run changes: same getter, same + run_func, same dispatch, same measurement, so a replay observes exactly what the + campaign observes. The run leaves ``collector_replay.json`` in the output root; + packaging must keep such a result out of shipped tables (it is a diagnosis, not a + dataset). This is a selection mechanism like ``--case-filter``: it decides WHICH + planned cases run, never whether a case may be skipped once queued. + """ + + MARKER = "collector_replay.json" + + def __init__(self, source: str, wanted: set[str], control: int = 0): + self.source = source + self.wanted = set(wanted) + self.control = int(control) + self.matched: dict[str, set[str]] = {} + self.controls: dict[str, int] = {} + + @staticmethod + def case_key(case) -> str: + """The string a failure record stores for this case (see collect_module_safe).""" + if isinstance(case, dict) and "id" in case and "params" in case: + return str(case["params"]) + return str(case) + + def select(self, op: str, cases: list, rng: random.Random) -> list: + wanted = [case for case in cases if self.case_key(case) in self.wanted] + self.matched[op] = {self.case_key(case) for case in wanted} + controls: list = [] + if self.control > 0: + rest = [case for case in cases if self.case_key(case) not in self.wanted] + controls = rng.sample(rest, min(self.control, len(rest))) + self.controls[op] = len(controls) + return wanted + controls + + def unmatched(self) -> list[str]: + seen: set[str] = set().union(*self.matched.values()) if self.matched else set() + return sorted(self.wanted - seen) + + def report(self) -> dict: + return { + "source": self.source, + "requested": len(self.wanted), + "matched": sum(len(v) for v in self.matched.values()), + "matched_by_op": {op: len(v) for op, v in self.matched.items()}, + "controls_by_op": dict(self.controls), + "unmatched": self.unmatched(), + } + + +def load_replay_cases(path: "str | Path") -> set[str]: + """Case strings for ``--cases-from``: an ``errors_*.json[.gz]`` or + ``collection_summary_*.json[.gz]`` written by an earlier run (``task_params`` of every + entry that has one), a JSON list of case strings, or a text file with one case per line.""" + import gzip + + p = Path(path) + if p.suffix == ".gz": + with gzip.open(p, "rt", encoding="utf-8") as fh: + raw = fh.read() + else: + raw = p.read_text(encoding="utf-8") + try: + doc = json.loads(raw) + except json.JSONDecodeError: + doc = [line.strip() for line in raw.splitlines() if line.strip()] + if isinstance(doc, dict): + doc = doc.get("errors", []) + out: set[str] = set() + for item in doc: + if isinstance(item, str): + out.add(item) + elif isinstance(item, dict) and item.get("task_params") not in (None, "None", ""): + out.add(str(item["task_params"])) + if not out: + raise ValueError(f"--cases-from {path}: no case strings found") + return out + + +def _load_replay_spec(args, log) -> "ReplaySpec | None": + if not getattr(args, "cases_from", None): + return None + wanted = load_replay_cases(args.cases_from) + spec = ReplaySpec(str(args.cases_from), wanted, control=args.replay_control or 0) + log.info(f"Replay mode: {len(wanted)} requested cases from {args.cases_from}; control sample {spec.control} per op") + return spec + + +def _finish_replay(spec: "ReplaySpec | None", output_root: Path, log) -> None: + if spec is None: + return + report = spec.report() + (output_root / ReplaySpec.MARKER).write_text(json.dumps(report, indent=1) + "\n", encoding="utf-8") + if report["unmatched"]: + log.warning( + f"Replay: {len(report['unmatched'])}/{report['requested']} requested cases are not in this run's " + f"case plan (first: {report['unmatched'][:3]})" + ) + log.info( + f"Replay: matched {report['matched']}/{report['requested']} requested cases; " + f"{ReplaySpec.MARKER} written — this result is a diagnosis, not a dataset" + ) + + def _requested_ops(ops: list[str] | None, case_plan=None) -> set[str]: if ops is not None: return set(ops) @@ -1229,13 +1371,15 @@ def __init__( def load_existing(self): """Load an existing checkpoint for resume. Raises on mismatch.""" if not self._path.parent.exists() and not self._path.parent.is_symlink(): - logger.info(f"{self.module_name}: no checkpoint found, starting fresh") + if logger is not None: + logger.info(f"{self.module_name}: no checkpoint found, starting fresh") return with _locked_checkpoint_directory(self._path) as locked: canonical_path = _checkpoint_path_in_locked_directory(self._path, locked) _normalize_atomic_replace_state_at(locked.file_descriptor, canonical_path) if _entry_state_at(locked.file_descriptor, canonical_path.name) is None: - logger.info(f"{self.module_name}: no checkpoint found, starting fresh") + if logger is not None: + logger.info(f"{self.module_name}: no checkpoint found, starting fresh") return try: @@ -1276,7 +1420,8 @@ def load_existing(self): self._source_digest = snapshot.digest self._source_device = snapshot.device self._source_inode = snapshot.inode - logger.info(f"{self.module_name}: loaded checkpoint — {len(self._done)} passed, {len(self._failed)} failed") + if logger is not None: + logger.info(f"{self.module_name}: loaded checkpoint — {len(self._done)} passed, {len(self._failed)} failed") # -- public API ------------------------------------------------------- @@ -1440,6 +1585,21 @@ def __exit__(self, exc_type, exc_val, exc_tb): logger.info(f"Full profile saved to: {profile_file}") +KERNEL_LIMIT_PREFIX = "FIXME(kernel-limit)" + + +def _classify_exception(error: BaseException) -> str: + """How the error report labels a failed case. + + ``kernel_limit``: the collector itself refused the cell with a cited, + deterministic framework limit (message starts with FIXME(kernel-limit) — + divisibility guards, missing kernels, unsupported head ranges). These are + expected on every rerun and must be counted apart from ``unexpected`` + failures (pipeline summaries matched message text by hand until 2026-10-08). + """ + return "kernel_limit" if str(error).lstrip().startswith(KERNEL_LIMIT_PREFIX) else "unexpected" + + def _failure_group(task) -> str | None: """Group label for failure aggregation: one (model, dtype) family within an op. @@ -1472,6 +1632,11 @@ def _is_cuda_fatal_exception(exc, torch_mod) -> bool: error_text = str(exc).lower() fatal_markers = ( "illegal memory access", + "cudaerrorillegaladdress", + # FlashInfer's TVM-FFI launchers report a sticky error as + # "Error in function 'run' at : Got CUDA error. See above for + # details." -- the error text itself only goes to stderr. + "got cuda error", "unspecified launch failure", "cuda_error_launch_failed", "cublas_status_execution_failed", @@ -1544,6 +1709,7 @@ def worker( module_name="unknown", current_task_ids=None, consumed_sentinel=None, + heartbeat=None, ): """worker with automatic logging setup""" @@ -1565,6 +1731,15 @@ def worker( get_device_module().set_device(device) worker_logger.info(f"Worker {device_id} initialized for {module_name}") + # Liveness for the executor's stall watchdog: every measurement beats through + # helper.worker_heartbeat (benchmark_with_power), so one long task counts as + # progress before it finishes. + def _beat(): + if heartbeat is not None: + heartbeat.value = time.time() + + set_worker_heartbeat(_beat if heartbeat is not None else None) + # Process tasks while True: task_info = queue.get() @@ -1594,6 +1769,7 @@ def worker( try: worker_logger.debug(f"Starting task {task_id}") + _beat() result = func(*task, device=device) # Only the dedicated sentinel requests a recycle: entrypoints also # return plain ints (row counts), which must never be mistaken for @@ -1632,7 +1808,7 @@ def worker( "task_params": str(task), "error_type": type(e).__name__, "error_message": str(e), - "classification": "unexpected", + "classification": _classify_exception(e), "group": _failure_group(task), "traceback": traceback.format_exc(), "timestamp": datetime.now().isoformat(), @@ -1658,7 +1834,18 @@ def worker( for handler in worker_logger.handlers: handler.flush() - if _is_cuda_fatal_exception(e, torch_mod): + # Message matching alone misses launcher wrappers that hide the CUDA + # error text (FlashInfer's "Got CUDA error. See above for details."): + # the worker then runs the next case on a dead context and books an + # unrelated failure for it (gb200 job 512476645, 2026-10-27: one + # healthy case lost per Llama-4 autotune crash). Probe the context + # after every failure and recycle the worker when it is gone. + context_error = None if _is_cuda_fatal_exception(e, torch_mod) else cuda_context_error(device) + if context_error is not None: + worker_logger.warning( + f"CUDA context unusable after task {task_id} failed with {type(e).__name__}: {context_error}" + ) + if context_error is not None or _is_cuda_fatal_exception(e, torch_mod): worker_logger.warning( f"Fatal {type(e).__name__} encountered on task {task_id}. " f"Worker {device_id} exiting to reset GPU context. " @@ -1755,6 +1942,9 @@ def _unresolved_failure_errors(): manager = mp.Manager() progress_value = manager.Value("i", 0) lock = manager.Lock() + # Last time any worker reported liveness (helper.worker_heartbeat); the + # stall watchdog treats a fresh beat like a finished task. + heartbeat = mp.Value("d", 0.0, lock=False) # Track process health process_stats = {i: {"restarts": 0, "errors": []} for i in range(num_processes)} @@ -1788,6 +1978,7 @@ def start_process(device_id): module_name, current_task_ids, consumed_sentinel, + heartbeat, ), ) p.start() @@ -1873,9 +2064,79 @@ def sync_done_to_checkpoint(): # Monitor progress with error collection errors = [] + # Worker slots the stall watchdog killed on purpose. The health check below + # restarts them through start_process without recording a WorkerSignalCrash: + # their in-flight task is already a WorkerStallTimeout failure. + watchdog_killed: set[int] = set() + + def _dump_and_kill_stalled_workers(stalled_for: float, *, stall_round: int, close_op: bool): + """Stall watchdog: stacks first (SIGUSR1 -> faulthandler in the worker), + then SIGTERM/SIGKILL. The in-flight task of every live worker becomes a + WorkerStallTimeout failure; the killed workers are restarted by the + health check and the queue keeps draining. Only when ``close_op`` is set + (STALL_MAX_ROUNDS consecutive rounds with nothing finished in between) + are the slots abandoned so the orphan path below closes the op.""" + live = [(i, p) for i, p in enumerate(processes) if p is not None and p.is_alive()] + action = "closing the op" if close_op else "restarting the workers" + logger.error( + f"{module_name}: no progress for {int(stalled_for)} s with {len(live)} live worker(s) " + f"(STALL_TIMEOUT_SEC={STALL_TIMEOUT_SEC}, stall round {stall_round}/{STALL_MAX_ROUNDS}); " + f"dumping worker stacks, failing the in-flight tasks and {action}" + ) + for _, p in live: + try: + os.kill(p.pid, signal.SIGUSR1) + except Exception: + pass + time.sleep(5) # let faulthandler write before the kill + for i, p in live: + active_task_id = current_task_ids.get(i) + if active_task_id is not None and active_task_id not in accounted: + errors.append( + { + "module": module_name, + "device_id": i, + "task_id": active_task_id, + "task_params": None, + "error_type": "WorkerStallTimeout", + "error_message": f"worker made no progress for {int(stalled_for)} s; stacks dumped via SIGUSR1", + "classification": "unexpected", + "group": None, + "traceback": "", + "timestamp": datetime.now().isoformat(), + } + ) + try: + failed_tasks[active_task_id] = True + except Exception: + pass + try: + current_task_ids[i] = None + except Exception: + pass + p.terminate() + p.join(timeout=10) + if p.is_alive(): + p.kill() + p.join(timeout=10) + if p.is_alive(): + # wedged beyond SIGKILL (driver call): its GPU cannot serve a + # replacement worker either, so the slot is abandoned + logger.error(f"Process {i} ({p.pid}) survived SIGKILL; abandoning its worker slot") + processes[i] = None + elif close_op: + processes[i] = None + else: + watchdog_killed.add(i) + with tqdm(total=len(task_infos), desc=f"{module_name}", dynamic_ncols=True, leave=True) as pbar: last_progress = 0 + last_heartbeat = heartbeat.value stall_count = 0 + stall_since = None + stall_diag_sent = False + stall_rounds = 0 + accounted_after_stall_kill = None last_error_count = 0 if num_processes == 0: @@ -1904,7 +2165,7 @@ def sync_done_to_checkpoint(): "task_params": str(task_params), "error_type": type(e).__name__, "error_message": str(e), - "classification": "unexpected", + "classification": _classify_exception(e), "group": _failure_group(task_params), "traceback": traceback.format_exc(), "timestamp": datetime.now().isoformat(), @@ -1932,14 +2193,65 @@ def sync_done_to_checkpoint(): pbar.set_postfix({"errors": len(errors)}) last_error_count = len(errors) - if len(accounted) == last_progress: + if len(accounted) == last_progress and heartbeat.value > last_heartbeat: + # No task finished, but a worker measured another point of a long + # task since the last check (helper.worker_heartbeat): progress. + last_heartbeat = heartbeat.value + stall_count = 0 + stall_since = None + stall_diag_sent = False + elif len(accounted) == last_progress: stall_count += 1 + stall_since = stall_since or time.time() if stall_count > STALL_THRESHOLD: - logger.warning(f"Progress stalled at {len(accounted)}/{len(task_infos)}") + if not accounted: + # the first case carries module import + JIT/autotune: slow is normal, silent is not + logger.info( + f"waiting for the first result ({int(time.time() - stall_since)} s, 0/{len(task_infos)}; " + f"stacks dumped at {STALL_DIAG_SEC} s, workers restarted at {STALL_TIMEOUT_SEC} s of no " + f"progress, op stopped after {STALL_MAX_ROUNDS} such rounds without progress)" + ) + else: + logger.warning(f"Progress stalled at {len(accounted)}/{len(task_infos)}") stall_count = 0 + if STALL_DIAG_SEC > 0 and not stall_diag_sent and time.time() - stall_since > STALL_DIAG_SEC: + stall_diag_sent = True + live = [p for p in processes if p is not None and p.is_alive()] + logger.warning( + f"{module_name}: no progress for {int(time.time() - stall_since)} s; requesting thread " + f"stacks from {len(live)} live worker(s) (SIGUSR1 -> faulthandler, workers keep running)" + ) + for p in live: + try: + os.kill(p.pid, signal.SIGUSR1) + except Exception: + pass + if ( + STALL_TIMEOUT_SEC > 0 + and time.time() - stall_since > STALL_TIMEOUT_SEC + and any(p is not None and p.is_alive() for p in processes) + ): + # Rounds are consecutive only while no task has been finished + # (by a worker) since the previous round's kill. + if accounted_after_stall_kill is not None and len(accounted) == accounted_after_stall_kill: + stall_rounds += 1 + else: + stall_rounds = 1 + _dump_and_kill_stalled_workers( + time.time() - stall_since, + stall_round=stall_rounds, + close_op=stall_rounds >= STALL_MAX_ROUNDS, + ) + sync_done_to_checkpoint() # the WorkerStallTimeout tasks are accounted from here on + accounted_after_stall_kill = len(accounted) + stall_since = None + stall_diag_sent = False else: stall_count = 0 + stall_since = None + stall_diag_sent = False last_progress = len(accounted) + last_heartbeat = heartbeat.value # Check process health — only restart if there is still work # remaining. Workers that consumed a None sentinel or finished @@ -1985,7 +2297,13 @@ def sync_done_to_checkpoint(): with lock: progress_value.value += 1 - crash_error = create_process_exit_error(i, exit_code) + if i in watchdog_killed: + # killed by the stall watchdog on purpose: the in-flight + # task is already a WorkerStallTimeout failure + watchdog_killed.discard(i) + crash_error = None + else: + crash_error = create_process_exit_error(i, exit_code) if crash_error: errors.append(crash_error) process_stats[i]["errors"].append("process_exit") @@ -2008,13 +2326,18 @@ def sync_done_to_checkpoint(): processes[i] = None # Escape hatch: every worker slot is permanently gone (restart - # limit, consumed sentinel, or no remaining work to justify a - # restart) while tasks are still unaccounted — e.g. a worker was - # SIGKILLed after queue.get() but before it recorded anything in - # done_tasks/current_task_ids. Without this the monitor loop - # would spin at 0.5s forever with nothing able to make progress. - # Record the orphaned ids as failures so the summary and resume - # checkpoint stay complete, then stop the loop. + # limit, consumed sentinel, stall watchdog closing the op, or no + # remaining work to justify a restart) while tasks are still + # unaccounted — e.g. a worker was SIGKILLed after queue.get() but + # before it recorded anything in done_tasks/current_task_ids. + # Without this the monitor loop would spin at 0.5s forever with + # nothing able to make progress. The orphaned ids are recorded in + # the error report (WorkerOrphanedTask) for the summary and the + # harness, but NOT in the failed ledger: nothing ran them, so a + # plain --resume must queue them again. 2026-10-11, b200 vllm + # dsa_context_module: 8,950 orphans written as failed were re-run + # by --resume-retry-failed behind the poison shapes that had + # stalled the op, which stalled it again. if all(p is None for p in processes) and len(accounted) < len(task_infos): while not error_queue.empty(): error = error_queue.get() @@ -2025,7 +2348,8 @@ def sync_done_to_checkpoint(): if orphaned: logger.error( f"All workers exited with {len(orphaned)} task(s) unaccounted; " - f"recording them as failed (orphaned by worker death) and stopping the monitor loop." + f"recording them as WorkerOrphanedTask (never dispatched — left out of the failed " + f"ledger so a plain --resume runs them) and stopping the monitor loop." ) for task_id in orphaned: errors.append( @@ -2035,14 +2359,14 @@ def sync_done_to_checkpoint(): "task_id": task_id, "task_params": None, "error_type": "WorkerOrphanedTask", - "error_message": "all workers exited before this task was accounted", + "error_message": "all workers exited before this task was dispatched; " + "not recorded as failed — a plain --resume queues it again", "classification": "unexpected", "group": None, "traceback": "", "timestamp": datetime.now().isoformat(), } ) - failed_tasks[task_id] = True sync_done_to_checkpoint() break @@ -2068,6 +2392,14 @@ def sync_done_to_checkpoint(): if p.is_alive(): logger.warning(f"Process {p.pid} did not terminate, forcing...") p.terminate() + p.join(timeout=10) + if p.is_alive(): + # a worker wedged in a CUDA call ignores SIGTERM; without SIGKILL the + # interpreter's multiprocessing atexit join waits forever (pipeline + # watchdogs had to reap collect.py itself, 2026-10-05 campaign) + logger.warning(f"Process {p.pid} survived SIGTERM, killing") + p.kill() + p.join(timeout=10) # Shutdown manager to clean up resources (semaphores, etc.) manager.shutdown() @@ -2107,6 +2439,7 @@ def collect_ops( case_plan=None, sm_version: int | None = None, case_filters: list[str] | None = None, + replay: ReplaySpec | None = None, ) -> list[dict]: """Run collection for a list of resolved collection entries. @@ -2176,11 +2509,15 @@ class CompatibilityError(RuntimeError): get_func = getattr(get_module, collection["get_func"]) run_func = getattr(run_module, collection["run_func"]) - perf_filename = collection["perf_filename"] - worker_filename = collection.get("worker_perf_filename") - if worker_filename is not None: - perf_filename = str(Path(perf_filename).with_name(worker_filename)) - run_func = functools.partial(run_func, perf_filename=perf_filename) + run_func = functools.partial(run_func, perf_filename=collection["perf_filename"]) + if collection.get("extra_perf_filenames"): + # a multi-table producer (registry OpEntry.extra_perf_filenames) receives + # every table it must write; finalize binds all of them to its checkpoint + run_func = functools.partial(run_func, extra_perf_filenames=tuple(collection["extra_perf_filenames"])) + if collection.get("run_kwargs"): + # registry OpEntry.run_kwargs: fixed mode arguments for a run_func shared by + # several ops (sglang dsa_*_module_skip_indexer -> skip_indexer=True) + run_func = functools.partial(run_func, **dict(collection["run_kwargs"])) def get_func_with_limit(get_func=get_func, op=collection["type"]): from collector.capabilities import filter_cases @@ -2191,6 +2528,13 @@ def get_func_with_limit(get_func=get_func, op=collection["type"]): before_count = len(cases) cases = [case for case in cases if any(fragment in str(case) for fragment in case_filters)] logger.info(f"{op}: --case-filter kept {len(cases)}/{before_count} cases") + if replay is not None: + before_count = len(cases) + cases = replay.select(op, cases, random.Random(shuffle_seed)) + logger.info( + f"{op}: replay selected {len(replay.matched[op])} requested + " + f"{replay.controls[op]} control of {before_count} cases" + ) if shuffle: rng = random.Random(shuffle_seed) rng.shuffle(cases) @@ -2238,6 +2582,7 @@ def collect_sglang( case_plan=None, sm_version: int | None = None, case_filters: list[str] | None = None, + replay: ReplaySpec | None = None, ): """Collect performance data for SGLang with enhanced error tracking""" os.environ["FLASHINFER_LOG_LEVEL"] = "ERROR" @@ -2297,6 +2642,7 @@ def collect_sglang( case_plan=case_plan, sm_version=sm_version, case_filters=case_filters, + replay=replay, ) for collection in fullnode_collections: @@ -2313,6 +2659,7 @@ def collect_sglang( case_plan=case_plan, sm_version=sm_version, case_filters=case_filters, + replay=replay, get_test_cases_for_model=_get_test_cases_for_model, resume_checkpoint_cls=ResumeCheckpoint, logger=logger, @@ -2340,6 +2687,7 @@ def collect_vllm( case_plan=None, sm_version: int | None = None, case_filters: list[str] | None = None, + replay: ReplaySpec | None = None, ): """Collect performance data for vLLM""" from collector.version_resolver import build_collections @@ -2397,6 +2745,7 @@ def collect_vllm( case_plan=case_plan, sm_version=sm_version, case_filters=case_filters, + replay=replay, ) generate_collection_summary(all_errors, "vllm", version) @@ -2420,6 +2769,7 @@ def collect_trtllm( case_plan=None, sm_version: int | None = None, case_filters: list[str] | None = None, + replay: ReplaySpec | None = None, ): """Collect performance data for TensorRT LLM with enhanced error tracking""" from collector.trtllm.registry import REGISTRY @@ -2468,6 +2818,7 @@ def collect_trtllm( case_plan=case_plan, sm_version=sm_version, case_filters=case_filters, + replay=replay, ) generate_collection_summary(all_errors, "trtllm", version) @@ -2569,7 +2920,15 @@ def _all_op_names() -> list[str]: def _git_collector_ref(repo_root: Path) -> str: - """The repo SHA the collector ran from (design §5), "unknown" outside a repo.""" + """The repo SHA the collector ran from (design §5), "unknown" outside a repo. + + ``AISIM_COLLECTOR_REF`` (legacy ``AIC_COLLECTOR_REF``) wins when set: the + framework images carry no git, so pipelines that check the collector out + by SHA pass it in (every b200 shard run of 2026-10-05 recorded 'unknown'). + """ + forced = (os.environ.get("AISIM_COLLECTOR_REF") or os.environ.get("AIC_COLLECTOR_REF") or "").strip() + if forced: + return forced try: result = subprocess.run( ["git", "rev-parse", "HEAD"], @@ -2731,8 +3090,24 @@ def _resume_tracker_for_collection( ) -def _registered_checkpoint_table(identity: dict, *, backend: str) -> str: - """Resolve a checkpoint producer to its registry-owned table.""" +def _collection_perf_filenames(collection: dict) -> list[str]: + """Every staged table a provenance collection entry declares (primary first).""" + return [str(collection["perf_filename"]), *(str(name) for name in collection.get("extra_perf_filenames") or ())] + + +def _collection_tables(collection: dict) -> tuple[str, ...]: + return tuple(sorted(Path(name).stem for name in _collection_perf_filenames(collection))) + + +def _registered_checkpoint_tables(identity: dict, *, backend: str) -> frozenset[str]: + """Resolve a checkpoint producer to the set of registry-owned tables it writes. + + One producer (module + run_func) owns one or more tables — the primary + ``perf_filename`` plus any ``extra_perf_filenames`` its OpEntry declares + (owner decision 2026-09-28: compute_scale writes computescale_perf and + scale_matrix_perf from one measurement, so the executor binds both to the + same checkpoint instead of splitting the op). + """ from collector.version_resolver import resolve_module if set(identity) != set(_CHECKPOINT_IDENTITY_FIELDS) or identity.get("backend") != backend: @@ -2756,16 +3131,16 @@ def _registered_checkpoint_table(identity: dict, *, backend: str) -> str: registry = list(registry_module.REGISTRY_XPU) else: registry = [*registry_module.REGISTRY, *_wideep_registry_for_backend(backend)] - owned_tables = { - Path(str(entry.perf_filename)).stem + owned_table_sets = { + frozenset(Path(str(name)).stem for name in entry.perf_filenames) for entry in registry if identity["module"] == f"{backend}.{entry.op}" and identity["run_func"] == entry.run_func and resolve_module(entry, framework_version) is not None } - if len(owned_tables) != 1: - raise RuntimeError(f"checkpoint producer has no unambiguous registered table: {identity!r}") - return owned_tables.pop() + if len(owned_table_sets) != 1: + raise RuntimeError(f"checkpoint producer has no unambiguous registered table set: {identity!r}") + return owned_table_sets.pop() def _load_selected_producer_checkpoint( @@ -2796,14 +3171,14 @@ def _load_selected_producer_checkpoint( f"{context}: staged table {staging_path} has no checkpoint for selected producer " f"{resume_tracker.module_name} at {checkpoint_path}" ) - if _registered_checkpoint_table(resume_tracker._metadata, backend=backend) != staging_path.stem: + if staging_path.stem not in _registered_checkpoint_tables(resume_tracker._metadata, backend=backend): raise RuntimeError( f"{context}: checkpoint producer {resume_tracker.module_name} does not own staged table {staging_path}" ) return resume_tracker -def _producer_checkpoint_plan(resume_tracker: ResumeCheckpoint, table: str) -> _ProducerCheckpointPlan: +def _producer_checkpoint_plan(resume_tracker: ResumeCheckpoint, tables: Iterable[str]) -> _ProducerCheckpointPlan: if ( resume_tracker._source_digest is None or resume_tracker._source_device is None @@ -2817,7 +3192,7 @@ def _producer_checkpoint_plan(resume_tracker: ResumeCheckpoint, table: str) -> _ device=resume_tracker._source_device, inode=resume_tracker._source_inode, ), - table=table, + tables=tuple(sorted(tables)), identity=tuple((field, resume_tracker._metadata[field]) for field in _CHECKPOINT_IDENTITY_FIELDS), done=frozenset(resume_tracker._done), attempted=frozenset(resume_tracker._attempted), @@ -2845,10 +3220,9 @@ def _revalidate_producer_plan(producer_plan: dict[Path, _ProducerCheckpointPlan] ) checkpoint = json.loads(snapshot.contents) identity = {field: checkpoint.get(field) for field in _CHECKPOINT_IDENTITY_FIELDS} - if ( - identity != plan.identity_dict() - or _registered_checkpoint_table(identity, backend=identity["backend"]) != plan.table - ): + if identity != plan.identity_dict() or _registered_checkpoint_tables( + identity, backend=identity["backend"] + ) != frozenset(plan.tables): raise RuntimeError("checkpoint producer identity changed after preflight") if ( _checkpoint_case_ids(checkpoint, "done") != set(plan.done) @@ -2881,16 +3255,18 @@ def _pending_resume_perf_outputs( """ producers_by_output: dict[Path, list[dict]] = {} for collection in provenance_ctx.get("collections") or []: - perf_path = Path(str(collection["perf_filename"])) - if not perf_path.is_absolute(): - perf_path = output_root / perf_path - if perf_path.name.endswith("_perf.txt"): - producers_by_output.setdefault(perf_path, []).append(collection) + for perf_filename in _collection_perf_filenames(collection): + perf_path = Path(perf_filename) + if not perf_path.is_absolute(): + perf_path = output_root / perf_path + if perf_path.name.endswith("_perf.txt"): + producers_by_output.setdefault(perf_path, []).append(collection) pending_outputs: set[Path] = set() for perf_path, producers in producers_by_output.items(): staging_present = perf_path.exists() or perf_path.is_symlink() has_pending_attempts = False + has_completed_cases = False for collection in producers: resume_tracker = _load_selected_producer_checkpoint( collection, @@ -2903,8 +3279,20 @@ def _pending_resume_perf_outputs( context="resume finalization", ) has_pending_attempts = has_pending_attempts or bool(resume_tracker and resume_tracker._attempted) + has_completed_cases = has_completed_cases or bool(resume_tracker and resume_tracker._done) if has_pending_attempts: if perf_path.is_symlink() or not perf_path.is_file(): + if not has_completed_cases: + # Every attempted case of every producer failed (e.g. a lane guard + # rejecting the whole shard: sglang moe/int4_wo on 0.5.21/SM100, + # job 469988017): there is legitimately nothing staged. The failures + # are already in the error report; do not turn them into a + # finalization crash that also hides the other tables of the run. + logger.warning( + f"resume finalization: {perf_path} has an open checkpoint event but no completed case " + "and no staging table — all attempted cases failed; nothing to finalize for it" + ) + continue raise RuntimeError( f"resume finalization: open checkpoint event has no regular staging table {perf_path}" ) @@ -3042,10 +3430,11 @@ def _preflight_collector_finalization_inputs( } producers_by_path: dict[Path, list[dict]] = {} for collection in provenance_ctx.get("collections") or []: - staging_path = Path(str(collection["perf_filename"])) - if not staging_path.is_absolute(): - staging_path = output_root / staging_path - producers_by_path.setdefault(staging_path, []).append(collection) + for perf_filename in _collection_perf_filenames(collection): + staging_path = Path(perf_filename) + if not staging_path.is_absolute(): + staging_path = output_root / staging_path + producers_by_path.setdefault(staging_path, []).append(collection) producer_plan: dict[Path, _ProducerCheckpointPlan] = {} seen_attempted_case_ids: set[str] = set() @@ -3067,7 +3456,7 @@ def _preflight_collector_finalization_inputs( ) if resume_tracker is not None: checkpoint_path = resume_tracker._path - checkpoint_plan = _producer_checkpoint_plan(resume_tracker, staging_path.stem) + checkpoint_plan = _producer_checkpoint_plan(resume_tracker, _collection_tables(collection)) prior_plan = producer_plan.get(checkpoint_path) if prior_plan is None: duplicate_case_ids = set(checkpoint_plan.attempted) & seen_attempted_case_ids @@ -3234,7 +3623,7 @@ def _perf_checkpoint_record(record: _ProducerCheckpointPlan) -> dict: "digest": record.attestation.digest, "device": record.attestation.device, "inode": record.attestation.inode, - "table": record.table, + "tables": list(record.tables), "done": sorted(record.done), "failed": sorted(record.failed), "attempted": sorted(record.attempted), @@ -3292,7 +3681,7 @@ def _validate_perf_transaction_document( "digest", "device", "inode", - "table", + "tables", "done", "failed", "attempted", @@ -3300,6 +3689,14 @@ def _validate_perf_transaction_document( } if not isinstance(checkpoint, dict) or set(checkpoint) != expected_fields: raise RuntimeError(f"Invalid collector perf checkpoint record in {journal_path}") + recorded_tables = checkpoint["tables"] + if ( + not isinstance(recorded_tables, list) + or not recorded_tables + or any(not isinstance(table, str) or not table for table in recorded_tables) + or len(recorded_tables) != len(set(recorded_tables)) + ): + raise RuntimeError(f"Invalid collector perf checkpoint tables in {journal_path}") attestation = _attestation_from_record( {field: checkpoint[field] for field in ("path", "digest", "device", "inode")}, None, @@ -3310,8 +3707,8 @@ def _validate_perf_transaction_document( if not isinstance(identity, dict) or set(identity) != set(_CHECKPOINT_IDENTITY_FIELDS): raise RuntimeError(f"Invalid collector perf checkpoint identity in {journal_path}") expected_path = _checkpoint_path(checkpoint_root, identity["module"]) - table = _registered_checkpoint_table(identity, backend=backend) - if attestation.path != expected_path or checkpoint["table"] != table or table not in tables: + owned_tables = _registered_checkpoint_tables(identity, backend=backend) + if attestation.path != expected_path or set(recorded_tables) != owned_tables or not owned_tables <= tables: raise RuntimeError(f"Unowned collector perf checkpoint in {journal_path}: {attestation.path}") if attestation.path in seen_checkpoint_paths: raise RuntimeError(f"Duplicate collector perf checkpoint in {journal_path}: {attestation.path}") @@ -3329,10 +3726,10 @@ def _validate_perf_transaction_document( if attempted & seen_attempted or (not attempted and not (checkpoint["done"] or checkpoint["failed"])): raise RuntimeError(f"Invalid collector perf checkpoint attempts in {journal_path}") if attempted: - attempted_tables.add(table) + attempted_tables.update(owned_tables) seen_checkpoint_paths.add(attestation.path) seen_attempted.update(attempted) - if {checkpoint["table"] for checkpoint in checkpoint_records} != tables: + if {table for checkpoint in checkpoint_records for table in checkpoint["tables"]} != tables: raise RuntimeError(f"Collector perf transaction tables lack checkpoint owners in {journal_path}") if attempted_tables != tables: raise RuntimeError(f"Collector perf transaction tables lack attempted checkpoint case IDs in {journal_path}") @@ -3967,7 +4364,7 @@ def _validated_transaction_checkpoints( raise ValueError("attempted case IDs must be unique across checkpoint participants") if not isinstance(identity, dict) or set(identity) != set(_CHECKPOINT_IDENTITY_FIELDS): raise TypeError(f"checkpoint identity must contain exactly {_CHECKPOINT_IDENTITY_FIELDS!r}") - table = _registered_checkpoint_table(identity, backend=backend) + tables = tuple(sorted(_registered_checkpoint_tables(identity, backend=backend))) checkpoint_path = Path(path_text) expected_path = _checkpoint_path(checkpoint_root, identity["module"]) if path_text != str(expected_path) or checkpoint_path in seen_checkpoint_paths: @@ -3976,12 +4373,12 @@ def _validated_transaction_checkpoints( raise RuntimeError(f"Invalid checkpoint participant in {journal_path}: {error}") from error seen_checkpoint_paths.add(checkpoint_path) seen_attempted_case_ids.update(attempted_case_ids) - parsed_participants.append((checkpoint_path, identity, recorded_ledgers, attempted_case_ids, table)) + parsed_participants.append((checkpoint_path, identity, recorded_ledgers, attempted_case_ids, tables)) validated_participants: list[_ValidatedCheckpoint] = [] with _locked_checkpoint_root(checkpoint_root) as locked: preflight_participants = [] - for checkpoint_path, identity, recorded_ledgers, attempted_case_ids, table in parsed_participants: + for checkpoint_path, identity, recorded_ledgers, attempted_case_ids, tables in parsed_participants: try: canonical_path = _checkpoint_path_in_locked_directory(checkpoint_path, locked) effective_state = _effective_checkpoint_state_at( @@ -4008,14 +4405,14 @@ def _validated_transaction_checkpoints( preflight_participants.append( ( checkpoint_path, - table, + tables, attempted_case_ids, checkpoint, effective_state, ) ) - for checkpoint_path, table, attempted_case_ids, checkpoint, effective_state in preflight_participants: + for checkpoint_path, tables, attempted_case_ids, checkpoint, effective_state in preflight_participants: canonical_path = _checkpoint_path_in_locked_directory(checkpoint_path, locked) _require_effective_checkpoint_state_at( locked.file_descriptor, @@ -4035,7 +4432,7 @@ def _validated_transaction_checkpoints( validated_participants.append( _ValidatedCheckpoint( attestation=snapshot.attest(checkpoint_path), - table=table, + tables=tables, attempted=frozenset(attempted_case_ids), document=checkpoint, ) @@ -4067,7 +4464,8 @@ def _validate_transaction_table_ownership( allowed_staging_by_table = {Path(str(perf_file)).stem: output_root / str(perf_file) for perf_file in PerfFile} checkpoints_by_table: dict[str, list[_ValidatedCheckpoint]] = {} for checkpoint in checkpoints: - checkpoints_by_table.setdefault(checkpoint.table, []).append(checkpoint) + for table in checkpoint.tables: + checkpoints_by_table.setdefault(table, []).append(checkpoint) if any(table not in allowed_staging_by_table for table in checkpoints_by_table): raise RuntimeError(f"Invalid table in collector sidecar transaction {journal_path}") expected_staging_paths = {allowed_staging_by_table[table] for table in checkpoints_by_table} @@ -5385,7 +5783,8 @@ def _commit_collector_provenance_transaction( journal_attestation=journal_attestation, locked_output_root=locked_output_root, ) - logger.info(f"Wrote collector provenance sidecar: {meta_path}") + if logger is not None: + logger.info(f"Wrote collector provenance sidecar: {meta_path}") for checkpoint in participants: _close_checkpoint_attempts( checkpoint.path, @@ -5478,15 +5877,16 @@ def _write_collector_provenance( module_by_table: dict[str, str] = {} staging_by_table: dict[str, Path] = {} for collection in collections: - table = Path(str(collection["perf_filename"])).stem full_name = f"{collection['name']}.{collection['type']}" - ops_by_table.setdefault(table, []).append(full_name) collection_by_full_name[full_name] = collection - module_by_table.setdefault(table, collection["module"]) - staging_path = Path(str(collection["perf_filename"])) - if not staging_path.is_absolute(): - staging_path = output_root / staging_path - staging_by_table.setdefault(table, staging_path) + for perf_filename in _collection_perf_filenames(collection): + table = Path(perf_filename).stem + ops_by_table.setdefault(table, []).append(full_name) + module_by_table.setdefault(table, collection["module"]) + staging_path = Path(perf_filename) + if not staging_path.is_absolute(): + staging_path = output_root / staging_path + staging_by_table.setdefault(table, staging_path) module_failure_names = {e["module"] for e in run_errors if e.get("error_type") == "ModuleCollectionFailure"} checkpoint_root = _checkpoint_backend_root(checkpoint_dir, backend) @@ -5506,7 +5906,10 @@ def _write_collector_provenance( full_names = ops_by_table.get(table) module = module_by_table.get(table) if not full_names or module is None: - logger.warning(f"collection_meta: {table} has no registry mapping this run; skipping its provenance entry") + if logger is not None: + logger.warning( + f"collection_meta: {table} has no registry mapping this run; skipping its provenance entry" + ) continue if parquet_path not in finalization_info: @@ -5544,7 +5947,7 @@ def _write_collector_provenance( ) if resume_tracker is None: continue - checkpoint_plan = _producer_checkpoint_plan(resume_tracker, table) + checkpoint_plan = _producer_checkpoint_plan(resume_tracker, _collection_tables(collection)) else: resume_tracker = _resume_tracker_for_collection( collection, @@ -5556,7 +5959,7 @@ def _write_collector_provenance( checkpoint_plan = producer_plan.get(resume_tracker._path) if ( checkpoint_plan is None - or checkpoint_plan.table != table + or table not in checkpoint_plan.tables or checkpoint_plan.identity_dict() != resume_tracker._metadata ): raise RuntimeError( @@ -5902,6 +6305,24 @@ def main(): help="Run only cases whose string form contains SUBSTR (repeatable, OR semantics). " "Ephemeral healing filter — never persisted to YAML.", ) + parser.add_argument( + "--cases-from", + type=str, + default=None, + metavar="FILE", + help="Failure replay: run exactly the cases listed in FILE — an errors_*.json[.gz] or " + "collection_summary_*.json[.gz] from an earlier run, a JSON list of case strings, or one case per " + "line — matched by exact case string. Writes collector_replay.json beside the results; such a run " + "is a diagnosis and must never ship.", + ) + parser.add_argument( + "--replay-control", + type=int, + default=0, + metavar="N", + help="With --cases-from: also run N random cases per op that were NOT requested (seed 42) as a " + "control group for before/after comparison.", + ) parser.add_argument( "--plan-only", action="store_true", @@ -5972,9 +6393,16 @@ def main(): ops = [op for op in planned_ops if op in requested_ops] missing_ops = requested_ops - set(ops) if missing_ops: - parser.error( - "Requested ops are not present in the collector v2 case plan: " + ", ".join(sorted(missing_ops)) + # Not a usage error: the op is registered but the model plan keeps it + # registry-only (or the caller's shard plan is stale). A dedicated exit + # code lets a scheduler mark the shard terminal instead of retrying an + # argparse 2 until max_attempts (2026-10-11, sm100 shard plans). + print( + f"{parser.prog}: error: Requested ops are not present in the collector v2 case plan: " + + ", ".join(sorted(missing_ops)), + file=sys.stderr, ) + sys.exit(EXIT_CODE_OPS_NOT_IN_PLAN) if (args.model_path or args.model_architecture) and not case_plan.model_cases_paths: logger_message = ( @@ -5998,7 +6426,9 @@ def main(): os.environ.pop("COLLECTOR_MODEL_PATH", None) # Setup logging - debug flag is handled inside setup_logging - if logger is None: + global _LOGGER_CONFIGURED + if not _LOGGER_CONFIGURED: + _LOGGER_CONFIGURED = True if args.model_cases_full: log_scope = ["model_cases_full"] else: @@ -6112,6 +6542,8 @@ def was_touched_by_run(path: Path) -> bool: resolved = path.resolve() return resolved not in existing_perf_outputs or path.stat().st_mtime_ns != existing_perf_outputs[resolved] + replay = _load_replay_spec(args, logger) + # Use profiling context manager with ProfilerContext(args.backend, enabled=args.profile): collect_backend = {"trtllm": collect_trtllm, "sglang": collect_sglang, "vllm": collect_vllm}[args.backend] @@ -6125,7 +6557,9 @@ def was_touched_by_run(path: Path) -> bool: case_plan=case_plan, sm_version=sm_version, case_filters=args.case_filters, + replay=replay, ) + _finish_replay(replay, output_root, logger) converted: list[Path] = [] if args.keep_csv: diff --git a/python/aisimulate/collector/evidence_exceptions.yaml b/python/aisimulate/collector/evidence_exceptions.yaml index 578d441a9e..7dc741bbdb 100644 --- a/python/aisimulate/collector/evidence_exceptions.yaml +++ b/python/aisimulate/collector/evidence_exceptions.yaml @@ -80,6 +80,58 @@ evidence_exceptions: standards_exceptions: +# --- compute_scale (scale_matrix_perf) is an eager-only standalone kernel --- +# Owner decision (tianhaox, 2026-09-25): keep the compute_scale rows as an +# APPROXIMATION. Under vLLM 0.29's default execution mode (torch.compile with +# the norm_quant / act_quant fusion passes) the per-tensor fp8 activation +# quant never runs as its own kernel — it is folded into Inductor kernels +# (triton_*_fused__to_copy_clamp_cutlass_scaled_mm_* on Llama-3.1-70B-FP8, +# framework-mode probe 2026-09-24), so the op-level identity gate has no +# serving instance to compare against and the SDK bills a kernel serving +# does not launch separately. Direction: conservative (over-charges the +# quant boundary by a standalone kernel's launch + one extra pass over the +# activation). The rows stay collected as-is; consumers see the deviation +# through this entry and the opharness finding +# full_reprobe_framework_mode_2026_09_24. +- rule: serving-dispatch parity (layer_permissions.md) — compute_scale standalone quant kernel under torch.compile + scope: + tables: [scale_matrix_perf] + backend: vllm + versions: ["0.30.0"] # 0.29.0 was never collected; the fusion is unchanged on 0.30.0 (opharness compute_scale gate, 2026-09-27) + systems: [h100_sxm, h200_sxm] + retirement: "model the fused quant (fold compute_scale into the gemm row when the deployment compiles) or drop the standalone row for compiled deployments" + approved_by: tianhaox + expires: 2026-12-31 + +# --- fp8_static GEMM is DERIVED, and on CC>=100 serving runs a different kernel --- +# Owner decision (tianhaox, 2026-09-30): keep fp8_static as an APPROXIMATION, +# not a collected row — models needing it (ModelOpt static per-tensor FP8 +# checkpoints) will stay few. The SDK derives fp8_static from the dynamic +# per-token `fp8` gemm row plus compute_scale/scale_matrix +# (perf_database._gemm_key_names), which assumes both activation schemes run +# the SAME GEMM kernel. vLLM 0.30.0 breaks that on CC>=100: the static +# per-tensor scheme selects FlashInferFP8ScaledMMLinearKernel +# (model_executor/kernels/linear/__init__.py:421-430 priority; scaled_mm/ +# flashinfer.py:50-63 is_supported CC>=100 + per-tensor act/weight) while the +# dynamic scheme falls to CutlassFP8ScaledMMLinearKernel. Observed on the SM120 +# probe box: serving Llama-3.1-70B-Instruct-FP8 logs "Selected +# FlashInferFP8ScaledMMLinearKernel" and runs cutlass +# MainloopSm120TmaWarpSpecializedBlockwiseScaling + cuBLAS sm89_xmma at M=1; +# the `fp8` row runs the per-tensor Cutlass mainloop (opharness finding +# sm120_collector_gemm_fp8_static_row; gate gemm_fp8_Llama-3.1-70B-FP8 is an +# explained deviation, not a gate). Direction: unknown sign (different kernel, +# not a superset) — treat fp8_static predictions on these systems as +# approximate. Below CC 100 both schemes select Cutlass and the derivation holds. +- rule: serving-dispatch parity (layer_permissions.md) — fp8_static derived from the dynamic fp8 gemm row while CC>=100 serving selects a different scaled-mm kernel + scope: + tables: [gemm_perf] + backend: vllm + versions: ["0.30.0"] + systems: [rtx_pro_6000_server, b200_sxm, b300_sxm, gb200, gb300] # every CC>=100 system; only rtx_pro_6000_server-class (SM120) hardware-observed so far + retirement: "collect a static per-tensor fp8 gemm row (Fp8Config activation_scheme=static) on CC>=100 systems and let perf_database prefer the measured fp8_static row, or drop fp8_static support for ModelOpt FP8 checkpoints on those systems" + approved_by: tianhaox + expires: 2026-12-31 + # --- PR #1486: Blackwell mHC fused_hc serving-dispatch parity ------------- # Owner decision (tianhaox, 2026-08-19/20): the fused_hc collection on # SM100/103 is REJECTED for now. Default rc23 Blackwell mHC predictions diff --git a/python/aisimulate/collector/framework_manifest.yaml b/python/aisimulate/collector/framework_manifest.yaml index 6f9afdad2a..2e031f59fd 100644 --- a/python/aisimulate/collector/framework_manifest.yaml +++ b/python/aisimulate/collector/framework_manifest.yaml @@ -14,27 +14,18 @@ schema_version: 2 frameworks: sglang: source_repo: "https://github.com/sgl-project/sglang.git" + # 2026-10-01 (H20/sm90 upgrade audit, benchmarks/evidence/collector/op-probe-2026-10/findings.yaml + # hopper_sglang_collector_port_0514_to_0521_2026_10_01): default pin moved + # 0.5.14 -> 0.5.21 (newest release tag). Both tags resolve to the same + # multi-arch index digest, as the 0.5.17 model pins below do. The former + # kda (kimi-k3 branch build, 0.5.16) and msa (official v0.5.16) family pins + # are gone: 0.5.21 ships models/kimi_linear.py + the KDA Triton lanes and + # models/minimax_m3.py, and both op families were smoked/gated on it. default: - version: "0.5.14" + version: "0.5.21" images: - default: "lmsysorg/sglang:v0.5.14@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - cu130: "lmsysorg/sglang:v0.5.14-cu130@sha256:5027e95bf6ec536856b1b52a91d1f35ff5c564ab83e8a94758a169ff09bb8df3" - families: - # Kimi-K3 KDA kernels exist only on the sglang kimi-k3 branch - # (https://github.com/sgl-project/sglang/tree/kimi-k3), whose build - # reports 0.5.16. Pin the branch image for the kda family only. - kda: - version: "0.5.16" - images: - default: "lmsysorg/sglang@sha256:4b8a7542e00a4d82801d1276108438b024137c9a07f33215d03b04003e500f2b" - # MiniMax-M3 (minimax_m3.py + minimax_sparse_ops) landed in the official - # v0.5.16 release; the stock 0.5.14 default has no M3. Pin the official - # tag for the msa family only (NOT the kimi-k3 branch build above, which - # is a different container). - msa: - version: "0.5.16" - images: - default: "lmsysorg/sglang:v0.5.16@sha256:7b6a35df9839fd593a94a1eaee82d7777f472225d9f3ad1f8a2e0cb2bd1785d0" + default: "lmsysorg/sglang:v0.5.21@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469" + cu130: "lmsysorg/sglang:v0.5.21-cu130@sha256:b1259f3ea3275f66237c498ea388919729018bc9f01c3d638391e06e2cf3f469" # Model-scoped collector runtime pins: when the resolved run model_path # exactly equals one of these keys, the pin overrides BOTH the family # pins above and the framework default for every op in the run — the @@ -49,8 +40,12 @@ frameworks: # every other op's family/default resolution) unchanged. # # SGLang v0.5.17 registers Qwen3_5MoeForCausalLM / - # Qwen3_5ForCausalLM; v0.5.14 predates that model integration and cannot - # collect these cases. Both tags below resolve to the same multi-arch + # Qwen3_5ForCausalLM; v0.5.14 predated that model integration and could + # not collect these cases. The 0.5.21 default now covers the registration, + # but these model runs stay pinned to the runtime they were validated on + # until a Qwen3.8 collection is re-validated on 0.5.21 (the 0.5.21 serving + # matrix fails Qwen3.8-2.4T-A95B / -FP8 at boot, results/sm90/ + # sglang-0.5.21.yaml). Both tags below resolve to the same multi-arch # index digest. These are collector-runtime declarations, not claims of # whole-checkpoint serving validation (in particular for the RadixArk # NVFP4 collection fixture). @@ -98,88 +93,47 @@ frameworks: "nvidia/GLM-5.3-Flash-NVFP4": *glm53flash_sglang_runtime trtllm: source_repo: "https://github.com/NVIDIA/TensorRT-LLM.git" + # 2026-10-02 (H20/sm90 upgrade audit, benchmarks/evidence/collector/op-probe-2026-10/findings.yaml + # hopper_trtllm_collector_pin_rc20_to_rc29_2026_10_02): default pin moved + # 1.3.0rc20 -> 1.3.0rc29 (newest release container). The rc23 moe / + # sparse_attention / msa family pins are retired: every trtllm registry op + # smokes on rc29 (21/23 clean; moe's first plan case and mamba2's largest + # cells fail by their own classified raises) and all 17 path gates align on + # it, DSV4 included. Retired-pin history, kept for the data it explains: + # * rc23 moe (2026-08-08): FP4 w4a8_mxfp4_mxfp8 rows show reproducible + # non-monotonic num_tokens segments on Blackwell (median ratio vs rc20 + # 1.003-1.013; per-bucket tactic/tile-wave quantization, not a campaign + # artifact; GitLab aic-auto-collector pipelines 61692150-61692160, + # 61771569-61838704). Existing moe shapes still reuse rc20 data through + # the declared reuse.yaml channel; the rc23 evidence exception in + # evidence_exceptions.yaml keeps its own scope. + # * rc23 sparse_attention: DSV4 attention needed trtllm>=1.3.0rc21 and the + # DSA modules travelled with it (SM90 smoke at bring-up). + # * rc23 msa: the true-sparse fmha_sm100 backend (SM100/103) landed in rc23; + # SM90 msa data stayed at rc20 (Triton reference path, identical-or-worse + # measured at rc23). default: - version: "1.3.0rc20" + version: "1.3.0rc29" images: - default: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20@sha256:1532b38814b3faf2affdb5ef01ca91468685d314ffb7e8926a0567595355ed88" - families: - # trtllm FP4 (w4a8_mxfp4_mxfp8) MoE rows show non-monotonic - # num_tokens segments (4096->8192, 12288->16384 dips) on all four - # Blackwell systems. An rc23 re-collection reproduced the dips - # (2026-08-08; exact consumer-key join vs rc20: median ratio - # 1.003-1.013, median |per-row delta| 2.4-2.9%, ~25% of rows >5% — - # benchmark run-to-run spread, not a systematic shift), so the dips - # are REAL, reproducible kernel behavior (per-bucket tactic/tile-wave - # quantization), not a campaign artifact. Evidence: GitLab - # aic-auto-collector (project 205688) pipelines 61692150-61692160 - # (moe re-collection, 2026-08-08); DSV4 attention campaigns ran as - # pipelines 61771569-61838704 on feat/sbatch-partition-override. The DSV4 moe shapes stay collected on this - # runtime for version coherence with the DSV4 attention family; the - # remaining moe shapes reuse rc20 via the declared reuse.yaml channel. - moe: - version: "1.3.0rc23" - images: - default: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23@sha256:316b840a08a8174fc3f6b5716828bdfe1daaf629ee1ac2a8b7a22526d141a007" - # DeepSeek-V4 attention modules require trtllm>=1.3.0rc21 (the model - # class and sparse/deepseek_v4 backend first ship there; see - # collector/trtllm/collect_dsv4_attn.py __compat__). The catalog family - # also carries the DSA modules (dsa_*_module), so trtllm DSA collection - # moves to this runtime with it — DSA on rc23 smoke-verified on SM90 - # (H20) at bring-up; the rc20 serving citations in collect_mla_module - # are re-audited on this bump per layer_permissions.md. - sparse_attention: - version: "1.3.0rc23" - images: - default: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23@sha256:316b840a08a8174fc3f6b5716828bdfe1daaf629ee1ac2a8b7a22526d141a007" - # MiniMax-M3: the true-sparse fmha_sm100 backend (implementation="msa", - # SM100/103-only) landed in rc23; rc20 has only the Triton reference - # path (measured identical-or-worse on SM90 at rc23, so the existing - # existing SM90 msa data stays at rc20). Pin the msa family to rc23 for the - # Blackwell collection. - msa: - version: "1.3.0rc23" - images: - default: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc23@sha256:316b840a08a8174fc3f6b5716828bdfe1daaf629ee1ac2a8b7a22526d141a007" + default: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc29@sha256:124da3f10b73afbf4242f5c68fb04b48736c61fbe662e715202371fd7f046f43" vllm: source_repo: "https://github.com/vllm-project/vllm.git" default: - version: "0.24.0" + version: "0.30.0" images: - default: "vllm/vllm-openai:v0.24.0@sha256:251eba5cc7c12fed0b75da22a9240e582b1c9e39f6fbc064f86781b963bd814f" - cu129: "vllm/vllm-openai:v0.24.0-cu129@sha256:6b005c78cd17a1f0215f31e95d183833f4dfd6167c4473241b80d1c8843b91e8" - families: - # Kimi-K3 KDA kernels exist on the official kimi-k3 preview image - # (CUDA 13 build with fused_kda_decode + FlashKDA; setuptools-scm - # version 0.1.dev19262+gb6bbf29dd) and upstreamed in v0.27.0 — but the - # family pin STAYS on the preview until the coordinated consumer fix - # lands (review jasonqinzhou #1559, comment 3808798129): at 0.27.0 the - # DS conv-state layout makes is_fused_kda_decode_supported false, so - # serving decode runs causal_conv1d_update + - # fused_recurrent_kda_packed_decode + explicit gated RMSNorm, while - # KimiK3Model._build_generation_ops still prices every vLLM non-spec - # decode as fused_kda_decode and drops the norm. The 0.27.0 gb300 kda - # rows collected by this PR are checked in as evidence - # (systems/data/gb300/kda/vllm/0.27.0); until the consumer routes by - # version, loading that version fills 44 fused_kda_decode rows from - # this preview sibling via the shared layer and misprices generation - # KDA. Re-pin when the consumer emits the conv-update + packed - # decode pair (and blocks preview fused fill) for 0.27.0+. - kda: - version: "0.1.dev19262" - images: - default: "vllm/vllm-openai@sha256:e90e2603b2781936651ba019804137714367c69e10a7b25a2e57b46995225616" - # mla: pin 0.27.0 so gb300/vllm MLA module rows are collected on the - # same runtime as the K3 serving lane (KDA-family sibling) instead of - # silently consuming the trtllm 1.3.0 low-fidelity fallback. Collector - # API drift 0.24.0 -> 0.27.0 is audited before the pin lands in data. - mla: - version: "0.27.0" - images: - default: &vllm_0270_image "vllm/vllm-openai:v0.27.0@sha256:07ea4e292adf3a26b05ac97114b28849cf4551a26beb1fbe7decd3842d752ed7" - mla_bmm: - version: "0.27.0" - images: - default: *vllm_0270_image + default: "vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90" + # kda family pin RETIRED (owner decision 2026-09-28): the family ran on the + # kimi-k3 preview image (0.1.dev19262) while the consumer's fused-decode + # pricing did not match 0.27's packed-decode serving path (review #1559). + # On 0.30.0 the opharness framework-mode probe of Kimi-K3 runs the fused + # decode kernel (kda_decode_fusion_many_heads_kernel) and FlashKDA prefill + # (opharness findings vllm_0300_upgrade_2026_09_27), and the kda collector + # follows 0.30's is_flashkda_supported contract — so kda resolves to the + # default 0.30.0 like every other vllm op. + # mla / mla_bmm family pins (0.27.0) were folded into the 0.29.0 + # default 2026-09-20 per the pin-consolidation rule: pins BELOW the + # target version upgrade with it; the kda preview pin (an odd dev-build + # pin) was revisited at the 0.30.0 move and retired, see above. # moe: NO family pin (owner decision 2026-08-02, reverting the # 2026-08-01 family-wide preview pin — a temporary preview build must # not own a family default; a family pin here hijacks image selection diff --git a/python/aisimulate/collector/hash_closures.yaml b/python/aisimulate/collector/hash_closures.yaml index b69833881e..67062ed89e 100644 --- a/python/aisimulate/collector/hash_closures.yaml +++ b/python/aisimulate/collector/hash_closures.yaml @@ -277,13 +277,6 @@ collector.vllm.collect_mla_module: - collector/vllm/utils.py - collector/cases/base_ops/mla_module.yaml - __model_cases__ -collector.vllm.collect_mla_module_027: - - collector/vllm/utils.py - - collector/vllm/collect_mla_module.py - # 0.27.0-route shim whose get/run functions are re-exported from - # collector/vllm/collect_mla_module.py — identical case reads. - - collector/cases/base_ops/mla_module.yaml - - __model_cases__ collector.vllm.collect_moe: - collector/vllm/utils.py - collector/cases/base_ops/moe.yaml diff --git a/python/aisimulate/collector/helper.py b/python/aisimulate/collector/helper.py index 38a307db70..605cdbdbfa 100644 --- a/python/aisimulate/collector/helper.py +++ b/python/aisimulate/collector/helper.py @@ -40,6 +40,10 @@ # Exit codes EXIT_CODE_RESTART = 10 # Exit code to indicate restart is needed +# collect.py exits with this when --ops names an op the collector v2 case plan does not +# contain (registry-only op, or a stale shard plan). Distinct from argparse's 2 so a +# scheduler can mark the shard terminal instead of retrying it. +EXIT_CODE_OPS_NOT_IN_PLAN = 3 PERF_TRANSACTION_FILENAME = ".perf-finalization.transaction.json" SIDECAR_TRANSACTION_FILENAME = ".collection_meta.transaction.json" SIDECAR_STAGING_FILENAME = ".collection_meta.pending.yaml" @@ -72,6 +76,40 @@ class PerfLogError(RuntimeError): _NVML_INITIALIZED = False _NVML_LOCK = threading.Lock() +# Stall-watchdog heartbeat. collect.py's worker registers a sink in each worker +# process and benchmark_with_power beats once per measurement, so one task that +# measures hundreds of points (a DSA context series over the whole isl x prefix +# grid) keeps counting as progress, while a point wedged inside a kernel still +# goes silent and gets the watchdog's kill after STALL_TIMEOUT_SEC. +_WORKER_HEARTBEAT: Callable[[], None] | None = None + + +def set_worker_heartbeat(sink: Callable[[], None] | None) -> None: + """Register (or clear) the callable that reports liveness to the executor.""" + global _WORKER_HEARTBEAT + _WORKER_HEARTBEAT = sink + + +def worker_heartbeat() -> None: + """Report that the running task is still measuring; a no-op outside the executor.""" + global _WORKER_HEARTBEAT + sink = _WORKER_HEARTBEAT + if sink is None: + # collect.py imports this module as ``helper`` while the op modules import + # ``collector.helper``: two module objects, so a sink registered through + # one has to be found from the other. + for name in ("helper", "collector.helper"): + other = sys.modules.get(name) + if other is not None and getattr(other, "_WORKER_HEARTBEAT", None) is not None: + sink = _WORKER_HEARTBEAT = other._WORKER_HEARTBEAT + break + if sink is None: + return + try: + sink() + except Exception: # liveness must never fail a measurement + pass + def _parse_bool_env(env_var: str, default: bool = False) -> bool: """ @@ -207,6 +245,97 @@ def _monitoring_loop(self): self._stop_event.wait(self.interval_s) +class PhaseClock: + """Wall-clock split of one collector case or sweep point into named phases. + + ``mark(name)`` closes the phase that began at the previous mark (or at + construction); ``absorb`` folds a nested breakdown (for example the + ``phase_seconds`` that ``benchmark_with_power`` reports) into the clock + under a prefix; ``add_to`` accumulates the phases into a per-sweep total + that ``render_phase_totals`` prints. Collectors print these lines so the + campaign log shows where a case spends its seconds; they never change + what is measured. + """ + + def __init__(self) -> None: + self.start = self.last = time.perf_counter() + self.phases: dict[str, float] = {} + + def mark(self, name: str) -> None: + now = time.perf_counter() + self.phases[name] = self.phases.get(name, 0.0) + (now - self.last) + self.last = now + + def absorb(self, phases, prefix: str = "") -> None: + for name, seconds in (phases or {}).items(): + key = f"{prefix}{name}" + self.phases[key] = self.phases.get(key, 0.0) + float(seconds) + + def total(self) -> float: + return time.perf_counter() - self.start + + def render(self) -> str: + parts = [f"{name}={seconds:.2f}s" for name, seconds in self.phases.items()] + return " ".join(parts + [f"total={self.total():.2f}s"]) + + def add_to(self, totals: dict[str, float]) -> None: + for name, seconds in self.phases.items(): + totals[name] = totals.get(name, 0.0) + seconds + totals["total"] = totals.get("total", 0.0) + self.total() + totals["points"] = totals.get("points", 0) + 1 + + +def render_phase_totals(totals: dict[str, float]) -> str: + points = int(totals.get("points", 0)) + parts = [f"{name}={seconds:.1f}s" for name, seconds in totals.items() if name != "points"] + return f"{points} point(s): " + " ".join(parts) + + +class PeriodicGarbageCollector: + """Run ``gc.collect()`` every ``every`` calls, or sooner when allocated CUDA + memory sits more than ``growth_bytes`` above the lowest level seen after a + collection. + + A full collection in a process that holds a loaded model costs 0.2-0.4 s; + running it after every case or sweep point was half of an SGLang DSV4 + context sweep and 0.17 s of every TRT-LLM module case. The collectors + release framework state (KV pools, CUDA graphs, workspace buffers) + explicitly, so the collection only has to catch reference cycles, and the + memory guard keeps a cycle-heavy sweep from growing unbounded. + """ + + def __init__(self, every: int = 16, growth_bytes: int = 2 << 30) -> None: + self.every = max(1, int(every)) + self.growth_bytes = int(growth_bytes) + self.calls = 0 + self.floor: int | None = None + + @staticmethod + def _allocated() -> int: + try: + import torch + + if torch.cuda.is_available(): + return int(torch.cuda.memory_allocated()) + except Exception: + pass + return 0 + + def maybe_collect(self) -> bool: + import gc + + self.calls += 1 + allocated = self._allocated() + if self.floor is None: + self.floor = allocated + if self.calls < self.every and allocated <= self.floor + self.growth_bytes: + return False + gc.collect() + self.calls = 0 + self.floor = min(self.floor, self._allocated()) + return True + + @contextmanager def benchmark_with_power( device, @@ -220,6 +349,7 @@ def benchmark_with_power( use_cuda_graph: bool = True, # Set False only for explicitly eager measurements. *, explicit_graph_cleanup: bool = False, + num_eager_warmups: int | None = None, ): """ Context manager that handles warmup, graph capture, timing, and power monitoring. @@ -243,6 +373,10 @@ def benchmark_with_power( when capture fails or a traceback retains it. Opted into by the vLLM MLA/DSA collector; other callers retain their existing graph lifetime and synchronization behavior. + num_eager_warmups: Eager passes before capture (default: num_warmups). Graph + replay warmups stay at num_warmups. Set lower when the + kernel already ran (a dry run) and the measurement is + graph-based; eager measurements should keep the default. Yields: dict with keys: @@ -255,6 +389,8 @@ def benchmark_with_power( import torch dev_mod = get_device_module(device) + # One beat per measurement: the executor's stall watchdog counts it as progress. + worker_heartbeat() # Auto-detect configuration from environment if not explicitly provided if measure_power is None: @@ -264,19 +400,35 @@ def benchmark_with_power( # Adaptive num_runs calculation actual_num_runs = num_runs + # Eager passes before capture. They exist so lazily initialised paths (JIT, + # autotune, plan caches) run outside the capture; a caller whose kernel + # already executed (a dry run) can ask for fewer than the replay warmups, + # since the Python-heavy eager forward costs several times a graph replay. + eager_warmups = num_warmups if num_eager_warmups is None else max(int(num_eager_warmups), 0) + # Wall time of each protocol phase, reported as results["phase_seconds"]. + phase_seconds: dict[str, float] = {} + _phase_start = time.perf_counter() + + def _close_phase(name: str) -> None: + nonlocal _phase_start + now = time.perf_counter() + phase_seconds[name] = phase_seconds.get(name, 0.0) + (now - _phase_start) + _phase_start = now + if measure_power: # Estimate single iteration time with warmup start_warmup = dev_mod.Event(enable_timing=True) end_warmup = dev_mod.Event(enable_timing=True) + estimate_runs = max(eager_warmups, 1) dev_mod.synchronize() start_warmup.record() - for _ in range(num_warmups): + for _ in range(estimate_runs): kernel_func() end_warmup.record() dev_mod.synchronize() - single_iter_time = start_warmup.elapsed_time(end_warmup) / num_warmups / 1000.0 # seconds + single_iter_time = start_warmup.elapsed_time(end_warmup) / estimate_runs / 1000.0 # seconds # Adaptive duration: use shorter duration for very fast kernels to reduce memory pressure target_duration = power_min_duration @@ -292,10 +444,12 @@ def benchmark_with_power( else: # Normal warmup dev_mod.synchronize() - for _ in range(num_warmups): + for _ in range(eager_warmups): kernel_func() dev_mod.synchronize() + _close_phase("eager_warmup") + # ═══════════════════════════════════════════════════════════════════ # Device Graph Capture with Optional Fallback (CUDA: CUDAGraph, XPU: XPUGraph) # ═══════════════════════════════════════════════════════════════════ @@ -347,19 +501,20 @@ def benchmark_with_power( dev_mod.empty_cache() use_graph = False + _close_phase("capture") capture_phase_completed = True # ═══════════════════════════════════════════════════════════════ # Warmup the ACTUAL execution path (after graph capture) # ═══════════════════════════════════════════════════════════════ - dev_mod.synchronize() - for _ in range(num_warmups): - if use_graph: + # Eager callers already ran their warmups before the capture step; + # repeating them here re-ran the same eager path (23% of an SGLang + # MSA context sweep) ahead of an identical timed loop. + if use_graph: + dev_mod.synchronize() + for _ in range(num_warmups): g.replay() - else: - # Fallback: Direct execution matching actual execution path - for _ in range(repeat_n): - kernel_func() - dev_mod.synchronize() + dev_mod.synchronize() + _close_phase("graph_warmup") # Initialize power monitor if enabled power_monitor = None @@ -396,6 +551,7 @@ def benchmark_with_power( kernel_func() end_event.record() dev_mod.synchronize() + _close_phase("timed") # Check for throttling throttled = False @@ -428,6 +584,7 @@ def benchmark_with_power( "throttled": throttled, "num_runs_executed": actual_num_runs, "used_cuda_graph": use_graph, # NEW: Inform caller which path was used + "phase_seconds": phase_seconds, } finally: # Preserve default cleanup after timing starts. Explicit cleanup also @@ -440,6 +597,7 @@ def benchmark_with_power( if not explicit_graph_cleanup: dev_mod.synchronize() dev_mod.empty_cache() + _close_phase("cleanup") @contextmanager @@ -474,6 +632,409 @@ def power_monitoring_only(device, measure_power: bool | None = None): pass +# Set to 0/false to time FlashInfer-backed kernels on FlashInfer's untuned default +# tactic (the pre-2026-10-27 behavior). Default on. +FLASHINFER_AUTOTUNE_ENV = "COLLECTOR_FLASHINFER_AUTOTUNE" +FLASHINFER_AUTOTUNE_BUDGET_ENV = "COLLECTOR_FLASHINFER_AUTOTUNE_BUDGET" +FLASHINFER_AUTOTUNE_BUDGET_FLASHINFER = "flashinfer" + + +@dataclass(frozen=True) +class FlashInferTuneBudget: + """Per-tactic profiling budget applied to FlashInfer's autotuner during the tuning pass. + + FlashInfer profiles every candidate tactic with ``warmup`` untimed launches, ``repeat`` + timed launches and a ``stream_delay_us`` delay kernel in front of the timed block (its + defaults: 3 / 10 / 5000, enough to rank tactics for a server that tunes once per + process). The collector tunes once per (shape, EP, num_tokens), so that budget is the + dominant cost of a FlashInfer-backed sweep (the 2026-10-27 b200 probe: +3.6 s per + row, 28x the untuned collection time). The collector only needs the *ranking* -- the + final measurement is still ``benchmark_with_power``'s own warm-ups and runs -- and the + tactics that matter differ by 2x, so a smaller budget ranks them just as well while + near-ties stay inside collector noise. + """ + + warmup: int = 1 + repeat: int = 3 + stream_delay_us: int = 1000 + + def __str__(self) -> str: + return f"warmup={self.warmup} repeat={self.repeat} delay_us={self.stream_delay_us}" + + +def flashinfer_autotune_enabled() -> bool: + return _parse_bool_env(FLASHINFER_AUTOTUNE_ENV, default=True) + + +def flashinfer_autotune_budget() -> FlashInferTuneBudget | None: + """``COLLECTOR_FLASHINFER_AUTOTUNE_BUDGET``: ``"warmup,repeat,delay_us"`` (default + ``"1,3,1000"``), or ``"flashinfer"`` to leave FlashInfer's own 3/10/5000 untouched + (returns ``None``).""" + raw = os.environ.get(FLASHINFER_AUTOTUNE_BUDGET_ENV, "").strip() + if not raw: + return FlashInferTuneBudget() + if raw.lower() == FLASHINFER_AUTOTUNE_BUDGET_FLASHINFER: + return None + parts = [part.strip() for part in raw.split(",")] + try: + warmup, repeat, delay_us = (int(part) for part in parts) + except ValueError: + raise ValueError( + f"{FLASHINFER_AUTOTUNE_BUDGET_ENV}={raw!r}: expected 'warmup,repeat,delay_us' " + f"(three integers) or '{FLASHINFER_AUTOTUNE_BUDGET_FLASHINFER}'" + ) from None + if warmup < 0 or repeat < 1 or delay_us < 0: + raise ValueError(f"{FLASHINFER_AUTOTUNE_BUDGET_ENV}={raw!r}: warmup and delay_us must be >= 0, repeat >= 1") + return FlashInferTuneBudget(warmup, repeat, delay_us) + + +def cuda_context_error(device=None) -> str | None: + """Probe whether the CUDA context of ``device`` (default: current) is still usable. + + A sticky CUDA error (illegal memory access, launch failure) poisons the whole + context: every later kernel on this process fails, so a case that merely runs + next on the worker is reported as failed for a crash it did not cause. The probe + synchronises the device and launches one trivial kernel; it returns the error + text when either raises, ``None`` when the context is healthy or torch is absent. + """ + try: + import torch + except ImportError: + return None + try: + if device is None: + device = torch.cuda.current_device() + torch.cuda.synchronize(device) + torch.ones(1, device=device).sum().item() + except Exception as exc: # any failure here means the context is gone + return f"{type(exc).__name__}: {exc}" + return None + + +def _flashinfer_failed_tactics(tuner) -> str: + stats = getattr(tuner, "stats", None) + failed = getattr(stats, "failed_tactics", None) + if not failed: + return "no failed tactics recorded" + try: + parts = [] + for op, tactics in failed.items(): + names = sorted(str(t) for t in tactics) + shown = ", ".join(names[:4]) + (", ..." if len(names) > 4 else "") + parts.append(f"{op}: {len(names)} failed tactics ({shown})") + return "; ".join(parts) + except Exception: # diagnostics only + return f"failed tactics: {failed!r}"[:400] + + +_FLASHINFER_TUNER_BUDGET_ATTRS = ( + ("warmup", "warmup"), + ("repeat", "repeat"), + ("stream_delay_us", "stream_delay_micro_secs"), +) + + +@contextmanager +def _flashinfer_tuner_budget(tuner, budget: FlashInferTuneBudget | None): + """Apply ``budget`` to the ``AutoTuner`` singleton's plain profiling attributes for the + duration of the block, restoring whatever was there afterwards.""" + if budget is None: + yield + return + saved = {} + for field_name, attr in _FLASHINFER_TUNER_BUDGET_ATTRS: + if hasattr(tuner, attr): + saved[attr] = getattr(tuner, attr) + setattr(tuner, attr, getattr(budget, field_name)) + try: + yield + finally: + for attr, value in saved.items(): + setattr(tuner, attr, value) + + +_FLASHINFER_TUNER_DIAG_MARKERS = ( + # MoERunner.get_valid_tactics: C++ enumeration error swallowed, [] returned (DEBUG). + "failed to get valid tactics", + # Per-tactic profiling failure: "Failed when profiling ... Error + # occurred: " (DEBUG) and the "Skipped N unsupported tactic(s)" summary (INFO). + "error occurred", + "unsupported tactic", + # Nothing profiled at all: the op is served by the fallback tactic. + "no valid tactic", + "fallback", + "oom", +) + + +def _flashinfer_loggers() -> list[logging.Logger]: + """The loggers FlashInfer's autotuner writes to. ``flashinfer.jit.core.logger`` is a + ``FlashInferJITLogger`` built directly (not via ``logging.getLogger``), so it has to be + imported; the stdlib name is included for builds that register it normally.""" + loggers: list[logging.Logger] = [] + try: + from flashinfer.jit.core import logger as fi_logger + + loggers.append(fi_logger) + except Exception: # optional dependency: any import failure means "absent" + pass + std = logging.getLogger("flashinfer") + if all(lg is not std for lg in loggers): + loggers.append(std) + return loggers + + +class _LogMessageCapture(logging.Handler): + def __init__(self): + super().__init__(logging.DEBUG) + self.messages: list[str] = [] + + def emit(self, record): + try: + self.messages.append(record.getMessage()) + except Exception: # diagnostics must never break the tuning pass + pass + + +@contextmanager +def _flashinfer_tuner_log_capture(): + """Collect FlashInfer's autotuner log at DEBUG for the block without changing what it + prints: the loggers' pre-existing handlers are pinned at the old effective threshold + while the logger level is lowered, and everything is restored afterwards.""" + handler = _LogMessageCapture() + saved: list[tuple[logging.Logger, int, list[tuple[logging.Handler, int]]]] = [] + for lg in _flashinfer_loggers(): + effective = lg.getEffectiveLevel() + pinned = [] + for existing in lg.handlers: + pinned.append((existing, existing.level)) + existing.setLevel(max(existing.level, effective)) + saved.append((lg, lg.level, pinned)) + lg.addHandler(handler) + lg.setLevel(logging.DEBUG) + try: + yield handler.messages + finally: + for lg, level, pinned in saved: + lg.removeHandler(handler) + lg.setLevel(level) + for existing, old_level in pinned: + existing.setLevel(old_level) + + +def _flashinfer_tuner_diagnostics(messages: Iterable[str]) -> list[str]: + """Autotuner messages that explain a tactic *not* being profiled or chosen: the + per-runner ``get_valid_tactics`` failure (``core.py`` swallows the C++ exception and + returns ``[]`` at DEBUG), per-tactic profiling failures (every candidate runs the full + op, routing kernel included, on synthetic inputs; a failing one is marked inf at DEBUG), + empty tactic sets, fallback selection and OOM. Duplicates are collapsed.""" + out: list[str] = [] + seen: set[str] = set() + for msg in messages: + low = msg.lower() + if not any(marker in low for marker in _FLASHINFER_TUNER_DIAG_MARKERS): + continue + text = " ".join(msg.split())[:400] + if text not in seen: + seen.add(text) + out.append(text) + return out + + +def _flashinfer_failed_tactic_counts(tuner) -> dict[str, int]: + """``{op::runner: failed tactic count}`` from the tuner's statistics (empty if absent).""" + try: + failed = tuner.stats.failed_tactics + return {str(k): len(v) for k, v in failed.items()} + except Exception: # diagnostics only + return {} + + +def _flashinfer_moe_tactic_tables() -> dict[str, dict]: + """``{module: MoERunner.valid_tactics_dict}`` for the FlashInfer trtllm-gen MoE modules + already built in this process. The class-level dict caches, per instance key + ``(dtype_act, dtype_weights, fp8_quant, top_k, hidden, intermediate, local_experts, + activation, shuffled, layout, per_token, num_tokens, lora)``, the tactic list the C++ + launcher enumerated -- exactly what the autotuner had to choose from. Only modules whose + ``functools.cache`` is populated are read, so nothing is JIT-built here.""" + tables: dict[str, dict] = {} + try: + import flashinfer.fused_moe.core as fi_core + except Exception: # optional dependency + return tables + impl = getattr(fi_core, "_get_trtllm_moe_sm100_module_impl", None) + cache_info = getattr(impl, "cache_info", None) + try: + if cache_info is not None and cache_info().currsize > 0: + table = getattr(fi_core.get_trtllm_moe_sm100_module().MoERunner, "valid_tactics_dict", None) + if isinstance(table, dict): + tables["trtllm_sm100"] = table + except Exception: # diagnostics only + pass + return tables + + +def _flashinfer_new_tactic_entries(before: dict[str, set]) -> list[str]: + """Human-readable ``module key -> N tactics [...]`` lines for tactic-table keys that + appeared since ``before`` (a ``{module: set(keys)}`` snapshot).""" + lines = [] + for module, table in _flashinfer_moe_tactic_tables().items(): + for key in table: + if key in before.get(module, set()): + continue + tactics = table[key] + try: + shown = [tuple(int(x) for x in t) if isinstance(t, (list, tuple)) else t for t in list(tactics)[:8]] + count = len(tactics) + except Exception: # diagnostics only + shown, count = [repr(tactics)[:200]], -1 + more = f" (+{count - 8} more)" if count > 8 else "" + lines.append(f"{module} {key!r} -> {count} tactics {shown}{more}") + return lines + + +def _flashinfer_device_tag() -> str: + """`` dev`` for the current CUDA device, so the per-process tuning decisions of + parallel workers can be told apart in a shared log; empty without CUDA.""" + try: + import torch + + if torch.cuda.is_available(): + return f" dev{torch.cuda.current_device()}" + except Exception: # diagnostics only + pass + return "" + + +def _flashinfer_autotune_api(): + """FlashInfer's ``autotune`` context manager and ``AutoTuner`` class, or ``(None, None)``.""" + try: + from flashinfer.autotuner import AutoTuner, autotune + except ImportError: + return None, None + return autotune, AutoTuner + + +@contextmanager +def flashinfer_autotuned(num_tokens: int, tune_fn: Callable[[], None], *, label: str = ""): + """Time FlashInfer-backed kernels on the tactic FlashInfer's autotuner picks, as serving does. + + vLLM (``enable_flashinfer_autotune``, default on for Blackwell) and SGLang (warm-up + autotune, plus the extend bucket since sglang #32556) both run FlashInfer kernels on + autotuned tactics. Without this context the collector times FlashInfer's *untuned* + fallback: tile = clamp(next_pow2(num_tokens * top_k / local_num_experts), 8, 128) and + the first config of an unstable ``std::sort`` over the kernel manifest -- a value that + moved between FlashInfer 0.6.12 and 0.6.18 (flashinfer #6838) and broke every SM100 + nvfp4 MoE EP sweep whose average tokens per local expert exceeded 128 (vLLM 0.30.0 / + SGLang 0.5.21 campaign tables, 2026-10-27). + + ``tune_fn()`` runs once inside ``autotune(True, tuning_buckets=(num_tokens,))`` so + every FlashInfer op it reaches is profiled at exactly this token count (with the + caller's real routing, so the EP-local expert load is what gets tuned). The ``with`` + body then runs under ``autotune(False, ...)`` with the same bucket: lookups hit the + entries just profiled, nothing is profiled inside the body (CUDA-graph capture safe), + and an op the tuner has no entry for keeps FlashInfer's fallback exactly as before. + + The tuning pass runs under :func:`flashinfer_autotune_budget` (default 1 warm-up, 3 + timed launches, 1 ms delay per tactic instead of FlashInfer's 3 / 10 / 5 ms; see + :class:`FlashInferTuneBudget`), restored afterwards. The log line records the pass's + wall time and budget so a probe can read the tuning cost per token point. FlashInfer's + own autotuner log is captured at DEBUG during the pass and its fallback / failure + messages (an op whose ``get_valid_tactics`` raised is silently timed untuned) are + printed as a ``FlashInfer autotune WARNING`` line next to the case. + + Yields ``True`` when a tuning pass ran, ``False`` when FlashInfer is not importable or + ``COLLECTOR_FLASHINFER_AUTOTUNE`` is off (then the body runs untouched and ``tune_fn`` + is not called). Non-FlashInfer kernels inside the body are unaffected either way. + """ + autotune, auto_tuner_cls = _flashinfer_autotune_api() + if autotune is None or not flashinfer_autotune_enabled(): + yield False + return + budget = flashinfer_autotune_budget() + tuner = auto_tuner_cls.get() + cache = getattr(tuner, "profiling_cache", None) + entries_before = len(cache) if hasattr(cache, "__len__") else None + cache_keys_before = set(cache.keys()) if isinstance(cache, dict) else set() + buckets = (int(num_tokens),) + started = time.perf_counter() + tune_error: BaseException | None = None + tuner_log: list[str] = [] + failed_before = _flashinfer_failed_tactic_counts(tuner) + tactic_keys_before = {module: set(table) for module, table in _flashinfer_moe_tactic_tables().items()} + try: + with ( + _flashinfer_tuner_log_capture() as tuner_log, + _flashinfer_tuner_budget(tuner, budget), + autotune(True, tuning_buckets=buckets), + ): + tune_fn() + except Exception as exc: # re-raised below, after the context probe + tune_error = exc + elapsed = time.perf_counter() - started + # FlashInfer profiles every candidate tactic and swallows per-tactic exceptions + # (logged at DEBUG, tactic marked inf). A tactic that faults leaves a sticky CUDA + # error behind: the chosen tactic then fails with an unrelated-looking "Got CUDA + # error", and the next case on this worker inherits the dead context. Probe here so + # the failure record names the tuning pass and the worker is recycled right away. + poisoned = cuda_context_error() + if poisoned is not None: + raise RuntimeError( + f"FlashInfer autotune pass at num_tokens={num_tokens} {label} left the CUDA context " + f"in error state ({poisoned}); {_flashinfer_failed_tactics(tuner)}" + ) from tune_error + if tune_error is not None: + raise tune_error + budget_desc = str(budget) if budget is not None else "flashinfer defaults" + if entries_before is not None: + outcome = f"{len(cache) - entries_before} new tactic entries" + else: + outcome = "tuning pass done" + dev = _flashinfer_device_tag() + print( + f"FlashInfer autotune: {outcome} at num_tokens={num_tokens} {label}".rstrip() + + f" [{elapsed:.2f}s, {budget_desc}]{dev}" + ) + # An op the tuner could not enumerate tactics for is timed on the untuned fallback + # with no visible trace (b300 sglang 0.5.21 Nemotron-3-Nano nvfp4 relu2 shape, + # 2026-10-28: unchanged against the untuned table while every other shape moved). + # Surface FlashInfer's own DEBUG explanation next to the case. + # What the tuner chose: every cache entry this pass added, with its tactic. Two worker + # processes tune the same key independently (per-process cache), so a slow cell whose + # sibling distributions are fine is told apart from a slow kernel by comparing these. + if isinstance(cache, dict): + for key in cache: + if key in cache_keys_before: + continue + entry = cache[key] + tactic = entry[0] if isinstance(entry, (tuple, list)) and entry else entry + print( + f"FlashInfer autotune chosen at num_tokens={num_tokens} {label}".rstrip() + + f": {' '.join(str(key).split())[:300]} -> tactic {tactic!r}{dev}" + ) + # What the tuner had to choose from: the launcher-enumerated tactic list per instance + # key (shows the intermediate size / activation the kernel really saw and whether the + # candidate set was a single tactic, i.e. tuned == untuned by construction). + for line in _flashinfer_new_tactic_entries(tactic_keys_before): + print(f"FlashInfer autotune tactics at num_tokens={num_tokens} {label}".rstrip() + f": {line}{dev}") + diagnostics = _flashinfer_tuner_diagnostics(tuner_log) + failed_after = _flashinfer_failed_tactic_counts(tuner) + newly_failed = {k: n - failed_before.get(k, 0) for k, n in failed_after.items() if n > failed_before.get(k, 0)} + if diagnostics or newly_failed: + parts = [] + if newly_failed: + parts.append( + "tactics failed while profiling: " + ", ".join(f"{k}={n}" for k, n in sorted(newly_failed.items())) + ) + if diagnostics: + more = f" (+{len(diagnostics) - 4} more)" if len(diagnostics) > 4 else "" + parts.append(f"{len(diagnostics)} autotuner messages: " + "; ".join(diagnostics[:4]) + more) + head = f"FlashInfer autotune WARNING at num_tokens={num_tokens} {label}".rstrip() + print(f"{head}: " + " | ".join(parts) + dev) + with autotune(False, tuning_buckets=buckets): + yield True + + def setup_signal_handlers(worker_id): """Setup signal handlers to log crashes.""" import traceback as _tb @@ -499,6 +1060,18 @@ def signal_handler(signum, frame): for sig in [signal.SIGTERM, signal.SIGABRT]: signal.signal(sig, signal_handler) + # SIGUSR1 = "dump every thread's stack to stderr and keep running". The + # executor's stall watchdog sends it before killing a wedged worker, so a + # hang leaves the exact frame in the log (sglang 0.5.21 moe/bf16 on l40s + # and b300, 2026-10-06: 8 workers took their first task and never returned, + # 76 min of silence, nothing to diagnose from). + try: + import faulthandler + + faulthandler.register(signal.SIGUSR1, all_threads=True, chain=False) + except Exception: + pass + # SIGSEGV might not be catchable on all platforms try: signal.signal(signal.SIGSEGV, signal_handler) @@ -607,7 +1180,9 @@ def setup_logging(scope=["all"], debug=False, worker_id=None): # Create log directory time_stamp = datetime.now().strftime("%Y%m%d_%H%M%S") - _LOG_DIR = Path(f"{log_scope_dirname(scope)}_{time_stamp}") + # Absolute: workers and the profiler read it back from the environment, + # possibly after the working directory changed. + _LOG_DIR = Path.cwd() / f"{log_scope_dirname(scope)}_{time_stamp}" if not _LOG_DIR.is_dir(): _LOG_DIR.mkdir() @@ -678,7 +1253,19 @@ def save_error_report(errors, filename): def get_sm_version(): - """Get CUDA compute capability (SM version)""" + """Get CUDA compute capability (SM version). + + ``AIS_SM`` (``sm100`` or ``100``; legacy ``AIC_SM``) overrides the probe so + case getters can be enumerated for a target SM on a box without that GPU + (the op-probe harness's case inventory: the case set is a pure function + of (SM, framework version, case files) and must be derivable without + collecting). Real collection never sets it. + """ + forced = os.environ.get("AIS_SM") or os.environ.get("AIC_SM") + if forced: + digits = "".join(ch for ch in str(forced) if ch.isdigit()) + if digits: + return int(digits) try: import torch @@ -1911,6 +2498,7 @@ def _prepare_perf_file( snapshot_file.seek(0) table = pc_csv.read_csv(snapshot_file) _validate_moe_default_eligibility(table, parquet_path, pa=pa) + table, _duplicate_rows = _dedupe_staged_perf_rows(table, parquet_path, pa=pa, pc=pc_compute) new_rows = table.num_rows merged_existing = False merge_target = None @@ -1922,7 +2510,7 @@ def _prepare_perf_file( merge_target_was_absent = True else: old_table, merge_target = _read_attested_parquet(parquet_path, pq=pq, locked_root=locked_root) - table, merged_existing, new_rows = _merge_perf_rows(table, old_table, parquet_path, pa=pa) + table, merged_existing, new_rows = _merge_perf_rows(table, old_table, parquet_path, pa=pa, pc=pc_compute) table = _normalize_power_metrics(table, pa=pa, pc=pc_compute) temporary = perf_preparation_path(parquet_path) @@ -2566,13 +3154,71 @@ def _validate_moe_default_eligibility(table, parquet_path: Path, *, pa): raise ValueError(f"{parquet_path}: default_eligible=false requires a nonblank string kernel_source") -def _merge_perf_rows(new_table, old_table, parquet_path: Path, *, pa): +def _dedupe_staged_perf_rows(table, parquet_path: Path, *, pa, pc): + """Keep the last row per identity key within one staging file. + + The staging ``*_perf.txt`` is opened in append mode and kept across resumed + windows, and ``--resume --resume-retry-failed`` re-runs a task that failed + after logging some of its rows, so those rows are appended a second time + (2026-10-11: rtxpro msa_context 10,342 of 13,972 rows, gb200 moe/fp8_block + 2,590 pairs). The merge-with-existing path already dedupes on the identity + key; this applies the same rule to the rows of the staging file itself, + keeping the newest measurement and the original row order. Identity = every + column except the measured metrics and MoE eligibility metadata. + """ + metadata_columns = {"default_eligible"} if parquet_path.name == "moe_perf.parquet" else set() + identity = [c for c in table.column_names if c not in PERF_METRIC_COLUMNS and c not in metadata_columns] + deduped, dropped = _keep_last_row_per_identity(table, identity, pa=pa, pc=pc) + if dropped: + logging.getLogger(__name__).info( + "convert_perf_csv_to_parquet: %s: dropped %d re-appended duplicate rows " + "(%d identity keys kept the newest measurement)", + parquet_path.name, + dropped, + deduped.num_rows, + ) + return deduped, dropped + + +def _keep_last_row_per_identity(table, identity: list[str], *, pa, pc): + """Keep the last row per identity key, in the original order of the kept rows. + + pyarrow only: the collector images ship pyarrow (the harness installs it + into ``pyarrow-site``) but not pandas, and the merge path used to need + pandas — the second finalize of a shard (resume / retry-failed) died with + ``No module named 'pandas'`` (b200 vllm dsa_context_module, 2026-10-12). + Returns ``(deduped_table, dropped_rows)``. + """ + if table.num_rows < 2 or not identity: + return table, 0 + index_name = "__row_index__" + while index_name in table.column_names: + index_name = "_" + index_name + keyed = table.select(identity) + # group_by cannot hash an all-null (null-typed) column; its values carry no identity anyway + for i, field in enumerate(keyed.schema): + if pa.types.is_null(field.type): + keyed = keyed.set_column(i, field.name, keyed.column(i).cast(pa.string())) + keyed = keyed.append_column(index_name, pa.array(range(table.num_rows), type=pa.int64())) + last = keyed.group_by(identity).aggregate([(index_name, "max")]) + keep = last.column(f"{index_name}_max") + keep = pc.take(keep, pc.sort_indices(keep)) + deduped = table.take(keep) + return deduped, table.num_rows - deduped.num_rows + + +def _identity_keys(table, identity: list[str]) -> list[tuple]: + """One hashable key per row, in row order.""" + columns = [table.column(name).to_pylist() for name in identity] + return list(zip(*columns, strict=True)) if columns else [()] * table.num_rows + + +def _merge_perf_rows(new_table, old_table, parquet_path: Path, *, pa, pc): """Merge freshly-collected rows into an existing perf parquet, keeping the newest row per identity key. Returns ``(table, merged_existing, current_event_rows)``; schema incompatibility returns the new table with - ``merged_existing=False`` and its unchanged row count.""" - import pandas as pd - + ``merged_existing=False`` and its unchanged row count. pyarrow only — see + ``_keep_last_row_per_identity``.""" log = logging.getLogger(__name__) _validate_moe_default_eligibility(old_table, parquet_path, pa=pa) @@ -2595,8 +3241,8 @@ def _merge_perf_rows(new_table, old_table, parquet_path: Path, *, pa): ) # Compare (name, type) pairs for IDENTITY columns only: matching names - # with drifted types would otherwise round-trip through pandas and - # silently rewrite the parquet under a different schema. Metric columns + # with drifted types would otherwise be concatenated and silently rewrite + # the parquet under a different schema. Metric columns # are exempt from the type check — pyarrow.csv infers an all-empty # optional metric column as `null` while populated runs infer `double`, # and treating that drift as a mismatch made the merge silently OVERWRITE @@ -2604,10 +3250,21 @@ def _merge_perf_rows(new_table, old_table, parquet_path: Path, *, pa): # the finalizer then casts power metrics to double and replaces nulls with # the repository's 0.0 unavailable-measurement sentinel. Order-insensitive # (the merge realigns column order below); Arrow metadata is ignored - # (pandas round-trips change it). + # (parquets written by the pandas-based merges before 2026-10-12 carry + # pandas metadata). + # `string` and `large_string` are one logical type: pyarrow.csv infers + # `string`, but a parquet that went through one of those pandas merges + # comes back as `large_string`, so the THIRD finalize of a table (run -> + # resume -> retry-failed) used to read as a schema mismatch and OVERWRITE + # the accumulated rows (sm120 attention_generation fp8kv, 2026-10-01: + # 27,303 rows replaced by 5,067). Compare the logical string type. + def _logical(type_str: str) -> str: + return {"large_string": "string", "large_binary": "binary"}.get(type_str, type_str) + def fields(schema): return sorted( - (f.name, str(f.type)) if f.name not in PERF_METRIC_COLUMNS else (f.name, "") for f in schema + (f.name, _logical(str(f.type))) if f.name not in PERF_METRIC_COLUMNS else (f.name, "") + for f in schema ) old_fields = fields(old_table.schema) @@ -2644,30 +3301,50 @@ def fields(schema): new_table.column(f.name).cast(f.type), ) - new_df = new_table.to_pandas() - old_df = old_table.to_pandas() + # align string width variants (string vs large_string) on the new side so the + # concatenation below does not widen or fail on a logical-type-equal column + for f in old_table.schema: + if f.name in PERF_METRIC_COLUMNS: + continue + new_field = new_table.schema.field(f.name) + if new_field.type != f.type and _logical(str(new_field.type)) == _logical(str(f.type)): + new_table = new_table.set_column( + new_table.schema.get_field_index(f.name), f.name, new_table.column(f.name).cast(f.type) + ) - new_df = new_df[old_df.columns.tolist()] # align column order - identity = [c for c in old_df.columns if c not in PERF_METRIC_COLUMNS and c not in metadata_columns] + new_table = new_table.select(old_table.column_names) # align column order + identity = [c for c in old_table.column_names if c not in PERF_METRIC_COLUMNS and c not in metadata_columns] if inherit_eligibility: - annotations = old_df.drop_duplicates(subset=identity, keep="last")[identity + ["default_eligible"]] - new_df = new_df.drop(columns="default_eligible").merge(annotations, on=identity, how="left", sort=False) - flags = new_df["default_eligible"] - new_df["default_eligible"] = flags.where(flags.notna(), True).astype(bool) - new_df = new_df[old_df.columns.tolist()] - current_event_rows = len(new_df.drop_duplicates(subset=identity, keep="last")) - combined = pd.concat([old_df, new_df], ignore_index=True) - deduped = combined.drop_duplicates(subset=identity, keep="last").reset_index(drop=True) + # newest existing annotation per identity key; keys the existing table + # never annotated (or annotated null) stay eligible + annotations: dict[tuple, bool] = {} + existing_flags = old_table.column("default_eligible").to_pylist() + for key, flag in zip(_identity_keys(old_table, identity), existing_flags, strict=True): + annotations[key] = True if flag is None else bool(flag) + flags = [annotations.get(key, True) for key in _identity_keys(new_table, identity)] + new_table = new_table.set_column( + new_table.schema.get_field_index("default_eligible"), + "default_eligible", + pa.array(flags, type=pa.bool_()), + ) + current_event_rows = _keep_last_row_per_identity(new_table, identity, pa=pa, pc=pc)[0].num_rows + # Drop schema metadata on both sides: a parquet written by an earlier + # pandas-based merge carries pandas index metadata that would otherwise + # make concat_tables see two different schemas (and be re-published). + old_table = old_table.replace_schema_metadata(None) + new_table = new_table.replace_schema_metadata(None) + combined = pa.concat_tables([old_table, new_table], promote_options="default") + deduped, replaced = _keep_last_row_per_identity(combined, identity, pa=pa, pc=pc) log.info( "convert_perf_csv_to_parquet: merged %s: %d existing + %d new -> %d rows " "(%d identity keys replaced by newer measurements)", parquet_path.name, - len(old_df), - len(new_df), - len(deduped), - len(combined) - len(deduped), + old_table.num_rows, + new_table.num_rows, + deduped.num_rows, + replaced, ) - return pa.Table.from_pandas(deduped, preserve_index=False), True, current_event_rows + return deduped, True, current_event_rows def find_perf_csv_outputs(output_root: str | os.PathLike = ".", *, recursive: bool = False) -> list[Path]: diff --git a/python/aisimulate/collector/kernel_source_backends.yaml b/python/aisimulate/collector/kernel_source_backends.yaml index 885f8cdc27..62912b0dac 100644 --- a/python/aisimulate/collector/kernel_source_backends.yaml +++ b/python/aisimulate/collector/kernel_source_backends.yaml @@ -137,6 +137,7 @@ mappings: - {framework: sglang, kernel_source: sglang_torch_linear, backend: cublas, source: "PR#1342 collect_gemm.py (torch.matmul/linear)"} - {framework: sglang, kernel_source: sglang_torch_bmm, backend: cublas, source: "PR#1342 collect_mla_bmm.py (torch.bmm)"} - {framework: sglang, kernel_source: sglang_sgl_kernel_bmm_fp8, backend: cublas, source: "PR#1342 collect_mla_bmm.py (sgl_kernel bmm_fp8 = cublasLt)"} + - {framework: sglang, kernel_source: sglang_flashinfer_bmm_fp8, backend: cublas, source: "collector/sglang/collect_mla_bmm.py:22-26 (sglang>=0.5.21: fp8_utils.bmm_fp8 wraps flashinfer.bmm_fp8 as custom op flashinfer_bmm_fp8_batched, docstring 'via flashinfer's cuBLAS backend', fp8_utils.py:246-273 @0.5.21); flashinfer csrc/bmm_fp8.cu bmm_fp8_internal_cublaslt"} - {framework: sglang, kernel_source: sglang_sgl_kernel_fp8_scaled_mm, backend: cutlass, source: "PR#1342 collect_gemm.py (sgl_kernel fp8_scaled_mm)"} - {framework: sglang, kernel_source: sglang_deepgemm_gemm_nt_f8f8bf16, backend: deepgemm, source: "PR#1342 collect_gemm.py"} - {framework: sglang, kernel_source: sglang_triton_w8a8_block_fp8, backend: triton, source: "sglang/collect_gemm_0520.py (fp8_utils.py:1094-1107 @v0.5.20 N%64/K%128 fallback)"} @@ -152,6 +153,8 @@ mappings: # GDN sub-kernels and mHC. - {framework: sglang, kernel_source: chunk_gated_delta_rule, backend: fla_triton, source: "PR#1342 collect_gdn.py (fla triton chunk kernel)"} - {framework: sglang, kernel_source: fused_recurrent_gated_delta_rule_packed_decode, backend: fla_triton, source: "PR#1342 collect_gdn.py"} + - {framework: sglang, kernel_source: flashinfer_chunk_gated_delta_rule, backend: flashinfer, source: "collector/sglang/collect_gdn.py:366-371 (FlashInferGDNKernel.extend lane); sglang srt/layers/attention/linear/kernels/gdn_flashinfer.py:129-131,465 @0.5.21 imports flashinfer.gdn_prefill.chunk_gated_delta_rule"} + - {framework: sglang, kernel_source: fused_qkvzba_causal_conv1d_update_contiguous, backend: triton, source: "collector/sglang/collect_gdn.py:505-514 (decode fused proj + conv update); sglang kernels/ops/attention/triton_gdn_fused_proj.py:576 @0.5.21 (@triton.jit kernels), gdn_backend.py:668-760"} - {framework: sglang, kernel_source: sglang_tilelang_mhc_pre, backend: tilelang, source: "PR#1342 collect_mhc_module.py"} - {framework: sglang, kernel_source: sglang_tilelang_mhc_post, backend: tilelang, source: "PR#1342 collect_mhc_module.py"} - {framework: sglang, kernel_source: sglang_torch_mhc_pre, backend: torch, source: "PR#1342 collect_mhc_module.py (torch fallback leaf)"} @@ -220,10 +223,15 @@ mappings: # Mamba/GDN kernels. - {framework: trtllm, kernel_source: causal_conv1d_fn, backend: causal_conv1d, source: "trtllm _torch mamba modules"} + - {framework: trtllm, kernel_source: causal_conv1d_fn_token_major, backend: causal_conv1d, source: "collector/trtllm/collect_mamba2.py _run_prefill_conv_token_major: channel-last view + token-major out, the 1.3.0rc29 Mamba2Mixer default (mamba2_mixer.py:319,541-569; causal_conv1d_channellast_fwd_kernel)"} - {framework: trtllm, kernel_source: causal_conv1d_update, backend: causal_conv1d, source: "trtllm _torch mamba modules"} - {framework: trtllm, kernel_source: chunk_gated_delta_rule, backend: fla_triton, source: "trtllm _torch GDN (triton)"} - - {framework: trtllm, kernel_source: fused_recurrent_gated_delta_rule, backend: fla_triton, source: "trtllm _torch GDN (triton)"} + - {framework: trtllm, kernel_source: fused_recurrent_gated_delta_rule, backend: fla_triton, source: "trtllm _torch GDN (triton) — FLA recurrent fallback lane; serving's pure-decode entry is the fused_sigmoid_gating row below (rc23)"} + - {framework: trtllm, kernel_source: chunk_gated_delta_rule_flashinfer, backend: flashinfer, source: "collector/trtllm/collect_gdn.py: gdn_mixer._resolve_chunk_gated_delta_rule picked fla/flashinfer_chunk (FlashInfer cute-DSL GDN chunk, SM90/SM100; gdn_mixer.py:57-70 @1.3.0rc23)"} + - {framework: trtllm, kernel_source: fused_sigmoid_gating_delta_rule_update_flashinfer, backend: flashinfer, source: "collector/trtllm/collect_gdn.py:_bench_serving_decode_entry — fla/fused_sigmoid_gating_recurrent.py:452-458 dispatched to FlashInfer gdn_decode_bf16_state (kernel_cutlass_gdn_decode_*) @1.3.0rc23"} + - {framework: trtllm, kernel_source: fused_sigmoid_gating_delta_rule_update_triton, backend: fla_triton, source: "collector/trtllm/collect_gdn.py:_bench_serving_decode_entry — same entry, _can_use_flashinfer_gdn_decode False (non-SM90/100, K/V != 128, T != N) -> fused_sigmoid_gating_delta_rule_update_kernel (Triton)"} - {framework: trtllm, kernel_source: mamba_chunk_scan_combined, backend: flashinfer, source: "collector/trtllm/collect_mamba2.py:290 (mamba2 SSD chunk scan; tensorrt_llm/_torch/modules/mamba/ssd_combined.py:437-444@1.3.0rc15 flashinfer SSDCombined on SM100+ when _flashinfer_ssd_supported; Triton fallback on non-SM100 or unsupported tile sizes — no shipped row takes it: all rc15 scan rows are (chunk=128, dstate=128, headdim=64) on b200_sxm, flashinfer-eligible by construction; label split for future fallback coverage: AIC-1677)"} + - {framework: trtllm, kernel_source: selective_state_update_flashinfer, backend: flashinfer, source: "collector/trtllm/collect_mamba2.py:194-219 (_decode_state_update_kernel imports flashinfer.mamba.selective_state_update when serving selects it; mamba2_mixer.py:23 @1.3.0rc29)"} # Comm. - {framework: trtllm, kernel_source: TRTLLM, backend: custom_allreduce, source: "collector/network/collect_all_reduce.py (trtllm custom AR)"} @@ -274,7 +282,13 @@ mappings: - {framework: vllm, kernel_source: vllm_flashinfer_fidecode, backend: flashinfer, source: "PR#1344 collect_attn.py"} - {framework: vllm, kernel_source: vllm_flashinfer_trtllmprefill, backend: trtllm_mha, source: "PR#1344 collect_attn.py (flashinfer trtllm-gen prefill)"} - {framework: vllm, kernel_source: vllm_flashinfer_trtllmdecode, backend: trtllm_mha, source: "PR#1344 collect_attn.py (flashinfer trtllm-gen decode)"} - - {framework: vllm, kernel_source: vllm_flashinfer_flashinfertrtllmapidecode, backend: unverified, source: "collector/vllm/collect_attn.py:_dense_kernel_source records only the metadata class; vllm/v1/attention/backends/flashinfer.py:502-509,1539-1546@v0.25.0 uses FlashInferTrtllmAPIDecode for both XQA and TRTLLM_GEN. The label omits the kernel discriminator, so it cannot identify one backend across systems. https://github.com/vllm-project/vllm/blob/v0.25.0/vllm/v1/attention/backends/flashinfer.py"} + # vLLM 0.30.0 merged the trtllm-gen and XQA decode metadata into FlashInferTrtllmAPIDecode (flashinfer.py:608, kernel field + # FlashInferDecodeKernel xqa|trtllm-gen :568-572); collect_attn appends the kernel to the label. Both are TRT-LLM MHA kernels + # (XQA = kernel_mha in the opharness sm120 records, trtllm-gen = the SM100 fmha cubins) -> trtllm_mha. The bare class label is + # kept for rows collected before the kernel suffix (2026-10-01). + - {framework: vllm, kernel_source: vllm_flashinfer_flashinfertrtllmapidecode_xqa, backend: trtllm_mha, source: "collect_attn.py _dense_kernel_source (FlashInferTrtllmAPIDecode.kernel == XQA; sm120 fp8-KV decode, opharness serving record kernel_mha)"} + - {framework: vllm, kernel_source: vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen, backend: trtllm_mha, source: "collect_attn.py _dense_kernel_source (FlashInferTrtllmAPIDecode.kernel == TRTLLM_GEN; SM100 decode)"} + - {framework: vllm, kernel_source: vllm_flashinfer_flashinfertrtllmapidecode, backend: trtllm_mha, source: "collect_attn.py before the kernel suffix (0.30.0 rows collected 2026-09-29..10-01 on B200/RTX 5000)"} # MLA module labels: the AttentionBackendEnum member that ran. - {framework: vllm, kernel_source: FLASH_ATTN, backend: flash_attn, source: "PR#1344 collect_mla_module.py (enum-name labels)"} - {framework: vllm, kernel_source: FLASH_ATTN_MLA, backend: flash_attn, source: "PR#1344 collect_mla_module.py"} @@ -283,6 +297,9 @@ mappings: - {framework: vllm, kernel_source: FLASHMLA_SPARSE_DSV4, backend: flashmla, source: "PR#1344 collect_dsv4_attn.py"} - {framework: vllm, kernel_source: FLASHINFER_MLA, backend: flashinfer, source: "PR#1344 collect_mla_module.py"} - {framework: vllm, kernel_source: FLASHINFER_MLA_SPARSE, backend: flashinfer, source: "PR#1344 collect_mla_module.py"} + - {framework: vllm, kernel_source: FLASHINFER_MLA_SPARSE_SM120, backend: flashinfer, source: "collect_mla_module.py enum-name label; vllm v0.30.0 v1/attention/backends/registry.py:82 FlashInferMLASparseSM120Backend -> mla/flashinfer_mla_sparse_sm120.py:221-226 flashinfer_trtllm_batch_decode_with_kv_cache_mla (flashinfer.decode)"} + - {framework: vllm, kernel_source: FLASHINFER_MLA_SPARSE_DSV4, backend: flashinfer, source: "collect_dsv4_attn.py enum-name label; vllm v0.30.0 registry.py:100 DeepseekV4FlashInferMLASparseBackend -> models/deepseek_v4/nvidia/flashinfer_sparse.py:27,519 flashinfer_trtllm_batch_decode_sparse_mla_dsv4 (flashinfer.decode)"} + - {framework: vllm, kernel_source: FLASH_ATTN_MLA_SPARSE, backend: flash_attn, source: "collect_mla_module.py enum-name label; vllm v0.30.0 registry.py:119 FlashAttnMLASparseBackend -> mla/flashattn_mla_sparse.py:31,315 vllm_flash_attn flash_attn_varlen_func (fa_utils.flash_attn_supports_mla: FA3 on SM90)"} - {framework: vllm, kernel_source: TRITON_MLA, backend: triton, source: "PR#1344 collect_mla_module.py"} - {framework: vllm, kernel_source: TRTLLM_RAGGED, backend: trtllm_ragged, source: "PR#1344 collect_mla_module.py:438-444 (MLAPrefillBackendEnum.TRTLLM_RAGGED, trtllm-gen ragged prefill)"} # GEMM / quant labels: vllm kernel class names. @@ -293,6 +310,7 @@ mappings: - {framework: vllm, kernel_source: FlashInferFp8BlockScaledMMKernel, backend: flashinfer, source: "PR#1344 collect_gemm.py"} - {framework: vllm, kernel_source: FlashInferCutlassNvFp4LinearKernel, backend: flashinfer_cutlass, source: "PR#1344 collect_gemm.py"} - {framework: vllm, kernel_source: FlashInferCuteDslNvFp4LinearKernel, backend: flashinfer_cutedsl, source: "PR#1344 collect_gemm.py"} + - {framework: vllm, kernel_source: MarlinFP8ScaledMMLinearKernel, backend: marlin, source: "collect_gemm.py kernel class label; vllm v0.30.0 model_executor/kernels/linear/scaled_mm/marlin.py (apply_fp8_marlin_linear)"} - {framework: vllm, kernel_source: "torch.nn.functional.linear", backend: cublas, source: "PR#1344 collect_gemm.py (bf16)"} - {framework: vllm, kernel_source: static_scaled_fp8_quant, backend: vllm_kernel, source: "PR#1344 collect_computescale.py:73 (vllm _custom_ops)"} - {framework: vllm, kernel_source: dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant, backend: vllm_kernel, source: "PR#1344 collect_computescale.py:63 (derived overhead measure)"} @@ -305,12 +323,15 @@ mappings: - {framework: vllm, kernel_source: vllm_fp8moe_flashinfer_cutlass_flashinferexperts, backend: flashinfer_cutlass, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular, backend: flashinfer_trtllm_gen, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic, backend: flashinfer_trtllm_gen, source: "PR#1344 collect_moe.py"} + - {framework: vllm, kernel_source: vllm_fp8moe_deepgemm_deepgemmexperts, backend: deepgemm, source: "collect_moe.py:795 label = quant method + oracle backend + experts class; vllm v0.30.0 fused_moe/experts/deep_gemm_moe.py:125 DeepGemmExperts (TritonOrDeepGemmExperts leaf, triton_deep_gemm_moe.py:29)"} - {framework: vllm, kernel_source: vllm_modeloptfp8moe_triton_tritonexperts, backend: triton_fused_moe, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts, backend: flashinfer_cutlass, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic, backend: flashinfer_trtllm_gen, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts, backend: flashinfer_cutlass, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular, backend: flashinfer_trtllm_gen, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic, backend: flashinfer_trtllm_gen, source: "PR#1344 collect_moe.py"} + - {framework: vllm, kernel_source: vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts, backend: flashinfer_cutedsl, source: "collect_moe.py:795; vllm v0.30.0 fused_moe/experts/flashinfer_cutedsl_moe.py:31 FlashInferCuteDSLExperts"} + - {framework: vllm, kernel_source: vllm_compressedtensorswna16moe_marlin_marlinexperts, backend: marlin, source: "collect_moe.py:795; vllm v0.30.0 fused_moe/experts/marlin_moe.py:648 MarlinExperts"} - {framework: vllm, kernel_source: vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic, backend: flashinfer_trtllm_gen, source: "vllm/collect_moe.py GLM-5.3-Flash ModelOptNvFp4Config lane (trtllm_nvfp4_moe.py:498-520 @v0.31.0; :486-508 @v0.30.0)"} - {framework: vllm, kernel_source: vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts, backend: marlin, source: "PR#1344 collect_moe.py"} - {framework: vllm, kernel_source: vllm_gptossmxfp4moe_marlin_marlinexperts, backend: marlin, source: "PR#1344 collect_moe.py"} diff --git a/python/aisimulate/collector/network/slurm/collect_trtllm_alltoall.py b/python/aisimulate/collector/network/slurm/collect_trtllm_alltoall.py index 1225fa97c7..1d6b868ab0 100644 --- a/python/aisimulate/collector/network/slurm/collect_trtllm_alltoall.py +++ b/python/aisimulate/collector/network/slurm/collect_trtllm_alltoall.py @@ -679,27 +679,80 @@ class AlltoallBenchmarkResult: combine_low_precision_error: Optional[BaseException] = field(default=None, compare=False) +def _agree_graph_capturable(func, label, device) -> None: + """Capture ``func`` once on every rank and agree on the outcome BEFORE any + rank replays a graph. + + benchmark_with_power captures locally and then replays the graph for its + warmups. These kernels are collectives: had capture failed on one rank + while its peers' capture succeeded, the peers' first replay would wait for + a transfer the failed rank never issues and the job would hang instead of + failing. The probe runs after one eager call (lazy kernel/workspace + initialisation is not capturable) and makes the capture outcome symmetric: + every rank raises, the run loop records the classified failure and all + ranks skip the case together — the agreement the loop already performs for + measured cases. The moe_a2a row has no column that could record an + eager-timed measurement, so the eager fallback used until 2026-10-12 is + gone (layer_permissions.md: measurement-method degradation is allowed only + when recorded in the output row). + """ + import torch + import torch.distributed as dist + + func() + torch.cuda.synchronize() + capture_error: BaseException | None = None + probe = torch.cuda.CUDAGraph() + try: + with torch.cuda.graph(probe): + func() + torch.cuda.synchronize() + except Exception as error: + capture_error = error + finally: + del probe + + failed_here = capture_error is not None + failed_anywhere = failed_here + if dist.is_available() and dist.is_initialized() and dist.get_world_size() > 1: + agreement = torch.tensor([1 if failed_here else 0], dtype=torch.int32, device=device) + dist.all_reduce(agreement, op=dist.ReduceOp.MAX) + failed_anywhere = bool(int(agreement.item())) + if failed_anywhere: + detail = f"{type(capture_error).__name__}: {capture_error}" if failed_here else "capture failed on a peer rank" + raise TrtllmAlltoallBenchmarkError( + f"CUDA graph capture of {label} failed ({detail}); the case fails classified instead of " + "shipping an eager launch-loop floor the moe_a2a row could not label" + ) + + def _benchmark_op(func, label, device, ep_rank, num_warmup, num_iterations): """Benchmark a single operation using shared benchmark_with_power. + CUDA-graph timed only: ``_agree_graph_capturable`` runs first so a capture + failure fails the case on every rank at once (see its docstring). + ``measure_power=False``: the moe_a2a table emits NO power column BY RULING (Task 4 review, see ``collect_moe_a2a._power_columns``): the ruling is per TABLE — HT/LL and trtllm writers share one file and one header, and ``helper.log_perf`` writes the header from the first row — so this writer must not sample or emit power either. """ + _agree_graph_capturable(func, label, device) with benchmark_with_power( device=device, kernel_func=func, num_warmups=num_warmup, num_runs=num_iterations, measure_power=False, - allow_graph_fail=True, ) as results: latency = results["latency_ms"] + if not results.get("used_cuda_graph", False): + raise TrtllmAlltoallBenchmarkError( + f"{label} was not timed under CUDA-graph replay; refusing to record an eager measurement" + ) if ep_rank == 0: - mode = "CUDA Graph" if results["used_cuda_graph"] else "Eager" - print(f" [{label}] {mode} timing: {latency:.3f} ms") + print(f" [{label}] CUDA Graph timing: {latency:.3f} ms") return latency diff --git a/python/aisimulate/collector/op_backend_catalog.yaml b/python/aisimulate/collector/op_backend_catalog.yaml index 055014b31b..ee0ccd3b86 100644 --- a/python/aisimulate/collector/op_backend_catalog.yaml +++ b/python/aisimulate/collector/op_backend_catalog.yaml @@ -118,12 +118,8 @@ families: op_files: - dsa_context_module_perf - dsa_generation_module_perf - - dsa_context_module_skip_indexer_perf - - dsa_generation_module_skip_indexer_perf - dsv4_csa_context_module_perf - dsv4_csa_generation_module_perf - - dsv4_csa_attn_module_perf - - dsv4_hca_attn_module_perf - dsv4_hca_context_module_perf - dsv4_hca_generation_module_perf - dsv4_paged_mqa_logits_module_perf diff --git a/python/aisimulate/collector/op_backend_facts.yaml b/python/aisimulate/collector/op_backend_facts.yaml index 0b15378cee..5d423cd26f 100644 --- a/python/aisimulate/collector/op_backend_facts.yaml +++ b/python/aisimulate/collector/op_backend_facts.yaml @@ -210,14 +210,14 @@ ops: - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill", "vllm_triton_attn"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2"], kernel_sources: ["vllm_flash_attn_fa2"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3"], kernel_sources: ["vllm_flash_attn_fa3"]} - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} @@ -226,14 +226,14 @@ ops: - {framework: "vllm", version: "0.26.0", system: "b60", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flash_attn"], kernel_sources: ["vllm_flash_attn"]} - {framework: "vllm", version: "0.28.0", system: "b60", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["flash_attn"], kernel_sources: ["vllm_flash_attn"]} - {framework: "vllm", version: "0.28.0", system: "b60", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flash_attn"], kernel_sources: ["vllm_flash_attn"]} - - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} - - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill"]} + - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_flashinfer_trtllmprefill", "vllm_triton_attn"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3", "triton"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_triton_attn"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} @@ -241,7 +241,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "flashinfer"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill", "vllm_triton_attn"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "flashinfer"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_fidecode", "vllm_flashinfer_fiprefill"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton", "unverified"], kernel_sources: ["vllm_flashinfer_fiprefill", "vllm_flashinfer_flashinfertrtllmapidecode_xqa", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "context_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_fiprefill", "vllm_flashinfer_flashinfertrtllmapidecode_xqa", "vllm_triton_attn"]} - op_file: context_mla_perf family: mla axes: [op_name, kv_cache_dtype, mla_dtype] @@ -856,54 +856,54 @@ ops: - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_context_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - op_file: dsa_generation_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1366,54 +1366,54 @@ ops: - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "bfloat16", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flash_attn"], kernel_sources: ["FLASH_ATTN_MLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "DeepseekV32ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "bfloat16", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsa_generation_module_skip_indexer", architecture: "GlmMoeDsaForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "nvfp4", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_SM120"]} - op_file: dsv41_module_perf family: dsv41 axes: [] @@ -1478,7 +1478,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_csa_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} - op_file: dsv4_csa_generation_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1537,7 +1537,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_csa_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} - op_file: dsv4_csa_topk_calib_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1554,22 +1554,6 @@ ops: - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "dsv4_csa_topk_calib", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8_e4m3", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["sgl_kernel"], kernel_sources: ["topk_transform_v1", "topk_transform_v2"]} - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "dsv4_csa_topk_calib", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8_e4m3", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["sgl_kernel"], kernel_sources: ["topk_transform_v1", "topk_transform_v2"]} - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "dsv4_csa_topk_calib", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8_e4m3", mla_dtype: "fp8_e4m3", gemm_type: "fp8_block", backends: ["sgl_kernel"], kernel_sources: ["topk_transform_v1", "topk_transform_v2"]} - - op_file: dsv4_hca_attn_module_perf - family: sparse_attention - axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] - facts: - - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.24.0", system: "b300_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.24.0", system: "gb200", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.24.0", system: "gb300", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.24.0", system: "h100_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.24.0", system: "h200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "dsv4_hca_attn_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - op_file: dsv4_hca_context_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1630,7 +1614,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_hca_context_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} - op_file: dsv4_hca_generation_module_perf family: sparse_attention axes: [op_name, architecture, kv_cache_dtype, mla_dtype, gemm_type] @@ -1691,7 +1675,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashmla"], kernel_sources: ["FLASHMLA_SPARSE_DSV4"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["unverified"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "dsv4_hca_generation_module", architecture: "DeepseekV4ForCausalLM", kv_cache_dtype: "fp8", mla_dtype: "bfloat16", gemm_type: "fp8_block", backends: ["flashinfer"], kernel_sources: ["FLASHINFER_MLA_SPARSE_DSV4"]} - op_file: dsv4_megamoe_module_perf family: moe axes: [op_name, phase, moe_dtype, kernel_dtype] @@ -1826,21 +1810,21 @@ ops: - {framework: "sglang", version: "0.5.17", system: "gb300", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - {framework: "sglang", version: "0.5.17", system: "gb300", op_name: "gdn", phase: "generation", backends: ["fla_triton", "flashinfer", "unverified"], kernel_sources: ["causal_conv1d_update", "flashinfer_gated_delta_rule_decode", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "gdn", phase: "context", backends: ["unverified"], kernel_sources: ["causal_conv1d_fn", "flashinfer_chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "gdn", phase: "context", backends: ["unverified"], kernel_sources: ["causal_conv1d_fn", "flashinfer_chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn", "flashinfer_chunk_gated_delta_rule"]} + - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn", "flashinfer_chunk_gated_delta_rule"]} + - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "gdn", phase: "context", backends: ["fla_triton", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "gdn", phase: "generation", backends: ["fla_triton", "unverified"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} + - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["fused_qkvzba_causal_conv1d_update_contiguous", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "trtllm", version: "1.3.0rc20", system: "b200_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - {framework: "trtllm", version: "1.3.0rc20", system: "b200_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule"]} - {framework: "trtllm", version: "1.3.0rc20", system: "b300_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} @@ -1857,22 +1841,22 @@ ops: - {framework: "trtllm", version: "1.3.0rc20", system: "l40s", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule"]} - {framework: "trtllm", version: "1.3.0rc20", system: "rtx_pro_6000_server", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - {framework: "trtllm", version: "1.3.0rc20", system: "rtx_pro_6000_server", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_flashinfer"]} - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_triton"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_triton"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_triton"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "gdn", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "gdn", phase: "generation", backends: ["causal_conv1d", "fla_triton"], kernel_sources: ["causal_conv1d_update", "fused_sigmoid_gating_delta_rule_update_triton"]} - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} - {framework: "vllm", version: "0.24.0", system: "b200_sxm", op_name: "gdn", phase: "generation", backends: ["fla_triton", "triton"], kernel_sources: ["causal_conv1d_update", "fused_recurrent_gated_delta_rule_packed_decode"]} - {framework: "vllm", version: "0.24.0", system: "b300_sxm", op_name: "gdn", phase: "context", backends: ["flashinfer", "triton"], kernel_sources: ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"]} @@ -2176,7 +2160,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm", "flashinfer"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel", "FlashInferFp8BlockScaledMMKernel"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["unverified"], kernel_sources: ["MarlinFP8ScaledMMLinearKernel"]} + - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["marlin"], kernel_sources: ["MarlinFP8ScaledMMLinearKernel"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "gemm", gemm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["torch.nn.functional.linear"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "gemm", gemm_dtype: "fp8", backends: ["cutlass"], kernel_sources: ["CutlassFP8ScaledMMLinearKernel"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "gemm", gemm_dtype: "fp8_block", backends: ["cutlass", "deepgemm"], kernel_sources: ["CutlassFp8BlockScaledMMKernel", "DeepGemmFp8BlockScaledMMKernel"]} @@ -2288,14 +2272,14 @@ ops: - {framework: "vllm", version: "0.24.0", system: "l40s", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_triton_attn"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2"], kernel_sources: ["vllm_flash_attn_fa2"]} - {framework: "vllm", version: "0.24.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} - - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} - - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} - - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} - - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode"]} + - {framework: "vllm", version: "0.25.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode", "vllm_triton_attn"]} - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} - {framework: "vllm", version: "0.25.0", system: "h100_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3"], kernel_sources: ["vllm_flash_attn_fa3"]} - {framework: "vllm", version: "0.25.0", system: "h200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} @@ -2304,14 +2288,14 @@ ops: - {framework: "vllm", version: "0.26.0", system: "b60", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flash_attn"], kernel_sources: ["vllm_flash_attn"]} - {framework: "vllm", version: "0.28.0", system: "b60", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["flash_attn"], kernel_sources: ["vllm_flash_attn"]} - {framework: "vllm", version: "0.28.0", system: "b60", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flash_attn"], kernel_sources: ["vllm_flash_attn"]} - - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} - - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} - - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} - - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} - - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "unverified"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} - - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} + - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} + - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} + - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "trtllm_mha"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen"]} + - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", "vllm_triton_attn"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["fa3", "triton"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_triton_attn"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa3", "fa4"], kernel_sources: ["vllm_flash_attn_fa3", "vllm_flash_attn_fa4"]} @@ -2319,7 +2303,7 @@ ops: - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "flashinfer"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_fidecode"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["flashinfer", "triton"], kernel_sources: ["vllm_flashinfer_fidecode", "vllm_triton_attn"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "bfloat16", backends: ["fa2", "flashinfer"], kernel_sources: ["vllm_flash_attn_fa2", "vllm_flashinfer_fidecode"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "unverified"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_xqa", "vllm_triton_attn"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "generation_attention", attn_dtype: "bfloat16", kv_cache_dtype: "fp8", backends: ["triton", "trtllm_mha"], kernel_sources: ["vllm_flashinfer_flashinfertrtllmapidecode_xqa", "vllm_triton_attn"]} - op_file: generation_mla_perf family: mla axes: [op_name, kv_cache_dtype, mla_dtype] @@ -2541,22 +2525,22 @@ ops: - {framework: "trtllm", version: "1.3.0rc20", system: "l40s", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d"], kernel_sources: ["causal_conv1d_update"]} - {framework: "trtllm", version: "1.3.0rc20", system: "rtx_pro_6000_server", op_name: "mamba2", phase: "context", backends: ["causal_conv1d"], kernel_sources: ["causal_conv1d_fn"]} - {framework: "trtllm", version: "1.3.0rc20", system: "rtx_pro_6000_server", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d"], kernel_sources: ["causal_conv1d_update"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "mamba2", phase: "context", backends: ["flashinfer", "unverified"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} - - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "unverified"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b200_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "b300_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb200", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "gb300", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h100_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "h200_sxm", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "l40s", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "mamba2", phase: "context", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_fn_token_major", "mamba_chunk_scan_combined"]} + - {framework: "trtllm", version: "1.3.0rc29", system: "rtx_pro_6000_server", op_name: "mamba2", phase: "generation", backends: ["causal_conv1d", "flashinfer"], kernel_sources: ["causal_conv1d_update", "selective_state_update_flashinfer"]} - op_file: mhc_module_perf family: mhc axes: [op_name, architecture] @@ -2746,37 +2730,37 @@ ops: - {framework: "sglang", version: "0.5.14", system: "rtx_pro_6000_server", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - {framework: "sglang", version: "0.5.14", system: "rtx_pro_6000_server", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_sgl_kernel_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "b200_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "b300_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "gb200", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "gb300", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "h100_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "h200_sxm", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "l40s", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "mla_gen_post", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["sglang_torch_bmm"]} - - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["unverified"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} + - {framework: "sglang", version: "0.5.21", system: "rtx_pro_6000_server", op_name: "mla_gen_pre", bmm_dtype: "fp8", backends: ["cublas"], kernel_sources: ["sglang_flashinfer_bmm_fp8"]} - {framework: "trtllm", version: "1.0.0", system: "a100_sxm", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["default"]} - {framework: "trtllm", version: "1.0.0", system: "a100_sxm", op_name: "mla_gen_pre", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["default"]} - {framework: "trtllm", version: "1.0.0", system: "l40s", op_name: "mla_gen_post", bmm_dtype: "bfloat16", backends: ["cublas"], kernel_sources: ["default"]} @@ -3693,50 +3677,50 @@ ops: - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_trtllm_gen", "unverified"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutedsl", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "b200_sxm", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_trtllm_gen", "unverified"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutedsl", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "b300_sxm", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_trtllm_gen", "unverified"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutedsl", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "gb200", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts", "vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular", "vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_trtllm_gen", "unverified"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} + - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutedsl", "flashinfer_trtllm_gen"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular", "vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["flashinfer_trtllm_gen", "marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "gb300", op_name: "moe", moe_dtype: "w4a8_mxfp4_mxfp8", backends: ["flashinfer_trtllm_gen"], kernel_sources: ["vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_cutlass", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_cutlass_flashinferexperts", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - {framework: "vllm", version: "0.30.0", system: "h100_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["marlin", "triton_fused_moe"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "moe", moe_dtype: "bfloat16", backends: ["triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_cutlass", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_cutlass_flashinferexperts", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - {framework: "vllm", version: "0.30.0", system: "h200_sxm", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["marlin", "triton_fused_moe"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "moe", moe_dtype: "bfloat16", backends: ["triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "moe", moe_dtype: "fp8", backends: ["triton_fused_moe"], kernel_sources: ["vllm_modeloptfp8moe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "moe", moe_dtype: "fp8_block", backends: ["triton_fused_moe"], kernel_sources: ["vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - {framework: "vllm", version: "0.30.0", system: "l40s", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_marlin_marlinexperts"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "bfloat16", backends: ["flashinfer_cutlass", "triton_fused_moe"], kernel_sources: ["vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts", "vllm_unquantizedfusedmoe_triton_tritonexperts"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "fp8", backends: ["flashinfer_cutlass"], kernel_sources: ["vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "fp8_block", backends: ["triton_fused_moe", "unverified"], kernel_sources: ["vllm_fp8moe_deepgemm_deepgemmexperts", "vllm_fp8moe_triton_tritonexperts"]} - - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "int4_wo", backends: ["unverified"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "fp8_block", backends: ["deepgemm", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_deepgemm_deepgemmexperts", "vllm_fp8moe_triton_tritonexperts"]} + - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "int4_wo", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorswna16moe_marlin_marlinexperts"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "nvfp4", backends: ["flashinfer_cutlass"], kernel_sources: ["vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts"]} - {framework: "vllm", version: "0.30.0", system: "rtx_pro_6000_server", op_name: "moe", moe_dtype: "w4a16_mxfp4", backends: ["marlin"], kernel_sources: ["vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu", "vllm_gptossmxfp4moe_marlin_marlinexperts"]} - {framework: "vllm", version: "0.31.0", system: "b300_sxm", op_name: "moe", moe_dtype: "fp8_block", backends: ["flashinfer_trtllm_gen", "triton_fused_moe"], kernel_sources: ["vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic", "vllm_fp8moe_triton_tritonexperts"]} diff --git a/python/aisimulate/collector/registry_types.py b/python/aisimulate/collector/registry_types.py index bdeace7149..99d57cd2f3 100644 --- a/python/aisimulate/collector/registry_types.py +++ b/python/aisimulate/collector/registry_types.py @@ -7,6 +7,7 @@ from dataclasses import dataclass, field from enum import Enum +from typing import Any class PerfFile(str, Enum): @@ -35,6 +36,12 @@ def __str__(self) -> str: KDA = "kda_perf.txt" MAMBA2 = "mamba2_perf.txt" COMPUTESCALE = "computescale_perf.txt" + # computescale's second table (dynamic-vs-static fp8 quant matrix): both + # backends have always written it and the SDK consumes + # scale_matrix_perf.parquet (aisimulate_core/sdk/common.py) — the enum + # entry was simply missing, which the fail-closed finalize rejected on + # the first full vllm computescale run (2026-09-21). + SCALE_MATRIX = "scale_matrix_perf.txt" WIDEEP_CONTEXT_MLA = "wideep_context_mla_perf.txt" WIDEEP_GENERATION_MLA = "wideep_generation_mla_perf.txt" WIDEEP_CONTEXT_MOE = "wideep_context_moe_perf.txt" @@ -51,10 +58,11 @@ def __str__(self) -> str: DSA_GENERATION_MODULE = "dsa_generation_module_perf.txt" # GLM-5.2 shares one topk index across `index_topk_freq` layers: only 1 # layer per group computes the indexer (mqa+topk+index-K store), the rest - # reuse it (skip_indexer). These names are worker selectors; skip rows - # share the canonical DSA phase tables above, distinguished by op_name. - DSA_CONTEXT_MODULE_SKIP_INDEXER = "dsa_context_module_skip_indexer_perf.txt" - DSA_GENERATION_MODULE_SKIP_INDEXER = "dsa_generation_module_skip_indexer_perf.txt" + # reuse it (skip_indexer). The skip-layer rows live in the SAME two DSA + # tables above, tagged by an op_name "_skip_indexer" suffix (the SDK reads + # them from there: sdk/database test_dsa_skip_indexer_fail_open), so the + # skip ops declare DSA_*_MODULE and select the variant through + # OpEntry.run_kwargs. per_layer = (1/freq)*full + (1-1/freq)*skip. # MiniMax-M3 MSA (block-sparse GQA) full-module data — same row schema as # the DSA module files, keyed by architecture. MSA_CONTEXT_MODULE = "msa_context_module_perf.txt" @@ -67,11 +75,12 @@ def __str__(self) -> str: DSV4_HCA_CONTEXT_MODULE = "dsv4_hca_context_module_perf.txt" DSV4_CSA_GENERATION_MODULE = "dsv4_csa_generation_module_perf.txt" DSV4_HCA_GENERATION_MODULE = "dsv4_hca_generation_module_perf.txt" - # DeepSeek-V4 sparse-kernel data — bench-collected (paged_mqa_logits + - # hca_attn + csa_attn), each 1:1 with its owning CSA/HCA module rows. + # DeepSeek-V4 sparse-kernel data — the CSA indexer scoring sub-kernel, + # bench-collected 1:1 with its owning CSA module rows. The hca_attn / + # csa_attn FMLA sidecars were removed end to end on 2026-10-15: their SDK + # readers had been retired (no query path; csa_attn never shipped data) + # and the registry-only ops failed every shard that named them. DSV4_PAGED_MQA_LOGITS_MODULE = "dsv4_paged_mqa_logits_module_perf.txt" - DSV4_HCA_ATTN_MODULE = "dsv4_hca_attn_module_perf.txt" - DSV4_CSA_ATTN_MODULE = "dsv4_csa_attn_module_perf.txt" # DeepSeek-V4 CSA topk_512 degenerate-vs-representative DELTA calibration. # SGLang 0.5.14 rows qualify flat/top_last by the executed v1/v2 variant. DSV4_CSA_TOPK_CALIB = "dsv4_csa_topk_calib_perf.txt" @@ -104,9 +113,10 @@ class OpEntry: Exactly one of ``module`` (unversioned) or ``versions`` (versioned) must be provided. This invariant is validated at construction time. - ``perf_filename`` is the physical output table. ``worker_perf_filename`` - optionally overrides only the basename passed to the worker's legacy - ``perf_filename`` dispatch argument, preserving the output directory. + ``perf_filename`` is the physical output table. Ops that share one + run_func and one table in different modes select the mode with + ``run_kwargs`` (sglang dsa_*_module_skip_indexer); multi-table producers + name their extra tables in ``extra_perf_filenames``. """ op: str @@ -123,10 +133,46 @@ class OpEntry: # unverified_sms=(120,) — debugged elsewhere, not validated on these SMs unverified: bool = False unverified_sms: tuple[int, ...] = () - worker_perf_filename: str | None = None + # Further tables the SAME run_func writes from one measurement (passed to + # it as ``extra_perf_filenames``); finalize binds every one of them to + # this producer's checkpoint. compute_scale is the only user so far + # (computescale_perf + scale_matrix_perf; owner decision 2026-09-28). + extra_perf_filenames: tuple[str, ...] = () + # Fixed keyword arguments the executor binds to run_func (functools.partial, + # next to perf_filename / extra_perf_filenames). This is how one run_func + # serves several registry ops that write the SAME table in different modes: + # sglang dsa_*_module_skip_indexer passes skip_indexer=True and shares the + # dsa_*_module table with the full-module op. Before this field the worker + # inferred the mode from a "skip_indexer" perf_filename that no collector + # ever wrote, and --resume finalization (which trusts the declared table) + # raised "open checkpoint event has no regular staging table" on every + # skip_indexer shard (b200_sxm sglang 0.5.21, GitLab job 469988017, + # 2026-10-05). A tuple of (name, value) pairs keeps the entry hashable. + run_kwargs: tuple[tuple[str, Any], ...] = () def __post_init__(self) -> None: if not self.module and not self.versions: raise ValueError(f"OpEntry '{self.op}': must specify 'module' or 'versions'") if self.module and self.versions: raise ValueError(f"OpEntry '{self.op}': cannot specify both 'module' and 'versions'") + names = [str(name) for name in self.perf_filenames] + if len(names) != len(set(names)): + raise ValueError(f"OpEntry '{self.op}': duplicate perf tables {names}") + keys = [pair[0] for pair in self.run_kwargs] + if any(not isinstance(key, str) or not key for key in keys) or len(keys) != len(set(keys)): + raise ValueError(f"OpEntry '{self.op}': run_kwargs must be unique non-empty names, got {keys}") + reserved = {"perf_filename", "extra_perf_filenames", "device"} + if reserved & set(keys): + raise ValueError( + f"OpEntry '{self.op}': run_kwargs may not rebind executor-owned {sorted(reserved & set(keys))}" + ) + + @property + def run_kwargs_dict(self) -> dict[str, Any]: + """The fixed keyword arguments as a dict (what the executor binds).""" + return dict(self.run_kwargs) + + @property + def perf_filenames(self) -> tuple[str, ...]: + """Every table this producer writes, primary first.""" + return (self.perf_filename, *self.extra_perf_filenames) diff --git a/python/aisimulate/collector/sglang/collect_attn.py b/python/aisimulate/collector/sglang/collect_attn.py index bab7c862ad..7273ed1abb 100644 --- a/python/aisimulate/collector/sglang/collect_attn.py +++ b/python/aisimulate/collector/sglang/collect_attn.py @@ -9,14 +9,16 @@ KV-cache setup, backend dispatch, and perf logging for the SGLang runtime. """ -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" import math import os +from importlib.metadata import version as _dist_version # setuptools/pkg_resources is absent from sglang>=0.5.21 images from types import SimpleNamespace from typing import NamedTuple -import pkg_resources import torch from collector.case_generator import ( get_attention_context_shape_sweeps, @@ -41,6 +43,9 @@ class Timing(NamedTuple): # Mock objects to satisfy RadixAttention dependencies class MockModelConfig: + def get_max_num_attention_heads(self) -> int: # model_config.py:1504 @0.5.21 (triton_backend.py:255) + return int(self.num_attention_heads) + def __init__( self, num_attention_heads, @@ -68,6 +73,11 @@ def __init__( self.is_multimodal = False self.hidden_size = num_attention_heads * head_dim self.is_local_attention_model = attention_chunk_size is not None + # sglang 0.5.16: configs/hybrid_arch.py:30-31 reads + # model_config.linear_attn_registry_result (ModelConfig property, + # model_config.py:707) to detect hybrid linear-attention archs; the + # dense-attention mock has none. + self.linear_attn_registry_result = None class MockHFConfig: def __init__(self, *, num_attention_heads, num_key_value_heads, head_dim, v_head_dim, architecture): @@ -81,6 +91,13 @@ def __init__(self, *, num_attention_heads, num_key_value_heads, head_dim, v_head self.hidden_size = num_attention_heads * head_dim self.attn_logit_softcapping = None + def get_text_config(self): + # HF PretrainedConfig API: sglang 0.5.16 unwraps multimodal + # configs through it (configs/hybrid_arch.py:49, + # linear_attn_model_registry.py:56); a text-only mock is its own + # text config. + return self + self.hf_config = MockHFConfig( num_attention_heads=num_attention_heads, num_key_value_heads=num_key_value_heads, @@ -91,7 +108,7 @@ def __init__(self, *, num_attention_heads, num_key_value_heads, head_dim, v_head self.hf_text_config = self.hf_config self.dtype = torch.bfloat16 - def get_num_kv_heads(self, tp_size): + def get_num_kv_heads(self, tp_size, dcp_size=1): # model_config.py:1512 @0.5.21 adds dcp_size (DCP replicates KV) return self.num_key_value_heads // tp_size @@ -121,9 +138,15 @@ def __init__(self, page_size: int): self.enable_two_batch_overlap = False self.disable_attn_tp_gather = False self.moe_dense_tp_size = None + # sglang 0.5.16: flashattention_backend.py:51 reads + # server_args.enable_prefill_cp (prefill context parallelism) — off + self.enable_prefill_cp = False class MockModelRunner: + def decode_num_tokens_per_req(self, *, num_draft_tokens=None) -> int: + return 1 + def __init__( self, device, @@ -136,6 +159,7 @@ def __init__( architecture=None, runtime_window_size=-1, attention_chunk_size=None, + attn_backend_name=None, ): self.device = device self.req_to_token_pool = None @@ -143,7 +167,56 @@ def __init__( self.attn_backend = None self.server_args = MockServerArgs(page_size=page_size) self.attn_cp_size = 1 # Context parallelism size; required by FlashAttentionBackend in sglang >=0.5.10 + # sglang 0.5.21 TRTLLMHAAttnBackend (the dense backend sglang picks on SM100/103) reads + # model_runner.prefill_attention_backend_str / decode_attention_backend_str to decide its + # prefill/decode split; fa3/flashinfer on Hopper never asked (B200 smoke 2026-10-04: + # AttributeError 'MockModelRunner' object has no attribute 'prefill_attention_backend_str'). + self.prefill_attention_backend_str = attn_backend_name + self.decode_attention_backend_str = attn_backend_name + # trtllm_mha_backend.py:310@0.5.21 scans model_runner.model.modules() for ENCODER_ONLY attention + # layers (expanded TARGET_VERIFY metadata); the collector has no model — an empty Module + # declares none, which is the dense-decoder answer. + self.model = torch.nn.Module() + # TRTLLMHAAttnBackend also sizes its workspaces from model_runner.max_running_requests + # (512 = the generator's --max-running-requests default). + self.max_running_requests = 512 + # sglang 0.5.16 reads the parallel geometry from ``model_runner.ps`` + # (flashattention_backend.py:183 attn_cp_size, :271-274 tp_size; the + # runner sets it at model_runner.py:262 from a ParallelState). Provide + # the real trivial ParallelState when the wrapper exists, else a + # namespace with the same fields (parallel_state_wrapper.py:6-24). + try: + from sglang.srt.distributed.parallel_state_wrapper import ParallelState + + self.ps = ParallelState.trivial(gpu_id=0) + except ImportError: + from types import SimpleNamespace + + self.ps = SimpleNamespace( + tp_rank=0, + tp_size=1, + pp_rank=0, + pp_size=1, + dp_rank=0, + dp_size=1, + attn_tp_rank=0, + attn_tp_size=1, + attn_cp_rank=0, + attn_cp_size=1, + attn_dp_rank=0, + attn_dp_size=1, + moe_ep_rank=0, + moe_ep_size=1, + moe_dp_rank=0, + moe_dp_size=1, + dcp_size=1, + gpu_id=0, + ) self.is_draft_worker = False + # sglang>=0.5.21 backends ask the runner for logits rows per decode slot + # (model_runner.py:796 decode_num_tokens_per_req; triton_backend.py:218,222); + # the kernel collectors never run speculative decoding -> 1 + self.spec_algorithm = None self.model_is_mrope = False self.sliding_window_size = attention_chunk_size if self.sliding_window_size is None and runtime_window_size >= 0: @@ -161,6 +234,11 @@ def __init__( attention_chunk_size, ) self.kv_cache_dtype = kv_cache_dtype # Default + # sglang>=0.5.21 FlashAttentionBackend reads the server-args string next to the + # torch dtype (flashattention_backend.py:192-194 kv_cache_dtype_str, drives the + # mxfp8 branch); serving sets it from server_args.kv_cache_dtype + # (model_runner.py:1481). The collector's kv_cache_dtype is that same string. + self.kv_cache_dtype_str = kv_cache_dtype self.page_size = page_size self.tp_size = 1 self.is_hybrid = False @@ -236,11 +314,22 @@ def get_context_attention_test_cases(): continue if b * s * num_kv_heads * head_dim * 2 >= max_kv_elements: continue + # the backend this head config runs on (per-model map, else the SM default table) — + # the same resolution run_attention_torch makes before refusing a combo + lane = head_config.kernel_source or default_attention_backend( + sm_version, head_config.has_attention_sink + ) for precision_case in shape_sweep["precision_cases"]: use_fp8_kv_cache = bool(precision_case["fp8_kv_cache"]) use_fp8_context_fmha = bool(precision_case["fp8_context_fmha"]) if skip_fp8 and use_fp8_kv_cache: continue + if use_fp8_context_fmha and lane == "flashinfer": + # FlashInfer has no FP8 prefill compute path (BF16 Q reads the FP8 cache through + # descales), so run_attention_torch refuses this case by design; scheduling it on + # the flashinfer SMs (sm89 / sm120) only made a third of the op's tasks fail + # (sm89 sample 2026-10-04: 13 of 40). Not an SM gate: the fp8_kv_cache case stays. + continue test_cases.append( [ b, @@ -362,6 +451,72 @@ def get_generation_attention_test_cases(): attn_dp_size=1, attn_dp_rank=0, ) +def default_attention_backend(sm_version: int, has_attention_sink: bool) -> str | None: + """SGLang's default dense-attention backend per SM (the table run_attention_torch documents + above its call site); None for SMs outside the supported set.""" + return { + 89: "triton" if has_attention_sink else "flashinfer", + 90: "fa3", + 100: "trtllm_mha", + 103: "trtllm_mha", + 120: "triton" if has_attention_sink else "flashinfer", + }.get(sm_version) + + +_FLASHINFER_WORKSPACE_MIN_BYTES = 4 << 30 + + +def _ensure_flashinfer_workspace() -> int | None: + """Raise SGLang's FlashInfer workspace floor BEFORE the first backend of this + process is built, through the knob the pinned version actually reads. + + 0.5.21 sizes the shared decode/prefill plan buffer from + ``sglang.srt.environ.envs.SGLANG_FLASHINFER_WORKSPACE_SIZE`` (EnvInt, default + 384 MiB) and allocates it ONCE per process via ``get_buffer("flashinfer_workspace")`` + (flashinfer_backend.py:430-436); serving itself bumps the knob to 512 MiB for + Qwen2/Qwen3/MiMo architectures (:394-404). The decode sweep's wide cells need + more than the default — l40s 0.5.21: "Buffer overflow when allocating + batch_prefill_tmp_v with size 536870912 ... only 402653184 bytes available", + 671 attention_generation cases (heads 128 / kv 1 / head_dim 128, batch 32 x isl + 255..4095). ed9109ec wrote ``global_config.flashinfer_workspace_size``, an + attribute 0.5.21 no longer has — a no-op (pipeline 72343559 reproduced all 671). + Older series keep the global_config path as a fallback. Returns the floor + applied (bytes) or None when no knob was found. + + Floor history: 1 GiB (5c0f54a5) cleared 447 of the 671 l40s failures (pipeline + 72369799, job 475655072); the remaining 224 — all heads 128 / kv 1 / head_dim 128, + bf16 and fp8 KV alike, batch x isl from 32x255 up to 4x16383 — asked the tensor-core + decode planner for batch_prefill_tmp_v of 1.5-2.22 GiB ("only 1073741824 bytes + available"). 4 GiB covers the largest observed request with margin; the buffer is one + torch.empty per worker process, never touched beyond what the plan uses, and the + planner's split decision does not depend on the buffer size, so the measured kernels + are unchanged (flashinfer prefill planner sizes tmp_v from the split count, not the + workspace). + """ + try: + from sglang.srt.environ import envs + + knob = getattr(envs, "SGLANG_FLASHINFER_WORKSPACE_SIZE", None) + if knob is not None: + current = int(knob.get()) + if current < _FLASHINFER_WORKSPACE_MIN_BYTES: + knob.set(_FLASHINFER_WORKSPACE_MIN_BYTES) + return max(current, _FLASHINFER_WORKSPACE_MIN_BYTES) + except Exception: + pass + try: + from sglang.global_config import global_config + + current = getattr(global_config, "flashinfer_workspace_size", None) + if isinstance(current, int): + if current < _FLASHINFER_WORKSPACE_MIN_BYTES: + global_config.flashinfer_workspace_size = _FLASHINFER_WORKSPACE_MIN_BYTES + return max(current, _FLASHINFER_WORKSPACE_MIN_BYTES) + except Exception: + pass + return None + + def run_attention_torch( batch_size, input_len, @@ -384,6 +539,10 @@ def run_attention_torch( device="cuda:0", page_size: int | None = None, ): + from collector.sglang.runtime_compat import ensure_offline_runtime_published + + ensure_offline_runtime_published() # sglang>=0.5.20 backends read get_exec()/get_parallel() + _ensure_flashinfer_workspace() if use_fp8_context_fmha: assert use_fp8_kv_cache, "If you want to use fp8 context fmha, kv cache must be fp8" kvtype = torch.float8_e4m3fn if use_fp8_kv_cache else torch.bfloat16 @@ -415,13 +574,7 @@ def run_attention_torch( # sglang_backends map as an explicit attn_backend_name, never through # this default table. SM80/86 are outside the # supported platform set {89, 90, 100, 103, 120} and fail closed below. - attn_backend_name = { - 89: "triton" if has_attention_sink else "flashinfer", - 90: "fa3", - 100: "trtllm_mha", - 103: "trtllm_mha", - 120: "triton" if has_attention_sink else "flashinfer", - }.get(sm_version) + attn_backend_name = default_attention_backend(sm_version, has_attention_sink) if attn_backend_name is None: raise ValueError(f"No SGLang 0.5.14 attention backend mapping for SM{sm_version}") if page_size is None: @@ -438,8 +591,15 @@ def run_attention_torch( architecture=architecture, runtime_window_size=runtime_window_size, attention_chunk_size=attention_chunk_size, + attn_backend_name=attn_backend_name, ) model_runner.kv_cache_dtype = kvtype + # sglang>=0.5.21 backends ask the runner which backend serves each phase + # (model_runner.py:1020-1021 resolved.prefill/decode; flashinfer_backend.py:331-332 + # gate the kv-access checks on it). One lane = one backend for both phases. Hopper's + # fa3 / Blackwell's trtllm_mha never read it, so only the sm89/sm120 flashinfer lane failed. + model_runner.prefill_attention_backend_str = attn_backend_name + model_runner.decode_attention_backend_str = attn_backend_name total_len = input_len if is_context_phase else input_len + 1 # TRTLLM MHA sizes its page table from context_len. @@ -474,10 +634,14 @@ def run_attention_torch( page_size=model_runner.page_size, get_kvcache=lambda: model_runner.token_to_kv_pool, ) + from collector.sglang.runtime_compat import attach_kv_index_translator + + attach_kv_index_translator(model_runner) if attn_backend_name == "flashinfer": from sglang.srt.layers.attention.flashinfer_backend import FlashInferAttnBackend + _ensure_flashinfer_workspace() # the shared workspace is allocated by the first backend of the process attn_backend = FlashInferAttnBackend(model_runner) elif attn_backend_name == "trtllm_mha": from sglang.srt.layers.attention.trtllm_mha_backend import TRTLLMHAAttnBackend @@ -640,7 +804,10 @@ def run_attention_torch( if is_context_phase: if use_fp8_context_fmha: if attn_backend_name == "flashinfer": - raise ValueError("SGLang 0.5.14 flashinfer has no FP8 prefill compute path") + raise ValueError( + "FIXME(kernel-limit): SGLang flashinfer has no FP8 prefill compute path " + "(plan-time skip missed this case)" + ) if attn_backend_name != "trtllm_mha": q = q.to(kvtype) k = k.to(kvtype) @@ -649,7 +816,7 @@ def run_attention_torch( attn_backend_name == "trtllm_mha" or (attn_backend_name == "fa3" and head_dim <= 256) ): raise ValueError( - f"SGLang 0.5.14 {attn_backend_name} quantizes Q to FP8 internally when the KV cache " + f"FIXME(kernel-limit): SGLang {attn_backend_name} quantizes Q to FP8 internally when the KV cache " "is FP8, so a BF16-compute prefill on an FP8 KV cache does not exist; this " "combination is the fp8_context_fmha case" ) @@ -707,7 +874,7 @@ def run_iter(): } ], framework="SGLang", - version=pkg_resources.get_distribution("sglang").version, + version=_dist_version("sglang"), device_name=torch.cuda.get_device_name(device), op_name=op_name, kernel_source=attn_backend_name, diff --git a/python/aisimulate/collector/sglang/collect_attn_encoder.py b/python/aisimulate/collector/sglang/collect_attn_encoder.py index 81be051301..b695753698 100644 --- a/python/aisimulate/collector/sglang/collect_attn_encoder.py +++ b/python/aisimulate/collector/sglang/collect_attn_encoder.py @@ -5,23 +5,42 @@ SM dispatch mirrors ``VisionAttention._determine_attention_backend``: -- CUDA SM == 90 (Hopper) -> ``flash_attn_varlen_func`` (FA3) -- CUDA SM == 100 (Blackwell) -> ``flash_attn_varlen_func(ver=4)`` (FA4) -- other CUDA (SM<90, SM120) -> ``context_attention_fwd`` (Triton) +- capability major 9 (SM90, Hopper) -> ``flash_attn_varlen_func`` (FA3) +- capability major 10 (SM100/SM103, B200/B300) -> ``flash_attn_varlen_func(ver=4)`` (FA4) +- other CUDA (SM<90, SM120) -> ``context_attention_fwd`` (Triton) + +Serving branches on the capability MAJOR (layers/attention/vision.py:1298-1303 @0.5.21: +``major == 9 -> fa3``, ``major == 10 -> fa4``, else ``triton_attn``), so SM103 shares the FA4 +lane with SM100. The collector's earlier ``sm == 100`` test sent SM103 down the Triton path +(B300 encoder gate failed, 2026-10-05) — a wrong-kernel measurement, not a crash. Quant: bf16 only. SGLang upstream does not support fp8 ViT FMHA. """ -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +from importlib.metadata import ( + version as _dist_version, # setuptools/pkg_resources is absent from recent framework images +) from typing import NamedTuple -import pkg_resources import torch from collector.case_generator import get_attention_encoder_head_configs, get_attention_encoder_shape_sweeps from collector.helper import benchmark_with_power, get_sm_version, log_perf +def serving_vision_backend(sm_version: int) -> str: + """The vision-attention backend sglang selects for a CUDA capability (vision.py:1298-1303 @0.5.21).""" + major = sm_version // 10 + if major == 9: + return "fa3" + if major == 10: + return "fa4" + return "triton_attn" + + class Timing(NamedTuple): mean: float @@ -80,11 +99,15 @@ def _build_kernel_runner( max_seqlen = seq_len softmax_scale = head_dim**-0.5 - sm = get_sm_version() # 90 / 100 / 120 / ... + sm = get_sm_version() # 90 / 100 / 103 / 120 / ... - if sm == 90: + backend = serving_vision_backend(sm) + if backend == "fa3": # Matches VisionFlash3Attention.forward. - from sglang.jit_kernel.flash_attention import flash_attn_varlen_func + try: # sglang>=0.5.21 (layers/attention/vision.py:55) + from sglang.kernels.ops.attention.flash_attention import flash_attn_varlen_func + except ImportError: + from sglang.jit_kernel.flash_attention import flash_attn_varlen_func def run_iter(): flash_attn_varlen_func( @@ -101,9 +124,12 @@ def run_iter(): return run_iter, "flash_attention_v3" - if sm == 100: + if backend == "fa4": # Matches VisionFlash4Attention.forward. - from sglang.jit_kernel.flash_attention import flash_attn_varlen_func + try: # sglang>=0.5.21 (layers/attention/vision.py:55) + from sglang.kernels.ops.attention.flash_attention import flash_attn_varlen_func + except ImportError: + from sglang.jit_kernel.flash_attention import flash_attn_varlen_func def run_iter(): flash_attn_varlen_func( @@ -120,10 +146,13 @@ def run_iter(): return run_iter, "flash_attention_v4" - # SM<90 or SM>100: Triton path matching VisionTritonAttention.forward. - from sglang.srt.layers.attention.triton_ops.prefill_attention import ( - context_attention_fwd, - ) + # every other capability (SM<90, SM120): Triton path matching VisionTritonAttention.forward. + try: # sglang>=0.5.21 (layers/attention/vision.py imports it from kernels/ops/attention) + from sglang.kernels.ops.attention.prefill_attention import context_attention_fwd + except ImportError: + from sglang.srt.layers.attention.triton_ops.prefill_attention import ( + context_attention_fwd, + ) seq_lens = torch.full( (batch_size,), @@ -158,6 +187,9 @@ def run_encoder_attention_torch( perf_filename, device="cuda:0", ): + from collector.sglang.runtime_compat import ensure_offline_runtime_published + + ensure_offline_runtime_published() # sglang>=0.5.20 backends read get_exec()/get_parallel() torch_device = torch.device(device) torch.cuda.set_device(device) @@ -193,7 +225,7 @@ def run_encoder_attention_torch( } ], framework="SGLang", - version=pkg_resources.get_distribution("sglang").version, + version=_dist_version("sglang"), device_name=torch.cuda.get_device_name(device), op_name="encoder_attention", kernel_source=backend_tag, diff --git a/python/aisimulate/collector/sglang/collect_computescale.py b/python/aisimulate/collector/sglang/collect_computescale.py index 81a1c1dbce..a9caa1599f 100644 --- a/python/aisimulate/collector/sglang/collect_computescale.py +++ b/python/aisimulate/collector/sglang/collect_computescale.py @@ -1,16 +1,28 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Measure SGLang FP8 activation quantization overhead for static-FP8 GEMM.""" +"""Measure SGLang FP8 activation quantization overhead for static-FP8 GEMM. -__compat__ = "sglang==0.5.14" +One measurement, two tables — the registry entry declares the second one as +``extra_perf_filenames`` and the executor hands both names in, so finalize +binds both to this producer's checkpoint: + perf_filename -> computescale_perf.txt (dynamic per-token quant minus static quant) + extra_perf_filenames[0] -> scale_matrix_perf.txt (the static per-tensor quant alone) +""" + +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" + +from importlib.metadata import version as _dist_version # setuptools/pkg_resources is absent from sglang>=0.5.21 images -import pkg_resources import torch from collector.case_generator import get_compute_scale_case_specs from collector.helper import benchmark_with_power, get_sm_version, log_perf from sgl_kernel import sgl_per_token_quant_fp8 +_OUTSIDE_LOOP_COUNT = 5 + def get_computescale_test_cases(): if get_sm_version() <= 86: @@ -24,37 +36,48 @@ def _static_quantize_e4m3_per_tensor(x: torch.Tensor, scale: torch.Tensor, out: return out -def run_computescale(m, k, *, perf_filename, device="cuda:0"): +def _setup(m, k, device): device = torch.device(device) torch.cuda.set_device(device) torch.set_default_device(device) - x = torch.randn((m, k), dtype=torch.bfloat16, device=device) + return device, x + + +def _bench_dynamic(device, x): + m = x.shape[0] dynamic_out = torch.empty_like(x, dtype=torch.float8_e4m3fn) dynamic_scale = torch.empty((m, 1), dtype=torch.float32, device=device) - static_out = torch.empty_like(x, dtype=torch.float8_e4m3fn) - static_scale = torch.tensor([1.0], dtype=torch.float32, device=device) - outside_loop_count = 5 - def dynamic_kernel_func(): - for _ in range(outside_loop_count): + def kernel_func(): + for _ in range(_OUTSIDE_LOOP_COUNT): sgl_per_token_quant_fp8(x, dynamic_out, dynamic_scale) - with benchmark_with_power(device=device, kernel_func=dynamic_kernel_func, repeat_n=1) as dynamic_results: + with benchmark_with_power(device=device, kernel_func=kernel_func, repeat_n=1) as results: pass + return results["latency_ms"] / _OUTSIDE_LOOP_COUNT, results["power_stats"] + - dynamic_latency = dynamic_results["latency_ms"] / outside_loop_count +def _bench_static(device, x): + static_out = torch.empty_like(x, dtype=torch.float8_e4m3fn) + static_scale = torch.tensor([1.0], dtype=torch.float32, device=device) - def static_kernel_func(): - for _ in range(outside_loop_count): + def kernel_func(): + for _ in range(_OUTSIDE_LOOP_COUNT): _static_quantize_e4m3_per_tensor(x, static_scale, static_out) - with benchmark_with_power(device=device, kernel_func=static_kernel_func, repeat_n=1) as static_results: + with benchmark_with_power(device=device, kernel_func=kernel_func, repeat_n=1) as results: pass + return results["latency_ms"] / _OUTSIDE_LOOP_COUNT, results["power_stats"] + - static_latency = static_results["latency_ms"] / outside_loop_count +def run_computescale(m, k, *, perf_filename, extra_perf_filenames, device="cuda:0"): + (scale_matrix_filename,) = extra_perf_filenames + device, x = _setup(m, k, device) + dynamic_latency, dynamic_power = _bench_dynamic(device, x) + static_latency, static_power = _bench_static(device, x) compute_scale_latency = max(0.0, dynamic_latency - static_latency) - version = pkg_resources.get_distribution("sglang").version + version = _dist_version("sglang") if not log_perf( item_list=[{"m": m, "k": k, "quant_dtype": "fp8", "latency": compute_scale_latency}], @@ -64,9 +87,10 @@ def static_kernel_func(): op_name="compute_scale", kernel_source="sglang", perf_filename=perf_filename, - power_stats=dynamic_results["power_stats"], + power_stats=dynamic_power, ): raise RuntimeError(f"Failed to persist SGLang compute scale performance row to {perf_filename}") + if not log_perf( item_list=[{"m": m, "k": k, "quant_dtype": "fp8", "latency": static_latency}], framework="SGLang", @@ -74,7 +98,7 @@ def static_kernel_func(): device_name=torch.cuda.get_device_name(device), op_name="scale_matrix", kernel_source="sglang", - perf_filename="scale_matrix_perf.txt", - power_stats=static_results["power_stats"], + perf_filename=scale_matrix_filename, + power_stats=static_power, ): - raise RuntimeError("Failed to persist SGLang scale matrix performance row to scale_matrix_perf.txt") + raise RuntimeError(f"Failed to persist SGLang scale matrix performance row to {scale_matrix_filename}") diff --git a/python/aisimulate/collector/sglang/collect_dsv4_attn.py b/python/aisimulate/collector/sglang/collect_dsv4_attn.py index 7f9f42c567..9c7da934fd 100644 --- a/python/aisimulate/collector/sglang/collect_dsv4_attn.py +++ b/python/aisimulate/collector/sglang/collect_dsv4_attn.py @@ -26,7 +26,9 @@ # Requires stock SGLang 0.5.14 with its matching ``sgl-kernel`` package. from __future__ import annotations -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" import argparse import contextlib @@ -54,10 +56,24 @@ # few shapes which the bench's own warmup JIT-compiles on first use. os.environ["SGLANG_JIT_DEEPGEMM_PRECOMPILE"] = "0" try: - from helper import _resolve_local_model_path, benchmark_with_power, log_perf + from helper import ( + PeriodicGarbageCollector, + PhaseClock, + _resolve_local_model_path, + benchmark_with_power, + log_perf, + render_phase_totals, + ) except ModuleNotFoundError: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - from helper import _resolve_local_model_path, benchmark_with_power, log_perf + from helper import ( + PeriodicGarbageCollector, + PhaseClock, + _resolve_local_model_path, + benchmark_with_power, + log_perf, + render_phase_totals, + ) try: from collector.sglang.runtime_limits import ( @@ -504,7 +520,16 @@ def _resolve_model_path( # model_type to "deepseek_v3" mirrors what sglang's # _load_deepseek_temp_model produces internally, so AutoConfig succeeds # and the V4 model class is still selected via the architectures field. - config["model_type"] = "deepseek_v3" + try: + # sglang>=0.5.21 ships a native DeepseekV4Config (configs/deepseek_v4.py, model_type + # "deepseek_v4") and its DSV4 model reads V4-only fields from it + # (hc_pre_from_prev_sublayer, models/deepseek_v4.py:2679); the v3 rewrite below + # would hand the model a DeepseekV3Config and fail there. + import sglang.srt.configs.deepseek_v4 # noqa: F401 + + config["model_type"] = "deepseek_v4" + except ImportError: + config["model_type"] = "deepseek_v3" # gemm_type "fp8_block" overrides disable_weight_quant: we MUST keep the # fp8 quantization_config so sglang dispatches projections to DeepGEMM. @@ -689,7 +714,11 @@ def _derive_csa_context_pool_cap( if not isinstance(configurator, DSV4PoolConfigurator): raise TypeError(f"expected SGLang 0.5.14 DSV4PoolConfigurator, got {type(configurator).__name__}") - page_size = int(model_runner.server_args.page_size) + from collector.sglang.runtime_compat import resolved_arg + + page_size = int( + resolved_arg(model_runner.server_args, "page_size") + ) # 0.5.21: derived, read through the resolving view if page_size != _DSV4_CUDA_PAGE_SIZE: raise RuntimeError(f"expected DSV4 CUDA page_size={_DSV4_CUDA_PAGE_SIZE}, got {page_size}") window_size = int(model_runner.model_config.window_size) @@ -705,7 +734,17 @@ def _derive_csa_context_pool_cap( if not chunk_eligible_shapes: raise RuntimeError(f"DSV4 CSA context pool derivation has no shape within effective_chunk={effective_chunk}") - profiled_bytes = int(model_runner._profile_available_bytes(model_runner.pre_model_load_memory)) + # sglang>=0.5.21 moved the pool profiling off ModelRunner onto the KV-cache + # configurator (mem_cache/kv_cache_configurator.py:2177 _profile_available_bytes, + # same free-memory-minus-slack arithmetic). Serving builds that object inside + # alloc_memory_pool (model_runner.py:885 init_kv_cache_configurator, then + # configure); build it the same way here, before the pool exists. + _profile = getattr(model_runner, "_profile_available_bytes", None) + if _profile is None: + if getattr(model_runner, "kv_cache_configurator", None) is None: + model_runner.init_kv_cache_configurator() + _profile = model_runner.kv_cache_configurator._profile_available_bytes + profiled_bytes = int(_profile(model_runner.pre_model_load_memory)) profiled_config = configurator.calculate_pool_sizes(profiled_bytes, page_size) compress_ratio = ATTN_KIND_TO_COMPRESS_RATIO["csa"] if page_size % compress_ratio != 0: @@ -908,20 +947,18 @@ def _load_model_runner( _set_envs_and_config(server_args) model_config = ModelConfig.from_server_args(server_args) + from collector.sglang.runtime_compat import init_runtime_config + + _runner_parallel_kwargs = init_runtime_config(server_args, gpu_id, nccl_port=nccl_port, model_config=model_config) with _tp_load_model_patch(tp_size): model_runner = ModelRunner( model_config=model_config, # Use sglang's own __post_init__-derived value, not a collector knob. mem_fraction_static=server_args.mem_fraction_static, gpu_id=gpu_id, - tp_rank=0, - tp_size=1, - pp_rank=0, - pp_size=1, - moe_ep_rank=0, - moe_ep_size=1, nccl_port=nccl_port, server_args=server_args, + **_runner_parallel_kwargs, ) derived_requirements = None if csa_context_shapes is not None: @@ -931,7 +968,18 @@ def _load_model_runner( model_runner, csa_context_shapes, ) - model_runner.server_args.max_total_tokens = derived_requirements[0] + try: + model_runner.server_args.max_total_tokens = derived_requirements[0] + except AttributeError: + # sglang>=0.5.21: ServerArgs is read-only once resolved (server_args.py:396); + # post-publish config changes go through the context's config bags + # (runtime_context.py:1080 override(source, **fields) — the same call + # serving uses for control-plane updates, tokenizer_manager.py:2204). The + # configurator reads the cap from get_schedule().max_total_tokens + # (kv_cache_configurator.py:2269 _apply_token_constraints). + from sglang.srt.runtime_context import get_context + + get_context().override("collector-dsv4-csa-context", max_total_tokens=derived_requirements[0]) # SGLang 0.5.14 separates model construction from serving-state setup. model_runner.alloc_memory_pool() if derived_requirements is not None: @@ -1036,7 +1084,11 @@ def _make_reqs( req.full_untruncated_fill_ids = array("q", req.origin_input_ids) req.fill_len = full_len req.logprob_start_len = 0 - req.set_extend_input_len(seq_len if prefix_len else full_len) + _n = seq_len if prefix_len else full_len + if hasattr(req, "set_extend_input_len"): + req.set_extend_input_len(_n) + else: # sglang 0.5.16: schedule_batch.py:1153 set_extend_range(start, end) + req.set_extend_range(full_len - _n, full_len) req.swa_evicted_seqlen = swa_evicted_seqlen if decode: req.cached_tokens = 0 @@ -1113,7 +1165,16 @@ def _build_forward_batch( req.output_ids.append(0) batch.prepare_for_decode() - forward_batch = ForwardBatch.init_new(batch, model_runner) + import inspect as _inspect + + # sglang 0.5.16: return_hidden_states_before_norm is a required keyword of + # ForwardBatch.init_new; older pins reject it (collect_msa_module.py:1015 pattern) + _fb_kw = ( + {"return_hidden_states_before_norm": False} + if "return_hidden_states_before_norm" in _inspect.signature(ForwardBatch.init_new).parameters + else {} + ) + forward_batch = ForwardBatch.init_new(batch, model_runner, **_fb_kw) model_runner.attn_backend.init_forward_metadata(forward_batch) return forward_batch @@ -1209,6 +1270,7 @@ def timed_kernel(): "used_cuda_graph": True, "power_stats": result.get("power_stats"), "throttled": bool(result.get("throttled", False)), + "phase_seconds": result.get("phase_seconds"), } @@ -1345,10 +1407,29 @@ def run_dsv4_mla_module( seq_lens_by_prefix.get(cur_prefix, default_seq_lens) if seq_lens_by_prefix is not None else default_seq_lens ) ] + phase_totals: dict[str, float] = {} + # gc.collect() after every point cost 0.2-0.4 s (half of a context sweep); collect periodically instead. + point_gc = PeriodicGarbageCollector() # SGLang derives compressed-pool sizes from max_total_tokens. Direct # probes with an explicit value must account for both the full pool and # the smaller page-rounded SWA tail pool before ModelRunner construction. + # + # HCA always takes this path since sglang 0.5.21: the ratio-128 stage prices a + # full token at ~1/128 of a paged slot (pool_configurator.py:1264-1283 + # _compressed_bytes_per_full_token / _get_bytes_per_full_token), so the + # framework-derived pool on an otherwise empty 140 GB device reaches ~2e9 full + # tokens, and the hybrid allocator's int64 full_to_swa_index_mapping + # (mem_cache/allocator/swa.py:99-108, size + page_size entries, not priced by the + # configurator) then needs ~15 GiB that the sizing already handed to the pools + # (OOM in SWATokenToKVPoolAllocator.__init__, kv_cache_configurator.py:2080, + # H20 2026-10-01). Serving never meets this corner because weights leave far + # less free memory; the collector sizes the pool to what the planned shapes + # allocate instead. sglang treats the value as an upper bound only + # (kv_cache_configurator.py:2269-2277 _apply_token_constraints: min(profiled, + # user cap)), so no cell gains capacity it would not have in serving. + if max_total_tokens is None and attn_kind == "hca": + max_total_tokens = 0 if max_total_tokens is not None: max_full_alloc = 0 max_swa_alloc = 0 @@ -1495,6 +1576,7 @@ def run_dsv4_mla_module( skipped_shapes.append((batch_size, seq_len, cur_prefix, "KVPoolSWACapacity")) continue print(f"\n{mode}: batch_size={batch_size}, seq_len={seq_len}, prefix_len={cur_prefix}") + point_clock = PhaseClock() try: forward_batch = _build_forward_batch( model_runner, @@ -1503,6 +1585,7 @@ def run_dsv4_mla_module( is_prefill=is_prefill, prefix_len=cur_prefix, ) + point_clock.mark("forward_batch") hidden_states, positions = _make_inputs( model_runner, batch_size=batch_size, @@ -1511,6 +1594,7 @@ def run_dsv4_mla_module( device=device, prefix_len=cur_prefix, ) + point_clock.mark("inputs") def kernel_func(model_runner=model_runner): from sglang.srt.model_executor.forward_context import ForwardContext, forward_context @@ -1536,6 +1620,8 @@ def kernel_func(model_runner=model_runner): graph_repeat=graph_repeat, device=device, ) + point_clock.mark("bench") + point_clock.absorb(stats.get("phase_seconds"), prefix="bench_") print( f" latency mean={stats['mean_ms']:.4f} ms, " f"median={stats['median_ms']:.4f} ms, " @@ -1593,7 +1679,7 @@ def kernel_func(model_runner=model_runner): ("req_to_token_pool.clear", model_runner.req_to_token_pool.clear), ("token_to_kv_pool_allocator.clear", model_runner.token_to_kv_pool_allocator.clear), ("torch.cuda.empty_cache", torch.cuda.empty_cache), - ("gc.collect", gc.collect), + ("gc.collect", point_gc.maybe_collect), ): try: _cleanup_step() @@ -1606,6 +1692,12 @@ def kernel_func(model_runner=model_runner): f"dsv4-flash {sweep_label} bs={batch_size} sl={seq_len} " f"prefix={cur_prefix}: cleanup failed: {'; '.join(cleanup_errors)}" ) + point_clock.mark("cleanup") + point_clock.add_to(phase_totals) + print( + f" [dsv4 point timing] {sweep_label} bs={batch_size} sl={seq_len} prefix={cur_prefix}: " + + point_clock.render() + ) finally: final_cleanup_errors = [] del model_runner @@ -1633,6 +1725,7 @@ def kernel_func(model_runner=model_runner): if skipped_shapes: skipped_str = ", ".join(f"(bs={b},sl={s},prefix={p},reason={r})" for b, s, p, r in skipped_shapes) print(f"[WARN] dsv4-flash {sweep_label}: skipped or errored shapes: {skipped_str}") + print(f"[dsv4 timing] {sweep_label}: {render_phase_totals(phase_totals)}") print(f"[dsv4-collector] {sweep_label}: {summary}") if ok_count == 0 or error_count > 0: raise RuntimeError(f"dsv4-flash {sweep_label}: {summary}") diff --git a/python/aisimulate/collector/sglang/collect_dsv4_megamoe.py b/python/aisimulate/collector/sglang/collect_dsv4_megamoe.py index 077455c711..079b8301b6 100644 --- a/python/aisimulate/collector/sglang/collect_dsv4_megamoe.py +++ b/python/aisimulate/collector/sglang/collect_dsv4_megamoe.py @@ -446,7 +446,10 @@ def copy_pre_dispatch(hidden_states, topk_ids, topk_weights, buffer, num_tokens: try: from sglang.kernels.ops.attention.dsv4 import mega_moe_pre_dispatch except ModuleNotFoundError: - from sglang.jit_kernel.deepseek_v4 import mega_moe_pre_dispatch + try: + from sglang.jit_kernel.deepseek_v4 import mega_moe_pre_dispatch + except ImportError: # sglang>=0.5.21: kernels/ops/moe/dsv4.py:156 + from sglang.kernels.ops.moe.dsv4 import mega_moe_pre_dispatch def sglang_jit_pre_dispatch(hidden_states, topk_ids, topk_weights, buffer, num_tokens: int): del num_tokens diff --git a/python/aisimulate/collector/sglang/collect_gdn.py b/python/aisimulate/collector/sglang/collect_gdn.py index 209e5f3a33..9ded6052cf 100644 --- a/python/aisimulate/collector/sglang/collect_gdn.py +++ b/python/aisimulate/collector/sglang/collect_gdn.py @@ -86,7 +86,9 @@ # accepted, narrow gap: the framework_manifest digest-pinned gate is the # true version enforcement upstream and only ever supplies exactly 0.5.14 # or 0.5.17 in a sanctioned run, so the leak is unreachable there. -__compat__ = "sglang>=0.5.14,<=0.5.17,!=0.5.15,!=0.5.16" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19,!=0.5.20" import gc import os @@ -194,13 +196,16 @@ def run_gdn_context_benchmark( perf_filename: str, sglang_version: str, device: str = "cuda:0", + mamba_ssm_dtype: str = "float32", ): """ Benchmark GDN operations for context (prefill) phase using SGLang's Triton FLA kernels. Benchmarks: 1. causal_conv1d_fn — Conv1D over packed Q+K+V channels - 2. chunk_gated_delta_rule — GDN scan (Q, K, V, g, beta) via SGLang's vendored FLA + 2. chunk_gated_delta_rule — GDN scan (Q, K, V, g, beta) via SGLang's vendored FLA, + or flashinfer_chunk_gated_delta_rule when sglang>=0.5.21 serving defaults the + prefill backend to FlashInfer for the case (see _resolve_flashinfer_gdn_prefill) """ device = torch.device(device) torch.cuda.set_device(device) @@ -219,6 +224,7 @@ def run_gdn_context_benchmark( ) conv_weight = torch.randn(conv_channels, d_conv, dtype=dtype, device=device) + flashinfer_prefill = _resolve_flashinfer_gdn_prefill(head_k_dim, head_v_dim, mamba_ssm_dtype) successful_points = 0 failed_points = 0 @@ -244,9 +250,14 @@ def run_gdn_context_benchmark( # DSA-FUSED-KS-4G-OFFSET row. Raise instead of launching the # corrupting kernel: the async illegal access otherwise poisons # the CUDA context and aborts every remaining sweep cell. + # NOT relaxed for sglang 0.5.21: its causal_conv1d_triton casts the sequence + # offsets to tl.int64 (:95,:185), yet the >=2**31 cells still fault with + # cudaErrorIllegalAddress on H20 (0.5.21 smoke 2026-10-01, facts/upgrade_rc29/ + # sgl_smoke_r6) — silicon evidence keeps the guard; the int32 arithmetic is + # elsewhere in the kernel's per-block view strides. if total_tokens * conv_channels >= 2**31: raise ValueError( - "SGLang 0.5.14 causal_conv1d Triton kernel int32 token-offset overflow: " + "SGLang causal_conv1d Triton kernel int32 token-offset overflow: " f"total_tokens={total_tokens} * conv_channels={conv_channels} >= 2**31 " "(causal_conv1d_triton.py:373-379)" ) @@ -336,19 +347,52 @@ def run_conv1d(): ): raise RuntimeError(f"failed to persist SGLang GDN context row to {perf_filename}") - def run_gdn_scan(): - chunk_gated_delta_rule( - q, - k, - v, - g, - beta, - initial_state=recurrent_state, - initial_state_indices=state_indices, - cu_seqlens=cu_seqlens, - head_first=False, - use_qk_l2norm_in_kernel=True, - ) + if flashinfer_prefill is None: + scan_kernel_source = "chunk_gated_delta_rule" + + def run_gdn_scan(): + chunk_gated_delta_rule( + q, + k, + v, + g, + beta, + initial_state=recurrent_state, + initial_state_indices=state_indices, + cu_seqlens=cu_seqlens, + head_first=False, + use_qk_l2norm_in_kernel=True, + ) + + else: + # FlashInferGDNKernel.extend @0.5.21 (kernels/gdn_flashinfer.py:465-581): + # l2norm/layout prepare, exp(g) and beta as fp32, the state rows gathered + # (fp32 pool on SM90) and written back with index_copy_, int64 cu_seqlens, + # use_qk_l2norm_in_kernel=False because the prepare already normalized. + scan_kernel_source = "flashinfer_chunk_gated_delta_rule" + prefill_fn, qkv_prepare = flashinfer_prefill + cu_seqlens_i64 = cu_seqlens.to(torch.int64) + state_rows = state_indices.to(torch.int64) + initial_state_fi = recurrent_state[state_rows].to(torch.float32) + output_state_fi = torch.empty_like(initial_state_fi) + + def run_gdn_scan(): + q_fi, k_fi, v_fi = qkv_prepare(q[0], k[0], v[0]) + prefill_fn( + q=q_fi, + k=k_fi, + v=v_fi, + g=torch.exp(g[0].to(torch.float32)), + beta=beta[0].to(torch.float32), + scale=None, + initial_state=initial_state_fi, + output_final_state=True, + cu_seqlens=cu_seqlens_i64, + use_qk_l2norm_in_kernel=False, + output=None, + output_state=output_state_fi, + ) + recurrent_state.index_copy_(0, state_rows, output_state_fi.to(recurrent_state.dtype)) with benchmark_with_power( device=device, @@ -363,7 +407,7 @@ def run_gdn_scan(): version=sglang_version, device_name=torch.cuda.get_device_name(device), op_name="gdn", - kernel_source="chunk_gated_delta_rule", + kernel_source=scan_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ): @@ -447,6 +491,113 @@ def _resolve_flashinfer_gdn_decode(mamba_ssm_dtype: str = "float32"): return gated_delta_rule_decode_pretranspose, None +def _sglang_gdn_lanes_0521(): + """Feature-probe the sglang>=0.5.21 GDN lane surface (None on older runtimes). + + 0.5.21 moved the GDN backend to srt/layers/attention/linear/gdn_backend.py and + changed what serving runs inside the GDN role (H20 serving golden 2026-10-01, + results/pathdiff/sm90/sglang-0.5.21/gdn_{ctx,gen}_Qwen3.5-0.8B.json): + * prefill: attention_registry.py:425-430 asks flashinfer_gdn_prefill_default + (gdn_backend.py:188-234) and, when it answers "flashinfer", the extend path + runs FlashInferGDNKernel.extend (kernels/gdn_flashinfer.py:465-581): + gdn_prefill_qkv_prepare_fwd + flashinfer.gdn_prefill.chunk_gated_delta_rule + instead of the vendored FLA chunk_gated_delta_rule; + * decode: Qwen3.5 hands the backend the projected (qkvz, ba) pair + (models/qwen3_5.py:160-162, 940-953; SGLANG_ENABLE_GDN_DECODE_FUSED_PROJ_CONV + defaults True, environ.py:1800) and forward_decode runs + fused_qkvzba_causal_conv1d_update_contiguous (gdn_backend.py:668-760) in place + of the split + causal_conv1d_update. + """ + try: + from sglang.kernels.ops.attention.fla.l2norm import gdn_prefill_qkv_prepare_fwd + from sglang.kernels.ops.attention.triton_gdn_fused_proj import ( + can_use_fused_qkvzba_causal_conv1d_update_contiguous, + fused_qkvzba_causal_conv1d_update_contiguous, + ) + from sglang.srt import environ as _environ + from sglang.srt.layers.attention.linear.gdn_backend import ( # noqa: F401 — lane marker + flashinfer_gdn_prefill_default, + ) + from sglang.srt.layers.attention.linear.kernels.gdn_flashinfer import ( + _get_flashinfer_gdn_kernels, + is_flashinfer_gdn_prefill_available, + ) + except ImportError: + return None + return { + "qkv_prepare": gdn_prefill_qkv_prepare_fwd, + "fused_conv_eligible": can_use_fused_qkvzba_causal_conv1d_update_contiguous, + "fused_conv": fused_qkvzba_causal_conv1d_update_contiguous, + "fused_conv_enabled": bool(_environ.envs.SGLANG_ENABLE_GDN_DECODE_FUSED_PROJ_CONV.get()), + "flashinfer_prefill_available": is_flashinfer_gdn_prefill_available, + "flashinfer_kernels": _get_flashinfer_gdn_kernels, + } + + +def _resolve_flashinfer_gdn_prefill(head_k_dim: int, head_v_dim: int, mamba_ssm_dtype: str): + """Mirror sglang>=0.5.21's own prefill-backend default for the case. + + gdn_backend.py:188-234 flashinfer_gdn_prefill_default: FlashInfer only when + no explicit --linear-attn-prefill-backend, linear_attn_backend == triton, + no deterministic / page-major mode, SM major 9 or 10 (SM100 additionally + CUDA >= 13 and a bf16 state pool; SM90 an fp32 state pool), key/value head + dim 128, a static chunked_prefill_size within the validated chunk bound + (serving default sizing: derived, not None, dynamic chunking off), and the + FlashInfer prefill kernels importable. Everything else keeps Triton FLA, + and so does this collector. Returns ``(prefill_fn, qkv_prepare)`` or + ``None`` (FLA lane); FlashInfer missing is serving's own Triton fallback + (:230-231), so it is not an error here. + """ + lanes = _sglang_gdn_lanes_0521() + if lanes is None: + return None + sm_version = get_sm_version() + sm_major = sm_version // 10 + if sm_major not in (9, 10): + print(f" SM{sm_version}: sglang>=0.5.21 keeps Triton FLA GDN prefill (flashinfer default is SM90/SM100 only).") + return None + if sm_major == 10: + cuda_major = int(torch.version.cuda.split(".", 1)[0]) if torch.version.cuda else 0 + if cuda_major < 13 or mamba_ssm_dtype != "bfloat16": + print( + f" SM{sm_version}, cuda={torch.version.cuda}, mamba_ssm_dtype={mamba_ssm_dtype!r}: " + "sglang>=0.5.21 FlashInfer GDN prefill needs CUDA>=13 and a bf16 state pool on SM100; Triton FLA." + ) + return None + elif mamba_ssm_dtype != "float32": + print( + f" SM{sm_version}, mamba_ssm_dtype={mamba_ssm_dtype!r}: " + "FlashInfer GDN prefill needs an fp32 state pool on SM90; Triton FLA." + ) + return None + if head_k_dim != 128 or head_v_dim != 128: + print( + f" head_k_dim={head_k_dim}, head_v_dim={head_v_dim}: FlashInfer GDN prefill is 128/128 only; Triton FLA." + ) + return None + if not lanes["flashinfer_prefill_available"](): + print( + " FlashInfer GDN prefill kernels unavailable in this runtime: " + "serving falls back to Triton FLA, so does this lane." + ) + return None + _available, prefill_fn, *_rest = lanes["flashinfer_kernels"]() + return prefill_fn, lanes["qkv_prepare"] + + +def _resolve_fused_gdn_decode_conv(): + """sglang>=0.5.21 decode conv lane: the fused qkvz/ba unpack + indexed Conv1D + update (gdn_backend.py:668-760), on by default (environ.py:1800). Returns + ``(fused_fn, eligibility_fn)`` or ``None`` for the split + causal_conv1d_update + lane. When the fused kernel's own eligibility check refuses a shape, serving + logs and falls back to the unfused pair (gdn_backend.py:790-815), which the + caller mirrors and kernel_source records.""" + lanes = _sglang_gdn_lanes_0521() + if lanes is None or not lanes["fused_conv_enabled"]: + return None + return lanes["fused_conv"], lanes["fused_conv_eligible"] + + def run_gdn_generation_benchmark( d_model: int, d_conv: int, @@ -503,6 +654,7 @@ def run_gdn_generation_benchmark( # Resolve serving's exact decode dispatch: SM-major 10 + bf16 uses FI; # current FP32 recipes stay on FLA, and a required-but-missing FI raises. flashinfer_gdn_decode_fn, flashinfer_gdn_decode_error = _resolve_flashinfer_gdn_decode(mamba_ssm_dtype) + fused_decode_conv = _resolve_fused_gdn_decode_conv() successful_points = 0 failed_points = 0 failures = [] @@ -555,7 +707,36 @@ def run_gdn_generation_benchmark( "model_name": model_name, } + conv_kernel_source = "causal_conv1d_update" + fused_conv_args = None + if fused_decode_conv is not None: + # Serving input at 0.5.21: the projection outputs as laid out by Qwen3.5 + # (qwen3_5.py:940-953) — [Q|K|V|Z] of width qkv_dim + v_dim and [B|A] of + # width 2*num_v_heads, both model dtype; the kernel unpacks and runs the + # indexed Conv1D update in one launch (gdn_backend.py:749-761). + projected_qkvz = torch.randn(batch_size, conv_channels + value_dim, dtype=dtype, device=device) + projected_ba = torch.randn(batch_size, 2 * num_v_heads, dtype=dtype, device=device) + fused_conv_fn, fused_conv_eligible = fused_decode_conv + fused_conv_args = (projected_qkvz, projected_ba, conv_state, conv_weight, None, state_indices) + fused_conv_kwargs = dict( + qkv_dim=conv_channels, + v_dim=value_dim, + num_v_heads=num_v_heads, + activation="silu", + ) + eligible, reason = fused_conv_eligible(*fused_conv_args, **fused_conv_kwargs) + if eligible: + conv_kernel_source = "fused_qkvzba_causal_conv1d_update_contiguous" + else: + # gdn_backend.py:790-815: serving logs the reason and runs the unfused + # split + causal_conv1d_update for this shape. + print(f" fused GDN decode conv ineligible ({reason}); serving falls back to causal_conv1d_update") + fused_conv_args = None + def run_conv1d_update(): + if fused_conv_args is not None: + fused_conv_fn(*fused_conv_args, head_v_dim=head_v_dim, **fused_conv_kwargs) + return causal_conv1d_update( mixed_qkv, conv_state, @@ -578,7 +759,7 @@ def run_conv1d_update(): version=sglang_version, device_name=torch.cuda.get_device_name(device), op_name="gdn", - kernel_source="causal_conv1d_update", + kernel_source=conv_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ): @@ -820,6 +1001,7 @@ def run_gdn_torch( perf_filename=perf_filename, sglang_version=sglang_version, device=device, + mamba_ssm_dtype=mamba_ssm_dtype, ) elif phase == "generation": run_gdn_generation_benchmark( diff --git a/python/aisimulate/collector/sglang/collect_gemm.py b/python/aisimulate/collector/sglang/collect_gemm.py index e35cf02e6d..59fb19bdee 100644 --- a/python/aisimulate/collector/sglang/collect_gemm.py +++ b/python/aisimulate/collector/sglang/collect_gemm.py @@ -50,6 +50,8 @@ # framework_manifest digest-pinned gate is the true version enforcement # upstream and only ever supplies exactly 0.5.14 or 0.5.17 in a sanctioned # run, so the leak is unreachable there. +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. # SGLang 0.5.20 (GLM-5.3-Flash model pin; audited 2026-09-30 against tag # commit 94602c9c2b7c): serving changed WHICH kernels this module's raw # calls map to (bf16 CuTe DSL/split-K dispatch, JIT per-token fp8 quant, @@ -61,7 +63,7 @@ # instead of a registry VersionRoute because tools/perf_database/ # changed_ops.py fails closed on versioned registry entries. 0.5.18/0.5.19 # are not audited and stay excluded (same != caveat as above). -__compat__ = "sglang>=0.5.14,<=0.5.20,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19" +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19" import os import random @@ -70,6 +72,7 @@ # Set it before any SGLang imports so a task compiles only its requested M. os.environ.setdefault("SGLANG_JIT_DEEPGEMM_PRECOMPILE", "0") +# setuptools/pkg_resources is absent from recent framework images. from importlib.metadata import version as _dist_version import torch @@ -494,8 +497,9 @@ def kernel_func(): # Whole-implementation version fork (see the __compat__ comment): SGLang -# 0.5.20 collects through serving-built linear layers. -if _sglang_release() >= (0, 5, 20): +# 0.5.20 collects through serving-built linear layers. Exactly 0.5.20: the +# 0.5.21 default keeps this module's own (0.5.21-validated) implementation. +if _sglang_release()[:3] == (0, 5, 20): from collector.sglang import collect_gemm_0520 as _fork_0520 get_gemm_test_cases = _fork_0520.get_gemm_test_cases diff --git a/python/aisimulate/collector/sglang/collect_kda.py b/python/aisimulate/collector/sglang/collect_kda.py index 4bc206f3de..d4ba1019c0 100644 --- a/python/aisimulate/collector/sglang/collect_kda.py +++ b/python/aisimulate/collector/sglang/collect_kda.py @@ -53,12 +53,18 @@ """ # The kimi-k3 branch build (https://github.com/sgl-project/sglang/tree/kimi-k3) -# reports 0.5.16. 0.5.20 adds GLM-5.3-Flash (glm5_next) KDA, audited against +# reports 0.5.16 and was the only KDA runtime until the kernels landed in the +# official releases (0.5.21: models/kimi_linear.py, layers/attention/linear/ +# kda_backend.py with the same Triton lanes — chunk_kda, fused_recurrent_kda_ +# packed_decode / fused_sigmoid_gating_delta_rule_update, kda_fused_decode, +# causal_conv1d_fn/update — kda_backend.py:6-34, kernels/kda_triton.py:109-237; +# H20 smoke 2026-10-01, 91 cells ok, the remaining 21 are the >=2**31 conv +# guard cells). 0.5.20 adds GLM-5.3-Flash (glm5_next) KDA, audited against # the GB300 0.5.20 image (see the GLM section below). The file-level range -# spans both; each KDA architecture is additionally gated at runtime to its -# own audited release by _KDA_ARCHITECTURE_COMPAT (Kimi-K3 0.5.16, GLM-5.3- -# Flash 0.5.20), raising KdaRuntimeNotAuditedError otherwise. -__compat__ = "sglang>=0.5.16,<=0.5.20,!=0.5.17,!=0.5.18,!=0.5.19" +# spans all three; each KDA architecture is additionally gated at runtime to +# its own audited releases by _KDA_ARCHITECTURE_COMPAT (Kimi-K3 0.5.16 and +# 0.5.21, GLM-5.3-Flash 0.5.20), raising KdaRuntimeNotAuditedError otherwise. +__compat__ = "sglang>=0.5.16,<=0.5.21,!=0.5.17,!=0.5.18,!=0.5.19" import gc import os @@ -213,6 +219,11 @@ def run_kda_context_benchmark( # [2**31, 3*2**31) crash with cudaErrorIllegalAddress on both # Hopper SM90 (2026-07 campaign coverage boundary) and SM100 # (B200, 2026-07-28), while every cell under this bound passes. + # NOT relaxed for sglang 0.5.21: its causal_conv1d_triton casts the sequence + # offsets to tl.int64 (:95,:185), yet the >=2**31 cells still fault with + # cudaErrorIllegalAddress on H20 (0.5.21 smoke 2026-10-01, facts/upgrade_rc29/ + # sgl_smoke_r6) — silicon evidence keeps the guard; the int32 arithmetic is + # elsewhere in the kernel's per-block view strides. if total_tokens * conv_channels >= 2**31: raise ValueError( "SGLang causal_conv1d Triton kernel int32 token-offset overflow: " @@ -370,6 +381,30 @@ def _fused_decode_module(): return kda_fused_decode +def _serving_decode_lane_for_lower_bound() -> str: + """Which recurrence kernel serving runs for a KDA layer that carries a + ``lower_bound`` (Kimi-K3: KDA_LOWER_BOUND). + + sglang>=0.5.21 kda_backend.KDAAttnBackend.forward_decode:737-741 takes the + packed Triton kernel only when ``supports_packed_decode`` AND the layer has + no lower_bound; a lower-bound layer goes through kernel_dispatcher.decode + (:771-786) = TritonKDAKernel.decode -> fused_sigmoid_gating_delta_rule_update + with lower_bound (kernels/kda_triton.py:125-157). The kimi-k3 branch build + (0.5.16) packed every T=1 decode. Probed from the backend source rather than + the version string so a backport is followed too. H20 serving golden + 2026-10-01 (archive raw 7ef3a64cae10 decode_kernels) carries + fused_sigmoid_gating_delta_rule_update_kernel, not the packed kernel. + """ + try: + import inspect + + from sglang.srt.layers.attention.linear.kda_backend import KDAAttnBackend + except ImportError: + return "packed" + src = inspect.getsource(KDAAttnBackend.forward_decode) + return "plain" if 'getattr(layer, "lower_bound", None) is None' in src else "packed" + + def run_kda_generation_benchmark( d_model: int, d_conv: int, @@ -543,20 +578,57 @@ def run_conv1d_update(): ): raise RuntimeError(f"failed to persist SGLang KDA generation row to {perf_filename}") - def run_kda_packed_decode(): - fused_recurrent_kda_packed_decode( - mixed_qkv=mixed_qkv, - a=a, - b=b, - A_log=a_log, - dt_bias=dt_bias, - scale=head_k_dim**-0.5, - initial_state=recurrent_state, - out=output, - ssm_state_indices=state_indices, - use_qk_l2norm_in_kernel=True, - lower_bound=KDA_LOWER_BOUND, + if _serving_decode_lane_for_lower_bound() == "plain" and KDA_LOWER_BOUND is not None: + # kda_backend.py:771-786 @0.5.21: after the conv update, q/k/v are the + # [Q|K|V] split viewed as 1 x n x h x d, and the plain Triton decode runs + # fused_sigmoid_gating_delta_rule_update over cu_seqlens with lower_bound. + recurrence_kernel_source = "fused_sigmoid_gating_delta_rule_update" + plain_q, plain_k, plain_v = ( + t.unflatten(-1, (-1, dim)).unsqueeze(0) + for t, dim in zip( + mixed_qkv.split([proj_size] * 3, dim=-1), (head_k_dim, head_k_dim, head_v_dim), strict=True + ) ) + plain_a = a.view(1, batch_size, num_v_heads, head_k_dim) + plain_b = b.view(1, batch_size, num_v_heads) + plain_cu_seqlens = torch.arange(0, batch_size + 1, dtype=torch.int32, device=device) + + def run_kda_packed_decode(): + fused_sigmoid_gating_delta_rule_update( + A_log=a_log, + dt_bias=dt_bias, + q=plain_q, + k=plain_k, + v=plain_v, + a=plain_a, + b=plain_b, + initial_state_source=recurrent_state, + initial_state_indices=state_indices, + cu_seqlens=plain_cu_seqlens, + use_qk_l2norm_in_kernel=True, + softplus_beta=1.0, + softplus_threshold=20.0, + is_kda=True, + lower_bound=KDA_LOWER_BOUND, + ) + + else: + recurrence_kernel_source = "fused_recurrent_kda_packed_decode" + + def run_kda_packed_decode(): + fused_recurrent_kda_packed_decode( + mixed_qkv=mixed_qkv, + a=a, + b=b, + A_log=a_log, + dt_bias=dt_bias, + scale=head_k_dim**-0.5, + initial_state=recurrent_state, + out=output, + ssm_state_indices=state_indices, + use_qk_l2norm_in_kernel=True, + lower_bound=KDA_LOWER_BOUND, + ) with benchmark_with_power( device=device, @@ -571,7 +643,7 @@ def run_kda_packed_decode(): version=sglang_version, device_name=torch.cuda.get_device_name(device), op_name="kda", - kernel_source="fused_recurrent_kda_packed_decode", + kernel_source=recurrence_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ): @@ -951,7 +1023,7 @@ def _is_glm5_next_kda(model_name: str) -> bool: # replicates; any other installed release raises KdaRuntimeNotAuditedError (a # classified failure) instead of timing a possibly different kernel path. _KDA_ARCHITECTURE_COMPAT = { - KIMI_K3_KDA_ARCHITECTURE: "sglang==0.5.16", + KIMI_K3_KDA_ARCHITECTURE: "sglang>=0.5.16,<=0.5.21,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20", GLM5_NEXT_KDA_ARCHITECTURE: "sglang==0.5.20", } diff --git a/python/aisimulate/collector/sglang/collect_mhc_module.py b/python/aisimulate/collector/sglang/collect_mhc_module.py index 97be868c57..80f7a4e5cc 100644 --- a/python/aisimulate/collector/sglang/collect_mhc_module.py +++ b/python/aisimulate/collector/sglang/collect_mhc_module.py @@ -8,11 +8,13 @@ # capable image or put a matching SGLang source tree on PYTHONPATH. from __future__ import annotations -# The file-level range spans the two audited routes; each architecture is -# additionally gated to its own audited release by _ARCHITECTURE_COMPAT, so -# the DeepSeek-V4 path still runs only on 0.5.14 (its ModelRunner/ServerArgs -# API use is 0.5.14-specific) and GLM only on 0.5.20. -__compat__ = "sglang>=0.5.14,<=0.5.20" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# The file-level range spans the audited routes; each architecture is +# additionally gated to its own audited releases by _ARCHITECTURE_COMPAT: +# DeepSeek-V4 on 0.5.14 and 0.5.21 (the 0.5.21 port uses the native +# DeepseekV4Config), GLM only on 0.5.20. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19" import argparse import copy @@ -57,7 +59,7 @@ # Per-architecture audited SGLang releases; anything else raises # MhcRuntimeNotAuditedError (a classified failure) instead of running. _ARCHITECTURE_COMPAT = { - DSV4_ARCHITECTURE: "sglang==0.5.14", + DSV4_ARCHITECTURE: "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20", GLM5NEXT_ARCHITECTURE: "sglang==0.5.20", } @@ -224,7 +226,13 @@ def _patched_model_dir(model_id: str) -> str: # Match collect_dsv4_attn.py: current Transformers does not know a # native deepseek_v4 config, while SGLang selects the V4 model class from # the architectures field. - config["model_type"] = "deepseek_v3" + try: + # sglang>=0.5.21: native DeepseekV4Config (V4-only fields such as hc_pre_from_prev_sublayer) + import sglang.srt.configs.deepseek_v4 # noqa: F401 + + config["model_type"] = "deepseek_v4" + except ImportError: + config["model_type"] = "deepseek_v3" tmp_dir = os.path.join( tempfile.gettempdir(), @@ -283,18 +291,17 @@ def _load_one_layer_runner( _set_envs_and_config(server_args) model_config = ModelConfig.from_server_args(server_args) + nccl_port = 29500 + random.randint(0, 10000) + from collector.sglang.runtime_compat import init_runtime_config + + _runner_parallel_kwargs = init_runtime_config(server_args, gpu_id, nccl_port=nccl_port, model_config=model_config) return ModelRunner( model_config=model_config, mem_fraction_static=mem_fraction_static, gpu_id=gpu_id, - tp_rank=0, - tp_size=1, - pp_rank=0, - pp_size=1, - moe_ep_rank=0, - moe_ep_size=1, - nccl_port=29500 + random.randint(0, 10000), + nccl_port=nccl_port, server_args=server_args, + **_runner_parallel_kwargs, ) diff --git a/python/aisimulate/collector/sglang/collect_mla.py b/python/aisimulate/collector/sglang/collect_mla.py index 9391b32e59..f26a6a7e26 100644 --- a/python/aisimulate/collector/sglang/collect_mla.py +++ b/python/aisimulate/collector/sglang/collect_mla.py @@ -9,13 +9,17 @@ mocking, runtime dispatch, and perf logging. """ -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +import contextlib +import inspect import math import os import random +from importlib.metadata import version as _dist_version # setuptools/pkg_resources is absent from sglang>=0.5.21 images -import pkg_resources import sglang.srt.layers.dp_attention import sglang.srt.server_args import torch @@ -93,6 +97,9 @@ def _mla_compute_dtype(backend: str, kv_cache_dtype: torch.dtype) -> str: class MockModelConfig: + def get_max_num_attention_heads(self) -> int: # model_config.py:1504 @0.5.21 (triton_backend.py:255) + return int(self.num_attention_heads) + def __init__( self, context_len: int = 32768, @@ -120,10 +127,26 @@ def __init__( self.head_dim = 256 self.v_head_dim = v_head_dim self.hf_text_config = self + # sglang>=0.5.21 TritonAttnBackend.__init__ (the sm89 / sm120 MLA default) asks + # mambaish_config(model_config) whether this is a hybrid linear-attention arch + # (triton_backend.py:287 -> configs/hybrid_arch.py reads hf_config / get_text_config() + # / linear_attn_registry_result); an MLA module is none of them. fa3 / trtllm_mla + # (sm90 / sm100) never reach that probe, so only the Triton lane needs this. + self.linear_attn_registry_result = None + self.is_draft_model = False + + class _MockHFConfig: + architectures = ["DeepseekV3ForCausalLM"] # noqa: RUF012 + model_type = "deepseek_v3" + + def get_text_config(self): + return self + + self.hf_config = _MockHFConfig() self.scaling = scaling self.is_local_attention_model = False - def get_num_kv_heads(self, tp_size: int): + def get_num_kv_heads(self, tp_size: int, dcp_size: int = 1): # model_config.py:1512 @0.5.21 return 1 @@ -160,9 +183,14 @@ def __init__(self, kv_cache_dtype: torch.dtype, page_size: int): # stays off exactly as in production (triton_backend.py:214-217). # The SM90 fa3 / SM100 trtllm_mla init paths never read this field. self.cuda_graph_config = None + self.enable_prefill_cp = False # sglang 0.5.16 flashattention_backend.py:51 + self.enable_dp_attention = False # same site: prefill-CP/DP-attention gate class MockModelRunner: + def decode_num_tokens_per_req(self, *, num_draft_tokens=None) -> int: + return 1 + def __init__( self, device: torch.device, @@ -175,6 +203,8 @@ def __init__( self.gpu_id = device.index if device.index is not None else torch.cuda.current_device() self.tp_size = 1 self.kv_cache_dtype = kv_cache_dtype + # sglang>=0.5.21 backends read the runner's server-arg spelling (model_runner.py:1481) + self.kv_cache_dtype_str = "fp8_e4m3" if kv_cache_dtype == torch.float8_e4m3fn else "auto" self.dtype = torch.bfloat16 self.page_size = page_size self.req_to_token_pool = None @@ -190,7 +220,48 @@ def __init__( # Keep attributes for compatibility across sglang versions (older code ignores them) self.is_hybrid_swa = self.model_config.is_hybrid_swa self.attn_cp_size = 1 # Context parallelism size; required by FlashAttentionBackend in sglang >=0.5.10 + # sglang 0.5.21 TRTLLMMLABackend (the MLA backend sglang picks on SM100, _get_backends) reads + # model_runner.max_running_requests (workspace sizing) and model_runner.is_draft_worker; + # on Hopper fa3/flashinfer never asked, so the attributes were missing until the first + # Blackwell smoke (B200 2026-10-04: AttributeError 'MockModelRunner' has no attribute + # 'max_running_requests'). 512 = the generator's --max-running-requests default. + self.max_running_requests = 512 + self.is_draft_worker = False + # sglang 0.5.16 reads the parallel geometry from model_runner.ps + # (flashattention_backend.py:183,271-274; parallel_state_wrapper.py:6-24) + try: + from sglang.srt.distributed.parallel_state_wrapper import ParallelState + + self.ps = ParallelState.trivial(gpu_id=0) + except ImportError: + from types import SimpleNamespace + + self.ps = SimpleNamespace( + tp_rank=0, + tp_size=1, + pp_rank=0, + pp_size=1, + dp_rank=0, + dp_size=1, + attn_tp_rank=0, + attn_tp_size=1, + attn_cp_rank=0, + attn_cp_size=1, + attn_dp_rank=0, + attn_dp_size=1, + moe_ep_rank=0, + moe_ep_size=1, + moe_dp_rank=0, + moe_dp_size=1, + dcp_size=1, + gpu_id=0, + ) self.server_args = MockServerArgs(kv_cache_dtype, page_size) + self.is_draft_worker = False + # sglang>=0.5.21 backends ask the runner for logits rows per decode slot + # (model_runner.py:796 decode_num_tokens_per_req; triton_backend.py:218,222); + # the kernel collectors never run speculative decoding -> 1 + self.spec_algorithm = None # read by the 0.5.16 attention backends (draft-worker branches) self.use_mla_backend = True @@ -214,8 +285,91 @@ def create_req_to_token_pool( return pool, token_matrix.contiguous() +def _extend_seq_lens_cpu(forward_batch): + """CPU int32 mirror of the per-request extend lengths of a plain EXTEND batch, else None. + + The values are sglang's own ``forward_batch.extend_seq_lens_cpu`` — the list its + trtllm_mla backend already reads for ``max_seq_len`` (trtllm_mla_backend.py:850 + @v0.5.21) — so nothing is reconstructed, only mirrored into a CPU tensor. + """ + if forward_batch.forward_mode != ForwardMode.EXTEND: + return None + extend_seq_lens = getattr(forward_batch, "extend_seq_lens_cpu", None) + if not extend_seq_lens: + return None + return torch.tensor([int(n) for n in extend_seq_lens], dtype=torch.int32, device="cpu") + + +@contextlib.contextmanager +def _ragged_prefill_with_cpu_seq_lens(seq_lens_cpu): + """Hand flashinfer the CPU length mirrors that keep trtllm_ragged_attention_deepseek capturable. + + flashinfer 0.6.18 (sglang 0.5.21's pin) takes optional ``q_seq_lens_cpu`` / + ``kv_seq_lens_cpu`` (prefill.py:4797-4798). Without them the wrapper derives + the per-row lengths from the device indptrs with a host sync and refuses to + run under CUDA-graph capture ("q_seq_lens_cpu and kv_seq_lens_cpu must be + provided during CUDA graph capture", prefill.py:5083-5091); with them it + validates the mirrors on the host (prefill.py:5053-5077) and launches the + same trtllm-gen kernel with the same device arguments. Older flashinfer + builds without the parameters never synchronized here and are left alone. + """ + import flashinfer.prefill as flashinfer_prefill + + original = flashinfer_prefill.trtllm_ragged_attention_deepseek + parameters = inspect.signature(original).parameters + if "q_seq_lens_cpu" not in parameters or "kv_seq_lens_cpu" not in parameters: + yield + return + + def _with_cpu_seq_lens(*args, **kwargs): + kwargs.setdefault("q_seq_lens_cpu", seq_lens_cpu) + kwargs.setdefault("kv_seq_lens_cpu", seq_lens_cpu) + return original(*args, **kwargs) + + flashinfer_prefill.trtllm_ragged_attention_deepseek = _with_cpu_seq_lens + try: + yield + finally: + flashinfer_prefill.trtllm_ragged_attention_deepseek = original + + +class _GraphCapturableTRTLLMMLABackend(TRTLLMMLABackend): + """sglang's trtllm_mla backend with a CUDA-graph-capturable prefill kernel call. + + Serving never graph-captures prefill, so sglang 0.5.21's + ``TRTLLMMLABackend._run_prefill_kernel`` (trtllm_mla_backend.py:1098-1141) + calls ``flashinfer.prefill.trtllm_ragged_attention_deepseek`` without the + CPU length mirrors and flashinfer refuses the capture (see + ``_ragged_prefill_with_cpu_seq_lens``). The collector times the kernel under + graph replay — eager timing of this lane floors at the Python launch loop + (0.13 ms on B200, 0.29 ms on GB200 for every isl <= 1024 cell; data + validation 2026-10-11) — so the no-prefix prefill call gets the mirrors + here. This changes how the kernel is invoked, never which kernel runs: the + chunked-prefix calls (their own ``cum_seq_lens_kv``) and every other + backend path go through sglang unchanged. + """ + + _prefill_seq_lens_cpu = None + + def init_forward_metadata(self, forward_batch): + super().init_forward_metadata(forward_batch) + self._prefill_seq_lens_cpu = _extend_seq_lens_cpu(forward_batch) + + def _run_prefill_kernel(self, *args, **kwargs): + seq_lens_cpu = self._prefill_seq_lens_cpu + cum_seq_lens_q = kwargs.get("cum_seq_lens_q") + cum_seq_lens_kv = kwargs.get("cum_seq_lens_kv") + if seq_lens_cpu is None or cum_seq_lens_q is None or cum_seq_lens_kv is not cum_seq_lens_q: + return super()._run_prefill_kernel(*args, **kwargs) + with _ragged_prefill_with_cpu_seq_lens(seq_lens_cpu): + return super()._run_prefill_kernel(*args, **kwargs) + + def benchmark_layer(layer, forward_batch, q, k, v, q_rope, k_rope, **kwargs): - # Use benchmark_with_power context manager + # CUDA-graph replay for every lane: a capture failure raises (helper default + # allow_graph_fail=False) and an eager result is refused below, so a kernel + # that stops being capturable fails the case instead of shipping its launch + # floor (sglang mla_context on B200/GB200, data validation 2026-10-11). device = q.device def kernel_func(): @@ -236,6 +390,8 @@ def kernel_func(): ) as results: pass + if results.get("used_cuda_graph") is not True: + raise RuntimeError("SGLang MLA requires CUDA graph timing; refusing to publish eager timing") return results["latency_ms"], results["power_stats"] @@ -349,6 +505,9 @@ def run_mla( perf_filename, device="cuda:0", ): + from collector.sglang.runtime_compat import ensure_offline_runtime_published + + ensure_offline_runtime_published() # sglang>=0.5.20 backends read get_exec()/get_parallel() torch.cuda.set_device(device) torch_device = torch.device(device) random.seed(0) @@ -363,6 +522,28 @@ def run_mla( expected_page_size = 64 if selected_backend == "trtllm_mla" else 1 if tokens_per_block != expected_page_size: raise ValueError(f"SGLang {selected_backend} requires page_size={expected_page_size}, got {tokens_per_block}") + # FIXME(kernel-limit): sglang 0.5.21 trtllm_mla DECODE on Blackwell with 64 < local heads < 128. + # flashinfer's trtllm-gen MLA decode refuses that head range (flashinfer mla/_core.py:3948) and + # falls back to the cute-dsl runner, but sglang 0.5.21's trtllm_mla_backend passes + # multi_ctas_kv_counter_buffer unconditionally on Blackwell -> "multi_ctas_kv_counter_buffer is + # only supported when a trtllm-gen runner is selected" (_core.py:4278). Serving hits the same + # raise (B200 identity probe: Kimi-K3 bf16 tp1, 96 heads; finding + # sm100_probe_failures_recheck_2026_10_04 (i)), so the cell is serving-unreachable at this tp; + # the b200_sxm shard run (GitLab job 469988017) failed exactly the 362 num_heads=96 decode cases + # (bf16 + fp8 KV) and nothing else. Prefill (mla_context, 96 heads) is unaffected. Fail-closed: + # recorded as a classified failure, never a predicted skip or a silent backend swap. + if ( + not is_context_phase + and selected_backend == "trtllm_mla" + and get_sm_version() in {100, 103} + and 64 < local_num_heads < 128 + ): + raise RuntimeError( + "FIXME(kernel-limit): sglang 0.5.21 trtllm_mla decode on SM100/103 does not support " + f"64 < local_num_heads < 128 (num_heads={num_heads}, tp_size={tp_size} -> {local_num_heads}); " + "flashinfer trtllm-gen MLA decode rejects the head range and the cute-dsl fallback rejects " + "multi_ctas_kv_counter_buffer (mla/_core.py:3948, :4278); serving fails identically (Kimi-K3 tp1)" + ) model_runner = MockModelRunner( torch_device, @@ -385,7 +566,12 @@ def run_mla( model_runner.server_args.prefill_attention_backend = selected_backend model_runner.server_args.decode_attention_backend = selected_backend # Set global args after potential overrides. - sglang.srt.server_args.set_global_server_args_for_scheduler(model_runner.server_args) + try: + from sglang.srt.runtime_context import publish as _publish # noqa: F401 + except ImportError: + sglang.srt.server_args.set_global_server_args_for_scheduler(model_runner.server_args) + # >=0.5.20: set_global_server_args_for_scheduler publishes (resolve_once on a real ServerArgs record); + # ensure_offline_runtime_published() already installed the process record, the mock drives the backends directly. # Define dimensions based on phase kv_lora_rank = KV_LORA_RANK @@ -432,6 +618,9 @@ def run_mla( enable_memory_saver=False, ) model_runner.token_to_kv_pool = kv_pool + from collector.sglang.runtime_compat import attach_kv_index_translator + + attach_kv_index_translator(model_runner) if selected_backend == "trtllm_mla": # TRTLLMMLABackend inherits FlashInferMLAAttnBackend which creates @@ -440,7 +629,7 @@ def run_mla( import gc gc.collect() - attn_backend = TRTLLMMLABackend(model_runner) + attn_backend = _GraphCapturableTRTLLMMLABackend(model_runner) kernel_source = "trtllm_mla" elif selected_backend == "triton": attn_backend = TritonAttnBackend(model_runner) @@ -628,7 +817,7 @@ def run_mla( } ], framework="SGLang", - version=pkg_resources.get_distribution("sglang").version, + version=_dist_version("sglang"), device_name=torch.cuda.get_device_name(device), op_name=f"mla_{'context' if is_context_phase else 'generation'}", kernel_source=kernel_source, diff --git a/python/aisimulate/collector/sglang/collect_mla_bmm.py b/python/aisimulate/collector/sglang/collect_mla_bmm.py index 3a253d4791..d487a2f7ac 100644 --- a/python/aisimulate/collector/sglang/collect_mla_bmm.py +++ b/python/aisimulate/collector/sglang/collect_mla_bmm.py @@ -8,16 +8,31 @@ shapes, selects SGLang kernel helpers, and logs the resulting MLA BMM perf rows. """ -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" + +from importlib.metadata import version as _dist_version # setuptools/pkg_resources is absent from sglang>=0.5.21 images -import pkg_resources import torch from collector.case_generator import get_mla_bmm_case_specs from collector.helper import benchmark_with_power, get_sm_version, log_perf -from sgl_kernel import bmm_fp8 -from sglang.srt.layers.quantization.fp8_kernel import ( - per_tensor_quant_mla_fp8, -) + +try: + from sgl_kernel import bmm_fp8 + + _BMM_FP8_SOURCE = "sglang_sgl_kernel_bmm_fp8" +except ImportError: # sglang>=0.5.21: sgl_kernel no longer exports bmm_fp8; serving wraps flashinfer's + # (layers/quantization/fp8_utils.py:246-273 @0.5.21, custom op flashinfer_bmm_fp8_batched) + from sglang.srt.layers.quantization.fp8_utils import bmm_fp8 + + _BMM_FP8_SOURCE = "sglang_flashinfer_bmm_fp8" +try: + from sglang.srt.layers.quantization.fp8_kernel import per_tensor_quant_mla_fp8 +except ModuleNotFoundError: # sglang>=0.5.21: the Triton quant kernels moved to sglang.kernels.ops + # (kernels/ops/quantization/fp8_kernel.py:956 @0.5.21; same kernel, the serving MLA fp8 lane + # imports it from there — deepseek_common/attention_forward_methods/forward_mla_fused_rope_rocm.py:8) + from sglang.kernels.ops.quantization.fp8_kernel import per_tensor_quant_mla_fp8 def _supported_dtypes() -> set[str]: @@ -97,10 +112,10 @@ def kernel_func(): } ], framework="SGLang", - version=pkg_resources.get_distribution("sglang").version, + version=_dist_version("sglang"), device_name=torch.cuda.get_device_name(device), op_name="mla_gen_pre", - kernel_source="sglang_sgl_kernel_bmm_fp8" if dtype == "fp8" else "sglang_torch_bmm", + kernel_source=_BMM_FP8_SOURCE if dtype == "fp8" else "sglang_torch_bmm", perf_filename=perf_filename, power_stats=results["power_stats"], ): @@ -181,10 +196,10 @@ def kernel_func(): } ], framework="SGLang", - version=pkg_resources.get_distribution("sglang").version, + version=_dist_version("sglang"), device_name=torch.cuda.get_device_name(device), op_name="mla_gen_post", - kernel_source="sglang_sgl_kernel_bmm_fp8" if dtype == "fp8" else "sglang_torch_bmm", + kernel_source=_BMM_FP8_SOURCE if dtype == "fp8" else "sglang_torch_bmm", perf_filename=perf_filename, power_stats=results["power_stats"], ): diff --git a/python/aisimulate/collector/sglang/collect_mla_module.py b/python/aisimulate/collector/sglang/collect_mla_module.py index f7cd50debb..77ba8f8b59 100644 --- a/python/aisimulate/collector/sglang/collect_mla_module.py +++ b/python/aisimulate/collector/sglang/collect_mla_module.py @@ -22,7 +22,9 @@ python collect_mla_module.py --mode generation --attn-type mla """ -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" import argparse import gc @@ -44,12 +46,13 @@ from importlib.metadata import version as get_version import torch +from collector.sglang.runtime_compat import resolved_arg # 0.5.21: derived server args live behind the resolving view try: - from helper import benchmark_with_power, get_sm_version, log_perf + from helper import PhaseClock, benchmark_with_power, get_sm_version, log_perf, render_phase_totals except ModuleNotFoundError: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - from helper import benchmark_with_power, get_sm_version, log_perf + from helper import PhaseClock, benchmark_with_power, get_sm_version, log_perf, render_phase_totals try: from collector.case_generator import ( @@ -130,6 +133,10 @@ _NATIVE_NVFP4_SMS = {100, 103, 120} +# Per-subprocess sum of the point phase times printed by run_mla_module. +_POINT_PHASE_TOTALS: dict[str, float] = {} + + def _env_flag(name: str, default: bool = False) -> bool: value = os.environ.get(name) if value is None: @@ -186,8 +193,8 @@ def _is_glm5_dsa_model(model_id: str) -> bool: # Set by run_mla_module() at the start of each benchmark subprocess from its -# skip_indexer arg (threaded down from run_mla_module_worker, which derives it -# from the op's perf_filename). Process-local: each subprocess runs exactly one +# skip_indexer arg (threaded down from run_mla_module_worker, which receives it +# from the registry entry's run_kwargs). Process-local: each subprocess runs exactly one # run_mla_module, so this is never shared across cases. Replaces the old # AIC_DSA_SKIP_INDEXER env that previously crossed the subprocess boundary. _SKIP_INDEXER_PASS = False @@ -218,7 +225,7 @@ def _generation_cuda_graph_enabled_for_tokens(model_runner, num_tokens: int) -> that coverage using sglang's own settings -- no AIC env override -- so the decode benchmark uses graph timing exactly where serve would. """ - decode_config = model_runner.server_args.cuda_graph_config.decode + decode_config = resolved_arg(model_runner.server_args, "cuda_graph_config").decode if decode_config.backend == "disabled": return False capture_bs = decode_config.bs @@ -879,7 +886,7 @@ def get_dsa_context_module_skip_indexer_test_cases(): patched out (GLM-5.2 index_topk_freq>1 reuse layers). Same shapes as the full context module — the skip behaviour is applied in - the subprocess (run_func detects the skip_indexer perf_filename). Only emit + the subprocess (the registry binds skip_indexer=True via run_kwargs). Only emit cases for models that actually share the index across layers (index_topk_freq > 1); for freq==1 models the skip layer == full layer, so a separate file would just duplicate dsa_context_module. @@ -1154,9 +1161,6 @@ def load_model_runner( from sglang.srt.configs.model_config import ModelConfig from sglang.srt.entrypoints.engine import _set_envs_and_config - from sglang.srt.layers.moe import initialize_moe_config - from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config - from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.server_args import ServerArgs from sglang.srt.utils import suppress_other_loggers @@ -1231,9 +1235,6 @@ def load_model_runner( server_args.json_model_override_args = json.dumps(override_args) _set_envs_and_config(server_args) - initialize_moe_config(server_args) - initialize_fp8_gemm_config(server_args) - initialize_fp4_gemm_config(server_args) nccl_port = 29500 + random.randint(0, 10000) + gpu_id * 100 @@ -1255,18 +1256,19 @@ def load_model_runner( if native_quant == "fp8_block": _ensure_fp8_block_quant_config(model_config.hf_config) + # parallel geometry / config publish / distributed bootstrap: one helper per sglang generation + from collector.sglang.runtime_compat import init_runtime_config + + _runner_parallel_kwargs = init_runtime_config( + server_args, gpu_id, nccl_port=nccl_port, model_config=model_config, tp_rank=gpu_id, tp_size=server_args.tp_size + ) model_runner = ModelRunner( model_config=model_config, mem_fraction_static=server_args.mem_fraction_static, gpu_id=gpu_id, - tp_rank=gpu_id, - tp_size=server_args.tp_size, - pp_rank=0, - pp_size=1, - moe_ep_rank=0, - moe_ep_size=1, nccl_port=nccl_port, server_args=server_args, + **_runner_parallel_kwargs, ) model_runner.alloc_memory_pool() @@ -1471,7 +1473,10 @@ def _run_prefill( from array import array is_wideep_mla = attn_type == "mla" and not ordinary_mla - from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context + try: # sglang>=0.5.21: layers/layer_boundary/adapters/attention.py:45,167 + from sglang.srt.layers.layer_boundary.adapters.attention import AttentionInputs, get_attn_tp_context + except ImportError: + from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.chunk_cache import ChunkCache @@ -1483,6 +1488,7 @@ def _run_prefill( forward_context_type, forward_context = _import_sglang_forward_context() print(f"\nPrefill: batch_size={batch_size}, seq_length={seq_length}, prefix_len={prefix_len}") + point_clock = PhaseClock() try: model_runner.req_to_token_pool.clear() @@ -1502,9 +1508,18 @@ def _run_prefill( req.prefix_indices = prefix_indices[i] req.full_untruncated_fill_ids = array("q", req.origin_input_ids) req.fill_len = full_length - req.set_extend_input_len(seq_length if prefix_len else full_length) + # sglang 0.5.16 replaced Req.set_extend_input_len(n) with + # set_extend_range(start, end) (schedule_batch.py:1153: the extend + # window is the last n of fill_len tokens); keep the old call on + # pins that still have it. + _n = seq_length if prefix_len else full_length + if hasattr(req, "set_extend_input_len"): + req.set_extend_input_len(_n) + else: + req.set_extend_range(full_length - _n, full_length) req.logprob_start_len = 0 reqs.append(req) + point_clock.mark("reqs") cache_params = CacheInitParams( disable=True, @@ -1525,8 +1540,20 @@ def _run_prefill( ) with _temporarily_chunked_alloc_extend(model_runner, batch_size * seq_length): batch.prepare_for_extend() - forward_batch = ForwardBatch.init_new(batch, model_runner) + point_clock.mark("batch") + # sglang 0.5.16 made return_hidden_states_before_norm a required + # keyword (ForwardBatch.init_new); older pins do not accept it — + # same dispatch collect_msa_module.py:1015 uses. + import inspect as _inspect + + _fb_kw = ( + {"return_hidden_states_before_norm": False} + if "return_hidden_states_before_norm" in _inspect.signature(ForwardBatch.init_new).parameters + else {} + ) + forward_batch = ForwardBatch.init_new(batch, model_runner, **_fb_kw) model_runner.attn_backend.init_forward_metadata(forward_batch) + point_clock.mark("metadata") hidden_states = torch.randn( batch_size * seq_length, @@ -1983,6 +2010,7 @@ def call_attention_module(): **_skip_kwargs(), ) + point_clock.mark("setup") last_can_run_graph = None def call_full_model_piecewise_replay(): @@ -2034,6 +2062,7 @@ def call_full_model_piecewise_replay(): "refusing to record a skip row with full-indexer latency." ) + point_clock.mark("warmup") module_cuda_graph = None if use_module_cuda_graph: try: @@ -2047,6 +2076,7 @@ def call_full_model_piecewise_replay(): print(" Module CUDA graph capture failed") raise + point_clock.mark("capture") # Timed runs — run ALL iterations back-to-back, ONE sync at the end, then # divide by count. Per-iteration cuda.synchronize would insert GPU-idle # bubbles + per-launch latency between modules (NOT representative); the @@ -2064,6 +2094,7 @@ def call_full_model_piecewise_replay(): end_event.record() torch.cuda.synchronize() avg_time_ms = start_event.elapsed_time(end_event) / num_iterations + point_clock.mark("timed") # Log perf — wideep MLA uses old filename/op_name/kernel_source conventions try: @@ -2143,6 +2174,11 @@ def call_full_model_piecewise_replay(): cleanup_errors.append(f"{cleanup_name}: {type(cleanup_exc).__name__}: {cleanup_exc}") if cleanup_errors: raise RuntimeError(f"DSA/MLA prefill cleanup failed: {'; '.join(cleanup_errors)}") + point_clock.mark("cleanup") + point_clock.add_to(_POINT_PHASE_TOTALS) + print( + f" [DSA point timing] prefill b={batch_size}, s={seq_length}, prefix={prefix_len}: {point_clock.render()}" + ) def _run_decode( @@ -2173,7 +2209,10 @@ def _run_decode( from array import array is_wideep_mla = attn_type == "mla" - from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context + try: # sglang>=0.5.21: layers/layer_boundary/adapters/attention.py:45,167 + from sglang.srt.layers.layer_boundary.adapters.attention import AttentionInputs, get_attn_tp_context + except ImportError: + from sglang.srt.layers.communicator import AttentionInputs, get_attn_tp_context from sglang.srt.managers.schedule_batch import Req, ScheduleBatch from sglang.srt.mem_cache.cache_init_params import CacheInitParams from sglang.srt.mem_cache.chunk_cache import ChunkCache @@ -2186,6 +2225,7 @@ def _run_decode( forward_context_type, forward_context = _import_sglang_forward_context() print(f"\nDecode: batch_size={batch_size}, kv_cache_length={seq_length}") + point_clock = PhaseClock() try: model_runner.req_to_token_pool.clear() @@ -2202,11 +2242,15 @@ def _run_decode( req.prefix_indices = torch.empty((0,), dtype=torch.int64) req.full_untruncated_fill_ids = array("q", req.origin_input_ids) req.fill_len = len(req.origin_input_ids) - req.set_extend_input_len(req.fill_len) + if hasattr(req, "set_extend_input_len"): + req.set_extend_input_len(req.fill_len) + else: # sglang 0.5.16: schedule_batch.py:1153 set_extend_range(start, end) + req.set_extend_range(0, req.fill_len) req.logprob_start_len = 0 req.cached_tokens = 0 req.already_computed = 0 reqs.append(req) + point_clock.mark("reqs") cache_params = CacheInitParams( disable=True, @@ -2230,8 +2274,17 @@ def _run_decode( for req in batch.reqs: req.output_ids.append(0) batch.prepare_for_decode() - forward_batch_decode = ForwardBatch.init_new(batch, model_runner) + point_clock.mark("batch") + import inspect as _inspect + + _fb_kw = ( + {"return_hidden_states_before_norm": False} + if "return_hidden_states_before_norm" in _inspect.signature(ForwardBatch.init_new).parameters + else {} + ) # sglang 0.5.16 required keyword (see the context path above) + forward_batch_decode = ForwardBatch.init_new(batch, model_runner, **_fb_kw) model_runner.attn_backend.init_forward_metadata(forward_batch_decode) + point_clock.mark("metadata") decode_hidden = torch.randn( batch_size, @@ -2244,6 +2297,7 @@ def _run_decode( attn_inputs_decode = AttentionInputs(decode_hidden, forward_batch_decode, dummy_qkv_latent_func) get_attn_tp_context().set_attn_inputs(attn_inputs_decode) + point_clock.mark("inputs") if use_module_cuda_graph or _env_flag("AIC_ENABLE_MODULE_CUDA_GRAPH"): _patch_nsa_indexer_compile_for_module_cuda_graph(attention_module) @@ -2322,6 +2376,7 @@ def kernel_func(): if _skip_indexer: attention_module.next_skip_topk = original_next_skip_topk + point_clock.mark("prewarm") print(f" Decode module CUDA graph: {use_benchmark_cuda_graph} (tokens={batch_size})") with benchmark_with_power( @@ -2333,6 +2388,8 @@ def kernel_func(): use_cuda_graph=use_benchmark_cuda_graph, ) as results: pass + point_clock.mark("bench") + point_clock.absorb(results.get("phase_seconds"), prefix="bench_") if _skip_indexer and _skip_state["prev_topk"] is None: raise RuntimeError( @@ -2428,6 +2485,9 @@ def kernel_func(): cleanup_errors.append(f"{cleanup_name}: {type(cleanup_exc).__name__}: {cleanup_exc}") if cleanup_errors: raise RuntimeError(f"DSA/MLA decode cleanup failed: {'; '.join(cleanup_errors)}") + point_clock.mark("cleanup") + point_clock.add_to(_POINT_PHASE_TOTALS) + print(f" [DSA point timing] decode b={batch_size}, kv={seq_length}: {point_clock.render()}") def _resolve_perf_path(output_path: str | None, filename: str) -> str: @@ -2467,9 +2527,9 @@ def run_mla_module( (batch_size, seq_length) combos for the specified phase. ``skip_indexer`` selects the GLM-5.2 reuse-layer variant (the per-layer - indexer is patched out and rows are tagged ``_skip_indexer``). It mirrors - ``is_prefill``: the worker derives it from the op's perf_filename and passes - it down. Recorded in the ``_SKIP_INDEXER_PASS`` process global so the + indexer is patched out and rows are tagged ``_skip_indexer``). The worker + receives it from the registry entry's run_kwargs and passes it down. + Recorded in the ``_SKIP_INDEXER_PASS`` process global so the existing ``_dsa_skip_indexer_enabled`` call sites need no signature change. """ global _SKIP_INDEXER_PASS @@ -2725,6 +2785,7 @@ def run_mla_module( error_count = len(cases) - logged_count summary = f"ok={logged_count} error={error_count} skip=0 total={len(cases)}" print(f"[{attn_type.upper()}] {phase_name.lower()} {summary}") + print(f"[{attn_type.upper()} timing] {phase_name.lower()}: {render_phase_totals(_POINT_PHASE_TOTALS)}") if error_count > 0: raise RuntimeError( f"{attn_type.upper()} module {phase_name.lower()} failed strict completeness: {summary}; " @@ -2833,6 +2894,7 @@ def run_mla_module_worker( perf_filename: str, device: str = "cuda:0", chunked_prefill_size: int | None = None, + skip_indexer: bool = False, ): """Worker-compatible wrapper used by collector/collect.py. @@ -2846,7 +2908,10 @@ def run_mla_module_worker( For DSA test cases, it defaults to None and _get_backends() is used. perf_filename and device are keyword-only arguments supplied by - collect.py via functools.partial and the worker dispatch loop. + collect.py via functools.partial and the worker dispatch loop; + skip_indexer is bound the same way from the registry entry's run_kwargs + (dsa_*_module_skip_indexer ops), so the full and skip ops share one + table and one run_func and differ only in this explicit argument. """ device_str = str(device) if not isinstance(device, str) else device gpu_id = int(device_str.split(":")[-1]) if ":" in device_str else 0 @@ -2857,9 +2922,9 @@ def run_mla_module_worker( # module; the only difference is (1) the per-layer indexer (mqa+topk) is # patched out via skip_topk in _run_prefill/_run_generation and (2) the rows # are tagged with an op_name "_skip_indexer" suffix in the same perf file. - # Derived from the op's perf_filename and threaded down as an explicit arg — - # exactly like is_prefill above; no env var crosses the subprocess boundary. - skip_indexer = "skip_indexer" in os.path.basename(perf_filename) + # Threaded down to the subprocess as an explicit arg; no env var crosses + # the subprocess boundary and nothing is inferred from the table name. + skip_indexer = bool(skip_indexer) print(f"\n{'=' * 60}") print( diff --git a/python/aisimulate/collector/sglang/collect_moe.py b/python/aisimulate/collector/sglang/collect_moe.py index d8d76999b2..55568066c2 100644 --- a/python/aisimulate/collector/sglang/collect_moe.py +++ b/python/aisimulate/collector/sglang/collect_moe.py @@ -55,6 +55,8 @@ # framework_manifest digest-pinned gate is the true version enforcement # upstream and only ever supplies exactly 0.5.14 or 0.5.17 in a sanctioned # run, so the leak is unreachable there. +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. # 0.5.20 audit (2026-09-30; source tag v0.5.20 == 94602c9c2b7c vs v0.5.17, # source-only). 0.5.18/0.5.19 # were never verified and are carved out with != (same caveat as above). @@ -88,14 +90,14 @@ # (moe_runner/flashinfer_cutlass.py:17,208-209), which the single-rank # parallel patch below does not replace, so flashinfer_cutlass lanes (the # SM120 nvfp4 map) are expected to fail until that module is added. -__compat__ = "sglang>=0.5.14,<=0.5.20,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19" +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19" import gc import importlib -import itertools import os import sys import tempfile +from collections.abc import Callable from contextlib import contextmanager # The SGLang 0.5.20 image ships setuptools without pkg_resources; the @@ -282,6 +284,7 @@ balanced_logits, benchmark_with_power, build_rank0_local_workload, + flashinfer_autotuned, get_sm_version, log_perf, power_law_logits_v3, @@ -304,6 +307,7 @@ balanced_logits, benchmark_with_power, build_rank0_local_workload, + flashinfer_autotuned, get_sm_version, log_perf, power_law_logits_v3, @@ -345,6 +349,13 @@ def _ensure_writable_flashinfer_cubin_dir() -> None: def get_moe_test_cases(): + """One case per (quant mode, shape, tp, ep, distribution) carrying the whole token + sweep, like the vLLM and TRT-LLM MoE collectors: the framework layer (weights, quant + processing, backend pins) is built once per case and every token point is timed and + persisted inside it. The previous one-case-per-token layout rebuilt the layer for + each of the 27 token points, which made the SGLang nvfp4 tables ~2x the fixed cost of + the vLLM ones (2026-10-28 re-collection: 4.4-5.8 h per SGLang shard vs 1-1.4 h per + vLLM shard on the same shapes).""" sm_version = get_sm_version() moe_list = get_moe_quantization_modes("sglang", sm_version=sm_version) @@ -358,7 +369,7 @@ def get_moe_test_cases(): model_name = common_moe_testcase.model_name num_tokens_list = common_moe_testcase.num_tokens_list - for moe_type, num_tokens in itertools.product(moe_list, num_tokens_list): + for moe_type in moe_list: if not moe_model_allows_quantization("sglang", model_name, moe_type): quant_policy_drops[model_name] = quant_policy_drops.get(model_name, 0) + 1 continue @@ -370,7 +381,7 @@ def get_moe_test_cases(): moe_backend = get_sglang_moe_backend(common_moe_testcase, moe_type, sm_version) base_case = [ moe_type, - num_tokens, + list(num_tokens_list), common_moe_testcase.hidden_size, common_moe_testcase.inter_size, common_moe_testcase.topk, @@ -399,7 +410,7 @@ def get_moe_test_cases(): ] physical_key = ( moe_type, - num_tokens, + tuple(num_tokens_list), common_moe_testcase.hidden_size, common_moe_testcase.inter_size, common_moe_testcase.topk, @@ -636,6 +647,7 @@ def _patch_framework_moe_parallel(*, moe_tp_size: int, moe_ep_size: int): parallel = SimpleNamespace( tp_size=moe_tp_size, tp_rank=0, + tp_group=None, # 0.5.21 topk.py:720 use_symmetric_memory(get_parallel().tp_group, disabled=True) only reads it moe_tp_size=moe_tp_size, moe_tp_rank=0, moe_ep_size=moe_ep_size, @@ -653,6 +665,11 @@ def replace(module, name, value): "sglang.srt.layers.moe.moe_runner.flashinfer_trtllm", "sglang.srt.layers.moe.moe_runner.flashinfer_mxfp4", "sglang.srt.layers.quantization.mxfp4_flashinfer_trtllm_moe", + # 0.5.21 flashinfer_cutlass runner reads get_parallel().tp_group / + # is_allocation_symmetric() at module level (moe_runner/flashinfer_cutlass.py:21, + # :27, :283, :510) — every Nemotron-3 BF16 flashinfer_cutlass case raised + # "parallel name 'tp_group' has not been written" in the 2026-10-02 H20 campaign. + "sglang.srt.layers.moe.moe_runner.flashinfer_cutlass", ): try: modules.append(importlib.import_module(module_name)) @@ -662,15 +679,9 @@ def replace(module, name, value): for module in modules: replace(module, "get_tp_group", lambda: None) replace(module, "is_allocation_symmetric", lambda: False) - if module in { - _moe_layer_mod, - _std_dispatch_mod, - _topk_mod, - _mxint4_mod, - _mxfp4_mod, - _fp8_mod, - _modelopt_mod, - }: + # Any module that imported get_parallel at module level reads our + # single-process stand-in (the dynamically imported runners included). + if hasattr(module, "get_parallel"): replace(module, "get_parallel", lambda: parallel) replace(_moe_layer_mod, "get_moe_expert_parallel_world_size", lambda: moe_ep_size) replace(_moe_layer_mod, "get_moe_expert_parallel_rank", lambda: 0) @@ -813,7 +824,8 @@ def _benchmark_framework_quantized_moe( *, moe_type: str, moe_backend: str, - num_tokens: int, + num_tokens_list: list[int], + on_result: Callable[[int, float, dict, str], None], hidden_size: int, inter_size: int, topk: int, @@ -840,8 +852,14 @@ def _benchmark_framework_quantized_moe( is_fp4_experts: bool, int4_group_size: int, device: str, -) -> tuple[float, dict, str]: - """Benchmark model-aware paths through SGLang FusedMoE (0.5.14/0.5.17).""" +) -> str: + """Benchmark model-aware paths through SGLang FusedMoE (0.5.14/0.5.17). + + The layer is constructed and its weights processed once; each token point in + ``num_tokens_list`` is then timed on it and handed to ``on_result(num_tokens, + latency_ms, power_stats, kernel_source)`` as soon as it is measured, so a crash + later in the sweep keeps the rows already persisted. Returns the kernel source. + """ if moe_backend.startswith("flashinfer"): _ensure_writable_flashinfer_cubin_dir() @@ -1011,11 +1029,21 @@ def _benchmark_framework_quantized_moe( f"requested={moe_backend}, actual={actual_backend}" ) if actual_backend == "flashinfer_mxfp4": - if quant_method._fi_kernel != "cutlass_sm90": + # Mxfp4MoEMethod dispatches flashinfer_mxfp4 to one of three leaves by GPU + # (mxfp4.py: SM90 cutlass_fused_moe w4-group-scaling, SM100 trtllm_fp4_block_scale_moe, + # SM120 cutlass MXFP8xMXFP4); name the kernel source after the leaf that actually ran + # (B200 2026-10-05: the Hopper-only check rejected the 'trtllm_sm100' leaf gpt-oss + # serves with on Blackwell — identity records: Mxfp4MoE -> trtllm_gen_moe). + _fi_leaf_sources = { + "cutlass_sm90": "sglang_flashinfer_cutlass_moe", + "trtllm_sm100": "sglang_mxfp4_flashinfer_trtllm_moe", + "flashinfer_cutlass_sm120": "sglang_flashinfer_cutlass_moe_sm120", + } + if quant_method._fi_kernel not in _fi_leaf_sources: raise RuntimeError( f"SGLang {moe_type} has unexpected FlashInfer MXFP4 leaf {quant_method._fi_kernel!r}" ) - kernel_source = "sglang_flashinfer_cutlass_moe" + kernel_source = _fi_leaf_sources[quant_method._fi_kernel] else: kernel_source = source_by_backend[actual_backend] elif ( @@ -1071,55 +1099,82 @@ def _benchmark_framework_quantized_moe( correction_bias = ( torch.zeros(num_experts, dtype=torch.float32, device=device) if has_correction_bias else None ) - topk_layer = TopK( - top_k=topk, - layer_id=0, - use_grouped_topk=num_expert_group is not None and topk_group is not None, - num_expert_group=num_expert_group, - topk_group=topk_group, - renormalize=renormalize, - scoring_func=scoring_func, - correction_bias=correction_bias, - output_format=( - TopKOutputFormat.STANDARD - if routing_method_type is None and moe_backend not in {"flashinfer_mxfp4", "triton_kernel"} - else None - ), - routed_scaling_factor=routed_scaling_factor, - apply_routed_scaling_factor_on_output=( - routed_scaling_factor is not None and moe_layer.should_fuse_routed_scaling_factor_in_topk - ), - is_fp4_experts=is_fp4_experts, - ) - hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device) - if distributed == "balanced": - logits = [ - balanced_logits(num_tokens, num_experts, topk).to(device=device, dtype=torch.float32) - for _ in range(5) - ] - elif distributed == "power_law": - logits = [ - power_law_logits_v3(num_tokens, num_experts, topk, moe_ep_size, power_law_alpha).to( - device=device, dtype=torch.float32 - ) - for _ in range(5) - ] + if routing_method_type == "Llama4": + # Serving builds the Llama-4 router with a custom routing function + # (fast_topk on the raw logits, then sigmoid; renormalize=False) — + # models/llama4.py:78-92 (function) and :114-118 (TopK construction) + # @0.5.21, unchanged since 0.5.14. The generic sigmoid TopK the + # collector used before routes through moe_fused_gate on 0.5.21, which + # asserts "bias is required for non-softmax routing" (kernels/ops/moe/ + # moe_fused_gate.py:384) — a kernel serving never runs for Llama-4. + from sglang.srt.models.llama4 import Llama4MoE + + topk_layer = TopK( + top_k=topk, + layer_id=0, + renormalize=False, + custom_routing_function=Llama4MoE.custom_routing_function, + ) else: - logits = [torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device) for _ in range(5)] - - def kernel_func(): - for router_logits in logits: - moe_layer(hidden_states, topk_layer(hidden_states, router_logits)) - - with benchmark_with_power( - device=device, - kernel_func=kernel_func, - num_warmups=5, - num_runs=10, - repeat_n=1, - ) as results: - pass - return results["latency_ms"] / len(logits), results["power_stats"], kernel_source + topk_layer = TopK( + top_k=topk, + layer_id=0, + use_grouped_topk=num_expert_group is not None and topk_group is not None, + num_expert_group=num_expert_group, + topk_group=topk_group, + renormalize=renormalize, + scoring_func=scoring_func, + correction_bias=correction_bias, + output_format=( + TopKOutputFormat.STANDARD + if routing_method_type is None and moe_backend not in {"flashinfer_mxfp4", "triton_kernel"} + else None + ), + routed_scaling_factor=routed_scaling_factor, + apply_routed_scaling_factor_on_output=( + routed_scaling_factor is not None and moe_layer.should_fuse_routed_scaling_factor_in_topk + ), + is_fp4_experts=is_fp4_experts, + ) + for num_tokens in num_tokens_list: + print(f"num_tokens {num_tokens} ({kernel_source})") + hidden_states = torch.randn(num_tokens, hidden_size, dtype=torch.bfloat16, device=device) + if distributed == "balanced": + logits = [ + balanced_logits(num_tokens, num_experts, topk).to(device=device, dtype=torch.float32) + for _ in range(5) + ] + elif distributed == "power_law": + logits = [ + power_law_logits_v3(num_tokens, num_experts, topk, moe_ep_size, power_law_alpha).to( + device=device, dtype=torch.float32 + ) + for _ in range(5) + ] + else: + logits = [ + torch.randn(num_tokens, num_experts, dtype=torch.float32, device=device) for _ in range(5) + ] + + def kernel_func(): + for router_logits in logits: + moe_layer(hidden_states, topk_layer(hidden_states, router_logits)) + + # FlashInfer-backed runners (flashinfer_trtllm / _mxfp4 / _cutlass / _cutedsl) + # are timed on the autotuned tactic like serving; see helper.flashinfer_autotuned. + with ( + flashinfer_autotuned(num_tokens, kernel_func, label=kernel_source), + benchmark_with_power( + device=device, + kernel_func=kernel_func, + num_warmups=5, + num_runs=10, + repeat_n=1, + ) as results, + ): + pass + on_result(num_tokens, results["latency_ms"] / len(logits), results["power_stats"], kernel_source) + return kernel_source finally: if moe_layer is not None: for parameter in moe_layer.parameters(): @@ -1299,10 +1354,57 @@ def _raise_if_unverified_moe_lane(moe_type: str) -> str: _check_compat(specifier, installed_version) for specifier in ("sglang>=0.5.14,<0.5.15", "sglang>=0.5.17,<0.5.18") ) + # 0.5.21 (tag commit e00930c5489053f26d86b179cee0d087f846acbb) re-verified + # for the Hopper/Ada lanes only (2026-10-04): + # * int4_wo: compressed_tensors.py:838-877 — CompressedTensorsMxInt4MoE + # only under an explicit flashinfer_trtllm runner, the Triton WNA16 MoE + # on SM100/103 auto or explicit triton, Marlin for every other CUDA + # path -> SM89/90 stay on Marlin (declared 89/90: marlin). SM100 auto + # moved from flashinfer_trtllm to Triton, so the SM100/103 declaration + # is NOT verified on this series. + # * w4a16_mxfp4 (GPT-OSS): no gpt-oss runner override exists in + # arg_groups/overrides.py any more (model_hook.py:524-528 still points + # at a `_gpt_oss_overrides` that is gone); Mxfp4MoEMethod resolves + # auto in create_moe_runner (mxfp4.py:1423-1430) to TRITON because + # use_triton_kernels is only set by an explicit triton_kernel backend + # (:382) -> SM89/90 declared `triton` matches serving. The SM100/103 + # flashinfer_mxfp4 declaration that 0.5.17 derived from the removed + # override is NOT verified on this series. + # * w4a8_mxfp4_mxfp8 (DSV4 FP4 experts, SM100-only): not re-verified. + if not verified and _check_compat("sglang>=0.5.21,<0.5.22", installed_version): + verified = moe_type in ("int4_wo", "w4a16_mxfp4") and get_sm_version() in (89, 90) + # SM100/103 w4a16_mxfp4 re-verified on 0.5.21 from the B200 identity probes (2026-10-04, + # benchmarks/evidence/collector/op-probe-2026-10/: results/sm100/sglang-0.5.21.yaml + # + results/retests/sm100/sglang-0.5.21.moe_auto.yaml): + # openai/gpt-oss-120b and gpt-oss-20b serve with Mxfp4MoEMethod -> MoeRunnerBackend.FLASHINFER_MXFP4 + # (kernel family trtllm_gen_moe) under the default render AND under an explicit AUTO, i.e. the + # flashinfer_mxfp4 declaration the cases carry for SM100/103 still matches serving even though the + # gpt-oss server-args override is gone (the selection now lives in Mxfp4MoEMethod itself). + if not verified and moe_type == "w4a16_mxfp4" and get_sm_version() in (100, 103): + verified = True + # w4a8_mxfp4_mxfp8 (DeepSeek-V4 native fp4 experts, SM100-only lane) re-verified the same way: the five + # deepseek-ai/DeepSeek-V4* checkpoints serve on B200 with MoeRunnerBackend.FLASHINFER_MXFP4 (trtllm_gen_moe) + # under AUTO (results/retests/sm100/sglang-0.5.21.moe_auto.yaml); the harness's marlin prescription is the + # SM90 workaround ("fp4-experts on SM90: runner=auto crashes"), not the Blackwell path. + if not verified and moe_type == "w4a8_mxfp4_mxfp8" and get_sm_version() in (100, 103): + verified = True + # int4_wo on SM100/103: the source reading above ("SM100 auto moved to Triton") is contradicted + # by the B200 identity records of the same day, which the guard update (fd07bf6b, done on SM90) + # never consumed: moonshotai/Kimi-K2.5 (compressed-tensors W4A16) serves on B200 / 0.5.21 as + # CompressedTensorsFusedMoE -> trtllm_gen_moe under the default render + # (results/sm100/sglang-0.5.21.yaml) AND as moe_runner FLASHINFER_TRTLLM under an explicit AUTO + # (results/retests/sm100/sglang-0.5.21.moe_auto.yaml), i.e. the flashinfer_trtllm declaration the + # SM100/103 cases carry still matches serving. Hardware evidence outranks the source claim + # (harness rule: no claim from reading code). Re-opened 2026-10-06 after the b200_sxm shard run + # (GitLab job 469988017) rejected all 3,078 int4_wo cases on this guard alone. + if not verified and moe_type == "int4_wo" and get_sm_version() in (100, 103): + verified = True if not verified: raise RuntimeError( f"SGLang {moe_type} collection is verified only for the 0.5.14 and 0.5.17 series " - f"(installed: {installed_version}); re-verify framework dispatch before extending this guard." + f"(and 0.5.21 on SM89/SM90/SM100/SM103 for int4_wo / w4a16_mxfp4, SM100/SM103 for " + f"w4a8_mxfp4_mxfp8); installed: {installed_version}, SM{get_sm_version()}; " + "re-verify framework dispatch before extending this guard." ) return installed_version @@ -1314,7 +1416,7 @@ def _triton_config_namespaces() -> list: def run_moe_torch( moe_type, - num_tokens, + num_tokens_list, hidden_size, inter_size, topk, @@ -1346,6 +1448,11 @@ def run_moe_torch( ): torch.cuda.set_device(device) torch.set_default_device(device) + # One case carries the whole token sweep; a single int is accepted for fills and + # direct callers that re-measure one point. + num_tokens_list = [int(num_tokens_list)] if isinstance(num_tokens_list, int) else [int(n) for n in num_tokens_list] + if not num_tokens_list: + raise ValueError("SGLang MoE case has an empty num_tokens_list") assert moe_type in [ "fp8_block", @@ -1388,14 +1495,25 @@ def run_moe_torch( and (hidden_size % 8 != 0 or local_inter_size % 8 != 0) ): raise ValueError( - "SGLang FlashInfer CUTLASS BF16 MoE requires hidden_size and local_inter_size " + "FIXME(kernel-limit): SGLang FlashInfer CUTLASS BF16 MoE requires hidden_size and local_inter_size " f"to be divisible by 8, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) activation_vector_size = 16 if sm_version >= 100 else 8 - if moe_backend == "triton" and is_gated and activation == "gelu" and local_inter_size % activation_vector_size != 0: + if ( + moe_backend == "triton" + and is_gated + and activation in ("gelu", "silu") + and local_inter_size % activation_vector_size != 0 + ): + # sgl-kernel JIT silu_and_mul / gelu_and_mul: activation.cuh:186 + # RuntimeCheck(hidden_size % kVecSize == 0, "hidden size must be divisible + # by vector size"), kVecSize = kMaxVecBytes / sizeof(T) (16 bf16 on SM100+, + # 8 below). 2026-10-11: silu hit it on b200/gb300 (1,053 each) and rtxpro + # (1,377) at local_inter_size 168/84/40 (Nemotron-3-Super inter 2688 at TP16+). raise ValueError( - "SGLang Triton gated BF16 GELU requires local_inter_size " - f"to be divisible by {activation_vector_size}, got local_inter_size={local_inter_size}" + "FIXME(kernel-limit): SGLang Triton gated BF16 GELU/SiLU (sgl-kernel act_and_mul, " + f"activation.cuh:186) requires local_inter_size to be divisible by {activation_vector_size}, " + f"got local_inter_size={local_inter_size} (activation={activation})" ) if ( moe_type == "w4a16_mxfp4" @@ -1405,7 +1523,7 @@ def run_moe_torch( and (hidden_size % 128 != 0 or local_inter_size % 128 != 0) ): raise ValueError( - "SGLang SM90 DeepSeek-V4 W4A16 FP4 experts require hidden_size and local_inter_size " + "FIXME(kernel-limit): SGLang SM90 DeepSeek-V4 W4A16 FP4 experts require hidden_size and local_inter_size " f"to be divisible by 128, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) if ( @@ -1428,6 +1546,7 @@ def run_moe_torch( # runtime series. (The separate serialized-MXFP4 Mxfp4MoEMethod does # pad, but is not the is_fp4_experts path guarded here.) raise ValueError( + "FIXME(kernel-limit): " "SGLang SM100/103 DeepSeek-V4 W4A8 FP4 experts require hidden_size and local_inter_size " f"to be divisible by 128, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) @@ -1459,7 +1578,7 @@ def run_moe_torch( int4_group_size = int(int4_config.get("group_size", 128)) if hidden_size % int4_group_size != 0 or local_inter_size % int4_group_size != 0: raise ValueError( - "SGLang INT4-WO group quantization requires hidden_size and local_inter_size " + "FIXME(kernel-limit): SGLang INT4-WO group quantization requires hidden_size and local_inter_size " f"to be divisible by group_size={int4_group_size}, got " f"hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) @@ -1467,7 +1586,7 @@ def run_moe_torch( elif moe_type == "fp8_block": if moe_backend == "triton" and (hidden_size % 128 != 0 or local_inter_size % 128 != 0): raise ValueError( - "SGLang Triton fp8_block requires hidden_size and local_inter_size " + "FIXME(kernel-limit): SGLang Triton fp8_block requires hidden_size and local_inter_size " f"to be divisible by 128, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) block_shape = [128, 128] @@ -1486,15 +1605,68 @@ def run_moe_torch( or apply_router_weight_on_input or (moe_backend != "triton" and not use_int4_w4a16) ) - # Record (never choose) whether the timed Triton call used a tuned JSON or + distribution = "power_law_" + str(power_law_alpha) if distributed == "power_law" else distributed + + # Record (never choose) whether each timed Triton call used a tuned JSON or # SGLang's default config; both the framework runner and the direct path - # resolve it through the same two config functions. + # resolve it through the same two config functions. One task times a whole + # num_tokens series, so the counters cover one point at a time: _persist + # hands them to the sidecar with that point's row and resets them. with observe_triton_config_selection(_triton_config_namespaces()) as triton_config_observed: + + def _persist(num_tokens: int, latency: float, power_stats: dict, kernel_source: str) -> None: + persisted = log_perf( + item_list=[ + { + "moe_dtype": moe_type, + "num_tokens": num_tokens, + "hidden_size": hidden_size, + "inter_size": inter_size, + "topk": topk, + "num_experts": num_experts, + "moe_tp_size": moe_tp_size, + "moe_ep_size": moe_ep_size, + "distribution": distribution, + "latency": latency, + } + ], + framework="SGLang", + version=installed_sglang_version, + device_name=torch.cuda.get_device_name(device), + op_name="moe", + kernel_source=kernel_source, + perf_filename=perf_filename, + power_stats=power_stats, + ) + if not persisted: + raise RuntimeError("Failed to persist SGLang MoE performance row") + point_observed = dict(triton_config_observed) + for key in triton_config_observed: + triton_config_observed[key] = 0 + if moe_backend == "triton": + append_triton_config_record( + perf_filename, + { + "moe_dtype": moe_type, + "num_tokens": num_tokens, + "hidden_size": hidden_size, + "inter_size": inter_size, + "topk": topk, + "num_experts": num_experts, + "moe_tp_size": moe_tp_size, + "moe_ep_size": moe_ep_size, + "device_name": torch.cuda.get_device_name(device), + }, + kernel_source, + point_observed, + ) + if use_framework_layer: - latency, power_stats, kernel_source = _benchmark_framework_quantized_moe( + _benchmark_framework_quantized_moe( moe_type=moe_type, moe_backend=moe_backend, - num_tokens=num_tokens, + num_tokens_list=num_tokens_list, + on_result=_persist, hidden_size=hidden_size, inter_size=inter_size, topk=topk, @@ -1523,103 +1695,60 @@ def run_moe_torch( device=device, ) else: - rank0_workloads: list[Rank0Workload] | None = None - if moe_ep_size > 1 and distributed in ("power_law", "balanced"): - rank0_workloads = build_rank0_workloads( - num_workloads=5, - num_tokens=num_tokens, - hidden_size=hidden_size, - topk=topk, - num_experts=num_experts, - moe_ep_size=moe_ep_size, - distributed=distributed, - power_law_alpha=power_law_alpha if distributed == "power_law" else None, - dtype=torch.bfloat16, - device=torch.device(device), - ) + kernel_source = { + "triton": "sglang_fused_moe_triton", + "triton_kernel": "sglang_triton_kernel_moe", + "marlin": "sglang_marlin_moe", + "flashinfer_trtllm": "sglang_flashinfer_trtllm_moe", + "flashinfer_cutlass": "sglang_flashinfer_cutlass_moe", + }[moe_backend] try: - latency, power_stats = benchmark( - num_tokens, - num_local_experts, - (2 if is_gated else 1) * local_inter_size, - hidden_size, - topk, - torch.bfloat16, - moe_type == "fp8_block", - False, - False, - block_shape=block_shape, - distributed=distributed, - power_law_alpha=power_law_alpha, - workloads=rank0_workloads, - swiglu_limit=swiglu_limit, - activation=activation, - is_gated=is_gated, - gemm1_alpha=gemm1_alpha, - gemm1_clamp_limit=gemm1_clamp_limit, - ) + for num_tokens in num_tokens_list: + rank0_workloads: list[Rank0Workload] | None = None + if moe_ep_size > 1 and distributed in ("power_law", "balanced"): + rank0_workloads = build_rank0_workloads( + num_workloads=5, + num_tokens=num_tokens, + hidden_size=hidden_size, + topk=topk, + num_experts=num_experts, + moe_ep_size=moe_ep_size, + distributed=distributed, + power_law_alpha=power_law_alpha if distributed == "power_law" else None, + dtype=torch.bfloat16, + device=torch.device(device), + ) + latency, power_stats = benchmark( + num_tokens, + num_local_experts, + (2 if is_gated else 1) * local_inter_size, + hidden_size, + topk, + torch.bfloat16, + moe_type == "fp8_block", + False, + False, + block_shape=block_shape, + distributed=distributed, + power_law_alpha=power_law_alpha, + workloads=rank0_workloads, + swiglu_limit=swiglu_limit, + activation=activation, + is_gated=is_gated, + gemm1_alpha=gemm1_alpha, + gemm1_clamp_limit=gemm1_clamp_limit, + ) + _persist(num_tokens, latency, power_stats, kernel_source) except Exception: _fmoe_kernels_mod._B_DESC_CACHE.clear() gc.collect() torch.cuda.empty_cache() raise - - if not use_framework_layer: - _fmoe_kernels_mod._B_DESC_CACHE.clear() - gc.collect() - torch.cuda.empty_cache() + _fmoe_kernels_mod._B_DESC_CACHE.clear() + gc.collect() + torch.cuda.empty_cache() restart_worker = torch.cuda.memory_allocated(device) > torch.cuda.get_device_properties(device).total_memory // 4 - if not use_framework_layer: - kernel_source = { - "triton": "sglang_fused_moe_triton", - "triton_kernel": "sglang_triton_kernel_moe", - "marlin": "sglang_marlin_moe", - "flashinfer_trtllm": "sglang_flashinfer_trtllm_moe", - "flashinfer_cutlass": "sglang_flashinfer_cutlass_moe", - }[moe_backend] - persisted = log_perf( - item_list=[ - { - "moe_dtype": moe_type, - "num_tokens": num_tokens, - "hidden_size": hidden_size, - "inter_size": inter_size, - "topk": topk, - "num_experts": num_experts, - "moe_tp_size": moe_tp_size, - "moe_ep_size": moe_ep_size, - "distribution": "power_law_" + str(power_law_alpha) if distributed == "power_law" else distributed, - "latency": latency, - } - ], - framework="SGLang", - version=installed_sglang_version, - device_name=torch.cuda.get_device_name(device), - op_name="moe", - kernel_source=kernel_source, - perf_filename=perf_filename, - power_stats=power_stats, - ) - if not persisted: - raise RuntimeError("Failed to persist SGLang MoE performance row") - if moe_backend == "triton": - append_triton_config_record( - perf_filename, - { - "moe_dtype": moe_type, - "num_tokens": num_tokens, - "hidden_size": hidden_size, - "inter_size": inter_size, - "topk": topk, - "num_experts": num_experts, - "moe_tp_size": moe_tp_size, - "moe_ep_size": moe_ep_size, - "device_name": torch.cuda.get_device_name(device), - }, - kernel_source, - triton_config_observed, - ) if restart_worker: return WORKER_RESTART diff --git a/python/aisimulate/collector/sglang/collect_msa_module.py b/python/aisimulate/collector/sglang/collect_msa_module.py index 30f66baa17..5a1d89cf2d 100644 --- a/python/aisimulate/collector/sglang/collect_msa_module.py +++ b/python/aisimulate/collector/sglang/collect_msa_module.py @@ -78,10 +78,12 @@ get_mla_module_sweep_spec, ) from collector.helper import ( + PhaseClock, _resolve_local_model_path, benchmark_with_power, get_sm_version, log_perf, + render_phase_totals, ) try: @@ -136,6 +138,10 @@ # All three are emulated on one GPU by shrinking the config head counts. +# Per-subprocess sum of the point phase times printed by run_msa_module. +_POINT_PHASE_TOTALS: dict[str, float] = {} + + def _ceil_div(a: int, b: int) -> int: return (a + b - 1) // b @@ -323,17 +329,31 @@ def _generation_shapes(max_pos: int | None): _M3_NATIVE_GQA_RATIO = 16 # 64 q / 4 kv heads (bundled config) +def _loaded_num_layers() -> int: + """Layers the worker loads (SGLANG_TEST_NUM_LAYERS, default 2) — every one of + them owns a slot in the main pool and the index-K pool, so the plan-time + footprint is per token x layers (memory_pool.py:5404 MiniMaxSparseKVPool + builds one MHA pool over start_layer..end_layer plus one index pool per + sparse layer @0.5.21).""" + return int(os.environ.get("SGLANG_TEST_NUM_LAYERS", "2")) + + def _estimated_kv_bytes_per_token(num_heads: int, kv_cache_dtype: str) -> int: - """Upper-bound bytes/token of the M3 pools at this per-GPU head count: - main paged K+V (kv-head sharded; fp8 stores 1 B/elem, else bf16) plus the - index-K side cache (always model dtype, kv_cache_configurator.py:1246 - index_dtype=model_dtype @v0.5.16).""" + """Upper-bound bytes/token of the M3 pools at this per-GPU head count, for + ALL loaded layers: main paged K+V (kv-head sharded; fp8 stores 1 B/elem, else + bf16) plus the index-K side cache (model dtype unless fp8 attn-GEMM, + memory_pool.py:5392 get_minimax_sparse_index_dtype @0.5.21; index_dtype= + model_dtype @v0.5.16 kv_cache_configurator.py:1246). The 0.5.16-era + estimate priced one layer while the worker loads two: the three + largest-batch decode cells (b=256/512/1024 x 131072/65536/32768) passed the + plan filter and then failed the worker's execute-or-raise capacity check + (33.6M planned tokens > 24.5M real pool, H20 0.5.21 smoke 2026-10-01).""" kv_heads = max(1, num_heads // _M3_NATIVE_GQA_RATIO) main_elem = 1 if kv_cache_dtype == "fp8" else 2 main = 2 * kv_heads * _M3_HEAD_DIM * main_elem index_heads = min(_M3_INDEX_HEADS, num_heads) index = index_heads * _M3_INDEX_DIM * 2 - return main + index + return (main + index) * _loaded_num_layers() def _plan_memory_filter(shapes, *, num_heads: int, kv_cache_dtype: str, is_prefill: bool): @@ -400,11 +420,12 @@ def _build_module_test_cases(mode: str): empty_groups = 0 for model_spec in model_specs: for compute_dtype, kv_dtype, gemm_type in _get_precision_combos(mode): - for target_tp in sweep.module_tp_sizes: + # the model row's declared shards win over the sweep's module_tp_sizes + for target_tp in model_spec.tensor_parallel_sizes or sweep.module_tp_sizes: if model_spec.native_num_heads % target_tp != 0: continue num_heads = model_spec.native_num_heads // target_tp - if num_heads not in sweep.inner_sweep_head_counts: + if model_spec.tensor_parallel_sizes is None and num_heads not in sweep.inner_sweep_head_counts: continue batch_sizes = sweep.context_batch_sizes if mode == "context" else [0] for batch_size in batch_sizes: @@ -656,11 +677,7 @@ def load_model_runner( (server_args.py _handle_gpu_memory_settings) and would go negative. """ from sglang.srt.configs.model_config import ModelConfig - from sglang.srt.distributed.parallel_state_wrapper import ParallelState from sglang.srt.entrypoints.engine import _set_envs_and_config - from sglang.srt.layers.moe import initialize_moe_config - from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config - from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config from sglang.srt.model_executor.model_runner import ModelRunner from sglang.srt.server_args import ServerArgs from sglang.srt.utils import suppress_other_loggers @@ -725,9 +742,6 @@ def load_model_runner( ) _set_envs_and_config(server_args) - initialize_moe_config(server_args) - initialize_fp8_gemm_config(server_args) - initialize_fp4_gemm_config(server_args) model_config = ModelConfig.from_server_args(server_args) actual_architecture = (model_config.hf_config.architectures or [None])[0] @@ -745,12 +759,15 @@ def load_model_runner( _sock.bind(("127.0.0.1", 0)) nccl_port = _sock.getsockname()[1] + from collector.sglang.runtime_compat import init_runtime_config + + _runner_parallel_kwargs = init_runtime_config(server_args, gpu_id, nccl_port=nccl_port, model_config=model_config) model_runner = ModelRunner( model_config=model_config, mem_fraction_static=server_args.mem_fraction_static, gpu_id=gpu_id, - ps=ParallelState.trivial(gpu_id=gpu_id), nccl_port=nccl_port, + **_runner_parallel_kwargs, server_args=server_args, ) model_runner.alloc_memory_pool() @@ -840,7 +857,13 @@ def _decode_graph_covered(model_runner, num_tokens: int) -> bool: graph: cuda_graph_config.decode coverage (backend != disabled; bs list or max_bs — resolved per GPU tier by server_args _handle_gpu_memory_settings, e.g. 256 on the SM90 validation node at tp<4).""" - decode_cfg = model_runner.server_args.cuda_graph_config.decode + cuda_graph_config = model_runner.server_args.cuda_graph_config + if cuda_graph_config is None: + # sglang>=0.5.21: derived config lives behind the resolving view (arg_groups/model_override_base.py:120) + from sglang.srt.arg_groups.model_override_base import resolving_view + + cuda_graph_config = resolving_view(model_runner.server_args).cuda_graph_config + decode_cfg = cuda_graph_config.decode if decode_cfg.backend == "disabled": return False if decode_cfg.bs: @@ -996,6 +1019,7 @@ def _run_prefill_point( """ from sglang.srt.model_executor.forward_batch_info import ForwardBatch + point_clock = PhaseClock() model_runner.req_to_token_pool.clear() model_runner.token_to_kv_pool_allocator.clear() @@ -1010,6 +1034,7 @@ def _run_prefill_point( batch = _make_schedule_batch(model_runner, reqs) with temporarily_chunked_alloc_extend(model_runner, batch_size * seq_len): batch.prepare_for_extend() + point_clock.mark("batch") # return_hidden_states_before_norm=False mirrors the serving worker # (managers/tp_worker.py:267,548@v0.5.16). forward_batch = ForwardBatch.init_new(batch, model_runner, return_hidden_states_before_norm=False) @@ -1019,6 +1044,7 @@ def _run_prefill_point( # minimax_sparse_backend.py:505-507, base_attn_backend.py:48-54). with _forward_ctx(model_runner): model_runner.attn_backend.init_forward_metadata(forward_batch) + point_clock.mark("metadata") hidden_states = torch.randn( batch_size * seq_len, @@ -1033,6 +1059,7 @@ def _run_prefill_point( .contiguous() .flatten() ) + point_clock.mark("inputs") def kernel_func(): with _forward_ctx(model_runner): @@ -1053,11 +1080,16 @@ def kernel_func(): use_cuda_graph=False, ) as results: pass + point_clock.mark("bench") + point_clock.absorb(results.get("phase_seconds"), prefix="bench_") return results["latency_ms"], results["power_stats"] finally: model_runner.req_to_token_pool.clear() model_runner.token_to_kv_pool_allocator.clear() torch.cuda.empty_cache() + point_clock.mark("cleanup") + point_clock.add_to(_POINT_PHASE_TOTALS) + print(f" [MSA point timing] context b={batch_size}, s={seq_len}, prefix={prefix_len}: {point_clock.render()}") def _run_decode_point( @@ -1083,6 +1115,7 @@ def _run_decode_point( """ from sglang.srt.model_executor.forward_batch_info import ForwardBatch + point_clock = PhaseClock() model_runner.req_to_token_pool.clear() model_runner.token_to_kv_pool_allocator.clear() @@ -1094,6 +1127,7 @@ def _run_decode_point( req.output_ids.append(0) batch.prepare_for_decode() forward_batch = ForwardBatch.init_new(batch, model_runner, return_hidden_states_before_norm=False) + point_clock.mark("batch") use_graph = _decode_graph_covered(model_runner, batch_size) with _forward_ctx(model_runner): @@ -1108,6 +1142,7 @@ def _run_decode_point( # the serving decode graph runner owns — the module bench never # executes a dense layer, so its eager metadata is kept. model_runner.attn_backend.sparse.init_forward_metadata_out_graph(forward_batch, in_capture=True) + point_clock.mark("metadata") hidden_states = torch.randn( batch_size, @@ -1116,6 +1151,7 @@ def _run_decode_point( device="cuda", ) positions = torch.full((batch_size,), kv_len, device="cuda", dtype=torch.int64) + point_clock.mark("inputs") def kernel_func(): with _forward_ctx(model_runner): @@ -1136,11 +1172,16 @@ def kernel_func(): use_cuda_graph=use_graph, ) as results: pass + point_clock.mark("bench") + point_clock.absorb(results.get("phase_seconds"), prefix="bench_") return results["latency_ms"], results["power_stats"], use_graph finally: model_runner.req_to_token_pool.clear() model_runner.token_to_kv_pool_allocator.clear() torch.cuda.empty_cache() + point_clock.mark("cleanup") + point_clock.add_to(_POINT_PHASE_TOTALS) + print(f" [MSA point timing] generation b={batch_size}, kv={kv_len}: {point_clock.render()}") # ═══════════════════════════════════════════════════════════════════════ @@ -1342,6 +1383,7 @@ def run_msa_module( summary = f"ok={ok} error={len(failures)} skip=0 total={len(shapes)}" print(f"[MSA] {phase} {summary}") + print(f"[MSA timing] {phase}: {render_phase_totals(_POINT_PHASE_TOTALS)}") if ok == 0: raise RuntimeError(f"MSA module {phase} persisted no rows; {summary}") if failures: diff --git a/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py b/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py index 54083fc9bf..b6fec83881 100644 --- a/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py +++ b/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py @@ -5,35 +5,38 @@ Collects the DeepSeek-V4 NSA sparse-attention sub-kernels at the kernel level. DeepSeek-V4's primary AIC path uses prefix-aware full CSA/HCA attention-module -rows; paged_mqa_logits / csa_attn / hca_attn here are supporting kernel-level -data for prefix/past_kv correction and residual analysis, while the topk_calib -DELTA is actively consumed by perf_database's topK correction. +rows; paged_mqa_logits here is supporting kernel-level data for the SDK CP +chunk walk, while the topk_calib DELTA is actively consumed by +perf_database's topK correction. -The four sub-kernels form ONE sparse-op family — the CSA/HCA sparse path in +The two sub-kernels form ONE sparse-op family — the CSA sparse path in order — all collected, none modeled analytically: 1. ``deep_gemm.fp8_paged_mqa_logits`` CSA indexer scoring 2. ``topk_transform`` (topk_512/_1024) CSA selection — phase-specific v1/v2 flat-vs-top_last DELTA calib - 3. ``flash_mla_with_kvcache`` (csa_attn) CSA sparse FMLA over the - topk-selected c4 positions - 4. ``flash_mla_with_kvcache`` (hca_attn) HCA c128 sparse FMLA + +The standalone FMLA sidecars (``flash_mla_with_kvcache`` as csa_attn / hca_attn) +were removed 2026-10-15: their SDK readers were retired design placeholders with +no query path (csa_attn never shipped data) and, as registry-only ops, they +failed every shard plan generated from the registry. Inputs are SAME-SOURCE: every sub-kernel derives its ``(prefix, isl, bs[, tp])`` -shapes STRICTLY 1:1 from the CSA/HCA attention-module CSV it belongs to -(paged_mqa / topk / csa_attn ← CSA module; hca_attn ← HCA module) — no separate -sweep grid — benched at upstream layouts (DeepGEMM ``test_attention.py`` for the -indexer GEMM, FlashMLA ``MODEL1_FP8Sparse`` quant for the FMLA kernel). +shapes STRICTLY 1:1 from the CSA attention-module INPUT sweep it belongs to — no +separate sweep grid — benched at the upstream layout (DeepGEMM +``test_attention.py`` for the indexer GEMM). CSV schema matches existing aic dsv4 module CSVs (so loaders can be shared): -``isl`` carries M, ``step`` carries past_kv, ``compress_ratio`` distinguishes -CSA(=4) / HCA(=128). +``isl`` carries M, ``step`` carries past_kv, ``compress_ratio`` carries the CSA +c4 path (=4). """ # Requires stock SGLang 0.5.14 with its matching ``sgl-kernel`` package. from __future__ import annotations -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" import functools import json @@ -63,10 +66,10 @@ def _dsv4_sparse_kernel_cases(kernel, models=None): # One task per (model, bs); collect.py spreads bs across GPU workers (no - # single-worker cuda-graph buildup). All sparse kernels run a single fixed - # head config: the FMLA pads to its required head count OUTSIDE the kernel - # (model TP zero-pad), so the kernel is TP-independent -> one config per bs, - # no tp sweep. Each task sweeps (isl, prefix) for its bs. + # single-worker cuda-graph buildup). The sparse kernels run a single fixed + # head config (the indexer heads are not TP-sharded), so each is + # TP-independent -> one config per bs, no tp sweep. Each task sweeps + # (isl, prefix) for its bs. if models is None: try: from collector.case_generator import _selected_dsv4_models @@ -87,24 +90,12 @@ def get_dsv4_paged_mqa_logits_test_cases(): return _dsv4_sparse_kernel_cases("paged_mqa_logits") -def get_dsv4_hca_attn_test_cases(): - return _dsv4_sparse_kernel_cases("hca_attn") - - -def get_dsv4_csa_attn_test_cases(): - return _dsv4_sparse_kernel_cases("csa_attn") - - get_dsv4_flash_paged_mqa_logits_test_cases = get_dsv4_paged_mqa_logits_test_cases -get_dsv4_flash_hca_attn_test_cases = get_dsv4_hca_attn_test_cases __all__ = [ "DEFAULT_MODEL", - "get_dsv4_csa_attn_test_cases", - "get_dsv4_flash_hca_attn_test_cases", "get_dsv4_flash_paged_mqa_logits_test_cases", - "get_dsv4_hca_attn_test_cases", "get_dsv4_paged_mqa_logits_test_cases", "get_dsv4_topk_calib_test_cases", "run_dsv4_sparse_kernel_worker", @@ -180,25 +171,6 @@ def _dsv4_sparse_config(model_path: str) -> SimpleNamespace: ) -# ═══════════════════════════════════════════════════════════════════════ -# DeepSeek-V4 sparse architectural constants -# ═══════════════════════════════════════════════════════════════════════ -# Shape fields below are identical across DSV4 variants (Flash/Pro), so the -# default model config is authoritative; read them via the sparse config. -_DEFAULT_SC = _dsv4_sparse_config(DEFAULT_MODEL) -V_HEAD_DIM = _DEFAULT_SC.head_dim # 512 -FMLA_D_QK = V_HEAD_DIM # FlashMLA d_qk == head_dim (NOT 576) -FMLA_D_ROPE = _DEFAULT_SC.d_rope # 64 -FMLA_D_NOPE = _DEFAULT_SC.d_nope # 448 - -# Kernel-layout invariants — NOT in the model config. The FlashMLA -# MODEL1_FP8Sparse fp8 quant tile size and the deep_gemm / FlashMLA paged block -# size are hardcoded in the kernels themselves (deep_gemm blocksize 64; -# flashmla_backend.PAGE_SIZE = 64), so they are fixed constants here. -# bytes_per_token = d_nope + d_rope*2 + num_tiles + 1 pad = 448 + 128 + 7 + 1 = 584 -FMLA_TILE_SIZE = 64 -FMLA_NUM_TILES = FMLA_D_NOPE // FMLA_TILE_SIZE # 448 / 64 = 7 - DEFAULT_ARCHITECTURE = "DeepseekV4ForCausalLM" @@ -224,49 +196,36 @@ def _device_num_sms(device: str | torch.device) -> int: return torch.cuda.get_device_properties(device).multi_processor_count -# The DSV4 sparse-op family — all four collected by ONE worker +# The DSV4 sparse-op family — both collected by ONE worker # (run_dsv4_sparse_kernel_worker), all driven by the KERNEL_* maps below, all -# same-source (shapes read 1:1 from the CSA/HCA attention-module CSV they belong -# to), all writing the SAME row schema (one ``latency`` per row): +# same-source (shapes derived 1:1 from the CSA attention-module INPUT sweep), +# all writing the SAME row schema (one ``latency`` per row): # - paged_mqa_logits: CSA indexer scoring (deep_gemm) -> 1 row/shape # - topk: CSA selection — context runs v1 and generation runs v2, # each under two score distributions, emitted as TWO # rows/shape (score_mode=v1_flat | v1_top_last or # v2_flat | v2_top_last); perf_database takes the matching # DELTA = flat.latency - top_last.latency -> 2 rows/shape -# - csa_attn: CSA's flash_mla over the topk-selected c4 -# positions (K_per_query = min(index_topk, full_s//4)) -# - hca_attn: HCA's flash_mla over c128 cache. -# (production DSV4 at TP>1 also runs FlashMLA with -# the native h_q — sglang pads Q to full native heads, -# then slices output back; see deepseek_v4.py:847. So -# TP=1 data is valid for any deployment TP.) KERNEL_TO_OP_NAME = { "paged_mqa_logits": "dsv4_paged_mqa_logits_module", "topk": "dsv4_csa_topk_calib", - "hca_attn": "dsv4_hca_attn_module", - "csa_attn": "dsv4_csa_attn_module", } KERNEL_TO_KERNEL_SOURCE = { "paged_mqa_logits": "deep_gemm.fp8_paged_mqa_logits", - "hca_attn": "flash_mla_with_kvcache", - "csa_attn": "flash_mla_with_kvcache", } -# compress_ratio: 4 for the CSA c4 path (indexer/topk/csa_attn), 128 for HCA c128. +# compress_ratio: 4 for the CSA c4 path (indexer/topk). KERNEL_TO_COMPRESS_RATIO = { "paged_mqa_logits": 4, "topk": 4, - "hca_attn": 128, - "csa_attn": 4, } # Each sparse sub-kernel derives its (prefix, isl, bs) shapes from its owning -# attention module's INPUT sweep (paged_mqa/topk/csa_attn <- CSA, hca_attn <- -# HCA) via _dsv4_context_derived_shapes / _dsv4_generation_derived_shapes — -# NOT from the already-collected module CSVs. -# ALL four sparse kernels are TP-independent — the indexer/FMLA latency does not +# attention module's INPUT sweep (paged_mqa/topk <- CSA) via +# _dsv4_context_derived_shapes / _dsv4_generation_derived_shapes — NOT from the +# already-collected module CSVs. +# Both sparse kernels are TP-independent — the indexer latency does not # change with tp (the module's per-tp rows differ only by the attention module's # comms/sharding, not the sparse sub-kernel). So each is benched ONCE per unique # (prefix, isl, bs) module shape and written one row per shape (tp_size=1), @@ -326,7 +285,7 @@ def timed_kernel(): # ═══════════════════════════════════════════════════════════════════════ -# Cache packing helpers (mirror DeepGEMM/FlashMLA test code) +# Cache packing helpers (mirror DeepGEMM test code) # ═══════════════════════════════════════════════════════════════════════ @@ -348,65 +307,6 @@ def _kv_cache_cast_to_fp8_indexer(x: torch.Tensor) -> torch.Tensor: return out.view(num_blocks, block_size, 1, head_dim + 4) -def _quantize_k_cache_model1(k_bf16: torch.Tensor) -> torch.Tensor: - """FlashMLA MODEL1_FP8Sparse pack (DSV4 sparse layout). - - k_bf16: (num_blocks, block_size, 1, d_qk=512) bf16 - out: (num_blocks, block_size, 1, bytes_per_token=584) packed fp8 - - Layout per token (inside the bytes_per_token slab): - [d_nope=448 fp8 nope][d_rope*2=128 bf16 rope][num_tiles=7 fp8 e8m0 sf][1 pad] - """ - num_blocks, block_size, num_heads, d_qk = k_bf16.shape - assert num_heads == 1 and d_qk == FMLA_D_QK - k = k_bf16.squeeze(2) # (num_blocks, block_size, d_qk) - - bytes_per_token = FMLA_D_NOPE + 2 * FMLA_D_ROPE + FMLA_NUM_TILES + 1 # 584 - size_per_block_padded = (block_size * bytes_per_token + 576 - 1) // 576 * 576 - # Allocate padded (so memory layout matches kernel TMA expectations) but - # SLICE back to exact ``block_size*bytes_per_token`` so the final view has - # last-dim = bytes_per_token (kernel asserts on this). - out = torch.empty((num_blocks, size_per_block_padded), dtype=torch.float8_e4m3fn, device=k_bf16.device) - out_view = out[:, : block_size * bytes_per_token] - - # nope+rope region - nope_rope_part = out_view[:, : block_size * (FMLA_D_NOPE + 2 * FMLA_D_ROPE)].view( - num_blocks, block_size, FMLA_D_NOPE + 2 * FMLA_D_ROPE - ) - nope = nope_rope_part[:, :, :FMLA_D_NOPE] - rope = nope_rope_part[:, :, FMLA_D_NOPE:].view(k_bf16.dtype) - - sf = ( - out_view[:, block_size * (FMLA_D_NOPE + 2 * FMLA_D_ROPE) :] - .view(num_blocks, block_size, 8)[:, :, :7] - .view(torch.float8_e8m0fnu) - ) - - rope[:] = k[..., FMLA_D_NOPE:] - for tile_idx in range(FMLA_NUM_TILES): - s, e = tile_idx * FMLA_TILE_SIZE, (tile_idx + 1) * FMLA_TILE_SIZE - scale_inv = (k[..., s:e].abs().float().amax(dim=-1).float() / 448.0).clamp_min(1e-4) - scale_inv = torch.pow(2, scale_inv.log2().ceil()).to(torch.float32) - sf[:, :, tile_idx] = scale_inv.to(torch.float8_e8m0fnu) - - scale_inv = scale_inv.unsqueeze(-1) - nope[:, :, s:e] = (k[..., s:e].float() / scale_inv.float()).to(torch.float8_e4m3fn) - - # Return a view whose stride(0) is the padded per-block size — FlashMLA's - # MODEL1 path asserts ``k_cache.stride(0) % TMA_K_STRIDE == 0`` (with - # TMA_K_STRIDE = D_NOPE + 2*D_ROPE = 576), and ``bytes_per_token * block_size`` - # = 37376 is *not* a multiple of 576, so we need the per-block padding to be - # visible in the tensor's stride. ``.view`` collapses stride(0) to the - # contiguous value when num_blocks == 1, breaking the assertion for small - # shapes; ``as_strided`` lets us pin stride(0) to size_per_block_padded for - # any num_blocks. - return out.as_strided( - size=(num_blocks, block_size, 1, bytes_per_token), - stride=(size_per_block_padded, bytes_per_token, bytes_per_token, 1), - storage_offset=0, - ) - - def _expand_block_table(num_blocks: int, batch_size: int, device) -> torch.Tensor: """Block list [0..num_blocks) replicated to (batch_size, num_blocks) int32. All requests share the same physical blocks (KV cache is shared in-bench).""" @@ -414,12 +314,6 @@ def _expand_block_table(num_blocks: int, batch_size: int, device) -> torch.Tenso return row.unsqueeze(0).expand(batch_size, num_blocks).contiguous() -def _expand_indices(k: int, batch_size: int, m: int, device) -> torch.Tensor: - """Sequential KV indices [0..k) replicated to (batch_size, m, k) int32.""" - base = torch.arange(k, dtype=torch.int32, device=device) - return base.view(1, 1, k).expand(batch_size, m, k).contiguous() - - # ═══════════════════════════════════════════════════════════════════════ # Kernel 1: deep_gemm.fp8_paged_mqa_logits # ═══════════════════════════════════════════════════════════════════════ @@ -440,7 +334,19 @@ def _bench_paged_mqa_logits( to ``batch_dim`` with ``next_n=1``. Per-request causal lengths remain distinct even though those token rows share one physical KV cache. """ - from deep_gemm import fp8_paged_mqa_logits, get_paged_mqa_logits_metadata + from deep_gemm import fp8_paged_mqa_logits + + try: + # sglang>=0.5.21 builds the paged-MQA schedule with its own JIT kernel + # (sglang::paged_mqa_metadata_tiny_kernel) whenever SGLANG_OPT_USE_JIT_INDEXER_METADATA + # is set (default True, environ.py:1570) or the query row count exceeds 11673 + # (dsv4/metadata.py:213-224, force_deep_gemm_metadata defaults False :184); the + # DeepGEMM sched kernel is only the opt-out path. Same (seq_lens, page_size, num_sm) + # contract (kernels/ops/attention/dsv4/attn.py:57). H20 gate 2026-10-01: the serving + # decode trace carries the tiny kernel, never deep_gemm::sched::sm90_paged_mqa_logits_metadata. + from sglang.kernels.ops.attention.dsv4.attn import get_paged_mqa_logits_metadata + except ImportError: + from deep_gemm import get_paged_mqa_logits_metadata if batch_size <= 0 or M % batch_size: raise ValueError(f"M={M} must be divisible by positive batch_size={batch_size}") @@ -481,173 +387,6 @@ def kernel_fn(): return _bench_cuda_graph(kernel_fn, allow_graph_fail=False, device=device) -# ═══════════════════════════════════════════════════════════════════════ -# Helpers for FlashMLA HCA -# ═══════════════════════════════════════════════════════════════════════ - - -def _build_flash_mla_inputs( - M: int, # noqa: N803 - past_kv: int, - *, - K_per_query: int, # noqa: N803 - batch_size: int, - n_local_heads: int, - device: str, -) -> tuple[torch.Tensor, ...]: - """Build fp8 paged K cache + Q + indices + scheduler metadata. - - Layout = MODEL1_FP8Sparse (DSV4 NSA): d_qk=512 with 584-byte fp8 cache. - """ - M_per_req = M // batch_size if batch_size > 1 else M # noqa: N806 - # Per-request sequence length: each of the ``batch_size`` requests has - # ``M_per_req`` new query tokens over ``past_kv`` prefix (NOT the flattened - # total M), so cache_seqlens / indices mirror the owning module - # (prefix, isl, bs) row 1:1 instead of one giant bs*isl request. - full_s = M_per_req + past_kv - K_per_query = max(min(K_per_query, full_s), 1) # noqa: N806 - - # Q: (batch, M_per_req, n_local_heads, FMLA_D_QK=512) bf16 - q = torch.randn(batch_size, M_per_req, n_local_heads, FMLA_D_QK, dtype=torch.bfloat16, device=device) - - # K cache: SHARED across batch entries to avoid b-fold blowup. - blocks_per_req = (full_s + _dsv4_kv_page_size() - 1) // _dsv4_kv_page_size() - k_bf16 = torch.randn(blocks_per_req, _dsv4_kv_page_size(), 1, FMLA_D_QK, dtype=torch.bfloat16, device=device) - k_cache = _quantize_k_cache_model1(k_bf16) - - block_table = _expand_block_table(blocks_per_req, batch_size, device) - - # indices_in_kvcache: (batch, M_per_req, K_per_query) int32 — first K_per_query positions per Q - indices = _expand_indices(K_per_query, batch_size, M_per_req, device) - - # Pad indices to multiple of 64 (FlashMLA assertion) - if K_per_query % 64 != 0: - pad = 64 - K_per_query % 64 - pad_t = torch.full((batch_size, M_per_req, pad), -1, dtype=torch.int32, device=device) - indices = torch.cat([indices, pad_t], dim=-1).contiguous() - - cache_seqlens = torch.full((batch_size,), full_s, dtype=torch.int32, device=device) - - return q, k_cache, block_table, indices, cache_seqlens - - -# ═══════════════════════════════════════════════════════════════════════ -# Kernel 2 (HCA): flash_mla_with_kvcache -# ═══════════════════════════════════════════════════════════════════════ - - -def _bench_flash_mla_sparse( - M: int, # noqa: N803 - past_kv: int, - *, - K_per_query: int, # noqa: N803 - native_heads: int, - sliding_window: int, - batch_size: int = 1, - tp_size: int = 1, - device: str = "cuda:0", -) -> float: - """Benchmark sparse FlashMLA matching V4's V4 backend call shape. - - Real V4 HCA backend (deepseek_v4_backend_radix.py:1087+) passes: - - main K cache + indices = SWA window (128 fixed positions) - - extra K cache + indices = c128 (HCA) positions - Total K attended per query = SWA_WINDOW + extra_K_per_query. - - TP zero-pad (mirrors ``sglang/srt/models/deepseek_v4.py:847``): - 1. Projection produces ``q_local`` of shape (..., native_heads//tp, d_qk) — - the rank's actual computed heads. - 2. ``q_padded`` is allocated full (..., native_heads, d_qk) and the rank's - ``tp_slice`` is filled from ``q_local``; other heads are zeros. - 3. FlashMLA always receives the full native head count. - """ - from flash_mla import flash_mla_with_kvcache, get_mla_metadata - - # rank-local head count (what the upstream projection actually produces) - n_local_heads = max(1, native_heads // tp_size) - M_per_req = M // batch_size if batch_size > 1 else M # noqa: N806 - _full_s = M_per_req + past_kv # per-request sequence length (bs requests, M_per_req new tokens each) - - # Build main K cache + ``q_local`` (per-rank Q at n_local_heads) - q_local, k_cache_main, _, _, cache_seqlens = _build_flash_mla_inputs( - M, - past_kv, - K_per_query=K_per_query, - batch_size=batch_size, - n_local_heads=n_local_heads, - device=device, - ) - - # Zero-pad ``q_local`` to the full native head count before the FlashMLA - # call. Passing the unpadded TP-local head count trips ``Unsupported h_q``. - if n_local_heads == native_heads: - q = q_local - else: - q = torch.zeros(batch_size, M_per_req, native_heads, FMLA_D_QK, dtype=torch.bfloat16, device=device) - q[:, :, :n_local_heads, :] = q_local # rank-0's tp_slice - - # Kernel always sees full h_q. - n_local_heads = native_heads - swa_window = sliding_window - swa_indices = _expand_indices(swa_window, batch_size, M_per_req, device) - swa_topk_lengths = torch.full((batch_size,), swa_window, dtype=torch.int32, device=device) - - # Build extra K cache (c128 or c4) + extra indices - _pg = _dsv4_kv_page_size() - extra_blocks = (K_per_query + _pg - 1) // _pg - extra_k_bf16 = torch.randn(max(1, extra_blocks), _pg, 1, FMLA_D_QK, dtype=torch.bfloat16, device=device) - extra_k_cache = _quantize_k_cache_model1(extra_k_bf16) - extra_K = max(64, ((K_per_query + 63) // 64) * 64) # noqa: N806 - extra_indices = _expand_indices(extra_K, batch_size, M_per_req, device) - extra_topk_lengths = torch.full((batch_size,), K_per_query, dtype=torch.int32, device=device) - - sched_meta, _ = get_mla_metadata( - cache_seqlens=cache_seqlens, - num_q_tokens_per_head_k=M_per_req * n_local_heads, - num_heads_k=1, - num_heads_q=n_local_heads, - is_fp8_kvcache=True, - topk=swa_indices.size(-1), - ) - - softmax_scale = 1.0 / (FMLA_D_QK**0.5) - attn_sink = torch.zeros(n_local_heads, dtype=torch.float32, device=device) - - def kernel_fn(): - flash_mla_with_kvcache( - q=q, - k_cache=k_cache_main, - block_table=None, - cache_seqlens=None, - head_dim_v=V_HEAD_DIM, - tile_scheduler_metadata=sched_meta, - num_splits=None, - softmax_scale=softmax_scale, - causal=False, - is_fp8_kvcache=True, - indices=swa_indices, - topk_length=swa_topk_lengths, - attn_sink=attn_sink, - extra_k_cache=extra_k_cache, - extra_indices_in_kvcache=extra_indices, - extra_topk_length=extra_topk_lengths, - ) - - return _bench_cuda_graph(kernel_fn, allow_graph_fail=False, device=device) - - -# HCA and CSA are the SAME FlashMLA kernel (``_bench_flash_mla_sparse``) — they -# differ only in how many KV positions each query attends to: -# - hca_attn: all c128 positions -> full_s // 128 -# - csa_attn: the topk-selected c4 positions -> min(index_topk, full_s // 4) -# (csa uses the same sequential-index approximation as hca; the real topk -# scatter pattern is not reproducible in a kernel-level bench.) -_FMLA_K_PER_QUERY = { - "hca_attn": lambda full_s, sc: max(1, full_s // 128), - "csa_attn": lambda full_s, sc: max(1, min(sc.index_topk, full_s // 4)), -} - - # ═══════════════════════════════════════════════════════════════════════ # CSV write helper # ═══════════════════════════════════════════════════════════════════════ @@ -678,7 +417,7 @@ def _write_row( ) -> None: os.makedirs(os.path.dirname(os.path.abspath(perf_filename)) or ".", exist_ok=True) - mla_dtype = "bfloat16" if kernel in ("hca_attn", "csa_attn") else "fp8_e4m3" + mla_dtype = "fp8_e4m3" kv_cache_dtype = "fp8_e4m3" gemm_type = "fp8_block" @@ -745,29 +484,13 @@ def _guarded_bench(bench_fn: Callable[[], object], label: str): def _bench_sparse_kernel_shape(kernel, prefix, isl, bs, sc, device, *, topk_variant=None): """Bench one module ``(prefix, isl, bs)`` shape; return a list of ``(score_mode, latency_ms)`` rows. TP-independent (paged_mqa maps tokens to - b=M while preserving request-local causal lengths; FMLA pads to full native - heads), so this is computed once per shape. + b=M while preserving request-local causal lengths), so this is computed + once per shape. Single-latency kernels return one ``(None, latency)``; topk returns the phase-qualified flat-vs-representative pair whose matching-variant DELTA perf_database consumes (flat.latency - top_last.latency).""" M = bs * isl # noqa: N806 - if kernel in _FMLA_K_PER_QUERY: - # FMLA mirrors the module row 1:1: bs requests, each isl new query tokens - # over `prefix` past_kv. Pass batch_size=bs (so M_per_req=isl) and base - # K_per_query on the PER-REQUEST sequence length (isl+prefix), not the - # flattened total (bs*isl+prefix). - full_s = isl + prefix - lat = _bench_flash_mla_sparse( - M, - prefix, - K_per_query=_FMLA_K_PER_QUERY[kernel](full_s, sc), - native_heads=sc.num_attention_heads, - sliding_window=sc.sliding_window, - batch_size=bs, - device=device, - )["latency_ms"] - return [(None, lat)] if kernel == "topk": if topk_variant not in ("v1", "v2"): raise ValueError(f"topk requires variant v1 or v2, got {topk_variant!r}") @@ -843,7 +566,12 @@ def _sglang_chunked_prefill_size(): if _CHUNKED_PREFILL is None: try: from sglang.srt.model_executor.cuda_graph_config import default_cuda_graph_config - from sglang.srt.server_args import ServerArgs, get_device_memory_capacity + from sglang.srt.server_args import ServerArgs + + try: # 0.5.21: moved to utils/common.py:904 (server_args no longer re-exports it) + from sglang.srt.utils.common import get_device_memory_capacity + except ImportError: + from sglang.srt.server_args import get_device_memory_capacity except ModuleNotFoundError: from srt.model_executor.cuda_graph_config import default_cuda_graph_config from srt.server_args import ServerArgs, get_device_memory_capacity @@ -852,16 +580,23 @@ def _sglang_chunked_prefill_size(): gpu_mem = get_device_memory_capacity("cuda") except Exception: pass - sa = ServerArgs.__new__(ServerArgs) - sa.chunked_prefill_size = None - sa.cuda_graph_config = default_cuda_graph_config() - sa.tp_size = 1 - sa.device = "cuda" + chunked = None try: - sa._handle_gpu_memory_settings(gpu_mem) - except Exception: - pass # chunked_prefill_size is set first, before any model-dependent step - chunked = sa.chunked_prefill_size + sa = ServerArgs.__new__(ServerArgs) + except TypeError: + # sglang>=0.5.20: ServerArgs is a msgspec Struct (no bare __new__); the + # GPU-memory tiering below is sglang's own and needs no instance. + sa = None + if sa is not None: + sa.chunked_prefill_size = None + sa.cuda_graph_config = default_cuda_graph_config() + sa.tp_size = 1 + sa.device = "cuda" + try: + sa._handle_gpu_memory_settings(gpu_mem) + except Exception: + pass # chunked_prefill_size is set first, before any model-dependent step + chunked = sa.chunked_prefill_size if chunked is None and gpu_mem is not None: # Newer sglang (0.0.0.dev / >=0.5.x) refactored cuda_graph_max_bs/_bs # into a cuda_graph_config object that _handle_gpu_memory_settings @@ -887,7 +622,7 @@ def _sglang_chunked_prefill_size(): def _dsv4_context_derived_shapes(model_path): """DSV4 sparse-kernel context shapes derived from the csa/hca context MODULE INPUT sweep (same source as GLM5's dsa derive) — NOT read back from the - module CSV. paged_mqa_logits / topk / csa_attn / hca_attn therefore cover + module CSV. paged_mqa_logits / topk therefore cover the full-chunk isl the module drops at runtime; CSA and HCA share one grid (the module input validity does not depend on attn_kind). @@ -988,9 +723,9 @@ def run_dsv4_sparse_kernel_worker( """Single worker for the whole DSV4 sparse-op family (invoked by collect.py, one case per model from get_func). - Reads the kernel's owning module CSV(s) — paged_mqa_logits/topk/csa_attn ← - CSA, hca_attn ← HCA — dedups to the unique ``(prefix, isl, bs)`` module shapes - (the kernels are TP-independent), and benches each once. + Derives the kernel's shapes from its owning CSA module INPUT sweep + (paged_mqa_logits / topk ← CSA) — dedups to the unique ``(prefix, isl, bs)`` + module shapes (the kernels are TP-independent), and benches each once. ``_bench_sparse_kernel_shape`` returns one or more ``(score_mode, latency)`` rows per shape — one for the single-latency kernels, two for topk (variant-qualified flat + top_last) — each written as one row (tp_size=1).""" @@ -1131,11 +866,30 @@ def _bench_topk_512( uses planned v2. Both execute inside production CUDA graphs, so capture is mandatory here too. """ - from sglang.jit_kernel.dsv4.topk import ( - plan_topk_v2, - topk_transform_512, - topk_transform_512_v2, - ) + try: + from sglang.jit_kernel.dsv4.topk import ( + plan_topk_v2, + topk_transform_512, + topk_transform_512_v2, + ) + except ImportError: + # sglang>=0.5.21 moved the JIT ops to sglang.kernels.ops.attention.dsv4.topk and + # renamed the entry points: topk_transform_512 -> topk_transform_paged (CUDA branch = + # the same _jit_topk_v1_module().topk_transform, topk.py:104-125; the ROCm branch is the + # deepseek_v4_topk_transform_512 AOT op) and topk_transform_512_v2 -> + # topk_transform_paged_v2 (topk.py:222, identical positional signature, out_raw_indices + # now optional). Serving DSV4 dispatches exactly these two from + # indexer.topk_transform_paged_from_metadata (srt/layers/attention/dsv4/indexer.py:446-486, + # v2 iff metadata.use_topk_v2), so the v1/v2 calib lanes keep their meaning. + from sglang.kernels.ops.attention.dsv4.topk import ( + plan_topk_v2, + ) + from sglang.kernels.ops.attention.dsv4.topk import ( + topk_transform_paged as topk_transform_512, + ) + from sglang.kernels.ops.attention.dsv4.topk import ( + topk_transform_paged_v2 as topk_transform_512_v2, + ) if variant not in ("v1", "v2"): raise ValueError(f"unknown topk variant: {variant}") diff --git a/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py b/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py index 0e87f785c8..9170f6bfaf 100644 --- a/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py +++ b/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py @@ -27,7 +27,9 @@ from __future__ import annotations -__compat__ = "sglang==0.5.14" +# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" import os import sys diff --git a/python/aisimulate/collector/sglang/registry.py b/python/aisimulate/collector/sglang/registry.py index 086641ac4a..fd2409f2a1 100644 --- a/python/aisimulate/collector/sglang/registry.py +++ b/python/aisimulate/collector/sglang/registry.py @@ -32,6 +32,9 @@ get_func="get_computescale_test_cases", run_func="run_computescale", perf_filename=PerfFile.COMPUTESCALE, + # one measurement, two tables: the static quant row goes to + # scale_matrix_perf; finalize binds both to this producer + extra_perf_filenames=(PerfFile.SCALE_MATRIX,), ), OpEntry( op="mla_context", @@ -123,16 +126,21 @@ ), # GLM-5.2 skip-indexer layers (index_topk_freq>1): same shapes as the full # DSA module, but the per-layer indexer (mqa+topk+index-K store) is patched - # out so the captured cost is the reuse-layer cost. run_func derives a - # skip_indexer bool from the worker_perf_filename selector and passes it to - # the benchmark subprocess as an explicit arg (no env var). + # out so the captured cost is the reuse-layer cost. The rows go into the + # SAME dsa_*_module table as the full op, tagged by an op_name + # "_skip_indexer" suffix (that is what the SDK reads), so the entry declares + # that shared table and selects the mode with run_kwargs — the executor + # binds skip_indexer=True to run_func. (Until 2026-10-06 the entry declared + # a dsa_*_skip_indexer_perf.txt table nobody wrote and the worker sniffed + # the mode from that filename; --resume finalization trusted the + # declaration and failed every skip_indexer shard: job 469988017.) OpEntry( op="dsa_context_module_skip_indexer", module="collector.sglang.collect_mla_module", get_func="get_dsa_context_module_skip_indexer_test_cases", run_func="run_mla_module_worker", perf_filename=PerfFile.DSA_CONTEXT_MODULE, - worker_perf_filename=PerfFile.DSA_CONTEXT_MODULE_SKIP_INDEXER, + run_kwargs=(("skip_indexer", True),), # Hardware-validated on SM100, on SM103 via the B300 probe # (2026-07-13, pipeline 57747474: 8,506 rows, skip_indexer # trtllm+flashmla buckets clean, 1 error), and on SM90 via the @@ -151,7 +159,7 @@ get_func="get_dsa_generation_module_skip_indexer_test_cases", run_func="run_mla_module_worker", perf_filename=PerfFile.DSA_GENERATION_MODULE, - worker_perf_filename=PerfFile.DSA_GENERATION_MODULE_SKIP_INDEXER, + run_kwargs=(("skip_indexer", True),), unverified_sms=(120,), ), # DeepSeek-V4 module-level data (csa/hca x ctx/gen = 4 ops, 1 file each). @@ -225,28 +233,6 @@ # hardware-validate through the sanctioned plan. unverified_sms=(120,), ), - # The standalone HCA/CSA FMLA sub-kernel collectors require the - # ``flash_mla`` package, which the pinned 0.5.14 image does not ship (the - # serving path uses the bundled sgl-kernel FlashMLA instead); no platform - # has ever produced rows for them. Keep them parked as unverified until a - # wired backend exists, instead of letting the getter silently enumerate - # zero cases. - OpEntry( - op="dsv4_hca_attn_module", - module="collector.sglang.deepseekv4_sparse_modules", - get_func="get_dsv4_hca_attn_test_cases", - run_func="run_dsv4_sparse_kernel_worker", - perf_filename=PerfFile.DSV4_HCA_ATTN_MODULE, - unverified=True, - ), - OpEntry( - op="dsv4_csa_attn_module", - module="collector.sglang.deepseekv4_sparse_modules", - get_func="get_dsv4_csa_attn_test_cases", - run_func="run_dsv4_sparse_kernel_worker", - perf_filename=PerfFile.DSV4_CSA_ATTN_MODULE, - unverified=True, - ), # CSA topk_512 DELTA calibration (flat vs top_last) — feeds the # degenerate->representative topK correction in perf_database. OpEntry( @@ -300,10 +286,11 @@ perf_filename=PerfFile.GLM5_DSA_ATTN_MODULE, unverified_sms=(120,), ), - # MiniMax-M3 MSA sparse-attention modules — requires the msa-family image - # pin (framework_manifest sglang families.msa → official v0.5.16, the - # first release with models/minimax_m3.py; the module declares - # __compat__ = "sglang>=0.5.16"). Hardware-validated per SM: + # MiniMax-M3 MSA sparse-attention modules — need models/minimax_m3.py, + # first shipped in the official v0.5.16 release (the module declares + # __compat__ = "sglang>=0.5.16"; the 0.5.14-era framework_manifest carried + # an msa family pin to v0.5.16, retired when the default moved to 0.5.21 + # on 2026-10-01). Hardware-validated per SM: # SM90 (h100/h200 — SGLang's own M3 server-args override selects # fa3 + page 128 and the Triton sparse path, # arg_groups/overrides.py:521-537@v0.5.16), SM100/103 diff --git a/python/aisimulate/collector/sglang/runtime_compat.py b/python/aisimulate/collector/sglang/runtime_compat.py new file mode 100644 index 0000000000..3f51dc38a0 --- /dev/null +++ b/python/aisimulate/collector/sglang/runtime_compat.py @@ -0,0 +1,181 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Process-wide sglang runtime config for the module collectors, across sglang generations. + +sglang 0.5.21 (runtime_context "bags"): a worker process must ``publish(server_args, role=..., +ranks=SpawnRanks(...))`` first and the config initializers then take NO arguments +(``benchmark/one_batch.py:893-902`` @0.5.21: publish -> initialize_moe_config() -> +initialize_fp8_gemm_config() -> initialize_fp4_gemm_config()); ``ModelRunner.__init__`` no longer +accepts the parallel geometry (``model_executor/model_runner.py``: model_config, mem_fraction_static, +gpu_id, nccl_port, server_args, ...) — it reads ``get_parallel()``. +sglang 0.5.16: the initializers take ``server_args`` and the geometry travels as +``ps=ParallelState.trivial(gpu_id)`` (parallel_state_wrapper.py:6-24). +Older pins: initializers take ``server_args`` and ModelRunner takes tp_rank/tp_size/... keywords. + +This only changes HOW the same engine state is published (layer_permissions.md API-compat shim +rule); the kernels the runner selects are untouched. +""" + +from __future__ import annotations + +import inspect + + +def init_runtime_config( + server_args, + gpu_id: int, + *, + nccl_port: int | None = None, + model_config=None, + tp_rank: int = 0, + tp_size: int | None = None, +) -> dict: + """Run the framework's own config initializers and return the ModelRunner parallel kwargs. + + 0.5.21 additionally needs the distributed world built BEFORE ModelRunner (model_runner.py + init_torch_distributed -> bootstrap.measure_pre_model_load_memory reads get_world_group()): + scheduler.py:525-530 and benchmark/one_batch.py load_model run bootstrap.init_parallel_runtime + (publish -> groups) then bootstrap.init_layer_runtime(model_config). Pass nccl_port + model_config. + """ + from sglang.srt.layers.moe import initialize_moe_config + from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config + from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config + + if inspect.signature(initialize_moe_config).parameters: # <= 0.5.20: seeded from ServerArgs directly + initialize_moe_config(server_args) + initialize_fp8_gemm_config(server_args) + initialize_fp4_gemm_config(server_args) + try: + from sglang.srt.distributed.parallel_state_wrapper import ParallelState + + return {"ps": ParallelState.trivial(gpu_id=gpu_id)} + except ImportError: + return { + "tp_rank": tp_rank, + "tp_size": tp_size or getattr(server_args, "tp_size", 1), + "pp_rank": 0, + "pp_size": 1, + "moe_ep_rank": 0, + "moe_ep_size": 1, + } + from sglang.srt.runtime_context import SpawnRanks, publish, spawn_world_rank + + publish( + server_args, + role="scheduler", + ranks=SpawnRanks(world_rank=spawn_world_rank(server_args, tp_rank=tp_rank, pp_rank=0), gpu_id=gpu_id), + ) + initialize_moe_config() + initialize_fp8_gemm_config() + initialize_fp4_gemm_config() + from sglang.srt.distributed import bootstrap + + if nccl_port is None or model_config is None: + raise RuntimeError( + "sglang>=0.5.21 runner init needs nccl_port and model_config " + "(bootstrap.init_parallel_runtime/init_layer_runtime)" + ) + try: + from sglang.srt.runtime_context import get_device + + device = get_device().device + except (ImportError, AttributeError): + device = getattr(server_args, "device", "cuda") + bootstrap.init_parallel_runtime(server_args=server_args, device=device, dist_port=nccl_port) + bootstrap.init_layer_runtime(model_config=model_config) + return {} + + +_OFFLINE_PUBLISHED = False + + +def ensure_offline_runtime_published(**server_args_fields) -> bool: + """Publish a minimal process-wide runtime config for the kernel-level collectors + that build attention backends on a mock runner (no engine, no model). + + sglang>=0.5.20 projects ``get_exec()`` / ``get_parallel()`` from a published + ServerArgs; the backends read them (flashattention_backend.py:320,349 get_exec(). + deterministic; parallel_state world_group). sglang's own unit tests publish an + offline record exactly this way (``ServerArgs(model_path="dummy", **fields)`` + + ``publish(role=..., ranks=SpawnRanks(world_rank=0))``, test/test_utils.py:1990-2023 + @0.5.21; the resolution pipeline returns early on the dummy/absent-model path). + Older sglang has no ``publish`` -> no-op. Returns True when a publish happened. + """ + global _OFFLINE_PUBLISHED + if _OFFLINE_PUBLISHED: + return True + try: + from sglang.srt.runtime_context import SpawnRanks, publish + from sglang.srt.server_args import ServerArgs + except ImportError: + return False + publish(ServerArgs(model_path="dummy", **server_args_fields), role="scheduler", ranks=SpawnRanks(world_rank=0)) + _OFFLINE_PUBLISHED = True + return True + + +def attach_kv_index_translator(mock_runner) -> None: + """sglang>=0.5.21 attention backends take ``model_runner.kv_index_translator`` + (flashattention_backend.py:205; built by ModelRunner.init_kv_index_translator at + model_runner.py:860-868 from req_to_token / allocator / pool / page_size / device). + Build the real one when the mock runner carries those objects; otherwise a + non-translating stand-in (the backends only consult ``is_translating`` on the + non-DCP, non-hybrid paths these kernel collectors exercise). No-op on older sglang. + """ + try: + from sglang.srt.mem_cache.kv_index_translator import KVIndexTranslator + except ImportError: + return + try: + mock_runner.kv_index_translator = KVIndexTranslator( + req_to_token=mock_runner.req_to_token_pool.req_to_token, + token_to_kv_pool_allocator=mock_runner.token_to_kv_pool_allocator, + token_to_kv_pool=mock_runner.token_to_kv_pool, + page_size=getattr(mock_runner, "page_size", None) or getattr(mock_runner.server_args, "page_size", 1) or 1, + device=str(mock_runner.device), + ) + except Exception as exc: + from types import SimpleNamespace + + print( + f"[sglang-compat] KVIndexTranslator not constructible on the mock runner ({type(exc).__name__}: {exc}); " + "using a non-translating stand-in" + ) + mock_runner.kv_index_translator = SimpleNamespace(is_translating=False, reads_are_translated=False) + + +def resolved_arg(server_args, name: str, default=None): + """A ServerArgs field as serving resolves it: the raw record first, else the + sglang>=0.5.21 resolving view (arg_groups/model_override_base.py:120) where derived + values (page_size, chunked_prefill_size, cuda_graph_config, ...) live.""" + value = getattr(server_args, name, None) + if value is not None: + return value + try: + from sglang.srt.arg_groups.model_override_base import resolving_view + + value = getattr(resolving_view(server_args), name, None) + except ImportError: + value = None + return default if value is None else value + + +def causal_conv1d_uses_int64_offsets() -> bool: + """Whether the installed causal_conv1d Triton kernel computes token offsets in + int64. The 0.5.14 kernel used int32 pointer arithmetic (causal_conv1d_triton.py:373-379, + collector guards at total_tokens*conv_channels >= 2**31); sglang>=0.5.21 casts the + sequence/token offsets to tl.int64 (kernels/ops/mamba/causal_conv1d_triton.py:95,185), + so that limit no longer exists there. Source-probed, never version-sniffed.""" + import importlib + import inspect + + for name in ( + "sglang.kernels.ops.mamba.causal_conv1d_triton", + "sglang.srt.layers.attention.mamba.causal_conv1d_triton", + ): + try: + src = inspect.getsource(importlib.import_module(name)) + except Exception: + continue + return "tl.int64" in src and "stride_x_token" in src + return False diff --git a/python/aisimulate/collector/sglang/runtime_limits.py b/python/aisimulate/collector/sglang/runtime_limits.py index f840f44a29..66c872fa4f 100644 --- a/python/aisimulate/collector/sglang/runtime_limits.py +++ b/python/aisimulate/collector/sglang/runtime_limits.py @@ -149,6 +149,23 @@ def sglang_dsa_mqa_logits_chunking_supported() -> bool: has_chunk_kernel = "logits_chunk" in source if has_chunk_decision and has_chunk_loop and has_chunk_kernel: return True + # sglang>=0.5.21 restructured the contract: the free-memory row chunking moved + # out of Indexer into dsa_indexer_kpool._mqa_logits_row_chunks (:85, driven by + # mqa_logits_should_chunk / mqa_logits_rows_per_chunk / mqa_logits_budget_bytes) + # and the SM100 batch>num_sms split lives in Indexer._get_topk_ragged's local + # _chunked_fp8_paged_mqa_logits (dsa_indexer.py:1027). Same behaviour the + # collector relies on (logits never exceed the free-memory budget), new shape. + try: + kpool = importlib.import_module("sglang.srt.layers.attention.dsa.dsa_indexer_kpool") + indexer_mod = importlib.import_module("sglang.srt.layers.attention.dsa.dsa_indexer") + indexer_cls = getattr(indexer_mod, "Indexer", None) + row_chunks = getattr(kpool, "_mqa_logits_row_chunks", None) + if indexer_cls is not None and row_chunks is not None and getattr(indexer_cls, "_get_topk_ragged", None): + source = inspect.getsource(row_chunks) + if "mqa_logits_should_chunk" in source and "mqa_logits_rows_per_chunk" in source: + return True + except Exception: + pass raise RuntimeError("SGLang DSA MQA-logits chunking source contract was not detected") @@ -177,6 +194,16 @@ def required_prefill_extend_tokens(batch_size: int, seq_len: int) -> int: def runtime_chunk_size(model_runner) -> int: server_args = getattr(model_runner, "server_args", None) sglang_chunk = getattr(server_args, "chunked_prefill_size", None) if server_args else None + if not (isinstance(sglang_chunk, int) and sglang_chunk > 0) and server_args is not None: + # sglang>=0.5.21: derived values are read through the resolving view + # (arg_groups/model_override_base.py:120 resolving_view; one_batch.load_model + # reads cfg.mem_fraction_static the same way), not off the raw record. + try: + from sglang.srt.arg_groups.model_override_base import resolving_view + + sglang_chunk = getattr(resolving_view(server_args), "chunked_prefill_size", None) + except ImportError: + pass if isinstance(sglang_chunk, int) and sglang_chunk > 0: return sglang_chunk raise RuntimeError("SGLang did not initialize server_args.chunked_prefill_size") diff --git a/python/aisimulate/collector/trtllm/collect_attn.py b/python/aisimulate/collector/trtllm/collect_attn.py index 8d154fb76f..4d36121235 100644 --- a/python/aisimulate/collector/trtllm/collect_attn.py +++ b/python/aisimulate/collector/trtllm/collect_attn.py @@ -27,13 +27,31 @@ ) from collector.helper import benchmark_with_power, get_sm_version, log_perf from collector.registry_types import PerfFile -from tensorrt_llm._torch.attention_backend import TrtllmAttentionMetadata -from tensorrt_llm._torch.attention_backend.interface import ( - AttentionRuntimeFeatures, - PositionalEmbeddingParams, - RopeParams, -) -from tensorrt_llm._torch.attention_backend.utils import create_attention + +# trtllm >=1.3.0rc29 moved _torch.attention_backend.* -> _torch.attention.backends.* and +# _torch.modules.fused_moe -> _torch.moe.fused_moe (the old package root is a deprecation shim +# without submodules). Path-only compat: same classes, same kernels (layer_permissions.md +# 'API-compat shims may only change HOW the same kernel is constructed'). +try: + from tensorrt_llm._torch.attention.backends import TrtllmAttentionMetadata +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend import TrtllmAttentionMetadata +try: + from tensorrt_llm._torch.attention.backends.interface import ( + AttentionRuntimeFeatures, + PositionalEmbeddingParams, + RopeParams, + ) +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.interface import ( + AttentionRuntimeFeatures, + PositionalEmbeddingParams, + RopeParams, + ) +try: + from tensorrt_llm._torch.attention.backends.utils import create_attention +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.utils import create_attention from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager from tensorrt_llm.functional import PositionEmbeddingType @@ -82,6 +100,28 @@ def _skip_trtllm_sm120_fp8_context_fmha( ) +def _deployment_tokens_per_block() -> int: + """KV page the deployment actually serves with, not a collector constant. + + The generator renders every trtllm engine with ``kv_cache_config.tokens_per_block: 32`` + (TRT-LLM's own KvCacheConfig default); the collector used to pin 64 here, which on + Blackwell selects a different FMHA cubin family — the sm100 kernel names carry the page + (fmhaSm100a…PagedKvCausalP64… vs serving's …P32…), so the sm100 attention gates + DIVERGED on nothing but the page while the sm90 names hide it (B200, 2026-10-04, + opharness finding sm100_sglang_trtllm_gate_verdicts_2026_10_04). Ask the framework + for its default; ``AIS_TRTLLM_TOKENS_PER_BLOCK`` is the A/B hook. + """ + override = os.environ.get("AIS_TRTLLM_TOKENS_PER_BLOCK") + if override: + return int(override) + try: + from tensorrt_llm.llmapi import KvCacheConfig + + return int(KvCacheConfig().tokens_per_block) + except Exception: # older builds without the pydantic config + return 32 + + def run_attention_torch( batch_size, input_len, @@ -98,7 +138,12 @@ def run_attention_torch( device="cuda:0", ): device = torch.device(device) - torch.set_default_device(device) + # No torch.set_default_device(device): the FLASHINFER sub-backend builds its HOST block + # tables with torch.zeros(..., pin_memory=True) (attention/backends/flashinfer.py:1566 + # @1.3.0rc29); under a CUDA default device that allocation lands on the GPU and pinning + # raises "Only dense CPU tensors can be pinned" — 2,942 FLASHINFER decode cases (every + # Gemma-4 head_dim 256/512 x window 1024/0 cell) on b200 rc29, job 469988228. Serving + # never sets a default device; every tensor this collector creates names its device. torch.cuda.set_device(device) if attn_backend_name not in {"TRTLLM", "FLASHINFER"}: @@ -118,7 +163,7 @@ def run_attention_torch( # supported-arch set against fmhaRunner.cuh on the next framework version bump. if is_flashinfer and get_sm_version() not in (100, 103): raise ValueError( - f"FlashInfer trtllm-gen FMHA is Blackwell-only (SM100/103); Gemma4 dense " + f"FIXME(kernel-limit): FlashInfer trtllm-gen FMHA is Blackwell-only (SM100/103); Gemma4 dense " f"attention has no kernel on SM{get_sm_version()} " f"(fmhaRunner.cuh:37 + modeling_gemma4.py:270 @1.3.0rc20)" ) @@ -133,9 +178,48 @@ def run_attention_torch( # TRTLLM-backend contract flashinfer does not consume, so an fp8-labeled # flashinfer row would record BF16 compute under an fp8 label. Fail closed. raise ValueError("TRT-LLM 1.3.0rc20 FlashInferAttention has no FP8 FMHA compute path") + # FIXME(kernel-limit): trtllm-gen has no FMHA for head_dim 192 on SM100/103 — context + # ("Missing TRTLLM-GEN kernel (context): headDimQk=192", 2,904 b200 rc29 cases) and decode + # ("Missing TRTLLM-GEN kernel (decode)", 1,624 b200 / 1,641 b300). Serving routes the same + # kernel. Classified before the layer is built; re-verify on the next TRT-LLM bump. + if head_dim == 192 and get_sm_version() in (100, 103): + raise ValueError( + "FIXME(kernel-limit): TRT-LLM trtllm-gen FMHA has no head_dim=192 kernel on SM100/103 " + f"({'context' if is_context_phase else 'decode'}; trtllm_fmha_kernel_launcher.cu:320)" + ) + # FIXME(kernel-limit): SM90 FP8-KV decode crashes ("CUDA error: unspecified launch failure" + # + SIGABRT) in two measured envelopes (h200 rc29, job 2026-10-08, 681 + 681 cases; the + # passing rows of the same shard bound them exactly): + # (a) num_heads 96 / kv_heads 4 (GQA 24): EVERY head_dim (64/128/256), every batch, isl + # and window — 512 cases, 0 passing rows; + # (b) head_dim 256 at GQA 32 (32/1, 64/2, 128/4), full attention (window 0): isl >= 255 + # — 169 cases; isl <= 127 passes at every batch. + # GQA 32/64 at other head_dims and GQA 24 never crashed and keep collecting (the first + # version of this guard, ed9109ec, refused every GQA >= 24 cell and lost 4,119 good rows: + # pipeline 72346027). Same family as the rc23 campaign's "decode fails for fp8 KV at GQA + # ratio 24/32" (finding campaign_1002 (3)). Re-verify on the next TRT-LLM bump. + if not is_context_phase and use_fp8_kv_cache and get_sm_version() == 90 and num_key_value_heads: + gqa_ratio = num_heads // num_key_value_heads + if (num_heads, num_key_value_heads) == (96, 4): + raise ValueError( + "FIXME(kernel-limit): TRT-LLM SM90 FP8-KV decode crashes for 96 q-heads over 4 kv-heads " + f"(head_dim={head_dim}); launch failure + SIGABRT on every cell" + ) + if head_dim == 256 and gqa_ratio == 32 and attention_window_size == 0 and input_len >= 255: + raise ValueError( + "FIXME(kernel-limit): TRT-LLM SM90 FP8-KV decode crashes for head_dim 256 at GQA ratio 32 " + f"with kv length >= 255 (num_heads={num_heads}, kv_heads={num_key_value_heads}, isl={input_len})" + ) # if XQA JIT is enabled, the context phase will also trigger XQA prepare which causes the error # with specifc q/kv head and seq setting. + # NOTE (1.3.0rc23, verified 2026-09-24): this env no longer exists in the + # binaries (only TRTLLM_FORCE_XQA / TRTLLM_XQA_BLOCKS_PER_SEQUENCE do). + # XQA vs MMHA for decode is the op's own heuristic ("JIT XQA is not + # used: maybe no performance gain" at short KV; XQA selected at long KV) — + # the collector follows it per (batch, kv_len) cell exactly as serving + # does, which path_diff confirmed at kv 4095 (XQA) vs kv 1 (MMHA). Kept + # for older builds that still read it. if is_context_phase: os.environ["TRTLLM_ENABLE_XQA_JIT"] = "0" else: @@ -144,7 +228,7 @@ def run_attention_torch( layer_idx = 0 world_size = 1 tp_size = 1 - tokens_per_block = 64 + tokens_per_block = _deployment_tokens_per_block() warming_up = 10 test_ite = 6 output_len = 1 @@ -352,7 +436,10 @@ def run_attention_torch( # with backend-agnostic kwargs (pyexecutor/model_engine.py:1784, # 1818-1830@1.3.0rc20); ``workspace`` is a TrtllmAttentionMetadata-only # field, flashinfer manages its own workspace_buffer. - from tensorrt_llm._torch.attention_backend.flashinfer import FlashInferAttentionMetadata + try: + from tensorrt_llm._torch.attention.backends.flashinfer import FlashInferAttentionMetadata + except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.flashinfer import FlashInferAttentionMetadata attn_metadata = FlashInferAttentionMetadata(**metadata_kwargs) else: @@ -368,10 +455,10 @@ def run_attention_torch( else: num_tokens = batch_size - sinks = torch.randn(num_heads, dtype=torch.float32) if head_dim == 64 else None - q = torch.randn([num_tokens, num_heads * head_dim]).bfloat16().to(torch.device(device)) - k = torch.randn([num_tokens, num_key_value_heads * head_dim]).bfloat16().to(torch.device(device)) - v = torch.randn([num_tokens, num_key_value_heads * head_dim]).bfloat16().to(torch.device(device)) + sinks = torch.randn(num_heads, dtype=torch.float32, device=device) if head_dim == 64 else None + q = torch.randn([num_tokens, num_heads * head_dim], dtype=torch.bfloat16, device=device) + k = torch.randn([num_tokens, num_key_value_heads * head_dim], dtype=torch.bfloat16, device=device) + v = torch.randn([num_tokens, num_key_value_heads * head_dim], dtype=torch.bfloat16, device=device) if is_flashinfer: # Serving splits Q/K/V for backends without fused-QKV support # (modules/attention.py:619,641-646@1.3.0rc20; FlashInferAttention @@ -392,6 +479,20 @@ def run_attention_torch( "attention_sinks": sinks, "out_scale": out_scale, } + # FIXME(kernel-limit): H20 campaign 2026-10-02 (1.3.0rc29, TRTLLM backend, + # attention/backends/fmha/fallback.py:122): the context FMHA faults with + # cudaErrorIllegalAddress exactly when the LAST sequence's packed Q/O byte + # offset (batch-1) * input_len * num_heads * head_dim * 2 >= 2**31 — + # verified cell-by-cell on the (96,*,256), (64,*,256) and (96,8,128) grids + # (b16 s3072 h96 d256 crashes, b8 s6144 passes at the same token count), + # i.e. a 32-bit per-sequence offset inside the kernel. Serving reaches such + # batches only with max_num_tokens >= ~44k tokens per context forward. + # Unverified against the kernel source; cases above the boundary keep + # failing into the classified log until the claim is confirmed (or a bump + # fixes it). Generation has a second family: fp8 KV with GQA ratio 24 or + # 32 (96/4 at d64/128/256; 32/1, 64/2, 128/4 at d256) fails with + # cudaErrorLaunchFailure / IMA at every batch and KV length while the same + # shapes pass with a bf16 KV cache. attn.forward(*forward_args, attn_metadata, **forward_kwargs) # Use benchmark_with_power context manager diff --git a/python/aisimulate/collector/trtllm/collect_attn_encoder.py b/python/aisimulate/collector/trtllm/collect_attn_encoder.py index 7ba7f9a6de..06170f11ce 100644 --- a/python/aisimulate/collector/trtllm/collect_attn_encoder.py +++ b/python/aisimulate/collector/trtllm/collect_attn_encoder.py @@ -21,12 +21,29 @@ from collector.case_generator import get_attention_encoder_head_configs, get_attention_encoder_shape_sweeps from collector.helper import benchmark_with_power, log_perf from collector.registry_types import PerfFile -from tensorrt_llm._torch.attention_backend import TrtllmAttentionMetadata -from tensorrt_llm._torch.attention_backend.interface import ( - AttentionRuntimeFeatures, - PredefinedAttentionMask, -) -from tensorrt_llm._torch.attention_backend.utils import create_attention + +# trtllm >=1.3.0rc29 moved _torch.attention_backend.* -> _torch.attention.backends.* and +# _torch.modules.fused_moe -> _torch.moe.fused_moe (the old package root is a deprecation shim +# without submodules). Path-only compat: same classes, same kernels (layer_permissions.md +# 'API-compat shims may only change HOW the same kernel is constructed'). +try: + from tensorrt_llm._torch.attention.backends import TrtllmAttentionMetadata +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend import TrtllmAttentionMetadata +try: + from tensorrt_llm._torch.attention.backends.interface import ( + AttentionRuntimeFeatures, + PredefinedAttentionMask, + ) +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.interface import ( + AttentionRuntimeFeatures, + PredefinedAttentionMask, + ) +try: + from tensorrt_llm._torch.attention.backends.utils import create_attention +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.utils import create_attention from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantConfig diff --git a/python/aisimulate/collector/trtllm/collect_computescale.py b/python/aisimulate/collector/trtllm/collect_computescale.py index a052b60a99..eab200a839 100644 --- a/python/aisimulate/collector/trtllm/collect_computescale.py +++ b/python/aisimulate/collector/trtllm/collect_computescale.py @@ -7,15 +7,22 @@ quantization for each YAML-backed shape. The reported latency isolates the scale-computation portion so support matrix data can track the cost separately from the static quantize kernel. + +One measurement, two tables — the registry entry declares the second one as +``extra_perf_filenames`` and the executor hands both names in, so finalize +binds both to this producer's checkpoint: + perf_filename -> computescale_perf.txt (dynamic quant minus static quant) + extra_perf_filenames[0] -> scale_matrix_perf.txt (the static per-tensor quant alone) """ __compat__ = "trtllm>=1.3.0rc20" import tensorrt_llm import torch -from case_generator import get_compute_scale_case_specs +from collector.case_generator import get_compute_scale_case_specs +from collector.helper import benchmark_with_power, get_sm_version, log_perf -from helper import benchmark_with_power, get_sm_version, log_perf +_OUTSIDE_LOOP_COUNT = 5 # to reduce impact of L2 cache hit def get_computescale_test_cases(): @@ -30,90 +37,63 @@ def get_computescale_test_cases(): return test_cases -def run_computescale(m, k, *, perf_filename, device="cuda:0"): +def _setup(m, k, device): device = torch.device(device) torch.cuda.set_device(device) torch.set_default_device(device) + x = torch.randn((m, k), dtype=torch.bfloat16).to(device) + return device, x - dtype = torch.bfloat16 - x = torch.randn((m, k), dtype=dtype).to(torch.device(device)) - scale = torch.tensor([1.0], dtype=torch.float32, device=device) - outside_loop_count = 5 # to reduce impact of L2 cache hit - - # Build op lists for both dynamic and static quantization - dynamic_op_list = [] - static_op_list = [] - for _ in range(outside_loop_count): - dynamic_op_list.append(torch.ops.tensorrt_llm.quantize_e4m3_per_tensor) - static_op_list.append(torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor) - - # Benchmark dynamic quantization (compute scale + scale matrix) - def dynamic_kernel_func(): - for op in dynamic_op_list: - op(x) - - with benchmark_with_power( - device=device, - kernel_func=dynamic_kernel_func, - repeat_n=1, - ) as dynamic_results: +def _bench_dynamic(device, x): + # dynamic quantization = compute scale + scale matrix + def kernel_func(): + for _ in range(_OUTSIDE_LOOP_COUNT): + torch.ops.tensorrt_llm.quantize_e4m3_per_tensor(x) + + with benchmark_with_power(device=device, kernel_func=kernel_func, repeat_n=1) as results: pass + return results["latency_ms"] / _OUTSIDE_LOOP_COUNT, results["power_stats"] - dynamic_latency = dynamic_results["latency_ms"] / outside_loop_count - # Benchmark static quantization (scale matrix only) - def static_kernel_func(): - for op in static_op_list: - op(x, scale) +def _bench_static(device, x): + # static quantization = scale matrix only + scale = torch.tensor([1.0], dtype=torch.float32, device=device) + + def kernel_func(): + for _ in range(_OUTSIDE_LOOP_COUNT): + torch.ops.tensorrt_llm.static_quantize_e4m3_per_tensor(x, scale) - with benchmark_with_power( - device=device, - kernel_func=static_kernel_func, - repeat_n=1, - ) as static_results: + with benchmark_with_power(device=device, kernel_func=kernel_func, repeat_n=1) as results: pass + return results["latency_ms"] / _OUTSIDE_LOOP_COUNT, results["power_stats"] - static_latency = static_results["latency_ms"] / outside_loop_count - # compute_scale latency = dynamic - static - compute_scale_latency = dynamic_latency - static_latency - compute_scale_latency = max(0.0, compute_scale_latency) +def run_computescale(m, k, *, perf_filename, extra_perf_filenames, device="cuda:0"): + (scale_matrix_filename,) = extra_perf_filenames + device, x = _setup(m, k, device) + dynamic_latency, dynamic_power = _bench_dynamic(device, x) + static_latency, static_power = _bench_static(device, x) + compute_scale_latency = max(0.0, dynamic_latency - static_latency) - # Log compute_scale performance log_perf( - item_list=[ - { - "m": m, - "k": k, - "quant_dtype": "fp8", - "latency": compute_scale_latency, - } - ], + item_list=[{"m": m, "k": k, "quant_dtype": "fp8", "latency": compute_scale_latency}], framework="TRTLLM", version=tensorrt_llm.__version__, device_name=torch.cuda.get_device_name(device), op_name="compute_scale", kernel_source="torch_ops", perf_filename=perf_filename, - power_stats=dynamic_results["power_stats"], + power_stats=dynamic_power, ) - # Log scale_matrix performance log_perf( - item_list=[ - { - "m": m, - "k": k, - "quant_dtype": "fp8", - "latency": static_latency, - } - ], + item_list=[{"m": m, "k": k, "quant_dtype": "fp8", "latency": static_latency}], framework="TRTLLM", version=tensorrt_llm.__version__, device_name=torch.cuda.get_device_name(device), op_name="scale_matrix", kernel_source="torch_ops", - perf_filename="scale_matrix_perf.txt", - power_stats=static_results["power_stats"], + perf_filename=scale_matrix_filename, + power_stats=static_power, ) diff --git a/python/aisimulate/collector/trtllm/collect_dsv4_attn.py b/python/aisimulate/collector/trtllm/collect_dsv4_attn.py index 95f8fd30cf..f6590f24f3 100644 --- a/python/aisimulate/collector/trtllm/collect_dsv4_attn.py +++ b/python/aisimulate/collector/trtllm/collect_dsv4_attn.py @@ -57,12 +57,12 @@ try: from registry_types import PerfFile - from helper import benchmark_with_power, log_perf + from helper import benchmark_with_power, get_sm_version, log_perf except ModuleNotFoundError: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from registry_types import PerfFile - from helper import benchmark_with_power, log_perf + from helper import benchmark_with_power, get_sm_version, log_perf ARCHITECTURE = "DeepseekV4ForCausalLM" @@ -161,6 +161,31 @@ def _drop(reason: str) -> None: return shapes +# Perf-row / case label of each KV pool dtype = the SDK's KVCacheQuantMode member name. +_KV_CACHE_DTYPE_LABELS = {"bf16": "bfloat16", "bfloat16": "bfloat16", "fp8": "fp8"} + + +def _serving_kv_cache_dtype() -> str: + """KV pool dtype the deployment actually serves DSV4 with on this SM. + + Hopper (and the documented SM120/121 path) serve the packed fp8_ds_mla sparse-MLA pool. + On SM100 the generator's same facts (kv_cache_config.dtype fp8_ds_mla, page 128) are NOT + servable with 1.3.0rc29: the manager first demands tokens_per_block=256, then the fp8_ds_mla + sparse GENERATION step has no FMHA library ("No TRT-LLM attention FMHA library supports this + request"); the only configuration that boots is kv dtype auto -> bf16 (DeepseekV4CacheManager + BF16), which serves fmhaSm100f QkvBfloat16 ... DynamicTokenSparse cubins (B200 2026-10-04, + opharness finding sm100_trtllm_dsv4_facts_unservable_2026_10_04). Mirror that: the sm100 + collector lane is bf16-KV; the perf row carries kv_cache_dtype accordingly, spelled as the + SDK's KVCacheQuantMode member name ("bfloat16" / "fp8": engine_table_view rehydrates the + column through that enum, and "bf16" raised KeyError on every SM100 DSV4 table, Full CI + 2026-10-29). AIS_DSV4_KV_CACHE_DTYPE=fp8|bfloat16 (alias bf16) is the A/B hook. + """ + override = os.environ.get("AIS_DSV4_KV_CACHE_DTYPE") + if override: + return _KV_CACHE_DTYPE_LABELS.get(override, override) + return "fp8" if get_sm_version() in (90, 120, 121) else "bfloat16" + + def _build_dsv4_test_cases(mode: str, attn_kind: str) -> list[dict]: cases: list[dict] = [] tp_sizes = [1] if "--smoke" in sys.argv else _DSV4_MODULE_TP_SIZES @@ -179,7 +204,7 @@ def _build_dsv4_test_cases(mode: str, attn_kind: str) -> list[dict]: for model_path in model_paths: for tp_size in tp_sizes: for bs, sl, prefix in shapes: - params = [sl, bs, tp_size, "fp8", "bfloat16", "fp8_block", model_path, attn_kind] + params = [sl, bs, tp_size, _serving_kv_cache_dtype(), "bfloat16", "fp8_block", model_path, attn_kind] case_id = f"dsv4_{attn_kind}_{mode}_b{bs}_s{sl}_tp{tp_size}_{model_path.replace('/', '_')}" if mode == "context": params.append(prefix) @@ -222,10 +247,14 @@ def get_dsv4_hca_generation_test_cases() -> list[dict]: # untouched, and reusing one module across batches is exactly what serving # does. Owner-approved perf change 2026-08-09. _MODULE_CACHE: dict = {} +# Lower bound on the per-case KV token quota (see create_dsv4_kv_cache_and_metadata). +_KV_POOL_MIN_TOKENS = 2 * 1024 * 1024 -def _cached_dsv4_attention_module(model_path: str, attn_kind: str, tp_size: int, device: str): - key = (model_path, attn_kind, int(tp_size), device) +def _cached_dsv4_attention_module( + model_path: str, attn_kind: str, tp_size: int, device: str, kv_cache_dtype: str = "fp8" +): + key = (model_path, attn_kind, int(tp_size), device, kv_cache_dtype) hit = _MODULE_CACHE.get(key) if hit is not None: return hit @@ -235,7 +264,9 @@ def _cached_dsv4_attention_module(model_path: str, attn_kind: str, tp_size: int, _MODULE_CACHE.clear() gc.collect() torch.cuda.empty_cache() - entry = create_dsv4_attention_module(model_path=model_path, attn_kind=attn_kind, tp_size=tp_size, device=device) + entry = create_dsv4_attention_module( + model_path=model_path, attn_kind=attn_kind, tp_size=tp_size, device=device, kv_cache_dtype=kv_cache_dtype + ) _MODULE_CACHE[key] = entry return entry @@ -322,6 +353,7 @@ def create_dsv4_attention_module( attn_kind: str, tp_size: int, device: str = "cuda:0", + kv_cache_dtype: str = "fp8", ): """Build DeepseekV4Attention through TRT-LLM's own config path. @@ -379,7 +411,11 @@ def create_dsv4_attention_module( "The framework dispatch contract changed; re-audit against the runtime version." ) - _apply_gemm_type_quant(model_config, "fp8_block", use_fp8_kv_cache=True) + # The KV quant algo follows the lane: fp8 (Hopper fp8_ds_mla pool) sets kv_cache_quant_algo=FP8 and the + # module runs the E4m3 FMHA cubins; bf16 (the only servable SM100 pool, see _serving_kv_cache_dtype) + # leaves it unset so the module runs the QkvBfloat16 cubins serving runs — with FP8 pinned here the + # sm100 bf16 pool still produced E4m3 kernels (B200 capture 2026-10-04). + _apply_gemm_type_quant(model_config, "fp8_block", use_fp8_kv_cache=(kv_cache_dtype == "fp8")) # Provenance: print the RESOLVED kernel-selection knobs (same auto-build # code path default serving takes when the user sets no @@ -405,12 +441,43 @@ def create_dsv4_attention_module( ) ) + if get_sm_version() == 90: + # Serving publishes the requested pool dtype to the attention modules through the + # model's extra attrs (pyexecutor/model_loader.py:1934 extra_attrs['kv_cache_dtype'] + # = kv_cache_config.dtype; read by attention/mla.py:341). On Hopper that is + # 'fp8_ds_mla' (see the KvCacheConfig below) and it also switches the DSV4 module + # OFF the fused Q-FP8/KV-norm prologue (sparse/deepseek_v4/module.py:470-480 + # _is_fused_q_fp8_quant_enabled: "fp8_ds_mla does not use the fused Q FP8 path"). + # Without it the module takes the fused path, hands the footer-scale cache a + # strided raw latent and mla_rope_inplace rejects it ("data must be contiguous", + # H20 ctx capture 2026-10-01) — a path serving never runs on Hopper. + model_config.extra_attrs["kv_cache_dtype"] = "fp8_ds_mla" + aux_stream = torch.cuda.Stream(device=device) - attn_module = DeepseekV4Attention( - model_config=model_config, - layer_idx=0, - aux_stream=aux_stream, - ) + import inspect + + _ctor_params = inspect.signature(DeepseekV4Attention.__init__).parameters + if "aux_stream_dict" in _ctor_params: + # 1.3.0rc29: DeepseekV4Attention takes the model's per-purpose stream dict + # (modeling_deepseekv4.py:1371-1376, built by DeepseekV4ForCausalLM :1521-1570), + # as collect_mla_module already does for the MLA module. + from tensorrt_llm._torch.utils import AuxStreamType + + attn_module = DeepseekV4Attention( + model_config=model_config, + layer_idx=0, + aux_stream_dict={ + AuxStreamType.Attention: aux_stream, + AuxStreamType.MoeShared: aux_stream, + AuxStreamType.MoeChunkingOverlap: torch.cuda.Stream(device=device), + }, + ) + else: + attn_module = DeepseekV4Attention( + model_config=model_config, + layer_idx=0, + aux_stream=aux_stream, + ) # Serving applies QuantConfig.exclude_modules before weight creation # (apply_quant_config_exclude_modules, modeling_utils.py @runtime version); @@ -452,6 +519,7 @@ def create_dsv4_kv_cache_and_metadata( is_context: bool, prefix_len: int = 0, device: str = "cuda:0", + kv_cache_dtype_str: str = "fp8", ): """DSV4 cache manager + attention metadata, following the serving construction path with a DIRECT pinned citation on every hand-set field @@ -474,11 +542,20 @@ def create_dsv4_kv_cache_and_metadata( (is_gen: req.prompt_len = token_num - 1; py_prompt_len = prompt_len), consumed at model_engine.py:4336 """ - from tensorrt_llm._torch.attention_backend.interface import ( - AttentionRuntimeFeatures, - KVCacheParams, - ) - from tensorrt_llm._torch.attention_backend.utils import get_attention_backend + try: + from tensorrt_llm._torch.attention.backends.interface import ( + AttentionRuntimeFeatures, + KVCacheParams, + ) + except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.interface import ( + AttentionRuntimeFeatures, + KVCacheParams, + ) + try: + from tensorrt_llm._torch.attention.backends.utils import get_attention_backend + except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.utils import get_attention_backend from tensorrt_llm.bindings import DataType from tensorrt_llm.bindings.internal.batch_manager import CacheType @@ -537,20 +614,69 @@ def create_dsv4_kv_cache_and_metadata( # equivalent byte quota directly (half of currently-free device memory, # matching the DSV4 example's free_gpu_memory_fraction=0.5, # examples/models/core/deepseek_v4/README.md:148-151 @v1.3.0rc23). A - # main-KV-shaped max_tokens cap under-counts on V2 (the byte quota spans - # main KV + SWA + compressor/indexer caches) and made large-KV shapes - # fail dummy-request allocation ("Request ID not found in IndexMapper", - # B200 smoke round 1 2026-08-06). + # collector-computed main-KV-shaped byte cap under-counted on V2 (the + # byte quota spans main KV + SWA + compressor/indexer caches) and made + # large-KV shapes fail dummy-request allocation ("Request ID not found in + # IndexMapper", B200 smoke round 1 2026-08-06) — hence the token cap below + # is handed to the manager's own arithmetic, never computed here. free_bytes, _ = torch.cuda.mem_get_info(torch.device(device)) - kv_cache_config = KvCacheConfig( + # Pool size is a per-case cost, not a measurement input: the manager + # physically backs the whole quota at construction (runtime/ + # kv_cache_manager_v2/_storage_manager.py:248-249 slot counts from the + # GPU-tier quota, expand_pool_group :826 @1.3.0rc29) and this collector + # builds and tears one down per case. Cap the quota at what this batch + # can touch via kv_cache_config.max_tokens — the manager converts it with + # its own DSV4 arithmetic (DeepseekV4CacheManager._get_quota_from_max_tokens, + # sparse/deepseek_v4/cache_manager.py:926-981 @1.3.0rc29: non-sliding KV + + # SWA per token/request + indexer/compress pools + per-type padding) and + # takes min(max_gpu_total_bytes, quota_from_max_tokens) (pyexecutor/kv_cache/ + # kv_cache_manager_v2.py:1384-1398). The envelope is doubled and rounded + # to whole pages so dummy-request allocation never sits at the edge; a + # pool that is still too small fails allocation loudly (classified), + # never silently. The free-memory half stays as the upper bound, matching + # the DSV4 example's free_gpu_memory_fraction=0.5. + planned_tokens = batch_size * (max(max_seq, request_tokens) + tokens_per_block) + planned_tokens = -(-2 * planned_tokens // tokens_per_block) * tokens_per_block + # Floor: the pool must stay large enough that inductor emits the SAME + # kernels serving runs. footer_scale_kv.dequant_gather (fp8_ds_mla path, + # @maybe_compile(dynamic=True)) indexes the whole footer-scale pool; when + # that pool extent is < 2**31 bytes inductor generates int32-indexed + # triton kernels (xnumel i32), above it int64 — serving pools are tens of + # GiB, so serving always runs the int64 variant. H20 calibration + # (2026-10-03, DeepSeek-V4-Flash, b64 s128 p2048): quota from 1,048,576 + # tokens = 11.4 GiB -> i32 (53.9 ms), 1,572,864 tokens = 17.1 GiB -> i64 + # (45.8 ms); 2M tokens (22.7 GiB on Flash, i64 verified in the same sweep) + # sits 1.33x past the threshold, and every other DSV4 geometry carries more + # footer bytes per planned token than Flash, so the margin only grows. The + # footer row format is fixed (584 B/token), so the threshold is in tokens, + # not model geometry. 4M was tried first: on DeepSeek-V4-Pro it reproduced + # the campaign's free*0.5 pool size and left 20 GiB for the 262k-295k token + # context cells, whose forward allocates a 32 GiB intermediate -> OOM. + planned_tokens = max(planned_tokens, _KV_POOL_MIN_TOKENS) + kv_cache_kwargs = dict( tokens_per_block=tokens_per_block, max_gpu_total_bytes=int(free_bytes * 0.5), + max_tokens=int(planned_tokens), enable_block_reuse=False, ) + if get_sm_version() == 90: + # 1.3.0rc29 on Hopper: DeepseekV4CacheManager refuses any pool but the packed + # FP8 sparse-MLA cache (attention/backends/sparse/deepseek_v4/cache_manager.py: + # 343-349 "DeepSeek-V4 on Hopper requires kv_cache_config.dtype='fp8_ds_mla'"), + # the option llm_args.py:4256-4262 documents for SM90/SM120/SM121; serving + # deployments on H20 carry kv_cache_config.dtype=fp8_ds_mla (harness targets + # customization, matrix pass+custom). Earlier rc KvCacheConfig has no such + # literal and used the plain FP8 pool — keep that path for them. + try: + kv_cache_config = KvCacheConfig(dtype="fp8_ds_mla", **kv_cache_kwargs) + except (ValueError, TypeError): + kv_cache_config = KvCacheConfig(**kv_cache_kwargs) + else: + kv_cache_config = KvCacheConfig(**kv_cache_kwargs) kv_cache_manager_cls = get_kv_cache_manager_cls(model_config, kv_cache_config) # fp8 KV rows -> DataType.FP8, matching serving's kv_cache_dtype # resolution from kv_cache_quant_algo (set by _apply_gemm_type_quant). - kv_cache_dtype = DataType.FP8 + kv_cache_dtype = DataType.BF16 if kv_cache_dtype_str == "bfloat16" else DataType.FP8 # Serving construction site: pyexecutor/_util.py:1843-1867 @1.3.0rc23 # (is_mla branch): CacheType.SELFKONLY :1846, num_kv_heads=1 :1848, @@ -709,8 +835,8 @@ def run_dsv4_attn( if attn_kind not in ATTN_KIND_TO_COMPRESS_RATIO: raise ValueError(f"unsupported DSV4 attn_kind: {attn_kind}") - if kv_cache_dtype != "fp8": - raise ValueError(f"DSV4 module rows are fp8-KV only (got {kv_cache_dtype!r})") + if kv_cache_dtype not in ("fp8", "bfloat16"): + raise ValueError(f"DSV4 module rows are fp8-KV or bfloat16-KV only (got {kv_cache_dtype!r})") if compute_dtype != "bfloat16": raise ValueError(f"DSV4 module rows are bfloat16-compute only (got {compute_dtype!r})") if gemm_type != "fp8_block": @@ -720,7 +846,9 @@ def run_dsv4_attn( torch_device = torch.device(device) torch.cuda.set_device(torch_device) - attn_module, model_config, head_info = _cached_dsv4_attention_module(model_path, attn_kind, tp_size, device) + attn_module, model_config, head_info = _cached_dsv4_attention_module( + model_path, attn_kind, tp_size, device, kv_cache_dtype + ) # Ownership: the KV pool and the process-global extra-attrs slot are # released/restored on EVERY exit path (success, dry-run failure, @@ -738,6 +866,7 @@ def run_dsv4_attn( is_context=is_context, prefix_len=prefix_len, device=device, + kv_cache_dtype_str=kv_cache_dtype, ) hidden_size = model_config.pretrained_config.hidden_size diff --git a/python/aisimulate/collector/trtllm/collect_gdn.py b/python/aisimulate/collector/trtllm/collect_gdn.py index 5307edbe6a..f34bdc8078 100644 --- a/python/aisimulate/collector/trtllm/collect_gdn.py +++ b/python/aisimulate/collector/trtllm/collect_gdn.py @@ -42,6 +42,8 @@ from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule from tensorrt_llm._torch.modules.mamba.causal_conv1d import causal_conv1d_fn, causal_conv1d_update + _CHUNK_KERNEL_SOURCE: str = "" # bound with the kernels by the runtime import (globals().update) + import torch try: @@ -167,6 +169,13 @@ def run_gdn_context_benchmark( conv_weight = torch.randn(conv_channels, d_conv, dtype=dtype, device=device) conv_bias = torch.randn(conv_channels, dtype=dtype, device=device) + # Every cell is timed under CUDA-graph replay; a failed capture raises + # (benchmark_with_power default allow_graph_fail=False) and the cell is + # counted below, so the case ends as a classified failure instead of an + # eager-timed row whose latency floor is the Python launch loop. + successful_points = 0 + failed_points = 0 + failures: list[str] = [] for batch_size in batch_size_list: for seq_len in seq_len_list: if aic_debug: @@ -199,10 +208,22 @@ def run_gdn_context_benchmark( q = torch.randn(batch_size, seq_len, num_k_heads, head_k_dim, dtype=dtype, device=device) k = torch.randn(batch_size, seq_len, num_k_heads, head_k_dim, dtype=dtype, device=device) v = torch.randn(batch_size, seq_len, num_v_heads, head_v_dim, dtype=dtype, device=device) + # g is fp32 in serving (fused_gdn_gating output); FlashInfer's + # chunk kernel asserts it ("g must be fp32"), Triton accepts both g = torch.nn.functional.logsigmoid( - torch.randn(batch_size, seq_len, num_v_heads, dtype=dtype, device=device) + torch.randn(batch_size, seq_len, num_v_heads, dtype=torch.float32, device=device) ) beta = torch.sigmoid(torch.randn(batch_size, seq_len, num_v_heads, dtype=dtype, device=device)) + # serving feeds the chunk kernel PACKED tokens [1, B*S, H, D] plus + # cu_seqlens (gdn_mixer.forward_extend); FlashInfer's chunk kernel + # accepts only that layout ("q must be [1, T, H_q, D_k]") + q, k, v, g, beta = (t.reshape(1, batch_size * seq_len, *t.shape[2:]) for t in (q, k, v, g, beta)) + cu_seqlens = torch.arange(0, batch_size * seq_len + 1, seq_len, dtype=torch.int64, device=device) + # serving passes the per-request recurrent state and asks for the + # final state back (gdn_mixer.py:847-855); FlashInfer requires it + gdn_init_state = torch.zeros( + batch_size, num_v_heads, head_k_dim, head_v_dim, dtype=torch.bfloat16, device=device + ) # --- Benchmark causal_conv1d_fn --- torch.cuda.synchronize() @@ -218,7 +239,6 @@ def run_conv1d(_ki=k_input, _cs=conv_state): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -233,11 +253,15 @@ def run_conv1d(_ki=k_input, _cs=conv_state): # --- Benchmark chunk_gated_delta_rule --- torch.cuda.synchronize() - chunk_gated_delta_rule(q, k, v, g, beta) + chunk_gated_delta_rule( + q, k, v, g, beta, initial_state=gdn_init_state, output_final_state=True, cu_seqlens=cu_seqlens + ) torch.cuda.synchronize() - def run_gdn_scan(_q=q, _k=k, _v=v, _g=g, _beta=beta): - chunk_gated_delta_rule(_q, _k, _v, _g, _beta) + def run_gdn_scan(_q=q, _k=k, _v=v, _g=g, _beta=beta, _cu=cu_seqlens, _st=gdn_init_state): + chunk_gated_delta_rule( + _q, _k, _v, _g, _beta, initial_state=_st, output_final_state=True, cu_seqlens=_cu + ) with benchmark_with_power( device=device, @@ -245,7 +269,6 @@ def run_gdn_scan(_q=q, _k=k, _v=v, _g=g, _beta=beta): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -253,7 +276,7 @@ def run_gdn_scan(_q=q, _k=k, _v=v, _g=g, _beta=beta): version=trtllm_version, device_name=torch.cuda.get_device_name(device), op_name="gdn", - kernel_source="chunk_gated_delta_rule", + kernel_source=_CHUNK_KERNEL_SOURCE, perf_filename=perf_filename, power_stats=results["power_stats"], ) @@ -262,16 +285,22 @@ def run_gdn_scan(_q=q, _k=k, _v=v, _g=g, _beta=beta): input_pool = _make_input_pool( { "k_input": (batch_size, conv_channels, seq_len), - "q": (batch_size, seq_len, num_k_heads, head_k_dim), - "k": (batch_size, seq_len, num_k_heads, head_k_dim), - "v": (batch_size, seq_len, num_v_heads, head_v_dim), - "g": (batch_size, seq_len, num_v_heads), - "beta": (batch_size, seq_len, num_v_heads), + # packed [1, B*S, ...] + cu_seqlens, as serving feeds the chunk kernel + "q": (1, batch_size * seq_len, num_k_heads, head_k_dim), + "k": (1, batch_size * seq_len, num_k_heads, head_k_dim), + "v": (1, batch_size * seq_len, num_v_heads, head_v_dim), + "g": (1, batch_size * seq_len, num_v_heads), + "beta": (1, batch_size * seq_len, num_v_heads), }, total_iters, dtype, device, ) + input_pool["g"] = [t.float() for t in input_pool["g"]] # serving g is fp32 + cu_seqlens = torch.arange(0, batch_size * seq_len + 1, seq_len, dtype=torch.int64, device=device) + gdn_init_state = torch.zeros( + batch_size, num_v_heads, head_k_dim, head_v_dim, dtype=torch.bfloat16, device=device + ) for i in range(total_iters): input_pool["g"][i] = torch.nn.functional.logsigmoid(input_pool["g"][i]) input_pool["beta"][i] = torch.sigmoid(input_pool["beta"][i]) @@ -298,7 +327,6 @@ def run_conv1d(_pool=input_pool, _cs=conv_state, _idx=conv1d_iter_idx): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -319,6 +347,9 @@ def run_conv1d(_pool=input_pool, _cs=conv_state, _idx=conv1d_iter_idx): input_pool["v"][0], input_pool["g"][0], input_pool["beta"][0], + initial_state=gdn_init_state, + output_final_state=True, + cu_seqlens=cu_seqlens, ) torch.cuda.synchronize() @@ -333,6 +364,9 @@ def run_gdn_scan(_pool=input_pool, _idx=gdn_iter_idx): _pool["v"][idx], _pool["g"][idx], _pool["beta"][idx], + initial_state=gdn_init_state, + output_final_state=True, + cu_seqlens=cu_seqlens, ) with benchmark_with_power( @@ -341,7 +375,6 @@ def run_gdn_scan(_pool=input_pool, _idx=gdn_iter_idx): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -349,7 +382,7 @@ def run_gdn_scan(_pool=input_pool, _idx=gdn_iter_idx): version=trtllm_version, device_name=torch.cuda.get_device_name(device), op_name="gdn", - kernel_source="chunk_gated_delta_rule", + kernel_source=_CHUNK_KERNEL_SOURCE, perf_filename=perf_filename, power_stats=results["power_stats"], ) @@ -361,11 +394,108 @@ def run_gdn_scan(_pool=input_pool, _idx=gdn_iter_idx): del input_pool, conv_state gc.collect() torch.cuda.empty_cache() + successful_points += 1 except Exception as e: + failed_points += 1 + failures.append(f"batch_size={batch_size}, seq_len={seq_len}: {type(e).__name__}: {e}") print(f" Error at batch_size={batch_size}, seq_len={seq_len}: {e}") continue + summary = f"ok={successful_points} error={failed_points} skip=0" + print(f"GDN context summary: {summary}") + if failed_points or successful_points == 0: + raise RuntimeError( + f"TRTLLM GDN context collection failed strict completeness: {summary}; failed cells: {'; '.join(failures)}" + ) + + +def _bench_serving_decode_entry( + *, + batch_size: int, + num_k_heads: int, + head_k_dim: int, + num_v_heads: int, + head_v_dim: int, + dtype, + device, + common_log_data: dict, + trtllm_version: str, + perf_filename: str, + num_warmups: int, + num_runs: int, +) -> None: + """Measure the entry serving actually calls for pure decode. + + gdn_mixer.forward_decode (_torch/modules/mamba/gdn_mixer.py:670 @1.3.0rc23) + calls fused_sigmoid_gating_delta_rule_update with the raw gating inputs + (A_log, dt_bias, a, b), the SSM state pool + slot indices and + cu_seqlens; that function dispatches to FlashInfer's bf16-state GDN decode + kernel when _can_use_flashinfer_gdn_decode holds (SM90/SM100, K==V==128, + bf16 state, one token per sequence) and to the FLA Triton kernel otherwise + (fla/fused_sigmoid_gating_recurrent.py:205-238,452-458). kernel_source + records which branch ran. Builds without this entry (pre-rc23) skip it — + their serving truth is the fused_recurrent row above. + """ + fn = globals().get("fused_sigmoid_gating_delta_rule_update") + gate = globals().get("_can_use_flashinfer_gdn_decode") + if fn is None: + return + B, H, K, V = batch_size, num_v_heads, head_k_dim, head_v_dim # noqa: N806 + q = torch.randn(1, B, num_k_heads, K, dtype=dtype, device=device) + k = torch.randn(1, B, num_k_heads, K, dtype=dtype, device=device) + v = torch.randn(1, B, H, V, dtype=dtype, device=device) + a = torch.randn(B, H, dtype=dtype, device=device) + b = torch.randn(B, H, dtype=dtype, device=device) + A_log = torch.zeros(H, dtype=torch.float32, device=device) # noqa: N806 + dt_bias = 0.1 * torch.randn(H, dtype=torch.float32, device=device) + ssm_states = torch.randn(B, H, K, V, dtype=torch.bfloat16, device=device) + slot_indices = torch.arange(B, dtype=torch.int32, device=device) + cu_seqlens = torch.arange(B + 1, dtype=torch.int64, device=device) + uses_flashinfer = bool(gate(ssm_states, K, V, B, B)) if gate is not None else False + + def run_serving_decode(): + fn( + A_log=A_log, + dt_bias=dt_bias, + q=q, + k=k, + v=v, + a=a, + b=b, + initial_state_source=ssm_states, + initial_state_indices=slot_indices, + cu_seqlens=cu_seqlens, + use_qk_l2norm_in_kernel=True, + softplus_beta=1.0, + softplus_threshold=20.0, + ) + + torch.cuda.synchronize() + run_serving_decode() + torch.cuda.synchronize() + with benchmark_with_power( + device=device, + kernel_func=run_serving_decode, + num_warmups=num_warmups, + num_runs=num_runs, + repeat_n=1, + ) as results: + log_perf( + item_list=[{**common_log_data, "latency": results["latency_ms"]}], + framework="TRTLLM", + version=trtllm_version, + device_name=torch.cuda.get_device_name(device), + op_name="gdn", + kernel_source=( + "fused_sigmoid_gating_delta_rule_update_flashinfer" + if uses_flashinfer + else "fused_sigmoid_gating_delta_rule_update_triton" + ), + perf_filename=perf_filename, + power_stats=results["power_stats"], + ) + def run_gdn_generation_benchmark( d_model: int, @@ -404,6 +534,11 @@ def run_gdn_generation_benchmark( conv_weight = torch.randn(conv_channels, d_conv, dtype=dtype, device=device) conv_bias = torch.randn(conv_channels, dtype=dtype, device=device) + # CUDA-graph replay only (see run_gdn_context_benchmark): a failed capture + # raises and the batch is counted as a failed cell below. + successful_points = 0 + failed_points = 0 + failures: list[str] = [] for batch_size in batch_size_list: if aic_debug: print(f" Benchmarking batch_size={batch_size}") @@ -431,12 +566,31 @@ def run_gdn_generation_benchmark( "model_name": model_name, } + # serving's pure-decode entry (dispatch-labeled); the rows below keep + # the conv update and the FLA recurrent fallback lane + _bench_serving_decode_entry( + batch_size=batch_size, + num_k_heads=num_k_heads, + head_k_dim=head_k_dim, + num_v_heads=num_v_heads, + head_v_dim=head_v_dim, + dtype=dtype, + device=device, + common_log_data=common_log_data, + trtllm_version=trtllm_version, + perf_filename=perf_filename, + num_warmups=num_warmups, + num_runs=num_runs, + ) + if aic_cached_inputs: k_input = torch.randn(batch_size, conv_channels, dtype=dtype, device=device) q = torch.randn(batch_size, 1, num_k_heads, head_k_dim, dtype=dtype, device=device) k = torch.randn(batch_size, 1, num_k_heads, head_k_dim, dtype=dtype, device=device) v = torch.randn(batch_size, 1, num_v_heads, head_v_dim, dtype=dtype, device=device) - g = torch.nn.functional.logsigmoid(torch.randn(batch_size, 1, num_v_heads, dtype=dtype, device=device)) + g = torch.nn.functional.logsigmoid( + torch.randn(batch_size, 1, num_v_heads, dtype=torch.float32, device=device) + ) # serving g is fp32 beta = torch.sigmoid(torch.randn(batch_size, 1, num_v_heads, dtype=dtype, device=device)) # --- Benchmark causal_conv1d_update --- @@ -453,7 +607,6 @@ def run_conv1d_update(_k=k_input, _cs=conv_state): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -466,48 +619,52 @@ def run_conv1d_update(_k=k_input, _cs=conv_state): power_stats=results["power_stats"], ) - # --- Benchmark fused_recurrent_gated_delta_rule --- - torch.cuda.synchronize() - fused_recurrent_gated_delta_rule( - q, - k, - v, - g, - beta, - initial_state=gdn_state, - output_final_state=True, - ) - torch.cuda.synchronize() - - def run_gdn_update(_q=q, _k=k, _v=v, _g=g, _beta=beta, _state=gdn_state): + if fused_recurrent_gated_delta_rule is None: + print( + "[gdn] fused_recurrent_gated_delta_rule row omitted: kernel absent in this trtllm (>=1.3.0rc29)" + ) + else: + # --- Benchmark fused_recurrent_gated_delta_rule --- + torch.cuda.synchronize() fused_recurrent_gated_delta_rule( - _q, - _k, - _v, - _g, - _beta, - initial_state=_state, + q, + k, + v, + g, + beta, + initial_state=gdn_state, output_final_state=True, ) + torch.cuda.synchronize() - with benchmark_with_power( - device=device, - kernel_func=run_gdn_update, - num_warmups=num_warmups, - num_runs=num_runs, - repeat_n=1, - allow_graph_fail=True, - ) as results: - log_perf( - item_list=[{**common_log_data, "latency": results["latency_ms"]}], - framework="TRTLLM", - version=trtllm_version, - device_name=torch.cuda.get_device_name(device), - op_name="gdn", - kernel_source="fused_recurrent_gated_delta_rule", - perf_filename=perf_filename, - power_stats=results["power_stats"], - ) + def run_gdn_update(_q=q, _k=k, _v=v, _g=g, _beta=beta, _state=gdn_state): + fused_recurrent_gated_delta_rule( + _q, + _k, + _v, + _g, + _beta, + initial_state=_state, + output_final_state=True, + ) + + with benchmark_with_power( + device=device, + kernel_func=run_gdn_update, + num_warmups=num_warmups, + num_runs=num_runs, + repeat_n=1, + ) as results: + log_perf( + item_list=[{**common_log_data, "latency": results["latency_ms"]}], + framework="TRTLLM", + version=trtllm_version, + device_name=torch.cuda.get_device_name(device), + op_name="gdn", + kernel_source="fused_recurrent_gated_delta_rule", + perf_filename=perf_filename, + power_stats=results["power_stats"], + ) else: input_pool = _make_input_pool( @@ -523,6 +680,7 @@ def run_gdn_update(_q=q, _k=k, _v=v, _g=g, _beta=beta, _state=gdn_state): dtype, device, ) + input_pool["g"] = [t.float() for t in input_pool["g"]] # serving g is fp32 for i in range(total_iters): input_pool["g"][i] = torch.nn.functional.logsigmoid(input_pool["g"][i]) input_pool["beta"][i] = torch.sigmoid(input_pool["beta"][i]) @@ -545,7 +703,6 @@ def run_conv1d_update(_pool=input_pool, _cs=conv_state, _idx=conv1d_iter_idx): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -558,52 +715,56 @@ def run_conv1d_update(_pool=input_pool, _cs=conv_state, _idx=conv1d_iter_idx): power_stats=results["power_stats"], ) - # --- Benchmark fused_recurrent_gated_delta_rule --- - torch.cuda.synchronize() - fused_recurrent_gated_delta_rule( - input_pool["q"][0], - input_pool["k"][0], - input_pool["v"][0], - input_pool["g"][0], - input_pool["beta"][0], - initial_state=gdn_state, - output_final_state=True, - ) - torch.cuda.synchronize() - - gdn_iter_idx = [0] - - def run_gdn_update(_pool=input_pool, _state=gdn_state, _idx=gdn_iter_idx): - idx = _idx[0] % total_iters - _idx[0] += 1 + if fused_recurrent_gated_delta_rule is None: + print( + "[gdn] fused_recurrent_gated_delta_rule row omitted: kernel absent in this trtllm (>=1.3.0rc29)" + ) + else: + # --- Benchmark fused_recurrent_gated_delta_rule --- + torch.cuda.synchronize() fused_recurrent_gated_delta_rule( - _pool["q"][idx], - _pool["k"][idx], - _pool["v"][idx], - _pool["g"][idx], - _pool["beta"][idx], - initial_state=_state, + input_pool["q"][0], + input_pool["k"][0], + input_pool["v"][0], + input_pool["g"][0], + input_pool["beta"][0], + initial_state=gdn_state, output_final_state=True, ) + torch.cuda.synchronize() - with benchmark_with_power( - device=device, - kernel_func=run_gdn_update, - num_warmups=num_warmups, - num_runs=num_runs, - repeat_n=1, - allow_graph_fail=True, - ) as results: - log_perf( - item_list=[{**common_log_data, "latency": results["latency_ms"]}], - framework="TRTLLM", - version=trtllm_version, - device_name=torch.cuda.get_device_name(device), - op_name="gdn", - kernel_source="fused_recurrent_gated_delta_rule", - perf_filename=perf_filename, - power_stats=results["power_stats"], - ) + gdn_iter_idx = [0] + + def run_gdn_update(_pool=input_pool, _state=gdn_state, _idx=gdn_iter_idx): + idx = _idx[0] % total_iters + _idx[0] += 1 + fused_recurrent_gated_delta_rule( + _pool["q"][idx], + _pool["k"][idx], + _pool["v"][idx], + _pool["g"][idx], + _pool["beta"][idx], + initial_state=_state, + output_final_state=True, + ) + + with benchmark_with_power( + device=device, + kernel_func=run_gdn_update, + num_warmups=num_warmups, + num_runs=num_runs, + repeat_n=1, + ) as results: + log_perf( + item_list=[{**common_log_data, "latency": results["latency_ms"]}], + framework="TRTLLM", + version=trtllm_version, + device_name=torch.cuda.get_device_name(device), + op_name="gdn", + kernel_source="fused_recurrent_gated_delta_rule", + perf_filename=perf_filename, + power_stats=results["power_stats"], + ) # Cleanup if aic_cached_inputs: @@ -612,11 +773,22 @@ def run_gdn_update(_pool=input_pool, _state=gdn_state, _idx=gdn_iter_idx): del input_pool, conv_state, gdn_state gc.collect() torch.cuda.empty_cache() + successful_points += 1 except Exception as e: + failed_points += 1 + failures.append(f"batch_size={batch_size}: {type(e).__name__}: {e}") print(f" Error at batch_size={batch_size}: {e}") continue + summary = f"ok={successful_points} error={failed_points} skip=0" + print(f"GDN generation summary: {summary}") + if failed_points or successful_points == 0: + raise RuntimeError( + f"TRTLLM GDN generation collection failed strict completeness: {summary}; " + f"failed cells: {'; '.join(failures)}" + ) + def run_gdn_torch( phase: str, @@ -647,17 +819,60 @@ def run_gdn_torch( contextlib.redirect_stderr(_devnull_file), ): import tensorrt_llm - from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule - from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule from tensorrt_llm._torch.modules.mamba.causal_conv1d import causal_conv1d_fn, causal_conv1d_update + # Extra FLA row (the vendored single-step recurrence). trtllm >=1.3.0rc29 + # removed fused_recurrent_gated_delta_rule from fla/fused_recurrent.py + # (only the *_update variants remain; serving never called the removed + # one — gdn_mixer.forward_decode uses fused_sigmoid_gating_delta_rule_update + # at rc23:670 and rc29:706 alike). When the kernel does not exist the row + # cannot exist: the serving-dispatch row below is still produced, and the + # omission is printed once per case, never silent. + try: + from tensorrt_llm._torch.modules.fla.fused_recurrent import fused_recurrent_gated_delta_rule + except ImportError: + fused_recurrent_gated_delta_rule = None + + # Serving dispatch, never the vendored kernel directly: the GDN mixer + # resolves chunk_gated_delta_rule to FlashInfer's cute-DSL chunk kernel on + # SM90/SM100 (TLLM_USE_FLASHINFER_GDN_PREFILL=1 default) and to the FLA + # Triton kernel elsewhere (_torch/modules/mamba/gdn_mixer.py:57-70 + # @1.3.0rc23; arch gate _utils.is_flashinfer_gdn_supported_arch). Older + # builds have no resolver and only the Triton kernel — import that. + chunk_label = "chunk_gated_delta_rule" + try: + from tensorrt_llm._torch.modules.mamba.gdn_mixer import ( + _resolve_chunk_gated_delta_rule, + chunk_gated_delta_rule, + ) + + if "flashinfer" in getattr(_resolve_chunk_gated_delta_rule(), "__module__", ""): + chunk_label = "chunk_gated_delta_rule_flashinfer" + except ImportError: + from tensorrt_llm._torch.modules.fla.chunk import chunk_gated_delta_rule + # Pure-decode serving entry (gdn_mixer.py:670): fused sigmoid gating + + # recurrent update, itself dispatching to FlashInfer's bf16-state GDN + # decode kernel when _can_use_flashinfer_gdn_decode holds + # (fla/fused_sigmoid_gating_recurrent.py:205-238,452-458 @1.3.0rc23). + try: + from tensorrt_llm._torch.modules.fla.fused_sigmoid_gating_recurrent import ( + _can_use_flashinfer_gdn_decode, + fused_sigmoid_gating_delta_rule_update, + ) + except ImportError: + _can_use_flashinfer_gdn_decode = None + fused_sigmoid_gating_delta_rule_update = None + globals().update( { "tensorrt_llm": tensorrt_llm, "causal_conv1d_fn": causal_conv1d_fn, "causal_conv1d_update": causal_conv1d_update, "chunk_gated_delta_rule": chunk_gated_delta_rule, + "_CHUNK_KERNEL_SOURCE": chunk_label, "fused_recurrent_gated_delta_rule": fused_recurrent_gated_delta_rule, + "fused_sigmoid_gating_delta_rule_update": fused_sigmoid_gating_delta_rule_update, + "_can_use_flashinfer_gdn_decode": _can_use_flashinfer_gdn_decode, } ) diff --git a/python/aisimulate/collector/trtllm/collect_gemm.py b/python/aisimulate/collector/trtllm/collect_gemm.py index 7053d953a7..2465b9e971 100644 --- a/python/aisimulate/collector/trtllm/collect_gemm.py +++ b/python/aisimulate/collector/trtllm/collect_gemm.py @@ -19,13 +19,12 @@ import tensorrt_llm import torch import torch.nn.functional as F -from case_generator import get_gemm_case_specs +from collector.case_generator import get_gemm_case_specs +from collector.helper import benchmark_with_power, get_sm_version, log_perf from tensorrt_llm._torch.modules.linear import Linear from tensorrt_llm._utils import is_sm_100f from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig -from helper import benchmark_with_power, get_sm_version, log_perf - def pad_up(x: int, y: int) -> int: return ((x + y - 1) // y) * y @@ -50,7 +49,47 @@ def _compute_fp8_block_weight_scales(weight: torch.Tensor, group_size: int) -> t # Per-worker-process weight cache for large shapes (outside_loop_count == 1). # Keyed by (gemm_type, n, k). Only the most recently used entry is kept so # that GPU memory is not accumulated across (n, k) groups. -_weight_cache: dict = {} +# One worker builds the Linear modules (and their weights) once per +# (gemm_type, n, k) and reuses them for every m that follows on this worker: +# the modules do not depend on m, and the per-case rebuild was the shard's +# cost — nvfp4 unswizzles its block scales on the CPU and the sub-L2 shapes +# build up to 5 copies per case (b300 base run, job 471159002: gemm/nvfp4 +# 1.3 GPU-s/row and gemm/fp8_block 1.7 GPU-s/row against 0.04 for bf16). +# get_gemm_test_cases orders the plan by (n, k) then gemm_type then m, so a +# worker sees runs of equal keys. The cache holds one key at a time, so peak +# memory is unchanged. +_gemm_op_cache: dict = {} + + +def _gemm_ops(gemm_type: str, n: int, k: int, *, quant_config, group_size, outside_loop_count: int, device, dtype, x): + """The ``outside_loop_count`` Linear copies for (gemm_type, n, k), built on first use.""" + key = (gemm_type, n, k, outside_loop_count) + op_list = _gemm_op_cache.get(key) + if op_list is not None: + return op_list + _gemm_op_cache.clear() # release the previous key's modules before building + torch.cuda.empty_cache() + op_list = [] + for _i in range(outside_loop_count): + # nvfp4's per-tensor activation scale is a calibration constant in + # serving; the first x of this key provides it, and the kernel's + # timing does not depend on its value. + weights = _build_weights(gemm_type, n, k, device, dtype, group_size, x) + gemm = Linear( + k, + n, + bias=False, + dtype=dtype, + quant_config=quant_config, + force_dynamic_quantization=False, + ) + gemm.load_weights([weights]) + if gemm_type == "fp8_block" and callable(getattr(gemm, "post_load_weights", None)): + gemm.post_load_weights() + gemm.to(torch.device(device)) + op_list.append(gemm) + _gemm_op_cache[key] = op_list + return op_list def _get_l2_cache_bytes(device_id: int = 0) -> int: @@ -157,6 +196,17 @@ def run_gemm(gemm_type, m, n, k, *, perf_filename, device="cuda:0"): torch.set_default_device(device) sm_version = get_sm_version() + if gemm_type == "fp8" and (n % 16 or k % 16): + # FP8QDQLinearMethod.apply -> torch.ops.trtllm.cublas_scaled_mm / cuda_scaled_mm, + # which TORCH_CHECK the 16-element alignment of the output row stride (= n) + # and of mat_b's column stride (= k): cpp/tensorrt_llm/thop/cublasScaledMM.cpp:260 + # and cutlassScaledMM.cpp:152 @1.3.0rc29. The shared GEMM grid carries + # n in {1, 4, 8, 12, 24} (1,110 cases per system on gb200/gb300). + raise RuntimeError( + "FIXME(kernel-limit): TRT-LLM fp8 GEMM (cublas/cutlass scaled_mm) requires n and k to be " + f"multiples of 16 (out.strides()[0] % 16 == 0 && mat_b.strides()[1] % 16 == 0), got n={n}, k={k}" + ) + dtype = torch.bfloat16 x = torch.randn((m, k), dtype=dtype).to(torch.device(device)) @@ -167,7 +217,12 @@ def run_gemm(gemm_type, m, n, k, *, perf_filename, device="cuda:0"): group_size = 128 qc = QuantConfig(quant_algo=QuantAlgo.FP8_BLOCK_SCALES, group_size=group_size) elif gemm_type == "nvfp4": - group_size = 128 + # NVFP4 scale blocks are 16 elements (the weights above are quantized with + # fp4_quantize(..., 16, ...)); 1.3.0rc29 NVFP4LinearMethod.resolve_scaling_vector_size + # rejects any other declared group_size ("supports NVFP4 scale blocks of (16,) elements, + # but the checkpoint declares group_size=128" — first seen on B200 2026-10-04; on Hopper + # the nvfp4 lane is a platform floor and never constructed the module). + group_size = 16 qc = QuantConfig(quant_algo=QuantAlgo.NVFP4, group_size=group_size) else: qc = None @@ -177,63 +232,20 @@ def run_gemm(gemm_type, m, n, k, *, perf_filename, device="cuda:0"): _bytes_per_elem = {"bfloat16": 2, "fp8": 1, "fp8_block": 1, "nvfp4": 0.5} _weight_bytes = int(n * k * _bytes_per_elem[gemm_type]) outside_loop_count = max(1, min(5, math.ceil(_l2_cache_bytes / _weight_bytes))) - op_list = [] + op_list = _gemm_ops( + gemm_type, + n, + k, + quant_config=qc, + group_size=group_size, + outside_loop_count=outside_loop_count, + device=device, + dtype=dtype, + x=x, + ) + for op in op_list: + op.forward(x) # dry run at this m (lazy init / JIT) before graph capture - if outside_loop_count == 1: - # Large weight (exceeds GPU L2): reuse cached tensor across consecutive - # tasks with the same (gemm_type, n, k) to avoid re-running the slow - # torch.randn call for every batch-size variant. - cache_key = (gemm_type, n, k) - if cache_key not in _weight_cache: - _weight_cache.clear() # release the previous (n,k) weight - _weight_cache[cache_key] = _build_weights(gemm_type, n, k, device, dtype, group_size, x) - - weights = _weight_cache[cache_key] - - # For nvfp4 the input_scale is x-dependent; recompute it cheaply. - if gemm_type == "nvfp4": - x_sf_global = (448 * 6) / x.abs().max().float() - weights = {**weights, "input_scale": 1.0 / x_sf_global.cpu()} - - # The fp8 weights carry no input_scale, so TRT-LLM's FP8QDQ method runs - # its dynamic-quantization path and the collected fp8 GEMM includes the - # compute-scale overhead. AIC derives fp8_static at query time by - # subtracting that overhead, so no separate fp8_static GEMM is collected. - gemm = Linear( - k, - n, - bias=False, - dtype=dtype, - quant_config=qc, - force_dynamic_quantization=False, - ) - gemm.load_weights([weights]) - if gemm_type == "fp8_block" and callable(getattr(gemm, "post_load_weights", None)): - gemm.post_load_weights() - gemm.to(torch.device(device)) - gemm.forward(x) # dry run to init - op_list.append(gemm) - else: - # Small weight: create distinct copies so that repeated calls span more - # than the GPU L2 cache, giving a realistic memory-pressure measurement. - for _i in range(outside_loop_count): - weights = _build_weights(gemm_type, n, k, device, dtype, group_size, x) - gemm = Linear( - k, - n, - bias=False, - dtype=dtype, - quant_config=qc, - force_dynamic_quantization=False, - ) - gemm.load_weights([weights]) - if gemm_type == "fp8_block" and callable(getattr(gemm, "post_load_weights", None)): - gemm.post_load_weights() - gemm.to(torch.device(device)) - gemm.forward(x) # dry run to init - op_list.append(gemm) - - # Use benchmark_with_power context manager def kernel_func(): for op in op_list: op.forward(x) diff --git a/python/aisimulate/collector/trtllm/collect_mamba2.py b/python/aisimulate/collector/trtllm/collect_mamba2.py index c0bbe59073..a77c37c01e 100644 --- a/python/aisimulate/collector/trtllm/collect_mamba2.py +++ b/python/aisimulate/collector/trtllm/collect_mamba2.py @@ -125,6 +125,117 @@ def get_mamba2_test_cases(): return test_cases +def _token_major_prefill_conv() -> bool: + """Serving's prefill conv layout at this runtime. + + 1.3.0rc29 Mamba2Mixer defaults TRTLLM_MAMBA_TOKEN_MAJOR_CONV=1 (mamba2_mixer.py:319) + and feeds causal_conv1d_fn a channel-last VIEW of the token-major projection with a + separate token-major ``out`` (mamba2_mixer.py:541-569; causal_conv1d.py:26-60 picks + causal_conv1d_channellast_fwd_kernel off x.stride(0) == 1). Older runtimes have no + ``out`` parameter and run the channel-major kernel on (batch, dim, seq) inputs. The + H20 mamba2_ctx gate (2026-10-02) diverged on exactly this kernel pair. + """ + import inspect + + return ( + "out" in inspect.signature(causal_conv1d_fn).parameters + and os.environ.get("TRTLLM_MAMBA_TOKEN_MAJOR_CONV", "1") == "1" + ) + + +def _prefill_conv_token_major_inputs(xbc_batched, batch_size, seq_len, device): + """(batch, dim, seq) random input -> the serving varlen token-major buffer + [tokens, dim] (sequences concatenated), its ``out`` twin and the varlen + metadata serving passes (mamba2_mixer.py:536-569: cu_seqlens int32, + cache_indices, has_initial_state=False for a cache-cold prefill).""" + conv_dim = xbc_batched.shape[1] + x_tm = xbc_batched.permute(0, 2, 1).reshape(batch_size * seq_len, conv_dim).contiguous() + out_tm = torch.empty_like(x_tm) + cu_seqlens = torch.arange(0, batch_size * seq_len + 1, seq_len, dtype=torch.int32, device=device) + cache_indices = torch.arange(batch_size, dtype=torch.int32, device=device) + has_initial_state = torch.zeros(batch_size, dtype=torch.bool, device=device) + return x_tm, out_tm, cu_seqlens, cache_indices, has_initial_state + + +def _run_prefill_conv_token_major( + x_tm, out_tm, conv_weight, conv_bias, conv_state, cu_seqlens, cache_indices, has_initial_state +): + causal_conv1d_fn( + x_tm.t(), + conv_weight, + conv_bias, + query_start_loc=cu_seqlens, + cache_indices=cache_indices, + has_initial_state=has_initial_state, + conv_states=conv_state, + activation="silu", + out=out_tm.t(), + ) + + +# Both SSM kernels take the output buffer from the caller: serving hands them a +# view of one preallocated per-forward buffer (mamba2_mixer.py:453-457 alloc, +# :634 prefill `out=`, :906 decode `out=` @1.3.0rc29; rc20 :335/:403/:629) and +# neither kernel allocates one when it is missing — ssd_chunk_scan.py:667 +# `assert out.shape == x.shape` and selective_state_update.py `out.dim()` +# (rc20 :85, rc29 same) both fail on None. The collector omitted `out` until +# 2026-10-12, so every scan/state-update cell raised and the per-cell except +# dropped the lane: no shipped mamba2 table has those rows. + +# Decode SSM state update: serving picks FlashInfer's kernel or TRT-LLM's own +# Triton kernel per layer geometry — Mamba2Mixer.__init__ (mamba2_mixer.py: +# 284-293 gate, :322-328 selection @1.3.0rc29; rc20 mixer :14-23 had the ratio +# list [1, 8, 16]). The collector measures the full (unsharded) layer, i.e. the +# TP=1 geometry the gate sees as tp_nheads / tp_ngroups. +# FIXME(re-verify): re-check the three lists on every TRT-LLM version bump. +_FLASHINFER_SSU_HEAD_DIMS = (64, 128) +_FLASHINFER_SSU_HEAD_GROUP_RATIOS = (1, 2, 4, 8, 16, 32, 64) +_FLASHINFER_SSU_D_STATES = (64, 128, 256) +SSU_KERNEL_SOURCE_FLASHINFER = "selective_state_update_flashinfer" +SSU_KERNEL_SOURCE_TRITON = "selective_state_update_triton" + + +def _serving_selects_flashinfer_ssu(nheads: int, n_groups: int, head_dim: int, d_state: int) -> bool: + head_group_ratio = nheads // n_groups if n_groups > 0 else 0 + return ( + head_dim in _FLASHINFER_SSU_HEAD_DIMS + and head_group_ratio in _FLASHINFER_SSU_HEAD_GROUP_RATIOS + and d_state in _FLASHINFER_SSU_D_STATES + ) + + +def _decode_state_update_kernel(nheads: int, n_groups: int, head_dim: int, d_state: int): + """The (callable, kernel_source) serving's Mamba2Mixer runs for this layer's decode. + + kernel_source records the function serving calls (`selective_state_update_fi` + from flashinfer.mamba, or TRT-LLM's own Triton `selective_state_update`), the + way the GDN tables record `chunk_gated_delta_rule_{flashinfer,triton}`. No + fallback: if the FlashInfer import fails, serving fails identically at mixer + import (mamba2_mixer.py:23 @1.3.0rc29). + """ + if _serving_selects_flashinfer_ssu(nheads, n_groups, head_dim, d_state): + from flashinfer.mamba import selective_state_update as selective_state_update_flashinfer + + return selective_state_update_flashinfer, SSU_KERNEL_SOURCE_FLASHINFER + return selective_state_update, SSU_KERNEL_SOURCE_TRITON + + +def _decode_dt(batch_size: int, nheads: int, head_dim: int, device) -> torch.Tensor: + """Decode dt laid out the way serving hands it to the state-update kernel. + + Serving casts the per-head dt to fp32 and broadcasts it over head_dim with + einops.repeat (mamba2_mixer.py:743-746 and :764 @1.3.0rc29): an expanded + VIEW with strides (nheads, 1, 0), like the A / D / dt_bias buffers. FlashInfer's + binding checks exactly that layout (csrc/selective_state_update.cu:187-188 + @v0.6.18: dt.stride(1) == 1, dt.stride(2) == 0 — the b200 diagnostic on + 76ce215e failed every decode cell with "dt.stride(1) must be 1, got 64"), and + TRT-LLM's Triton kernel keys its tie_hdim fast path off dt.stride(-1) == 0. + A dense (batch, nheads, head_dim) tensor is rejected by the former and + mis-tuned for the latter. + """ + return repeat(torch.randn(batch_size, nheads, dtype=torch.float32, device=device), "b h -> b h p", p=head_dim) + + def run_mamba2_context_benchmark( d_model: int, d_state: int, @@ -146,7 +257,9 @@ def run_mamba2_context_benchmark( This benchmarks: 1. causal_conv1d_fn - Conv1D for context phase - 2. mamba_chunk_scan_combined - SSM scan + 2. mamba_chunk_scan_combined - SSM scan (the one entry point serving calls; + it routes to FlashInfer's SSD on SM100-family parts itself, + ssd_combined.py:469-497 @1.3.0rc29) Together these represent the core compute of a Mamba2 layer (excluding in_proj/out_proj GEMMs which use existing GEMM model). @@ -176,6 +289,17 @@ def run_mamba2_context_benchmark( D = torch.randn(nheads, device=device) # noqa: N806 dt_bias = torch.rand(nheads, device=device) - 4.0 + # Every cell is timed under CUDA-graph replay; a failed capture raises + # (benchmark_with_power default allow_graph_fail=False) and the cell is + # counted as failed below. Until 2026-10-12 allow_graph_fail=True timed the + # Python launch loop eagerly and shipped that floor as the kernel latency, + # and the per-cell `except` dropped a failed kernel lane with no failure + # record: every shipped mamba2 table (rc20, all eight systems) and the rc29 + # b200 package carry only the causal_conv1d lanes. + successful_points = 0 + failed_points = 0 + failures: list[str] = [] + # Sweep over batch_size x seq_len combinations for batch_size in batch_size_list: for seq_len in seq_len_list: @@ -215,20 +339,34 @@ def run_mamba2_context_benchmark( C = torch.randn(batch_size, seq_len, n_groups, d_state, dtype=dtype, device=device) # noqa: N806 # --- Benchmark causal_conv1d_fn --- + conv_kernel_source = "causal_conv1d_fn" + if _token_major_prefill_conv(): + conv_kernel_source = "causal_conv1d_fn_token_major" + x_tm, out_tm, cu_seqlens, cache_idx, has_init = _prefill_conv_token_major_inputs( + xbc_input, batch_size, seq_len, device + ) + del xbc_input + + def run_conv1d( + _x=x_tm, _o=out_tm, _conv_state=conv_state, _cu=cu_seqlens, _ci=cache_idx, _hi=has_init + ): + _run_prefill_conv_token_major(_x, _o, conv_weight, conv_bias, _conv_state, _cu, _ci, _hi) + + else: + + def run_conv1d(_xbc=xbc_input, _conv_state=conv_state): + causal_conv1d_fn(_xbc, conv_weight, conv_bias, activation="silu", conv_states=_conv_state) + torch.cuda.synchronize() - causal_conv1d_fn(xbc_input, conv_weight, conv_bias, activation="silu", conv_states=conv_state) + run_conv1d() torch.cuda.synchronize() - def run_conv1d(_xbc=xbc_input, _conv_state=conv_state): - causal_conv1d_fn(_xbc, conv_weight, conv_bias, activation="silu", conv_states=_conv_state) - with benchmark_with_power( device=device, kernel_func=run_conv1d, num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -236,12 +374,15 @@ def run_conv1d(_xbc=xbc_input, _conv_state=conv_state): version=tensorrt_llm.__version__, device_name=torch.cuda.get_device_name(device), op_name="mamba2", - kernel_source="causal_conv1d_fn", + kernel_source=conv_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ) # --- Benchmark mamba_chunk_scan_combined --- + # Caller-owned output, as serving's preallocated buffer view + # (mamba2_mixer.py:634 @1.3.0rc29); the kernel asserts on it. + ssm_out = torch.empty_like(x) torch.cuda.synchronize() mamba_chunk_scan_combined( x, @@ -255,10 +396,11 @@ def run_conv1d(_xbc=xbc_input, _conv_state=conv_state): dt_bias=dt_bias, dt_softplus=True, return_final_states=True, + out=ssm_out, ) torch.cuda.synchronize() - def run_ssm_scan(_x=x, _dt=dt, _b=B, _c=C): + def run_ssm_scan(_x=x, _dt=dt, _b=B, _c=C, _out=ssm_out): mamba_chunk_scan_combined( _x, _dt, @@ -271,6 +413,7 @@ def run_ssm_scan(_x=x, _dt=dt, _b=B, _c=C): dt_bias=dt_bias, dt_softplus=True, return_final_states=True, + out=_out, ) with benchmark_with_power( @@ -279,7 +422,6 @@ def run_ssm_scan(_x=x, _dt=dt, _b=B, _c=C): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -307,29 +449,55 @@ def run_ssm_scan(_x=x, _dt=dt, _b=B, _c=C): device, ) + conv_kernel_source = "causal_conv1d_fn" + conv1d_iter_idx = [0] + if _token_major_prefill_conv(): + conv_kernel_source = "causal_conv1d_fn_token_major" + tm_pool = [] + for _xb in input_pool["xbc"]: + x_tm, _, cu_seqlens, cache_idx, has_init = _prefill_conv_token_major_inputs( + _xb, batch_size, seq_len, device + ) + tm_pool.append(x_tm) + input_pool["xbc"] = tm_pool # channel-major originals released + out_tm = torch.empty_like(tm_pool[0]) + + def run_conv1d( + _pool=tm_pool, + _o=out_tm, + _conv_state=conv_state, + _idx=conv1d_iter_idx, + _cu=cu_seqlens, + _ci=cache_idx, + _hi=has_init, + ): + idx = _idx[0] % total_iters + _idx[0] += 1 + _run_prefill_conv_token_major( + _pool[idx], _o, conv_weight, conv_bias, _conv_state, _cu, _ci, _hi + ) + + else: + + def run_conv1d(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter_idx): + idx = _idx[0] % total_iters + _idx[0] += 1 + causal_conv1d_fn( + _pool["xbc"][idx], conv_weight, conv_bias, activation="silu", conv_states=_conv_state + ) + # Warmup with first set of inputs torch.cuda.synchronize() - causal_conv1d_fn( - input_pool["xbc"][0], conv_weight, conv_bias, activation="silu", conv_states=conv_state - ) + run_conv1d() + conv1d_iter_idx[0] = 0 torch.cuda.synchronize() - conv1d_iter_idx = [0] - - def run_conv1d(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter_idx): - idx = _idx[0] % total_iters - _idx[0] += 1 - causal_conv1d_fn( - _pool["xbc"][idx], conv_weight, conv_bias, activation="silu", conv_states=_conv_state - ) - with benchmark_with_power( device=device, kernel_func=run_conv1d, num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -337,12 +505,15 @@ def run_conv1d(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter_idx): version=tensorrt_llm.__version__, device_name=torch.cuda.get_device_name(device), op_name="mamba2", - kernel_source="causal_conv1d_fn", + kernel_source=conv_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ) # --- Benchmark mamba_chunk_scan_combined --- + # One caller-owned output buffer for every iteration, as + # serving's preallocated view (mamba2_mixer.py:634 @1.3.0rc29). + ssm_out = torch.empty_like(input_pool["x"][0]) torch.cuda.synchronize() mamba_chunk_scan_combined( input_pool["x"][0], @@ -356,12 +527,13 @@ def run_conv1d(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter_idx): dt_bias=dt_bias, dt_softplus=True, return_final_states=True, + out=ssm_out, ) torch.cuda.synchronize() ssm_iter_idx = [0] - def run_ssm_scan(_pool=input_pool, _idx=ssm_iter_idx): + def run_ssm_scan(_pool=input_pool, _idx=ssm_iter_idx, _out=ssm_out): idx = _idx[0] % total_iters _idx[0] += 1 @@ -377,6 +549,7 @@ def run_ssm_scan(_pool=input_pool, _idx=ssm_iter_idx): dt_bias=dt_bias, dt_softplus=True, return_final_states=True, + out=_out, ) with benchmark_with_power( @@ -385,7 +558,6 @@ def run_ssm_scan(_pool=input_pool, _idx=ssm_iter_idx): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -400,16 +572,27 @@ def run_ssm_scan(_pool=input_pool, _idx=ssm_iter_idx): # Cleanup if aic_cached_inputs: - del x, dt, B, C, xbc_input, conv_state + del x, dt, B, C, xbc_input, conv_state, ssm_out else: - del input_pool, conv_state + del input_pool, conv_state, ssm_out gc.collect() torch.cuda.empty_cache() + successful_points += 1 except Exception as e: + failed_points += 1 + failures.append(f"batch_size={batch_size}, seq_len={seq_len}: {type(e).__name__}: {e}") print(f" Error at batch_size={batch_size}, seq_len={seq_len}: {e}") continue + summary = f"ok={successful_points} error={failed_points} skip=0" + print(f"Mamba2 context summary: {summary}") + if failed_points or successful_points == 0: + raise RuntimeError( + f"TRTLLM Mamba2 context collection failed strict completeness: {summary}; " + f"failed cells: {'; '.join(failures)}" + ) + def run_mamba2_generation_benchmark( d_model: int, @@ -429,7 +612,9 @@ def run_mamba2_generation_benchmark( This benchmarks: 1. causal_conv1d_update - Conv1D state update for decode - 2. selective_state_update - SSM state update + 2. the SSM state update serving selects for this geometry: FlashInfer's + selective_state_update or TRT-LLM's Triton one (see + _decode_state_update_kernel); kernel_source names the one that ran Together these represent the core compute of a Mamba2 layer during decode (excluding in_proj/out_proj GEMMs which use existing GEMM model). @@ -464,6 +649,14 @@ def run_mamba2_generation_benchmark( D = repeat(d_base, "h -> h p", p=head_dim) # noqa: N806 dt_bias = repeat(dt_bias_base, "h -> h p", p=head_dim) + # The decode kernel serving's gate selects for this geometry, and its label. + ssu_func, ssu_kernel_source = _decode_state_update_kernel(nheads, n_groups, head_dim, d_state) + + # Same contract as the context runner: graph-timed or counted as failed. + successful_points = 0 + failed_points = 0 + failures: list[str] = [] + for batch_size in batch_size_list: if aic_debug == 1: print(f" Benchmarking batch_size={batch_size}") @@ -477,6 +670,11 @@ def run_mamba2_generation_benchmark( # SSM state: [batch, nheads, head_dim, d_state] ssm_state = torch.randn(batch_size, nheads, head_dim, d_state, dtype=dtype, device=device) + # Serving addresses the state cache through slot indices and hands + # the kernel a view of its preallocated output buffer + # (mamba2_mixer.py:901-907 @1.3.0rc29); the state here is dense, so + # the slots are the identity. + state_batch_indices = torch.arange(batch_size, dtype=torch.int32, device=device) if aic_cached_inputs: # Common log data for both kernels @@ -497,9 +695,10 @@ def run_mamba2_generation_benchmark( # Cached mode: use same inputs for all iterations (saves memory) xbc_input = torch.randn(batch_size, conv_dim, dtype=dtype, device=device) x = torch.randn(batch_size, nheads, head_dim, dtype=dtype, device=device) - dt = torch.randn(batch_size, nheads, head_dim, dtype=dtype, device=device) + dt = _decode_dt(batch_size, nheads, head_dim, device) B = torch.randn(batch_size, n_groups, d_state, dtype=dtype, device=device) # noqa: N806 C = torch.randn(batch_size, n_groups, d_state, dtype=dtype, device=device) # noqa: N806 + ssu_out = torch.empty_like(x) # --- Benchmark causal_conv1d_update --- torch.cuda.synchronize() @@ -515,7 +714,6 @@ def run_conv1d_update(_xbc=xbc_input, _conv_state=conv_state): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -528,9 +726,9 @@ def run_conv1d_update(_xbc=xbc_input, _conv_state=conv_state): power_stats=results["power_stats"], ) - # --- Benchmark selective_state_update --- + # --- Benchmark the serving-selected selective_state_update --- torch.cuda.synchronize() - selective_state_update( + ssu_func( ssm_state, x, dt, @@ -541,11 +739,13 @@ def run_conv1d_update(_xbc=xbc_input, _conv_state=conv_state): z=None, dt_bias=dt_bias, dt_softplus=True, + state_batch_indices=state_batch_indices, + out=ssu_out, ) torch.cuda.synchronize() - def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C): - selective_state_update( + def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C, _out=ssu_out): + ssu_func( _ssm_state, _x, _dt, @@ -556,6 +756,8 @@ def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C): z=None, dt_bias=dt_bias, dt_softplus=True, + state_batch_indices=state_batch_indices, + out=_out, ) with benchmark_with_power( @@ -564,7 +766,6 @@ def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -572,7 +773,7 @@ def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C): version=tensorrt_llm.__version__, device_name=torch.cuda.get_device_name(device), op_name="mamba2", - kernel_source="selective_state_update", + kernel_source=ssu_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ) @@ -597,7 +798,6 @@ def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C): { "xbc": (batch_size, conv_dim), "x": (batch_size, nheads, head_dim), - "dt": (batch_size, nheads, head_dim), "B": (batch_size, n_groups, d_state), "C": (batch_size, n_groups, d_state), }, @@ -605,6 +805,8 @@ def run_state_update(_ssm_state=ssm_state, _x=x, _dt=dt, _b=B, _c=C): dtype, device, ) + input_pool["dt"] = [_decode_dt(batch_size, nheads, head_dim, device) for _ in range(total_iters)] + ssu_out = torch.empty_like(input_pool["x"][0]) # --- Benchmark causal_conv1d_update --- torch.cuda.synchronize() @@ -624,7 +826,6 @@ def run_conv1d_update(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -637,9 +838,9 @@ def run_conv1d_update(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter power_stats=results["power_stats"], ) - # --- Benchmark selective_state_update --- + # --- Benchmark the serving-selected selective_state_update --- torch.cuda.synchronize() - selective_state_update( + ssu_func( ssm_state, input_pool["x"][0], input_pool["dt"][0], @@ -650,15 +851,17 @@ def run_conv1d_update(_pool=input_pool, _conv_state=conv_state, _idx=conv1d_iter z=None, dt_bias=dt_bias, dt_softplus=True, + state_batch_indices=state_batch_indices, + out=ssu_out, ) torch.cuda.synchronize() ssm_iter_idx = [0] - def run_state_update(_pool=input_pool, _ssm_state=ssm_state, _idx=ssm_iter_idx): + def run_state_update(_pool=input_pool, _ssm_state=ssm_state, _idx=ssm_iter_idx, _out=ssu_out): idx = _idx[0] % total_iters _idx[0] += 1 - selective_state_update( + ssu_func( _ssm_state, _pool["x"][idx], _pool["dt"][idx], @@ -669,6 +872,8 @@ def run_state_update(_pool=input_pool, _ssm_state=ssm_state, _idx=ssm_iter_idx): z=None, dt_bias=dt_bias, dt_softplus=True, + state_batch_indices=state_batch_indices, + out=_out, ) with benchmark_with_power( @@ -677,7 +882,6 @@ def run_state_update(_pool=input_pool, _ssm_state=ssm_state, _idx=ssm_iter_idx): num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: log_perf( item_list=[{**common_log_data, "latency": results["latency_ms"]}], @@ -685,23 +889,34 @@ def run_state_update(_pool=input_pool, _ssm_state=ssm_state, _idx=ssm_iter_idx): version=tensorrt_llm.__version__, device_name=torch.cuda.get_device_name(device), op_name="mamba2", - kernel_source="selective_state_update", + kernel_source=ssu_kernel_source, perf_filename=perf_filename, power_stats=results["power_stats"], ) # Cleanup if aic_cached_inputs: - del ssm_state, conv_state, x, dt, B, C, xbc_input + del ssm_state, conv_state, x, dt, B, C, xbc_input, ssu_out, state_batch_indices else: - del ssm_state, conv_state, input_pool + del ssm_state, conv_state, input_pool, ssu_out, state_batch_indices gc.collect() torch.cuda.empty_cache() + successful_points += 1 except Exception as e: + failed_points += 1 + failures.append(f"batch_size={batch_size}: {type(e).__name__}: {e}") print(f" Error at batch_size={batch_size}: {e}") continue + summary = f"ok={successful_points} error={failed_points} skip=0" + print(f"Mamba2 generation summary: {summary}") + if failed_points or successful_points == 0: + raise RuntimeError( + f"TRTLLM Mamba2 generation collection failed strict completeness: {summary}; " + f"failed cells: {'; '.join(failures)}" + ) + def _make_input_pool( shapes: dict[str, tuple[int, ...]], diff --git a/python/aisimulate/collector/trtllm/collect_mla.py b/python/aisimulate/collector/trtllm/collect_mla.py index a7faa6105f..3d08d5955e 100644 --- a/python/aisimulate/collector/trtllm/collect_mla.py +++ b/python/aisimulate/collector/trtllm/collect_mla.py @@ -17,13 +17,29 @@ import torch from collector.case_generator import get_context_mla_case_specs, get_generation_mla_case_specs from collector.helper import benchmark_with_power, get_sm_version, log_perf -from tensorrt_llm._torch.attention_backend.interface import ( - AttentionInputType, - MLAParams, - PositionalEmbeddingParams, - RopeParams, -) -from tensorrt_llm._torch.attention_backend.utils import get_attention_backend + +# trtllm >=1.3.0rc29 moved _torch.attention_backend.* -> _torch.attention.backends.* and +# _torch.modules.fused_moe -> _torch.moe.fused_moe (the old package root is a deprecation shim +# without submodules). Path-only compat: same classes, same kernels (layer_permissions.md +# 'API-compat shims may only change HOW the same kernel is constructed'). +try: + from tensorrt_llm._torch.attention.backends.interface import ( + AttentionInputType, + MLAParams, + PositionalEmbeddingParams, + RopeParams, + ) +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.interface import ( + AttentionInputType, + MLAParams, + PositionalEmbeddingParams, + RopeParams, + ) +try: + from tensorrt_llm._torch.attention.backends.utils import get_attention_backend +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.utils import get_attention_backend from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.pyexecutor.resource_manager import KVCacheManager from tensorrt_llm.bindings.executor import KvCacheConfig @@ -208,8 +224,8 @@ def run_mla( # (cpp/kernels/fmha_v2/setup.py:6926-6931@v1.3.0rc20); the hand-added # sm89 576x512 generation cubin entry from 1.0.0 was dropped in the same # window. Layout-level, so it kills every dtype at once. Fail closed with - # a cited, classified raise (Gemma4/DSA precedent). Re-verify on the next - # framework version bump. + # a cited, classified raise (Gemma4/DSA precedent). Re-verified on 1.3.0rc29 / L40 2026-10-04 (the module + # collector's twin guard, collect_mla_module.py): same assert, now attentionOp.cpp:3234 — still SM89-less. if get_sm_version() < 90: phase = "context" if is_context_phase else "generation" raise ValueError( @@ -247,6 +263,15 @@ def run_mla( assert num_heads % tp_size == 0, "num_heads != N * tp_size" num_heads = num_heads // tp_size num_kv_heads = num_heads + # FIXME(kernel-limit): TRT-LLM's MLA decode FMHA on SM100/103 tiles 16 q-heads per CTA and + # rejects head counts above 16 that are not a multiple of 16 ("Internal error numHeadsQ=24, + # numHeadsPerCta=16", 724 cases = every heads-24 decode cell of the 96-head model at tp4, + # b200 rc29 job 469988228). Classified before the layer is built. + if not is_context_phase and get_sm_version() in (100, 103) and num_heads > 16 and num_heads % 16: + raise RuntimeError( + "FIXME(kernel-limit): TRT-LLM MLA decode on SM100/103 needs per-rank heads <= 16 or a multiple " + f"of 16 (numHeadsPerCta=16); got {num_heads}" + ) context_sequence_lengths = [input_len for _ in range(batch_size)] diff --git a/python/aisimulate/collector/trtllm/collect_mla_bmm.py b/python/aisimulate/collector/trtllm/collect_mla_bmm.py index 272bcced0b..0581fe3bfd 100644 --- a/python/aisimulate/collector/trtllm/collect_mla_bmm.py +++ b/python/aisimulate/collector/trtllm/collect_mla_bmm.py @@ -36,9 +36,8 @@ import tensorrt_llm import tensorrt_llm.quantization.utils.fp8_utils as fp8_utils import torch -from case_generator import get_mla_bmm_case_specs - -from helper import benchmark_with_power, get_sm_version, log_perf +from collector.case_generator import get_mla_bmm_case_specs +from collector.helper import benchmark_with_power, get_sm_version, log_perf def _supported_dtypes() -> set[str]: diff --git a/python/aisimulate/collector/trtllm/collect_mla_module.py b/python/aisimulate/collector/trtllm/collect_mla_module.py index fac6ba25a3..a1c8d8103a 100644 --- a/python/aisimulate/collector/trtllm/collect_mla_module.py +++ b/python/aisimulate/collector/trtllm/collect_mla_module.py @@ -82,13 +82,25 @@ import gc import os import sys +import time import traceback import weakref import tensorrt_llm import torch -from tensorrt_llm._torch.attention_backend.interface import AttentionRuntimeFeatures -from tensorrt_llm._torch.attention_backend.utils import get_attention_backend + +# trtllm >=1.3.0rc29 moved _torch.attention_backend.* -> _torch.attention.backends.* and +# _torch.modules.fused_moe -> _torch.moe.fused_moe (the old package root is a deprecation shim +# without submodules). Path-only compat: same classes, same kernels (layer_permissions.md +# 'API-compat shims may only change HOW the same kernel is constructed'). +try: + from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.interface import AttentionRuntimeFeatures +try: + from tensorrt_llm._torch.attention.backends.utils import get_attention_backend +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.utils import get_attention_backend from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_deepseekv3 import ( @@ -125,7 +137,13 @@ sys.path.insert(0, _REPO_ROOT) from collector.case_generator import get_mla_module_model_specs, get_mla_module_sweep_spec -from collector.helper import _resolve_local_model_path, benchmark_with_power, get_sm_version, log_perf +from collector.helper import ( + PeriodicGarbageCollector, + _resolve_local_model_path, + benchmark_with_power, + get_sm_version, + log_perf, +) from collector.registry_types import PerfFile if "glm_moe_dsa" not in _CONFIG_REGISTRY: @@ -259,13 +277,42 @@ def _build_module_test_cases(attn_type: str, mode: str): base_cases = get_context_test_cases(attn_type) if mode == "context" else get_generation_test_cases(attn_type) model_specs = get_mla_module_model_specs(attention_type=attn_type, backend="trtllm") cases = [] + dropped_small_heads = 0 + dropped_non_shard_heads = 0 + sm_version = get_sm_version() for model_spec in model_specs: + # Model-correlated narrowing (case_authoring.md; get_msa_head_counts + # precedent): a row declaring its real tensor_parallel_sizes measures + # only the per-rank head counts those shards produce; the shared head + # grid otherwise sweeps 128..1 for the 64-head GLM-5.x too. Rows without + # the declaration keep the full grid. + shard_heads = None + declared_tp_sizes = getattr(model_spec, "tensor_parallel_sizes", None) + if declared_tp_sizes: + shard_heads = {model_spec.native_num_heads // tp for tp in declared_tp_sizes} for base_case in base_cases: s, b, h, kv_dtype, compute_dtype, gemm_type, *rest = base_case + if shard_heads is not None and h not in shard_heads: + dropped_non_shard_heads += 1 + continue + if attn_type == "dsa" and sm_version in (100, 103) and h < 8: + # FIXME(kernel-limit): TRT-LLM's sparse-MLA path on SM100/103 falls back to the FMHA + # that requires "Num. rows must be a multiple of 8" — per-rank heads 1/2/4 (TP 32/64/128 + # of a 128-head model) failed 15,408 of the b200 rc29 DSA cases after a full module load + # each (job 469988228); vLLM's trtllm-gen sparse decode has no kernel for the same cells + # (tileSizeQ < 8). No serving deployment reaches them. Dropped at plan time, counted. + dropped_small_heads += 1 + continue case = [s, b, h, kv_dtype, compute_dtype, gemm_type, model_spec.model_path, attn_type] if rest: case.append(rest[0]) cases.append(case) + if dropped_small_heads or dropped_non_shard_heads: + print( + f"[trtllm-mla-module-cases] {attn_type}/{mode}: dropped {dropped_small_heads} cases with per-rank " + f"heads < 8 (SM{sm_version} sparse-MLA FMHA row-multiple-of-8 domain) and {dropped_non_shard_heads} " + "cases whose per-rank heads no declared tensor_parallel_sizes shard produces" + ) return cases @@ -368,6 +415,12 @@ def _apply_gemm_type_quant(model_config, gemm_type: str, use_fp8_kv_cache: bool) _replace_quant_config( model_config.quant_config, quant_algo=QuantAlgo.NVFP4, + # NVFP4 scale blocks are 16 elements; without an explicit value the replaced + # config keeps the checkpoint's/fp8_block group_size=128 and 1.3.0rc29 + # NVFP4LinearMethod.resolve_scaling_vector_size rejects module construction + # ("supports NVFP4 scale blocks of (16,) elements, but the checkpoint declares + # group_size=128" — B200 smoke 2026-10-04; nvfp4 is a platform floor on Hopper). + group_size=16, kv_cache_quant_algo=kv_algo, exclude_modules=None, ), @@ -612,6 +665,39 @@ def create_attention_layer( # ═══════════════════════════════════════════════════════════════════════ +# One attention layer per (model, heads, kv dtype, gemm type) per worker. The layer +# build (ModelConfig.from_pretrained, DeepseekV3DecoderLayer, dummy weights, +# post_load_weights) does not depend on the point: create_attention_layer sizes +# the config with fixed max_num_tokens / max_seq_len, so one layer serves every +# (batch, seq, prefix) of its key exactly as serving reuses one module. The +# per-case KV cache manager, metadata, dry run, CUDA-graph capture and timing are +# untouched. Same pattern as collect_dsv4_attn._cached_dsv4_attention_module +# (owner-approved perf change 2026-08-09); b300 base run (job 471159002): +# dsa_context_module 1.2-2.4 GPU-s and dsa_generation_module 1.8 GPU-s per +# one-row case, most of it this build. At most one layer is retained. +_LAYER_CACHE: dict = {} + + +def _cached_attention_layer(model_path: str, num_heads: int, use_fp8_kv_cache: bool, gemm_type: str, device: str): + key = (model_path, int(num_heads), bool(use_fp8_kv_cache), gemm_type, device) + hit = _LAYER_CACHE.get(key) + if hit is not None: + return hit + if _LAYER_CACHE: + _LAYER_CACHE.clear() + gc.collect() + torch.cuda.empty_cache() + entry = create_attention_layer( + model_path=model_path, + num_heads=num_heads, + use_fp8_kv_cache=use_fp8_kv_cache, + gemm_type=gemm_type, + device=device, + ) + _LAYER_CACHE[key] = entry + return entry + + def create_kv_cache_and_metadata( model_config: ModelConfig, attn_module, @@ -823,12 +909,15 @@ def run_mla_module( # (attentionOp.cpp:3097) / generation (:3091). Hardware-observed on L40 # (SM89) 2026-07-26: module smoke 0/8, C++ SIGABRT per case; serving hits # the identical assert. Same fail-closed treatment as the stock MLA - # collector (collect_mla.py). Re-verify on the next framework version bump. + # collector (collect_mla.py). Re-verified on 1.3.0rc29 / L40 (SM89) 2026-10-04: with only this SM guard + # patched out, a bf16 DeepSeek-V3 MLA module cell still aborts in AttentionOp with "Deepseek should be + # supported by fmha in generation part" (attentionOp.cpp:3234 @1.3.0rc29; the assert moved from :3091), SIGSEGV + # in the worker (facts/mla_bypass_*.log). The wall is unchanged across rc20 -> rc29: keep failing closed. if attn_type == "mla" and get_sm_version() < 90: raise ValueError( f"TRT-LLM MLA has no pre-Hopper FMHA kernel; MLA modules are " f"unsupported on SM{get_sm_version()} " - f"(attentionOp.cpp:3091/:3097 assert @1.3.0rc20)" + f"(attentionOp.cpp:3234 assert @1.3.0rc29, :3091/:3097 @1.3.0rc20)" ) # FIXME(kernel-limit): SM120 MLA context takes serving's dense-expand path # (forward_context_default, attention.py:2015-2040@1.3.0rc20 — SM100/103 @@ -854,14 +943,23 @@ def run_mla_module( f"compute={compute_dtype}, kv={kv_cache_dtype}, model={model_path}" ) + # Per-phase wall time of this case, printed after cleanup so the campaign + # log shows where a module case spends its seconds (layer cache hit vs + # KV manager + metadata vs graph capture + timing vs teardown). + _t_case = _t_last = time.perf_counter() + phase_timing: dict[str, float] = {} + + def _mark(name: str) -> None: + nonlocal _t_last + now = time.perf_counter() + phase_timing[name] = now - _t_last + _t_last = now + # 1. Create attention layer (from_pretrained reads config.json) - attn_module, model_config, original_architecture = create_attention_layer( - model_path=model_path, - num_heads=num_heads, - use_fp8_kv_cache=use_fp8_kv_cache, - gemm_type=gemm_type, - device=device, + attn_module, model_config, original_architecture = _cached_attention_layer( + model_path, num_heads, use_fp8_kv_cache, gemm_type, device ) + _mark("layer") # 2. Create KV cache + metadata kv_cache_manager, attn_metadata = create_kv_cache_and_metadata( @@ -875,6 +973,8 @@ def run_mla_module( device=device, ) + _mark("kv_meta") + # 3. Input tensors hidden_size = model_config.pretrained_config.hidden_size if is_context: @@ -902,6 +1002,8 @@ def run_mla_module( device=torch_device, ) + _mark("inputs") + # 4. Dry run with model_extra_attrs(model_config.extra_attrs): get_model_extra_attrs()["attention_metadata"] = weakref.ref(attn_metadata) @@ -918,12 +1020,20 @@ def run_mla_module( _cleanup(kv_cache_manager) raise + _mark("dry_run") + # 5. Benchmark import tensorrt_llm._torch.utils as _trtllm_utils _trtllm_utils._model_extra_attrs.attrs = model_config.extra_attrs _trtllm_utils._model_extra_attrs.attrs["attention_metadata"] = weakref.ref(attn_metadata) + # FIXME(kernel-limit): same 32-bit per-sequence offset boundary as + # collect_attn.py (H20 campaign 2026-10-02, rc29): DeepSeek-V3 context cells + # with (batch-1) * input_len * local_heads * 192 * 2 >= 2**31 (tp1: 81920+ + # total tokens) fault with cudaErrorIllegalAddress while the TP-sharded rows + # of the same (batch, seq) pass. Unverified against the kernel source; left + # failing into the classified log. def kernel_func(): attn_module.forward(position_ids, hidden_states, attn_metadata) @@ -931,6 +1041,9 @@ def kernel_func(): device=torch_device, kernel_func=kernel_func, num_warmups=warming_up, + # The case's dry run already executed this forward; three eager passes + # cover lazily initialised paths before capture. + num_eager_warmups=3, num_runs=test_ite, repeat_n=1, allow_graph_fail=False, @@ -938,6 +1051,7 @@ def kernel_func(): pass latency = results["latency_ms"] + _mark("bench") # 6. Log results if is_context: @@ -983,6 +1097,13 @@ def kernel_func(): ) _cleanup(kv_cache_manager) + _mark("cleanup") + print( + f" [{variant} module timing] {phase} b={batch_size}, s={seq_len}, prefix={prefix_len}, " + f"heads={num_heads}, gemm={gemm_type}, kv={kv_cache_dtype}: " + + " ".join(f"{name}={seconds:.2f}s" for name, seconds in phase_timing.items()) + + f" total={time.perf_counter() - _t_case:.2f}s" + ) return latency @@ -1016,11 +1137,15 @@ def run_mla_module_worker( ) +# gc.collect() after every case cost 0.17 s of a ~1 s module case; collect periodically instead. +_CASE_GC = PeriodicGarbageCollector() + + def _cleanup(kv_cache_manager): if kv_cache_manager is not None: kv_cache_manager.shutdown() torch.cuda.empty_cache() - gc.collect() + _CASE_GC.maybe_collect() # ═══════════════════════════════════════════════════════════════════════ diff --git a/python/aisimulate/collector/trtllm/collect_moe.py b/python/aisimulate/collector/trtllm/collect_moe.py index 4021cad9df..f49ba56158 100755 --- a/python/aisimulate/collector/trtllm/collect_moe.py +++ b/python/aisimulate/collector/trtllm/collect_moe.py @@ -13,6 +13,7 @@ import gc import glob +import importlib import inspect import json import os @@ -26,7 +27,15 @@ from tensorrt_llm._torch.autotuner import AutoTuner, autotune from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_deepseekv3 import DeepseekV3Gate -from tensorrt_llm._torch.modules.fused_moe import RenormalizeMoeRoutingMethod, create_moe + +# trtllm >=1.3.0rc29 moved _torch.attention_backend.* -> _torch.attention.backends.* and +# _torch.modules.fused_moe -> _torch.moe.fused_moe (the old package root is a deprecation shim +# without submodules). Path-only compat: same classes, same kernels (layer_permissions.md +# 'API-compat shims may only change HOW the same kernel is constructed'). +try: + from tensorrt_llm._torch.moe.fused_moe import RenormalizeMoeRoutingMethod, create_moe +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.modules.fused_moe import RenormalizeMoeRoutingMethod, create_moe from tensorrt_llm._utils import is_sm_100f from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig @@ -56,6 +65,7 @@ log_perf, power_law_logits_v3, ) +from collector.version_resolver import _check_compat aic_debug = int(os.getenv("aic_moe_debug", "0")) # noqa: SIM112 @@ -63,6 +73,45 @@ _TRTLLM_VERSION = tensorrt_llm.__version__ +_DEEPGEMM_WORKSPACE_POOL_WARNED = False + + +def _clear_deepgemm_moe_workspaces() -> None: + """Drop DeepGEMM's fp8_block MoE workspace pool between token counts. + + DeepGemmFusedMoE sizes its workspaces per num_tokens and parks every size + in the memory_buffer_utils pool (``DeepGemmFusedMoE.buffers``); left alone + over the 27-token-count sweep the pool reached 251 GB on gb300 and the + next 63 GiB request OOMed (2026-10-11: gb300 60 / gb200 108 fp8_block + cases, Kimi-K2 / DeepSeek-V3 geometries). 1.3.0rc29 moved the module from + ``_torch.modules.fused_moe`` to ``_torch.moe.fused_moe``; the previous + single-path import failed silently inside a try/except and the pool was + never cleared. Both layouts are tried here and an unreachable pool is + reported once instead of swallowed. + """ + global _DEEPGEMM_WORKSPACE_POOL_WARNED + cls = None + for module_name in ( + "tensorrt_llm._torch.moe.fused_moe.fused_moe_deepgemm", # >= 1.3.0rc29 + "tensorrt_llm._torch.modules.fused_moe.fused_moe_deepgemm", # < rc29 + ): + try: + cls = importlib.import_module(module_name).DeepGemmFusedMoE + break + except (ImportError, AttributeError): + continue + pool = getattr(getattr(cls, "buffers", None), "buffers", None) + if pool is None: + if not _DEEPGEMM_WORKSPACE_POOL_WARNED: + _DEEPGEMM_WORKSPACE_POOL_WARNED = True + print( + "[trtllm-moe] DeepGemmFusedMoE workspace pool is not reachable on this TRT-LLM build; " + "fp8_block MoE workspaces will accumulate across token counts" + ) + return + pool.clear() + + def _moe_model_behavior(model_name: str) -> str: """Resolve model-name branches that change the synthetic invocation.""" if any(pattern in model_name for pattern in NON_GATED_MOE_MODELS): @@ -207,7 +256,9 @@ def get_moe_test_cases(): min_latency_mode_options = [False] - if moe_type == "nvfp4" and get_sm_version() == 100 and common_moe_testcase.num_experts <= 256: + if moe_type == "nvfp4" and get_sm_version() in (100, 103) and common_moe_testcase.num_experts <= 256: + # SM103 treated as SM100 (owner decision 2026-10-05: "sm103 可以认为等价sm100" — + # same capability major, same trtllm-gen kernel family; re-verify on a B300 box). # FIXME: recent version only supports SM100 for min-latency mode. # current support, DS router only support up to 256 experts. # Renormalize router only support <=128 experts. trtllmgen kernels only @@ -270,7 +321,19 @@ def get_moe_test_cases(): return test_cases -def run_moe_torch( +def _declared_moe_routing(model_name: str): + """The model row's routing declaration (cases/models/*_cases.yaml + model_case_values.moe: sglang_moe_routing_method_type / _num_expert_group / + _topk_group / _routed_scaling_factor). Framework-neutral model facts despite + the sglang_ prefix: TensorRT-LLM reads the same numbers from the checkpoint + config (n_group, topk_group, routed_scaling_factor) when it builds the gate.""" + for case in get_common_moe_test_cases(): + if case.model_name == model_name: + return case + return None + + +def _run_moe_case( moe_type, num_tokens_lists, hidden_size, @@ -287,7 +350,7 @@ def run_moe_torch( perf_filename, device="cuda:0", ): - """Run MoE forward passes and log latency/power to perf file (trtllm >= 1.1 collector).""" + """Run one MoE case (its token sweep) and log latency/power rows (trtllm >= 1.1 collector).""" device = torch.device(device) torch.cuda.set_device(device) torch.set_default_device(device) @@ -298,18 +361,37 @@ def run_moe_torch( print("MOE Allocated GDRAM:", torch.cuda.memory_allocated(device.index) / 1024**2, "MB") print("MOE Reserved GDRAM:", torch.cuda.memory_reserved(device) / 1024**2, "MB") # moe type support bfloat16, fp8_qdq, fp8_block, w4a8, nvfp4(not implemented yet) + # ``dtype`` is the MODEL/activation dtype serving hands to create_moe + # (modeling_deepseekv3.py:798 ``dtype=config.torch_dtype`` -> DeepseekV3MoE + # -> create_moe(dtype=dtype) @1.3.0rc29; identical at rc23). Quantization + # travels in quant_config only. Passing float8 here (pre-rc29 habit) is + # rejected by rc29's MoE resolution ("CutlassFusedMoE FP8_BLOCK_SCALES + # requires torch.bfloat16, got torch.float8_e4m3fn") and never matched + # serving, where activations are bf16 and quantized inside the kernel path. dtype = torch.bfloat16 quant_group_size = 128 quant_algo = None + # FIXME(kernel-limit): TRT-LLM >= 1.3.0rc29 MoE resolution has no implementation for W4A16 on + # SM100/103 ("no MoE implementation can serve this layer ... CutlassFusedMoE: quant_unsupported + # (quant_algo=W4A16)", 114/114 int4_wo cases, b200 job 469988228). Serving cannot load the + # W4A16 checkpoint either (benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml: + # Kimi-K2.5 fail, finding sm100_probe_failures_recheck_2026_10_04 (g)). The lane is unreachable on this pin, not a + # per-shape failure; refuse before building weights. + if ( + moe_type == "int4_wo" + and get_sm_version() in (100, 103) + and _check_compat("tensorrt_llm>=1.3.0rc29", _TRTLLM_VERSION) + ): + raise RuntimeError( + "FIXME(kernel-limit): TRT-LLM int4_wo (W4A16) MoE has no implementation on SM100/103 at " + f"{_TRTLLM_VERSION}; serving fails identically (Kimi-K2.5 identity probe)" + ) if moe_type == "fp8_block": quant_algo = QuantAlgo.FP8_BLOCK_SCALES - dtype = torch.float8_e4m3fn elif moe_type == "w4afp8": quant_algo = QuantAlgo.W4A8_AWQ - dtype = torch.float8_e4m3fn elif moe_type == "fp8": quant_algo = QuantAlgo.FP8 - dtype = torch.float8_e4m3fn elif moe_type == "int4_wo": quant_algo = QuantAlgo.W4A16 int4_config = get_moe_quantization_module_config("trtllm", moe_type, model_name=model_name) @@ -392,7 +474,7 @@ def run_moe_torch( # next framework version bump. if moe_type == "fp8_block" and (hidden_size % 128 != 0 or (inter_size // moe_tp_size) % 128 != 0): raise ValueError( - f"fp8_block MoE requires 128-aligned hidden_size and TP-sharded intermediate " + f"FIXME(kernel-limit): fp8_block MoE requires 128-aligned hidden_size and TP-sharded intermediate " f"size (128x128-blocked weight scales; deepgemm layout.hpp:78 on SM90/100/103, " f"Triton block-scale on SM120); got hidden_size={hidden_size}, " f"inter_size={inter_size} / moe_tp={moe_tp_size} = {inter_size // moe_tp_size}" @@ -400,7 +482,7 @@ def run_moe_torch( if moe_type == "w4afp8" and (inter_size // moe_tp_size) % 128 != 0: raise ValueError( - f"w4afp8 MoE requires a 128-aligned TP-sharded intermediate size (grouped-GEMM " + f"FIXME(kernel-limit): w4afp8 MoE requires a 128-aligned TP-sharded intermediate size (grouped-GEMM " f"k alignment); got inter_size={inter_size} / moe_tp={moe_tp_size} = " f"{inter_size // moe_tp_size}" ) @@ -426,7 +508,7 @@ def run_moe_torch( native_mxfp4_padding = moe_type in _MXFP4_MOE_TYPES and 100 <= sm_version < 120 if not native_mxfp4_padding and (inter_size // moe_tp_size) % (256 // _weight_bits) != 0: raise ValueError( - f"TRT-LLM fused MoE requires the TP-sharded intermediate size to be a multiple " + f"FIXME(kernel-limit): TRT-LLM fused MoE requires the TP-sharded intermediate size to be a multiple " f"of 256/weight_bits = {256 // _weight_bits} for {moe_type} (TLLM_CHECK_WITH_INFO " f"weight-layout alignment); got inter_size={inter_size} / moe_tp={moe_tp_size} = " f"{inter_size // moe_tp_size}" @@ -505,14 +587,38 @@ def run_moe_torch( ).routing_method router_logits_dtype = torch.float32 else: - # for low latency mode in fp4, experts > 128 is not supported. - # Native GPT-OSS uses BF16 routing weights for TRTLLM, FP32 otherwise: - # modeling_gpt_oss.py:159-162@1.3.0rc14 (93cb6518), :165-168@1.3.0rc20. - if is_gpt_oss: + declared = _declared_moe_routing(model_name) + if declared is not None and declared.sglang_moe_routing_method_type == "DeepSeekV3": + # Serving routes DeepSeek-V3-family / GLM / Nemotron-H MoE through + # DeepseekV3Gate.routing_method (noaux_tc, grouped top-k with the + # e_score_correction_bias; Deepseekv3RoutingImpl is_fused=True -> + # deepseek_v3_topk_kernel) built from the checkpoint's n_group / + # topk_group / routed_scaling_factor and the resolved moe_backend — + # modeling_deepseekv3.py:979-990, modeling_glm.py:273-285, + # modeling_nemotron_h.py:236 @1.3.0rc29. RenormalizeMoeRoutingMethod + # (customMoeRoutingKernel) is the softmax-top-k family (Qwen3-MoE, + # Mixtral, ...), and running it for DeepSeek-V3 diverged the + # moe_fp8block gate on H20 (2026-10-01). The router GEMM that feeds the + # gate (dsv3_router_gemm_op) stays outside this op, as before. + routing_method = DeepseekV3Gate( + hidden_size, + num_experts, + top_k=topk, + n_group=int(declared.sglang_moe_num_expert_group or 1), + topk_group=int(declared.sglang_moe_topk_group or 1), + routed_scaling_factor=float(declared.sglang_moe_routed_scaling_factor or 1.0), + dtype=dtype, + moe_backend=model_config.moe_backend, + ).routing_method + router_logits_dtype = torch.float32 + elif is_gpt_oss: + # Native GPT-OSS uses BF16 routing weights for TRTLLM, FP32 otherwise: + # modeling_gpt_oss.py:159-162@1.3.0rc14 (93cb6518), :165-168@1.3.0rc20. routing_method = RenormalizeMoeRoutingMethod( topk, output_dtype=torch.bfloat16 if model_config.moe_backend == "trtllm" else torch.float32 ) else: + # softmax top-k renormalized routing (fused custom routing kernel). routing_method = RenormalizeMoeRoutingMethod(topk) create_moe_kwargs = { @@ -725,12 +831,7 @@ def fp32_to_mxfp4(tensor): del hidden_states_max_tokens, logits_max_tokens if moe_type == "fp8_block": - try: - from tensorrt_llm._torch.modules.fused_moe.fused_moe_deepgemm import DeepGemmFusedMoE - - DeepGemmFusedMoE.buffers.buffers.clear() - except (ImportError, AttributeError): - pass + _clear_deepgemm_moe_workspaces() torch.cuda.empty_cache() for num_tokens in num_tokens_lists: @@ -766,29 +867,28 @@ def run_forward_pass(): moe.forward(hidden_states, actual_logits, do_finalize=not min_latency_mode) # noqa: F821 # ═══════════════════════════════════════════════════════════════════════════════ - # Benchmark with automatic power measurement and graph fallback + # Benchmark under CUDA-graph replay with automatic power measurement # ═══════════════════════════════════════════════════════════════════════════════ # Determine base warmups and runs based on distribution mode num_warmups = 1 if distributed == "power_law" else 3 num_runs = 1 if distributed == "power_law" else 6 - # Use benchmark_with_power with graceful graph fallback + # CUDA-graph timing is mandatory: a failed capture raises + # (benchmark_with_power default allow_graph_fail=False) and the case + # fails classified. The moe row has no column that could record an + # eager-timed measurement, and the fallback used until 2026-10-12 + # shipped the Python launch-loop floor as kernel latency. with benchmark_with_power( device=device, kernel_func=run_forward_pass, num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, # Enable graceful fallback to eager execution ) as results: # Calculate per-iteration latency (accounting for internal iterations) latency = results["latency_ms"] / num_iter power_stats = results["power_stats"] - # Log if CUDA graph capture failed (for debugging) - if not results["used_cuda_graph"] and aic_debug == 1: - print(f"CUDA graph capture failed for {num_tokens} tokens, used eager execution fallback") - if moe_type == "fp8_block" and is_sm_100f(sm_version): source = "deepgemm" elif moe_type == "fp8_block" and get_sm_version() == 120: @@ -847,18 +947,168 @@ def run_forward_pass(): del actual_logits del hidden_states if moe_type == "fp8_block" and num_tokens != max_tokens: - try: - from tensorrt_llm._torch.modules.fused_moe.fused_moe_deepgemm import DeepGemmFusedMoE - - DeepGemmFusedMoE.buffers.buffers.clear() - except (ImportError, AttributeError): - pass + _clear_deepgemm_moe_workspaces() torch.cuda.empty_cache() - if os.getenv("TRTLLM_MOE_RESTART_WORKER", "1") != "0": - # Exit the worker process after completing MOE task to ensure complete resource cleanup. - # This forces OS to reclaim all GPU memory, CUDA context, and other resources. - sys.exit(EXIT_CODE_RESTART) + +# ── Worker recycling ───────────────────────────────────────────────────────── +# AIConfigurator's collector (40bc989a9) exited the worker after EVERY case so the +# OS reclaimed all GPU memory. collect.py spawns its workers, so each recycle +# re-imports tensorrt_llm and re-creates the CUDA context (10-15 s); on the b300 +# base run (job 471159002) that was most of a 14-20 GPU-s case of 27 token +# points. The policy is now: +# TRTLLM_MOE_RESTART_EVERY=N recycle after N completed cases on this worker +# (default 16; 1 = the legacy per-case recycle; +# 0 = never recycle on the count) +# TRTLLM_MOE_RESTART_LEAK_MB=M recycle at once when more than M MiB are still +# allocated after the case's own cleanup (default +# 1024; 0 disables the guard) — the residue the +# per-case exit used to hide +# TRTLLM_MOE_RESTART_WORKER=0 legacy switch: never recycle (overrides both) +# A recycle is always announced on one greppable line so a run log shows how +# often, and why, workers were replaced. +_cases_since_worker_start = 0 + + +def _worker_restart_policy() -> tuple[int, int] | None: + """(cases per worker, leak guard MiB), or None when recycling is disabled.""" + if os.getenv("TRTLLM_MOE_RESTART_WORKER", "1") == "0": + return None + every = int(os.getenv("TRTLLM_MOE_RESTART_EVERY", "16")) + leak_mb = int(os.getenv("TRTLLM_MOE_RESTART_LEAK_MB", "1024")) + if every < 0 or leak_mb < 0: + raise ValueError( + f"TRTLLM_MOE_RESTART_EVERY and TRTLLM_MOE_RESTART_LEAK_MB must be >= 0; got {every} and {leak_mb}" + ) + return every, leak_mb + + +def _release_trtllm_memory_buffers() -> int: + """Drop TRT-LLM's global workspace buffer pool between cases; returns MiB released. + + b200 probe 2026-10-29 (job 514855922, TRTLLM_MOE_LEAK_DEBUG=1): everything still + allocated after a case was `BufferBlock` tensors of + tensorrt_llm/_torch/memory_buffer_utils.py — the module-level ``_buffer = Buffers()`` + pool the fused-MoE kernels draw their workspaces from (2.7-86 GiB, the two + largest blocks in the 2:1 shape of w3_w1 / w2). It is a reuse pool, not a leak: + the next case replaces it, so memory did not grow across 117 cases. Serving keeps + it alive for the model's lifetime; between collector cases nothing is live, so + every block (reserved ones included — ``clear_memory_buffers`` keeps those) is + released here and the leak guard measures real residue again. @1.3.0rc29; a + TRT-LLM without this module (older pins) makes this a no-op. + """ + try: + from tensorrt_llm._torch import memory_buffer_utils + except ImportError: + return 0 + pool = memory_buffer_utils.get_memory_buffers() + released = 0 + for blocks in list(pool.buffers.values()): + for block in blocks: + tensor = getattr(block, "buffer", None) + if tensor is not None: + released += tensor.numel() * tensor.element_size() + block.buffer = None + pool.buffers.clear() + return released // 2**20 + + +def _print_live_cuda_tensors(device, top: int = 8) -> None: + """TRTLLM_MOE_LEAK_DEBUG=1: name what still pins CUDA memory after a case. + + b200 probe 2026-10-29 (jobs 514778748/514778867): 2.7-86 GiB stayed allocated after + every DeepSeek-V3 fp8_block case, so the leak guard recycled the worker each time and + the policy bought nothing. Lists the largest live CUDA tensors with the types of the + objects that reference them (gc view; a traceback-free best effort). + """ + sizes = [] + for obj in gc.get_objects(): + try: + if isinstance(obj, torch.Tensor) and obj.is_cuda and obj.device == device: + sizes.append((obj.numel() * obj.element_size(), obj)) + except Exception: # gc.get_objects can hand back half-built objects + continue + sizes.sort(key=lambda item: -item[0]) + total = sum(size for size, _ in sizes) + print(f"trtllm.moe: leak debug: {len(sizes)} live CUDA tensors, {total / 2**20:.0f} MiB; largest:") + for size, tensor in sizes[:top]: + referrers = [type(r).__name__ for r in gc.get_referrers(tensor) if r is not sizes][:4] + print( + f" {size / 2**20:8.0f} MiB {tuple(tensor.shape)} {tensor.dtype} " + f"param={isinstance(tensor, torch.nn.Parameter)} referrers={referrers}" + ) + + +def _recycle_worker_if_due(device) -> None: + """Exit this worker with EXIT_CODE_RESTART when the recycling policy says so. + + Runs after the case's rows are persisted; collect.py marks the task done and + starts a fresh worker on the same device (worker(): SystemExit handling). + """ + global _cases_since_worker_start + policy = _worker_restart_policy() + if policy is None: + return + every, leak_mb = policy + _cases_since_worker_start += 1 + workspace_mb = _release_trtllm_memory_buffers() + gc_collect() + allocated_mb = torch.cuda.memory_allocated(device) / 2**20 + reserved_mb = torch.cuda.memory_reserved(device) / 2**20 + print( + f"trtllm.moe: after case {_cases_since_worker_start} on this worker: " + f"{allocated_mb:.0f} MiB allocated, {reserved_mb:.0f} MiB reserved " + f"(released {workspace_mb} MiB of TRT-LLM workspace buffers)" + ) + if os.getenv("TRTLLM_MOE_LEAK_DEBUG", "0") == "1" and allocated_mb > 256: + _print_live_cuda_tensors(device) + due_count = every > 0 and _cases_since_worker_start >= every + due_leak = leak_mb > 0 and allocated_mb > leak_mb + if not (due_count or due_leak): + return + reason = "leak guard" if due_leak else f"every {every}" + print( + f"trtllm.moe: recycling worker after {_cases_since_worker_start} case(s) " + f"({allocated_mb:.0f} MiB still allocated, {reason})" + ) + sys.exit(EXIT_CODE_RESTART) + + +def run_moe_torch( + moe_type, + num_tokens_lists, + hidden_size, + inter_size, + topk, + num_experts, + moe_tp_size, + moe_ep_size, + min_latency_mode, + model_name, + distributed="power_law", + power_law_alpha=0.0, + *, + perf_filename, + device="cuda:0", +): + """Run one MoE case, then recycle the worker when the policy above is due.""" + _run_moe_case( + moe_type, + num_tokens_lists, + hidden_size, + inter_size, + topk, + num_experts, + moe_tp_size, + moe_ep_size, + min_latency_mode, + model_name, + distributed=distributed, + power_law_alpha=power_law_alpha, + perf_filename=perf_filename, + device=device, + ) + _recycle_worker_if_due(torch.device(device)) if __name__ == "__main__": diff --git a/python/aisimulate/collector/trtllm/collect_msa_module.py b/python/aisimulate/collector/trtllm/collect_msa_module.py index f3421b9c9c..7645993986 100644 --- a/python/aisimulate/collector/trtllm/collect_msa_module.py +++ b/python/aisimulate/collector/trtllm/collect_msa_module.py @@ -55,8 +55,19 @@ import tensorrt_llm import torch import transformers -from tensorrt_llm._torch.attention_backend.interface import AttentionRuntimeFeatures -from tensorrt_llm._torch.attention_backend.utils import get_attention_backend + +# trtllm >=1.3.0rc29 moved _torch.attention_backend.* -> _torch.attention.backends.* and +# _torch.modules.fused_moe -> _torch.moe.fused_moe (the old package root is a deprecation shim +# without submodules). Path-only compat: same classes, same kernels (layer_permissions.md +# 'API-compat shims may only change HOW the same kernel is constructed'). +try: + from tensorrt_llm._torch.attention.backends.interface import AttentionRuntimeFeatures +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.interface import AttentionRuntimeFeatures +try: + from tensorrt_llm._torch.attention.backends.utils import get_attention_backend +except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.utils import get_attention_backend from tensorrt_llm._torch.metadata import KVCacheParams from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.modeling_minimaxm3 import MiniMaxM3DecoderLayer @@ -104,8 +115,15 @@ get_mla_module_model_specs, get_mla_module_precision_specs, get_mla_module_sweep_spec, + get_msa_head_counts, +) +from collector.helper import ( + PeriodicGarbageCollector, + _resolve_local_model_path, + benchmark_with_power, + get_sm_version, + log_perf, ) -from collector.helper import _resolve_local_model_path, benchmark_with_power, get_sm_version, log_perf from collector.registry_types import PerfFile if not hasattr(_trtllm_configs, "PretrainedConfig"): @@ -134,6 +152,18 @@ def _get_precision_combos(phase: str): ] +def _msa_implementation(sm_version: int) -> str: + """Which MiniMax-M3 sparse implementation TRT-LLM can run on this SM. + + "msa" = the fmha_sm100 kernels, hard-gated to the SM100 family + (msa_availability.py:ensure_msa_available@1.3.0rc29 raises off SM100/103, + SM120 included); everything else runs the Triton reference + (triton_backend.py). Single source of truth for the plan-time budget + below and the layer construction in build_msa_layer. + """ + return "msa" if sm_version in (100, 103) else "triton" + + def get_context_test_cases(): """Context-phase test cases. @@ -142,12 +172,27 @@ def get_context_test_cases(): """ cases = [] sweep = get_mla_module_sweep_spec("trtllm") + head_counts = get_msa_head_counts("trtllm") # native // tp of the declared shards, not the shared head grid + # Declared budget for the Triton reference implementation + # (mla_module.yaml mla_module_trtllm.context.msa_triton_max_tokens): its + # masked-dense FP32 prefill is quadratic per sequence (24.5 s per forward + # at 64 heads / isl 32768 on rtxpro6000) and OOMs at b*isl=131072, so the + # SMs that only have it stop at 32K tokens; fmha_sm100 keeps the shared cap. + sm_version = get_sm_version() + max_tokens = sweep.context_max_tokens + triton_max_tokens = sweep.context_msa_triton_max_tokens + if triton_max_tokens and _msa_implementation(sm_version) == "triton": + max_tokens = min(max_tokens, triton_max_tokens) + dropped_triton_budget = 0 for compute_dtype, kv_dtype, gemm_type in _get_precision_combos("context"): - for num_heads in sweep.inner_sweep_head_counts: + for num_heads in head_counts: for b in sweep.context_batch_sizes: for s in sweep.context_sequence_lengths: if b * s > sweep.context_max_tokens: continue + if b * s > max_tokens: + dropped_triton_budget += len(sweep.context_prefix_lengths) + continue if ( sweep.context_large_sequence_min and s >= sweep.context_large_sequence_min @@ -156,6 +201,12 @@ def get_context_test_cases(): continue for prefix_len in sweep.context_prefix_lengths: cases.append([s, b, num_heads, kv_dtype, compute_dtype, gemm_type, prefix_len]) + if dropped_triton_budget: + print( + f"[trtllm-msa-module-cases] context: dropped {dropped_triton_budget} cases with batch*isl > " + f"{max_tokens} (SM{sm_version} runs the Triton reference implementation; " + "mla_module_trtllm.context.msa_triton_max_tokens)" + ) return cases @@ -167,8 +218,9 @@ def get_generation_test_cases(): """ cases = [] sweep = get_mla_module_sweep_spec("trtllm") + head_counts = get_msa_head_counts("trtllm") for compute_dtype, kv_dtype, gemm_type in _get_precision_combos("generation"): - for num_heads in sweep.inner_sweep_head_counts: + for num_heads in head_counts: for b in sweep.generation_batch_sizes: for s in sweep.generation_sequence_lengths: if b * s > sweep.generation_max_tokens: @@ -362,7 +414,7 @@ def _required_sparse_field(key: str) -> int: # elsewhere. kernel_source records which one actually ran. Field-presence # probe keeps rc19/rc20 (no such field, pydantic strict) working. if "implementation" in getattr(MiniMaxM3SparseAttentionConfig, "model_fields", {}): - sparse_kwargs["implementation"] = "msa" if get_sm_version() in (100, 103) else "triton" + sparse_kwargs["implementation"] = _msa_implementation(get_sm_version()) sparse_attention_config = MiniMaxM3SparseAttentionConfig(**sparse_kwargs) # NOTE: sparse_attention_config is intentionally NOT passed as a @@ -509,9 +561,14 @@ def _resolve_msa_metadata_cls(): instead of sniffing the version string. """ try: - from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_backend import ( - MiniMaxM3MsaSparseAttentionMetadata, - ) + try: + from tensorrt_llm._torch.attention.backends.sparse.minimax_m3.msa_backend import ( + MiniMaxM3MsaSparseAttentionMetadata, + ) + except ModuleNotFoundError: # < rc29 layout + from tensorrt_llm._torch.attention_backend.sparse.minimax_m3.msa_backend import ( + MiniMaxM3MsaSparseAttentionMetadata, + ) except ImportError: # rc19/rc20 layout — Triton contract only return None return MiniMaxM3MsaSparseAttentionMetadata @@ -540,6 +597,28 @@ def _resolve_attention_backend_cls(model_config: ModelConfig): ) +# One MiniMax-M3 sparse attention layer per (model, heads, gemm type) per worker — +# same pattern and rationale as collect_mla_module._cached_attention_layer / +# collect_dsv4_attn._cached_dsv4_attention_module: the layer build does not depend +# on the point; the per-case KV cache manager, metadata, dry run and timing are +# untouched (b300 base run: msa_context_module 1.8 GPU-s per one-row case). +_LAYER_CACHE: dict = {} + + +def _cached_msa_attention_layer(model_path: str, num_heads: int, gemm_type: str, device: str): + key = (model_path, int(num_heads), gemm_type, device) + hit = _LAYER_CACHE.get(key) + if hit is not None: + return hit + if _LAYER_CACHE: + _LAYER_CACHE.clear() + gc.collect() + torch.cuda.empty_cache() + entry = create_msa_attention_layer(model_path=model_path, num_heads=num_heads, gemm_type=gemm_type, device=device) + _LAYER_CACHE[key] = entry + return entry + + def create_kv_cache_and_metadata( model_config: ModelConfig, batch_size: int, @@ -756,7 +835,15 @@ class (never the version string): "the MSA cache-write buffers (msa_backend.py:549-609@1.3.0rc23); " "the KV cache manager is not the M3 sparse manager" ) - if not is_context and attn_metadata.msa_decode_proxy_plan is None: + # rc23 exposed the graph-safe decode plan as ``msa_decode_proxy_plan``; rc29's + # MiniMaxM3MsaSparseAttentionMetadata no longer has that attribute (AttributeError + # on every generation case: 2,760/2,760, b200 job 469988228). Check the plan only + # where the contract still names it; on later series rely on _msa_fields_ready + # above. FIXME(re-verify): confirm rc29's decode-plan field name and restore an + # explicit check once known. + unset = object() + proxy_plan = getattr(attn_metadata, "msa_decode_proxy_plan", unset) + if not is_context and proxy_plan is None: raise RuntimeError( "MSA decode plans were not built for a pure generation batch " "(msa_backend.py:428-460@1.3.0rc23); sparse_metadata_params " @@ -817,11 +904,8 @@ def run_msa_module( f"compute={compute_dtype}, kv={kv_cache_dtype}, model={model_path}" ) - attn_module, model_config, original_architecture = create_msa_attention_layer( - model_path=model_path, - num_heads=num_heads, - gemm_type=gemm_type, - device=device, + attn_module, model_config, original_architecture = _cached_msa_attention_layer( + model_path, num_heads, gemm_type, device ) kv_cache_manager, attn_metadata, kernel_source = create_kv_cache_and_metadata( @@ -907,6 +991,10 @@ def kernel_func(): device=torch_device, kernel_func=kernel_func, num_warmups=warming_up, + # Graph-timed cases: the dry run already executed this forward, three + # eager passes suffice before capture. Eager-timed MSA prefill keeps + # the full warmup since it IS the measurement's warmup. + num_eager_warmups=3 if use_cuda_graph else warming_up, num_runs=test_ite, repeat_n=1, allow_graph_fail=False, @@ -1000,11 +1088,15 @@ def run_msa_module_worker( ) +# gc.collect() after every case cost 0.17 s of a ~1 s module case; collect periodically instead. +_CASE_GC = PeriodicGarbageCollector() + + def _cleanup(kv_cache_manager): if kv_cache_manager is not None: kv_cache_manager.shutdown() torch.cuda.empty_cache() - gc.collect() + _CASE_GC.maybe_collect() # ═══════════════════════════════════════════════════════════════════════ diff --git a/python/aisimulate/collector/trtllm/registry.py b/python/aisimulate/collector/trtllm/registry.py index 47589e145a..2405110ffc 100644 --- a/python/aisimulate/collector/trtllm/registry.py +++ b/python/aisimulate/collector/trtllm/registry.py @@ -24,6 +24,9 @@ get_func="get_computescale_test_cases", run_func="run_computescale", perf_filename=PerfFile.COMPUTESCALE, + # one measurement, two tables: the static quant row goes to + # scale_matrix_perf; finalize binds both to this producer + extra_perf_filenames=(PerfFile.SCALE_MATRIX,), ), OpEntry( op="mla_context", diff --git a/python/aisimulate/collector/version_resolver.py b/python/aisimulate/collector/version_resolver.py index 8dc208930b..eaa7abbec7 100644 --- a/python/aisimulate/collector/version_resolver.py +++ b/python/aisimulate/collector/version_resolver.py @@ -167,7 +167,8 @@ def build_collections( "get_func": entry.get_func, "run_func": entry.run_func, "perf_filename": entry.perf_filename, - "worker_perf_filename": entry.worker_perf_filename, + "extra_perf_filenames": tuple(entry.extra_perf_filenames), + "run_kwargs": tuple(entry.run_kwargs), "unverified": entry.unverified, "unverified_sms": entry.unverified_sms, } diff --git a/python/aisimulate/collector/vllm/collect_attn.py b/python/aisimulate/collector/vllm/collect_attn.py index 8bbde7e404..2658d32d4c 100644 --- a/python/aisimulate/collector/vllm/collect_attn.py +++ b/python/aisimulate/collector/vllm/collect_attn.py @@ -1,13 +1,11 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 - -"""vLLM 0.24.0 dense-attention collector for CUDA backends.""" - -# B200 0.25.0 qualification (installed vLLM dd10e03f9), job 1967975: -# attention_context and attention_generation: 8/8 representative cases each. The native framework -# builders/selectors remain authoritative; no kernel fallback is introduced. -# The campaign manifest still selects one exact release per run. -__compat__ = "vllm>=0.24.0,<=0.25.0" +# +# Versions are upgraded IN PLACE (owner policy 2026-09-20): this file always +# targets the manifest pin; prior-version code is `git log -- ` +# away and prior-version DATA stays permanent under its version key. The +# 0.24->0.29 adaptations each carry a serving citation @0.29.0. +__compat__ = "vllm>=0.30.0,<=0.30.0" import os @@ -24,6 +22,7 @@ create_kv_cache_and_block_mappings, create_standard_kv_cache_spec, create_vllm_config, + framework_kv_block_size, get_attention_backend, with_exit_stack, ) @@ -32,7 +31,10 @@ from vllm.utils.import_utils import resolve_obj_by_qualname from vllm.utils.torch_utils import set_random_seed from vllm.v1.attention.backends.registry import AttentionBackendEnum -from vllm.v1.attention.backends.utils import set_kv_cache_layout +from vllm.v1.attention.backends.utils import ( + get_supported_kv_cache_layouts, + resolve_kv_cache_layout, +) from vllm.v1.attention.selector import AttentionSelectorConfig from vllm.version import __version__ as vllm_version @@ -54,6 +56,19 @@ def __init__(self, device: torch.device): # support MHA GQA MQA bfloat16 tensor and bfloat16/fp8 kv cache +def kv_cache_in_resolved_layout(kv_cache: torch.Tensor, layer_view_order: tuple[int, ...]) -> torch.Tensor: + """Return a tensor with the SAME logical [B, H, N, C] view as ``kv_cache`` + whose memory follows the resolved physical axis order ``layer_view_order`` + (vllm.v1.kv_cache_layout.KVCacheLayout.layer_view_order), i.e. exactly what + ``kv_cache.permute(*layer_view_order)`` in the backends expects to be + contiguous. Identity layout -> the input itself.""" + order = tuple(int(i) for i in layer_view_order) + if order == tuple(range(kv_cache.dim())): + return kv_cache.contiguous() + inverse = [order.index(i) for i in range(len(order))] + return kv_cache.permute(*order).contiguous().permute(*inverse) + + def _dense_kernel_source(backend_name_str, impl, attn_metadata): """Ground-truth kernel_source for the dense attention row.""" if backend_name_str == "FLASH_ATTN": @@ -73,7 +88,17 @@ def _dense_kernel_source(backend_name_str, impl, attn_metadata): phase_metadata = attn_metadata.decode if phase_metadata is None: raise RuntimeError("vLLM FlashInfer metadata has neither a prefill nor a decode portion") - return f"vllm_flashinfer_{type(phase_metadata).__name__}".lower() + label = f"vllm_flashinfer_{type(phase_metadata).__name__}".lower() + # 0.30 folds XQA and trtllm-gen decode into ONE metadata class, + # FlashInferTrtllmAPIDecode, and names the kernel in its `kernel` field + # (flashinfer.py:568-572 FlashInferDecodeKernel, :608-611 @v0.30.0; + # decode_with_xqa / decode_with_trtllm_gen at :1997-1998). The class + # name alone would label SM120's XQA row and SM100's trtllm-gen row + # identically — kernel_source must name the kernel that ran. + kernel = getattr(phase_metadata, "kernel", None) + if kernel is not None: + label += "_" + str(getattr(kernel, "value", kernel)).lower().replace("-", "_") + return label return f"vllm_{backend_name_str}".lower() @@ -96,7 +121,10 @@ def run_attention_torch( dtype = torch.bfloat16 model = os.path.join(os.path.dirname(__file__), "fake_hf_model") - block_size = 64 + # Backend selection runs at the framework default page (serving does the same: the + # model is built at CacheConfig.DEFAULT_BLOCK_SIZE, the page is re-derived from the + # selected backend afterwards — utils.framework_kv_block_size). + block_size = None # unconstrained selection (no --block-size); the page is asked from the backend below if is_context_phase: batch_spec = BatchSpec( @@ -132,14 +160,35 @@ def run_attention_torch( head_size=head_dim, dtype=dtype, kv_cache_dtype="fp8" if use_fp8_kv_cache else "auto", - block_size=block_size, + block_size=None, # serving passes None unless the user set --block-size (selector.py:131-135) use_mla=False, has_sink=False, use_sparse=False, use_mm_prefix=False, ) + # Backend selection for head_dim > 256 mirrors the framework's own + # heterogeneous-head-dim policy. head_dim > 256 is reachable only from + # Gemma-4 (global layers, global_head_dim=512), whose model-level config + # hook (Gemma4Config.verify_and_update_config, + # model_executor/models/config.py:209-256) forces a UNIFORM backend across + # its mixed 256/512 heads to avoid the FA3/FA4 mixed penalty. That policy, + # plus the per-group validity of head-512, resolves as follows on SM90 + # (live-engine kernel probe, 2026-09-22, gemma-4-26B-A4B + kv fp8/auto): + # bf16-KV -> FLASH_ATTN (FA4 cute flash_fwd_sm90) — the bare selector + # already returns this, so no override is needed. + # fp8-KV -> TRITON_ATTN (kernel_unified_attention). No FA/FlashInfer + # path compiles fp8+head512 on SM90, so the uniform-backend + # policy lands on Triton. The bare platform selector, lacking + # the model-level heterogeneous-head context, instead picks + # FlashInfer (whose SM90 fp8+head512 JIT cannot compile) — the + # attention fp8 collection failures. Force the serving backend. + # This is not a per-model special case: it reproduces the framework's own + # config-hook branch keyed on head_dim, with the metadata-parity citation. + selected_backend = None + if head_dim > 256 and use_fp8_kv_cache: + selected_backend = AttentionBackendEnum.TRITON_ATTN backend_path = current_platform.get_attn_backend_cls( - None, + selected_backend, attn_selector_config, num_heads=num_heads, ) @@ -147,6 +196,10 @@ def run_attention_torch( backend_name_str = backend_cls.get_name() backend_name = AttentionBackendEnum[backend_name_str] + # the page the framework would run THIS backend with (asked, not tabled) + block_size = framework_kv_block_size(backend_cls) + vllm_config.cache_config.block_size = block_size + kv_cache_spec = create_standard_kv_cache_spec(vllm_config, use_fp8_kv_cache) # Ensure the KV cache has enough blocks for all sequences. @@ -195,15 +248,33 @@ def run_attention_torch( randomize_blocks=True, ) - # The helper populates [2, num_blocks, ...]; vLLM 0.24 uses logical - # [num_blocks, 2, ...] and may impose a backend-specific physical stride. - kv_cache = kv_cache.transpose(0, 1).contiguous() - set_kv_cache_layout(backend_cls.get_required_kv_cache_layout()) - exit_stack.callback(set_kv_cache_layout, None) - stride_order = backend_cls.get_kv_cache_stride_order() - if stride_order != tuple(range(kv_cache.ndim)): - inverse_order = [stride_order.index(i) for i in range(kv_cache.ndim)] - kv_cache = kv_cache.permute(*stride_order).contiguous().permute(*inverse_order) + # serving-same layout resolution for the single backend under test + # (utils.py:240-307@v0.29.0); reset cache_config after the case so the + # next backend re-resolves instead of inheriting ("existing wins outright") + _supported = get_supported_kv_cache_layouts([backend_cls]) + resolve_kv_cache_layout(vllm_config, [[m.name for m in _supported]], [kv_cache_spec]) + exit_stack.callback(lambda: setattr(vllm_config.cache_config, "kv_cache_layout", None)) + # 0.29 impls consume ONE merged tensor [num_blocks, H, block, 2*D] with + # K/V split on the last dim (flash_attn.py:1013-1014 "(B,H,N,2*D) -> + # ((B,N,H,D),(B,N,H,D))"); the 0.24-era per-backend stride permutation + # (get_kv_cache_stride_order) no longer exists. Convert the populated + # helper layout [2, B, N, H, D] once, value-equivalent. + _k, _v = kv_cache[0], kv_cache[1] # [B, N, H, D] + kv_cache = torch.cat((_k.permute(0, 2, 1, 3), _v.permute(0, 2, 1, 3)), dim=-1).contiguous() # [B, H, N, 2*D] + # Serving allocates the per-layer cache in the RESOLVED physical layout and + # backends read it through kv_cache.permute(*layout.layer_view_order) + # (v1/kv_cache_layout.py KVCacheLayout.layer_view_order; flashinfer.py:2091 + # "kv_cache_permute = kv_cache.permute(*stride_order) # HND and contiguous" + # @v0.30.0). A contiguous [B, H, N, 2*D] tensor is only right for the LBHNC + # family (HND) that FLASH_ATTN and the SM100 trtllm-gen FlashInfer path + # resolve (flashinfer.py:543-549); FlashInfer elsewhere resolves the NHD + # default, whose XQA decode asserts "KV cache inner dims (block_size, + # head_size) must be contiguous" on this HND memory (flashinfer.py:2448) — + # every fp8-KV GQA generation case on SM120, 2026-10-01. Lay the memory out + # per the resolved layout and hand the backend the same logical view. + kv_cache = kv_cache_in_resolved_layout( + kv_cache, vllm_config.cache_config.get_resolved_kv_cache_layout().layer_view_order + ) builder_cls, impl_cls = get_attention_backend(backend_name) layer_names = ["placeholder"] diff --git a/python/aisimulate/collector/vllm/collect_attn_encoder.py b/python/aisimulate/collector/vllm/collect_attn_encoder.py index 7537a0612f..8b25ff8b00 100644 --- a/python/aisimulate/collector/vllm/collect_attn_encoder.py +++ b/python/aisimulate/collector/vllm/collect_attn_encoder.py @@ -14,7 +14,7 @@ # encoder_attention: 8/8 representative cases. The native framework # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. -__compat__ = "vllm>=0.24.0,<=0.25.0" +__compat__ = "vllm>=0.24.0,<=0.30.0" import torch from collector.case_generator import get_attention_encoder_head_configs, get_attention_encoder_shape_sweeps diff --git a/python/aisimulate/collector/vllm/collect_computescale.py b/python/aisimulate/collector/vllm/collect_computescale.py index 905b79232c..c136ecdb85 100644 --- a/python/aisimulate/collector/vllm/collect_computescale.py +++ b/python/aisimulate/collector/vllm/collect_computescale.py @@ -1,13 +1,20 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""Measure vLLM FP8 activation quantization overhead for static-FP8 GEMM.""" +"""Measure vLLM FP8 activation quantization overhead for static-FP8 GEMM. + +One measurement, two tables — the registry entry declares the second one as +``extra_perf_filenames`` and the executor hands both names in, so finalize +binds both to this producer's checkpoint: + perf_filename -> computescale_perf.txt (dynamic per-token quant minus static quant) + extra_perf_filenames[0] -> scale_matrix_perf.txt (the static per-tensor quant alone) +""" # B200 0.25.0 qualification (installed vLLM dd10e03f9), job 1968047: # compute_scale: 8/8 representative cases. The native framework # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. -__compat__ = "vllm>=0.24.0,<=0.25.0" +__compat__ = "vllm>=0.24.0,<=0.30.0" import torch from collector.case_generator import get_compute_scale_case_specs @@ -17,6 +24,8 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import GroupShape from vllm.version import __version__ as vllm_version +_OUTSIDE_LOOP_COUNT = 5 + def get_computescale_test_cases(): if get_sm_version() <= 86: @@ -24,37 +33,46 @@ def get_computescale_test_cases(): return [[case.m, case.k] for case in get_compute_scale_case_specs()] -def run_computescale(m, k, *, perf_filename, device="cuda:0"): +def _setup(m, k, device): setup_distributed(device) device = torch.device(device) torch.cuda.set_device(device) torch.set_default_device(device) - x = torch.randn((m, k), dtype=torch.bfloat16, device=device) - static_scale = torch.tensor([1.0], dtype=torch.float32, device=device) - outside_loop_count = 5 + return device, x + - def dynamic_kernel_func(): - for _ in range(outside_loop_count): +def _bench_dynamic(device, x): + def kernel_func(): + for _ in range(_OUTSIDE_LOOP_COUNT): ops.scaled_fp8_quant(x, scale=None, use_per_token_if_dynamic=True) - with benchmark_with_power(device=device, kernel_func=dynamic_kernel_func, repeat_n=1) as dynamic_results: + with benchmark_with_power(device=device, kernel_func=kernel_func, repeat_n=1) as results: pass + return results["latency_ms"] / _OUTSIDE_LOOP_COUNT, results["power_stats"] - dynamic_latency = dynamic_results["latency_ms"] / outside_loop_count - def static_kernel_func(): - for _ in range(outside_loop_count): +def _bench_static(device, x): + static_scale = torch.tensor([1.0], dtype=torch.float32, device=device) + + def kernel_func(): + for _ in range(_OUTSIDE_LOOP_COUNT): ops.scaled_fp8_quant( x, scale=static_scale, group_shape=(GroupShape.PER_TENSOR.row, GroupShape.PER_TENSOR.col), ) - with benchmark_with_power(device=device, kernel_func=static_kernel_func, repeat_n=1) as static_results: + with benchmark_with_power(device=device, kernel_func=kernel_func, repeat_n=1) as results: pass + return results["latency_ms"] / _OUTSIDE_LOOP_COUNT, results["power_stats"] - static_latency = static_results["latency_ms"] / outside_loop_count + +def run_computescale(m, k, *, perf_filename, extra_perf_filenames, device="cuda:0"): + (scale_matrix_filename,) = extra_perf_filenames + device, x = _setup(m, k, device) + dynamic_latency, dynamic_power = _bench_dynamic(device, x) + static_latency, static_power = _bench_static(device, x) compute_scale_latency = max(0.0, dynamic_latency - static_latency) log_perf( @@ -65,8 +83,9 @@ def static_kernel_func(): op_name="compute_scale", kernel_source="dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant", perf_filename=perf_filename, - power_stats=dynamic_results["power_stats"], + power_stats=dynamic_power, ) + log_perf( item_list=[{"m": m, "k": k, "quant_dtype": "fp8", "latency": static_latency}], framework="VLLM", @@ -74,6 +93,6 @@ def static_kernel_func(): device_name=torch.cuda.get_device_name(device), op_name="scale_matrix", kernel_source="static_scaled_fp8_quant", - perf_filename="scale_matrix_perf.txt", - power_stats=static_results["power_stats"], + perf_filename=scale_matrix_filename, + power_stats=static_power, ) diff --git a/python/aisimulate/collector/vllm/collect_dsv4_attn.py b/python/aisimulate/collector/vllm/collect_dsv4_attn.py index 8146fcda48..1208ab6612 100644 --- a/python/aisimulate/collector/vllm/collect_dsv4_attn.py +++ b/python/aisimulate/collector/vllm/collect_dsv4_attn.py @@ -42,7 +42,6 @@ _DSV4_SPARSE_ISL_LIST, _DSV4_SPARSE_MAX_FULL_S, _DSV4_SPARSE_PAST_KV_LIST, - _DSV4_SPARSE_TP_LIST_ATTN, _DSV4_SPARSE_TP_LIST_INDEXER, DSV4_ATTN_KINDS, DSV4_SPARSE_KERNELS, @@ -73,21 +72,22 @@ # CSA/HCA context/generation: 32/32; auxiliary sparse kernels: 4/4. The native framework # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. -__compat__ = "vllm>=0.24.0,<=0.25.0" +# Versions are upgraded IN PLACE (owner policy 2026-09-20): this file always +# targets the manifest pin; prior-version code is `git log -- ` +# away and prior-version DATA stays permanent under its version key. The +# 0.24->0.29 adaptations each carry a serving citation @0.29.0. +__compat__ = "vllm>=0.30.0,<=0.30.0" DEFAULT_MODEL = _DSV4_DEFAULT_MODELS[0] ARCHITECTURE = "DeepseekV4ForCausalLM" ATTN_KIND_TO_COMPRESS_RATIO = {"csa": 4, "hca": 128} -SPARSE_KERNEL_TO_ATTN_KIND = {"paged_mqa_logits": "csa", "hca_attn": "hca"} -SPARSE_KERNEL_TO_OP_NAME = { - "paged_mqa_logits": "dsv4_paged_mqa_logits_module", - "hca_attn": "dsv4_hca_attn_module", -} -SPARSE_KERNEL_TO_PERF_FILE = { - "paged_mqa_logits": PerfFile.DSV4_PAGED_MQA_LOGITS_MODULE, - "hca_attn": PerfFile.DSV4_HCA_ATTN_MODULE, -} +# paged_mqa_logits (the CSA indexer scoring GEMM) is the only standalone sparse +# sub-kernel; the hca_attn FMLA sidecar was removed 2026-10-15 (retired SDK +# placeholder, see case_generator.DSV4_SPARSE_KERNELS). +SPARSE_KERNEL_TO_ATTN_KIND = {"paged_mqa_logits": "csa"} +SPARSE_KERNEL_TO_OP_NAME = {"paged_mqa_logits": "dsv4_paged_mqa_logits_module"} +SPARSE_KERNEL_TO_PERF_FILE = {"paged_mqa_logits": PerfFile.DSV4_PAGED_MQA_LOGITS_MODULE} MODEL_CONFIGS_DIR = Path(__file__).resolve().parents[2] / "src" / "aisimulate_core" / "model_configs" SUPPORTED_GEMM_TYPES = {"fp8_block"} DEFAULT_MAX_SEQ_LEN = 65536 @@ -253,7 +253,7 @@ def _create_dsv4_attention_module( capability = current_platform.get_device_capability() if capability is not None and capability.to_int() == 89: raise RuntimeError( - "vLLM 0.24.0 DeepSeek-V4 attention is unsupported on SM89: " + "vLLM DeepSeek-V4 attention is unsupported on SM89 (capabilities.yaml op_min_sm floors it at 90): " "the production selector falls back to FlashMLA, whose DSV4 " "backend supports SM90 and SM10x only." ) @@ -361,23 +361,55 @@ def _remap_common_metadata(common, *, block_size: int, device: str): def _allocate_attention_kv_cache(backend, spec, num_blocks: int, cache_dtype: str, *, device: str) -> torch.Tensor: - shape_block_size = spec.storage_block_size if spec.storage_block_size != spec.block_size else spec.block_size - shape = backend.get_kv_cache_shape( - num_blocks, - shape_block_size, - spec.num_kv_heads, - spec.head_size, - cache_dtype, - ) + # 0.29: the shape contract lives in the generic spec pipeline. Serving + # allocates flat bytes and views each layer as the 4D logical [B,H,N,C] + # page (compute_layer_kv_cache_shape_bytes, v1/kv_cache_interface.py:253-269; + # per-layer element view = bytes.view(spec.dtype), create_kv_cache_views). + # The dsv4 layout facts (584B fp8_ds_mla state content, 576B alignment, + # tokens_per_state=compress_ratio) are spec fields at 0.29 + # (models/deepseek_v4/attention.py:741-753), so the helper output IS the + # serving shape. Page padding: mirror create_kv_cache_views' block stride + # (page_size_bytes per block) via empty_strided when the spec pads pages. + from vllm.v1.kv_cache_interface import compute_layer_kv_cache_shape_bytes + + shape_bytes = compute_layer_kv_cache_shape_bytes(spec, num_blocks) + dtype_size = torch.empty((), dtype=spec.dtype).element_size() + shape = (*shape_bytes[:-1], shape_bytes[-1] // dtype_size) if spec.page_size_padded is None: return torch.zeros(shape, dtype=spec.dtype, device=device) - dtype_size = torch.empty((), dtype=spec.dtype).element_size() strides = list(torch.empty(shape).stride()) strides[0] = spec.page_size_bytes // dtype_size return torch.empty_strided(shape, tuple(strides), dtype=spec.dtype, device=device).zero_() +def _make_builder(backend, spec, prefix, vllm_config, common, *, device: str): + """Construct the layer's metadata builder exactly as serving does at 0.29. + + Builders may demand kw-only block_table_width (DeepseekV32IndexerMetadataBuilder, + mla/indexer.py:519,531); serving computes it via get_block_table_width and + ALLOCATES the block table at that width (v1/worker/utils.py:276-283 + + v1/worker/block_table.py:29-49), so builders assume + table.shape[1] == width — the family-100 decode path copies the table into + a width-sized expanded buffer (indexer.py:609,745) and crashes on narrower + tables (found on B300 2026-09-20 in the mla 029 lane; same contract here). + Zero-pad our ceil(seq/block)-column table to the width. + """ + builder_cls = backend.get_builder_cls() + kwargs = {} + if getattr(builder_cls, "requires_block_table_width", False): + from vllm.v1.worker.block_table import get_block_table_width + + max_num_blocks = spec.max_num_blocks_per_req(vllm_config, vllm_config.model_config.max_model_len) + width = get_block_table_width(max_num_blocks, spec.block_size) + kwargs["block_table_width"] = width + _bt = common.block_table_tensor + if _bt.shape[1] < width: + _pad = torch.zeros((_bt.shape[0], width - _bt.shape[1]), dtype=_bt.dtype, device=_bt.device) + common.block_table_tensor = torch.cat([_bt, _pad], dim=1) + return builder_cls(spec, [prefix], vllm_config, torch.device(device), **kwargs) + + def _build_metadata_and_bind_caches(attn_module: DeepseekV4Attention, vllm_config, common, *, device: str): cache_blocks = _cache_blocks(int(common.num_reqs), int(common.max_seq_len)) metadata = {} @@ -392,19 +424,21 @@ def _build_metadata_and_bind_caches(attn_module: DeepseekV4Attention, vllm_confi if spec is None: continue backend = registered_layer.get_attn_backend() - metadata[layer.prefix] = backend.get_builder_cls()( - spec, - [layer.prefix], - vllm_config, - torch.device(device), - ).build(0, common) + metadata[layer.prefix] = _make_builder(backend, spec, layer.prefix, vllm_config, common, device=device).build( + 0, common + ) cache_dtype = getattr(spec, "cache_dtype_str", None) or "auto" - registered_layer.kv_cache = _allocate_attention_kv_cache( - backend, - spec, - cache_blocks, - cache_dtype, - device=device, + # serving binds through each layer's bind_kv_cache + # (v1/worker/utils.py:622@0.29.0) — layer overrides may reshape the + # raw view (e.g. squeeze the H=1 dim), so never assign .kv_cache raw + registered_layer.bind_kv_cache( + _allocate_attention_kv_cache( + backend, + spec, + cache_blocks, + cache_dtype, + device=device, + ) ) compressors = [attn_module.compressor] @@ -415,23 +449,22 @@ def _build_metadata_and_bind_caches(attn_module: DeepseekV4Attention, vllm_confi spec = state_cache.get_kv_cache_spec(vllm_config) backend = state_cache.get_attn_backend() compressor_common = _remap_common_metadata(common, block_size=spec.block_size, device=device) - metadata[state_cache.prefix] = backend.get_builder_cls()( - spec, - [state_cache.prefix], - vllm_config, - torch.device(device), + metadata[state_cache.prefix] = _make_builder( + backend, spec, state_cache.prefix, vllm_config, compressor_common, device=device ).build(0, compressor_common) state_cache_blocks = _cache_blocks_for_block_size( int(common.num_reqs), int(common.max_seq_len), spec.block_size, ) - state_cache.kv_cache = _allocate_attention_kv_cache( - backend, - spec, - state_cache_blocks, - getattr(spec, "cache_dtype_str", None) or "auto", - device=device, + state_cache.bind_kv_cache( + _allocate_attention_kv_cache( + backend, + spec, + state_cache_blocks, + getattr(spec, "cache_dtype_str", None) or "auto", + device=device, + ) ) return metadata @@ -674,46 +707,6 @@ def kernel_func(): return result -def _bench_mla_sparse_op( - *, - attn_module: DeepseekV4Attention, - vllm_config, - metadata, - positions: torch.Tensor, - num_tokens: int, - device: str, - warming_up: int, - test_ite: int, -) -> dict: - sparse_attn = attn_module - q = torch.full( - (num_tokens, sparse_attn.padded_heads, sparse_attn.head_dim), - 0.01, - dtype=torch.bfloat16, - device=device, - ) - kv = torch.zeros((num_tokens, sparse_attn.head_dim), dtype=torch.bfloat16, device=device) - output = torch.empty_like(q) - - with set_current_vllm_config(vllm_config), set_forward_context(metadata, vllm_config), torch.inference_mode(): - sparse_attn.forward_mqa(q, kv, positions, output) - torch.cuda.synchronize() - - def kernel_func(): - sparse_attn.forward_mqa(q, kv, positions, output) - - with benchmark_with_power( - device=torch.device(device), - kernel_func=kernel_func, - num_warmups=warming_up, - num_runs=test_ite, - repeat_n=1, - use_cuda_graph=True, - ) as result: - pass - return result - - def _bench_sparse_kernel_shape( *, model_path: str, @@ -734,64 +727,21 @@ def _bench_sparse_kernel_shape( raise ValueError(f"invalid sparse sequence length: isl={isl}, past_kv={past_kv}") attn_kind = SPARSE_KERNEL_TO_ATTN_KIND[kernel] - is_context = isl > 1 - query_len = isl - num_tokens = batch_size * query_len - - if kernel == "paged_mqa_logits": - hf_config = SimpleNamespace(**_read_model_config(model_path)) - architecture = hf_config.architectures[0] if hf_config.architectures else ARCHITECTURE - native_num_heads = int(hf_config.num_attention_heads) - local_num_heads = native_num_heads // tp_size - backend_name = "vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits" - result = _bench_paged_mqa_logits_kernel( - hf_config=hf_config, - num_query_rows=num_tokens, - past_kv=past_kv, - device=device, - warming_up=warming_up, - test_ite=test_ite, - ) - else: - with ( - _tp_simulation(tp_size), - _create_dsv4_attention_module( - model_path=model_path, - attn_kind=attn_kind, - batch_size=batch_size, - seq_len=full_seq_len, - tp_size=tp_size, - is_context=is_context, - device=device, - query_len=query_len, - ) as (attn_module, vllm_config), - ): - common = _make_common_metadata( - batch_size=batch_size, - seq_len=full_seq_len, - is_context=is_context, - query_len=query_len, - device=device, - ) - metadata = _build_metadata_and_bind_caches(attn_module, vllm_config, common, device=device) - positions = common.positions - hf_config = vllm_config.model_config.hf_config - architecture = hf_config.architectures[0] if hf_config.architectures else ARCHITECTURE - native_num_heads = int(hf_config.num_attention_heads) - local_num_heads = int(attn_module.n_local_heads) - concrete_layer = vllm_config.compilation_config.static_forward_context[attn_module.prefix] - backend_name = concrete_layer.get_attn_backend().get_name() - result = _bench_mla_sparse_op( - attn_module=attn_module, - vllm_config=vllm_config, - metadata=metadata, - positions=positions, - num_tokens=num_tokens, - device=device, - warming_up=warming_up, - test_ite=test_ite, - ) - del attn_module, vllm_config, metadata, common, positions + num_tokens = batch_size * isl + + hf_config = SimpleNamespace(**_read_model_config(model_path)) + architecture = hf_config.architectures[0] if hf_config.architectures else ARCHITECTURE + native_num_heads = int(hf_config.num_attention_heads) + local_num_heads = native_num_heads // tp_size + backend_name = "vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits" + result = _bench_paged_mqa_logits_kernel( + hf_config=hf_config, + num_query_rows=num_tokens, + past_kv=past_kv, + device=device, + warming_up=warming_up, + test_ite=test_ite, + ) latency = float(result["latency_ms"]) @@ -800,7 +750,7 @@ def _bench_sparse_kernel_shape( { "model": model_path, "architecture": architecture, - "mla_dtype": "fp8_e4m3" if kernel == "paged_mqa_logits" else "bfloat16", + "mla_dtype": "fp8_e4m3", "kv_cache_dtype": "fp8", "gemm_type": "fp8_block", # Sparse-op consumers key ``num_heads`` by the native model @@ -913,7 +863,7 @@ def _build_vllm_dsv4_sparse_test_cases(kernel: str) -> list[list]: if not model_paths: return [] - tp_list = _DSV4_SPARSE_TP_LIST_ATTN if kernel == "hca_attn" else _DSV4_SPARSE_TP_LIST_INDEXER + tp_list = _DSV4_SPARSE_TP_LIST_INDEXER if "--smoke" in sys.argv: smoke_shapes = [ (1, 1, 8192), @@ -1058,10 +1008,6 @@ def get_dsv4_paged_mqa_logits_test_cases(): return _build_vllm_dsv4_sparse_test_cases("paged_mqa_logits") -def get_dsv4_hca_attn_test_cases(): - return _build_vllm_dsv4_sparse_test_cases("hca_attn") - - def main() -> None: parser = argparse.ArgumentParser(description="Collect vLLM DeepSeek-V4 attention module latency.") parser.add_argument("--model-path", default=DEFAULT_MODEL) diff --git a/python/aisimulate/collector/vllm/collect_gdn.py b/python/aisimulate/collector/vllm/collect_gdn.py index 6a4e468a9d..badbbd0420 100644 --- a/python/aisimulate/collector/vllm/collect_gdn.py +++ b/python/aisimulate/collector/vllm/collect_gdn.py @@ -60,7 +60,7 @@ # 1968047). Existing framework construction/dispatch is unchanged. Invalid # model/quant shapes continue to raise and remain recorded failures; this is # API compatibility, not a claim that every Cartesian shape is supported. -__compat__ = "vllm>=0.24.0,<=0.27.1,!=0.25.1,!=0.26.0,!=0.27.0" +__compat__ = "vllm>=0.24.0,<=0.30.0,!=0.25.1,!=0.26.0,!=0.27.0" import gc import os @@ -369,6 +369,10 @@ def run_conv1d(_conv_input=conv_input, _conv_state=conv_state): # present and source-unchanged, so the claim is not dangling, but # the bug reproduction itself is unconfirmed at this version. # Re-verify on the next vLLM bump or the first 0.27.1 GPU run. + # RE-VERIFIED on hardware at 0.30.0, SM89 (L40, 2026-10-04, clean GPU, container per op): + # a 40-case --shuffle sample of the gdn op failed 21 tasks, 17 of them CUDA faults + # (9 illegal memory access + 8 SIGABRT) in this same chunked prefill at the large-token + # sub-points, 3 the grid-y limit below, 1 capacity OOM — the family is unchanged from 0.24.0. def run_gdn_scan(_q=q, _k=k, _v=v, _g=g, _beta=beta, _state=gdn_state): chunk_gdn( q=_q, diff --git a/python/aisimulate/collector/vllm/collect_gemm.py b/python/aisimulate/collector/vllm/collect_gemm.py index 0d5a94908e..304d1de7b0 100644 --- a/python/aisimulate/collector/vllm/collect_gemm.py +++ b/python/aisimulate/collector/vllm/collect_gemm.py @@ -58,9 +58,11 @@ # 322-363); NVFP4 still uses CT's factory and records the selected kernel # (schemes/compressed_tensors_w4a4_nvfp4.py:29-31,95-141). This adds the # exact 0.25.0 release to the existing lane, not 0.25.1/0.26.0/0.27.0. -# 0.30.0 source audit (2026-09-30: tag v0.30.0 == ced6857afa0e vs v0.27.1), -# the baseline of the 0.31.0 audit below. 0.28.0, 0.29.0 and 0.30.0 itself -# are not admitted: no consumer reads their tables. +# 0.30.0 source audit (2026-09-30: tag v0.30.0 == ced6857afa0e vs v0.27.1): +# the collector default (framework_manifest.yaml; the 2026-10 campaign +# tables) and the baseline of the 0.31.0 audit below, which admits the +# GLM-5.3-Flash model-pinned run. 0.28.0/0.29.0 stay admitted from the +# 0.27.1 -> 0.30.0 upgrade path. # bf16: UnquantizedLinearMethod now binds its GEMM at __init__ from # kernel_config.linear_backend (layers/linear.py:169-174, apply :234-238); # dispatch_unquantized_gemm (layers/utils.py:584-616) returns @@ -128,8 +130,8 @@ # 0.7.0.post1 (was 0.6.18.post1). # Helpers with identical bodies: scaled_fp4_quant (_custom_ops.py:1513-1588), # per_block_cast_to_fp8 (utils/deep_gemm.py:983-1002). All imports below -# resolve at 0.31.0. 0.28.0/0.29.0 are still not audited. -__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0,!=0.28.0,!=0.29.0,!=0.30.0" +# resolve at 0.31.0. +__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0" from types import SimpleNamespace @@ -204,6 +206,19 @@ def get_gemm_test_cases(): @with_exit_stack def run_gemm(exit_stack, gemm_type, m, n, k, *, perf_filename, device="cuda:0"): + if gemm_type == "fp8_block" and (n % 16 or k % 16): + # FIXME(kernel-limit): vLLM routes block-fp8 GEMMs whose weight dims are not multiples + # of 16 away from cutlass (_custom_ops.cutlass_scaled_mm: `cutlass_compatible_b = + # b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0` @0.30.0:819) into triton_scaled_mm, + # whose scale-shape assert cannot take 128x128 block scales (triton_scaled_mm.py:201). + # Deterministic per shape: gb200 1,110 cases, every one N in {1,4,8,12,24} (pipeline + # 72343556; b200/b300 ~130 of the same shard). The ed9109ec attempt to classify by a + # pre-flight forward never ran: create_gemm()'s own dry run asserts first. Refuse before + # the layer is built; serving linears with these N take the same path. + raise RuntimeError( + f"FIXME(kernel-limit): vLLM fp8_block GEMM needs n and k multiples of 16 for cutlass_scaled_mm " + f"(n={n}, k={k}); the triton_scaled_mm fallback asserts on 128x128 block scales" + ) setup_distributed(device) if envs.VLLM_BATCH_INVARIANT: diff --git a/python/aisimulate/collector/vllm/collect_kda.py b/python/aisimulate/collector/vllm/collect_kda.py index e5dcc0bbf9..36ebc67da2 100644 --- a/python/aisimulate/collector/vllm/collect_kda.py +++ b/python/aisimulate/collector/vllm/collect_kda.py @@ -48,21 +48,16 @@ kda_perf.txt — same column layout as the sglang kda collector. """ -# Serve-parity `metadata=` argument validated on the 0.27.0 release image -# (Kimi-K3 landed upstream there). The manifest kda family pin stays on the -# kimi-k3 preview build until the SDK consumer routes 0.27.0's split decode -# (see collector/framework_manifest.yaml), so this module exactly pins the -# preview version; it runs on either image — the DS-layout probe -# (is_fused_kda_decode_supported) yields fused_kda_decode rows on the -# preview and the packed conv-update + recurrence pair on 0.27.0. -# The file-level range spans the two audited routes (PEP 440 clauses are -# conjunctive, so it cannot name the two releases alone). Each KDA -# architecture is additionally gated at runtime to its own audited release by -# _KDA_ARCHITECTURE_COMPAT: Kimi-K3 only on the 0.1.dev19262 preview, GLM-5.3- -# Flash only on stock 0.31.0 (no overlay); every release in between -# (including 0.30.0, where the GLM layer lived under glm5next/nvidia/) raises -# KdaRuntimeNotAuditedError. -__compat__ = "vllm>=0.1.dev19262,<=0.31.0" +# Kimi-K3 runs on the manifest default 0.30.0 since the kda family pin on +# the kimi-k3 preview image was retired 2026-09-28: 0.30's DS-layout probe +# (is_fused_kda_decode_supported, now with recurrent_state_dtype) selects +# fused_kda_decode for the non-spec decode, and the packed conv-update + +# recurrence pair is measured alongside as the spec-decode serving path. +# GLM-5.3-Flash runs on stock 0.31.0 (no overlay; model-pinned run). The +# file-level range spans the two audited routes; each KDA architecture is +# additionally gated at runtime to its own audited release by +# _KDA_ARCHITECTURE_COMPAT, raising KdaRuntimeNotAuditedError otherwise. +__compat__ = "vllm>=0.30.0,<=0.31.0" import gc import os @@ -98,6 +93,17 @@ # (config.json linear_attn_config.gate_lower_bound). KDA_LOWER_BOUND = -5.0 + +def _decode_paths() -> set[str]: + """Which decode paths the generation benchmark measures. Default: both the + fused CUDA decode kernel (serving's non-spec path) and the packed + conv-update + Triton recurrence pair (serving's path under speculative + decoding). The opharness kda decode gate captures serving WITHOUT spec + decoding, so it sets AIS_KDA_DECODE_PATHS=fused to compare like with like + (recorded in the capture's env). Collection runs never set it.""" + return {p.strip() for p in os.environ.get("AIS_KDA_DECODE_PATHS", "fused,packed").split(",") if p.strip()} + + NUM_WARMUPS = 3 NUM_RUNS = 10 @@ -151,10 +157,16 @@ def _log(common, latency_ms, kernel_source, perf_filename, vllm_version, device, def _resolve_prefill_kernel(dtype: torch.dtype): """Mirror serving's resolve_kda_prefill_backend on this device: FlashKDA when supported AND importable, else the Triton chunk kernel - (vllm/models/kimi_k3/nvidia/kda.py resolve_kda_prefill_backend).""" + (vllm/models/kimi_k3/nvidia/kda.py:455-495@v0.30.0 resolve_kda_prefill_backend; + backend "auto" never selects FlashInfer, so on SM90 the choice is FlashKDA + or Triton). 0.30 added the recurrent-state dtype to the probe: serving + passes get_state_dtype()[1] (kda.py:665), which + MambaStateDtypeCalculator.kda_state_dtype resolves to float32 for the + default mamba_ssm_cache_dtype "auto" — the dtype this collector's + init_state uses below.""" from vllm.models.kimi_k3.nvidia.kda import is_flashkda_supported - if is_flashkda_supported(128, dtype, KDA_LOWER_BOUND): + if is_flashkda_supported(128, dtype, torch.float32, KDA_LOWER_BOUND): try: import vllm._flashkda_C # noqa: F401 from vllm.models.kimi_k3.nvidia.kda import _flashkda_prefill @@ -343,21 +355,58 @@ def run_conv_qkv3(): init_state = torch.zeros(batch_size, nh, hd, hd, dtype=torch.float32, device=device) if prefill_source == "flashkda_fwd": - - def run_prefill(): - flashkda_prefill( - q, - k, - v, - raw_g, - raw_beta, - a_log, - dt_bias, - KDA_LOWER_BOUND, - init_state, - cu, + import inspect + + # vLLM 0.29.0 FlashKDA writes into caller-owned buffers taken + # from the engine workspace (kda.py:536-547 sizes them: + # out (1,T,H,D) model dtype, final_state (N,H,D,D) fp32, + # workspace uint8 of _flashkda_C.get_workspace_size(T,H,N); + # the non-checkpoint prefill call is kda.py:939-953). The + # kimi-k3 preview build returned them instead — dispatch on + # the signature so both builds run the same kernel. + if "out" in inspect.signature(flashkda_prefill).parameters: + import vllm._flashkda_C # noqa: F401 + + fk_out = torch.empty_like(q) + fk_final = torch.zeros(batch_size, nh, hd, hd, dtype=torch.float32, device=device) + fk_ws = torch.empty( + int(torch.ops._flashkda_C.get_workspace_size(nt, nh, batch_size)), + dtype=torch.uint8, + device=device, ) + def run_prefill(): + flashkda_prefill( + q, + k, + v, + raw_g, + raw_beta, + a_log, + dt_bias, + KDA_LOWER_BOUND, + init_state, + cu, + out=fk_out, + final_state=fk_final, + workspace=fk_ws, + ) + else: + + def run_prefill(): + flashkda_prefill( + q, + k, + v, + raw_g, + raw_beta, + a_log, + dt_bias, + KDA_LOWER_BOUND, + init_state, + cu, + ) + else: def run_prefill(): @@ -447,7 +496,15 @@ def run_kda_generation_benchmark( nh, hd, cw = num_v_heads, head_v_dim, d_conv dim = nh * hd - fused_ok = is_fused_kda_decode_supported(nh, hd, cw, num_spec=0, input_dtype=dtype, conv_state_dtype=dtype) + # 0.30 added the recurrent-state dtype to the decode probe as well: serving + # resolves it from get_state_dtype()[1] (vllm/models/kimi_k3/nvidia/kda.py + # :665-680@v0.30.0, resolve_kda_decode_backend -> is_fused_kda_decode_supported + # :157-178), float32 for the default mamba_ssm_cache_dtype "auto" — the dtype + # of this collector's recurrent state below. + decode_paths = _decode_paths() + fused_ok = is_fused_kda_decode_supported( + nh, hd, cw, num_spec=0, input_dtype=dtype, conv_state_dtype=dtype, recurrent_state_dtype=torch.float32 + ) conv_weight = torch.randn(3 * dim, cw, dtype=torch.float32, device=device) fused_weight = conv_weight.reshape(3, dim, cw).transpose(1, 2).contiguous() norm_weight = torch.randn(hd, dtype=torch.float32, device=device) @@ -483,7 +540,7 @@ def run_kda_generation_benchmark( "model_name": model_name, } - if fused_ok: + if fused_ok and "fused" in decode_paths: def run_fused_decode(): ops.fused_kda_decode( @@ -520,55 +577,57 @@ def run_fused_decode(): results["power_stats"], ) - def run_conv_update(): - causal_conv1d_update( - x, - conv_state, - conv_weight, - None, - activation="silu", - conv_state_indices=idx, - validate_data=True, - out=conv_out, - ) + if "packed" in decode_paths: - with benchmark_with_power( - device=device, kernel_func=run_conv_update, num_warmups=NUM_WARMUPS, num_runs=NUM_RUNS, repeat_n=1 - ) as results: - _log( - common, - results["latency_ms"], - "causal_conv1d_update", - perf_filename, - vllm_version, - device, - results["power_stats"], - ) + def run_conv_update(): + causal_conv1d_update( + x, + conv_state, + conv_weight, + None, + activation="silu", + conv_state_indices=idx, + validate_data=True, + out=conv_out, + ) - def run_packed_decode(): - fused_recurrent_kda_packed_decode( - mixed_qkv=x, - raw_g=raw_g, - raw_beta=raw_beta, - A_log=a_log, - dt_bias=dt_bias_hd, - lower_bound=KDA_LOWER_BOUND, - initial_state=state, - state_indices=idx, - ) + with benchmark_with_power( + device=device, kernel_func=run_conv_update, num_warmups=NUM_WARMUPS, num_runs=NUM_RUNS, repeat_n=1 + ) as results: + _log( + common, + results["latency_ms"], + "causal_conv1d_update", + perf_filename, + vllm_version, + device, + results["power_stats"], + ) - with benchmark_with_power( - device=device, kernel_func=run_packed_decode, num_warmups=NUM_WARMUPS, num_runs=NUM_RUNS, repeat_n=1 - ) as results: - _log( - common, - results["latency_ms"], - "fused_recurrent_kda_packed_decode", - perf_filename, - vllm_version, - device, - results["power_stats"], - ) + def run_packed_decode(): + fused_recurrent_kda_packed_decode( + mixed_qkv=x, + raw_g=raw_g, + raw_beta=raw_beta, + A_log=a_log, + dt_bias=dt_bias_hd, + lower_bound=KDA_LOWER_BOUND, + initial_state=state, + state_indices=idx, + ) + + with benchmark_with_power( + device=device, kernel_func=run_packed_decode, num_warmups=NUM_WARMUPS, num_runs=NUM_RUNS, repeat_n=1 + ) as results: + _log( + common, + results["latency_ms"], + "fused_recurrent_kda_packed_decode", + perf_filename, + vllm_version, + device, + results["power_stats"], + ) ok += 1 except Exception as e: err += 1 @@ -823,7 +882,7 @@ def _is_glm5_next_kda(model_name: str) -> bool: # replicates; any other installed release raises KdaRuntimeNotAuditedError (a # classified failure) instead of timing a possibly different kernel path. _KDA_ARCHITECTURE_COMPAT = { - KIMI_K3_KDA_ARCHITECTURE: "vllm==0.1.dev19262", + KIMI_K3_KDA_ARCHITECTURE: "vllm==0.30.0", GLM5_NEXT_KDA_ARCHITECTURE: "vllm==0.31.0", } diff --git a/python/aisimulate/collector/vllm/collect_mhc_module.py b/python/aisimulate/collector/vllm/collect_mhc_module.py index 628bfc8331..a5c187b9fd 100644 --- a/python/aisimulate/collector/vllm/collect_mhc_module.py +++ b/python/aisimulate/collector/vllm/collect_mhc_module.py @@ -28,12 +28,11 @@ # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. # -# The file-level range spans the two audited routes; each architecture is +# The file-level range spans the audited routes; each architecture is # additionally gated to its own audited release(s) by ``_ARCHITECTURE_COMPAT`` -# below, so DeepSeek-V4 behaviour on 0.24.0-0.25.0 is unchanged and neither -# architecture runs on an unaudited release in between (it raises instead). -# GLM-5.3-Flash moved to stock vLLM 0.31.0 (no overlay); the upper bound -# admits it while DeepSeek-V4 stays gated to its own releases. +# below: DeepSeek-V4 on 0.24.0-0.30.0 (0.30.0 is the collector default), +# GLM-5.3-Flash on stock vLLM 0.31.0 only (no overlay); an architecture on +# any other release raises instead of running. __compat__ = "vllm>=0.24.0,<=0.31.0" # vLLM imports stay lazy in this module so that a mismatched install fails @@ -64,7 +63,7 @@ def _mhc_tilelang_kernels(): # audited on the installed release raises MhcRuntimeNotAuditedError (a # classified failure) instead of running an unverified dispatch. _ARCHITECTURE_COMPAT = { - ARCHITECTURE: "vllm>=0.24.0,<=0.25.0", + ARCHITECTURE: "vllm>=0.24.0,<=0.30.0", # Stock vllm 0.31.0 only (no overlay) — see the GLM audit below. GLM5NEXT_ARCHITECTURE: "vllm==0.31.0", } diff --git a/python/aisimulate/collector/vllm/collect_mla_bmm.py b/python/aisimulate/collector/vllm/collect_mla_bmm.py index 3218f680de..b8a73fe40c 100644 --- a/python/aisimulate/collector/vllm/collect_mla_bmm.py +++ b/python/aisimulate/collector/vllm/collect_mla_bmm.py @@ -23,9 +23,12 @@ # MLA bmm call site unchanged in-image on GB300). # B200 0.25.0 qualification: both pre/post torch.bmm callbacks passed in # job 1968047. The NVIDIA bf16 BMM contract and input geometry are unchanged. -__compat__ = "vllm>=0.25.0,<=0.27.0,!=0.25.1,!=0.26.0" +__compat__ = "vllm>=0.25.0,<=0.30.0,!=0.25.1,!=0.26.0" + +from importlib.metadata import ( + version as _dist_version, # setuptools/pkg_resources is absent from recent framework images +) -import pkg_resources import torch from collector.case_generator import get_mla_bmm_case_specs from collector.helper import benchmark_with_power, log_perf @@ -62,7 +65,7 @@ def _log_row(*, op_name, dtype, num_tokens, num_heads, results, perf_filename, d } ], framework="VLLM", - version=pkg_resources.get_distribution("vllm").version, + version=_dist_version("vllm"), device_name=torch.cuda.get_device_name(device), op_name=op_name, kernel_source="vllm_torch_bmm", diff --git a/python/aisimulate/collector/vllm/collect_mla_module.py b/python/aisimulate/collector/vllm/collect_mla_module.py index 82af603e83..bfd1e33c0d 100644 --- a/python/aisimulate/collector/vllm/collect_mla_module.py +++ b/python/aisimulate/collector/vllm/collect_mla_module.py @@ -1,13 +1,16 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Versions are upgraded IN PLACE (owner policy 2026-09-20): this file always +# targets the manifest pin; prior-version code is `git log -- ` +# away and prior-version DATA stays permanent under its version key. The +# 0.24->0.29 adaptations each carry a serving citation @0.29.0. On 0.29 the +# kv dtype FORKS the backend (bf16 -> FLASH_ATTN_MLA_SPARSE, fp8 -> +# FLASHMLA_SPARSE on SM90) — both keys are collected. +# # Portions adapted from vLLM dd10e03f95f94edbea1975c67ace3a35ec9a8a40. # Copyright contributors to the vLLM project. Modified for collector execution. - -# 0.27.0 audit (pod GB300 probe): DeepseekV2MLAAttention ctor params, -# _CONFIG_REGISTRY gap (glm_moe_dsa still unmapped), -# backend_supports_prefill_query_quantization (mla_attention.py) and the -# prefill selector surface are all unchanged vs the 0.24.0 citations below. -__compat__ = "vllm>=0.24.0,<=0.25.1" +__compat__ = "vllm>=0.30.0,<=0.30.0" """ MLA Module Collector for vLLM — unified MLA and DSA benchmarking. @@ -26,6 +29,12 @@ config, overrides the layer-local shape fields in-memory, and then instantiates just the attention module. +Case layout: one case per (batch, heads, kv dtype, compute dtype, gemm type, +model, attention type, prefix) carrying its seq_len points; run_mla_module +builds the module once per case and measures every point on it, each with its +own KV cache and metadata (the module build was most of the per-point cost in +the 2026-10 campaign: vllm dsa_context_module alone was 59% of its GPU time). + Usage: # MLA context phase (DeepSeek-V3 style) python collect_mla_module.py --mode context --model deepseek-ai/DeepSeek-V3 @@ -47,7 +56,9 @@ import math import os import tempfile +import time import traceback +from contextlib import ExitStack from functools import partial from pathlib import Path @@ -66,6 +77,7 @@ create_common_attn_metadata, create_vllm_config, enable_engine_fused_ops, + framework_kv_block_size, setup_distributed, with_exit_stack, ) @@ -95,11 +107,15 @@ # Local model config resolution — avoid HuggingFace Hub downloads # ═══════════════════════════════════════════════════════════════════════ -# Pre-cached HF configs live in src/aisimulate_core/model_configs/ as -# "--_config.json". vLLM's ModelConfig accepts a local -# directory containing config.json, so we create a temp dir with a -# symlink when the cached file exists. -_MODEL_CONFIGS_DIR = Path(__file__).resolve().parents[2] / "src" / "aisimulate_core" / "model_configs" +# Pre-cached HF configs ship with AISim as "--_config.json". +# The directory is helper.py's constant — never recomputed here, so a +# package rename cannot silently strand this lane (the aiconfigurator -> +# aisimulate consolidation did exactly that to a hand-built copy of it). +# vLLM's ModelConfig accepts a local directory containing config.json, so +# we create a temp dir with a symlink when the cached file exists. +from collector.helper import _AIC_MODEL_CONFIG_DIR as _AIS_MODEL_CONFIG_DIR + +_MODEL_CONFIGS_DIR = Path(_AIS_MODEL_CONFIG_DIR) # Cache of model_name -> temp dir path (created once per process). _local_config_cache: dict[str, str] = {} @@ -193,6 +209,34 @@ def _generation_kv_footprint_bytes(total_tokens: int, kv_cache_dtype: str) -> in return 2 * total_tokens * _MLA_KV_ENTRY_ELEMS * bytes_per_elem +def _dsa_context_point_allowed( + batch_size: int, + seq_len: int, + prefix_len: int, + *, + new_token_budget: int, + full_seq_budget: int | None, + max_position: int | None, +) -> bool: + """Whether one DSA context point (new tokens ``seq_len`` behind ``prefix_len`` cached + tokens, ``batch_size`` requests) is in the plan: serving's chunked-prefill new-token + budget, the full-sequence budget and the model's max_position_embeddings. The plan + generator and ``run_mla_module`` (which expands a case's isl x prefix grid) share it, so + a series measures exactly the points the plan declared.""" + if batch_size * seq_len > new_token_budget: + return False + if full_seq_budget and batch_size * (seq_len + prefix_len) > full_seq_budget: + return False + return max_position is None or prefix_len + seq_len <= max_position + + +def _dsa_context_budgets(sweep=None) -> tuple[int, int | None]: + """(new-token budget, full-sequence budget) of the vLLM DSA context sweep.""" + sweep = sweep or get_mla_module_sweep_spec("vllm") + new_token_budget = getattr(sweep, "context_dsa_chunk_prefill_size", None) or sweep.context_max_tokens + return int(new_token_budget), getattr(sweep, "context_dsa_max_full_sequence_length", None) + + def get_context_test_cases(attn_type: str): """Context-phase test cases. @@ -201,16 +245,37 @@ def get_context_test_cases(attn_type: str): """ cases = [] sweep = get_mla_module_sweep_spec("vllm") + # DSA context mirrors serving's chunked prefill (matches DSV4 sparse + + # sglang DSA): cap the per-forward NEW-token budget (b*s) at the chunk + # prefill size and let the long context enter via prefix, bounded by the + # full-sequence budget b*(s+prefix). This makes rows faithful to the + # engine's query shape (isl=chunk, prefix=context) and keeps the fp8 + # sparse-decode q stride int32-safe. Falls back to the shared context + # axes when the DSA-specific budgets are not declared. + dsa_new_token_budget, dsa_full_seq_budget = _dsa_context_budgets(sweep) + dsa_prefixes = getattr(sweep, "context_dsa_prefix_lengths", None) or sweep.context_prefix_lengths for compute_dtype, kv_dtype, gemm_type in _get_precision_combos("context", attn_type): for num_heads in sweep.inner_sweep_head_counts: for b in sweep.context_batch_sizes: for s in sweep.context_sequence_lengths: - if b * s > sweep.context_max_tokens: - continue if attn_type == "dsa": - for prefix_len in sweep.context_prefix_lengths: + # new-token budget per chunk (not the shared memory cap) and the + # full-sequence budget; the per-model max_position cap is applied + # by _build_module_test_cases with the same predicate. + for prefix_len in dsa_prefixes: + if not _dsa_context_point_allowed( + b, + s, + prefix_len, + new_token_budget=dsa_new_token_budget, + full_seq_budget=dsa_full_seq_budget, + max_position=None, + ): + continue cases.append([s, b, num_heads, kv_dtype, compute_dtype, gemm_type, prefix_len]) else: + if b * s > sweep.context_max_tokens: + continue cases.append([s, b, num_heads, kv_dtype, compute_dtype, gemm_type]) return cases @@ -252,22 +317,232 @@ def get_generation_test_cases(attn_type: str): return cases +def _strided_throwaway_copy(query): + """A scratch tensor with the query's shape AND strides — never a contiguous clone. + + vLLM's own rope kernel derives ONE head stride from the query and applies it to the + key as well (csrc/libtorch_stable/pos_encoding_kernels.cu:57,68,164-165 @0.30.0, + unchanged @0.31.0). The q_pe that attention.py:534 ropes alone is a last-dim slice of + q, so its head stride is qk_head_dim (256 for GLM-5.2, 192 for DeepSeek-V3.2) while a + contiguous clone's is 64: the kernel then addresses the clone at + token*heads*64 + head*256, i.e. up to 192*(heads-1) elements PAST its end, and ropes + in place — a CUDA illegal memory access when the overrun leaves the allocation, + silent corruption of the neighbouring tensor otherwise. That was the GLM-5.2 fp8-KV + dsa_context (isl <= index_topk) illegal-address / hang family on b200, b300, gb200 + and gb300 (2026-10-14). FlashInfer's rope takes per-tensor strides (csrc/rope.cu + q_stride_h / k_stride_h @0.6.18) and is safe with either layout; matching the + query's strides is correct for both kernels. + """ + shape = tuple(query.shape) + stride = tuple(query.stride()) + if any(s == 0 for s in shape): + return query.clone() + storage_numel = 1 + sum((s - 1) * st for s, st in zip(shape, stride, strict=True)) + key = query.new_empty(storage_numel).as_strided(shape, stride) + key.copy_(query) + return key + + +def _rope_tolerating_missing_key(forward): + """Wrap a rotary module's forward so `forward(positions, query)` (key=None) works. + + vLLM 0.30's DeepseekV32 attention ropes q_pe alone in its dense-MHA FP8 branch, but + both DeepseekScalingRotaryEmbedding implementations require a key (FlashInfer op: + key.view; native: assert). Rope the query against a throwaway copy that shares the + query's strides (see _strided_throwaway_copy) and drop it. + """ + import functools + + if getattr(forward, "_aisim_rope_shim", False): + # Idempotent: run_mla_module installs the shim per case on a module that may be reused, + # and named_modules() can visit a shared rotary twice. Re-wrapping stacked one closure + # per case until "maximum recursion depth exceeded" (b200/gb200 pipelines 72387095 / + # 72387091 at e598d380: 2,286 cases each). + return forward + + @functools.wraps(forward) + def wrapped(positions, query, key=None, *args, **kwargs): + # Forward only what the caller passed: vLLM 0.30's forward_cuda signatures take + # (positions, query, key) — no `offsets` — so a positional None for offsets raised + # "takes from 3 to 4 positional arguments but 5 were given" (508 GLM-5 cases, same runs). + if key is None: + out = forward(positions, query, _strided_throwaway_copy(query), *args, **kwargs) + q = out[0] if isinstance(out, tuple) else out + return q, None + return forward(positions, query, key, *args, **kwargs) + + wrapped._aisim_rope_shim = True + return wrapped + + +def _rope_module_rejects_missing_key(module) -> bool: + """True only for rotary classes whose forward cannot take key=None. + + Serving calls ``self.rotary_emb(positions, q_pe)`` with no key + (models/deepseek_v32/attention.py:534 @0.30.0). vLLM's base RotaryEmbedding and every + subclass that reaches ops.rotary_embedding accept that (the kernel gets a null key + pointer and never touches key memory; _custom_ops.rotary_embedding key: Tensor | None), + so GLM-5.x (rope_type "default" -> RotaryEmbedding) must run exactly serving's call — + wrapping it fed vLLM's kernel a contiguous key whose head stride differs from the + query's (see _strided_throwaway_copy). Only DeepseekScalingRotaryEmbedding's own + forward_cuda rejects None (deepseek_scaling_rope.py:203-221 @0.30.0: FlashInfer op + key.view / forward_native assert key is not None); its DeepseekV4 subclass overrides + forward_cuda onto ops.rotary_embedding and accepts None again. + """ + mro = type(module).__mro__ + if not any(c.__name__ == "DeepseekScalingRotaryEmbedding" for c in mro): + return False + owner = next((c for c in mro if "forward_cuda" in vars(c)), None) + return owner is not None and owner.__name__ == "DeepseekScalingRotaryEmbedding" + + +def _install_rope_missing_key_shims(attn_module) -> tuple[list[str], list[str]]: + """Shim only the rotary submodules that reject key=None. + + Returns the class names that were wrapped and the rotary class names left on + serving's own path. + """ + patched: list[str] = [] + left: list[str] = [] + modules = attn_module.named_modules() if hasattr(attn_module, "named_modules") else () + for _name, module in modules: + if not (type(module).__name__.endswith("RotaryEmbedding") and hasattr(module, "forward")): + continue + if _rope_module_rejects_missing_key(module): + module.forward = _rope_tolerating_missing_key(module.forward) + patched.append(type(module).__name__) + else: + left.append(type(module).__name__) + return patched, left + + +# SM100/103: trtllm-gen sparse-MLA decode has no kernel below tileSizeQ 8 (see the +# FIXME in _build_module_test_cases). Hopper/Ada/SM120 route sparse decode elsewhere. +_DSA_SPARSE_DECODE_MIN_HEADS = 8 +_DSA_SPARSE_DECODE_MIN_HEADS_SMS = (100, 103) + + +def _model_max_position_embeddings(model_id: str) -> int | None: + """The model's max context length from its cached config (same file the + sglang module collector reads); None when unavailable (no cap applied).""" + config_file = os.path.join(_AIS_MODEL_CONFIG_DIR, f"{model_id.replace('/', '--')}_config.json") + if not os.path.exists(config_file): + return None + try: + with open(config_file) as f: + value = json.load(f).get("max_position_embeddings") + return int(value) if value else None + except Exception: + return None + + def _build_module_test_cases(attn_type: str, mode: str): """Build module-level test cases for a specific attention type and phase. - Output format: [seq_len, batch_size, num_heads, kv_cache_dtype, - compute_dtype, gemm_type, model_path, attn_type] + Output format: [[seq_len, ...], batch_size, num_heads, kv_cache_dtype, + compute_dtype, gemm_type, model_path, attn_type, (prefix_len)] + — one case per module series, see the grouping at the end. """ base_cases = get_context_test_cases(attn_type) if mode == "context" else get_generation_test_cases(attn_type) cases = [] + dropped_small_heads = 0 + dropped_over_ceiling = 0 + dropped_non_shard_heads = 0 + sm_version = get_sm_version() for model_spec in get_mla_module_model_specs(attention_type=attn_type, backend="vllm"): + max_position = _model_max_position_embeddings(model_spec.model_path) if attn_type == "dsa" else None + # Model-correlated narrowing (case_authoring.md; the MSA collectors' + # get_msa_head_counts precedent): a row that declares its real + # tensor_parallel_sizes only measures the per-rank head counts those + # shards produce. The shared inner head grid also carries 96/48/24/12 + # for the 96-head Kimi-K3 MLA module and 128 for 128-head models; on + # the 64-head GLM-5.x and the 128-head DeepSeek-V3.2 DSA rows those + # were 40-50% of the b200 dsa_context rows with no deployment behind + # them (2026-10-29). Rows without the declaration keep the full grid. + shard_heads = None + declared_tp_sizes = getattr(model_spec, "tensor_parallel_sizes", None) + if declared_tp_sizes: + shard_heads = {model_spec.native_num_heads // tp for tp in declared_tp_sizes} for base_case in base_cases: s, b, h, kv_dtype, compute_dtype, gemm_type, *rest = base_case + if shard_heads is not None and h not in shard_heads: + dropped_non_shard_heads += 1 + continue + small_head_hole = sm_version in _DSA_SPARSE_DECODE_MIN_HEADS_SMS and h < _DSA_SPARSE_DECODE_MIN_HEADS + if attn_type == "dsa" and small_head_hole: + # FIXME(kernel-limit): FlashInfer's trtllm-gen sparse-MLA decode ships no kernel for + # tileSizeQ < 8 (= per-rank heads at q_len 1): "Missing TRTLLM-GEN kernel (decode) ... + # tileSizeQ=4, sparseMlaType=1" (trtllm_fmha_kernel_launcher.cu:320, flashinfer 0.6.x). + # Heads 1/2/4 (TP 32/64/128 of a 128-head model) failed 68,435 of the b200 vllm 0.30.0 + # DSA cases AFTER loading a full module each (job 471043976); trtllm's fallback FMHA + # rejects the same cells ("Num. rows must be a multiple of 8"). Serving has no such + # deployment either. Dropped at plan time, counted here, not silently. + dropped_small_heads += 1 + continue + prefix_len = rest[0] if rest else 0 + if attn_type == "dsa" and max_position is not None and prefix_len + s > max_position: + # The shared dsa_prefix_lengths axis reaches 1,048,575 (GLM-5.2's ceiling); a model + # with a smaller max_position_embeddings (DeepSeek-V3.2 163,840, GLM-5 202,752) cannot + # build an engine past it: vLLM rejects max_model_len > derived limit (7,050 cases, + # job 471043976). sglang's module collector applies the same per-model cap + # (_prefix_fits_model); mirror it here. + dropped_over_ceiling += 1 + continue case = [s, b, h, kv_dtype, compute_dtype, gemm_type, model_spec.model_path, attn_type] if rest: case.append(rest[0]) cases.append(case) - return cases + if dropped_small_heads or dropped_over_ceiling or dropped_non_shard_heads: + print( + f"[vllm-mla-module-cases] {attn_type}/{mode}: dropped {dropped_small_heads} cases with per-rank heads " + f"< {_DSA_SPARSE_DECODE_MIN_HEADS} (SM{sm_version} sparse-MLA decode kernel domain), " + f"{dropped_over_ceiling} cases whose prefix+isl exceeds the model's max_position_embeddings and " + f"{dropped_non_shard_heads} cases whose per-rank heads no declared tensor_parallel_sizes shard produces" + ) + # One case per (batch, heads, kv, compute, gemm, model, attn_type) carrying its + # seq_len points in slot 0 and, for DSA context, its prefix points in the last slot: + # run_mla_module builds the module once per case and measures every (isl, prefix) + # point on it (largest total context first). Every point keeps its own KV cache, + # metadata and rows; only the module build is shared. The runner expands the + # isl x prefix grid through _dsa_context_point_allowed, the same predicate that + # built ``cases`` above, and the check below pins that the two agree. + if attn_type == "dsa" and mode == "context": + grid: dict[tuple, set[tuple[int, int]]] = {} + for s, b, h, kv_dtype, compute_dtype, gemm_type, model_path, attn, prefix_len in cases: + key = (b, h, kv_dtype, compute_dtype, gemm_type, model_path, attn) + grid.setdefault(key, set()).add((int(s), int(prefix_len))) + new_token_budget, full_seq_budget = _dsa_context_budgets() + grouped = [] + for key, points in grid.items(): + b, model_path = key[0], key[5] + seq_lens = sorted({s for s, _ in points}, reverse=True) + prefixes = sorted({p for _, p in points}, reverse=True) + max_position = _model_max_position_embeddings(model_path) + expanded = { + (s, p) + for s in seq_lens + for p in prefixes + if _dsa_context_point_allowed( + b, + s, + p, + new_token_budget=new_token_budget, + full_seq_budget=full_seq_budget, + max_position=max_position, + ) + } + if expanded != points: + raise RuntimeError( + f"vllm dsa context series {key}: the isl x prefix grid expands to {len(expanded)} points " + f"but the plan holds {len(points)}; _dsa_context_point_allowed and get_context_test_cases " + "disagree" + ) + grouped.append([seq_lens, *key, prefixes]) + return grouped + series: dict[tuple, list[int]] = {} + for s, *rest in cases: + series.setdefault(tuple(rest), []).append(int(s)) + return [[sorted(set(points), reverse=True), *key] for key, points in series.items()] def get_mla_context_module_test_cases(): @@ -357,12 +632,55 @@ def _create_gemm_quant_config(gemm_type: str): def _move_module_preserving_buffers(module, device): - """Materialize meta weights without discarding initialized runtime buffers.""" + """Materialize meta weights without discarding initialized runtime buffers. + + Two construction paths reach here. The explicit-class path builds the + module on the host, so only serialized block-scaled FP8 weights are meta + and every buffer already holds its constructor value. The serving-parity + path instantiates the framework's whole model under torch.device("meta") + (_create_attention_module), so its buffers are meta as well, and each is + rebuilt from its owner's own constructor recipe: + + - rotary cos/sin tables: `rotary_emb` and `indexer_rope_emb` on + DeepseekV32Attention (models/deepseek_v32/attention.py:272-283 + @ v0.30.0), registered non-persistent by rotary_embedding/base.py:60-63 + and deepseek_scaling_rope.py:241-242 from a device-free recipe + (_compute_cos_sin_cache: torch.arange / einsum on the default device), + cast to the dtype the constructor chose (the meta tensor carries it); + - KV-quant scales `_k_scale/_v_scale/_q_scale/_prob_scale` that + MLAAttention registers through _init_kv_cache_quant -> + set_default_quant_scales(register_buffer=True) as 1.0 fp32 + (layers/attention/attention.py:129-135,184 @ v0.30.0), together with + the `_cpu` host copies the same call keeps beside them. + + Any other meta buffer is refused rather than filled: a filled table + silently changes what the kernels see (the pre-#404 collector filled RoPE + with 0.01 and these scales with 0.5). + """ if not any(parameter.is_meta for parameter in module.parameters()): return module.to(device) - buffers = dict(module.named_buffers()) - if any(buffer.is_meta for buffer in buffers.values()): - raise RuntimeError("Cannot preserve an uninitialized meta buffer in the attention module") + buffers = {} + for name, buffer in module.named_buffers(): + if buffer.is_meta: + parent_name, _, local_name = name.rpartition(".") + owner = module.get_submodule(parent_name) if parent_name else module + recompute = getattr(owner, "_compute_cos_sin_cache", None) + if local_name in ("_k_scale", "_v_scale", "_q_scale", "_prob_scale"): + buffer = torch.ones(buffer.shape, dtype=buffer.dtype) + host_name = f"{local_name}_cpu" + if getattr(getattr(owner, host_name, None), "is_meta", False): + setattr(owner, host_name, torch.ones((), dtype=torch.float32)) + elif local_name.startswith("cos_sin_cache") and callable(recompute): + table = recompute() + if table.is_meta or tuple(table.shape) != tuple(buffer.shape): + raise RuntimeError( + f"Rotary table rebuilt for {name} is {tuple(table.shape)} on {table.device}; " + f"the constructor registered {tuple(buffer.shape)}" + ) + buffer = table.to(dtype=buffer.dtype) + else: + raise RuntimeError(f"Cannot preserve an uninitialized meta buffer in the attention module: {name}") + buffers[name] = buffer module = module.to_empty(device=torch.device(device)) for name, buffer in buffers.items(): parent_name, _, local_name = name.rpartition(".") @@ -412,25 +730,35 @@ def _initialize_nvfp4_parameters(module): untouched. vLLM subsequently performs its own layout conversion/dispatch. """ from vllm import _custom_ops as ops - from vllm.model_executor.layers.quantization.modelopt import ModelOptNvFp4LinearMethod - - # Native API contracts at vLLM v0.25.1, commit 752a3a504485790a: - # modelopt.py:1111-1199 stores two E2M1 values per uint8, one E4M3 scale - # per 16 inputs, and dequantizing FP32 global scales. _custom_ops.py:1492- - # 1571 accepts the inverse global scale and can return unswizzled scales; - # modelopt.py:1203-1238 performs native conversion only after loading. + from vllm.model_executor.layers.quantization.utils.quant_utils import kNvfp4Static + + # Native API contracts at vLLM v0.30.0: every ModelOpt linear is the + # generic ModelOptLinearMethod carrying a (weight, activation) QuantSpec + # and a CkptCtx (modelopt.py:2448-2476). The NVFP4 weight scheme + # KNvfp4Static stores two E2M1 values per uint8, one E4M3 scale per + # ctx.group_size inputs (created as a NaN sentinel that process() refuses) + # and a per-partition FP32 global scale `weight_scale_2` + # (modelopt.py:1913-1990); the W4A4 activation scheme KNvfp4Dynamic adds + # the per-partition `input_scale` (modelopt.py:1993-2028) while W4A16 has + # none. Both global scales are ModelOpt's dequantizing amax/(6*448) + # values, raw-max'ed at process_weights_after_loading. _custom_ops.py: + # 1525-1531 accepts the inverse global scale and can return unswizzled + # block scales. (v0.25.1's ModelOptNvFp4LinearMethod no longer exists.) generator = torch.Generator(device="cpu").manual_seed(0) initialized = 0 with torch.no_grad(): for name, layer in module.named_modules(): - if not isinstance(getattr(layer, "quant_method", None), ModelOptNvFp4LinearMethod): + quant_method = getattr(layer, "quant_method", None) + spec = getattr(quant_method, "spec", None) + if spec is None or getattr(spec, "weight", None) != kNvfp4Static: continue weight, scales = layer.weight, layer.weight_scale if weight.dtype != torch.uint8 or scales.dtype != torch.float8_e4m3fn: raise ValueError(f"Unexpected native NVFP4 storage for {name}") if weight.ndim != 2 or tuple(scales.shape) != (weight.shape[0], weight.shape[1] // 8): raise ValueError(f"Unexpected native NVFP4 block layout for {name}") - if layer.quant_method.quant_config.group_size != 16 or weight.shape[1] % 8: + group_size = getattr(getattr(quant_method, "ctx", None), "group_size", None) + if group_size != 16 or weight.shape[1] % 8: raise ValueError(f"Native NVFP4 quantization requires 16-input blocks for {name}") input_size = weight.shape[1] * 2 rows_per_chunk = max(1, 1048576 // input_size) @@ -439,7 +767,9 @@ def _initialize_nvfp4_parameters(module): # These fixed source ranges are unrelated to timing/accuracy targets. weight_global_scale = 0.08 / (6.0 * 448.0) layer.weight_scale_2.fill_(weight_global_scale) - layer.input_scale.fill_(1.0 / (6.0 * 448.0)) + input_scale = getattr(layer, "input_scale", None) # W4A4 only (KNvfp4Dynamic) + if input_scale is not None: + input_scale.fill_(1.0 / (6.0 * 448.0)) inverse_scale = torch.tensor(1.0 / weight_global_scale, dtype=torch.float32, device=weight.device) for start in range(0, weight.shape[0], rows_per_chunk): end = min(start + rows_per_chunk, weight.shape[0]) @@ -506,12 +836,16 @@ def _create_attention_module( local_model_path = _resolve_model_path(model_path) - block_size = 64 + # The module is built at the framework default page; the real page is asked from the + # backend the module selected (_create_kv_cache_and_metadata, utils.framework_kv_block_size). + block_size = None # unconstrained selection; _create_kv_cache_and_metadata asks the selected backend for the page # seq_len includes the current token; generation caches only seq_len - 1. # Keep exact-limit models such as Kimi-K2-Instruct at their declared limit. max_model_len = max(max_seq_len, 4096) num_kv_cache_blocks = max( - 1 + math.ceil((max_seq_len + 1) / block_size) * max_batch_size, + 1 + + math.ceil((max_seq_len + 1) / 16) + * max_batch_size, # upper bound at the smallest page; recomputed per page below 8192, ) @@ -625,27 +959,70 @@ def _create_attention_module( # (MLA backends only support bfloat16, not float32). from vllm.utils.torch_utils import set_default_torch_dtype + use_legacy = bool(os.environ.get("AIS_DSA_LEGACY_MODULE") or os.environ.get("AIC_DSA_LEGACY_MODULE")) with set_current_vllm_config(vllm_config), set_default_torch_dtype(vllm_config.model_config.dtype): - attn_module = DeepseekV2MLAAttention( - vllm_config=vllm_config, - config=hf_config, - hidden_size=hf_config.hidden_size, - num_heads=num_heads, - qk_nope_head_dim=hf_config.qk_nope_head_dim, - qk_rope_head_dim=hf_config.qk_rope_head_dim, - v_head_dim=hf_config.v_head_dim, - q_lora_rank=hf_config.q_lora_rank if hasattr(hf_config, "q_lora_rank") else None, - kv_lora_rank=hf_config.kv_lora_rank, - max_position_embeddings=hf_config.max_position_embeddings, - cache_config=vllm_config.cache_config, - quant_config=vllm_config.quant_config, - prefix="model.layers.0.self_attn", - topk_indices_buffer=topk_indices_buffer, - ) + if attn_type == "dsa" and not use_legacy: + # GENERIC serving-parity construction (owner rule 2026-09-19: + # config in -> the module SERVING would build comes out; never a + # per-model class import — that is how the prelude-fusion drift + # was born). Resolve the model class through the framework's own + # registry (exactly serving's routing, registry.resolve_model_cls), + # instantiate the WHOLE model on the meta device (free), and + # extract layer 0's attention submodule. Attention layers register + # themselves into static_forward_context during init, which is + # what _create_kv_cache_and_metadata reads — same as serving. + from vllm.model_executor.models.registry import ModelRegistry + + model_cls, _resolved = ModelRegistry.resolve_model_cls(hf_config.architectures, vllm_config.model_config) + with torch.device("meta"): + _meta_model = model_cls(vllm_config=vllm_config, prefix="") + attn_module = None + for _n, _m in _meta_model.named_modules(): + if _n.endswith("layers.0.self_attn"): + attn_module = _m + break + if attn_module is None: # no fallback: raise, never substitute + raise RuntimeError( + f"framework model {model_cls.__name__} exposes no " + "layers.0.self_attn — extend the extraction, do not pin a class" + ) + # the model allocated its shared topk buffer on meta; swap in the + # real one the collector allocated (same shape contract) + for _n, _m in attn_module.named_modules(): + if getattr(_m, "topk_indices_buffer", None) is not None: + _m.topk_indices_buffer = topk_indices_buffer + if getattr(attn_module, "topk_indices_buffer", None) is not None: + attn_module.topk_indices_buffer = topk_indices_buffer + else: + if attn_type == "dsa" and use_legacy: + print( + " [AB-ONLY] AIS_DSA_LEGACY_MODULE=1: building the LEGACY generic " + "DeepseekV2MLAAttention — NOT serving-parity on 0.29; rows are for " + "comparison only, never for the database" + ) + attn_module = DeepseekV2MLAAttention( + vllm_config=vllm_config, + config=hf_config, + hidden_size=hf_config.hidden_size, + num_heads=num_heads, + qk_nope_head_dim=hf_config.qk_nope_head_dim, + qk_rope_head_dim=hf_config.qk_rope_head_dim, + v_head_dim=hf_config.v_head_dim, + q_lora_rank=hf_config.q_lora_rank if hasattr(hf_config, "q_lora_rank") else None, + kv_lora_rank=hf_config.kv_lora_rank, + max_position_embeddings=hf_config.max_position_embeddings, + cache_config=vllm_config.cache_config, + quant_config=vllm_config.quant_config, + prefix="model.layers.0.self_attn", + topk_indices_buffer=topk_indices_buffer, + ) # Serialized block-scaled FP8 creates weight params on meta device; # to_empty() also discards already-initialized buffers. Preserve their - # constructor values, including nonpersistent RoPE tables, across it. + # constructor values, including nonpersistent RoPE tables, across it. On + # the serving-parity path above the whole module is meta, so the RoPE + # tables and KV-quant scales are rebuilt from their owners' constructor + # recipe instead of preserved. attn_module = _move_module_preserving_buffers(attn_module, device) attn_module.eval() attn_module.requires_grad_(False) @@ -753,6 +1130,13 @@ def _populate_indexer_kv_cache( # ═══════════════════════════════════════════════════════════════════════ +# Per-case phase timings. run_mla_module prints one "[... module timing]" line +# per case so a run log shows where the module collectors spend their time +# (framework setup per case vs. the timed forwards); _create_kv_cache_and_metadata +# records the indexer-cache fill here because its return tuple is a contract. +_PHASE_TIMINGS: dict[str, float] = {} + + def _create_kv_cache_and_metadata( vllm_config, attn_type: str, @@ -765,10 +1149,15 @@ def _create_kv_cache_and_metadata( indexer=None, ): """Create KV cache and attention metadata for benchmarking.""" + _PHASE_TIMINGS["indexer_fill"] = 0.0 hf_config = vllm_config.model_config.hf_text_config kv_lora_rank = hf_config.kv_lora_rank qk_rope_head_dim = hf_config.qk_rope_head_dim - block_size = vllm_config.cache_config.block_size + # page = what the framework would run the SELECTED backend with (serving re-derives it + # from the built model, platforms/interface.py update_block_size_for_backend) + _attn_layer = vllm_config.compilation_config.static_forward_context["model.layers.0.self_attn.attn"] + block_size = framework_kv_block_size(_attn_layer.get_attn_backend()) + vllm_config.cache_config.block_size = block_size is_dsa = attn_type == "dsa" if is_dsa and indexer is None: @@ -889,17 +1278,67 @@ def _create_kv_cache_and_metadata( device=device, ) indexer_builder_cls = indexer_layer.get_attn_backend().get_builder_cls() - indexer_builder = indexer_builder_cls(indexer_spec, [indexer_layer_name], vllm_config, torch.device(device)) + # 0.29.0: builders may demand block_table_width (indexer.py:519,531); + # mirror serving's population exactly (v1/worker/utils.py:276-283) + # using vllm's own width helper (v1/worker/block_table.py:29-49). + builder_kwargs = {} + if getattr(indexer_builder_cls, "requires_block_table_width", False): + from vllm.v1.worker.block_table import get_block_table_width + + max_num_blocks = indexer_spec.max_num_blocks_per_req(vllm_config, vllm_config.model_config.max_model_len) + builder_kwargs["block_table_width"] = get_block_table_width(max_num_blocks, indexer_spec.block_size) + # Serving allocates the block table AT block_table_width columns + # (v1/worker/block_table.py), so builders may assume + # table.shape[1] == width. The family-100 decode path acts on + # that assumption: it copies the metadata's table into + # expanded_block_table_buffer sized by the width + # (v1/attention/backends/mla/indexer.py:609,745@0.29.0) and + # crashes on our ceil(seq/block)-column table ("expanded size + # (64) must match existing size (32)"). Zero-pad to the width — + # padding blocks are never dereferenced for absent tokens. SM90 + # paths never read the expanded buffer, which is why this stayed + # latent on H20 (found on B300, 2026-09-20). + _width = builder_kwargs["block_table_width"] + _bt = common_attn_metadata.block_table_tensor + if _bt.shape[1] < _width: + _pad = torch.zeros( + (_bt.shape[0], _width - _bt.shape[1]), + dtype=_bt.dtype, + device=_bt.device, + ) + common_attn_metadata.block_table_tensor = torch.cat([_bt, _pad], dim=1) + indexer_builder = indexer_builder_cls( + indexer_spec, [indexer_layer_name], vllm_config, torch.device(device), **builder_kwargs + ) indexer_metadata = indexer_builder.build( common_prefix_len=prefix_len, common_attn_metadata=common_attn_metadata, ) + if os.environ.get("AIS_DEBUG_IDXMETA") or os.environ.get( + "AIC_DEBUG_IDXMETA" + ): # diagnostic dump, no behavior change + _pre = getattr(indexer_metadata, "prefill", None) + print( + f"[idxmeta] type={type(indexer_metadata).__name__} " + f"prefill={type(_pre).__name__ if _pre is not None else None} " + f"decode={getattr(indexer_metadata, 'decode', None) is not None}" + ) + if _pre is not None: + _ch = getattr(_pre, "chunks", None) + print(f"[idxmeta] chunks={None if _ch is None else len(_ch)}") + for _c in (_ch or [])[:1]: + for _f in dir(_c): + _v = getattr(_c, _f, None) + if not _f.startswith("_") and isinstance(_v, (int, bool, float)): + print(f"[idxmeta] chunk.{_f} = {_v}") + _fill_start = time.perf_counter() _populate_indexer_kv_cache( indexer_kv_cache=indexer_kv_cache, common_attn_metadata=common_attn_metadata, context_lens=[tensor.shape[0] for tensor in kv_c_contexts], indexer=indexer, ) + _PHASE_TIMINGS["indexer_fill"] = time.perf_counter() - _fill_start return kv_cache, attn_metadata, common_attn_metadata, indexer_kv_cache, indexer_metadata @@ -909,17 +1348,85 @@ def _create_kv_cache_and_metadata( # ═══════════════════════════════════════════════════════════════════════ +def _series_seq_lens(seq_len) -> list[int]: + """One point (int) or the series of points (list) a case measures on one module. + + Series points run largest first: the module and every per-point buffer are + sized by the largest point, so a capacity failure shows up on the first + point instead of after the smaller ones were measured. + """ + if isinstance(seq_len, bool): + raise TypeError("seq_len must be an int or a list of ints") + if isinstance(seq_len, int): + return [int(seq_len)] + seq_lens = sorted({int(value) for value in seq_len}, reverse=True) + if not seq_lens: + raise ValueError("a module series needs at least one seq_len") + return seq_lens + + +def _series_points( + seq_lens: list[int], + prefix_len, + *, + batch_size: int, + model_path: str, +) -> list[tuple[int, int]]: + """The (seq_len, prefix_len) points a case measures on one module, largest total + context first (a capacity failure shows up on the first point). + + ``prefix_len`` is one prefix (int: every seq_len behind it) or the case's prefix + list (the plan's isl x prefix grid, expanded through _dsa_context_point_allowed + exactly as the plan generator filtered it).""" + if isinstance(prefix_len, bool): + raise TypeError("prefix_len must be an int or a list of ints") + if isinstance(prefix_len, int): + points = [(s, int(prefix_len)) for s in seq_lens] + else: + prefixes = sorted({int(value) for value in prefix_len}, reverse=True) + if not prefixes: + raise ValueError("a module series needs at least one prefix_len") + new_token_budget, full_seq_budget = _dsa_context_budgets() + max_position = _model_max_position_embeddings(model_path) + points = [ + (s, p) + for s in seq_lens + for p in prefixes + if _dsa_context_point_allowed( + batch_size, + s, + p, + new_token_budget=new_token_budget, + full_seq_budget=full_seq_budget, + max_position=max_position, + ) + ] + if not points: + raise ValueError("the module series' isl x prefix grid has no point inside the plan budgets") + return sorted(points, key=lambda sp: (sp[0] + sp[1], sp[0]), reverse=True) + + +def _is_fatal_cuda_error(exc: BaseException) -> bool: + """A sticky CUDA failure: nothing measured on this context afterwards is trustworthy.""" + accelerator_error = getattr(torch, "AcceleratorError", None) + if accelerator_error is not None and isinstance(exc, accelerator_error): + return True + text = str(exc).lower() + markers = ("cuda error", "illegal memory access", "cudaerrorillegaladdress", "device-side assert") + return any(marker in text for marker in markers) + + @with_exit_stack def run_mla_module( exit_stack, - seq_len: int, + seq_len: int | list[int], batch_size: int, num_heads: int, kv_cache_dtype: str, compute_dtype: str, gemm_type: str, perf_filename: str, - prefix_len: int = 0, + prefix_len: int | list[int] = 0, *, model_path: str, attn_type: str, @@ -927,7 +1434,23 @@ def run_mla_module( warming_up: int = 10, test_ite: int = 6, ): - """Run a single MLA / DSA module-level benchmark point.""" + """Run one MLA / DSA module series: one module, one measured point per (``seq_len``, prefix). + + ``seq_len`` is the query length (context) or the KV length (generation) of a + point, or the list of them that share (batch, heads, dtypes, gemm, model); + ``prefix_len`` is one cached-prefix length or, for a DSA context series, the + case's prefix list (the points are the isl x prefix grid inside the plan + budgets, see _series_points). The module (vLLM config, synthetic/quantized + weights, rope shims) is built once for the largest point; the KV cache, the indexer cache and the + attention metadata are still built per point exactly as a single-point run + does, so every row keeps its serving-parity construction. A point that fails + for a non-fatal reason (OOM, a kernel limit) is recorded and the remaining + points are still measured; the failure is raised after the series so + collect.py classifies the case (the point's own exception when it is the + only one). A sticky CUDA failure aborts the series at once. + + Returns the full-indexer latencies of the measured points, largest first. + """ if attn_type not in {"mla", "dsa"}: raise ValueError(f"unsupported vLLM attention type: {attn_type!r}") if kv_cache_dtype not in {"bfloat16", "fp8"}: @@ -940,6 +1463,16 @@ def run_mla_module( "(determine_prefill_query_data_type, mla_attention.py:1462-1466 " f"@0.24.0); got kv_cache_dtype={kv_cache_dtype!r}" ) + seq_lens = _series_seq_lens(seq_len) + + _t_series = _t_last = time.perf_counter() + _setup_phase: dict[str, float] = {} + + def _lap(phases: dict[str, float], name: str) -> None: + nonlocal _t_last + now = time.perf_counter() + phases[name] = now - _t_last + _t_last = now setup_distributed(device) torch.cuda.set_device(device) @@ -957,35 +1490,112 @@ def run_mla_module( use_fp8_kv_cache = kv_cache_dtype == "fp8" use_prefill_fp8 = compute_dtype == "fp8" is_context = "context" in perf_filename - prefix_len = int(prefix_len) if is_context else 0 + points = _series_points(seq_lens, prefix_len if is_context else 0, batch_size=batch_size, model_path=model_path) + prefix_lens = sorted({p for _, p in points}, reverse=True) + prefix_desc = prefix_lens[0] if len(prefix_lens) == 1 else prefix_lens phase = "context" if is_context else "generation" variant = attn_type.upper() + max_seq_len = max(s for s, _ in points) + max_total_len = max(s + p for s, p in points) print( - f"\n[{variant} module] {phase} b={batch_size}, s={seq_len}, " - f"prefix={prefix_len}, heads={num_heads}, gemm={gemm_type}, " + f"\n[{variant} module] {phase} b={batch_size}, s={seq_lens if len(seq_lens) > 1 else seq_lens[0]}, " + f"prefix={prefix_desc}, heads={num_heads}, gemm={gemm_type}, " f"compute={compute_dtype}, kv={kv_cache_dtype}, model={model_path}" ) - # 1. Create attention module + # 1. Create attention module, sized by the largest point of the series. The + # sizes only reach buffers and capacity declarations (max_model_len, + # max_num_batched_tokens, the top-k index buffer); the backend selection + # is dtype/SM driven and every point's KV cache + metadata is built below + # for that point's own shape. attn_module, vllm_config = _create_attention_module( model_path=model_path, attn_type=attn_type, num_heads=num_heads, use_fp8_kv_cache=use_fp8_kv_cache, use_prefill_fp8=use_prefill_fp8, - max_seq_len=prefix_len + seq_len, + max_seq_len=max_total_len, max_batch_size=batch_size, - max_query_len=seq_len, + max_query_len=max_seq_len, gemm_type=gemm_type, device=device, is_context=is_context, ) + _lap(_setup_phase, "module") + if attn_type == "dsa" and use_fp8_kv_cache: + # vLLM 0.30 DeepseekV32 attention: the short-prefill dense-MHA branch with an FP8 query + # calls `self.rotary_emb(positions, q_pe)[0]` (models/deepseek_v32/attention.py:534) — + # key=None. Serving takes that branch on the SM100 family with an FP8 KV cache + # (_fp8_query = fp8 KV and FlashInferMLASparseImpl.supports_quant_query_input, + # attention.py:239 / flashinfer_mla_sparse.py:481; _use_sparse_mha for prompts <= + # index_topk). The yarn rope of DeepSeek-V3.2 (DeepseekScalingRotaryEmbedding) rejects + # key=None — FlashInfer op `key.view(...)` ("'NoneType' object has no attribute 'view'", + # 3.9k b200 + ~3k b300 + ~3k gb200 cases) / forward_native `assert key is not None` + # (deepseek_scaling_rope.py:123; the ed9109ec attempt to run native moved every case + # onto that assert: gb200 pipeline 72343556) — so for THAT class the call's intent, + # "rope q_pe only", is given to the module: rope the query against a throwaway copy + # and return (query, None). GLM-5.x's default rope (RotaryEmbedding) accepts key=None + # natively and is left on serving's exact call: wrapping it too handed vLLM's own rope + # kernel a contiguous key with the wrong head stride -> out-of-bounds in-place writes, + # the GLM-5.2 fp8-KV illegal-address / stall family (2026-10-14, see + # _strided_throwaway_copy). Kernels and labels unchanged either way. Drop the shim once + # vLLM's :534 call or DeepseekScalingRotaryEmbedding accepts key=None. + _patched, _left = _install_rope_missing_key_shims(attn_module) + if _patched: + print( + f"[vllm-mla-module] dsa fp8: rope shim on {sorted(set(_patched))} (reject key=None; " + f"vLLM 0.30 attention.py:534 ropes q_pe alone); on serving's own call: {sorted(set(_left))}" + ) # 1b. Process weights (FP8 quantization + create W_UK_T / W_UV for MLA) _process_module_weights(attn_module, vllm_config, device) + _lap(_setup_phase, "weights") - # 2. Create KV cache + metadata - with set_current_vllm_config(vllm_config): + # set_current_vllm_config — needed by quantised layers and RoPE for every + # forward of the series; entered once, after _cleanup was registered, so it + # unwinds before the workspace is released. + exit_stack.enter_context(set_current_vllm_config(vllm_config)) + + attn_layer_name = "model.layers.0.self_attn.attn" + indexer_layer_name = "model.layers.0.self_attn.indexer.k_cache" + forward_ctx = vllm_config.compilation_config.static_forward_context + hidden_size = vllm_config.model_config.hf_text_config.hidden_size + + # Index reuse (GLM-5.x index_topk_freq / index_topk_pattern): the full dry run below + # populates the shared top-k buffer; serving's reuse layers run with skip_topk=True, + # keeping the MLA projections and sparse attention while skipping the indexer + # (vllm/models/deepseek_v32/attention.py:169-177,226,297,324,406,460 @v0.30.0; the + # generic DeepseekV2 wrapper keeps the flag on mla_attn, layers/mla.py:109,147,232). + # Emit both op_name variants for checkpoints declaring reuse; never a model-averaged + # latency in a full-indexer row. + hf_text_config = vllm_config.model_config.hf_text_config + reuse_pattern = getattr(hf_text_config, "index_topk_pattern", None) + has_reuse = attn_type == "dsa" and ( + "S" in reuse_pattern if reuse_pattern is not None else (getattr(hf_text_config, "index_topk_freq", 1) or 1) > 1 + ) + # 0.30 DeepseekV32Attention IS the MLA layer and owns skip_topk; the legacy generic + # DeepseekV2MLAAttention nests it under mla_attn. + skip_owner = next( + (c for c in (attn_module, getattr(attn_module, "mla_attn", None)) if c is not None and hasattr(c, "skip_topk")), + None, + ) + if has_reuse: + if skip_owner is None: + raise RuntimeError("vLLM DSA reuse requires the native skip_topk execution path") + exit_stack.callback(setattr, skip_owner, "skip_topk", skip_owner.skip_topk) + + # Record architecture to distinguish different DSA models in the perf CSV. + # perf_database uses this as a dict key when loading data. + # Aligns with sdk/models.py which uses architectures[0] throughout. + hf_cfg = vllm_config.model_config.hf_config + architecture = getattr(hf_cfg, "architectures", [getattr(hf_cfg, "model_type", "unknown")])[0] + # legacy generic module nests the layer (module.mla_attn.mla_attn); + # DeepseekV32Attention IS the MLAAttention layer itself + mla_layer = attn_module if not hasattr(attn_module, "mla_attn") else attn_module.mla_attn.mla_attn + actual_kv_cache_dtype = "fp8" if mla_layer.kv_cache_dtype.startswith("fp8") else "bfloat16" + + def _run_point(point_stack, seq_len: int, prefix_len: int, point_phase: dict[str, float]) -> float: + # 2. Create KV cache + metadata for this point kv_cache, attn_metadata, common_attn_metadata, indexer_kv_cache, indexer_metadata = ( _create_kv_cache_and_metadata( vllm_config=vllm_config, @@ -999,226 +1609,256 @@ def run_mla_module( indexer=getattr(attn_module, "indexer", None), ) ) + _lap(point_phase, "kv_metadata") - # 2b. Bind KV cache to the 0.24.0 attention layer. - attn_layer_name = "model.layers.0.self_attn.attn" - forward_ctx = vllm_config.compilation_config.static_forward_context - forward_ctx[attn_layer_name].kv_cache = kv_cache + # 2b. Bind KV cache to the 0.24.0 attention layer. + forward_ctx[attn_layer_name].kv_cache = kv_cache - # For DSA, also bind the indexer's KV cache. - indexer_layer_name = "model.layers.0.self_attn.indexer.k_cache" - if indexer_kv_cache is not None and indexer_layer_name in forward_ctx: - forward_ctx[indexer_layer_name].kv_cache = indexer_kv_cache + # For DSA, also bind the indexer's KV cache. + if indexer_kv_cache is not None and indexer_layer_name in forward_ctx: + forward_ctx[indexer_layer_name].kv_cache = indexer_kv_cache - # 3. Input tensors - hidden_size = vllm_config.model_config.hf_text_config.hidden_size - if is_context: - num_tokens = seq_len * batch_size - positions = ( - torch.arange(prefix_len, prefix_len + seq_len, device=device, dtype=torch.long) - .unsqueeze(0) - .expand(batch_size, -1) - .reshape(-1) - .contiguous() - ) - else: - num_tokens = batch_size - positions = torch.full( - (batch_size,), - seq_len - 1, + # 3. Input tensors + if is_context: + num_tokens = seq_len * batch_size + positions = ( + torch.arange(prefix_len, prefix_len + seq_len, device=device, dtype=torch.long) + .unsqueeze(0) + .expand(batch_size, -1) + .reshape(-1) + .contiguous() + ) + else: + num_tokens = batch_size + positions = torch.full( + (batch_size,), + seq_len - 1, + device=device, + dtype=torch.long, + ) + + hidden_states = torch.full( + (num_tokens, hidden_size), + 0.01, + dtype=torch.bfloat16, device=device, - dtype=torch.long, ) - hidden_states = torch.full( - (num_tokens, hidden_size), - 0.01, - dtype=torch.bfloat16, - device=device, - ) - - # 4. Dry run - # set_current_vllm_config — needed by quantised layers and RoPE. - # set_forward_context — provides attn_metadata + kv_cache to the - # MLAAttention.forward() path (it calls get_forward_context()). - # FIXME(kernel-limit): on SM100, FlashInfer's trtllm-gen sparse-MLA decode - # ships kernels only for tileSizeQ >= 8. DSA cases whose per-rank head - # count is 1/2/4 (tileSizeQ = heads for q_len 1) raise "Missing TRTLLM-GEN - # kernel (decode)" (flashinfer csrc/trtllm_fmha_kernel_launcher.cu:272, - # flashinfer 0.6.12) whenever vLLM selects FLASHINFER_MLA_SPARSE — always - # for FP8 KV, and for BF16 KV at heads <= 16 (platforms/cuda.py:98-116 - # @0.24.0). Heads 8..128 pass; boundary measured on B200. Serving fails - # identically, so the affected cases stay observed runtime failures. - # Re-verify on the next vLLM/FlashInfer bump. - # FIXME(kernel-limit): on SM120 and SM89, vLLM's dense-MLA decode backend - # is TRITON_MLA (SM120: platforms/cuda.py:130-134; SM89: the else-branch - # priority list at cuda.py:135-142, where FLASH_ATTN_MLA/FLASHMLA/ - # FLASHINFER_MLA all reject major != 9/10 and TRITON_MLA is the only - # eligible backend @0.24.0). Two measured limits (RTX PRO 6000 - # Blackwell; serving fails identically; re-verify on the next vLLM - # bump): - # 1. FP8 KV + q-heads >= 2 routes to the grouped decode kernel, which - # requests 102400B shared memory vs SM120's 101376B limit -> - # OutOfResources (heads == 1 takes the non-grouped kernel and - # passes; bf16 KV passes). vLLM's overflow guard only drops - # num_stages at BLOCK_DMODEL >= 1024, which the MLA Lk=576 path - # (BLOCK_DMODEL=512) never reaches (triton_decode_attention.py - # :490-532 @0.24.0). Upstream fix in flight: vllm#46728 (open PR, - # num_stages=1 fallback for exactly this tile, fixes vllm#46721). - # 2. Decode batches whose total cached tokens exceed ~2^31/576 raise - # a deterministic illegal memory access (reproduced in isolation - # on a clean GPU): largest passing batch*seq = 2.10M tokens, - # smallest failing = 4.19M, bracketing 2^31 / 576 elements = 3.73M - # — consistent with int32 offset overflow in the Triton decode - # kernel's KV indexing. Affects both KV dtypes (fp8 only at - # heads == 1, where limit 1 does not fire first). Same family on - # SM89 (L40S): 33/400 sampled generation cases, smallest failing - # batch*seq again 4.19M tokens (bf16 KV; fp8-KV combos are not - # declared below SM90). - # FIXME(kernel-limit): on SM120, every DSA case fails: the CUDA sparse - # attention indexer hard-requires DeepGEMM (sparse_attn_indexer.py:468-472 - # @0.24.0), whose fp8 MQA-logits kernels ship for SM90/SM100 only - # (vllm/third_party/deep_gemm .../impls/sm{90,100}_fp8_mqa_logits.cuh) and - # assert "Unsupported architecture" (deepgemm csrc/apis/attention.hpp:184); - # cases with fp8_block linears die even earlier in DeepGEMM's scale-factor - # layout transform ("Unknown SF transformation", layout.hpp:59). Serving - # fails identically. Upstream: vllm#45317 (gap report), TRITON_MLA_SPARSE - # backend for SM8x/11x/12x in vllm#38476/#47629 (open PRs), sparse decode - # in vllm#47527; DeepGEMM SM120 support in DeepGEMM#318 (open PR). - # Re-verify on the next vLLM/DeepGEMM bump. - exit_stack.enter_context(set_current_vllm_config(vllm_config)) - attn_metadata_dict = {attn_layer_name: attn_metadata} - if indexer_metadata is not None: - attn_metadata_dict[indexer_layer_name] = indexer_metadata - # Native MLA cache insertion reads the per-layer forward-context mapping, - # separately from the attention metadata. Omitting it silently suppresses - # current-token writes (vLLM mla_attention.py:1037-1058 and - # attention.py:726-766 @ vLLM v0.25.1, commit 752a3a504485790a). - slot_mapping_dict = {attn_layer_name: common_attn_metadata.slot_mapping} - if indexer_metadata is not None: - slot_mapping_dict[indexer_layer_name] = common_attn_metadata.slot_mapping - exit_stack.enter_context(set_forward_context(attn_metadata_dict, vllm_config, slot_mapping=slot_mapping_dict)) - try: - with torch.inference_mode(): - attn_module.forward(positions, hidden_states, None) - except torch.cuda.OutOfMemoryError as e: - print(f" Dry run OOM: {e}") - # Let collect.py record the capacity failure. Returning normally would - # mark a task done even though it emitted no performance row. - raise - except Exception as e: - print(f" Dry run failed: {e}") - traceback.print_exc() - # Propagate to collect.py's worker so the failure is recorded in the - # error queue. Swallowing here lets a whole op complete with zero - # rows while the pipeline reports success. - raise - - # The full dry run above populates the shared top-k indices. Native vLLM - # uses skip_topk to reuse that buffer while retaining MLA projections and - # sparse attention (deepseek_v2.py:1080-1104,1150-1162 and layers/mla.py - # MultiHeadLatentAttentionWrapper @ vLLM v0.25.1, commit 752a3a504485790a). - # Emit both existing op_name variants for checkpoints declaring reuse; - # no model-averaged latency is written into a full-indexer row. - hf_text_config = vllm_config.model_config.hf_text_config - reuse_pattern = getattr(hf_text_config, "index_topk_pattern", None) - has_reuse = attn_type == "dsa" and ( - "S" in reuse_pattern if reuse_pattern is not None else (getattr(hf_text_config, "index_topk_freq", 1) or 1) > 1 - ) - if has_reuse: - if not hasattr(attn_module.mla_attn, "skip_topk"): - raise RuntimeError("vLLM DSA reuse requires the native skip_topk execution path") - exit_stack.callback(setattr, attn_module.mla_attn, "skip_topk", attn_module.mla_attn.skip_topk) - - def measure_and_log(skip_indexer): - if has_reuse: - attn_module.mla_attn.skip_topk = skip_indexer + # 4. Dry run + # set_forward_context — provides attn_metadata + kv_cache to the + # MLAAttention.forward() path (it calls get_forward_context()). + # FIXME(kernel-limit): on SM100, FlashInfer's trtllm-gen sparse-MLA decode + # ships kernels only for tileSizeQ >= 8. DSA cases whose per-rank head + # count is 1/2/4 (tileSizeQ = heads for q_len 1) raise "Missing TRTLLM-GEN + # kernel (decode)" (flashinfer csrc/trtllm_fmha_kernel_launcher.cu:272, + # flashinfer 0.6.12) whenever vLLM selects FLASHINFER_MLA_SPARSE — always + # for FP8 KV, and for BF16 KV at heads <= 16 (platforms/cuda.py:98-116 + # @0.24.0). Heads 8..128 pass; boundary measured on B200. Serving fails + # identically, so the affected cases stay observed runtime failures. + # Re-verify on the next vLLM/FlashInfer bump. + # FIXME(kernel-limit): on SM120 and SM89, vLLM's dense-MLA decode backend + # is TRITON_MLA (SM120: platforms/cuda.py:130-134; SM89: the else-branch + # priority list at cuda.py:135-142, where FLASH_ATTN_MLA/FLASHMLA/ + # FLASHINFER_MLA all reject major != 9/10 and TRITON_MLA is the only + # eligible backend @0.24.0). Two measured limits (RTX PRO 6000 + # Blackwell; serving fails identically; re-verify on the next vLLM + # bump): + # 1. FP8 KV + q-heads >= 2 routes to the grouped decode kernel, which + # requests 102400B shared memory vs SM120's 101376B limit -> + # OutOfResources (heads == 1 takes the non-grouped kernel and + # passes; bf16 KV passes). vLLM's overflow guard only drops + # num_stages at BLOCK_DMODEL >= 1024, which the MLA Lk=576 path + # (BLOCK_DMODEL=512) never reaches (triton_decode_attention.py + # :490-532 @0.24.0). Upstream fix in flight: vllm#46728 (open PR, + # num_stages=1 fallback for exactly this tile, fixes vllm#46721). + # 2. Decode batches whose total cached tokens exceed ~2^31/576 raise + # a deterministic illegal memory access (reproduced in isolation + # on a clean GPU): largest passing batch*seq = 2.10M tokens, + # smallest failing = 4.19M, bracketing 2^31 / 576 elements = 3.73M + # — consistent with int32 offset overflow in the Triton decode + # kernel's KV indexing. Affects both KV dtypes (fp8 only at + # heads == 1, where limit 1 does not fire first). Same family on + # SM89 (L40S): 33/400 sampled generation cases, smallest failing + # batch*seq again 4.19M tokens (bf16 KV; fp8-KV combos are not + # declared below SM90). + # FIXME(kernel-limit): on SM120, every DSA case fails: the CUDA sparse + # attention indexer hard-requires DeepGEMM (sparse_attn_indexer.py:468-472 + # @0.24.0), whose fp8 MQA-logits kernels ship for SM90/SM100 only + # (vllm/third_party/deep_gemm .../impls/sm{90,100}_fp8_mqa_logits.cuh) and + # assert "Unsupported architecture" (deepgemm csrc/apis/attention.hpp:184); + # cases with fp8_block linears die even earlier in DeepGEMM's scale-factor + # layout transform ("Unknown SF transformation", layout.hpp:59). Serving + # fails identically. Upstream: vllm#45317 (gap report), TRITON_MLA_SPARSE + # backend for SM8x/11x/12x in vllm#38476/#47629 (open PRs), sparse decode + # in vllm#47527; DeepGEMM SM120 support in DeepGEMM#318 (open PR). + # Re-verify on the next vLLM/DeepGEMM bump. + attn_metadata_dict = {attn_layer_name: attn_metadata} + if indexer_metadata is not None: + attn_metadata_dict[indexer_layer_name] = indexer_metadata + # 0.29: DeepseekV32Attention reads per-layer slot mappings from + # forward_context.slot_mapping (attention.py:303-306@v0.29.0 — a dict + # keyed by layer name); mirror serving's population with the SAME slot + # tensor the metadata was built from. + _slot = getattr(common_attn_metadata, "slot_mapping", None) + _slot_map = dict.fromkeys(attn_metadata_dict, _slot) if _slot is not None else None + point_stack.enter_context(set_forward_context(attn_metadata_dict, vllm_config, slot_mapping=_slot_map)) + try: + with torch.inference_mode(): + _fwd_args = ( + (positions, hidden_states) + if type(attn_module).__name__ == "DeepseekV32Attention" + else (positions, hidden_states, None) + ) + attn_module.forward(*_fwd_args) + except torch.cuda.OutOfMemoryError as e: + print(f" Dry run OOM: {e}") + # Let collect.py record the capacity failure. Returning normally would + # mark a task done even though it emitted no performance row. + raise + except Exception as e: + print(f" Dry run failed: {e}") + traceback.print_exc() + # Propagate to collect.py's worker so the failure is recorded in the + # error queue. Swallowing here lets a whole op complete with zero + # rows while the pipeline reports success. + raise + + _lap(point_phase, "dry_run") # 5. Benchmark def kernel_func(): - attn_module.forward(positions, hidden_states, None) - - # DSA context previously bypassed graphs to avoid retained scratch pools. - # Keep graph timing for small as well as large cases: eager launch gaps can - # dominate small module latency. The helper resets each graph, and the - # ExitStack releases vLLM's workspace after its forward contexts unwind. - with benchmark_with_power( - device=torch.device(device), - kernel_func=kernel_func, - num_warmups=warming_up, - num_runs=test_ite, - repeat_n=1, - use_cuda_graph=True, - allow_graph_fail=False, - ) as results: - pass - - if results.get("used_cuda_graph") is not True: - raise RuntimeError("vLLM MLA/DSA module requires CUDA graph timing; refusing to publish eager timing") - latency = results["latency_ms"] - - # 6. Log results — schema aligned with TRT-LLM - if is_context: - isl = seq_len - step = prefix_len - else: - isl = 1 - # BatchSpec.seq_lens includes this forward's one new token. - # Both module loaders reconstruct total KV length as isl + step. - step = seq_len - 1 - - op_name = f"{attn_type}_{phase}_module" + ("_skip_indexer" if skip_indexer else "") - - # Record architecture to distinguish different DSA models in the perf CSV. - # perf_database uses this as a dict key when loading data. - # Aligns with sdk/models.py which uses architectures[0] throughout. - hf_cfg = vllm_config.model_config.hf_config - architecture = getattr(hf_cfg, "architectures", [getattr(hf_cfg, "model_type", "unknown")])[0] - mla_layer = attn_module.mla_attn.mla_attn - backend_name = _mla_backend_name(mla_layer, attn_type, is_context, attn_metadata) - actual_kv_cache_dtype = "fp8" if mla_layer.kv_cache_dtype.startswith("fp8") else "bfloat16" - - log_perf( - item_list=[ - { - "model": model_path, - "architecture": architecture, - "mla_dtype": "bfloat16" if compute_dtype == "bfloat16" else compute_dtype, - "kv_cache_dtype": actual_kv_cache_dtype, - "gemm_type": "bfloat16" if gemm_type == "bfloat16" else gemm_type, - "num_heads": num_heads, - "batch_size": batch_size, - "isl": isl, - "tp_size": 1, - "step": step, - "latency": f"{latency:.4f}", - } - ], - framework="VLLM", - version=vllm_version, - device_name=torch.cuda.get_device_name(device), - op_name=op_name, - kernel_source=backend_name, - perf_filename=perf_filename, - power_stats=results["power_stats"], - ) + attn_module.forward(*_fwd_args) + + def measure_and_log(skip_indexer): + if has_reuse: + skip_owner.skip_topk = skip_indexer + + # Graph timing for every MLA/DSA case. DSA context used to run eagerly to dodge a + # private-pool retention (~18 GiB flashmla-sparse scratch per capture, saturating + # at ~146 GiB per worker); that bought small cases a ~0.6 ms launch floor — 0.30 + # rows at isl <= 1024 measured 1.6-8x the graph-timed values of the same cells + # (b200/l40s vs #404's B200 tables, 2026-10-10). benchmark_with_power + # (explicit_graph_cleanup) resets each graph and the ExitStack releases vLLM's + # workspace after the forward contexts unwind. FIXME(re-verify): confirm on 0.30 + # with a small->large DSA context sweep that the retention is gone before a campaign. + with benchmark_with_power( + device=torch.device(device), + kernel_func=kernel_func, + num_warmups=warming_up, + # The point's dry run already executed this forward; three eager + # passes cover lazily initialised paths (eager_warmup was ~40% of a + # measurement's wall time at ten). + num_eager_warmups=3, + num_runs=test_ite, + repeat_n=1, + use_cuda_graph=True, + allow_graph_fail=False, + ) as results: + pass + _lap(point_phase, "measure_skip" if skip_indexer else "measure_full") + # Protocol phases of this measurement (eager_warmup, capture, graph_warmup, + # timed, cleanup) on the point's timing line, prefixed by the measurement. + _bench_prefix = "skip" if skip_indexer else "full" + for _phase_name, _phase_secs in (results.get("phase_seconds") or {}).items(): + point_phase[f"{_bench_prefix}_{_phase_name}"] = _phase_secs + + if results.get("used_cuda_graph") is not True: + raise RuntimeError("vLLM MLA/DSA module requires CUDA graph timing; refusing to publish eager timing") + latency = results["latency_ms"] + + # 6. Log results — schema aligned with TRT-LLM + if is_context: + isl = seq_len + step = prefix_len + else: + isl = 1 + # BatchSpec.seq_lens includes this forward's one new token. + # Both module loaders reconstruct total KV length as isl + step. + step = seq_len - 1 + + op_name = f"{attn_type}_{phase}_module" + ("_skip_indexer" if skip_indexer else "") + backend_name = _mla_backend_name(mla_layer, attn_type, is_context, attn_metadata) + + log_perf( + item_list=[ + { + "model": model_path, + "architecture": architecture, + "mla_dtype": "bfloat16" if compute_dtype == "bfloat16" else compute_dtype, + "kv_cache_dtype": actual_kv_cache_dtype, + "gemm_type": "bfloat16" if gemm_type == "bfloat16" else gemm_type, + "num_heads": num_heads, + "batch_size": batch_size, + "isl": isl, + "tp_size": 1, + "step": step, + "latency": f"{latency:.4f}", + } + ], + framework="VLLM", + version=vllm_version, + device_name=torch.cuda.get_device_name(device), + op_name=op_name, + kernel_source=backend_name, + perf_filename=perf_filename, + power_stats=results["power_stats"], + ) - print( - f" [{phase}] b={batch_size}, s={seq_len}, heads={num_heads}, " - f"prefix={prefix_len}, gemm={gemm_type}, compute={compute_dtype}, " - f"kv={kv_cache_dtype}, backend={backend_name}: {latency:.4f} ms" - ) + print( + f" [{phase}] b={batch_size}, s={seq_len}, heads={num_heads}, " + f"prefix={prefix_len}, gemm={gemm_type}, compute={compute_dtype}, " + f"kv={kv_cache_dtype}, backend={backend_name}: {latency:.4f} ms" + ) - return latency + return latency - full_latency = measure_and_log(False) - if has_reuse: - measure_and_log(True) - return full_latency + full_latency = measure_and_log(False) + if has_reuse: + measure_and_log(True) + return full_latency + + latencies: list[float] = [] + failures: list[tuple[tuple[int, int], BaseException]] = [] + for index, (point_seq_len, point_prefix) in enumerate(points): + _t_point = _t_last = time.perf_counter() + point_phase: dict[str, float] = dict(_setup_phase) if index == 0 else {} + try: + with ExitStack() as point_stack: + latencies.append(_run_point(point_stack, point_seq_len, point_prefix, point_phase)) + except Exception as exc: + if _is_fatal_cuda_error(exc) or len(points) == 1: + raise + failures.append(((point_seq_len, point_prefix), exc)) + print( + f" [{variant} module] {phase} b={batch_size}, s={point_seq_len}, prefix={point_prefix}: " + f"point failed ({type(exc).__name__}: {str(exc)[:200]}); the series continues" + ) + gc.collect() + torch.cuda.empty_cache() + continue + point_phase["indexer_fill"] = globals().get("_PHASE_TIMINGS", {}).get("indexer_fill", 0.0) + print( + f" [{variant} module timing] {phase} b={batch_size}, s={point_seq_len}, prefix={point_prefix}, " + f"heads={num_heads}, gemm={gemm_type}, kv={kv_cache_dtype}: " + + " ".join(f"{name}={seconds:.2f}s" for name, seconds in point_phase.items()) + + f" total={time.perf_counter() - _t_point:.2f}s" + ) + if len(points) > 1: + print( + f" [{variant} module series] {phase} b={batch_size}, prefix={prefix_desc}, heads={num_heads}, " + f"gemm={gemm_type}, kv={kv_cache_dtype}: {len(latencies)}/{len(points)} points in " + f"{time.perf_counter() - _t_series:.1f}s" + ) + if failures: + if len(failures) == 1: + raise failures[0][1] + summary = "; ".join(f"s={s} prefix={p}: {type(exc).__name__}: {str(exc)[:200]}" for (s, p), exc in failures) + raise RuntimeError(f"{len(failures)}/{len(points)} point(s) of the module series failed: {summary}") + return latencies def run_mla_module_worker( - seq_len: int, + seq_len: int | list[int], batch_size: int, num_heads: int, kv_cache_dtype: str, @@ -1226,12 +1866,34 @@ def run_mla_module_worker( gemm_type: str, model_path: str, attn_type: str, - prefix_len: int = 0, + prefix_len: int | list[int] = 0, *, perf_filename: str, device: str = "cuda:0", ): - """Worker-compatible positional wrapper used by collector/collect.py.""" + """Worker-compatible positional wrapper used by collector/collect.py. + + ``seq_len`` is one point (the CLI, single-point fills) or the case's list of + points measured on one module (the registry plan). + """ + # FIXME(kernel-limit): on SM100/103 with FP8 KV, vLLM selects FLASHINFER_MLA_SPARSE for DSA and + # FlashInfer's auto backend has nothing for 64 < per-rank heads < 128: trtllm-gen refuses the head + # range, cute-dsl refuses sparse_mla_top_k ("auto: no backend supports this configuration", 11,442 + # cases = every heads-96 fp8 cell, both phases, job 471043976). Serving fails identically (the same + # gap closes Kimi-K3 tp1 on sglang: finding sm100_probe_failures_recheck_2026_10_04 (i)). Raise + # before the module is built so the cell is a classified failure, not a 20 s load + crash. + if ( + attn_type == "dsa" + and "fp8" in str(kv_cache_dtype) + and get_sm_version() in (100, 103) # SM100 family (_DSA_SPARSE_DECODE_MIN_HEADS_SMS); inline so the + # worker stays self-contained for the CLI tests that load it alone + and 64 < num_heads < 128 + ): + raise RuntimeError( + "FIXME(kernel-limit): vLLM FLASHINFER_MLA_SPARSE on SM100/103 has no decode backend for " + f"64 < per-rank heads < 128 with FP8 KV (num_heads={num_heads}); trtllm-gen rejects the head " + "range and cute-dsl rejects sparse_mla_top_k (flashinfer mla/_core.py:4271); serving fails identically" + ) # Serving executes model forward under inference mode (vLLM dd10e03f9, # v1/worker/gpu_model_runner.py:4069-4070). Keep module initialization, # dry runs, graph warmup and timing in the SAME mode: FlashInfer 0.6.13 diff --git a/python/aisimulate/collector/vllm/collect_mla_module_027.py b/python/aisimulate/collector/vllm/collect_mla_module_027.py deleted file mode 100644 index f7b7919f50..0000000000 --- a/python/aisimulate/collector/vllm/collect_mla_module_027.py +++ /dev/null @@ -1,36 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""vLLM 0.27.0 lane for the MLA module collector. - -`collector.vllm.collect_mla_module` stays pinned to the manifest default -(0.24.0) because it also hosts the DSA ops whose family (`sparse_attention`) -collects on that runtime. The `mla` family moved to the 0.27.0 K3 serving -lane image; per the one-runtime-per-module invariant those ops resolve here. - -The bodies are thin re-dispatch wrappers around the base collector: the -0.27.0 audit (GB300 probe, 2026-08-17) confirmed -`DeepseekV2MLAAttention.__init__` params, the `glm_moe_dsa` config-registry -gap patch, `backend_supports_prefill_query_quantization`, and the prefill -selector surface are all unchanged from the 0.24.0 citations documented in -collect_mla_module.py, so the wrappers change neither WHICH kernel runs nor -HOW a case is timed. -""" - -# The historical filename also serves the source/runtime-qualified 0.25.0 -# MLA family override; each campaign manifest still selects one exact runtime. -__compat__ = "vllm>=0.25.0,<=0.27.0,!=0.25.1,!=0.26.0" - -from collector.vllm import collect_mla_module as _impl - - -def get_mla_context_module_test_cases(): - return _impl.get_mla_context_module_test_cases() - - -def get_mla_generation_module_test_cases(): - return _impl.get_mla_generation_module_test_cases() - - -def run_mla_module_worker(*args, **kwargs): - return _impl.run_mla_module_worker(*args, **kwargs) diff --git a/python/aisimulate/collector/vllm/collect_moe.py b/python/aisimulate/collector/vllm/collect_moe.py index 1e1483ff46..d058411701 100644 --- a/python/aisimulate/collector/vllm/collect_moe.py +++ b/python/aisimulate/collector/vllm/collect_moe.py @@ -72,9 +72,11 @@ # 1968047). Existing framework construction/dispatch is unchanged. Invalid # model/quant shapes continue to raise and remain recorded failures; this is # API compatibility, not a claim that every Cartesian shape is supported. -# 0.30.0 source audit (2026-09-30; tag v0.30.0 == ced6857afa0e vs v0.27.1), -# the baseline of the 0.31.0 audit below. 0.28.0, 0.29.0 and 0.30.0 itself -# are not admitted: no consumer reads their tables. +# 0.30.0 source audit (2026-09-30; tag v0.30.0 == ced6857afa0e vs v0.27.1): +# the collector default (framework_manifest.yaml; the 2026-10 campaign +# tables) and the baseline of the 0.31.0 audit below, which admits the +# GLM-5.3-Flash model-pinned run. 0.28.0/0.29.0 stay admitted from the +# 0.27.1 -> 0.30.0 upgrade path. # FusedMoEFactory (layer.py:88-131 @0.30.0, :99-140 @0.27.1): every kwarg # passed here is unchanged; 0.30.0 only adds defaulted params. swiglu_limit # exists at both (:108 / :119), so other models keep passing None. @@ -102,8 +104,9 @@ # TrtLlmNvFp4ExpertsMonolithic (trtllm_nvfp4_moe.py:486-508, clamp # :90-105,:175-181). Serving logs of both GLM checkpoints on this runtime # report "Using FLASHINFER_TRTLLM" for Fp8 and NvFp4 MoE. -# Known non-parity (unchanged from 0.27.1): no FlashInfer autotune pass -# before timing; the gate GEMM and shared expert are separate ops. +# Known non-parity: the gate GEMM and shared expert are separate ops. +# (FlashInfer lanes are timed on the autotuned tactic like serving; see +# helper.flashinfer_autotuned.) # 0.31.0 audit (2026-10-07, source-only: tag v0.31.0 == db9527a4 vs v0.30.0 # ced6857a; no releases in between). GLM-5.3-Flash now runs STOCK 0.31.0 # (no overlay). Construction surface unchanged: FusedMoEFactory @@ -160,7 +163,7 @@ # K3 SiTU: FusedMoEFactory exposes activation_situ_beta on stock 0.30.0 and # 0.31.0, so the probe below passes the betas natively there; the # situ-as-silu approximation applies only to older stock builds. -__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0,!=0.28.0,!=0.29.0,!=0.30.0" +__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0" import contextlib import json @@ -178,6 +181,7 @@ from collector.helper import ( balanced_logits, benchmark_with_power, + flashinfer_autotuned, get_sm_version, log_perf, power_law_logits_v3, @@ -632,7 +636,29 @@ def gemma4_routing_function(hidden_states, gating_output, topk, renormalize): # the Mxfp4Config path below. quant_config = CompressedTensorsConfig.from_config(checkpoint_qc) else: + # OCP "mxfp4" checkpoints of model_type gpt_oss are OVERRIDDEN by + # vLLM to the gpt_oss_mxfp4 method (GptOssMxfp4Config. + # override_quantization_method, quantization/mxfp4.py:119-136 + # @0.29.0). Its GptOssMxfp4MoEMethod selects through the gpt-oss + # priority list (fused_moe/oracle/mxfp4.py:584-602 @0.29.0) — TRITON + # matmul_ogs on SM90, as the gpt-oss-120b serving probe shows — + # while the generic Mxfp4Config resolves to Mxfp4MoEMethod -> MARLIN + # (path_diff moe_mxfp4 vs gpt-oss-120b was diverged on exactly this, + # 2026-09-23). Mirror the framework's own override here. A version + # without the class performs no override, so Mxfp4Config is that + # version's serving truth — this is dispatch, not a fallback. + model_cfg = _load_model_moe_config(model_name) quant_config = Mxfp4Config() + if ( + checkpoint_qc.get("quant_method") in ("mxfp4", "gpt_oss_mxfp4") + and model_cfg.get("model_type") == "gpt_oss" + ): + try: + from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4Config + except ImportError: # pre-override vLLM: no gpt_oss_mxfp4 method exists + GptOssMxfp4Config = None # noqa: N806 + if GptOssMxfp4Config is not None: + quant_config = GptOssMxfp4Config() elif moe_type == "w4a8_mxfp4_mxfp8": # Native DeepSeek-V4 (expert_dtype=fp4) serving path: vLLM overrides # the checkpoint's fp8 quant_method to DeepseekV4FP8Config @@ -955,8 +981,11 @@ def run_single_iteration(): forward_context.moe_layer_index = 0 moe_module(hidden_states, router_logits) + # FlashInfer-backed experts (trtllm-gen nvfp4/mxfp4, cutlass, cutedsl) are + # timed on the autotuned tactic like serving; see helper.flashinfer_autotuned. with ( set_forward_context({}, vllm_config), + flashinfer_autotuned(num_tokens, run_single_iteration, label=source), benchmark_with_power( device=device, kernel_func=run_single_iteration, diff --git a/python/aisimulate/collector/vllm/collect_moe_xpu.py b/python/aisimulate/collector/vllm/collect_moe_xpu.py index 59a859162d..7806c81c1d 100644 --- a/python/aisimulate/collector/vllm/collect_moe_xpu.py +++ b/python/aisimulate/collector/vllm/collect_moe_xpu.py @@ -293,7 +293,10 @@ def run_single_iteration(): ) def run_iterations(): - # Use benchmark_with_power context manager + # Use benchmark_with_power context manager. XPU graph capture follows + # vLLM's own VLLM_XPU_ENABLE_XPU_GRAPH (xpu_graph_measure_enabled), and + # a failed capture fails the case (allow_graph_fail stays at its False + # default) instead of silently timing eager. with benchmark_with_power( device=device, kernel_func=run_single_iteration, @@ -301,7 +304,6 @@ def run_iterations(): num_runs=num_runs, repeat_n=1, use_cuda_graph=xpu_graph_measure_enabled(), - allow_graph_fail=False, # graph mandatory; capture failure fails the case ) as results: pass diff --git a/python/aisimulate/collector/vllm/collect_msa_module.py b/python/aisimulate/collector/vllm/collect_msa_module.py index 7dff409487..f132d77ad4 100644 --- a/python/aisimulate/collector/vllm/collect_msa_module.py +++ b/python/aisimulate/collector/vllm/collect_msa_module.py @@ -3,14 +3,11 @@ # Portions adapted from vLLM dd10e03f95f94edbea1975c67ace3a35ec9a8a40. # Copyright contributors to the vLLM project. Modified for collector execution. -# MiniMax-M3 support landed in vLLM 0.24.0 (vllm/models/minimax_m3/ + -# MinimaxM3QKVParallelLinearWithIndexer + fused_minimax_m3_qknorm_rope_kv_insert); -# this collector follows the 0.24.0 APIs and is pinned to the manifest -# default runtime (vllm collectors pin exactly; see -# test_active_cuda_vllm_collectors_are_exactly_pinned_to_manifest_version). -# B200 0.25.0 validation: job 1968755 passed 8 context and 6 generation -# cases after matching serving inference mode, positions, and THK top-k layout. -__compat__ = "vllm>=0.24.0,<=0.25.0" +# Versions are upgraded IN PLACE (owner policy 2026-09-20): this file always +# targets the manifest pin; prior-version code is `git log -- ` +# away and prior-version DATA stays permanent under its version key. The +# 0.24->0.29 adaptations each carry a serving citation @0.29.0. +__compat__ = "vllm>=0.30.0,<=0.30.0" """ MSA Module Collector for vLLM — MiniMax-M3 sparse-attention benchmarking. @@ -80,6 +77,7 @@ get_mla_module_model_specs, get_mla_module_precision_specs, get_mla_module_sweep_spec, + get_msa_head_counts, ) from collector.helper import _resolve_local_model_path, benchmark_with_power, get_sm_version, log_perf from collector.registry_types import PerfFile @@ -166,8 +164,9 @@ def get_context_test_cases(): """ cases = [] sweep = get_mla_module_sweep_spec("vllm") + head_counts = get_msa_head_counts("vllm") # native // tp of the declared shards, not the shared head grid for compute_dtype, kv_dtype, gemm_type in _get_precision_combos("context"): - for num_heads in sweep.inner_sweep_head_counts: + for num_heads in head_counts: for b in sweep.context_batch_sizes: for s in sweep.context_sequence_lengths: if b * s > sweep.context_max_tokens: @@ -199,8 +198,9 @@ def get_generation_test_cases(): budget = None if total_memory is None else int(total_memory * _MEMORY_BUDGET_SAFETY_FACTOR) considered = 0 dropped = 0 + head_counts = get_msa_head_counts("vllm") for compute_dtype, kv_dtype, gemm_type in _get_precision_combos("generation"): - for num_heads in sweep.inner_sweep_head_counts: + for num_heads in head_counts: for b in sweep.generation_batch_sizes: for s in sweep.generation_sequence_lengths: if b * s > sweep.generation_max_tokens: @@ -302,6 +302,30 @@ def _create_gemm_quant_config(gemm_type: str): _INDEX_CACHE_LAYER_NAME = "model.layers.0.self_attn.attn.index_cache" +def msa_decode_scorer_kernel_limit( + num_idx_heads: int, num_heads: int, *, is_context: bool, query_len: int | None +) -> str | None: + """Classified kernel-limit message when the M3 DECODE indexer scorer + cannot tile this index-head count, else None. + + The scorer tiles BLOCK_SIZE_HQ = num_idx_heads * BLOCK_SIZE_Q through + tl.arange (power of 2 required). It runs for every generation cell and + for a one-token context query only; prefill queries take the other + kernel, which the 0.29 full-grid sweep collected at 48/96 heads + (904/976 rows per head, the 72 misses being exactly the isl=1 cells). + """ + decode_scorer_runs = (not is_context) or (query_len is not None and query_len <= 1) + if decode_scorer_runs and num_idx_heads & (num_idx_heads - 1): + return ( + f"kernel-limit: num_idx_heads={num_idx_heads} (from num_heads={num_heads}) " + "is not a power of 2; MiniMax-M3 sparse decode indexer Triton kernel requires " + "power-of-2 tl.arange tiles (models/minimax_m3/common/ops/" + "sparse_attn.py@0.29.0) and serving tp shards can never produce " + "this count" + ) + return None + + def _create_msa_attention_module( model_path: str, num_heads: int, @@ -311,10 +335,15 @@ def _create_msa_attention_module( max_batch_size: int, is_context: bool, device: str = "cuda:0", + query_len: int | None = None, ): """Create a ``MiniMaxM3SparseAttention`` module from vLLM's own modeling code. + ``query_len`` is the per-request query length of the case (context + ``seq_len``; ``1`` for generation) — it decides whether the decode-path + kernel-limit guard below applies. + Loads the real HF config from model_path, applies the TP-shard head emulation in-memory, and constructs the exact module serving builds for sparse layers (``MiniMaxM3DecoderLayer.__init__``, @@ -410,6 +439,28 @@ def _create_msa_attention_module( # (nvidia/model.py:769-781). sparse_cfg = dict(hf_config.sparse_attention_config) sparse_cfg["sparse_num_index_heads"] = num_kv_heads + + # KERNEL LIMIT GUARD (verified on the 0.29 full-grid sweep, 2026-09-21): + # the M3 DECODE indexer-score Triton kernel tiles BLOCK_SIZE_HQ = + # num_idx_heads * BLOCK_SIZE_Q through tl.arange, whose range must be a + # power of 2 (models/minimax_m3/common/ops/sparse_attn.py — the sweep's + # heads 48/96 shard to idx_heads 3/6 and fail compilation + # deterministically). Serving cannot reach these points either: real tp + # in {1,2,4,8} on the native 4 index heads yields idx_heads in {4,2,1}, + # always a power of 2. Raise the classified error up front instead of + # burning a Triton compile per case. + # SCOPE (0.30 full-grid sweep, 2026-09-28): the limit is the decode + # path's. On 0.29 the 48/96-head failures were every generation cell and + # ONLY the isl=1 context cells (the single-query-token context case runs + # the decode scorer): 736/736 generation, 72/976 context per head, while + # context isl >= 16 collected 904/904 rows per head. A guard on the head + # count alone dropped those 1808 prefill rows on 0.30 — so it now fires + # only where the decode scorer runs (generation, or a one-token query). + _limit = msa_decode_scorer_kernel_limit( + int(sparse_cfg["sparse_num_index_heads"]), num_heads, is_context=is_context, query_len=query_len + ) + if _limit: + raise ValueError(_limit) hf_config.sparse_attention_config = sparse_cfg # Reserved top-k indices buffer shared by the indexer and the attend — @@ -500,7 +551,9 @@ def _rebase_block_table_and_slots(common_attn_metadata, block_size: int): block_table += 1 query_start_loc = common_attn_metadata.query_start_loc_cpu - context_lens = common_attn_metadata.num_computed_tokens_cpu + # 0.30: property removed; seq_lens - query_lens on device + # (utils.py CommonAttentionMetadata.compute_num_computed_tokens@v0.30.0) + context_lens = common_attn_metadata.compute_num_computed_tokens().cpu() slot_mapping = common_attn_metadata.slot_mapping device = slot_mapping.device for i in range(common_attn_metadata.num_reqs): @@ -588,38 +641,44 @@ def _create_kv_caches_and_metadata( ) num_blocks = _rebase_block_table_and_slots(common_attn_metadata, block_size) - # Main paged K/V cache: shape from the layer's own backend - # ((num_blocks, 2, block_size, num_kv_heads, head_size) — MiniMaxM3 - # SparseBackend.get_kv_cache_shape, common/sparse_attention.py:90-98 - # @v0.24.0; NHD stride order is the natural layout, :100-114), dtype - # from the layer's KV-cache spec (nvidia/model.py:540-549@v0.24.0: - # spec.dtype = kv_cache_dtype_str_to_dtype(cache_dtype) — bf16 for - # "auto", uint8 storage for "fp8", utils/torch_utils.py:38,394-400). - # Serving allocates the same way: a raw per-layer buffer viewed with the - # spec dtype and backend shape (gpu_model_runner - # ._reshape_kv_cache_tensors:7081-7180@v0.24.0); the impl then - # reinterprets the uint8 storage as the platform fp8 (e4m3) at use - # (common/sparse_attention.py:298-306, view at :352). + # Main paged K/V cache. 0.29 removed AttentionBackend.get_kv_cache_shape; + # the shape contract moved into the generic spec pipeline: serving + # allocates a flat byte buffer and views each layer as the 4D logical + # [B, H, N, C] page, C in bytes (compute_layer_kv_cache_shape_bytes, + # v1/kv_cache_interface.py:253-269@v0.29.0; the per-layer element view is + # cache_logical.view(spec.dtype), create_kv_cache_views :371-376). For + # M3's main cache the spec is FullAttentionSpec (nvidia/model.py:556-565) + # so the element view is [num_blocks, num_kv_heads, block_size, + # 2*head_size] — merged K|V on the last dim: the impl splits it back with + # k_cache, v_cache = kv_cache.split(head_size, dim=-1) + # (nvidia/sparse_attention_msa.py:277) and the fused insert kernel reads + # the block size off dim 2 (kv_cache.size(2), nvidia/model.py:635). + # Dtype semantics are unchanged: spec.dtype is bf16 for "auto", uint8 + # storage for "fp8"; the impl reinterprets uint8 as e4m3 at use + # (nvidia/sparse_attention_msa.py:221-222). + from vllm.v1.kv_cache_interface import compute_layer_kv_cache_shape_bytes + backend_cls = attn_module.get_attn_backend() kv_cache_spec = attn_module.get_kv_cache_spec(vllm_config) kv_cache = torch.zeros( - backend_cls.get_kv_cache_shape(num_blocks, block_size, kv_cache_spec.num_kv_heads, kv_cache_spec.head_size), - dtype=kv_cache_spec.dtype, + compute_layer_kv_cache_shape_bytes(kv_cache_spec, num_blocks), + dtype=torch.uint8, device=torch_device, - ) + ).view(kv_cache_spec.dtype) - # Indexer key side-cache: (num_blocks, block_size, head_size), dtype from - # the index cache module (bf16 for the default indexer_kv_dtype "bf16", - # config/attention.py:55; MiniMaxM3IndexerBackend.get_kv_cache_shape, - # common/indexer.py:91-99@v0.24.0). + # Indexer key side-cache: MLAAttentionSpec, key-only — one head slot, one + # vector per token (common/indexer.py:144-151@v0.29.0), logical + # [num_blocks, 1, block_size, head_size] after the dtype view. Serving + # hands this 4D view to the layer's bind_kv_cache, which squeezes the + # H=1 dim (common/indexer.py:140-142); binding below mirrors that. index_cache_layer = attn_module.indexer.index_cache index_backend_cls = index_cache_layer.get_attn_backend() index_spec = index_cache_layer.get_kv_cache_spec(vllm_config) index_kv_cache = torch.zeros( - index_backend_cls.get_kv_cache_shape(num_blocks, block_size, 1, index_spec.head_size), - dtype=index_cache_layer.dtype, + compute_layer_kv_cache_shape_bytes(index_spec, num_blocks), + dtype=torch.uint8, device=torch_device, - ) + ).view(index_spec.dtype) main_builder = backend_cls.get_builder_cls()(kv_cache_spec, [_ATTN_LAYER_NAME], vllm_config, torch_device) attn_metadata = main_builder.build( @@ -692,6 +751,7 @@ def run_msa_module( max_batch_size=batch_size, is_context=is_context, device=device, + query_len=seq_len if is_context else 1, ) # 2. Create KV caches + metadata via the framework's builders. @@ -706,14 +766,16 @@ def run_msa_module( device=device, ) - # 2b. Bind the caches to the registered layers — the benchmark analog of - # serving's bind_kv_cache, which assigns each allocated per-layer tensor - # to forward_context[layer_name].kv_cache (v1/worker/utils.py:462-530 - # @v0.24.0; the model reads self.kv_cache / index_cache.kv_cache directly, - # nvidia/model.py:606-607). + # 2b. Bind the caches to the registered layers exactly as serving does: + # forward_context[layer_name].bind_kv_cache(kv_cache) + # (v1/worker/utils.py:622@v0.29.0). The main layer's bind is the + # AttentionLayerBase default (stores the view as-is, + # model_executor/layers/attention_layer_base.py:26-32); the indexer's + # override squeezes the H=1 head dim, [B,1,N,C] -> [B,N,C] + # (common/indexer.py:140-142). forward_ctx = vllm_config.compilation_config.static_forward_context - forward_ctx[_ATTN_LAYER_NAME].kv_cache = kv_cache - forward_ctx[_INDEX_CACHE_LAYER_NAME].kv_cache = index_kv_cache + forward_ctx[_ATTN_LAYER_NAME].bind_kv_cache(kv_cache) + forward_ctx[_INDEX_CACHE_LAYER_NAME].bind_kv_cache(index_kv_cache) # 2c. Prove the requested KV precision actually reached the framework # path — a benchmark that silently ran bf16 under an "fp8" label would be diff --git a/python/aisimulate/collector/vllm/registry.py b/python/aisimulate/collector/vllm/registry.py index 3e21c25c08..3967de160c 100644 --- a/python/aisimulate/collector/vllm/registry.py +++ b/python/aisimulate/collector/vllm/registry.py @@ -25,6 +25,9 @@ get_func="get_computescale_test_cases", run_func="run_computescale", perf_filename=PerfFile.COMPUTESCALE, + # one measurement, two tables: the static quant row goes to + # scale_matrix_perf; finalize binds both to this producer + extra_perf_filenames=(PerfFile.SCALE_MATRIX,), ), OpEntry( op="attention_context", @@ -70,14 +73,14 @@ ), OpEntry( op="mla_context_module", - module="collector.vllm.collect_mla_module_027", + module="collector.vllm.collect_mla_module", get_func="get_mla_context_module_test_cases", run_func="run_mla_module_worker", perf_filename=PerfFile.MLA_CONTEXT_MODULE, ), OpEntry( op="mla_generation_module", - module="collector.vllm.collect_mla_module_027", + module="collector.vllm.collect_mla_module", get_func="get_mla_generation_module_test_cases", run_func="run_mla_module_worker", perf_filename=PerfFile.MLA_GENERATION_MODULE, @@ -155,13 +158,6 @@ run_func="run_dsv4_sparse_kernel_worker", perf_filename=PerfFile.DSV4_PAGED_MQA_LOGITS_MODULE, ), - OpEntry( - op="dsv4_hca_attn_module", - module="collector.vllm.collect_dsv4_attn", - get_func="get_dsv4_hca_attn_test_cases", - run_func="run_dsv4_sparse_kernel_worker", - perf_filename=PerfFile.DSV4_HCA_ATTN_MODULE, - ), OpEntry( op="mhc_module", module="collector.vllm.collect_mhc_module", diff --git a/python/aisimulate/collector/vllm/utils.py b/python/aisimulate/collector/vllm/utils.py index 4e4885c1ad..96e2c18ff7 100644 --- a/python/aisimulate/collector/vllm/utils.py +++ b/python/aisimulate/collector/vllm/utils.py @@ -116,13 +116,17 @@ def create_common_attn_metadata( # Calculate max query length max_query_len = max(batch_spec.query_lens) + # Serving population site: vllm/v1/worker/gpu/attn_utils.py:456-475@v0.30.0 + # passes seq_lens + seq_lens_cpu_upper_bound (no private *_cpu fields — the + # 0.29 `_seq_lens_cpu` / `_num_computed_tokens_cpu` were removed in 0.30; + # computed-token counts derive from seq_lens and query_start_loc inside the + # dataclass). Version bump 2026-09-27 (opharness upgrade_op vllm 0.30.0). + del num_computed_tokens_cpu return CommonAttentionMetadata( query_start_loc=query_start_loc, query_start_loc_cpu=query_start_loc_cpu, seq_lens=seq_lens, seq_lens_cpu_upper_bound=seq_lens_cpu, - _seq_lens_cpu=seq_lens_cpu, - _num_computed_tokens_cpu=num_computed_tokens_cpu, num_reqs=batch_spec.batch_size, num_actual_tokens=num_tokens, max_query_len=max_query_len, @@ -164,7 +168,7 @@ def create_vllm_config( max_model_len: int = 1024, dtype: Union[ModelDType, torch.dtype] = "auto", num_gpu_blocks: int = 1000, - block_size: int = 16, + block_size: int | None = 16, max_num_seqs: int = 256, max_num_batched_tokens: int = 8192, enable_chunked_prefill: bool = True, @@ -195,6 +199,11 @@ def create_vllm_config( hf_overrides=hf_overrides or {}, ) + # block_size=None is NOT "16": pydantic's CacheConfig marks any explicit value as + # user_specified_block_size, and the selector (v1/attention/selector.py:131-135) then + # filters backends by it exactly as --block-size would. Serving without --block-size + # selects unconstrained and derives the page from the chosen backend afterwards + # (utils.framework_kv_block_size); pass None to get that behaviour. cache_config = CacheConfig( block_size=block_size, cache_dtype="fp8" if use_fp8_kv_cache else "auto", @@ -340,9 +349,13 @@ def create_and_prepopulate_kv_cache_mla( MLA KV cache tensor """ batch_size = len(kv_c_contexts) - seq_lens = common_attn_metadata.seq_lens_cpu + # 0.30 removed the seq_lens_cpu property; the CPU view serving carries is + # seq_lens_cpu_upper_bound (vllm/v1/worker/gpu/attn_utils.py:460@v0.30.0) + seq_lens = common_attn_metadata.seq_lens_cpu_upper_bound query_lens = common_attn_metadata.query_start_loc_cpu[1:] - common_attn_metadata.query_start_loc_cpu[:-1] - context_lens = common_attn_metadata.num_computed_tokens_cpu + # 0.30: property removed; seq_lens - query_lens on device + # (utils.py CommonAttentionMetadata.compute_num_computed_tokens@v0.30.0) + context_lens = common_attn_metadata.compute_num_computed_tokens().cpu() block_table = common_attn_metadata.block_table_tensor slot_mapping = common_attn_metadata.slot_mapping @@ -452,9 +465,13 @@ def create_kv_cache_and_block_mappings( Tuple of the empty KV cache and flattened history slot mapping """ batch_size = common_attn_metadata.num_reqs - seq_lens = common_attn_metadata.seq_lens_cpu + # 0.30 removed the seq_lens_cpu property; the CPU view serving carries is + # seq_lens_cpu_upper_bound (vllm/v1/worker/gpu/attn_utils.py:460@v0.30.0) + seq_lens = common_attn_metadata.seq_lens_cpu_upper_bound query_lens = common_attn_metadata.query_start_loc_cpu[1:] - common_attn_metadata.query_start_loc_cpu[:-1] - context_lens = common_attn_metadata.num_computed_tokens_cpu + # 0.30: property removed; seq_lens - query_lens on device + # (utils.py CommonAttentionMetadata.compute_num_computed_tokens@v0.30.0) + context_lens = common_attn_metadata.compute_num_computed_tokens().cpu() block_table = common_attn_metadata.block_table_tensor slot_mapping = common_attn_metadata.slot_mapping @@ -636,3 +653,57 @@ def wrapper(*args, **kwargs): return func(stack, *args, **kwargs) return wrapper + + +def framework_kv_block_size(backend_cls, *, user_block_size: int | None = None) -> int: + """The KV page the FRAMEWORK would run this backend with — asked, not tabled. + + Mirrors serving (vllm 0.30.0): the cache manager block size is + ``backend.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE)`` unless + the user passed --block-size (platforms/interface.py + update_block_size_for_backend, phase 1), and the kernel page is + ``select_common_block_size(cache_block_size, [backend])`` (v1/worker/utils.py: + the manager size when every backend accepts it, else the largest explicit + size that divides it). The per-SM table (kv_block_size) was the owner's + 2026-09-30 interim; the B200 re-run on 2026-10-03 showed it is not + equivalent: with mla=32 the bf16 DSA cell selected the TRT-LLM FMHA sparse + cubin while serving ran FA-cute (page 64) — the page is a per-(backend, kv + dtype, head shape) decision, so it must come from the backend the case + actually selected. AIS_KV_BLOCK_SIZE stays the single-variable A/B hook. + """ + import os + + override = os.environ.get("AIS_KV_BLOCK_SIZE") + if override: + return int(override) + from vllm.config.cache import CacheConfig + from vllm.v1.worker.utils import select_common_block_size + + cache_block = ( + user_block_size if user_block_size else backend_cls.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE) + ) + return int(select_common_block_size(int(cache_block), [backend_cls])) + + +def kv_block_size(sm_version: int, op: str = "attention") -> int: + """KV-cache page size the collector builds its fake engine with, per SM. + + Serving picks the page size per attention backend; SM90 backends (FA3, + FlashMLA, Triton) take 64 and the collectors used a 64 literal — which the + B200 campaign (2026-09-29) caught as a cross-arch defect: on SM100 vLLM + 0.30 serves dense attention through the flashinfer TRT-LLM FMHA at page + 16 (Llama/Qwen; Gemma-4 head-512 takes 64) and MLA decode at page 32, and + a 64-page collector cell ran a different cubin (attn_ctx P64 vs serving + P16; mla_gen_fp8 P64 vs P32). The A/B with page 16 aligned attn_ctx. + Owner decision 2026-09-30: branch by SM rather than query the framework + per case. SM120 keeps 64 (its 31 gates aligned with it). AIS_KV_BLOCK_SIZE + is the single-variable A/B hook and is never set in collection runs. + """ + import os + + override = os.environ.get("AIS_KV_BLOCK_SIZE") + if override: + return int(override) + if 100 <= sm_version < 120: + return 32 if op == "mla" else 16 + return 64 diff --git a/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py b/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py index f9d48fea64..e00d1ecc85 100644 --- a/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py +++ b/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py @@ -1210,29 +1210,27 @@ def run_forward_pass(): moe_ref.forward(hidden_states, actual_logits, do_finalize=not min_latency_mode) # ===================================================================== - # Benchmark with automatic power measurement and graph fallback - # (same pattern as collect_moe.py) + # Benchmark under CUDA-graph replay with automatic power measurement + # (same contract as collect_moe.py: a failed capture fails the case) # ===================================================================== num_warmups = 1 if distributed == "power_law" else 3 num_runs = 1 if distributed == "power_law" else 6 + # CUDA-graph timing is mandatory: a failed capture raises + # (benchmark_with_power default allow_graph_fail=False) and the case + # fails classified; the moe_ep row has no column that could record an + # eager-timed measurement. with benchmark_with_power( device=device, kernel_func=run_forward_pass, num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, - allow_graph_fail=True, ) as results: # Calculate per-iteration latency (accounting for internal iterations) latency = results["latency_ms"] / num_iter power_stats = results["power_stats"] - # Always print CUDA graph status for debugging - print(f"[DEBUG] used_cuda_graph={results['used_cuda_graph']}, latency_ms={results['latency_ms']:.4f}") - if not results["used_cuda_graph"]: - print(f"CUDA graph capture failed for {num_tokens} tokens, used eager execution fallback") - # Determine source name if min_latency_mode: source = f"wideep_compute_{moe_kernel}_min_latency" diff --git a/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml b/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml index 49ccea4374..6e1f8f3eb4 100644 --- a/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml +++ b/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml @@ -169,16 +169,25 @@ parameters: sglang: max-running-requests trtllm: max_batch_size +# Token budgets render as multiples of 64 on EVERY backend (one rule, here, +# never per formula): engines size per-cache-group slot-mapping rows and +# scheduler chunks by these budgets, and a non-idiomatic width misaligns +# every row after the first (vLLM 0.29 DSV4: 6012 crashes the cutedsl +# compress kernel at first forward, 6016 passes). Rounding is always UP — +# a lower budget can violate scheduler minimums. Owner decision 2026-09-24: +# align all budgets uniformly instead of case by case. - param_key: enable_chunked_prefill vllm: enable-chunked-prefill - sglang: + sglang: key: chunked-prefill-size - value: "max_num_tokens if max_num_tokens else -1" + value: "((max_num_tokens + 63) // 64) * 64 if max_num_tokens else -1" trtllm: enable_chunked_prefill - param_key: max_prefill_tokens vllm: null - sglang: max-prefill-tokens + sglang: + key: max-prefill-tokens + value: "((max_prefill_tokens + 63) // 64) * 64 if max_prefill_tokens is defined and max_prefill_tokens is not none else none" trtllm: null - param_key: enable_mixed_chunk @@ -187,9 +196,16 @@ parameters: trtllm: null - param_key: max_num_tokens - vllm: max-num-batched-tokens + # same 64-alignment rule as above (the vLLM 0.29 DSV4 slot-mapping A/B is + # where it was found: 6012 crashes, 6016 passes); trtllm's block rounding + # in trtllm.rule (tokens_per_block = 32) is compatible with it. + vllm: + key: max-num-batched-tokens + value: "((max_num_tokens + 63) // 64) * 64 if max_num_tokens is defined and max_num_tokens is not none else none" sglang: null - trtllm: max_num_tokens + trtllm: + key: max_num_tokens + value: "((max_num_tokens + 63) // 64) * 64 if max_num_tokens is defined and max_num_tokens is not none else none" - param_key: max_seq_len vllm: max-model-len @@ -261,6 +277,21 @@ parameters: sglang: null trtllm: sparse_attention_config.implementation +# MiniMax-M3 sparse-attention algorithm key (trtllm >= 1.3.0rc29 requires it to build +# the M3 KV-cache manager; rendered by the rc29 engine template; set by utils.msa_sparse_algorithm) +- param_key: msa_sparse_algorithm + vllm: null + sglang: null + trtllm: sparse_attention_config.algorithm + +# sglang >= 0.5.20 prefill CUDA-graph backend ({full,breakable,tc_piecewise,disabled}); +# mixed-chunk prefill requires `breakable` on models whose backend enables the prefill +# graph (MiniMax-M3 at 0.5.21: prefill_cuda_graph_runner.py:293). CLI only on sglang. +- param_key: cuda_graph_backend_prefill + vllm: null + sglang: cuda-graph-backend-prefill + trtllm: null + - param_key: disable_overlap_scheduler vllm: null sglang: disable-overlap-schedule diff --git a/python/aisimulate/src/aisimulate/generator/config/backend_templates/sglang/cli_args.0.5.20.j2 b/python/aisimulate/src/aisimulate/generator/config/backend_templates/sglang/cli_args.0.5.20.j2 index aa897c9285..f58afcb5b3 100644 --- a/python/aisimulate/src/aisimulate/generator/config/backend_templates/sglang/cli_args.0.5.20.j2 +++ b/python/aisimulate/src/aisimulate/generator/config/backend_templates/sglang/cli_args.0.5.20.j2 @@ -1,6 +1,14 @@ {# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 +SGLang >= 0.5.20: ServerArgs became a msgspec Struct whose argparse flags are +derived from field names, and the old spellings `--cuda-graph-bs` / +`--cuda-graph-max-bs` (aliases of the *-decode fields in 0.5.16-0.5.19) are +gone: 0.5.21 rejects `--cuda-graph-bs` as ambiguous (matches +--cuda-graph-bs-decode / --cuda-graph-bs-prefill). The unified param keys are +unchanged; only the rendered spelling differs from cli_args.0.5.11.j2. +Verified against lmsysorg/sglang:v0.5.21 `launch_server --help` (2026-10-01); +boundary from v0.5.20 server_args.py (Struct form, no alias). -#} {%- set args = [] -%} {%- if sglang['tensor-parallel-size'] is defined and sglang['tensor-parallel-size'] is not none -%}{%- set _ = args.append('--tensor-parallel-size "' ~ sglang['tensor-parallel-size'] ~ '"') -%}{%- endif -%} @@ -48,4 +56,5 @@ SPDX-License-Identifier: Apache-2.0 {%- if sglang['speculative-num-draft-tokens'] is defined and sglang['speculative-num-draft-tokens'] is not none -%}{%- set _ = args.append('--speculative-num-draft-tokens "' ~ sglang['speculative-num-draft-tokens'] ~ '"') -%}{%- endif -%} {%- if sglang['disable-overlap-schedule'] -%}{%- set _ = args.append('--disable-overlap-schedule') -%}{%- endif -%} {%- if sglang['load-balance-method'] is defined and sglang['load-balance-method'] is not none -%}{%- set _ = args.append('--load-balance-method "' ~ sglang['load-balance-method'] ~ '"') -%}{%- endif -%} +{%- if sglang['cuda-graph-backend-prefill'] is defined and sglang['cuda-graph-backend-prefill'] is not none -%}{%- set _ = args.append('--cuda-graph-backend-prefill "' ~ sglang['cuda-graph-backend-prefill'] ~ '"') -%}{%- endif -%} {{ args | join(' ') }} diff --git a/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 b/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 new file mode 100644 index 0000000000..16ab331702 --- /dev/null +++ b/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 @@ -0,0 +1,84 @@ +{# +SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +SPDX-License-Identifier: Apache-2.0 + +TRT-LLM >= 1.3.0rc29: copy of the rc23 template plus sparse_attention_config.algorithm +(MiniMax-M3 requires algorithm=minimax_m3 to build its KV-cache manager; op-probe harness +A/B 2026-10-01, facts/upgrade_rc29/ab_m3). Prior templates untouched. +-#} +backend: pytorch + +{% if is_moe is defined %} # is_moe from sdk +moe_expert_parallel_size: {{ moe_expert_parallel_size }} +moe_tensor_parallel_size: {{ moe_tensor_parallel_size }} + +moe_config: +{# moe_config.backend (facts / --generator-set params.agg.moe_config.backend) wins; the generic `moe_backend` param + (backend_config_mapping.yaml, the sglang --moe-runner-backend spelling) used to be silently dropped for trtllm — + it now feeds the same field, upper-cased to the TRT-LLM enum. #} + backend: {{ moe_config.backend | default((moe_backend | upper) if moe_backend is defined and moe_backend else 'CUTLASS') }} # MOE backend to use (CUTLASS, CUTEDSL, WIDEEP, TRTLLM, VANILLA, MARLIN, ...) + {% if moe_config.load_balancer is defined %} # Configuration for MoE load balancing + load_balancer: {{ moe_config.load_balancer }} # moe load balancer + {% endif %} +{% endif %} + +tensor_parallel_size: {{ tensor_parallel_size }} +pipeline_parallel_size: {{ pipeline_parallel_size }} +enable_attention_dp: {{ enable_attention_dp | default(false) }} +enable_chunked_prefill: {{ enable_chunked_prefill | default(false) }} + +{% set _max_seq_len = max_seq_len | default(none) %} + +max_batch_size: {{ max_batch_size }} +max_num_tokens: {{ max_num_tokens }} +{% if _max_seq_len is not none and _max_seq_len != '' %} +max_seq_len: {{ _max_seq_len }} +{% endif %} + +kv_cache_config: + {% if kv_cache_config.free_gpu_memory_fraction is defined %} + free_gpu_memory_fraction: {{ kv_cache_config.free_gpu_memory_fraction }} # Fraction of GPU memory usable for KV-cache + {% elif kv_cache_config.max_tokens is not defined %} + free_gpu_memory_fraction: 0.80 # Fraction of GPU memory usable for KV-cache + {% endif %} + {% if kv_cache_config.max_tokens is defined %} + max_tokens: {{ kv_cache_config.max_tokens }} + {% endif %} + dtype: {{ kv_cache_config.dtype | default('auto') }} + {% if kv_cache_config.tokens_per_block is defined %} + tokens_per_block: {{ kv_cache_config.tokens_per_block }} + {% endif %} + enable_block_reuse: {{ kv_cache_config.enable_block_reuse | default(false) }} + +{% if cache_transceiver_config.max_tokens_in_buffer is defined %} +cache_transceiver_config: + backend: {{ cache_transceiver_config.backend | default('DEFAULT') }} # The communication backend type to use for the cache transceiver ("DEFAULT","UCX","NIXL","MPI") + max_tokens_in_buffer: {{ cache_transceiver_config.max_tokens_in_buffer }} +{% endif %} + +cuda_graph_config: + enable_padding: {{ cuda_graph_config.enable_padding | default(false) }} # Pad CUDA graph + batch_sizes: [{% for s in cuda_graph_config.batch_sizes + | default([1,2,4,8,16,32,64,128,256]) -%} + {{- s }}{{ ',' if not loop.last else '' }} {%- endfor %}] + +disable_overlap_scheduler: {{ disable_overlap_scheduler | default(true) }} +print_iter_log: {{ print_iter_log | default(false) }} + +{% if speculative_config.decoding_type is defined %} # speculative decoding +speculative_config: + decoding_type: {{ speculative_config.decoding_type }} + {% if speculative_config.decoding_type == 'MTP' and speculative_config.num_nextn_predict_layers is defined %} + num_nextn_predict_layers: {{ speculative_config.num_nextn_predict_layers }} + {% endif %} +{% endif %} + +{% if msa_sparse_algorithm is defined or msa_sparse_implementation is defined %} # MiniMax-M3 MSA +sparse_attention_config: +{% if msa_sparse_algorithm is defined %} + algorithm: {{ msa_sparse_algorithm }} # required by rc29+ (modeling_minimaxm3.py:337) on every SM +{% endif %} +{% if msa_sparse_implementation is defined %} + implementation: {{ msa_sparse_implementation }} # SM100-family: run the fmha_sm100 path the perf data represents +{% endif %} +{% endif %} diff --git a/python/aisimulate/src/aisimulate/generator/facts/apply.py b/python/aisimulate/src/aisimulate/generator/facts/apply.py index 8818a148a5..8ff8b566b0 100644 --- a/python/aisimulate/src/aisimulate/generator/facts/apply.py +++ b/python/aisimulate/src/aisimulate/generator/facts/apply.py @@ -96,7 +96,48 @@ def apply_model_default_args( tokens.append(_stringify(value)) -def apply_moe_backend(context: dict[str, Any], hardware: dict[str, Any] | None, *, backend: str) -> None: +def quant_family_of(quantization: dict[str, Any] | None) -> str | None: + """Collapse an artifact's quantization block (config.json ``quantization_config`` merged with the + modelopt ``hf_quant_config``, see utils._bundled_quantization) onto the collector's quant families: + nvfp4 / mxfp4 / fp8_block / fp8 / int4_wo. None when the artifact declares no quantization.""" + if not quantization: + return None + algo = str(quantization.get("quant_algo") or "").upper() + method = str(quantization.get("quant_method") or "").lower() + if algo == "MIXED_PRECISION": + # modelopt per-layer table: the MoE experts' algo decides the MoE kernel family (nvidia/Qwen3.8-2.4T-A95B-NVFP4: + # 92 `*.mlp.experts` layers NVFP4, 506 linear-attention / dense layers FP8); fall back to the most common algo + layers = quantization.get("quantized_layers") or {} + algos = [str((v or {}).get("quant_algo") or "").upper() for k, v in layers.items() if isinstance(v, dict)] + experts = [ + str((v or {}).get("quant_algo") or "").upper() + for k, v in layers.items() + if isinstance(v, dict) and "experts" in str(k) + ] + pick = experts or algos + algo = max(set(pick), key=pick.count) if pick else "" + quantization = {**quantization, "quant_algo": algo} + if algo == "NVFP4" or method in ("modelopt_fp4", "nvfp4"): + return "nvfp4" + if algo == "MXFP4" or method == "mxfp4": + return "mxfp4" + if algo.startswith("FP8") or method in ("fp8", "modelopt_fp8", "modelopt"): + return "fp8_block" if quantization.get("weight_block_size") else "fp8" + if ( + method in ("awq", "gptq", "compressed-tensors") + and int(quantization.get("bits") or quantization.get("weight_bits") or 0) == 4 + ): + return "int4_wo" + return method or algo.lower() or None + + +def apply_moe_backend( + context: dict[str, Any], + hardware: dict[str, Any] | None, + *, + backend: str, + quantization: dict[str, Any] | None = None, +) -> None: """Apply the hardware-derived ``moe_backend`` selection (fill-if-absent, MoE-only). ``hardware`` is the resolved hardware-profile dict (``ResolvedFacts.hardware``). @@ -119,6 +160,12 @@ def apply_moe_backend(context: dict[str, Any], hardware: dict[str, Any] | None, by :func:`apply_model_default_args`), not this hardware fact. """ choice = ((hardware or {}).get("moe_backend") or {}).get(backend) + if isinstance(choice, dict): + # keyed by the artifact's quantization family: on SM89 trtllm 1.3.0rc29 serves NVFP4 MoE only + # through MARLIN (weight-only), which in turn rejects bf16/fp8 MoE layers, so one hardware-wide + # value cannot be right (opharness findings sm89_trtllm_rc29_probe_2026_10_04). Unknown family + # -> the profile's "default" entry, else nothing (the builder/template default applies). + choice = choice.get(quant_family_of(quantization) or "", choice.get("default")) if not choice: return # GUARD: hardware moe_backend applies to MoE deployments only. @@ -126,7 +173,13 @@ def apply_moe_backend(context: dict[str, Any], hardware: dict[str, Any] | None, return if backend == "trtllm": moe = context.setdefault("moe_config", {}) - if isinstance(moe, dict) and not moe.get("backend"): + # a user/recipe `moe_backend` param (the generic spelling the sglang branch below honours) counts as + # "already set" here too: until 2026-10-04 the hardware fill ignored it and the trtllm template never + # read it, so `--generator-set params.agg.moe_backend=...` was silently dropped on trtllm + user_set = context.get("moe_backend") or any( + context.get(f"{role}_moe_backend") for role in ("prefill", "decode", "agg", "encode") + ) + if isinstance(moe, dict) and not moe.get("backend") and not user_set: moe["backend"] = choice elif backend == "sglang": if not context.get("moe_backend"): diff --git a/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml b/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml index b193550d73..d22e7620e6 100644 --- a/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml +++ b/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml @@ -102,11 +102,19 @@ profiles: # src/aisimulate_core/systems/data/b200_sxm/moe/trtllm/ # /wideep_moe_perf.parquet data collected, # and cli/exps/deepseek_wideep_trtllm.yaml (gb200/b200 wideep configs). - trtllm: WIDEEP + # trtllm: WIDEEP (disabled, see note below) # SGLang: deepep_moe is the default wideep-compatible MoE runner on Blackwell. # Source: cli/main.py _sglang_moe_backend_override and # cli/exps/deepseek_disagg_sglang.yaml. - sglang: deepep_moe + # sglang: deepep_moe (disabled, see note below) + # 2026-10-04 (B200 probe, PR #228 sm100 round): NOT filled on Blackwell. + # `WIDEEP` is a TRT-LLM 1.0-era MoeConfig.backend literal; 1.3.0rc29 rejects it + # (pydantic literal_error) and sglang 0.5.21 rejects `--moe-runner-backend deepep_moe` + # (argparse invalid choice) — 156 trtllm + 150 sglang MoE renders failed to boot. + # Wide-EP is an EP-size/load-balancer deployment choice, not a per-layer MoE kernel + # backend; single-node MoE on sm100 boots and picks working kernels under the + # framework's AUTO (A/B: explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100). + # See benchmarks/evidence/collector/op-probe-2026-10/findings.yaml sm100_moe_backend_fact_rejected_2026_10_04. # --------------------------------------------------------------------------- # GB200 NVL72 — Blackwell sm100, arm64 Grace-Blackwell @@ -155,11 +163,19 @@ profiles: # (all gb200_wideep_* experiments), # src/aisimulate_core/systems/data/gb200/moe/trtllm/ # /wideep_moe_perf.parquet - trtllm: WIDEEP + # trtllm: WIDEEP (disabled, see note below) # SGLang: deepep_moe for the wideep DeepEP path on GB200 NVLink fabric. # Source: cli/main.py _sglang_moe_backend_override and # cli/exps/deepseek_disagg_sglang.yaml. - sglang: deepep_moe + # sglang: deepep_moe (disabled, see note below) + # 2026-10-04 (B200 probe, PR #228 sm100 round): NOT filled on Blackwell. + # `WIDEEP` is a TRT-LLM 1.0-era MoeConfig.backend literal; 1.3.0rc29 rejects it + # (pydantic literal_error) and sglang 0.5.21 rejects `--moe-runner-backend deepep_moe` + # (argparse invalid choice) — 156 trtllm + 150 sglang MoE renders failed to boot. + # Wide-EP is an EP-size/load-balancer deployment choice, not a per-layer MoE kernel + # backend; single-node MoE on sm100 boots and picks working kernels under the + # framework's AUTO (A/B: explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100). + # See benchmarks/evidence/collector/op-probe-2026-10/findings.yaml sm100_moe_backend_fact_rejected_2026_10_04. # --------------------------------------------------------------------------- # Intel Arc Pro B60 — XPU @@ -190,3 +206,76 @@ profiles: effect: NoSchedule moe_backend: trtllm: CUTLASS + + # --------------------------------------------------------------------------- + # RTX PRO Blackwell (PCIe, 8-GPU hosts) — sm120 + # Keyed by SM: the SDK carries several RTX PRO systems with the same hardware + # facts (rtx_pro_6000_server today; the RTX PRO 5000 campaign box will follow). + # Before this profile existed, resolve_facts() raised for every RTX render and + # the generator pipeline swallowed it, so NO model default (block-size, + # trust-remote-code, autotune flags) reached sm120 engine args + # (benchmarks/evidence/collector/op-probe-2026-10/findings.yaml: + # rtx_systems_have_no_generator_facts_hardware_profile, 2026-10-01). + # --------------------------------------------------------------------------- + sm120: + node_selector: + # GPU product label pattern NVIDIA-${name// /-} (same derivation as b200); + # device string observed in systems/data/rtx_pro_6000_server/*/*_perf.parquet + # `device`: "NVIDIA RTX PRO 6000 Blackwell Server Edition". + nvidia.com/gpu.product: NVIDIA-RTX-PRO-6000-Blackwell-Server-Edition + arch: x86_64 + # nccl_env: intentionally none. NCCL_CUMEM_ENABLE is unverified on sm120 and is + # not asserted. The sm120 campaign host routes cross-socket ring hops over + # NET/Socket by default (PCIe-only, two NUMA domains); the right interface + # setting is deployment-specific, not a hardware fact. + shared_memory: + # vLLM TP8 serving of GLM-5.2-NVFP4 on the 8x RTX PRO 5000 host ran with + # --shm-size 32g (2026-10-01, op-path vs forward-pass alignment run). + default: 32Gi + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + # moe_backend: intentionally unset. trtllm WIDEEP needs the sm100 wide-EP + # fabric (PCIe sm120 has none; the typed builder defaults to CUTLASS when + # unset) and sglang deepep_moe is a wide-EP selection. vLLM picks its MoE + # backend from MODEL defaults (observed on sm120: FLASHINFER_CUTLASS NvFp4). + + # --------------------------------------------------------------------------- + # L40S — Ada sm89 (SDK system `l40s`) + # Product-keyed like h100 / h200 / h20 (the SM-keyed sm120 profile exists because several + # RTX PRO products share one set of facts; Ada does not — an L4 host would carry a + # different node_selector, so it gets its own profile when a campaign needs it). + # Before this profile existed resolve_facts() raised "Unknown hardware profile l40s" for + # every l40s render and the generator pipeline swallowed it ("Fact resolution failed; + # continuing without facts."), so no MODEL default (block-size, trust-remote-code, + # autotune flags) reached the engine args — the failure first found on sm120 + # (findings: rtx_systems_have_no_generator_facts_hardware_profile; sm89 campaign 2026-10-04). + # --------------------------------------------------------------------------- + l40s: + node_selector: + # GPU product label pattern NVIDIA-${name// /-} (same derivation as b200); the SDK's + # l40s perf data carries device "NVIDIA L40S". (The 2026-10-04 opharness campaign box + # was an 8x L40 — same AD102 die, lower TDP — which the system id does not cover.) + nvidia.com/gpu.product: NVIDIA-L40S + arch: x86_64 + # nccl_env: intentionally none (PCIe-only host, no NVLink; nothing verified to assert). + shared_memory: + # Not measured: every opharness probe on the L40 box (tp1 and tp4) ran with --shm-size 16g; + # 32Gi mirrors the other PCIe 8-GPU profile (sm120) as a conservative deployment default. + default: 32Gi + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + # moe_backend: keyed by the artifact's quantization family (facts/apply.quant_family_of) because + # one hardware-wide value cannot be right on Ada: trtllm 1.3.0rc29 serves NVFP4 MoE only through + # MARLIN (weight-only; A/B on an L40: AUTO resolves to CUTLASS, which refuses NVFP4 below SM100, + # TRITON is SM90-only, DEEPGEMM needs SM100/103/107, VANILLA dies in CUDA-graph capture), while + # MARLIN rejects bf16 / fp8 MoE layers that CUTLASS serves. FP8-block and MXFP4 MoE have no + # implementation on SM89 at all, so no key is given for them (the template default CUTLASS + # fails exactly as every alternative does). No wide-EP fabric on PCIe Ada: no WIDEEP / + # deepep_moe. Evidence: opharness findings sm89_trtllm_rc29_probe_2026_10_04. + moe_backend: + trtllm: + nvfp4: MARLIN diff --git a/python/aisimulate/src/aisimulate/generator/facts/models.yaml b/python/aisimulate/src/aisimulate/generator/facts/models.yaml index c2a622724c..810f6e20fd 100644 --- a/python/aisimulate/src/aisimulate/generator/facts/models.yaml +++ b/python/aisimulate/src/aisimulate/generator/facts/models.yaml @@ -11,9 +11,11 @@ models: default_variant: pro default_precision: fp8 match: + # `DeepSeek-V4-*` (not `V4*`): DeepSeek-V4.1 is a different model with its + # own profile below; the broader glob used to hand it V4's defaults. model_paths: - - deepseek-ai/DeepSeek-V4* - - sgl-project/DeepSeek-V4* + - deepseek-ai/DeepSeek-V4-* + - sgl-project/DeepSeek-V4-* architectures: - DeepseekV4ForCausalLM model_types: @@ -113,3 +115,105 @@ models: job_name: model-download pvc_name: model-cache mount_path: /model-store + + minimax-m3: + id: minimax-m3 + match: + model_paths: + - MiniMaxAI/MiniMax-M3* + - nvidia/MiniMax-M3* + architectures: + - MiniMaxM3SparseForConditionalGeneration + - MiniMaxM3VLForConditionalGeneration + model_types: + - minimax_m3 + traits: + - moe + - sparse_attention + defaults: + # Provenance: op-probe harness on L40, vllm 0.30.0, 2026-10-05 + # (benchmarks/evidence/collector/op-probe-2026-10/findings.yaml + # minimax_m3_dummy_dense_head_2026_10_05). M3 is three dense-attention layers followed by sparse layers; vllm + # sizes the KV manager block to 16 by default, the dense layers' FlashAttention backend accepts MultipleOf(16) + # but MiniMaxM3SparseBackend / MiniMaxM3IndexerBackend accept exactly [128] (one sparse block per KV page, + # models/minimax_m3/common/sparse_attention.py), so select_common_block_size has no candidate and the engine + # dies at "No common block size for 16" (v1/worker/utils.py:386). 128 satisfies every layer's backend. An + # all-sparse dummy cut never showed this (its only backend forces 128), which is how the sm90 matrix passed. + - match: {backend: vllm} + roles: ["*"] + backend_args: + block-size: 128 + deepseek-v4.1: + id: deepseek-v4.1 + default_variant: flash + default_precision: fp8 + match: + model_paths: + - deepseek-ai/DeepSeek-V4.1* + - nvidia/DeepSeek-V4.1* + architectures: + - DeepseekV41ForCausalLM + model_types: + - deepseek_v41 + traits: + - moe + - mla + - mtp + - compressed_attention + - engram + requires: + capabilities: + - deepseek_v4_openai_parsing + - deepseek_v4_tokenization + defaults: + # Provenance: op-probe harness A/B on H20, vllm 0.30.0, 2026-10-01 + # (benchmarks/evidence/collector/op-probe-2026-10/findings.yaml + # dsv41_dummy_cut_and_vllm_block_size_2026_10_01). V4.1 inherits V4's + # trust-remote-code / autotune defaults but NOT block-size 256: its + # compressed layers (compress_ratio 1/2) run 64-token kernel blocks, and + # vllm 0.30.0 refuses to split a 256-token manager block 4-way under the + # BLHNC layout once a KV tensor is shared by several layers + # ("block stride != page", kv_cache_interface.py:373). 64 loads; it is the + # value vllm's own error message prescribes. + - match: {backend: vllm} + roles: ["*"] + backend_args: + trust-remote-code: true + block-size: 64 + no-enable-flashinfer-autotune: true + # Same sglang defaults as deepseek-v4 (V4.1 code lives in the same + # deepseek_v4 model file on sglang main; no release supports it yet). + - match: {backend: sglang} + roles: ["*"] + backend_args: + trust-remote-code: true + disable-flashinfer-autotune: true + precision_variants: + fp8: + weight_dtype: fp8 + activation_dtype: fp8 + expert_dtype: fp4 + kv_cache_dtype_default: fp8 + requires: + capabilities: + - fp8_kv_cache + - deepseek_v4_attention_indexer + - fp4_moe_runner + variants: + flash: + id: flash + match: + model_paths: + - deepseek-ai/DeepSeek-V4.1-Flash* + - nvidia/DeepSeek-V4.1-Flash* + default_precision: fp8 + artifacts: + include: + model_cache_pvc: + name: model-cache + storage: 400Gi + storageClassName: your-storage-class-name + model_download_job: + job_name: model-download + pvc_name: model-cache + mount_path: /model-store diff --git a/python/aisimulate/src/aisimulate/generator/facts/request_resolution.py b/python/aisimulate/src/aisimulate/generator/facts/request_resolution.py index dc7a023ec9..e568123478 100644 --- a/python/aisimulate/src/aisimulate/generator/facts/request_resolution.py +++ b/python/aisimulate/src/aisimulate/generator/facts/request_resolution.py @@ -42,6 +42,13 @@ "gb200": "gb200", "gb200_sxm": "gb200", "gb300": "gb200", + # RTX PRO Blackwell PCIe hosts share one SM-keyed profile; the RTX PRO 5000 + # system (SDK entry pending) maps here too once it exists. + "rtx_pro_6000_server": "sm120", + # Ada: product-keyed like the Hopper entries (the SDK has an l40s system and nothing else on + # sm89 yet; an l4 system would get its own profile). Explicit so the table stays the one place + # that says which systems have facts. + "l40s": "l40s", } # Sections of the params dict that may carry the SDK system id. diff --git a/python/aisimulate/src/aisimulate/generator/module_bridge.py b/python/aisimulate/src/aisimulate/generator/module_bridge.py index b68b494d5f..2a57f13112 100644 --- a/python/aisimulate/src/aisimulate/generator/module_bridge.py +++ b/python/aisimulate/src/aisimulate/generator/module_bridge.py @@ -28,6 +28,34 @@ def _msa_sparse_implementation(task_config) -> str | None: ) +def _msa_sparse_algorithm(task_config) -> str | None: + """Optimized-path wrapper over utils.msa_sparse_algorithm (shared with naive).""" + from .utils import msa_sparse_algorithm + + return msa_sparse_algorithm( + getattr(task_config, "primary_backend_name", None) or "", + task_config.primary_model_path, + ) + + +def _model_has_kda(model_path: str) -> bool: + from .utils import model_has_kda + + return model_has_kda(model_path) + + +def _vllm_dsa_kv_cache_dtype(task_config, gemm_quant_mode) -> str | None: + """Optimized-path wrapper over utils.vllm_dsa_kv_cache_dtype: NVFP4 DSA + checkpoints on vLLM must render an explicit fp8 kv-cache dtype.""" + from .utils import vllm_dsa_kv_cache_dtype + + return vllm_dsa_kv_cache_dtype( + getattr(task_config, "primary_backend_name", None) or "", + task_config.primary_model_path, + gemm_quant_mode, + ) + + def _deep_merge(target: dict, extra: dict | None) -> dict: """ Recursively merge the contents of the 'extra' dictionary into 'target', @@ -185,6 +213,11 @@ def _build_worker_params(prefix: str, extra_overrides: dict | None) -> tuple[dic worker_payload["memory"] = memory if quant.get("kvcache_quant_mode"): worker_payload["kv_cache_dtype"] = quant["kvcache_quant_mode"] + # NVFP4 DSA checkpoints pin the KV cache to FP8; vLLM's `auto` + # spelling of it is rejected by the sparse-MLA selector (utils docstring) + _dsa_kv = _vllm_dsa_kv_cache_dtype(task_config, quant.get("gemm_quant_mode")) + if _dsa_kv is not None: + worker_payload["kv_cache_dtype"] = _dsa_kv if encoder_dp is not None: worker_payload["enable_encoder_dp"] = encoder_dp if attention_backend is not None: @@ -270,6 +303,9 @@ def _build_worker_params(prefix: str, extra_overrides: dict | None) -> tuple[dic "nextn": task_config.nextn, "nextn_accepted": task_config.nextn_accepted if task_config.nextn else None, "msa_sparse_implementation": _msa_sparse_implementation(task_config), + "msa_sparse_algorithm": _msa_sparse_algorithm(task_config), + # KDA linear-attention layers: vllm.rule caps the decode batch (see there) + "has_kda": _model_has_kda(task_config.primary_model_path), } model_cfg = {k: v for k, v in model_cfg.items() if v is not None} model_cfg = _deep_merge(model_cfg, overrides.get("ModelConfig")) diff --git a/python/aisimulate/src/aisimulate/generator/naive.py b/python/aisimulate/src/aisimulate/generator/naive.py index 1ae8beb46c..edd2b8d5da 100644 --- a/python/aisimulate/src/aisimulate/generator/naive.py +++ b/python/aisimulate/src/aisimulate/generator/naive.py @@ -30,7 +30,7 @@ get_model_config_from_model_path, ) -from .utils import msa_sparse_implementation +from .utils import model_has_kda, msa_sparse_algorithm, msa_sparse_implementation, vllm_dsa_kv_cache_dtype logger = logging.getLogger(__name__) @@ -517,6 +517,12 @@ def build_naive_generator_params( "max_batch_size": max_batch_size, "gpus_per_worker": min_gpus, } + # NVFP4 DSA checkpoints on vLLM need an explicit fp8 kv-cache dtype (see + # utils.vllm_dsa_kv_cache_dtype); the naive path has no perf task, so the + # artifact fact comes from the bundled config + _kv_dtype = vllm_dsa_kv_cache_dtype(backend_name, model_name, model_config=model_config) + if _kv_dtype is not None: + worker_params["kv_cache_dtype"] = _kv_dtype name_prefix = _sanitize_rfc1123(model_name) @@ -548,6 +554,8 @@ def build_naive_generator_params( "is_moe": is_moe, "fits_in_memory": fits, "required_tp": required_tp, + # KDA linear-attention layers: vllm.rule caps the decode batch (see there) + "has_kda": model_has_kda(model_name, model_config), } # Shared MSA prescription (see utils.msa_sparse_implementation): the # naive entry point must emit the same MiniMax-M3/SM100-family @@ -557,6 +565,10 @@ def build_naive_generator_params( _msa_impl = msa_sparse_implementation(backend_name, model_name, system_name) if _msa_impl is not None: model_config["msa_sparse_implementation"] = _msa_impl + # TRT-LLM >= rc29 requires sparse_attention_config.algorithm for M3 (every SM) + _msa_algo = msa_sparse_algorithm(backend_name, model_name) + if _msa_algo is not None: + model_config["msa_sparse_algorithm"] = _msa_algo service = _deep_merge_dicts(service, _section_override(overrides, "ServiceConfig")) k8s = _deep_merge_dicts(k8s, _section_override(overrides, "K8sConfig")) diff --git a/python/aisimulate/src/aisimulate/generator/rendering/engine.py b/python/aisimulate/src/aisimulate/generator/rendering/engine.py index fd513efb5a..5f88847833 100644 --- a/python/aisimulate/src/aisimulate/generator/rendering/engine.py +++ b/python/aisimulate/src/aisimulate/generator/rendering/engine.py @@ -377,8 +377,16 @@ def render_backend_templates( # a model profile). No-op without resolved facts. if resolved_facts is not None and backend == "trtllm": from aisimulate.generator.facts.apply import apply_moe_backend - - apply_moe_backend(context, getattr(resolved_facts, "hardware", None), backend=backend) + from aisimulate.generator.utils import _bundled_quantization + + # the artifact's quantization decides the MoE kernel family on SMs where one value cannot + # serve every quant (hardware.yaml l40s); None for unquantized / unresolvable checkpoints + apply_moe_backend( + context, + getattr(resolved_facts, "hardware", None), + backend=backend, + quantization=_bundled_quantization(str(context.get("model_path") or "")), + ) # Find template files template_path = Path(templates_dir) diff --git a/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py b/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py index f7fc5e06e3..7e446f1c81 100644 --- a/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py +++ b/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py @@ -173,6 +173,35 @@ def _load_rule_path(base_dir: str, backend: str) -> Optional[str]: return p if os.path.exists(p) else None +_TOKEN_BUDGET_KEYS = ("max_num_tokens", "max_prefill_tokens") +_TOKEN_BUDGET_ALIGN = 64 + + +def _align_token_budgets(param_values: dict[str, Any]) -> None: + """Round every token budget UP to a multiple of 64, on every backend, in + every role — the ONE alignment rule (owner decision 2026-09-24). Engines + size per-cache-group slot-mapping rows and scheduler chunks by these + budgets and a non-idiomatic width misaligns every row after the first + (vLLM 0.29 DeepSeek-V4: 6012 crashes the cutedsl compress kernel, 6016 + passes). Applied here, after the rule formulas and before any rendering, + so the trtllm engine-yaml template — which the mapping transforms cannot + reach — gets it too; the mapping keeps the same rounding as a guard for + values injected past the rule engine. Never rounds down.""" + params = param_values.get("params") + if not isinstance(params, dict): + return + for role in params.values(): + if not isinstance(role, dict): + continue + for key in _TOKEN_BUDGET_KEYS: + v = role.get(key) + if isinstance(v, bool) or not isinstance(v, int) or v <= 0: + continue + aligned = ((v + _TOKEN_BUDGET_ALIGN - 1) // _TOKEN_BUDGET_ALIGN) * _TOKEN_BUDGET_ALIGN + if aligned != v: + role[key] = aligned + + def apply_rule_plugins( param_values: dict[str, Any], backend: str, @@ -182,6 +211,7 @@ def apply_rule_plugins( base = _resolve_rule_plugin_dir(rule_name, base_dir=dsl_dir) rule_path = _load_rule_path(base, backend) if not rule_path: + _align_token_budgets(param_values) return param_values with open(rule_path, encoding="utf-8") as f: content = f.read().splitlines() @@ -227,4 +257,5 @@ def apply_rule_plugins( _apply_line(assign, backend, param_values, default_scope) except Exception: logger.exception("rule apply failed at line %d", idx) + _align_token_budgets(param_values) return param_values diff --git a/python/aisimulate/src/aisimulate/generator/rule_plugin/trtllm.rule b/python/aisimulate/src/aisimulate/generator/rule_plugin/trtllm.rule index b752597546..bc0c0ef51e 100644 --- a/python/aisimulate/src/aisimulate/generator/rule_plugin/trtllm.rule +++ b/python/aisimulate/src/aisimulate/generator/rule_plugin/trtllm.rule @@ -72,3 +72,9 @@ when kv_cache_dtype in ("float16", "bfloat16"): # sm_version 100/103; byte-safe everywhere else. when (ModelConfig.msa_sparse_implementation or '') != '': agg_prefill_decode msa_sparse_implementation = ModelConfig.msa_sparse_implementation + +# TRT-LLM >= 1.3.0rc29 requires sparse_attention_config.algorithm='minimax_m3' for MiniMax-M3 on +# every SM (modeling_minimaxm3.py:337); utils.msa_sparse_algorithm sets it architecture-keyed, +# the rc29 engine template renders it (older templates ignore the variable). +when (ModelConfig.msa_sparse_algorithm or '') != '': + agg_prefill_decode msa_sparse_algorithm = ModelConfig.msa_sparse_algorithm diff --git a/python/aisimulate/src/aisimulate/generator/rule_plugin/vllm.rule b/python/aisimulate/src/aisimulate/generator/rule_plugin/vllm.rule index 81e690b19f..c9f024f545 100644 --- a/python/aisimulate/src/aisimulate/generator/rule_plugin/vllm.rule +++ b/python/aisimulate/src/aisimulate/generator/rule_plugin/vllm.rule @@ -7,6 +7,15 @@ when not (preserve_engine_limits or false): agg_decode max_batch_size = (512 if (max_batch_size or 0) < 512 else (max_batch_size * 2)) prefill max_batch_size = (max_batch_size if max_batch_size else 1) +# KDA (Kimi Delta Attention) models on vLLM: the 512-batch decode CUDA graph +# faults at replay right after capture (vLLM 0.29.0 / SM90, Kimi-K3: 64, 256, +# 384, 448, 480, 496 pass; 512 fails; graphs off at 512 passes — opharness +# findings followups_after_full_reprobe_2026_09_24 + k3 bisection). Owner +# decision 2026-09-25: cap the decode batch at 256 for KDA models. Applied +# BEFORE the token budgets below so max_num_tokens follows the capped batch. +when (ModelConfig.has_kda or false) and not (preserve_engine_limits or false): + agg_decode max_batch_size = (256 if (max_batch_size or 0) > 256 else max_batch_size) + # Enforce vLLM MoE parallelism: vLLM uses unified TP for both attention and MoE, # so tensor_parallel_size must account for moe_tp × moe_ep. # Without this, MoE models get tp=1 and the full model is replicated per GPU, causing OOM. @@ -23,6 +32,9 @@ agg_prefill_decode gpus_per_worker = (tensor_parallel_size or 1) * (pipeline_par agg_prefill_decode enable_expert_parallel = ((moe_expert_parallel_size or 1) > 1) when not (preserve_engine_limits or false): + # plain budgets; the 64-alignment every backend needs is applied ONCE in + # backend_config_mapping.yaml (max_num_tokens / max_prefill_tokens / + # chunked-prefill-size), not per formula prefill max_num_tokens = (SlaConfig.isl or 0) + 1500 decode max_num_tokens = max_batch_size agg max_num_tokens = (max_batch_size or 0) + (SlaConfig.isl or 0) + 1500 diff --git a/python/aisimulate/src/aisimulate/generator/utils.py b/python/aisimulate/src/aisimulate/generator/utils.py index 8c0d7f90e8..acbf86f4e5 100644 --- a/python/aisimulate/src/aisimulate/generator/utils.py +++ b/python/aisimulate/src/aisimulate/generator/utils.py @@ -173,3 +173,146 @@ def msa_sparse_implementation(backend_name: str, model_path: str, system_name: s if int(spec.get("gpu", {}).get("sm_version", -1)) in (100, 103): return "msa" return None + + +_MSA_ARCHITECTURES = ("MiniMaxM3ForCausalLM", "MiniMaxM3SparseForConditionalGeneration") + + +def msa_sparse_algorithm(backend_name: str, model_path: str) -> str | None: + """MiniMax-M3 x TRT-LLM: the ``sparse_attention_config.algorithm`` key. + + TRT-LLM >= 1.3.0rc29 refuses to build the M3 KV-cache manager without + ``sparse_attention_config.algorithm='minimax_m3'`` (modeling_minimaxm3.py:337 + ``_validate_sparse_attention_runtime_config`` @1.3.0rc29) on every SM — unlike + the ``implementation`` knob above, which is an SM100-family performance + prescription. Keyed on the checkpoint ARCHITECTURE; None for every other + model/backend so the field is dropped. Rendered by the >= rc29 engine + template only (older templates never emitted the key and rc23 did not read it). + """ + if backend_name != "trtllm": + return None + from aisimulate.sdk.utils import get_model_config_from_model_path + + try: + architecture = get_model_config_from_model_path(model_path).get("architecture") + except Exception: # FileNotFound / KeyError / ValueError / HuggingFaceDownloadError + return None + return "minimax_m3" if architecture in _MSA_ARCHITECTURES else None + + +def _model_architecture(model_path: str, model_config: dict | None = None) -> str | None: + """Architecture of a checkpoint config, None when unresolvable (a + user-local or unreachable checkpoint the SDK cannot load — the render + then carries no prescription, it never fails on this fact). A frozen + ``model_config`` (naive path) is used as-is: no resolution runs. + Indirection so tests can stub the SDK lookup.""" + if isinstance(model_config, dict): + return model_config.get("architecture") + from aisimulate.sdk.utils import get_model_config_from_model_path + + try: + return get_model_config_from_model_path(model_path).get("architecture") + except Exception: # FileNotFound / KeyError / ValueError / HuggingFaceDownloadError + return None + + +# DeepSeek sparse attention (DSA) architectures: the sparse-MLA selector is +# the one that rejects vLLM's auto-resolved kv dtype spelling (see below). +_DSA_ARCHITECTURES = ("GlmMoeDsaForCausalLM", "DeepseekV32ForCausalLM") + + +def _bundled_quantization(model_path: str, raw: dict | None = None) -> dict | None: + """The artifact's quantization facts as the SDK loader exposes them: + config.json ``quantization_config`` merged with the ``hf_quant_config`` + the loader attaches from the bundled ``_hf_quant_config.json`` (the + modelopt artifacts keep ``kv_cache_quant_algo`` ONLY there — their + config.json has no quantization block at all). None when the config is + unresolvable or carries neither. Indirection so tests can stub it.""" + if raw is None: + from aisimulate_core.sdk.utils import _load_model_config_from_model_path + + try: + raw = _load_model_config_from_model_path(model_path) + except Exception: # unresolvable checkpoint: no quantization fact + return None + merged: dict = {} + hfq = raw.get("hf_quant_config") + if isinstance(hfq, dict): + merged.update(hfq.get("quantization") if isinstance(hfq.get("quantization"), dict) else hfq) + if isinstance(raw.get("quantization_config"), dict): + merged.update(raw["quantization_config"]) + return merged or None + + +def _artifact_pins_fp8_kv(quantization: dict | None) -> bool: + """modelopt artifacts pin the KV cache either as ``kv_cache_quant_algo: + FP8`` (hf_quant_config.json) or as ``kv_cache_scheme: {num_bits: 8, + type: float}`` (config.json quantization block); either spelling counts.""" + if not quantization: + return False + if str(quantization.get("kv_cache_quant_algo") or "").upper() == "FP8": + return True + scheme = quantization.get("kv_cache_scheme") + if isinstance(scheme, str): + return scheme.upper() == "FP8" + scheme = scheme or {} + return scheme.get("num_bits") == 8 and str(scheme.get("type", "")).lower() == "float" + + +def vllm_dsa_kv_cache_dtype( + backend_name: str, model_path: str, gemm_quant_mode: Any = None, model_config: dict | None = None +) -> str | None: + """NVFP4 DSA checkpoints x vLLM: prescribe ``--kv-cache-dtype fp8``. + + The modelopt NVFP4 artifacts of the DSA models (GLM-5 / 5.1 / 5.2 / 5.3 + -NVFP4, DeepSeek-V3.2-NVFP4) pin the KV cache to FP8. vLLM 0.29.0 + resolves ``--kv-cache-dtype auto`` for them to the literal ``fp8_e4m3`` + and its sparse-MLA backend selector rejects that spelling + (FLASHMLA_SPARSE: "kv_cache_dtype not supported"); the same engine with + an explicit ``fp8`` loads FlashMLASparseImpl and serves. Serving-side + the KV IS fp8 either way (checkpoint scheme), so the explicit value only + says what ``auto`` means for these artifacts. + + Keyed on the checkpoint ARCHITECTURE (DSA) plus an artifact fact — never + a model-name pattern: the task's ``nvfp4`` GEMM quant mode on the + optimized path, or the bundled config's fp8 KV scheme on the naive + ``cli generate`` path (which has no perf task). Bundled configs without a + ``quantization_config`` (the Hub config of DeepSeek-V3.2-NVFP4 keeps its + quantization only in hf_quant_config.json) get no prescription until the + SDK bundles that fact — a known gap, recorded in the opharness findings. + Evidence: findings ``vllm_fp8kv`` (0.29 addendum), + ``glm53_onboarding_2026_09_24``; owner decision 2026-09-24. Returns None + everywhere else so the task's own kv mode stands. + """ + if backend_name != "vllm": + return None + if _model_architecture(model_path, model_config) not in _DSA_ARCHITECTURES: + return None + if str(gemm_quant_mode or "").lower() == "nvfp4": + return "fp8" + raw = model_config.get("raw_config") if isinstance(model_config, dict) else None + if _artifact_pins_fp8_kv(_bundled_quantization(model_path, raw if isinstance(raw, dict) else None)): + return "fp8" + return None + + +def model_has_kda(model_path: str, model_config: dict | None = None) -> bool: + """True when the checkpoint declares KDA (Kimi Delta Attention) linear-attention + layers: config.json (or text_config) linear_attn_config.kda_layers non-empty + (Kimi-K3, Kimi-Linear, GLM-5.3-Flash). Read from the SDK-loaded config so + bundled, local and Hub checkpoints resolve identically; any failure to load + is False (the caller renders the default, nothing is invented).""" + if isinstance(model_config, dict): # frozen config (naive path): no resolution + raw = model_config.get("raw_config") or {} + else: + try: + from aisimulate.sdk.utils import get_model_config_from_model_path + + raw = get_model_config_from_model_path(model_path).get("raw_config") or {} + except Exception: # unresolvable checkpoint: no fact, no override + return False + for cfg in (raw.get("text_config") or {}, raw): + la = cfg.get("linear_attn_config") if isinstance(cfg, dict) else None + if isinstance(la, dict) and la.get("kda_layers"): + return True + return False diff --git a/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json b/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json new file mode 100644 index 0000000000..e66d0dcb2d --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json @@ -0,0 +1,1033 @@ +{ + "architectures": [ + "Qwen3_5ForConditionalGeneration" + ], + "image_token_id": 248056, + "language_model_only": false, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": 248044, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 48, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 24, + "num_hidden_layers": 64, + "num_key_value_heads": 4, + "output_gate_type": "swish", + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.8.0.dev0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 5120, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053, + "quantization_config": { + "activation_scheme": "dynamic", + "fmt": "e4m3", + "quant_method": "fp8", + "modules_to_not_convert": [ + "model.visual.blocks.0.attn.proj", + "model.visual.blocks.0.attn.qkv", + "model.visual.blocks.0.mlp.linear_fc1", + "model.visual.blocks.0.mlp.linear_fc2", + "visual.blocks.0.attn.proj", + "visual.blocks.0.attn.qkv_proj", + "visual.blocks.0.mlp.linear_fc1", + "visual.blocks.0.mlp.linear_fc2", + "model.visual.blocks.1.attn.proj", + "model.visual.blocks.1.attn.qkv", + "model.visual.blocks.1.mlp.linear_fc1", + "model.visual.blocks.1.mlp.linear_fc2", + "visual.blocks.1.attn.proj", + "visual.blocks.1.attn.qkv_proj", + "visual.blocks.1.mlp.linear_fc1", + "visual.blocks.1.mlp.linear_fc2", + "model.visual.blocks.2.attn.proj", + "model.visual.blocks.2.attn.qkv", + "model.visual.blocks.2.mlp.linear_fc1", + "model.visual.blocks.2.mlp.linear_fc2", + "visual.blocks.2.attn.proj", + "visual.blocks.2.attn.qkv_proj", + "visual.blocks.2.mlp.linear_fc1", + "visual.blocks.2.mlp.linear_fc2", + "model.visual.blocks.3.attn.proj", + "model.visual.blocks.3.attn.qkv", + "model.visual.blocks.3.mlp.linear_fc1", + "model.visual.blocks.3.mlp.linear_fc2", + "visual.blocks.3.attn.proj", + "visual.blocks.3.attn.qkv_proj", + "visual.blocks.3.mlp.linear_fc1", + "visual.blocks.3.mlp.linear_fc2", + "model.visual.blocks.4.attn.proj", + "model.visual.blocks.4.attn.qkv", + "model.visual.blocks.4.mlp.linear_fc1", + "model.visual.blocks.4.mlp.linear_fc2", + "visual.blocks.4.attn.proj", + "visual.blocks.4.attn.qkv_proj", + "visual.blocks.4.mlp.linear_fc1", + "visual.blocks.4.mlp.linear_fc2", + "model.visual.blocks.5.attn.proj", + "model.visual.blocks.5.attn.qkv", + "model.visual.blocks.5.mlp.linear_fc1", + "model.visual.blocks.5.mlp.linear_fc2", + "visual.blocks.5.attn.proj", + "visual.blocks.5.attn.qkv_proj", + "visual.blocks.5.mlp.linear_fc1", + "visual.blocks.5.mlp.linear_fc2", + "model.visual.blocks.6.attn.proj", + "model.visual.blocks.6.attn.qkv", + "model.visual.blocks.6.mlp.linear_fc1", + "model.visual.blocks.6.mlp.linear_fc2", + "visual.blocks.6.attn.proj", + "visual.blocks.6.attn.qkv_proj", + "visual.blocks.6.mlp.linear_fc1", + "visual.blocks.6.mlp.linear_fc2", + "model.visual.blocks.7.attn.proj", + "model.visual.blocks.7.attn.qkv", + "model.visual.blocks.7.mlp.linear_fc1", + "model.visual.blocks.7.mlp.linear_fc2", + "visual.blocks.7.attn.proj", + "visual.blocks.7.attn.qkv_proj", + "visual.blocks.7.mlp.linear_fc1", + "visual.blocks.7.mlp.linear_fc2", + "model.visual.blocks.8.attn.proj", + "model.visual.blocks.8.attn.qkv", + "model.visual.blocks.8.mlp.linear_fc1", + "model.visual.blocks.8.mlp.linear_fc2", + "visual.blocks.8.attn.proj", + "visual.blocks.8.attn.qkv_proj", + "visual.blocks.8.mlp.linear_fc1", + "visual.blocks.8.mlp.linear_fc2", + "model.visual.blocks.9.attn.proj", + "model.visual.blocks.9.attn.qkv", + "model.visual.blocks.9.mlp.linear_fc1", + "model.visual.blocks.9.mlp.linear_fc2", + "visual.blocks.9.attn.proj", + "visual.blocks.9.attn.qkv_proj", + "visual.blocks.9.mlp.linear_fc1", + "visual.blocks.9.mlp.linear_fc2", + "model.visual.blocks.10.attn.proj", + "model.visual.blocks.10.attn.qkv", + "model.visual.blocks.10.mlp.linear_fc1", + "model.visual.blocks.10.mlp.linear_fc2", + "visual.blocks.10.attn.proj", + "visual.blocks.10.attn.qkv_proj", + "visual.blocks.10.mlp.linear_fc1", + "visual.blocks.10.mlp.linear_fc2", + "model.visual.blocks.11.attn.proj", + "model.visual.blocks.11.attn.qkv", + "model.visual.blocks.11.mlp.linear_fc1", + "model.visual.blocks.11.mlp.linear_fc2", + "visual.blocks.11.attn.proj", + "visual.blocks.11.attn.qkv_proj", + "visual.blocks.11.mlp.linear_fc1", + "visual.blocks.11.mlp.linear_fc2", + "model.visual.blocks.12.attn.proj", + "model.visual.blocks.12.attn.qkv", + "model.visual.blocks.12.mlp.linear_fc1", + "model.visual.blocks.12.mlp.linear_fc2", + "visual.blocks.12.attn.proj", + "visual.blocks.12.attn.qkv_proj", + "visual.blocks.12.mlp.linear_fc1", + "visual.blocks.12.mlp.linear_fc2", + "model.visual.blocks.13.attn.proj", + "model.visual.blocks.13.attn.qkv", + "model.visual.blocks.13.mlp.linear_fc1", + "model.visual.blocks.13.mlp.linear_fc2", + "visual.blocks.13.attn.proj", + "visual.blocks.13.attn.qkv_proj", + "visual.blocks.13.mlp.linear_fc1", + "visual.blocks.13.mlp.linear_fc2", + "model.visual.blocks.14.attn.proj", + "model.visual.blocks.14.attn.qkv", + "model.visual.blocks.14.mlp.linear_fc1", + "model.visual.blocks.14.mlp.linear_fc2", + "visual.blocks.14.attn.proj", + "visual.blocks.14.attn.qkv_proj", + "visual.blocks.14.mlp.linear_fc1", + "visual.blocks.14.mlp.linear_fc2", + "model.visual.blocks.15.attn.proj", + "model.visual.blocks.15.attn.qkv", + "model.visual.blocks.15.mlp.linear_fc1", + "model.visual.blocks.15.mlp.linear_fc2", + "visual.blocks.15.attn.proj", + "visual.blocks.15.attn.qkv_proj", + "visual.blocks.15.mlp.linear_fc1", + "visual.blocks.15.mlp.linear_fc2", + "model.visual.blocks.16.attn.proj", + "model.visual.blocks.16.attn.qkv", + "model.visual.blocks.16.mlp.linear_fc1", + "model.visual.blocks.16.mlp.linear_fc2", + "visual.blocks.16.attn.proj", + "visual.blocks.16.attn.qkv_proj", + "visual.blocks.16.mlp.linear_fc1", + "visual.blocks.16.mlp.linear_fc2", + "model.visual.blocks.17.attn.proj", + "model.visual.blocks.17.attn.qkv", + "model.visual.blocks.17.mlp.linear_fc1", + "model.visual.blocks.17.mlp.linear_fc2", + "visual.blocks.17.attn.proj", + "visual.blocks.17.attn.qkv_proj", + "visual.blocks.17.mlp.linear_fc1", + "visual.blocks.17.mlp.linear_fc2", + "model.visual.blocks.18.attn.proj", + "model.visual.blocks.18.attn.qkv", + "model.visual.blocks.18.mlp.linear_fc1", + "model.visual.blocks.18.mlp.linear_fc2", + "visual.blocks.18.attn.proj", + "visual.blocks.18.attn.qkv_proj", + "visual.blocks.18.mlp.linear_fc1", + "visual.blocks.18.mlp.linear_fc2", + "model.visual.blocks.19.attn.proj", + "model.visual.blocks.19.attn.qkv", + "model.visual.blocks.19.mlp.linear_fc1", + "model.visual.blocks.19.mlp.linear_fc2", + "visual.blocks.19.attn.proj", + "visual.blocks.19.attn.qkv_proj", + "visual.blocks.19.mlp.linear_fc1", + "visual.blocks.19.mlp.linear_fc2", + "model.visual.blocks.20.attn.proj", + "model.visual.blocks.20.attn.qkv", + "model.visual.blocks.20.mlp.linear_fc1", + "model.visual.blocks.20.mlp.linear_fc2", + "visual.blocks.20.attn.proj", + "visual.blocks.20.attn.qkv_proj", + "visual.blocks.20.mlp.linear_fc1", + "visual.blocks.20.mlp.linear_fc2", + "model.visual.blocks.21.attn.proj", + "model.visual.blocks.21.attn.qkv", + "model.visual.blocks.21.mlp.linear_fc1", + "model.visual.blocks.21.mlp.linear_fc2", + "visual.blocks.21.attn.proj", + "visual.blocks.21.attn.qkv_proj", + "visual.blocks.21.mlp.linear_fc1", + "visual.blocks.21.mlp.linear_fc2", + "model.visual.blocks.22.attn.proj", + "model.visual.blocks.22.attn.qkv", + "model.visual.blocks.22.mlp.linear_fc1", + "model.visual.blocks.22.mlp.linear_fc2", + "visual.blocks.22.attn.proj", + "visual.blocks.22.attn.qkv_proj", + "visual.blocks.22.mlp.linear_fc1", + "visual.blocks.22.mlp.linear_fc2", + "model.visual.blocks.23.attn.proj", + "model.visual.blocks.23.attn.qkv", + "model.visual.blocks.23.mlp.linear_fc1", + "model.visual.blocks.23.mlp.linear_fc2", + "visual.blocks.23.attn.proj", + "visual.blocks.23.attn.qkv_proj", + "visual.blocks.23.mlp.linear_fc1", + "visual.blocks.23.mlp.linear_fc2", + "model.visual.blocks.24.attn.proj", + "model.visual.blocks.24.attn.qkv", + "model.visual.blocks.24.mlp.linear_fc1", + "model.visual.blocks.24.mlp.linear_fc2", + "visual.blocks.24.attn.proj", + "visual.blocks.24.attn.qkv_proj", + "visual.blocks.24.mlp.linear_fc1", + "visual.blocks.24.mlp.linear_fc2", + "model.visual.blocks.25.attn.proj", + "model.visual.blocks.25.attn.qkv", + "model.visual.blocks.25.mlp.linear_fc1", + "model.visual.blocks.25.mlp.linear_fc2", + "visual.blocks.25.attn.proj", + "visual.blocks.25.attn.qkv_proj", + "visual.blocks.25.mlp.linear_fc1", + "visual.blocks.25.mlp.linear_fc2", + "model.visual.blocks.26.attn.proj", + "model.visual.blocks.26.attn.qkv", + "model.visual.blocks.26.mlp.linear_fc1", + "model.visual.blocks.26.mlp.linear_fc2", + "visual.blocks.26.attn.proj", + "visual.blocks.26.attn.qkv_proj", + "visual.blocks.26.mlp.linear_fc1", + "visual.blocks.26.mlp.linear_fc2", + "model.visual.deepstack_merger_list.0.linear_fc1", + "model.visual.deepstack_merger_list.0.linear_fc2", + "model.visual.deepstack_merger_list.0.norm", + "visual.deepstack_merger_list.0.linear_fc1", + "visual.deepstack_merger_list.0.linear_fc2", + "visual.deepstack_merger_list.0.norm", + "model.visual.deepstack_merger_list.1.linear_fc1", + "model.visual.deepstack_merger_list.1.linear_fc2", + "model.visual.deepstack_merger_list.1.norm", + "visual.deepstack_merger_list.1.linear_fc1", + "visual.deepstack_merger_list.1.linear_fc2", + "visual.deepstack_merger_list.1.norm", + "model.visual.deepstack_merger_list.2.linear_fc1", + "model.visual.deepstack_merger_list.2.linear_fc2", + "model.visual.deepstack_merger_list.2.norm", + "visual.deepstack_merger_list.2.linear_fc1", + "visual.deepstack_merger_list.2.linear_fc2", + "visual.deepstack_merger_list.2.norm", + "model.visual.merger.linear_fc1", + "model.visual.merger.linear_fc2", + "model.visual.merger.norm", + "model.visual.patch_embed.proj", + "model.visual.pos_embed", + "visual.merger.linear_fc1", + "visual.merger.linear_fc2", + "visual.merger.norm", + "visual.patch_embed.proj", + "visual.pos_embed", + "visual", + "model.visual", + "lm_head", + "model.embed_tokens", + "model.language_model.layers.0.input_layernorm", + "model.language_model.layers.0.mlp.shared_expert_gate", + "model.language_model.layers.0.post_attention_layernorm", + "model.language_model.layers.0.mlp.gate", + "model.language_model.layers.0.linear_attn.A_log", + "model.language_model.layers.0.linear_attn.conv1d", + "model.language_model.layers.0.linear_attn.dt_bias", + "model.language_model.layers.0.linear_attn.in_proj_ba", + "model.language_model.layers.0.linear_attn.in_proj_b", + "model.language_model.layers.0.linear_attn.in_proj_a", + "model.language_model.layers.0.linear_attn.norm", + "model.language_model.layers.1.input_layernorm", + "model.language_model.layers.1.mlp.shared_expert_gate", + "model.language_model.layers.1.post_attention_layernorm", + "model.language_model.layers.1.mlp.gate", + "model.language_model.layers.1.linear_attn.A_log", + "model.language_model.layers.1.linear_attn.conv1d", + "model.language_model.layers.1.linear_attn.dt_bias", + "model.language_model.layers.1.linear_attn.in_proj_ba", + "model.language_model.layers.1.linear_attn.in_proj_b", + "model.language_model.layers.1.linear_attn.in_proj_a", + "model.language_model.layers.1.linear_attn.norm", + "model.language_model.layers.2.input_layernorm", + "model.language_model.layers.2.mlp.shared_expert_gate", + "model.language_model.layers.2.post_attention_layernorm", + "model.language_model.layers.2.mlp.gate", + "model.language_model.layers.2.linear_attn.A_log", + "model.language_model.layers.2.linear_attn.conv1d", + "model.language_model.layers.2.linear_attn.dt_bias", + "model.language_model.layers.2.linear_attn.in_proj_ba", + "model.language_model.layers.2.linear_attn.in_proj_b", + "model.language_model.layers.2.linear_attn.in_proj_a", + "model.language_model.layers.2.linear_attn.norm", + "model.language_model.layers.3.input_layernorm", + "model.language_model.layers.3.mlp.shared_expert_gate", + "model.language_model.layers.3.post_attention_layernorm", + "model.language_model.layers.3.mlp.gate", + "model.language_model.layers.3.self_attn.k_norm", + "model.language_model.layers.3.self_attn.q_norm", + "model.language_model.layers.4.input_layernorm", + "model.language_model.layers.4.mlp.shared_expert_gate", + "model.language_model.layers.4.post_attention_layernorm", + "model.language_model.layers.4.mlp.gate", + "model.language_model.layers.4.linear_attn.A_log", + "model.language_model.layers.4.linear_attn.conv1d", + "model.language_model.layers.4.linear_attn.dt_bias", + "model.language_model.layers.4.linear_attn.in_proj_ba", + "model.language_model.layers.4.linear_attn.in_proj_b", + "model.language_model.layers.4.linear_attn.in_proj_a", + "model.language_model.layers.4.linear_attn.norm", + "model.language_model.layers.5.input_layernorm", + "model.language_model.layers.5.mlp.shared_expert_gate", + "model.language_model.layers.5.post_attention_layernorm", + "model.language_model.layers.5.mlp.gate", + "model.language_model.layers.5.linear_attn.A_log", + "model.language_model.layers.5.linear_attn.conv1d", + "model.language_model.layers.5.linear_attn.dt_bias", + "model.language_model.layers.5.linear_attn.in_proj_ba", + "model.language_model.layers.5.linear_attn.in_proj_b", + "model.language_model.layers.5.linear_attn.in_proj_a", + "model.language_model.layers.5.linear_attn.norm", + "model.language_model.layers.6.input_layernorm", + "model.language_model.layers.6.mlp.shared_expert_gate", + "model.language_model.layers.6.post_attention_layernorm", + "model.language_model.layers.6.mlp.gate", + "model.language_model.layers.6.linear_attn.A_log", + "model.language_model.layers.6.linear_attn.conv1d", + "model.language_model.layers.6.linear_attn.dt_bias", + "model.language_model.layers.6.linear_attn.in_proj_ba", + "model.language_model.layers.6.linear_attn.in_proj_b", + "model.language_model.layers.6.linear_attn.in_proj_a", + "model.language_model.layers.6.linear_attn.norm", + "model.language_model.layers.7.input_layernorm", + "model.language_model.layers.7.mlp.shared_expert_gate", + "model.language_model.layers.7.post_attention_layernorm", + "model.language_model.layers.7.mlp.gate", + "model.language_model.layers.7.self_attn.k_norm", + "model.language_model.layers.7.self_attn.q_norm", + "model.language_model.layers.8.input_layernorm", + "model.language_model.layers.8.mlp.shared_expert_gate", + "model.language_model.layers.8.post_attention_layernorm", + "model.language_model.layers.8.mlp.gate", + "model.language_model.layers.8.linear_attn.A_log", + "model.language_model.layers.8.linear_attn.conv1d", + "model.language_model.layers.8.linear_attn.dt_bias", + "model.language_model.layers.8.linear_attn.in_proj_ba", + "model.language_model.layers.8.linear_attn.in_proj_b", + "model.language_model.layers.8.linear_attn.in_proj_a", + "model.language_model.layers.8.linear_attn.norm", + "model.language_model.layers.9.input_layernorm", + "model.language_model.layers.9.mlp.shared_expert_gate", + "model.language_model.layers.9.post_attention_layernorm", + "model.language_model.layers.9.mlp.gate", + "model.language_model.layers.9.linear_attn.A_log", + "model.language_model.layers.9.linear_attn.conv1d", + "model.language_model.layers.9.linear_attn.dt_bias", + "model.language_model.layers.9.linear_attn.in_proj_ba", + "model.language_model.layers.9.linear_attn.in_proj_b", + "model.language_model.layers.9.linear_attn.in_proj_a", + "model.language_model.layers.9.linear_attn.norm", + "model.language_model.layers.10.input_layernorm", + "model.language_model.layers.10.mlp.shared_expert_gate", + "model.language_model.layers.10.post_attention_layernorm", + "model.language_model.layers.10.mlp.gate", + "model.language_model.layers.10.linear_attn.A_log", + "model.language_model.layers.10.linear_attn.conv1d", + "model.language_model.layers.10.linear_attn.dt_bias", + "model.language_model.layers.10.linear_attn.in_proj_ba", + "model.language_model.layers.10.linear_attn.in_proj_b", + "model.language_model.layers.10.linear_attn.in_proj_a", + "model.language_model.layers.10.linear_attn.norm", + "model.language_model.layers.11.input_layernorm", + "model.language_model.layers.11.mlp.shared_expert_gate", + "model.language_model.layers.11.post_attention_layernorm", + "model.language_model.layers.11.mlp.gate", + "model.language_model.layers.11.self_attn.k_norm", + "model.language_model.layers.11.self_attn.q_norm", + "model.language_model.layers.12.input_layernorm", + "model.language_model.layers.12.mlp.shared_expert_gate", + "model.language_model.layers.12.post_attention_layernorm", + "model.language_model.layers.12.mlp.gate", + "model.language_model.layers.12.linear_attn.A_log", + "model.language_model.layers.12.linear_attn.conv1d", + "model.language_model.layers.12.linear_attn.dt_bias", + "model.language_model.layers.12.linear_attn.in_proj_ba", + "model.language_model.layers.12.linear_attn.in_proj_b", + "model.language_model.layers.12.linear_attn.in_proj_a", + "model.language_model.layers.12.linear_attn.norm", + "model.language_model.layers.13.input_layernorm", + "model.language_model.layers.13.mlp.shared_expert_gate", + "model.language_model.layers.13.post_attention_layernorm", + "model.language_model.layers.13.mlp.gate", + "model.language_model.layers.13.linear_attn.A_log", + "model.language_model.layers.13.linear_attn.conv1d", + "model.language_model.layers.13.linear_attn.dt_bias", + "model.language_model.layers.13.linear_attn.in_proj_ba", + "model.language_model.layers.13.linear_attn.in_proj_b", + "model.language_model.layers.13.linear_attn.in_proj_a", + "model.language_model.layers.13.linear_attn.norm", + "model.language_model.layers.14.input_layernorm", + "model.language_model.layers.14.mlp.shared_expert_gate", + "model.language_model.layers.14.post_attention_layernorm", + "model.language_model.layers.14.mlp.gate", + "model.language_model.layers.14.linear_attn.A_log", + "model.language_model.layers.14.linear_attn.conv1d", + "model.language_model.layers.14.linear_attn.dt_bias", + "model.language_model.layers.14.linear_attn.in_proj_ba", + "model.language_model.layers.14.linear_attn.in_proj_b", + "model.language_model.layers.14.linear_attn.in_proj_a", + "model.language_model.layers.14.linear_attn.norm", + "model.language_model.layers.15.input_layernorm", + "model.language_model.layers.15.mlp.shared_expert_gate", + "model.language_model.layers.15.post_attention_layernorm", + "model.language_model.layers.15.mlp.gate", + "model.language_model.layers.15.self_attn.k_norm", + "model.language_model.layers.15.self_attn.q_norm", + "model.language_model.layers.16.input_layernorm", + "model.language_model.layers.16.mlp.shared_expert_gate", + "model.language_model.layers.16.post_attention_layernorm", + "model.language_model.layers.16.mlp.gate", + "model.language_model.layers.16.linear_attn.A_log", + "model.language_model.layers.16.linear_attn.conv1d", + "model.language_model.layers.16.linear_attn.dt_bias", + "model.language_model.layers.16.linear_attn.in_proj_ba", + "model.language_model.layers.16.linear_attn.in_proj_b", + "model.language_model.layers.16.linear_attn.in_proj_a", + "model.language_model.layers.16.linear_attn.norm", + "model.language_model.layers.17.input_layernorm", + "model.language_model.layers.17.mlp.shared_expert_gate", + "model.language_model.layers.17.post_attention_layernorm", + "model.language_model.layers.17.mlp.gate", + "model.language_model.layers.17.linear_attn.A_log", + "model.language_model.layers.17.linear_attn.conv1d", + "model.language_model.layers.17.linear_attn.dt_bias", + "model.language_model.layers.17.linear_attn.in_proj_ba", + "model.language_model.layers.17.linear_attn.in_proj_b", + "model.language_model.layers.17.linear_attn.in_proj_a", + "model.language_model.layers.17.linear_attn.norm", + "model.language_model.layers.18.input_layernorm", + "model.language_model.layers.18.mlp.shared_expert_gate", + "model.language_model.layers.18.post_attention_layernorm", + "model.language_model.layers.18.mlp.gate", + "model.language_model.layers.18.linear_attn.A_log", + "model.language_model.layers.18.linear_attn.conv1d", + "model.language_model.layers.18.linear_attn.dt_bias", + "model.language_model.layers.18.linear_attn.in_proj_ba", + "model.language_model.layers.18.linear_attn.in_proj_b", + "model.language_model.layers.18.linear_attn.in_proj_a", + "model.language_model.layers.18.linear_attn.norm", + "model.language_model.layers.19.input_layernorm", + "model.language_model.layers.19.mlp.shared_expert_gate", + "model.language_model.layers.19.post_attention_layernorm", + "model.language_model.layers.19.mlp.gate", + "model.language_model.layers.19.self_attn.k_norm", + "model.language_model.layers.19.self_attn.q_norm", + "model.language_model.layers.20.input_layernorm", + "model.language_model.layers.20.mlp.shared_expert_gate", + "model.language_model.layers.20.post_attention_layernorm", + "model.language_model.layers.20.mlp.gate", + "model.language_model.layers.20.linear_attn.A_log", + "model.language_model.layers.20.linear_attn.conv1d", + "model.language_model.layers.20.linear_attn.dt_bias", + "model.language_model.layers.20.linear_attn.in_proj_ba", + "model.language_model.layers.20.linear_attn.in_proj_b", + "model.language_model.layers.20.linear_attn.in_proj_a", + "model.language_model.layers.20.linear_attn.norm", + "model.language_model.layers.21.input_layernorm", + "model.language_model.layers.21.mlp.shared_expert_gate", + "model.language_model.layers.21.post_attention_layernorm", + "model.language_model.layers.21.mlp.gate", + "model.language_model.layers.21.linear_attn.A_log", + "model.language_model.layers.21.linear_attn.conv1d", + "model.language_model.layers.21.linear_attn.dt_bias", + "model.language_model.layers.21.linear_attn.in_proj_ba", + "model.language_model.layers.21.linear_attn.in_proj_b", + "model.language_model.layers.21.linear_attn.in_proj_a", + "model.language_model.layers.21.linear_attn.norm", + "model.language_model.layers.22.input_layernorm", + "model.language_model.layers.22.mlp.shared_expert_gate", + "model.language_model.layers.22.post_attention_layernorm", + "model.language_model.layers.22.mlp.gate", + "model.language_model.layers.22.linear_attn.A_log", + "model.language_model.layers.22.linear_attn.conv1d", + "model.language_model.layers.22.linear_attn.dt_bias", + "model.language_model.layers.22.linear_attn.in_proj_ba", + "model.language_model.layers.22.linear_attn.in_proj_b", + "model.language_model.layers.22.linear_attn.in_proj_a", + "model.language_model.layers.22.linear_attn.norm", + "model.language_model.layers.23.input_layernorm", + "model.language_model.layers.23.mlp.shared_expert_gate", + "model.language_model.layers.23.post_attention_layernorm", + "model.language_model.layers.23.mlp.gate", + "model.language_model.layers.23.self_attn.k_norm", + "model.language_model.layers.23.self_attn.q_norm", + "model.language_model.layers.24.input_layernorm", + "model.language_model.layers.24.mlp.shared_expert_gate", + "model.language_model.layers.24.post_attention_layernorm", + "model.language_model.layers.24.mlp.gate", + "model.language_model.layers.24.linear_attn.A_log", + "model.language_model.layers.24.linear_attn.conv1d", + "model.language_model.layers.24.linear_attn.dt_bias", + "model.language_model.layers.24.linear_attn.in_proj_ba", + "model.language_model.layers.24.linear_attn.in_proj_b", + "model.language_model.layers.24.linear_attn.in_proj_a", + "model.language_model.layers.24.linear_attn.norm", + "model.language_model.layers.25.input_layernorm", + "model.language_model.layers.25.mlp.shared_expert_gate", + "model.language_model.layers.25.post_attention_layernorm", + "model.language_model.layers.25.mlp.gate", + "model.language_model.layers.25.linear_attn.A_log", + "model.language_model.layers.25.linear_attn.conv1d", + "model.language_model.layers.25.linear_attn.dt_bias", + "model.language_model.layers.25.linear_attn.in_proj_ba", + "model.language_model.layers.25.linear_attn.in_proj_b", + "model.language_model.layers.25.linear_attn.in_proj_a", + "model.language_model.layers.25.linear_attn.norm", + "model.language_model.layers.26.input_layernorm", + "model.language_model.layers.26.mlp.shared_expert_gate", + "model.language_model.layers.26.post_attention_layernorm", + "model.language_model.layers.26.mlp.gate", + "model.language_model.layers.26.linear_attn.A_log", + "model.language_model.layers.26.linear_attn.conv1d", + "model.language_model.layers.26.linear_attn.dt_bias", + "model.language_model.layers.26.linear_attn.in_proj_ba", + "model.language_model.layers.26.linear_attn.in_proj_b", + "model.language_model.layers.26.linear_attn.in_proj_a", + "model.language_model.layers.26.linear_attn.norm", + "model.language_model.layers.27.input_layernorm", + "model.language_model.layers.27.mlp.shared_expert_gate", + "model.language_model.layers.27.post_attention_layernorm", + "model.language_model.layers.27.mlp.gate", + "model.language_model.layers.27.self_attn.k_norm", + "model.language_model.layers.27.self_attn.q_norm", + "model.language_model.layers.28.input_layernorm", + "model.language_model.layers.28.mlp.shared_expert_gate", + "model.language_model.layers.28.post_attention_layernorm", + "model.language_model.layers.28.mlp.gate", + "model.language_model.layers.28.linear_attn.A_log", + "model.language_model.layers.28.linear_attn.conv1d", + "model.language_model.layers.28.linear_attn.dt_bias", + "model.language_model.layers.28.linear_attn.in_proj_ba", + "model.language_model.layers.28.linear_attn.in_proj_b", + "model.language_model.layers.28.linear_attn.in_proj_a", + "model.language_model.layers.28.linear_attn.norm", + "model.language_model.layers.29.input_layernorm", + "model.language_model.layers.29.mlp.shared_expert_gate", + "model.language_model.layers.29.post_attention_layernorm", + "model.language_model.layers.29.mlp.gate", + "model.language_model.layers.29.linear_attn.A_log", + "model.language_model.layers.29.linear_attn.conv1d", + "model.language_model.layers.29.linear_attn.dt_bias", + "model.language_model.layers.29.linear_attn.in_proj_ba", + "model.language_model.layers.29.linear_attn.in_proj_b", + "model.language_model.layers.29.linear_attn.in_proj_a", + "model.language_model.layers.29.linear_attn.norm", + "model.language_model.layers.30.input_layernorm", + "model.language_model.layers.30.mlp.shared_expert_gate", + "model.language_model.layers.30.post_attention_layernorm", + "model.language_model.layers.30.mlp.gate", + "model.language_model.layers.30.linear_attn.A_log", + "model.language_model.layers.30.linear_attn.conv1d", + "model.language_model.layers.30.linear_attn.dt_bias", + "model.language_model.layers.30.linear_attn.in_proj_ba", + "model.language_model.layers.30.linear_attn.in_proj_b", + "model.language_model.layers.30.linear_attn.in_proj_a", + "model.language_model.layers.30.linear_attn.norm", + "model.language_model.layers.31.input_layernorm", + "model.language_model.layers.31.mlp.shared_expert_gate", + "model.language_model.layers.31.post_attention_layernorm", + "model.language_model.layers.31.mlp.gate", + "model.language_model.layers.31.self_attn.k_norm", + "model.language_model.layers.31.self_attn.q_norm", + "model.language_model.layers.32.input_layernorm", + "model.language_model.layers.32.mlp.shared_expert_gate", + "model.language_model.layers.32.post_attention_layernorm", + "model.language_model.layers.32.mlp.gate", + "model.language_model.layers.32.linear_attn.A_log", + "model.language_model.layers.32.linear_attn.conv1d", + "model.language_model.layers.32.linear_attn.dt_bias", + "model.language_model.layers.32.linear_attn.in_proj_ba", + "model.language_model.layers.32.linear_attn.in_proj_b", + "model.language_model.layers.32.linear_attn.in_proj_a", + "model.language_model.layers.32.linear_attn.norm", + "model.language_model.layers.33.input_layernorm", + "model.language_model.layers.33.mlp.shared_expert_gate", + "model.language_model.layers.33.post_attention_layernorm", + "model.language_model.layers.33.mlp.gate", + "model.language_model.layers.33.linear_attn.A_log", + "model.language_model.layers.33.linear_attn.conv1d", + "model.language_model.layers.33.linear_attn.dt_bias", + "model.language_model.layers.33.linear_attn.in_proj_ba", + "model.language_model.layers.33.linear_attn.in_proj_b", + "model.language_model.layers.33.linear_attn.in_proj_a", + "model.language_model.layers.33.linear_attn.norm", + "model.language_model.layers.34.input_layernorm", + "model.language_model.layers.34.mlp.shared_expert_gate", + "model.language_model.layers.34.post_attention_layernorm", + "model.language_model.layers.34.mlp.gate", + "model.language_model.layers.34.linear_attn.A_log", + "model.language_model.layers.34.linear_attn.conv1d", + "model.language_model.layers.34.linear_attn.dt_bias", + "model.language_model.layers.34.linear_attn.in_proj_ba", + "model.language_model.layers.34.linear_attn.in_proj_b", + "model.language_model.layers.34.linear_attn.in_proj_a", + "model.language_model.layers.34.linear_attn.norm", + "model.language_model.layers.35.input_layernorm", + "model.language_model.layers.35.mlp.shared_expert_gate", + "model.language_model.layers.35.post_attention_layernorm", + "model.language_model.layers.35.mlp.gate", + "model.language_model.layers.35.self_attn.k_norm", + "model.language_model.layers.35.self_attn.q_norm", + "model.language_model.layers.36.input_layernorm", + "model.language_model.layers.36.mlp.shared_expert_gate", + "model.language_model.layers.36.post_attention_layernorm", + "model.language_model.layers.36.mlp.gate", + "model.language_model.layers.36.linear_attn.A_log", + "model.language_model.layers.36.linear_attn.conv1d", + "model.language_model.layers.36.linear_attn.dt_bias", + "model.language_model.layers.36.linear_attn.in_proj_ba", + "model.language_model.layers.36.linear_attn.in_proj_b", + "model.language_model.layers.36.linear_attn.in_proj_a", + "model.language_model.layers.36.linear_attn.norm", + "model.language_model.layers.37.input_layernorm", + "model.language_model.layers.37.mlp.shared_expert_gate", + "model.language_model.layers.37.post_attention_layernorm", + "model.language_model.layers.37.mlp.gate", + "model.language_model.layers.37.linear_attn.A_log", + "model.language_model.layers.37.linear_attn.conv1d", + "model.language_model.layers.37.linear_attn.dt_bias", + "model.language_model.layers.37.linear_attn.in_proj_ba", + "model.language_model.layers.37.linear_attn.in_proj_b", + "model.language_model.layers.37.linear_attn.in_proj_a", + "model.language_model.layers.37.linear_attn.norm", + "model.language_model.layers.38.input_layernorm", + "model.language_model.layers.38.mlp.shared_expert_gate", + "model.language_model.layers.38.post_attention_layernorm", + "model.language_model.layers.38.mlp.gate", + "model.language_model.layers.38.linear_attn.A_log", + "model.language_model.layers.38.linear_attn.conv1d", + "model.language_model.layers.38.linear_attn.dt_bias", + "model.language_model.layers.38.linear_attn.in_proj_ba", + "model.language_model.layers.38.linear_attn.in_proj_b", + "model.language_model.layers.38.linear_attn.in_proj_a", + "model.language_model.layers.38.linear_attn.norm", + "model.language_model.layers.39.input_layernorm", + "model.language_model.layers.39.mlp.shared_expert_gate", + "model.language_model.layers.39.post_attention_layernorm", + "model.language_model.layers.39.mlp.gate", + "model.language_model.layers.39.self_attn.k_norm", + "model.language_model.layers.39.self_attn.q_norm", + "model.language_model.layers.40.input_layernorm", + "model.language_model.layers.40.mlp.shared_expert_gate", + "model.language_model.layers.40.post_attention_layernorm", + "model.language_model.layers.40.mlp.gate", + "model.language_model.layers.40.linear_attn.A_log", + "model.language_model.layers.40.linear_attn.conv1d", + "model.language_model.layers.40.linear_attn.dt_bias", + "model.language_model.layers.40.linear_attn.in_proj_ba", + "model.language_model.layers.40.linear_attn.in_proj_b", + "model.language_model.layers.40.linear_attn.in_proj_a", + "model.language_model.layers.40.linear_attn.norm", + "model.language_model.layers.41.input_layernorm", + "model.language_model.layers.41.mlp.shared_expert_gate", + "model.language_model.layers.41.post_attention_layernorm", + "model.language_model.layers.41.mlp.gate", + "model.language_model.layers.41.linear_attn.A_log", + "model.language_model.layers.41.linear_attn.conv1d", + "model.language_model.layers.41.linear_attn.dt_bias", + "model.language_model.layers.41.linear_attn.in_proj_ba", + "model.language_model.layers.41.linear_attn.in_proj_b", + "model.language_model.layers.41.linear_attn.in_proj_a", + "model.language_model.layers.41.linear_attn.norm", + "model.language_model.layers.42.input_layernorm", + "model.language_model.layers.42.mlp.shared_expert_gate", + "model.language_model.layers.42.post_attention_layernorm", + "model.language_model.layers.42.mlp.gate", + "model.language_model.layers.42.linear_attn.A_log", + "model.language_model.layers.42.linear_attn.conv1d", + "model.language_model.layers.42.linear_attn.dt_bias", + "model.language_model.layers.42.linear_attn.in_proj_ba", + "model.language_model.layers.42.linear_attn.in_proj_b", + "model.language_model.layers.42.linear_attn.in_proj_a", + "model.language_model.layers.42.linear_attn.norm", + "model.language_model.layers.43.input_layernorm", + "model.language_model.layers.43.mlp.shared_expert_gate", + "model.language_model.layers.43.post_attention_layernorm", + "model.language_model.layers.43.mlp.gate", + "model.language_model.layers.43.self_attn.k_norm", + "model.language_model.layers.43.self_attn.q_norm", + "model.language_model.layers.44.input_layernorm", + "model.language_model.layers.44.mlp.shared_expert_gate", + "model.language_model.layers.44.post_attention_layernorm", + "model.language_model.layers.44.mlp.gate", + "model.language_model.layers.44.linear_attn.A_log", + "model.language_model.layers.44.linear_attn.conv1d", + "model.language_model.layers.44.linear_attn.dt_bias", + "model.language_model.layers.44.linear_attn.in_proj_ba", + "model.language_model.layers.44.linear_attn.in_proj_b", + "model.language_model.layers.44.linear_attn.in_proj_a", + "model.language_model.layers.44.linear_attn.norm", + "model.language_model.layers.45.input_layernorm", + "model.language_model.layers.45.mlp.shared_expert_gate", + "model.language_model.layers.45.post_attention_layernorm", + "model.language_model.layers.45.mlp.gate", + "model.language_model.layers.45.linear_attn.A_log", + "model.language_model.layers.45.linear_attn.conv1d", + "model.language_model.layers.45.linear_attn.dt_bias", + "model.language_model.layers.45.linear_attn.in_proj_ba", + "model.language_model.layers.45.linear_attn.in_proj_b", + "model.language_model.layers.45.linear_attn.in_proj_a", + "model.language_model.layers.45.linear_attn.norm", + "model.language_model.layers.46.input_layernorm", + "model.language_model.layers.46.mlp.shared_expert_gate", + "model.language_model.layers.46.post_attention_layernorm", + "model.language_model.layers.46.mlp.gate", + "model.language_model.layers.46.linear_attn.A_log", + "model.language_model.layers.46.linear_attn.conv1d", + "model.language_model.layers.46.linear_attn.dt_bias", + "model.language_model.layers.46.linear_attn.in_proj_ba", + "model.language_model.layers.46.linear_attn.in_proj_b", + "model.language_model.layers.46.linear_attn.in_proj_a", + "model.language_model.layers.46.linear_attn.norm", + "model.language_model.layers.47.input_layernorm", + "model.language_model.layers.47.mlp.shared_expert_gate", + "model.language_model.layers.47.post_attention_layernorm", + "model.language_model.layers.47.mlp.gate", + "model.language_model.layers.47.self_attn.k_norm", + "model.language_model.layers.47.self_attn.q_norm", + "model.language_model.layers.48.input_layernorm", + "model.language_model.layers.48.mlp.shared_expert_gate", + "model.language_model.layers.48.post_attention_layernorm", + "model.language_model.layers.48.mlp.gate", + "model.language_model.layers.48.linear_attn.A_log", + "model.language_model.layers.48.linear_attn.conv1d", + "model.language_model.layers.48.linear_attn.dt_bias", + "model.language_model.layers.48.linear_attn.in_proj_ba", + "model.language_model.layers.48.linear_attn.in_proj_b", + "model.language_model.layers.48.linear_attn.in_proj_a", + "model.language_model.layers.48.linear_attn.norm", + "model.language_model.layers.49.input_layernorm", + "model.language_model.layers.49.mlp.shared_expert_gate", + "model.language_model.layers.49.post_attention_layernorm", + "model.language_model.layers.49.mlp.gate", + "model.language_model.layers.49.linear_attn.A_log", + "model.language_model.layers.49.linear_attn.conv1d", + "model.language_model.layers.49.linear_attn.dt_bias", + "model.language_model.layers.49.linear_attn.in_proj_ba", + "model.language_model.layers.49.linear_attn.in_proj_b", + "model.language_model.layers.49.linear_attn.in_proj_a", + "model.language_model.layers.49.linear_attn.norm", + "model.language_model.layers.50.input_layernorm", + "model.language_model.layers.50.mlp.shared_expert_gate", + "model.language_model.layers.50.post_attention_layernorm", + "model.language_model.layers.50.mlp.gate", + "model.language_model.layers.50.linear_attn.A_log", + "model.language_model.layers.50.linear_attn.conv1d", + "model.language_model.layers.50.linear_attn.dt_bias", + "model.language_model.layers.50.linear_attn.in_proj_ba", + "model.language_model.layers.50.linear_attn.in_proj_b", + "model.language_model.layers.50.linear_attn.in_proj_a", + "model.language_model.layers.50.linear_attn.norm", + "model.language_model.layers.51.input_layernorm", + "model.language_model.layers.51.mlp.shared_expert_gate", + "model.language_model.layers.51.post_attention_layernorm", + "model.language_model.layers.51.mlp.gate", + "model.language_model.layers.51.self_attn.k_norm", + "model.language_model.layers.51.self_attn.q_norm", + "model.language_model.layers.52.input_layernorm", + "model.language_model.layers.52.mlp.shared_expert_gate", + "model.language_model.layers.52.post_attention_layernorm", + "model.language_model.layers.52.mlp.gate", + "model.language_model.layers.52.linear_attn.A_log", + "model.language_model.layers.52.linear_attn.conv1d", + "model.language_model.layers.52.linear_attn.dt_bias", + "model.language_model.layers.52.linear_attn.in_proj_ba", + "model.language_model.layers.52.linear_attn.in_proj_b", + "model.language_model.layers.52.linear_attn.in_proj_a", + "model.language_model.layers.52.linear_attn.norm", + "model.language_model.layers.53.input_layernorm", + "model.language_model.layers.53.mlp.shared_expert_gate", + "model.language_model.layers.53.post_attention_layernorm", + "model.language_model.layers.53.mlp.gate", + "model.language_model.layers.53.linear_attn.A_log", + "model.language_model.layers.53.linear_attn.conv1d", + "model.language_model.layers.53.linear_attn.dt_bias", + "model.language_model.layers.53.linear_attn.in_proj_ba", + "model.language_model.layers.53.linear_attn.in_proj_b", + "model.language_model.layers.53.linear_attn.in_proj_a", + "model.language_model.layers.53.linear_attn.norm", + "model.language_model.layers.54.input_layernorm", + "model.language_model.layers.54.mlp.shared_expert_gate", + "model.language_model.layers.54.post_attention_layernorm", + "model.language_model.layers.54.mlp.gate", + "model.language_model.layers.54.linear_attn.A_log", + "model.language_model.layers.54.linear_attn.conv1d", + "model.language_model.layers.54.linear_attn.dt_bias", + "model.language_model.layers.54.linear_attn.in_proj_ba", + "model.language_model.layers.54.linear_attn.in_proj_b", + "model.language_model.layers.54.linear_attn.in_proj_a", + "model.language_model.layers.54.linear_attn.norm", + "model.language_model.layers.55.input_layernorm", + "model.language_model.layers.55.mlp.shared_expert_gate", + "model.language_model.layers.55.post_attention_layernorm", + "model.language_model.layers.55.mlp.gate", + "model.language_model.layers.55.self_attn.k_norm", + "model.language_model.layers.55.self_attn.q_norm", + "model.language_model.layers.56.input_layernorm", + "model.language_model.layers.56.mlp.shared_expert_gate", + "model.language_model.layers.56.post_attention_layernorm", + "model.language_model.layers.56.mlp.gate", + "model.language_model.layers.56.linear_attn.A_log", + "model.language_model.layers.56.linear_attn.conv1d", + "model.language_model.layers.56.linear_attn.dt_bias", + "model.language_model.layers.56.linear_attn.in_proj_ba", + "model.language_model.layers.56.linear_attn.in_proj_b", + "model.language_model.layers.56.linear_attn.in_proj_a", + "model.language_model.layers.56.linear_attn.norm", + "model.language_model.layers.57.input_layernorm", + "model.language_model.layers.57.mlp.shared_expert_gate", + "model.language_model.layers.57.post_attention_layernorm", + "model.language_model.layers.57.mlp.gate", + "model.language_model.layers.57.linear_attn.A_log", + "model.language_model.layers.57.linear_attn.conv1d", + "model.language_model.layers.57.linear_attn.dt_bias", + "model.language_model.layers.57.linear_attn.in_proj_ba", + "model.language_model.layers.57.linear_attn.in_proj_b", + "model.language_model.layers.57.linear_attn.in_proj_a", + "model.language_model.layers.57.linear_attn.norm", + "model.language_model.layers.58.input_layernorm", + "model.language_model.layers.58.mlp.shared_expert_gate", + "model.language_model.layers.58.post_attention_layernorm", + "model.language_model.layers.58.mlp.gate", + "model.language_model.layers.58.linear_attn.A_log", + "model.language_model.layers.58.linear_attn.conv1d", + "model.language_model.layers.58.linear_attn.dt_bias", + "model.language_model.layers.58.linear_attn.in_proj_ba", + "model.language_model.layers.58.linear_attn.in_proj_b", + "model.language_model.layers.58.linear_attn.in_proj_a", + "model.language_model.layers.58.linear_attn.norm", + "model.language_model.layers.59.input_layernorm", + "model.language_model.layers.59.mlp.shared_expert_gate", + "model.language_model.layers.59.post_attention_layernorm", + "model.language_model.layers.59.mlp.gate", + "model.language_model.layers.59.self_attn.k_norm", + "model.language_model.layers.59.self_attn.q_norm", + "model.language_model.layers.60.input_layernorm", + "model.language_model.layers.60.mlp.shared_expert_gate", + "model.language_model.layers.60.post_attention_layernorm", + "model.language_model.layers.60.mlp.gate", + "model.language_model.layers.60.linear_attn.A_log", + "model.language_model.layers.60.linear_attn.conv1d", + "model.language_model.layers.60.linear_attn.dt_bias", + "model.language_model.layers.60.linear_attn.in_proj_ba", + "model.language_model.layers.60.linear_attn.in_proj_b", + "model.language_model.layers.60.linear_attn.in_proj_a", + "model.language_model.layers.60.linear_attn.norm", + "model.language_model.layers.61.input_layernorm", + "model.language_model.layers.61.mlp.shared_expert_gate", + "model.language_model.layers.61.post_attention_layernorm", + "model.language_model.layers.61.mlp.gate", + "model.language_model.layers.61.linear_attn.A_log", + "model.language_model.layers.61.linear_attn.conv1d", + "model.language_model.layers.61.linear_attn.dt_bias", + "model.language_model.layers.61.linear_attn.in_proj_ba", + "model.language_model.layers.61.linear_attn.in_proj_b", + "model.language_model.layers.61.linear_attn.in_proj_a", + "model.language_model.layers.61.linear_attn.norm", + "model.language_model.layers.62.input_layernorm", + "model.language_model.layers.62.mlp.shared_expert_gate", + "model.language_model.layers.62.post_attention_layernorm", + "model.language_model.layers.62.mlp.gate", + "model.language_model.layers.62.linear_attn.A_log", + "model.language_model.layers.62.linear_attn.conv1d", + "model.language_model.layers.62.linear_attn.dt_bias", + "model.language_model.layers.62.linear_attn.in_proj_ba", + "model.language_model.layers.62.linear_attn.in_proj_b", + "model.language_model.layers.62.linear_attn.in_proj_a", + "model.language_model.layers.62.linear_attn.norm", + "model.language_model.layers.63.input_layernorm", + "model.language_model.layers.63.mlp.shared_expert_gate", + "model.language_model.layers.63.post_attention_layernorm", + "model.language_model.layers.63.mlp.gate", + "model.language_model.layers.63.self_attn.k_norm", + "model.language_model.layers.63.self_attn.q_norm", + "mtp.layers.0.input_layernorm", + "mtp.layers.0.mlp.gate", + "mtp.layers.0.mlp.shared_expert_gate", + "mtp.layers.0.post_attention_layernorm", + "mtp.layers.0.self_attn.k_norm", + "mtp.layers.0.self_attn.q_norm", + "mtp.fc", + "mtp.norm", + "mtp.pre_fc_norm_embedding", + "mtp.pre_fc_norm_hidden" + ], + "weight_block_size": [ + 128, + 128 + ] + } +} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json new file mode 100644 index 0000000000..706cebd746 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json @@ -0,0 +1,140 @@ +{ + "architectures": [ + "Qwen3_5ForConditionalGeneration" + ], + "image_token_id": 248056, + "language_model_only": false, + "model_type": "qwen3_5", + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": 248044, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 5120, + "initializer_range": 0.02, + "intermediate_size": 17408, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 48, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 24, + "num_hidden_layers": 64, + "num_key_value_heads": 4, + "output_gate_type": "swish", + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": false, + "use_cache": true, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.8.0.dev0", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 5120, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json new file mode 100644 index 0000000000..109d798e5d --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json @@ -0,0 +1,81 @@ +{ + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 4096, + "initializer_range": 0.02, + "intermediate_size": 12288, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "max_position_embeddings": 262144, + "mlp_only_layers": [], + "model_type": "qwen3_5_text", + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "use_cache": true, + "vocab_size": 248320, + "mamba_ssm_dtype": "float32", + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "rope_type": "default", + "rope_theta": 10000000, + "partial_rotary_factor": 0.25 + }, + "architectures": [ + "Qwen3_5ForCausalLM" + ], + "tie_word_embeddings": false, + "transformers_version": "5.16.1", + "modification_notice": "Modified by TokenRhythm: language-model weights fine-tuned from Qwen/Qwen3.5-9B; repackaged for text-only inference by changing config and tensor key prefixes. Tensor values are unchanged by repackaging. Original model: Copyright 2026 Alibaba Cloud." +} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json new file mode 100644 index 0000000000..b42385da33 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json @@ -0,0 +1,365 @@ +{ + "architectures": [ + "DeepseekV41ForCausalLM" + ], + "bos_token_id": 0, + "dtype": "bfloat16", + "eos_token_id": 1, + "image_token_id": 129264, + "model_type": "deepseek_v41", + "pad_token_id": 2, + "quantization_config": { + "activation_scheme": "dynamic", + "config_groups": { + "group_0": { + "input_activations": { + "dynamic": false, + "group_size": 16, + "num_bits": 4, + "type": "float" + }, + "targets": [ + "Linear" + ], + "weights": { + "dynamic": false, + "group_size": 16, + "num_bits": 4, + "type": "float" + } + } + }, + "expert_dtype": "fp4", + "group_size": 16, + "ignore": [ + "*.attn.*", + "*.ffn.shared_experts.*", + "head", + "mtp.*" + ], + "kv_cache_quant_algo": null, + "moe_quant_algo": "NVFP4", + "producer": { + "name": "modelopt", + "version": "dsv4-nvfp4-experts" + }, + "quant_algo": "MIXED_PRECISION", + "quant_method": "fp8", + "quantized_layers": { + "layers.0.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.1.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.10.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.11.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.12.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.13.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.14.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.15.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.16.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.17.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.18.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.19.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.2.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.20.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.21.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.22.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.23.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.24.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.25.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.26.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.27.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.28.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.29.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.3.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.30.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.31.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.32.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.33.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.34.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.35.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.36.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.37.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.38.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.39.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.4.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.5.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.6.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.7.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.8.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.9.ffn.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + } + }, + "scale_fmt": "ue8m0", + "weight_block_size": [ + 32, + 32 + ] + }, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "candidate_block_size": 8, + "candidate_source_layer_id": 20, + "candidate_topk_blocks": 2048, + "compress_ratios": [ + 0, + 0, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 2, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 1, + 0, + 0, + 0 + ], + "compress_rope_theta": 160000, + "dspark_block_size": 5, + "dspark_markov_rank": 256, + "dspark_n_routed_experts": 128, + "dspark_noise_token_id": 128799, + "dspark_num_experts_per_tok": 3, + "dspark_target_layer_ids": [ + 37, + 38, + 39 + ], + "engram_compressed_vocab_size": 99092, + "engram_head_dim": 256, + "engram_layer_ids": [ + 1, + 14 + ], + "engram_max_ngram_size": 4, + "engram_n_heads": 8, + "engram_num_embeddings": [ + 384006168, + 384016682 + ], + "engram_pad_token_id": 2, + "engram_vocab_size": 16000000, + "hc_eps": 1e-06, + "hc_mult": 4, + "hc_sinkhorn_iters": 20, + "head_dim": 512, + "hidden_act": "silu", + "hidden_size": 5120, + "index_head_dim": 128, + "index_n_heads": 32, + "index_source_layer_ids": [ + 2, + 8, + 14, + 20, + 24, + 28, + 32, + 36 + ], + "index_topk": 512, + "initializer_range": 0.02, + "kv_source_layer_ids": [ + 2, + 8, + 14, + 20 + ], + "max_position_embeddings": 1048576, + "model_type": "deepseek_v41_text", + "moe_intermediate_size": 2304, + "n_routed_experts": 384, + "n_shared_experts": 1, + "norm_topk_prob": true, + "num_attention_heads": 64, + "num_experts_per_tok": 6, + "num_hidden_layers": 40, + "num_key_value_heads": 1, + "num_nextn_predict_layers": 3, + "o_groups": 8, + "o_lora_rank": 1024, + "q_lora_rank": 1280, + "qk_rope_head_dim": 64, + "rms_norm_eps": 1e-20, + "rope_scaling": { + "beta_fast": 32, + "beta_slow": 1, + "factor": 16, + "original_max_position_embeddings": 65536, + "rope_type": "yarn" + }, + "rope_theta": 10000, + "routed_scaling_factor": 1.5, + "scoring_func": "sqrtsoftplus", + "sliding_window": 128, + "swiglu_limit": 10.0, + "tie_word_embeddings": false, + "topk_method": "noaux_tc", + "use_cache": true, + "vocab_size": 129280 + }, + "transformers_version": "5.6.0", + "vision_config": { + "downsample_ratio": 3, + "hidden_size": 1024, + "intermediate_size": 2816, + "max_image_tokens": 1024, + "max_wh_ratio": null, + "min_pixels": 295936, + "model_type": "deepseek_v41_vision", + "num_attention_heads": 16, + "num_hidden_layers": 32, + "patch_size": 14, + "rope_theta": 10000 + } +} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json new file mode 100644 index 0000000000..91a7f2a4f3 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json @@ -0,0 +1,179 @@ +{ + "producer": { + "name": "modelopt", + "version": "dsv4-nvfp4-experts" + }, + "quantization": { + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": null, + "group_size": 16, + "quantized_layers": { + "layers.0.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.1.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.10.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.11.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.12.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.13.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.14.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.15.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.16.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.17.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.18.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.19.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.2.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.20.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.21.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.22.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.23.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.24.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.25.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.26.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.27.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.28.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.29.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.3.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.30.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.31.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.32.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.33.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.34.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.35.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.36.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.37.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.38.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.39.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.4.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.5.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.6.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.7.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.8.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "layers.9.ffn.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + } + }, + "exclude_modules": [ + "*.attn.*", + "*.ffn.shared_experts.*", + "head", + "mtp.*" + ] + } +} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json new file mode 100644 index 0000000000..0e4b41bbe8 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json @@ -0,0 +1,14241 @@ +{ + "architectures": [ + "KimiK3ForConditionalGeneration" + ], + "auto_map": { + "AutoConfig": "configuration_kimi_k3.KimiK3Config", + "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration", + "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration" + }, + "bos_token_id": 163584, + "dtype": "bfloat16", + "eos_token_id": 163586, + "ignore_index": -100, + "image_placeholder": "<|kimi_image_placeholder|>", + "media_placeholder_token_id": 163605, + "model_type": "kimi_k3", + "pad_token_id": 163839, + "quantization_config": { + "config_groups": { + "group_0": { + "input_activations": { + "dynamic": false, + "group_size": 16, + "num_bits": 4, + "type": "float" + }, + "targets": [ + "language_model.model.layers.1.block_sparse_moe.experts", + "language_model.model.layers.1.mlp.experts", + "language_model.model.layers.10.block_sparse_moe.experts", + "language_model.model.layers.10.mlp.experts", + "language_model.model.layers.11.block_sparse_moe.experts", + "language_model.model.layers.11.mlp.experts", + "language_model.model.layers.12.block_sparse_moe.experts", + "language_model.model.layers.12.mlp.experts", + "language_model.model.layers.13.block_sparse_moe.experts", + "language_model.model.layers.13.mlp.experts", + "language_model.model.layers.14.block_sparse_moe.experts", + "language_model.model.layers.14.mlp.experts", + "language_model.model.layers.15.block_sparse_moe.experts", + "language_model.model.layers.15.mlp.experts", + "language_model.model.layers.16.block_sparse_moe.experts", + "language_model.model.layers.16.mlp.experts", + "language_model.model.layers.17.block_sparse_moe.experts", + "language_model.model.layers.17.mlp.experts", + "language_model.model.layers.18.block_sparse_moe.experts", + "language_model.model.layers.18.mlp.experts", + "language_model.model.layers.19.block_sparse_moe.experts", + "language_model.model.layers.19.mlp.experts", + "language_model.model.layers.2.block_sparse_moe.experts", + "language_model.model.layers.2.mlp.experts", + "language_model.model.layers.20.block_sparse_moe.experts", + "language_model.model.layers.20.mlp.experts", + "language_model.model.layers.21.block_sparse_moe.experts", + "language_model.model.layers.21.mlp.experts", + "language_model.model.layers.22.block_sparse_moe.experts", + "language_model.model.layers.22.mlp.experts", + "language_model.model.layers.23.block_sparse_moe.experts", + "language_model.model.layers.23.mlp.experts", + "language_model.model.layers.24.block_sparse_moe.experts", + "language_model.model.layers.24.mlp.experts", + "language_model.model.layers.25.block_sparse_moe.experts", + "language_model.model.layers.25.mlp.experts", + "language_model.model.layers.26.block_sparse_moe.experts", + "language_model.model.layers.26.mlp.experts", + "language_model.model.layers.27.block_sparse_moe.experts", + "language_model.model.layers.27.mlp.experts", + "language_model.model.layers.28.block_sparse_moe.experts", + "language_model.model.layers.28.mlp.experts", + "language_model.model.layers.29.block_sparse_moe.experts", + "language_model.model.layers.29.mlp.experts", + "language_model.model.layers.3.block_sparse_moe.experts", + "language_model.model.layers.3.mlp.experts", + "language_model.model.layers.30.block_sparse_moe.experts", + "language_model.model.layers.30.mlp.experts", + "language_model.model.layers.31.block_sparse_moe.experts", + "language_model.model.layers.31.mlp.experts", + "language_model.model.layers.32.block_sparse_moe.experts", + "language_model.model.layers.32.mlp.experts", + "language_model.model.layers.33.block_sparse_moe.experts", + "language_model.model.layers.33.mlp.experts", + "language_model.model.layers.34.block_sparse_moe.experts", + "language_model.model.layers.34.mlp.experts", + "language_model.model.layers.35.block_sparse_moe.experts", + "language_model.model.layers.35.mlp.experts", + "language_model.model.layers.36.block_sparse_moe.experts", + "language_model.model.layers.36.mlp.experts", + "language_model.model.layers.37.block_sparse_moe.experts", + "language_model.model.layers.37.mlp.experts", + "language_model.model.layers.38.block_sparse_moe.experts", + "language_model.model.layers.38.mlp.experts", + "language_model.model.layers.39.block_sparse_moe.experts", + "language_model.model.layers.39.mlp.experts", + "language_model.model.layers.4.block_sparse_moe.experts", + "language_model.model.layers.4.mlp.experts", + "language_model.model.layers.40.block_sparse_moe.experts", + "language_model.model.layers.40.mlp.experts", + "language_model.model.layers.41.block_sparse_moe.experts", + "language_model.model.layers.41.mlp.experts", + "language_model.model.layers.42.block_sparse_moe.experts", + "language_model.model.layers.42.mlp.experts", + "language_model.model.layers.43.block_sparse_moe.experts", + "language_model.model.layers.43.mlp.experts", + "language_model.model.layers.44.block_sparse_moe.experts", + "language_model.model.layers.44.mlp.experts", + "language_model.model.layers.45.block_sparse_moe.experts", + "language_model.model.layers.45.mlp.experts", + "language_model.model.layers.46.block_sparse_moe.experts", + "language_model.model.layers.46.mlp.experts", + "language_model.model.layers.47.block_sparse_moe.experts", + "language_model.model.layers.47.mlp.experts", + "language_model.model.layers.48.block_sparse_moe.experts", + "language_model.model.layers.48.mlp.experts", + "language_model.model.layers.49.block_sparse_moe.experts", + "language_model.model.layers.49.mlp.experts", + "language_model.model.layers.5.block_sparse_moe.experts", + "language_model.model.layers.5.mlp.experts", + "language_model.model.layers.50.block_sparse_moe.experts", + "language_model.model.layers.50.mlp.experts", + "language_model.model.layers.51.block_sparse_moe.experts", + "language_model.model.layers.51.mlp.experts", + "language_model.model.layers.52.block_sparse_moe.experts", + "language_model.model.layers.52.mlp.experts", + "language_model.model.layers.53.block_sparse_moe.experts", + "language_model.model.layers.53.mlp.experts", + "language_model.model.layers.54.block_sparse_moe.experts", + "language_model.model.layers.54.mlp.experts", + "language_model.model.layers.55.block_sparse_moe.experts", + "language_model.model.layers.55.mlp.experts", + "language_model.model.layers.56.block_sparse_moe.experts", + "language_model.model.layers.56.mlp.experts", + "language_model.model.layers.57.block_sparse_moe.experts", + "language_model.model.layers.57.mlp.experts", + "language_model.model.layers.58.block_sparse_moe.experts", + "language_model.model.layers.58.mlp.experts", + "language_model.model.layers.59.block_sparse_moe.experts", + "language_model.model.layers.59.mlp.experts", + "language_model.model.layers.6.block_sparse_moe.experts", + "language_model.model.layers.6.mlp.experts", + "language_model.model.layers.60.block_sparse_moe.experts", + "language_model.model.layers.60.mlp.experts", + "language_model.model.layers.61.block_sparse_moe.experts", + "language_model.model.layers.61.mlp.experts", + "language_model.model.layers.62.block_sparse_moe.experts", + "language_model.model.layers.62.mlp.experts", + "language_model.model.layers.63.block_sparse_moe.experts", + "language_model.model.layers.63.mlp.experts", + "language_model.model.layers.64.block_sparse_moe.experts", + "language_model.model.layers.64.mlp.experts", + "language_model.model.layers.65.block_sparse_moe.experts", + "language_model.model.layers.65.mlp.experts", + "language_model.model.layers.66.block_sparse_moe.experts", + "language_model.model.layers.66.mlp.experts", + "language_model.model.layers.67.block_sparse_moe.experts", + "language_model.model.layers.67.mlp.experts", + "language_model.model.layers.68.block_sparse_moe.experts", + "language_model.model.layers.68.mlp.experts", + "language_model.model.layers.69.block_sparse_moe.experts", + "language_model.model.layers.69.mlp.experts", + "language_model.model.layers.7.block_sparse_moe.experts", + "language_model.model.layers.7.mlp.experts", + "language_model.model.layers.70.block_sparse_moe.experts", + "language_model.model.layers.70.mlp.experts", + "language_model.model.layers.71.block_sparse_moe.experts", + "language_model.model.layers.71.mlp.experts", + "language_model.model.layers.72.block_sparse_moe.experts", + "language_model.model.layers.72.mlp.experts", + "language_model.model.layers.73.block_sparse_moe.experts", + "language_model.model.layers.73.mlp.experts", + "language_model.model.layers.74.block_sparse_moe.experts", + "language_model.model.layers.74.mlp.experts", + "language_model.model.layers.75.block_sparse_moe.experts", + "language_model.model.layers.75.mlp.experts", + "language_model.model.layers.76.block_sparse_moe.experts", + "language_model.model.layers.76.mlp.experts", + "language_model.model.layers.77.block_sparse_moe.experts", + "language_model.model.layers.77.mlp.experts", + "language_model.model.layers.78.block_sparse_moe.experts", + "language_model.model.layers.78.mlp.experts", + "language_model.model.layers.79.block_sparse_moe.experts", + "language_model.model.layers.79.mlp.experts", + "language_model.model.layers.8.block_sparse_moe.experts", + "language_model.model.layers.8.mlp.experts", + "language_model.model.layers.80.block_sparse_moe.experts", + "language_model.model.layers.80.mlp.experts", + "language_model.model.layers.81.block_sparse_moe.experts", + "language_model.model.layers.81.mlp.experts", + "language_model.model.layers.82.block_sparse_moe.experts", + "language_model.model.layers.82.mlp.experts", + "language_model.model.layers.83.block_sparse_moe.experts", + "language_model.model.layers.83.mlp.experts", + "language_model.model.layers.84.block_sparse_moe.experts", + "language_model.model.layers.84.mlp.experts", + "language_model.model.layers.85.block_sparse_moe.experts", + "language_model.model.layers.85.mlp.experts", + "language_model.model.layers.86.block_sparse_moe.experts", + "language_model.model.layers.86.mlp.experts", + "language_model.model.layers.87.block_sparse_moe.experts", + "language_model.model.layers.87.mlp.experts", + "language_model.model.layers.88.block_sparse_moe.experts", + "language_model.model.layers.88.mlp.experts", + "language_model.model.layers.89.block_sparse_moe.experts", + "language_model.model.layers.89.mlp.experts", + "language_model.model.layers.9.block_sparse_moe.experts", + "language_model.model.layers.9.mlp.experts", + "language_model.model.layers.90.block_sparse_moe.experts", + "language_model.model.layers.90.mlp.experts", + "language_model.model.layers.91.block_sparse_moe.experts", + "language_model.model.layers.91.mlp.experts", + "language_model.model.layers.92.block_sparse_moe.experts", + "language_model.model.layers.92.mlp.experts", + "layers.1.block_sparse_moe.experts", + "layers.1.mlp.experts", + "layers.10.block_sparse_moe.experts", + "layers.10.mlp.experts", + "layers.11.block_sparse_moe.experts", + "layers.11.mlp.experts", + "layers.12.block_sparse_moe.experts", + "layers.12.mlp.experts", + "layers.13.block_sparse_moe.experts", + "layers.13.mlp.experts", + "layers.14.block_sparse_moe.experts", + "layers.14.mlp.experts", + "layers.15.block_sparse_moe.experts", + "layers.15.mlp.experts", + "layers.16.block_sparse_moe.experts", + "layers.16.mlp.experts", + "layers.17.block_sparse_moe.experts", + "layers.17.mlp.experts", + "layers.18.block_sparse_moe.experts", + "layers.18.mlp.experts", + "layers.19.block_sparse_moe.experts", + "layers.19.mlp.experts", + "layers.2.block_sparse_moe.experts", + "layers.2.mlp.experts", + "layers.20.block_sparse_moe.experts", + "layers.20.mlp.experts", + "layers.21.block_sparse_moe.experts", + "layers.21.mlp.experts", + "layers.22.block_sparse_moe.experts", + "layers.22.mlp.experts", + "layers.23.block_sparse_moe.experts", + "layers.23.mlp.experts", + "layers.24.block_sparse_moe.experts", + "layers.24.mlp.experts", + "layers.25.block_sparse_moe.experts", + "layers.25.mlp.experts", + "layers.26.block_sparse_moe.experts", + "layers.26.mlp.experts", + "layers.27.block_sparse_moe.experts", + "layers.27.mlp.experts", + "layers.28.block_sparse_moe.experts", + "layers.28.mlp.experts", + "layers.29.block_sparse_moe.experts", + "layers.29.mlp.experts", + "layers.3.block_sparse_moe.experts", + "layers.3.mlp.experts", + "layers.30.block_sparse_moe.experts", + "layers.30.mlp.experts", + "layers.31.block_sparse_moe.experts", + "layers.31.mlp.experts", + "layers.32.block_sparse_moe.experts", + "layers.32.mlp.experts", + "layers.33.block_sparse_moe.experts", + "layers.33.mlp.experts", + "layers.34.block_sparse_moe.experts", + "layers.34.mlp.experts", + "layers.35.block_sparse_moe.experts", + "layers.35.mlp.experts", + "layers.36.block_sparse_moe.experts", + "layers.36.mlp.experts", + "layers.37.block_sparse_moe.experts", + "layers.37.mlp.experts", + "layers.38.block_sparse_moe.experts", + "layers.38.mlp.experts", + "layers.39.block_sparse_moe.experts", + "layers.39.mlp.experts", + "layers.4.block_sparse_moe.experts", + "layers.4.mlp.experts", + "layers.40.block_sparse_moe.experts", + "layers.40.mlp.experts", + "layers.41.block_sparse_moe.experts", + "layers.41.mlp.experts", + "layers.42.block_sparse_moe.experts", + "layers.42.mlp.experts", + "layers.43.block_sparse_moe.experts", + "layers.43.mlp.experts", + "layers.44.block_sparse_moe.experts", + "layers.44.mlp.experts", + "layers.45.block_sparse_moe.experts", + "layers.45.mlp.experts", + "layers.46.block_sparse_moe.experts", + "layers.46.mlp.experts", + "layers.47.block_sparse_moe.experts", + "layers.47.mlp.experts", + "layers.48.block_sparse_moe.experts", + "layers.48.mlp.experts", + "layers.49.block_sparse_moe.experts", + "layers.49.mlp.experts", + "layers.5.block_sparse_moe.experts", + "layers.5.mlp.experts", + "layers.50.block_sparse_moe.experts", + "layers.50.mlp.experts", + "layers.51.block_sparse_moe.experts", + "layers.51.mlp.experts", + "layers.52.block_sparse_moe.experts", + "layers.52.mlp.experts", + "layers.53.block_sparse_moe.experts", + "layers.53.mlp.experts", + "layers.54.block_sparse_moe.experts", + "layers.54.mlp.experts", + "layers.55.block_sparse_moe.experts", + "layers.55.mlp.experts", + "layers.56.block_sparse_moe.experts", + "layers.56.mlp.experts", + "layers.57.block_sparse_moe.experts", + "layers.57.mlp.experts", + "layers.58.block_sparse_moe.experts", + "layers.58.mlp.experts", + "layers.59.block_sparse_moe.experts", + "layers.59.mlp.experts", + "layers.6.block_sparse_moe.experts", + "layers.6.mlp.experts", + "layers.60.block_sparse_moe.experts", + "layers.60.mlp.experts", + "layers.61.block_sparse_moe.experts", + "layers.61.mlp.experts", + "layers.62.block_sparse_moe.experts", + "layers.62.mlp.experts", + "layers.63.block_sparse_moe.experts", + "layers.63.mlp.experts", + "layers.64.block_sparse_moe.experts", + "layers.64.mlp.experts", + "layers.65.block_sparse_moe.experts", + "layers.65.mlp.experts", + "layers.66.block_sparse_moe.experts", + "layers.66.mlp.experts", + "layers.67.block_sparse_moe.experts", + "layers.67.mlp.experts", + "layers.68.block_sparse_moe.experts", + "layers.68.mlp.experts", + "layers.69.block_sparse_moe.experts", + "layers.69.mlp.experts", + "layers.7.block_sparse_moe.experts", + "layers.7.mlp.experts", + "layers.70.block_sparse_moe.experts", + "layers.70.mlp.experts", + "layers.71.block_sparse_moe.experts", + "layers.71.mlp.experts", + "layers.72.block_sparse_moe.experts", + "layers.72.mlp.experts", + "layers.73.block_sparse_moe.experts", + "layers.73.mlp.experts", + "layers.74.block_sparse_moe.experts", + "layers.74.mlp.experts", + "layers.75.block_sparse_moe.experts", + "layers.75.mlp.experts", + "layers.76.block_sparse_moe.experts", + "layers.76.mlp.experts", + "layers.77.block_sparse_moe.experts", + "layers.77.mlp.experts", + "layers.78.block_sparse_moe.experts", + "layers.78.mlp.experts", + "layers.79.block_sparse_moe.experts", + "layers.79.mlp.experts", + "layers.8.block_sparse_moe.experts", + "layers.8.mlp.experts", + "layers.80.block_sparse_moe.experts", + "layers.80.mlp.experts", + "layers.81.block_sparse_moe.experts", + "layers.81.mlp.experts", + "layers.82.block_sparse_moe.experts", + "layers.82.mlp.experts", + "layers.83.block_sparse_moe.experts", + "layers.83.mlp.experts", + "layers.84.block_sparse_moe.experts", + "layers.84.mlp.experts", + "layers.85.block_sparse_moe.experts", + "layers.85.mlp.experts", + "layers.86.block_sparse_moe.experts", + "layers.86.mlp.experts", + "layers.87.block_sparse_moe.experts", + "layers.87.mlp.experts", + "layers.88.block_sparse_moe.experts", + "layers.88.mlp.experts", + "layers.89.block_sparse_moe.experts", + "layers.89.mlp.experts", + "layers.9.block_sparse_moe.experts", + "layers.9.mlp.experts", + "layers.90.block_sparse_moe.experts", + "layers.90.mlp.experts", + "layers.91.block_sparse_moe.experts", + "layers.91.mlp.experts", + "layers.92.block_sparse_moe.experts", + "layers.92.mlp.experts", + "model.layers.1.block_sparse_moe.experts", + "model.layers.1.mlp.experts", + "model.layers.10.block_sparse_moe.experts", + "model.layers.10.mlp.experts", + "model.layers.11.block_sparse_moe.experts", + "model.layers.11.mlp.experts", + "model.layers.12.block_sparse_moe.experts", + "model.layers.12.mlp.experts", + "model.layers.13.block_sparse_moe.experts", + "model.layers.13.mlp.experts", + "model.layers.14.block_sparse_moe.experts", + "model.layers.14.mlp.experts", + "model.layers.15.block_sparse_moe.experts", + "model.layers.15.mlp.experts", + "model.layers.16.block_sparse_moe.experts", + "model.layers.16.mlp.experts", + "model.layers.17.block_sparse_moe.experts", + "model.layers.17.mlp.experts", + "model.layers.18.block_sparse_moe.experts", + "model.layers.18.mlp.experts", + "model.layers.19.block_sparse_moe.experts", + "model.layers.19.mlp.experts", + "model.layers.2.block_sparse_moe.experts", + "model.layers.2.mlp.experts", + "model.layers.20.block_sparse_moe.experts", + "model.layers.20.mlp.experts", + "model.layers.21.block_sparse_moe.experts", + "model.layers.21.mlp.experts", + "model.layers.22.block_sparse_moe.experts", + "model.layers.22.mlp.experts", + "model.layers.23.block_sparse_moe.experts", + "model.layers.23.mlp.experts", + "model.layers.24.block_sparse_moe.experts", + "model.layers.24.mlp.experts", + "model.layers.25.block_sparse_moe.experts", + "model.layers.25.mlp.experts", + "model.layers.26.block_sparse_moe.experts", + "model.layers.26.mlp.experts", + "model.layers.27.block_sparse_moe.experts", + "model.layers.27.mlp.experts", + "model.layers.28.block_sparse_moe.experts", + "model.layers.28.mlp.experts", + "model.layers.29.block_sparse_moe.experts", + "model.layers.29.mlp.experts", + "model.layers.3.block_sparse_moe.experts", + "model.layers.3.mlp.experts", + "model.layers.30.block_sparse_moe.experts", + "model.layers.30.mlp.experts", + "model.layers.31.block_sparse_moe.experts", + "model.layers.31.mlp.experts", + "model.layers.32.block_sparse_moe.experts", + "model.layers.32.mlp.experts", + "model.layers.33.block_sparse_moe.experts", + "model.layers.33.mlp.experts", + "model.layers.34.block_sparse_moe.experts", + "model.layers.34.mlp.experts", + "model.layers.35.block_sparse_moe.experts", + "model.layers.35.mlp.experts", + "model.layers.36.block_sparse_moe.experts", + "model.layers.36.mlp.experts", + "model.layers.37.block_sparse_moe.experts", + "model.layers.37.mlp.experts", + "model.layers.38.block_sparse_moe.experts", + "model.layers.38.mlp.experts", + "model.layers.39.block_sparse_moe.experts", + "model.layers.39.mlp.experts", + "model.layers.4.block_sparse_moe.experts", + "model.layers.4.mlp.experts", + "model.layers.40.block_sparse_moe.experts", + "model.layers.40.mlp.experts", + "model.layers.41.block_sparse_moe.experts", + "model.layers.41.mlp.experts", + "model.layers.42.block_sparse_moe.experts", + "model.layers.42.mlp.experts", + "model.layers.43.block_sparse_moe.experts", + "model.layers.43.mlp.experts", + "model.layers.44.block_sparse_moe.experts", + "model.layers.44.mlp.experts", + "model.layers.45.block_sparse_moe.experts", + "model.layers.45.mlp.experts", + "model.layers.46.block_sparse_moe.experts", + "model.layers.46.mlp.experts", + "model.layers.47.block_sparse_moe.experts", + "model.layers.47.mlp.experts", + "model.layers.48.block_sparse_moe.experts", + "model.layers.48.mlp.experts", + "model.layers.49.block_sparse_moe.experts", + "model.layers.49.mlp.experts", + "model.layers.5.block_sparse_moe.experts", + "model.layers.5.mlp.experts", + "model.layers.50.block_sparse_moe.experts", + "model.layers.50.mlp.experts", + "model.layers.51.block_sparse_moe.experts", + "model.layers.51.mlp.experts", + "model.layers.52.block_sparse_moe.experts", + "model.layers.52.mlp.experts", + "model.layers.53.block_sparse_moe.experts", + "model.layers.53.mlp.experts", + "model.layers.54.block_sparse_moe.experts", + "model.layers.54.mlp.experts", + "model.layers.55.block_sparse_moe.experts", + "model.layers.55.mlp.experts", + "model.layers.56.block_sparse_moe.experts", + "model.layers.56.mlp.experts", + "model.layers.57.block_sparse_moe.experts", + "model.layers.57.mlp.experts", + "model.layers.58.block_sparse_moe.experts", + "model.layers.58.mlp.experts", + "model.layers.59.block_sparse_moe.experts", + "model.layers.59.mlp.experts", + "model.layers.6.block_sparse_moe.experts", + "model.layers.6.mlp.experts", + "model.layers.60.block_sparse_moe.experts", + "model.layers.60.mlp.experts", + "model.layers.61.block_sparse_moe.experts", + "model.layers.61.mlp.experts", + "model.layers.62.block_sparse_moe.experts", + "model.layers.62.mlp.experts", + "model.layers.63.block_sparse_moe.experts", + "model.layers.63.mlp.experts", + "model.layers.64.block_sparse_moe.experts", + "model.layers.64.mlp.experts", + "model.layers.65.block_sparse_moe.experts", + "model.layers.65.mlp.experts", + "model.layers.66.block_sparse_moe.experts", + "model.layers.66.mlp.experts", + "model.layers.67.block_sparse_moe.experts", + "model.layers.67.mlp.experts", + "model.layers.68.block_sparse_moe.experts", + "model.layers.68.mlp.experts", + "model.layers.69.block_sparse_moe.experts", + "model.layers.69.mlp.experts", + "model.layers.7.block_sparse_moe.experts", + "model.layers.7.mlp.experts", + "model.layers.70.block_sparse_moe.experts", + "model.layers.70.mlp.experts", + "model.layers.71.block_sparse_moe.experts", + "model.layers.71.mlp.experts", + "model.layers.72.block_sparse_moe.experts", + "model.layers.72.mlp.experts", + "model.layers.73.block_sparse_moe.experts", + "model.layers.73.mlp.experts", + "model.layers.74.block_sparse_moe.experts", + "model.layers.74.mlp.experts", + "model.layers.75.block_sparse_moe.experts", + "model.layers.75.mlp.experts", + "model.layers.76.block_sparse_moe.experts", + "model.layers.76.mlp.experts", + "model.layers.77.block_sparse_moe.experts", + "model.layers.77.mlp.experts", + "model.layers.78.block_sparse_moe.experts", + "model.layers.78.mlp.experts", + "model.layers.79.block_sparse_moe.experts", + "model.layers.79.mlp.experts", + "model.layers.8.block_sparse_moe.experts", + "model.layers.8.mlp.experts", + "model.layers.80.block_sparse_moe.experts", + "model.layers.80.mlp.experts", + "model.layers.81.block_sparse_moe.experts", + "model.layers.81.mlp.experts", + "model.layers.82.block_sparse_moe.experts", + "model.layers.82.mlp.experts", + "model.layers.83.block_sparse_moe.experts", + "model.layers.83.mlp.experts", + "model.layers.84.block_sparse_moe.experts", + "model.layers.84.mlp.experts", + "model.layers.85.block_sparse_moe.experts", + "model.layers.85.mlp.experts", + "model.layers.86.block_sparse_moe.experts", + "model.layers.86.mlp.experts", + "model.layers.87.block_sparse_moe.experts", + "model.layers.87.mlp.experts", + "model.layers.88.block_sparse_moe.experts", + "model.layers.88.mlp.experts", + "model.layers.89.block_sparse_moe.experts", + "model.layers.89.mlp.experts", + "model.layers.9.block_sparse_moe.experts", + "model.layers.9.mlp.experts", + "model.layers.90.block_sparse_moe.experts", + "model.layers.90.mlp.experts", + "model.layers.91.block_sparse_moe.experts", + "model.layers.91.mlp.experts", + "model.layers.92.block_sparse_moe.experts", + "model.layers.92.mlp.experts" + ], + "weights": { + "dynamic": false, + "group_size": 16, + "num_bits": 4, + "type": "float" + } + }, + "group_1": { + "quant_algo": "FP8_PB_WO", + "targets": [ + "language_model.model.layers.0.self_attn.b_proj", + "language_model.model.layers.0.self_attn.f_a_proj", + "language_model.model.layers.0.self_attn.f_b_proj", + "language_model.model.layers.0.self_attn.fused_qkvg_proj", + "language_model.model.layers.0.self_attn.g_proj", + "language_model.model.layers.0.self_attn.in_proj_qkvgfab", + "language_model.model.layers.0.self_attn.k_proj", + "language_model.model.layers.0.self_attn.o_proj", + "language_model.model.layers.0.self_attn.q_proj", + "language_model.model.layers.0.self_attn.v_proj", + "language_model.model.layers.1.self_attn.b_proj", + "language_model.model.layers.1.self_attn.f_a_proj", + "language_model.model.layers.1.self_attn.f_b_proj", + "language_model.model.layers.1.self_attn.fused_qkvg_proj", + "language_model.model.layers.1.self_attn.g_proj", + "language_model.model.layers.1.self_attn.in_proj_qkvgfab", + "language_model.model.layers.1.self_attn.k_proj", + "language_model.model.layers.1.self_attn.o_proj", + "language_model.model.layers.1.self_attn.q_proj", + "language_model.model.layers.1.self_attn.v_proj", + "language_model.model.layers.10.self_attn.b_proj", + "language_model.model.layers.10.self_attn.f_a_proj", + "language_model.model.layers.10.self_attn.f_b_proj", + "language_model.model.layers.10.self_attn.fused_qkvg_proj", + "language_model.model.layers.10.self_attn.g_proj", + "language_model.model.layers.10.self_attn.in_proj_qkvgfab", + "language_model.model.layers.10.self_attn.k_proj", + "language_model.model.layers.10.self_attn.o_proj", + "language_model.model.layers.10.self_attn.q_proj", + "language_model.model.layers.10.self_attn.v_proj", + "language_model.model.layers.11.self_attn.fused_qkv_a_proj", + "language_model.model.layers.11.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.11.self_attn.fused_qkvg_proj", + "language_model.model.layers.11.self_attn.g_proj", + "language_model.model.layers.11.self_attn.in_proj_qkvgfab", + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.11.self_attn.kv_b_proj", + "language_model.model.layers.11.self_attn.o_proj", + "language_model.model.layers.11.self_attn.q_a_proj", + "language_model.model.layers.11.self_attn.q_b_proj", + "language_model.model.layers.12.self_attn.b_proj", + "language_model.model.layers.12.self_attn.f_a_proj", + "language_model.model.layers.12.self_attn.f_b_proj", + "language_model.model.layers.12.self_attn.fused_qkvg_proj", + "language_model.model.layers.12.self_attn.g_proj", + "language_model.model.layers.12.self_attn.in_proj_qkvgfab", + "language_model.model.layers.12.self_attn.k_proj", + "language_model.model.layers.12.self_attn.o_proj", + "language_model.model.layers.12.self_attn.q_proj", + "language_model.model.layers.12.self_attn.v_proj", + "language_model.model.layers.13.self_attn.b_proj", + "language_model.model.layers.13.self_attn.f_a_proj", + "language_model.model.layers.13.self_attn.f_b_proj", + "language_model.model.layers.13.self_attn.fused_qkvg_proj", + "language_model.model.layers.13.self_attn.g_proj", + "language_model.model.layers.13.self_attn.in_proj_qkvgfab", + "language_model.model.layers.13.self_attn.k_proj", + "language_model.model.layers.13.self_attn.o_proj", + "language_model.model.layers.13.self_attn.q_proj", + "language_model.model.layers.13.self_attn.v_proj", + "language_model.model.layers.14.self_attn.b_proj", + "language_model.model.layers.14.self_attn.f_a_proj", + "language_model.model.layers.14.self_attn.f_b_proj", + "language_model.model.layers.14.self_attn.fused_qkvg_proj", + "language_model.model.layers.14.self_attn.g_proj", + "language_model.model.layers.14.self_attn.in_proj_qkvgfab", + "language_model.model.layers.14.self_attn.k_proj", + "language_model.model.layers.14.self_attn.o_proj", + "language_model.model.layers.14.self_attn.q_proj", + "language_model.model.layers.14.self_attn.v_proj", + "language_model.model.layers.15.self_attn.fused_qkv_a_proj", + "language_model.model.layers.15.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.15.self_attn.fused_qkvg_proj", + "language_model.model.layers.15.self_attn.g_proj", + "language_model.model.layers.15.self_attn.in_proj_qkvgfab", + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.15.self_attn.kv_b_proj", + "language_model.model.layers.15.self_attn.o_proj", + "language_model.model.layers.15.self_attn.q_a_proj", + "language_model.model.layers.15.self_attn.q_b_proj", + "language_model.model.layers.16.self_attn.b_proj", + "language_model.model.layers.16.self_attn.f_a_proj", + "language_model.model.layers.16.self_attn.f_b_proj", + "language_model.model.layers.16.self_attn.fused_qkvg_proj", + "language_model.model.layers.16.self_attn.g_proj", + "language_model.model.layers.16.self_attn.in_proj_qkvgfab", + "language_model.model.layers.16.self_attn.k_proj", + "language_model.model.layers.16.self_attn.o_proj", + "language_model.model.layers.16.self_attn.q_proj", + "language_model.model.layers.16.self_attn.v_proj", + "language_model.model.layers.17.self_attn.b_proj", + "language_model.model.layers.17.self_attn.f_a_proj", + "language_model.model.layers.17.self_attn.f_b_proj", + "language_model.model.layers.17.self_attn.fused_qkvg_proj", + "language_model.model.layers.17.self_attn.g_proj", + "language_model.model.layers.17.self_attn.in_proj_qkvgfab", + "language_model.model.layers.17.self_attn.k_proj", + "language_model.model.layers.17.self_attn.o_proj", + "language_model.model.layers.17.self_attn.q_proj", + "language_model.model.layers.17.self_attn.v_proj", + "language_model.model.layers.18.self_attn.b_proj", + "language_model.model.layers.18.self_attn.f_a_proj", + "language_model.model.layers.18.self_attn.f_b_proj", + "language_model.model.layers.18.self_attn.fused_qkvg_proj", + "language_model.model.layers.18.self_attn.g_proj", + "language_model.model.layers.18.self_attn.in_proj_qkvgfab", + "language_model.model.layers.18.self_attn.k_proj", + "language_model.model.layers.18.self_attn.o_proj", + "language_model.model.layers.18.self_attn.q_proj", + "language_model.model.layers.18.self_attn.v_proj", + "language_model.model.layers.19.self_attn.fused_qkv_a_proj", + "language_model.model.layers.19.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.19.self_attn.fused_qkvg_proj", + "language_model.model.layers.19.self_attn.g_proj", + "language_model.model.layers.19.self_attn.in_proj_qkvgfab", + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.19.self_attn.kv_b_proj", + "language_model.model.layers.19.self_attn.o_proj", + "language_model.model.layers.19.self_attn.q_a_proj", + "language_model.model.layers.19.self_attn.q_b_proj", + "language_model.model.layers.2.self_attn.b_proj", + "language_model.model.layers.2.self_attn.f_a_proj", + "language_model.model.layers.2.self_attn.f_b_proj", + "language_model.model.layers.2.self_attn.fused_qkvg_proj", + "language_model.model.layers.2.self_attn.g_proj", + "language_model.model.layers.2.self_attn.in_proj_qkvgfab", + "language_model.model.layers.2.self_attn.k_proj", + "language_model.model.layers.2.self_attn.o_proj", + "language_model.model.layers.2.self_attn.q_proj", + "language_model.model.layers.2.self_attn.v_proj", + "language_model.model.layers.20.self_attn.b_proj", + "language_model.model.layers.20.self_attn.f_a_proj", + "language_model.model.layers.20.self_attn.f_b_proj", + "language_model.model.layers.20.self_attn.fused_qkvg_proj", + "language_model.model.layers.20.self_attn.g_proj", + "language_model.model.layers.20.self_attn.in_proj_qkvgfab", + "language_model.model.layers.20.self_attn.k_proj", + "language_model.model.layers.20.self_attn.o_proj", + "language_model.model.layers.20.self_attn.q_proj", + "language_model.model.layers.20.self_attn.v_proj", + "language_model.model.layers.21.self_attn.b_proj", + "language_model.model.layers.21.self_attn.f_a_proj", + "language_model.model.layers.21.self_attn.f_b_proj", + "language_model.model.layers.21.self_attn.fused_qkvg_proj", + "language_model.model.layers.21.self_attn.g_proj", + "language_model.model.layers.21.self_attn.in_proj_qkvgfab", + "language_model.model.layers.21.self_attn.k_proj", + "language_model.model.layers.21.self_attn.o_proj", + "language_model.model.layers.21.self_attn.q_proj", + "language_model.model.layers.21.self_attn.v_proj", + "language_model.model.layers.22.self_attn.b_proj", + "language_model.model.layers.22.self_attn.f_a_proj", + "language_model.model.layers.22.self_attn.f_b_proj", + "language_model.model.layers.22.self_attn.fused_qkvg_proj", + "language_model.model.layers.22.self_attn.g_proj", + "language_model.model.layers.22.self_attn.in_proj_qkvgfab", + "language_model.model.layers.22.self_attn.k_proj", + "language_model.model.layers.22.self_attn.o_proj", + "language_model.model.layers.22.self_attn.q_proj", + "language_model.model.layers.22.self_attn.v_proj", + "language_model.model.layers.23.self_attn.fused_qkv_a_proj", + "language_model.model.layers.23.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.23.self_attn.fused_qkvg_proj", + "language_model.model.layers.23.self_attn.g_proj", + "language_model.model.layers.23.self_attn.in_proj_qkvgfab", + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.23.self_attn.kv_b_proj", + "language_model.model.layers.23.self_attn.o_proj", + "language_model.model.layers.23.self_attn.q_a_proj", + "language_model.model.layers.23.self_attn.q_b_proj", + "language_model.model.layers.24.self_attn.b_proj", + "language_model.model.layers.24.self_attn.f_a_proj", + "language_model.model.layers.24.self_attn.f_b_proj", + "language_model.model.layers.24.self_attn.fused_qkvg_proj", + "language_model.model.layers.24.self_attn.g_proj", + "language_model.model.layers.24.self_attn.in_proj_qkvgfab", + "language_model.model.layers.24.self_attn.k_proj", + "language_model.model.layers.24.self_attn.o_proj", + "language_model.model.layers.24.self_attn.q_proj", + "language_model.model.layers.24.self_attn.v_proj", + "language_model.model.layers.25.self_attn.b_proj", + "language_model.model.layers.25.self_attn.f_a_proj", + "language_model.model.layers.25.self_attn.f_b_proj", + "language_model.model.layers.25.self_attn.fused_qkvg_proj", + "language_model.model.layers.25.self_attn.g_proj", + "language_model.model.layers.25.self_attn.in_proj_qkvgfab", + "language_model.model.layers.25.self_attn.k_proj", + "language_model.model.layers.25.self_attn.o_proj", + "language_model.model.layers.25.self_attn.q_proj", + "language_model.model.layers.25.self_attn.v_proj", + "language_model.model.layers.26.self_attn.b_proj", + "language_model.model.layers.26.self_attn.f_a_proj", + "language_model.model.layers.26.self_attn.f_b_proj", + "language_model.model.layers.26.self_attn.fused_qkvg_proj", + "language_model.model.layers.26.self_attn.g_proj", + "language_model.model.layers.26.self_attn.in_proj_qkvgfab", + "language_model.model.layers.26.self_attn.k_proj", + "language_model.model.layers.26.self_attn.o_proj", + "language_model.model.layers.26.self_attn.q_proj", + "language_model.model.layers.26.self_attn.v_proj", + "language_model.model.layers.27.self_attn.fused_qkv_a_proj", + "language_model.model.layers.27.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.27.self_attn.fused_qkvg_proj", + "language_model.model.layers.27.self_attn.g_proj", + "language_model.model.layers.27.self_attn.in_proj_qkvgfab", + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.27.self_attn.kv_b_proj", + "language_model.model.layers.27.self_attn.o_proj", + "language_model.model.layers.27.self_attn.q_a_proj", + "language_model.model.layers.27.self_attn.q_b_proj", + "language_model.model.layers.28.self_attn.b_proj", + "language_model.model.layers.28.self_attn.f_a_proj", + "language_model.model.layers.28.self_attn.f_b_proj", + "language_model.model.layers.28.self_attn.fused_qkvg_proj", + "language_model.model.layers.28.self_attn.g_proj", + "language_model.model.layers.28.self_attn.in_proj_qkvgfab", + "language_model.model.layers.28.self_attn.k_proj", + "language_model.model.layers.28.self_attn.o_proj", + "language_model.model.layers.28.self_attn.q_proj", + "language_model.model.layers.28.self_attn.v_proj", + "language_model.model.layers.29.self_attn.b_proj", + "language_model.model.layers.29.self_attn.f_a_proj", + "language_model.model.layers.29.self_attn.f_b_proj", + "language_model.model.layers.29.self_attn.fused_qkvg_proj", + "language_model.model.layers.29.self_attn.g_proj", + "language_model.model.layers.29.self_attn.in_proj_qkvgfab", + "language_model.model.layers.29.self_attn.k_proj", + "language_model.model.layers.29.self_attn.o_proj", + "language_model.model.layers.29.self_attn.q_proj", + "language_model.model.layers.29.self_attn.v_proj", + "language_model.model.layers.3.self_attn.fused_qkv_a_proj", + "language_model.model.layers.3.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.3.self_attn.fused_qkvg_proj", + "language_model.model.layers.3.self_attn.g_proj", + "language_model.model.layers.3.self_attn.in_proj_qkvgfab", + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.3.self_attn.kv_b_proj", + "language_model.model.layers.3.self_attn.o_proj", + "language_model.model.layers.3.self_attn.q_a_proj", + "language_model.model.layers.3.self_attn.q_b_proj", + "language_model.model.layers.30.self_attn.b_proj", + "language_model.model.layers.30.self_attn.f_a_proj", + "language_model.model.layers.30.self_attn.f_b_proj", + "language_model.model.layers.30.self_attn.fused_qkvg_proj", + "language_model.model.layers.30.self_attn.g_proj", + "language_model.model.layers.30.self_attn.in_proj_qkvgfab", + "language_model.model.layers.30.self_attn.k_proj", + "language_model.model.layers.30.self_attn.o_proj", + "language_model.model.layers.30.self_attn.q_proj", + "language_model.model.layers.30.self_attn.v_proj", + "language_model.model.layers.31.self_attn.fused_qkv_a_proj", + "language_model.model.layers.31.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.31.self_attn.fused_qkvg_proj", + "language_model.model.layers.31.self_attn.g_proj", + "language_model.model.layers.31.self_attn.in_proj_qkvgfab", + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.31.self_attn.kv_b_proj", + "language_model.model.layers.31.self_attn.o_proj", + "language_model.model.layers.31.self_attn.q_a_proj", + "language_model.model.layers.31.self_attn.q_b_proj", + "language_model.model.layers.32.self_attn.b_proj", + "language_model.model.layers.32.self_attn.f_a_proj", + "language_model.model.layers.32.self_attn.f_b_proj", + "language_model.model.layers.32.self_attn.fused_qkvg_proj", + "language_model.model.layers.32.self_attn.g_proj", + "language_model.model.layers.32.self_attn.in_proj_qkvgfab", + "language_model.model.layers.32.self_attn.k_proj", + "language_model.model.layers.32.self_attn.o_proj", + "language_model.model.layers.32.self_attn.q_proj", + "language_model.model.layers.32.self_attn.v_proj", + "language_model.model.layers.33.self_attn.b_proj", + "language_model.model.layers.33.self_attn.f_a_proj", + "language_model.model.layers.33.self_attn.f_b_proj", + "language_model.model.layers.33.self_attn.fused_qkvg_proj", + "language_model.model.layers.33.self_attn.g_proj", + "language_model.model.layers.33.self_attn.in_proj_qkvgfab", + "language_model.model.layers.33.self_attn.k_proj", + "language_model.model.layers.33.self_attn.o_proj", + "language_model.model.layers.33.self_attn.q_proj", + "language_model.model.layers.33.self_attn.v_proj", + "language_model.model.layers.34.self_attn.b_proj", + "language_model.model.layers.34.self_attn.f_a_proj", + "language_model.model.layers.34.self_attn.f_b_proj", + "language_model.model.layers.34.self_attn.fused_qkvg_proj", + "language_model.model.layers.34.self_attn.g_proj", + "language_model.model.layers.34.self_attn.in_proj_qkvgfab", + "language_model.model.layers.34.self_attn.k_proj", + "language_model.model.layers.34.self_attn.o_proj", + "language_model.model.layers.34.self_attn.q_proj", + "language_model.model.layers.34.self_attn.v_proj", + "language_model.model.layers.35.self_attn.fused_qkv_a_proj", + "language_model.model.layers.35.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.35.self_attn.fused_qkvg_proj", + "language_model.model.layers.35.self_attn.g_proj", + "language_model.model.layers.35.self_attn.in_proj_qkvgfab", + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.35.self_attn.kv_b_proj", + "language_model.model.layers.35.self_attn.o_proj", + "language_model.model.layers.35.self_attn.q_a_proj", + "language_model.model.layers.35.self_attn.q_b_proj", + "language_model.model.layers.36.self_attn.b_proj", + "language_model.model.layers.36.self_attn.f_a_proj", + "language_model.model.layers.36.self_attn.f_b_proj", + "language_model.model.layers.36.self_attn.fused_qkvg_proj", + "language_model.model.layers.36.self_attn.g_proj", + "language_model.model.layers.36.self_attn.in_proj_qkvgfab", + "language_model.model.layers.36.self_attn.k_proj", + "language_model.model.layers.36.self_attn.o_proj", + "language_model.model.layers.36.self_attn.q_proj", + "language_model.model.layers.36.self_attn.v_proj", + "language_model.model.layers.37.self_attn.b_proj", + "language_model.model.layers.37.self_attn.f_a_proj", + "language_model.model.layers.37.self_attn.f_b_proj", + "language_model.model.layers.37.self_attn.fused_qkvg_proj", + "language_model.model.layers.37.self_attn.g_proj", + "language_model.model.layers.37.self_attn.in_proj_qkvgfab", + "language_model.model.layers.37.self_attn.k_proj", + "language_model.model.layers.37.self_attn.o_proj", + "language_model.model.layers.37.self_attn.q_proj", + "language_model.model.layers.37.self_attn.v_proj", + "language_model.model.layers.38.self_attn.b_proj", + "language_model.model.layers.38.self_attn.f_a_proj", + "language_model.model.layers.38.self_attn.f_b_proj", + "language_model.model.layers.38.self_attn.fused_qkvg_proj", + "language_model.model.layers.38.self_attn.g_proj", + "language_model.model.layers.38.self_attn.in_proj_qkvgfab", + "language_model.model.layers.38.self_attn.k_proj", + "language_model.model.layers.38.self_attn.o_proj", + "language_model.model.layers.38.self_attn.q_proj", + "language_model.model.layers.38.self_attn.v_proj", + "language_model.model.layers.39.self_attn.fused_qkv_a_proj", + "language_model.model.layers.39.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.39.self_attn.fused_qkvg_proj", + "language_model.model.layers.39.self_attn.g_proj", + "language_model.model.layers.39.self_attn.in_proj_qkvgfab", + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.39.self_attn.kv_b_proj", + "language_model.model.layers.39.self_attn.o_proj", + "language_model.model.layers.39.self_attn.q_a_proj", + "language_model.model.layers.39.self_attn.q_b_proj", + "language_model.model.layers.4.self_attn.b_proj", + "language_model.model.layers.4.self_attn.f_a_proj", + "language_model.model.layers.4.self_attn.f_b_proj", + "language_model.model.layers.4.self_attn.fused_qkvg_proj", + "language_model.model.layers.4.self_attn.g_proj", + "language_model.model.layers.4.self_attn.in_proj_qkvgfab", + "language_model.model.layers.4.self_attn.k_proj", + "language_model.model.layers.4.self_attn.o_proj", + "language_model.model.layers.4.self_attn.q_proj", + "language_model.model.layers.4.self_attn.v_proj", + "language_model.model.layers.40.self_attn.b_proj", + "language_model.model.layers.40.self_attn.f_a_proj", + "language_model.model.layers.40.self_attn.f_b_proj", + "language_model.model.layers.40.self_attn.fused_qkvg_proj", + "language_model.model.layers.40.self_attn.g_proj", + "language_model.model.layers.40.self_attn.in_proj_qkvgfab", + "language_model.model.layers.40.self_attn.k_proj", + "language_model.model.layers.40.self_attn.o_proj", + "language_model.model.layers.40.self_attn.q_proj", + "language_model.model.layers.40.self_attn.v_proj", + "language_model.model.layers.41.self_attn.b_proj", + "language_model.model.layers.41.self_attn.f_a_proj", + "language_model.model.layers.41.self_attn.f_b_proj", + "language_model.model.layers.41.self_attn.fused_qkvg_proj", + "language_model.model.layers.41.self_attn.g_proj", + "language_model.model.layers.41.self_attn.in_proj_qkvgfab", + "language_model.model.layers.41.self_attn.k_proj", + "language_model.model.layers.41.self_attn.o_proj", + "language_model.model.layers.41.self_attn.q_proj", + "language_model.model.layers.41.self_attn.v_proj", + "language_model.model.layers.42.self_attn.b_proj", + "language_model.model.layers.42.self_attn.f_a_proj", + "language_model.model.layers.42.self_attn.f_b_proj", + "language_model.model.layers.42.self_attn.fused_qkvg_proj", + "language_model.model.layers.42.self_attn.g_proj", + "language_model.model.layers.42.self_attn.in_proj_qkvgfab", + "language_model.model.layers.42.self_attn.k_proj", + "language_model.model.layers.42.self_attn.o_proj", + "language_model.model.layers.42.self_attn.q_proj", + "language_model.model.layers.42.self_attn.v_proj", + "language_model.model.layers.43.self_attn.fused_qkv_a_proj", + "language_model.model.layers.43.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.43.self_attn.fused_qkvg_proj", + "language_model.model.layers.43.self_attn.g_proj", + "language_model.model.layers.43.self_attn.in_proj_qkvgfab", + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.43.self_attn.kv_b_proj", + "language_model.model.layers.43.self_attn.o_proj", + "language_model.model.layers.43.self_attn.q_a_proj", + "language_model.model.layers.43.self_attn.q_b_proj", + "language_model.model.layers.44.self_attn.b_proj", + "language_model.model.layers.44.self_attn.f_a_proj", + "language_model.model.layers.44.self_attn.f_b_proj", + "language_model.model.layers.44.self_attn.fused_qkvg_proj", + "language_model.model.layers.44.self_attn.g_proj", + "language_model.model.layers.44.self_attn.in_proj_qkvgfab", + "language_model.model.layers.44.self_attn.k_proj", + "language_model.model.layers.44.self_attn.o_proj", + "language_model.model.layers.44.self_attn.q_proj", + "language_model.model.layers.44.self_attn.v_proj", + "language_model.model.layers.45.self_attn.b_proj", + "language_model.model.layers.45.self_attn.f_a_proj", + "language_model.model.layers.45.self_attn.f_b_proj", + "language_model.model.layers.45.self_attn.fused_qkvg_proj", + "language_model.model.layers.45.self_attn.g_proj", + "language_model.model.layers.45.self_attn.in_proj_qkvgfab", + "language_model.model.layers.45.self_attn.k_proj", + "language_model.model.layers.45.self_attn.o_proj", + "language_model.model.layers.45.self_attn.q_proj", + "language_model.model.layers.45.self_attn.v_proj", + "language_model.model.layers.46.self_attn.b_proj", + "language_model.model.layers.46.self_attn.f_a_proj", + "language_model.model.layers.46.self_attn.f_b_proj", + "language_model.model.layers.46.self_attn.fused_qkvg_proj", + "language_model.model.layers.46.self_attn.g_proj", + "language_model.model.layers.46.self_attn.in_proj_qkvgfab", + "language_model.model.layers.46.self_attn.k_proj", + "language_model.model.layers.46.self_attn.o_proj", + "language_model.model.layers.46.self_attn.q_proj", + "language_model.model.layers.46.self_attn.v_proj", + "language_model.model.layers.47.self_attn.fused_qkv_a_proj", + "language_model.model.layers.47.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.47.self_attn.fused_qkvg_proj", + "language_model.model.layers.47.self_attn.g_proj", + "language_model.model.layers.47.self_attn.in_proj_qkvgfab", + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.47.self_attn.kv_b_proj", + "language_model.model.layers.47.self_attn.o_proj", + "language_model.model.layers.47.self_attn.q_a_proj", + "language_model.model.layers.47.self_attn.q_b_proj", + "language_model.model.layers.48.self_attn.b_proj", + "language_model.model.layers.48.self_attn.f_a_proj", + "language_model.model.layers.48.self_attn.f_b_proj", + "language_model.model.layers.48.self_attn.fused_qkvg_proj", + "language_model.model.layers.48.self_attn.g_proj", + "language_model.model.layers.48.self_attn.in_proj_qkvgfab", + "language_model.model.layers.48.self_attn.k_proj", + "language_model.model.layers.48.self_attn.o_proj", + "language_model.model.layers.48.self_attn.q_proj", + "language_model.model.layers.48.self_attn.v_proj", + "language_model.model.layers.49.self_attn.b_proj", + "language_model.model.layers.49.self_attn.f_a_proj", + "language_model.model.layers.49.self_attn.f_b_proj", + "language_model.model.layers.49.self_attn.fused_qkvg_proj", + "language_model.model.layers.49.self_attn.g_proj", + "language_model.model.layers.49.self_attn.in_proj_qkvgfab", + "language_model.model.layers.49.self_attn.k_proj", + "language_model.model.layers.49.self_attn.o_proj", + "language_model.model.layers.49.self_attn.q_proj", + "language_model.model.layers.49.self_attn.v_proj", + "language_model.model.layers.5.self_attn.b_proj", + "language_model.model.layers.5.self_attn.f_a_proj", + "language_model.model.layers.5.self_attn.f_b_proj", + "language_model.model.layers.5.self_attn.fused_qkvg_proj", + "language_model.model.layers.5.self_attn.g_proj", + "language_model.model.layers.5.self_attn.in_proj_qkvgfab", + "language_model.model.layers.5.self_attn.k_proj", + "language_model.model.layers.5.self_attn.o_proj", + "language_model.model.layers.5.self_attn.q_proj", + "language_model.model.layers.5.self_attn.v_proj", + "language_model.model.layers.50.self_attn.b_proj", + "language_model.model.layers.50.self_attn.f_a_proj", + "language_model.model.layers.50.self_attn.f_b_proj", + "language_model.model.layers.50.self_attn.fused_qkvg_proj", + "language_model.model.layers.50.self_attn.g_proj", + "language_model.model.layers.50.self_attn.in_proj_qkvgfab", + "language_model.model.layers.50.self_attn.k_proj", + "language_model.model.layers.50.self_attn.o_proj", + "language_model.model.layers.50.self_attn.q_proj", + "language_model.model.layers.50.self_attn.v_proj", + "language_model.model.layers.51.self_attn.fused_qkv_a_proj", + "language_model.model.layers.51.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.51.self_attn.fused_qkvg_proj", + "language_model.model.layers.51.self_attn.g_proj", + "language_model.model.layers.51.self_attn.in_proj_qkvgfab", + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.51.self_attn.kv_b_proj", + "language_model.model.layers.51.self_attn.o_proj", + "language_model.model.layers.51.self_attn.q_a_proj", + "language_model.model.layers.51.self_attn.q_b_proj", + "language_model.model.layers.52.self_attn.b_proj", + "language_model.model.layers.52.self_attn.f_a_proj", + "language_model.model.layers.52.self_attn.f_b_proj", + "language_model.model.layers.52.self_attn.fused_qkvg_proj", + "language_model.model.layers.52.self_attn.g_proj", + "language_model.model.layers.52.self_attn.in_proj_qkvgfab", + "language_model.model.layers.52.self_attn.k_proj", + "language_model.model.layers.52.self_attn.o_proj", + "language_model.model.layers.52.self_attn.q_proj", + "language_model.model.layers.52.self_attn.v_proj", + "language_model.model.layers.53.self_attn.b_proj", + "language_model.model.layers.53.self_attn.f_a_proj", + "language_model.model.layers.53.self_attn.f_b_proj", + "language_model.model.layers.53.self_attn.fused_qkvg_proj", + "language_model.model.layers.53.self_attn.g_proj", + "language_model.model.layers.53.self_attn.in_proj_qkvgfab", + "language_model.model.layers.53.self_attn.k_proj", + "language_model.model.layers.53.self_attn.o_proj", + "language_model.model.layers.53.self_attn.q_proj", + "language_model.model.layers.53.self_attn.v_proj", + "language_model.model.layers.54.self_attn.b_proj", + "language_model.model.layers.54.self_attn.f_a_proj", + "language_model.model.layers.54.self_attn.f_b_proj", + "language_model.model.layers.54.self_attn.fused_qkvg_proj", + "language_model.model.layers.54.self_attn.g_proj", + "language_model.model.layers.54.self_attn.in_proj_qkvgfab", + "language_model.model.layers.54.self_attn.k_proj", + "language_model.model.layers.54.self_attn.o_proj", + "language_model.model.layers.54.self_attn.q_proj", + "language_model.model.layers.54.self_attn.v_proj", + "language_model.model.layers.55.self_attn.fused_qkv_a_proj", + "language_model.model.layers.55.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.55.self_attn.fused_qkvg_proj", + "language_model.model.layers.55.self_attn.g_proj", + "language_model.model.layers.55.self_attn.in_proj_qkvgfab", + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.55.self_attn.kv_b_proj", + "language_model.model.layers.55.self_attn.o_proj", + "language_model.model.layers.55.self_attn.q_a_proj", + "language_model.model.layers.55.self_attn.q_b_proj", + "language_model.model.layers.56.self_attn.b_proj", + "language_model.model.layers.56.self_attn.f_a_proj", + "language_model.model.layers.56.self_attn.f_b_proj", + "language_model.model.layers.56.self_attn.fused_qkvg_proj", + "language_model.model.layers.56.self_attn.g_proj", + "language_model.model.layers.56.self_attn.in_proj_qkvgfab", + "language_model.model.layers.56.self_attn.k_proj", + "language_model.model.layers.56.self_attn.o_proj", + "language_model.model.layers.56.self_attn.q_proj", + "language_model.model.layers.56.self_attn.v_proj", + "language_model.model.layers.57.self_attn.b_proj", + "language_model.model.layers.57.self_attn.f_a_proj", + "language_model.model.layers.57.self_attn.f_b_proj", + "language_model.model.layers.57.self_attn.fused_qkvg_proj", + "language_model.model.layers.57.self_attn.g_proj", + "language_model.model.layers.57.self_attn.in_proj_qkvgfab", + "language_model.model.layers.57.self_attn.k_proj", + "language_model.model.layers.57.self_attn.o_proj", + "language_model.model.layers.57.self_attn.q_proj", + "language_model.model.layers.57.self_attn.v_proj", + "language_model.model.layers.58.self_attn.b_proj", + "language_model.model.layers.58.self_attn.f_a_proj", + "language_model.model.layers.58.self_attn.f_b_proj", + "language_model.model.layers.58.self_attn.fused_qkvg_proj", + "language_model.model.layers.58.self_attn.g_proj", + "language_model.model.layers.58.self_attn.in_proj_qkvgfab", + "language_model.model.layers.58.self_attn.k_proj", + "language_model.model.layers.58.self_attn.o_proj", + "language_model.model.layers.58.self_attn.q_proj", + "language_model.model.layers.58.self_attn.v_proj", + "language_model.model.layers.59.self_attn.fused_qkv_a_proj", + "language_model.model.layers.59.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.59.self_attn.fused_qkvg_proj", + "language_model.model.layers.59.self_attn.g_proj", + "language_model.model.layers.59.self_attn.in_proj_qkvgfab", + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.59.self_attn.kv_b_proj", + "language_model.model.layers.59.self_attn.o_proj", + "language_model.model.layers.59.self_attn.q_a_proj", + "language_model.model.layers.59.self_attn.q_b_proj", + "language_model.model.layers.6.self_attn.b_proj", + "language_model.model.layers.6.self_attn.f_a_proj", + "language_model.model.layers.6.self_attn.f_b_proj", + "language_model.model.layers.6.self_attn.fused_qkvg_proj", + "language_model.model.layers.6.self_attn.g_proj", + "language_model.model.layers.6.self_attn.in_proj_qkvgfab", + "language_model.model.layers.6.self_attn.k_proj", + "language_model.model.layers.6.self_attn.o_proj", + "language_model.model.layers.6.self_attn.q_proj", + "language_model.model.layers.6.self_attn.v_proj", + "language_model.model.layers.60.self_attn.b_proj", + "language_model.model.layers.60.self_attn.f_a_proj", + "language_model.model.layers.60.self_attn.f_b_proj", + "language_model.model.layers.60.self_attn.fused_qkvg_proj", + "language_model.model.layers.60.self_attn.g_proj", + "language_model.model.layers.60.self_attn.in_proj_qkvgfab", + "language_model.model.layers.60.self_attn.k_proj", + "language_model.model.layers.60.self_attn.o_proj", + "language_model.model.layers.60.self_attn.q_proj", + "language_model.model.layers.60.self_attn.v_proj", + "language_model.model.layers.61.self_attn.b_proj", + "language_model.model.layers.61.self_attn.f_a_proj", + "language_model.model.layers.61.self_attn.f_b_proj", + "language_model.model.layers.61.self_attn.fused_qkvg_proj", + "language_model.model.layers.61.self_attn.g_proj", + "language_model.model.layers.61.self_attn.in_proj_qkvgfab", + "language_model.model.layers.61.self_attn.k_proj", + "language_model.model.layers.61.self_attn.o_proj", + "language_model.model.layers.61.self_attn.q_proj", + "language_model.model.layers.61.self_attn.v_proj", + "language_model.model.layers.62.self_attn.b_proj", + "language_model.model.layers.62.self_attn.f_a_proj", + "language_model.model.layers.62.self_attn.f_b_proj", + "language_model.model.layers.62.self_attn.fused_qkvg_proj", + "language_model.model.layers.62.self_attn.g_proj", + "language_model.model.layers.62.self_attn.in_proj_qkvgfab", + "language_model.model.layers.62.self_attn.k_proj", + "language_model.model.layers.62.self_attn.o_proj", + "language_model.model.layers.62.self_attn.q_proj", + "language_model.model.layers.62.self_attn.v_proj", + "language_model.model.layers.63.self_attn.fused_qkv_a_proj", + "language_model.model.layers.63.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.63.self_attn.fused_qkvg_proj", + "language_model.model.layers.63.self_attn.g_proj", + "language_model.model.layers.63.self_attn.in_proj_qkvgfab", + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.63.self_attn.kv_b_proj", + "language_model.model.layers.63.self_attn.o_proj", + "language_model.model.layers.63.self_attn.q_a_proj", + "language_model.model.layers.63.self_attn.q_b_proj", + "language_model.model.layers.64.self_attn.b_proj", + "language_model.model.layers.64.self_attn.f_a_proj", + "language_model.model.layers.64.self_attn.f_b_proj", + "language_model.model.layers.64.self_attn.fused_qkvg_proj", + "language_model.model.layers.64.self_attn.g_proj", + "language_model.model.layers.64.self_attn.in_proj_qkvgfab", + "language_model.model.layers.64.self_attn.k_proj", + "language_model.model.layers.64.self_attn.o_proj", + "language_model.model.layers.64.self_attn.q_proj", + "language_model.model.layers.64.self_attn.v_proj", + "language_model.model.layers.65.self_attn.b_proj", + "language_model.model.layers.65.self_attn.f_a_proj", + "language_model.model.layers.65.self_attn.f_b_proj", + "language_model.model.layers.65.self_attn.fused_qkvg_proj", + "language_model.model.layers.65.self_attn.g_proj", + "language_model.model.layers.65.self_attn.in_proj_qkvgfab", + "language_model.model.layers.65.self_attn.k_proj", + "language_model.model.layers.65.self_attn.o_proj", + "language_model.model.layers.65.self_attn.q_proj", + "language_model.model.layers.65.self_attn.v_proj", + "language_model.model.layers.66.self_attn.b_proj", + "language_model.model.layers.66.self_attn.f_a_proj", + "language_model.model.layers.66.self_attn.f_b_proj", + "language_model.model.layers.66.self_attn.fused_qkvg_proj", + "language_model.model.layers.66.self_attn.g_proj", + "language_model.model.layers.66.self_attn.in_proj_qkvgfab", + "language_model.model.layers.66.self_attn.k_proj", + "language_model.model.layers.66.self_attn.o_proj", + "language_model.model.layers.66.self_attn.q_proj", + "language_model.model.layers.66.self_attn.v_proj", + "language_model.model.layers.67.self_attn.fused_qkv_a_proj", + "language_model.model.layers.67.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.67.self_attn.fused_qkvg_proj", + "language_model.model.layers.67.self_attn.g_proj", + "language_model.model.layers.67.self_attn.in_proj_qkvgfab", + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.67.self_attn.kv_b_proj", + "language_model.model.layers.67.self_attn.o_proj", + "language_model.model.layers.67.self_attn.q_a_proj", + "language_model.model.layers.67.self_attn.q_b_proj", + "language_model.model.layers.68.self_attn.b_proj", + "language_model.model.layers.68.self_attn.f_a_proj", + "language_model.model.layers.68.self_attn.f_b_proj", + "language_model.model.layers.68.self_attn.fused_qkvg_proj", + "language_model.model.layers.68.self_attn.g_proj", + "language_model.model.layers.68.self_attn.in_proj_qkvgfab", + "language_model.model.layers.68.self_attn.k_proj", + "language_model.model.layers.68.self_attn.o_proj", + "language_model.model.layers.68.self_attn.q_proj", + "language_model.model.layers.68.self_attn.v_proj", + "language_model.model.layers.69.self_attn.b_proj", + "language_model.model.layers.69.self_attn.f_a_proj", + "language_model.model.layers.69.self_attn.f_b_proj", + "language_model.model.layers.69.self_attn.fused_qkvg_proj", + "language_model.model.layers.69.self_attn.g_proj", + "language_model.model.layers.69.self_attn.in_proj_qkvgfab", + "language_model.model.layers.69.self_attn.k_proj", + "language_model.model.layers.69.self_attn.o_proj", + "language_model.model.layers.69.self_attn.q_proj", + "language_model.model.layers.69.self_attn.v_proj", + "language_model.model.layers.7.self_attn.fused_qkv_a_proj", + "language_model.model.layers.7.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.7.self_attn.fused_qkvg_proj", + "language_model.model.layers.7.self_attn.g_proj", + "language_model.model.layers.7.self_attn.in_proj_qkvgfab", + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.7.self_attn.kv_b_proj", + "language_model.model.layers.7.self_attn.o_proj", + "language_model.model.layers.7.self_attn.q_a_proj", + "language_model.model.layers.7.self_attn.q_b_proj", + "language_model.model.layers.70.self_attn.b_proj", + "language_model.model.layers.70.self_attn.f_a_proj", + "language_model.model.layers.70.self_attn.f_b_proj", + "language_model.model.layers.70.self_attn.fused_qkvg_proj", + "language_model.model.layers.70.self_attn.g_proj", + "language_model.model.layers.70.self_attn.in_proj_qkvgfab", + "language_model.model.layers.70.self_attn.k_proj", + "language_model.model.layers.70.self_attn.o_proj", + "language_model.model.layers.70.self_attn.q_proj", + "language_model.model.layers.70.self_attn.v_proj", + "language_model.model.layers.71.self_attn.fused_qkv_a_proj", + "language_model.model.layers.71.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.71.self_attn.fused_qkvg_proj", + "language_model.model.layers.71.self_attn.g_proj", + "language_model.model.layers.71.self_attn.in_proj_qkvgfab", + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.71.self_attn.kv_b_proj", + "language_model.model.layers.71.self_attn.o_proj", + "language_model.model.layers.71.self_attn.q_a_proj", + "language_model.model.layers.71.self_attn.q_b_proj", + "language_model.model.layers.72.self_attn.b_proj", + "language_model.model.layers.72.self_attn.f_a_proj", + "language_model.model.layers.72.self_attn.f_b_proj", + "language_model.model.layers.72.self_attn.fused_qkvg_proj", + "language_model.model.layers.72.self_attn.g_proj", + "language_model.model.layers.72.self_attn.in_proj_qkvgfab", + "language_model.model.layers.72.self_attn.k_proj", + "language_model.model.layers.72.self_attn.o_proj", + "language_model.model.layers.72.self_attn.q_proj", + "language_model.model.layers.72.self_attn.v_proj", + "language_model.model.layers.73.self_attn.b_proj", + "language_model.model.layers.73.self_attn.f_a_proj", + "language_model.model.layers.73.self_attn.f_b_proj", + "language_model.model.layers.73.self_attn.fused_qkvg_proj", + "language_model.model.layers.73.self_attn.g_proj", + "language_model.model.layers.73.self_attn.in_proj_qkvgfab", + "language_model.model.layers.73.self_attn.k_proj", + "language_model.model.layers.73.self_attn.o_proj", + "language_model.model.layers.73.self_attn.q_proj", + "language_model.model.layers.73.self_attn.v_proj", + "language_model.model.layers.74.self_attn.b_proj", + "language_model.model.layers.74.self_attn.f_a_proj", + "language_model.model.layers.74.self_attn.f_b_proj", + "language_model.model.layers.74.self_attn.fused_qkvg_proj", + "language_model.model.layers.74.self_attn.g_proj", + "language_model.model.layers.74.self_attn.in_proj_qkvgfab", + "language_model.model.layers.74.self_attn.k_proj", + "language_model.model.layers.74.self_attn.o_proj", + "language_model.model.layers.74.self_attn.q_proj", + "language_model.model.layers.74.self_attn.v_proj", + "language_model.model.layers.75.self_attn.fused_qkv_a_proj", + "language_model.model.layers.75.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.75.self_attn.fused_qkvg_proj", + "language_model.model.layers.75.self_attn.g_proj", + "language_model.model.layers.75.self_attn.in_proj_qkvgfab", + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.75.self_attn.kv_b_proj", + "language_model.model.layers.75.self_attn.o_proj", + "language_model.model.layers.75.self_attn.q_a_proj", + "language_model.model.layers.75.self_attn.q_b_proj", + "language_model.model.layers.76.self_attn.b_proj", + "language_model.model.layers.76.self_attn.f_a_proj", + "language_model.model.layers.76.self_attn.f_b_proj", + "language_model.model.layers.76.self_attn.fused_qkvg_proj", + "language_model.model.layers.76.self_attn.g_proj", + "language_model.model.layers.76.self_attn.in_proj_qkvgfab", + "language_model.model.layers.76.self_attn.k_proj", + "language_model.model.layers.76.self_attn.o_proj", + "language_model.model.layers.76.self_attn.q_proj", + "language_model.model.layers.76.self_attn.v_proj", + "language_model.model.layers.77.self_attn.b_proj", + "language_model.model.layers.77.self_attn.f_a_proj", + "language_model.model.layers.77.self_attn.f_b_proj", + "language_model.model.layers.77.self_attn.fused_qkvg_proj", + "language_model.model.layers.77.self_attn.g_proj", + "language_model.model.layers.77.self_attn.in_proj_qkvgfab", + "language_model.model.layers.77.self_attn.k_proj", + "language_model.model.layers.77.self_attn.o_proj", + "language_model.model.layers.77.self_attn.q_proj", + "language_model.model.layers.77.self_attn.v_proj", + "language_model.model.layers.78.self_attn.b_proj", + "language_model.model.layers.78.self_attn.f_a_proj", + "language_model.model.layers.78.self_attn.f_b_proj", + "language_model.model.layers.78.self_attn.fused_qkvg_proj", + "language_model.model.layers.78.self_attn.g_proj", + "language_model.model.layers.78.self_attn.in_proj_qkvgfab", + "language_model.model.layers.78.self_attn.k_proj", + "language_model.model.layers.78.self_attn.o_proj", + "language_model.model.layers.78.self_attn.q_proj", + "language_model.model.layers.78.self_attn.v_proj", + "language_model.model.layers.79.self_attn.fused_qkv_a_proj", + "language_model.model.layers.79.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.79.self_attn.fused_qkvg_proj", + "language_model.model.layers.79.self_attn.g_proj", + "language_model.model.layers.79.self_attn.in_proj_qkvgfab", + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.79.self_attn.kv_b_proj", + "language_model.model.layers.79.self_attn.o_proj", + "language_model.model.layers.79.self_attn.q_a_proj", + "language_model.model.layers.79.self_attn.q_b_proj", + "language_model.model.layers.8.self_attn.b_proj", + "language_model.model.layers.8.self_attn.f_a_proj", + "language_model.model.layers.8.self_attn.f_b_proj", + "language_model.model.layers.8.self_attn.fused_qkvg_proj", + "language_model.model.layers.8.self_attn.g_proj", + "language_model.model.layers.8.self_attn.in_proj_qkvgfab", + "language_model.model.layers.8.self_attn.k_proj", + "language_model.model.layers.8.self_attn.o_proj", + "language_model.model.layers.8.self_attn.q_proj", + "language_model.model.layers.8.self_attn.v_proj", + "language_model.model.layers.80.self_attn.b_proj", + "language_model.model.layers.80.self_attn.f_a_proj", + "language_model.model.layers.80.self_attn.f_b_proj", + "language_model.model.layers.80.self_attn.fused_qkvg_proj", + "language_model.model.layers.80.self_attn.g_proj", + "language_model.model.layers.80.self_attn.in_proj_qkvgfab", + "language_model.model.layers.80.self_attn.k_proj", + "language_model.model.layers.80.self_attn.o_proj", + "language_model.model.layers.80.self_attn.q_proj", + "language_model.model.layers.80.self_attn.v_proj", + "language_model.model.layers.81.self_attn.b_proj", + "language_model.model.layers.81.self_attn.f_a_proj", + "language_model.model.layers.81.self_attn.f_b_proj", + "language_model.model.layers.81.self_attn.fused_qkvg_proj", + "language_model.model.layers.81.self_attn.g_proj", + "language_model.model.layers.81.self_attn.in_proj_qkvgfab", + "language_model.model.layers.81.self_attn.k_proj", + "language_model.model.layers.81.self_attn.o_proj", + "language_model.model.layers.81.self_attn.q_proj", + "language_model.model.layers.81.self_attn.v_proj", + "language_model.model.layers.82.self_attn.b_proj", + "language_model.model.layers.82.self_attn.f_a_proj", + "language_model.model.layers.82.self_attn.f_b_proj", + "language_model.model.layers.82.self_attn.fused_qkvg_proj", + "language_model.model.layers.82.self_attn.g_proj", + "language_model.model.layers.82.self_attn.in_proj_qkvgfab", + "language_model.model.layers.82.self_attn.k_proj", + "language_model.model.layers.82.self_attn.o_proj", + "language_model.model.layers.82.self_attn.q_proj", + "language_model.model.layers.82.self_attn.v_proj", + "language_model.model.layers.83.self_attn.fused_qkv_a_proj", + "language_model.model.layers.83.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.83.self_attn.fused_qkvg_proj", + "language_model.model.layers.83.self_attn.g_proj", + "language_model.model.layers.83.self_attn.in_proj_qkvgfab", + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.83.self_attn.kv_b_proj", + "language_model.model.layers.83.self_attn.o_proj", + "language_model.model.layers.83.self_attn.q_a_proj", + "language_model.model.layers.83.self_attn.q_b_proj", + "language_model.model.layers.84.self_attn.b_proj", + "language_model.model.layers.84.self_attn.f_a_proj", + "language_model.model.layers.84.self_attn.f_b_proj", + "language_model.model.layers.84.self_attn.fused_qkvg_proj", + "language_model.model.layers.84.self_attn.g_proj", + "language_model.model.layers.84.self_attn.in_proj_qkvgfab", + "language_model.model.layers.84.self_attn.k_proj", + "language_model.model.layers.84.self_attn.o_proj", + "language_model.model.layers.84.self_attn.q_proj", + "language_model.model.layers.84.self_attn.v_proj", + "language_model.model.layers.85.self_attn.b_proj", + "language_model.model.layers.85.self_attn.f_a_proj", + "language_model.model.layers.85.self_attn.f_b_proj", + "language_model.model.layers.85.self_attn.fused_qkvg_proj", + "language_model.model.layers.85.self_attn.g_proj", + "language_model.model.layers.85.self_attn.in_proj_qkvgfab", + "language_model.model.layers.85.self_attn.k_proj", + "language_model.model.layers.85.self_attn.o_proj", + "language_model.model.layers.85.self_attn.q_proj", + "language_model.model.layers.85.self_attn.v_proj", + "language_model.model.layers.86.self_attn.b_proj", + "language_model.model.layers.86.self_attn.f_a_proj", + "language_model.model.layers.86.self_attn.f_b_proj", + "language_model.model.layers.86.self_attn.fused_qkvg_proj", + "language_model.model.layers.86.self_attn.g_proj", + "language_model.model.layers.86.self_attn.in_proj_qkvgfab", + "language_model.model.layers.86.self_attn.k_proj", + "language_model.model.layers.86.self_attn.o_proj", + "language_model.model.layers.86.self_attn.q_proj", + "language_model.model.layers.86.self_attn.v_proj", + "language_model.model.layers.87.self_attn.fused_qkv_a_proj", + "language_model.model.layers.87.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.87.self_attn.fused_qkvg_proj", + "language_model.model.layers.87.self_attn.g_proj", + "language_model.model.layers.87.self_attn.in_proj_qkvgfab", + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.87.self_attn.kv_b_proj", + "language_model.model.layers.87.self_attn.o_proj", + "language_model.model.layers.87.self_attn.q_a_proj", + "language_model.model.layers.87.self_attn.q_b_proj", + "language_model.model.layers.88.self_attn.b_proj", + "language_model.model.layers.88.self_attn.f_a_proj", + "language_model.model.layers.88.self_attn.f_b_proj", + "language_model.model.layers.88.self_attn.fused_qkvg_proj", + "language_model.model.layers.88.self_attn.g_proj", + "language_model.model.layers.88.self_attn.in_proj_qkvgfab", + "language_model.model.layers.88.self_attn.k_proj", + "language_model.model.layers.88.self_attn.o_proj", + "language_model.model.layers.88.self_attn.q_proj", + "language_model.model.layers.88.self_attn.v_proj", + "language_model.model.layers.89.self_attn.b_proj", + "language_model.model.layers.89.self_attn.f_a_proj", + "language_model.model.layers.89.self_attn.f_b_proj", + "language_model.model.layers.89.self_attn.fused_qkvg_proj", + "language_model.model.layers.89.self_attn.g_proj", + "language_model.model.layers.89.self_attn.in_proj_qkvgfab", + "language_model.model.layers.89.self_attn.k_proj", + "language_model.model.layers.89.self_attn.o_proj", + "language_model.model.layers.89.self_attn.q_proj", + "language_model.model.layers.89.self_attn.v_proj", + "language_model.model.layers.9.self_attn.b_proj", + "language_model.model.layers.9.self_attn.f_a_proj", + "language_model.model.layers.9.self_attn.f_b_proj", + "language_model.model.layers.9.self_attn.fused_qkvg_proj", + "language_model.model.layers.9.self_attn.g_proj", + "language_model.model.layers.9.self_attn.in_proj_qkvgfab", + "language_model.model.layers.9.self_attn.k_proj", + "language_model.model.layers.9.self_attn.o_proj", + "language_model.model.layers.9.self_attn.q_proj", + "language_model.model.layers.9.self_attn.v_proj", + "language_model.model.layers.90.self_attn.b_proj", + "language_model.model.layers.90.self_attn.f_a_proj", + "language_model.model.layers.90.self_attn.f_b_proj", + "language_model.model.layers.90.self_attn.fused_qkvg_proj", + "language_model.model.layers.90.self_attn.g_proj", + "language_model.model.layers.90.self_attn.in_proj_qkvgfab", + "language_model.model.layers.90.self_attn.k_proj", + "language_model.model.layers.90.self_attn.o_proj", + "language_model.model.layers.90.self_attn.q_proj", + "language_model.model.layers.90.self_attn.v_proj", + "language_model.model.layers.91.self_attn.fused_qkv_a_proj", + "language_model.model.layers.91.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.91.self_attn.fused_qkvg_proj", + "language_model.model.layers.91.self_attn.g_proj", + "language_model.model.layers.91.self_attn.in_proj_qkvgfab", + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.91.self_attn.kv_b_proj", + "language_model.model.layers.91.self_attn.o_proj", + "language_model.model.layers.91.self_attn.q_a_proj", + "language_model.model.layers.91.self_attn.q_b_proj", + "language_model.model.layers.92.self_attn.fused_qkv_a_proj", + "language_model.model.layers.92.self_attn.fused_qkv_a_proj_with_mqa", + "language_model.model.layers.92.self_attn.fused_qkvg_proj", + "language_model.model.layers.92.self_attn.g_proj", + "language_model.model.layers.92.self_attn.in_proj_qkvgfab", + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa", + "language_model.model.layers.92.self_attn.kv_b_proj", + "language_model.model.layers.92.self_attn.o_proj", + "language_model.model.layers.92.self_attn.q_a_proj", + "language_model.model.layers.92.self_attn.q_b_proj", + "layers.0.self_attn.b_proj", + "layers.0.self_attn.f_a_proj", + "layers.0.self_attn.f_b_proj", + "layers.0.self_attn.fused_qkvg_proj", + "layers.0.self_attn.g_proj", + "layers.0.self_attn.in_proj_qkvgfab", + "layers.0.self_attn.k_proj", + "layers.0.self_attn.o_proj", + "layers.0.self_attn.q_proj", + "layers.0.self_attn.v_proj", + "layers.1.self_attn.b_proj", + "layers.1.self_attn.f_a_proj", + "layers.1.self_attn.f_b_proj", + "layers.1.self_attn.fused_qkvg_proj", + "layers.1.self_attn.g_proj", + "layers.1.self_attn.in_proj_qkvgfab", + "layers.1.self_attn.k_proj", + "layers.1.self_attn.o_proj", + "layers.1.self_attn.q_proj", + "layers.1.self_attn.v_proj", + "layers.10.self_attn.b_proj", + "layers.10.self_attn.f_a_proj", + "layers.10.self_attn.f_b_proj", + "layers.10.self_attn.fused_qkvg_proj", + "layers.10.self_attn.g_proj", + "layers.10.self_attn.in_proj_qkvgfab", + "layers.10.self_attn.k_proj", + "layers.10.self_attn.o_proj", + "layers.10.self_attn.q_proj", + "layers.10.self_attn.v_proj", + "layers.11.self_attn.fused_qkv_a_proj", + "layers.11.self_attn.fused_qkv_a_proj_with_mqa", + "layers.11.self_attn.fused_qkvg_proj", + "layers.11.self_attn.g_proj", + "layers.11.self_attn.in_proj_qkvgfab", + "layers.11.self_attn.kv_a_proj_with_mqa", + "layers.11.self_attn.kv_b_proj", + "layers.11.self_attn.o_proj", + "layers.11.self_attn.q_a_proj", + "layers.11.self_attn.q_b_proj", + "layers.12.self_attn.b_proj", + "layers.12.self_attn.f_a_proj", + "layers.12.self_attn.f_b_proj", + "layers.12.self_attn.fused_qkvg_proj", + "layers.12.self_attn.g_proj", + "layers.12.self_attn.in_proj_qkvgfab", + "layers.12.self_attn.k_proj", + "layers.12.self_attn.o_proj", + "layers.12.self_attn.q_proj", + "layers.12.self_attn.v_proj", + "layers.13.self_attn.b_proj", + "layers.13.self_attn.f_a_proj", + "layers.13.self_attn.f_b_proj", + "layers.13.self_attn.fused_qkvg_proj", + "layers.13.self_attn.g_proj", + "layers.13.self_attn.in_proj_qkvgfab", + "layers.13.self_attn.k_proj", + "layers.13.self_attn.o_proj", + "layers.13.self_attn.q_proj", + "layers.13.self_attn.v_proj", + "layers.14.self_attn.b_proj", + "layers.14.self_attn.f_a_proj", + "layers.14.self_attn.f_b_proj", + "layers.14.self_attn.fused_qkvg_proj", + "layers.14.self_attn.g_proj", + "layers.14.self_attn.in_proj_qkvgfab", + "layers.14.self_attn.k_proj", + "layers.14.self_attn.o_proj", + "layers.14.self_attn.q_proj", + "layers.14.self_attn.v_proj", + "layers.15.self_attn.fused_qkv_a_proj", + "layers.15.self_attn.fused_qkv_a_proj_with_mqa", + "layers.15.self_attn.fused_qkvg_proj", + "layers.15.self_attn.g_proj", + "layers.15.self_attn.in_proj_qkvgfab", + "layers.15.self_attn.kv_a_proj_with_mqa", + "layers.15.self_attn.kv_b_proj", + "layers.15.self_attn.o_proj", + "layers.15.self_attn.q_a_proj", + "layers.15.self_attn.q_b_proj", + "layers.16.self_attn.b_proj", + "layers.16.self_attn.f_a_proj", + "layers.16.self_attn.f_b_proj", + "layers.16.self_attn.fused_qkvg_proj", + "layers.16.self_attn.g_proj", + "layers.16.self_attn.in_proj_qkvgfab", + "layers.16.self_attn.k_proj", + "layers.16.self_attn.o_proj", + "layers.16.self_attn.q_proj", + "layers.16.self_attn.v_proj", + "layers.17.self_attn.b_proj", + "layers.17.self_attn.f_a_proj", + "layers.17.self_attn.f_b_proj", + "layers.17.self_attn.fused_qkvg_proj", + "layers.17.self_attn.g_proj", + "layers.17.self_attn.in_proj_qkvgfab", + "layers.17.self_attn.k_proj", + "layers.17.self_attn.o_proj", + "layers.17.self_attn.q_proj", + "layers.17.self_attn.v_proj", + "layers.18.self_attn.b_proj", + "layers.18.self_attn.f_a_proj", + "layers.18.self_attn.f_b_proj", + "layers.18.self_attn.fused_qkvg_proj", + "layers.18.self_attn.g_proj", + "layers.18.self_attn.in_proj_qkvgfab", + "layers.18.self_attn.k_proj", + "layers.18.self_attn.o_proj", + "layers.18.self_attn.q_proj", + "layers.18.self_attn.v_proj", + "layers.19.self_attn.fused_qkv_a_proj", + "layers.19.self_attn.fused_qkv_a_proj_with_mqa", + "layers.19.self_attn.fused_qkvg_proj", + "layers.19.self_attn.g_proj", + "layers.19.self_attn.in_proj_qkvgfab", + "layers.19.self_attn.kv_a_proj_with_mqa", + "layers.19.self_attn.kv_b_proj", + "layers.19.self_attn.o_proj", + "layers.19.self_attn.q_a_proj", + "layers.19.self_attn.q_b_proj", + "layers.2.self_attn.b_proj", + "layers.2.self_attn.f_a_proj", + "layers.2.self_attn.f_b_proj", + "layers.2.self_attn.fused_qkvg_proj", + "layers.2.self_attn.g_proj", + "layers.2.self_attn.in_proj_qkvgfab", + "layers.2.self_attn.k_proj", + "layers.2.self_attn.o_proj", + "layers.2.self_attn.q_proj", + "layers.2.self_attn.v_proj", + "layers.20.self_attn.b_proj", + "layers.20.self_attn.f_a_proj", + "layers.20.self_attn.f_b_proj", + "layers.20.self_attn.fused_qkvg_proj", + "layers.20.self_attn.g_proj", + "layers.20.self_attn.in_proj_qkvgfab", + "layers.20.self_attn.k_proj", + "layers.20.self_attn.o_proj", + "layers.20.self_attn.q_proj", + "layers.20.self_attn.v_proj", + "layers.21.self_attn.b_proj", + "layers.21.self_attn.f_a_proj", + "layers.21.self_attn.f_b_proj", + "layers.21.self_attn.fused_qkvg_proj", + "layers.21.self_attn.g_proj", + "layers.21.self_attn.in_proj_qkvgfab", + "layers.21.self_attn.k_proj", + "layers.21.self_attn.o_proj", + "layers.21.self_attn.q_proj", + "layers.21.self_attn.v_proj", + "layers.22.self_attn.b_proj", + "layers.22.self_attn.f_a_proj", + "layers.22.self_attn.f_b_proj", + "layers.22.self_attn.fused_qkvg_proj", + "layers.22.self_attn.g_proj", + "layers.22.self_attn.in_proj_qkvgfab", + "layers.22.self_attn.k_proj", + "layers.22.self_attn.o_proj", + "layers.22.self_attn.q_proj", + "layers.22.self_attn.v_proj", + "layers.23.self_attn.fused_qkv_a_proj", + "layers.23.self_attn.fused_qkv_a_proj_with_mqa", + "layers.23.self_attn.fused_qkvg_proj", + "layers.23.self_attn.g_proj", + "layers.23.self_attn.in_proj_qkvgfab", + "layers.23.self_attn.kv_a_proj_with_mqa", + "layers.23.self_attn.kv_b_proj", + "layers.23.self_attn.o_proj", + "layers.23.self_attn.q_a_proj", + "layers.23.self_attn.q_b_proj", + "layers.24.self_attn.b_proj", + "layers.24.self_attn.f_a_proj", + "layers.24.self_attn.f_b_proj", + "layers.24.self_attn.fused_qkvg_proj", + "layers.24.self_attn.g_proj", + "layers.24.self_attn.in_proj_qkvgfab", + "layers.24.self_attn.k_proj", + "layers.24.self_attn.o_proj", + "layers.24.self_attn.q_proj", + "layers.24.self_attn.v_proj", + "layers.25.self_attn.b_proj", + "layers.25.self_attn.f_a_proj", + "layers.25.self_attn.f_b_proj", + "layers.25.self_attn.fused_qkvg_proj", + "layers.25.self_attn.g_proj", + "layers.25.self_attn.in_proj_qkvgfab", + "layers.25.self_attn.k_proj", + "layers.25.self_attn.o_proj", + "layers.25.self_attn.q_proj", + "layers.25.self_attn.v_proj", + "layers.26.self_attn.b_proj", + "layers.26.self_attn.f_a_proj", + "layers.26.self_attn.f_b_proj", + "layers.26.self_attn.fused_qkvg_proj", + "layers.26.self_attn.g_proj", + "layers.26.self_attn.in_proj_qkvgfab", + "layers.26.self_attn.k_proj", + "layers.26.self_attn.o_proj", + "layers.26.self_attn.q_proj", + "layers.26.self_attn.v_proj", + "layers.27.self_attn.fused_qkv_a_proj", + "layers.27.self_attn.fused_qkv_a_proj_with_mqa", + "layers.27.self_attn.fused_qkvg_proj", + "layers.27.self_attn.g_proj", + "layers.27.self_attn.in_proj_qkvgfab", + "layers.27.self_attn.kv_a_proj_with_mqa", + "layers.27.self_attn.kv_b_proj", + "layers.27.self_attn.o_proj", + "layers.27.self_attn.q_a_proj", + "layers.27.self_attn.q_b_proj", + "layers.28.self_attn.b_proj", + "layers.28.self_attn.f_a_proj", + "layers.28.self_attn.f_b_proj", + "layers.28.self_attn.fused_qkvg_proj", + "layers.28.self_attn.g_proj", + "layers.28.self_attn.in_proj_qkvgfab", + "layers.28.self_attn.k_proj", + "layers.28.self_attn.o_proj", + "layers.28.self_attn.q_proj", + "layers.28.self_attn.v_proj", + "layers.29.self_attn.b_proj", + "layers.29.self_attn.f_a_proj", + "layers.29.self_attn.f_b_proj", + "layers.29.self_attn.fused_qkvg_proj", + "layers.29.self_attn.g_proj", + "layers.29.self_attn.in_proj_qkvgfab", + "layers.29.self_attn.k_proj", + "layers.29.self_attn.o_proj", + "layers.29.self_attn.q_proj", + "layers.29.self_attn.v_proj", + "layers.3.self_attn.fused_qkv_a_proj", + "layers.3.self_attn.fused_qkv_a_proj_with_mqa", + "layers.3.self_attn.fused_qkvg_proj", + "layers.3.self_attn.g_proj", + "layers.3.self_attn.in_proj_qkvgfab", + "layers.3.self_attn.kv_a_proj_with_mqa", + "layers.3.self_attn.kv_b_proj", + "layers.3.self_attn.o_proj", + "layers.3.self_attn.q_a_proj", + "layers.3.self_attn.q_b_proj", + "layers.30.self_attn.b_proj", + "layers.30.self_attn.f_a_proj", + "layers.30.self_attn.f_b_proj", + "layers.30.self_attn.fused_qkvg_proj", + "layers.30.self_attn.g_proj", + "layers.30.self_attn.in_proj_qkvgfab", + "layers.30.self_attn.k_proj", + "layers.30.self_attn.o_proj", + "layers.30.self_attn.q_proj", + "layers.30.self_attn.v_proj", + "layers.31.self_attn.fused_qkv_a_proj", + "layers.31.self_attn.fused_qkv_a_proj_with_mqa", + "layers.31.self_attn.fused_qkvg_proj", + "layers.31.self_attn.g_proj", + "layers.31.self_attn.in_proj_qkvgfab", + "layers.31.self_attn.kv_a_proj_with_mqa", + "layers.31.self_attn.kv_b_proj", + "layers.31.self_attn.o_proj", + "layers.31.self_attn.q_a_proj", + "layers.31.self_attn.q_b_proj", + "layers.32.self_attn.b_proj", + "layers.32.self_attn.f_a_proj", + "layers.32.self_attn.f_b_proj", + "layers.32.self_attn.fused_qkvg_proj", + "layers.32.self_attn.g_proj", + "layers.32.self_attn.in_proj_qkvgfab", + "layers.32.self_attn.k_proj", + "layers.32.self_attn.o_proj", + "layers.32.self_attn.q_proj", + "layers.32.self_attn.v_proj", + "layers.33.self_attn.b_proj", + "layers.33.self_attn.f_a_proj", + "layers.33.self_attn.f_b_proj", + "layers.33.self_attn.fused_qkvg_proj", + "layers.33.self_attn.g_proj", + "layers.33.self_attn.in_proj_qkvgfab", + "layers.33.self_attn.k_proj", + "layers.33.self_attn.o_proj", + "layers.33.self_attn.q_proj", + "layers.33.self_attn.v_proj", + "layers.34.self_attn.b_proj", + "layers.34.self_attn.f_a_proj", + "layers.34.self_attn.f_b_proj", + "layers.34.self_attn.fused_qkvg_proj", + "layers.34.self_attn.g_proj", + "layers.34.self_attn.in_proj_qkvgfab", + "layers.34.self_attn.k_proj", + "layers.34.self_attn.o_proj", + "layers.34.self_attn.q_proj", + "layers.34.self_attn.v_proj", + "layers.35.self_attn.fused_qkv_a_proj", + "layers.35.self_attn.fused_qkv_a_proj_with_mqa", + "layers.35.self_attn.fused_qkvg_proj", + "layers.35.self_attn.g_proj", + "layers.35.self_attn.in_proj_qkvgfab", + "layers.35.self_attn.kv_a_proj_with_mqa", + "layers.35.self_attn.kv_b_proj", + "layers.35.self_attn.o_proj", + "layers.35.self_attn.q_a_proj", + "layers.35.self_attn.q_b_proj", + "layers.36.self_attn.b_proj", + "layers.36.self_attn.f_a_proj", + "layers.36.self_attn.f_b_proj", + "layers.36.self_attn.fused_qkvg_proj", + "layers.36.self_attn.g_proj", + "layers.36.self_attn.in_proj_qkvgfab", + "layers.36.self_attn.k_proj", + "layers.36.self_attn.o_proj", + "layers.36.self_attn.q_proj", + "layers.36.self_attn.v_proj", + "layers.37.self_attn.b_proj", + "layers.37.self_attn.f_a_proj", + "layers.37.self_attn.f_b_proj", + "layers.37.self_attn.fused_qkvg_proj", + "layers.37.self_attn.g_proj", + "layers.37.self_attn.in_proj_qkvgfab", + "layers.37.self_attn.k_proj", + "layers.37.self_attn.o_proj", + "layers.37.self_attn.q_proj", + "layers.37.self_attn.v_proj", + "layers.38.self_attn.b_proj", + "layers.38.self_attn.f_a_proj", + "layers.38.self_attn.f_b_proj", + "layers.38.self_attn.fused_qkvg_proj", + "layers.38.self_attn.g_proj", + "layers.38.self_attn.in_proj_qkvgfab", + "layers.38.self_attn.k_proj", + "layers.38.self_attn.o_proj", + "layers.38.self_attn.q_proj", + "layers.38.self_attn.v_proj", + "layers.39.self_attn.fused_qkv_a_proj", + "layers.39.self_attn.fused_qkv_a_proj_with_mqa", + "layers.39.self_attn.fused_qkvg_proj", + "layers.39.self_attn.g_proj", + "layers.39.self_attn.in_proj_qkvgfab", + "layers.39.self_attn.kv_a_proj_with_mqa", + "layers.39.self_attn.kv_b_proj", + "layers.39.self_attn.o_proj", + "layers.39.self_attn.q_a_proj", + "layers.39.self_attn.q_b_proj", + "layers.4.self_attn.b_proj", + "layers.4.self_attn.f_a_proj", + "layers.4.self_attn.f_b_proj", + "layers.4.self_attn.fused_qkvg_proj", + "layers.4.self_attn.g_proj", + "layers.4.self_attn.in_proj_qkvgfab", + "layers.4.self_attn.k_proj", + "layers.4.self_attn.o_proj", + "layers.4.self_attn.q_proj", + "layers.4.self_attn.v_proj", + "layers.40.self_attn.b_proj", + "layers.40.self_attn.f_a_proj", + "layers.40.self_attn.f_b_proj", + "layers.40.self_attn.fused_qkvg_proj", + "layers.40.self_attn.g_proj", + "layers.40.self_attn.in_proj_qkvgfab", + "layers.40.self_attn.k_proj", + "layers.40.self_attn.o_proj", + "layers.40.self_attn.q_proj", + "layers.40.self_attn.v_proj", + "layers.41.self_attn.b_proj", + "layers.41.self_attn.f_a_proj", + "layers.41.self_attn.f_b_proj", + "layers.41.self_attn.fused_qkvg_proj", + "layers.41.self_attn.g_proj", + "layers.41.self_attn.in_proj_qkvgfab", + "layers.41.self_attn.k_proj", + "layers.41.self_attn.o_proj", + "layers.41.self_attn.q_proj", + "layers.41.self_attn.v_proj", + "layers.42.self_attn.b_proj", + "layers.42.self_attn.f_a_proj", + "layers.42.self_attn.f_b_proj", + "layers.42.self_attn.fused_qkvg_proj", + "layers.42.self_attn.g_proj", + "layers.42.self_attn.in_proj_qkvgfab", + "layers.42.self_attn.k_proj", + "layers.42.self_attn.o_proj", + "layers.42.self_attn.q_proj", + "layers.42.self_attn.v_proj", + "layers.43.self_attn.fused_qkv_a_proj", + "layers.43.self_attn.fused_qkv_a_proj_with_mqa", + "layers.43.self_attn.fused_qkvg_proj", + "layers.43.self_attn.g_proj", + "layers.43.self_attn.in_proj_qkvgfab", + "layers.43.self_attn.kv_a_proj_with_mqa", + "layers.43.self_attn.kv_b_proj", + "layers.43.self_attn.o_proj", + "layers.43.self_attn.q_a_proj", + "layers.43.self_attn.q_b_proj", + "layers.44.self_attn.b_proj", + "layers.44.self_attn.f_a_proj", + "layers.44.self_attn.f_b_proj", + "layers.44.self_attn.fused_qkvg_proj", + "layers.44.self_attn.g_proj", + "layers.44.self_attn.in_proj_qkvgfab", + "layers.44.self_attn.k_proj", + "layers.44.self_attn.o_proj", + "layers.44.self_attn.q_proj", + "layers.44.self_attn.v_proj", + "layers.45.self_attn.b_proj", + "layers.45.self_attn.f_a_proj", + "layers.45.self_attn.f_b_proj", + "layers.45.self_attn.fused_qkvg_proj", + "layers.45.self_attn.g_proj", + "layers.45.self_attn.in_proj_qkvgfab", + "layers.45.self_attn.k_proj", + "layers.45.self_attn.o_proj", + "layers.45.self_attn.q_proj", + "layers.45.self_attn.v_proj", + "layers.46.self_attn.b_proj", + "layers.46.self_attn.f_a_proj", + "layers.46.self_attn.f_b_proj", + "layers.46.self_attn.fused_qkvg_proj", + "layers.46.self_attn.g_proj", + "layers.46.self_attn.in_proj_qkvgfab", + "layers.46.self_attn.k_proj", + "layers.46.self_attn.o_proj", + "layers.46.self_attn.q_proj", + "layers.46.self_attn.v_proj", + "layers.47.self_attn.fused_qkv_a_proj", + "layers.47.self_attn.fused_qkv_a_proj_with_mqa", + "layers.47.self_attn.fused_qkvg_proj", + "layers.47.self_attn.g_proj", + "layers.47.self_attn.in_proj_qkvgfab", + "layers.47.self_attn.kv_a_proj_with_mqa", + "layers.47.self_attn.kv_b_proj", + "layers.47.self_attn.o_proj", + "layers.47.self_attn.q_a_proj", + "layers.47.self_attn.q_b_proj", + "layers.48.self_attn.b_proj", + "layers.48.self_attn.f_a_proj", + "layers.48.self_attn.f_b_proj", + "layers.48.self_attn.fused_qkvg_proj", + "layers.48.self_attn.g_proj", + "layers.48.self_attn.in_proj_qkvgfab", + "layers.48.self_attn.k_proj", + "layers.48.self_attn.o_proj", + "layers.48.self_attn.q_proj", + "layers.48.self_attn.v_proj", + "layers.49.self_attn.b_proj", + "layers.49.self_attn.f_a_proj", + "layers.49.self_attn.f_b_proj", + "layers.49.self_attn.fused_qkvg_proj", + "layers.49.self_attn.g_proj", + "layers.49.self_attn.in_proj_qkvgfab", + "layers.49.self_attn.k_proj", + "layers.49.self_attn.o_proj", + "layers.49.self_attn.q_proj", + "layers.49.self_attn.v_proj", + "layers.5.self_attn.b_proj", + "layers.5.self_attn.f_a_proj", + "layers.5.self_attn.f_b_proj", + "layers.5.self_attn.fused_qkvg_proj", + "layers.5.self_attn.g_proj", + "layers.5.self_attn.in_proj_qkvgfab", + "layers.5.self_attn.k_proj", + "layers.5.self_attn.o_proj", + "layers.5.self_attn.q_proj", + "layers.5.self_attn.v_proj", + "layers.50.self_attn.b_proj", + "layers.50.self_attn.f_a_proj", + "layers.50.self_attn.f_b_proj", + "layers.50.self_attn.fused_qkvg_proj", + "layers.50.self_attn.g_proj", + "layers.50.self_attn.in_proj_qkvgfab", + "layers.50.self_attn.k_proj", + "layers.50.self_attn.o_proj", + "layers.50.self_attn.q_proj", + "layers.50.self_attn.v_proj", + "layers.51.self_attn.fused_qkv_a_proj", + "layers.51.self_attn.fused_qkv_a_proj_with_mqa", + "layers.51.self_attn.fused_qkvg_proj", + "layers.51.self_attn.g_proj", + "layers.51.self_attn.in_proj_qkvgfab", + "layers.51.self_attn.kv_a_proj_with_mqa", + "layers.51.self_attn.kv_b_proj", + "layers.51.self_attn.o_proj", + "layers.51.self_attn.q_a_proj", + "layers.51.self_attn.q_b_proj", + "layers.52.self_attn.b_proj", + "layers.52.self_attn.f_a_proj", + "layers.52.self_attn.f_b_proj", + "layers.52.self_attn.fused_qkvg_proj", + "layers.52.self_attn.g_proj", + "layers.52.self_attn.in_proj_qkvgfab", + "layers.52.self_attn.k_proj", + "layers.52.self_attn.o_proj", + "layers.52.self_attn.q_proj", + "layers.52.self_attn.v_proj", + "layers.53.self_attn.b_proj", + "layers.53.self_attn.f_a_proj", + "layers.53.self_attn.f_b_proj", + "layers.53.self_attn.fused_qkvg_proj", + "layers.53.self_attn.g_proj", + "layers.53.self_attn.in_proj_qkvgfab", + "layers.53.self_attn.k_proj", + "layers.53.self_attn.o_proj", + "layers.53.self_attn.q_proj", + "layers.53.self_attn.v_proj", + "layers.54.self_attn.b_proj", + "layers.54.self_attn.f_a_proj", + "layers.54.self_attn.f_b_proj", + "layers.54.self_attn.fused_qkvg_proj", + "layers.54.self_attn.g_proj", + "layers.54.self_attn.in_proj_qkvgfab", + "layers.54.self_attn.k_proj", + "layers.54.self_attn.o_proj", + "layers.54.self_attn.q_proj", + "layers.54.self_attn.v_proj", + "layers.55.self_attn.fused_qkv_a_proj", + "layers.55.self_attn.fused_qkv_a_proj_with_mqa", + "layers.55.self_attn.fused_qkvg_proj", + "layers.55.self_attn.g_proj", + "layers.55.self_attn.in_proj_qkvgfab", + "layers.55.self_attn.kv_a_proj_with_mqa", + "layers.55.self_attn.kv_b_proj", + "layers.55.self_attn.o_proj", + "layers.55.self_attn.q_a_proj", + "layers.55.self_attn.q_b_proj", + "layers.56.self_attn.b_proj", + "layers.56.self_attn.f_a_proj", + "layers.56.self_attn.f_b_proj", + "layers.56.self_attn.fused_qkvg_proj", + "layers.56.self_attn.g_proj", + "layers.56.self_attn.in_proj_qkvgfab", + "layers.56.self_attn.k_proj", + "layers.56.self_attn.o_proj", + "layers.56.self_attn.q_proj", + "layers.56.self_attn.v_proj", + "layers.57.self_attn.b_proj", + "layers.57.self_attn.f_a_proj", + "layers.57.self_attn.f_b_proj", + "layers.57.self_attn.fused_qkvg_proj", + "layers.57.self_attn.g_proj", + "layers.57.self_attn.in_proj_qkvgfab", + "layers.57.self_attn.k_proj", + "layers.57.self_attn.o_proj", + "layers.57.self_attn.q_proj", + "layers.57.self_attn.v_proj", + "layers.58.self_attn.b_proj", + "layers.58.self_attn.f_a_proj", + "layers.58.self_attn.f_b_proj", + "layers.58.self_attn.fused_qkvg_proj", + "layers.58.self_attn.g_proj", + "layers.58.self_attn.in_proj_qkvgfab", + "layers.58.self_attn.k_proj", + "layers.58.self_attn.o_proj", + "layers.58.self_attn.q_proj", + "layers.58.self_attn.v_proj", + "layers.59.self_attn.fused_qkv_a_proj", + "layers.59.self_attn.fused_qkv_a_proj_with_mqa", + "layers.59.self_attn.fused_qkvg_proj", + "layers.59.self_attn.g_proj", + "layers.59.self_attn.in_proj_qkvgfab", + "layers.59.self_attn.kv_a_proj_with_mqa", + "layers.59.self_attn.kv_b_proj", + "layers.59.self_attn.o_proj", + "layers.59.self_attn.q_a_proj", + "layers.59.self_attn.q_b_proj", + "layers.6.self_attn.b_proj", + "layers.6.self_attn.f_a_proj", + "layers.6.self_attn.f_b_proj", + "layers.6.self_attn.fused_qkvg_proj", + "layers.6.self_attn.g_proj", + "layers.6.self_attn.in_proj_qkvgfab", + "layers.6.self_attn.k_proj", + "layers.6.self_attn.o_proj", + "layers.6.self_attn.q_proj", + "layers.6.self_attn.v_proj", + "layers.60.self_attn.b_proj", + "layers.60.self_attn.f_a_proj", + "layers.60.self_attn.f_b_proj", + "layers.60.self_attn.fused_qkvg_proj", + "layers.60.self_attn.g_proj", + "layers.60.self_attn.in_proj_qkvgfab", + "layers.60.self_attn.k_proj", + "layers.60.self_attn.o_proj", + "layers.60.self_attn.q_proj", + "layers.60.self_attn.v_proj", + "layers.61.self_attn.b_proj", + "layers.61.self_attn.f_a_proj", + "layers.61.self_attn.f_b_proj", + "layers.61.self_attn.fused_qkvg_proj", + "layers.61.self_attn.g_proj", + "layers.61.self_attn.in_proj_qkvgfab", + "layers.61.self_attn.k_proj", + "layers.61.self_attn.o_proj", + "layers.61.self_attn.q_proj", + "layers.61.self_attn.v_proj", + "layers.62.self_attn.b_proj", + "layers.62.self_attn.f_a_proj", + "layers.62.self_attn.f_b_proj", + "layers.62.self_attn.fused_qkvg_proj", + "layers.62.self_attn.g_proj", + "layers.62.self_attn.in_proj_qkvgfab", + "layers.62.self_attn.k_proj", + "layers.62.self_attn.o_proj", + "layers.62.self_attn.q_proj", + "layers.62.self_attn.v_proj", + "layers.63.self_attn.fused_qkv_a_proj", + "layers.63.self_attn.fused_qkv_a_proj_with_mqa", + "layers.63.self_attn.fused_qkvg_proj", + "layers.63.self_attn.g_proj", + "layers.63.self_attn.in_proj_qkvgfab", + "layers.63.self_attn.kv_a_proj_with_mqa", + "layers.63.self_attn.kv_b_proj", + "layers.63.self_attn.o_proj", + "layers.63.self_attn.q_a_proj", + "layers.63.self_attn.q_b_proj", + "layers.64.self_attn.b_proj", + "layers.64.self_attn.f_a_proj", + "layers.64.self_attn.f_b_proj", + "layers.64.self_attn.fused_qkvg_proj", + "layers.64.self_attn.g_proj", + "layers.64.self_attn.in_proj_qkvgfab", + "layers.64.self_attn.k_proj", + "layers.64.self_attn.o_proj", + "layers.64.self_attn.q_proj", + "layers.64.self_attn.v_proj", + "layers.65.self_attn.b_proj", + "layers.65.self_attn.f_a_proj", + "layers.65.self_attn.f_b_proj", + "layers.65.self_attn.fused_qkvg_proj", + "layers.65.self_attn.g_proj", + "layers.65.self_attn.in_proj_qkvgfab", + "layers.65.self_attn.k_proj", + "layers.65.self_attn.o_proj", + "layers.65.self_attn.q_proj", + "layers.65.self_attn.v_proj", + "layers.66.self_attn.b_proj", + "layers.66.self_attn.f_a_proj", + "layers.66.self_attn.f_b_proj", + "layers.66.self_attn.fused_qkvg_proj", + "layers.66.self_attn.g_proj", + "layers.66.self_attn.in_proj_qkvgfab", + "layers.66.self_attn.k_proj", + "layers.66.self_attn.o_proj", + "layers.66.self_attn.q_proj", + "layers.66.self_attn.v_proj", + "layers.67.self_attn.fused_qkv_a_proj", + "layers.67.self_attn.fused_qkv_a_proj_with_mqa", + "layers.67.self_attn.fused_qkvg_proj", + "layers.67.self_attn.g_proj", + "layers.67.self_attn.in_proj_qkvgfab", + "layers.67.self_attn.kv_a_proj_with_mqa", + "layers.67.self_attn.kv_b_proj", + "layers.67.self_attn.o_proj", + "layers.67.self_attn.q_a_proj", + "layers.67.self_attn.q_b_proj", + "layers.68.self_attn.b_proj", + "layers.68.self_attn.f_a_proj", + "layers.68.self_attn.f_b_proj", + "layers.68.self_attn.fused_qkvg_proj", + "layers.68.self_attn.g_proj", + "layers.68.self_attn.in_proj_qkvgfab", + "layers.68.self_attn.k_proj", + "layers.68.self_attn.o_proj", + "layers.68.self_attn.q_proj", + "layers.68.self_attn.v_proj", + "layers.69.self_attn.b_proj", + "layers.69.self_attn.f_a_proj", + "layers.69.self_attn.f_b_proj", + "layers.69.self_attn.fused_qkvg_proj", + "layers.69.self_attn.g_proj", + "layers.69.self_attn.in_proj_qkvgfab", + "layers.69.self_attn.k_proj", + "layers.69.self_attn.o_proj", + "layers.69.self_attn.q_proj", + "layers.69.self_attn.v_proj", + "layers.7.self_attn.fused_qkv_a_proj", + "layers.7.self_attn.fused_qkv_a_proj_with_mqa", + "layers.7.self_attn.fused_qkvg_proj", + "layers.7.self_attn.g_proj", + "layers.7.self_attn.in_proj_qkvgfab", + "layers.7.self_attn.kv_a_proj_with_mqa", + "layers.7.self_attn.kv_b_proj", + "layers.7.self_attn.o_proj", + "layers.7.self_attn.q_a_proj", + "layers.7.self_attn.q_b_proj", + "layers.70.self_attn.b_proj", + "layers.70.self_attn.f_a_proj", + "layers.70.self_attn.f_b_proj", + "layers.70.self_attn.fused_qkvg_proj", + "layers.70.self_attn.g_proj", + "layers.70.self_attn.in_proj_qkvgfab", + "layers.70.self_attn.k_proj", + "layers.70.self_attn.o_proj", + "layers.70.self_attn.q_proj", + "layers.70.self_attn.v_proj", + "layers.71.self_attn.fused_qkv_a_proj", + "layers.71.self_attn.fused_qkv_a_proj_with_mqa", + "layers.71.self_attn.fused_qkvg_proj", + "layers.71.self_attn.g_proj", + "layers.71.self_attn.in_proj_qkvgfab", + "layers.71.self_attn.kv_a_proj_with_mqa", + "layers.71.self_attn.kv_b_proj", + "layers.71.self_attn.o_proj", + "layers.71.self_attn.q_a_proj", + "layers.71.self_attn.q_b_proj", + "layers.72.self_attn.b_proj", + "layers.72.self_attn.f_a_proj", + "layers.72.self_attn.f_b_proj", + "layers.72.self_attn.fused_qkvg_proj", + "layers.72.self_attn.g_proj", + "layers.72.self_attn.in_proj_qkvgfab", + "layers.72.self_attn.k_proj", + "layers.72.self_attn.o_proj", + "layers.72.self_attn.q_proj", + "layers.72.self_attn.v_proj", + "layers.73.self_attn.b_proj", + "layers.73.self_attn.f_a_proj", + "layers.73.self_attn.f_b_proj", + "layers.73.self_attn.fused_qkvg_proj", + "layers.73.self_attn.g_proj", + "layers.73.self_attn.in_proj_qkvgfab", + "layers.73.self_attn.k_proj", + "layers.73.self_attn.o_proj", + "layers.73.self_attn.q_proj", + "layers.73.self_attn.v_proj", + "layers.74.self_attn.b_proj", + "layers.74.self_attn.f_a_proj", + "layers.74.self_attn.f_b_proj", + "layers.74.self_attn.fused_qkvg_proj", + "layers.74.self_attn.g_proj", + "layers.74.self_attn.in_proj_qkvgfab", + "layers.74.self_attn.k_proj", + "layers.74.self_attn.o_proj", + "layers.74.self_attn.q_proj", + "layers.74.self_attn.v_proj", + "layers.75.self_attn.fused_qkv_a_proj", + "layers.75.self_attn.fused_qkv_a_proj_with_mqa", + "layers.75.self_attn.fused_qkvg_proj", + "layers.75.self_attn.g_proj", + "layers.75.self_attn.in_proj_qkvgfab", + "layers.75.self_attn.kv_a_proj_with_mqa", + "layers.75.self_attn.kv_b_proj", + "layers.75.self_attn.o_proj", + "layers.75.self_attn.q_a_proj", + "layers.75.self_attn.q_b_proj", + "layers.76.self_attn.b_proj", + "layers.76.self_attn.f_a_proj", + "layers.76.self_attn.f_b_proj", + "layers.76.self_attn.fused_qkvg_proj", + "layers.76.self_attn.g_proj", + "layers.76.self_attn.in_proj_qkvgfab", + "layers.76.self_attn.k_proj", + "layers.76.self_attn.o_proj", + "layers.76.self_attn.q_proj", + "layers.76.self_attn.v_proj", + "layers.77.self_attn.b_proj", + "layers.77.self_attn.f_a_proj", + "layers.77.self_attn.f_b_proj", + "layers.77.self_attn.fused_qkvg_proj", + "layers.77.self_attn.g_proj", + "layers.77.self_attn.in_proj_qkvgfab", + "layers.77.self_attn.k_proj", + "layers.77.self_attn.o_proj", + "layers.77.self_attn.q_proj", + "layers.77.self_attn.v_proj", + "layers.78.self_attn.b_proj", + "layers.78.self_attn.f_a_proj", + "layers.78.self_attn.f_b_proj", + "layers.78.self_attn.fused_qkvg_proj", + "layers.78.self_attn.g_proj", + "layers.78.self_attn.in_proj_qkvgfab", + "layers.78.self_attn.k_proj", + "layers.78.self_attn.o_proj", + "layers.78.self_attn.q_proj", + "layers.78.self_attn.v_proj", + "layers.79.self_attn.fused_qkv_a_proj", + "layers.79.self_attn.fused_qkv_a_proj_with_mqa", + "layers.79.self_attn.fused_qkvg_proj", + "layers.79.self_attn.g_proj", + "layers.79.self_attn.in_proj_qkvgfab", + "layers.79.self_attn.kv_a_proj_with_mqa", + "layers.79.self_attn.kv_b_proj", + "layers.79.self_attn.o_proj", + "layers.79.self_attn.q_a_proj", + "layers.79.self_attn.q_b_proj", + "layers.8.self_attn.b_proj", + "layers.8.self_attn.f_a_proj", + "layers.8.self_attn.f_b_proj", + "layers.8.self_attn.fused_qkvg_proj", + "layers.8.self_attn.g_proj", + "layers.8.self_attn.in_proj_qkvgfab", + "layers.8.self_attn.k_proj", + "layers.8.self_attn.o_proj", + "layers.8.self_attn.q_proj", + "layers.8.self_attn.v_proj", + "layers.80.self_attn.b_proj", + "layers.80.self_attn.f_a_proj", + "layers.80.self_attn.f_b_proj", + "layers.80.self_attn.fused_qkvg_proj", + "layers.80.self_attn.g_proj", + "layers.80.self_attn.in_proj_qkvgfab", + "layers.80.self_attn.k_proj", + "layers.80.self_attn.o_proj", + "layers.80.self_attn.q_proj", + "layers.80.self_attn.v_proj", + "layers.81.self_attn.b_proj", + "layers.81.self_attn.f_a_proj", + "layers.81.self_attn.f_b_proj", + "layers.81.self_attn.fused_qkvg_proj", + "layers.81.self_attn.g_proj", + "layers.81.self_attn.in_proj_qkvgfab", + "layers.81.self_attn.k_proj", + "layers.81.self_attn.o_proj", + "layers.81.self_attn.q_proj", + "layers.81.self_attn.v_proj", + "layers.82.self_attn.b_proj", + "layers.82.self_attn.f_a_proj", + "layers.82.self_attn.f_b_proj", + "layers.82.self_attn.fused_qkvg_proj", + "layers.82.self_attn.g_proj", + "layers.82.self_attn.in_proj_qkvgfab", + "layers.82.self_attn.k_proj", + "layers.82.self_attn.o_proj", + "layers.82.self_attn.q_proj", + "layers.82.self_attn.v_proj", + "layers.83.self_attn.fused_qkv_a_proj", + "layers.83.self_attn.fused_qkv_a_proj_with_mqa", + "layers.83.self_attn.fused_qkvg_proj", + "layers.83.self_attn.g_proj", + "layers.83.self_attn.in_proj_qkvgfab", + "layers.83.self_attn.kv_a_proj_with_mqa", + "layers.83.self_attn.kv_b_proj", + "layers.83.self_attn.o_proj", + "layers.83.self_attn.q_a_proj", + "layers.83.self_attn.q_b_proj", + "layers.84.self_attn.b_proj", + "layers.84.self_attn.f_a_proj", + "layers.84.self_attn.f_b_proj", + "layers.84.self_attn.fused_qkvg_proj", + "layers.84.self_attn.g_proj", + "layers.84.self_attn.in_proj_qkvgfab", + "layers.84.self_attn.k_proj", + "layers.84.self_attn.o_proj", + "layers.84.self_attn.q_proj", + "layers.84.self_attn.v_proj", + "layers.85.self_attn.b_proj", + "layers.85.self_attn.f_a_proj", + "layers.85.self_attn.f_b_proj", + "layers.85.self_attn.fused_qkvg_proj", + "layers.85.self_attn.g_proj", + "layers.85.self_attn.in_proj_qkvgfab", + "layers.85.self_attn.k_proj", + "layers.85.self_attn.o_proj", + "layers.85.self_attn.q_proj", + "layers.85.self_attn.v_proj", + "layers.86.self_attn.b_proj", + "layers.86.self_attn.f_a_proj", + "layers.86.self_attn.f_b_proj", + "layers.86.self_attn.fused_qkvg_proj", + "layers.86.self_attn.g_proj", + "layers.86.self_attn.in_proj_qkvgfab", + "layers.86.self_attn.k_proj", + "layers.86.self_attn.o_proj", + "layers.86.self_attn.q_proj", + "layers.86.self_attn.v_proj", + "layers.87.self_attn.fused_qkv_a_proj", + "layers.87.self_attn.fused_qkv_a_proj_with_mqa", + "layers.87.self_attn.fused_qkvg_proj", + "layers.87.self_attn.g_proj", + "layers.87.self_attn.in_proj_qkvgfab", + "layers.87.self_attn.kv_a_proj_with_mqa", + "layers.87.self_attn.kv_b_proj", + "layers.87.self_attn.o_proj", + "layers.87.self_attn.q_a_proj", + "layers.87.self_attn.q_b_proj", + "layers.88.self_attn.b_proj", + "layers.88.self_attn.f_a_proj", + "layers.88.self_attn.f_b_proj", + "layers.88.self_attn.fused_qkvg_proj", + "layers.88.self_attn.g_proj", + "layers.88.self_attn.in_proj_qkvgfab", + "layers.88.self_attn.k_proj", + "layers.88.self_attn.o_proj", + "layers.88.self_attn.q_proj", + "layers.88.self_attn.v_proj", + "layers.89.self_attn.b_proj", + "layers.89.self_attn.f_a_proj", + "layers.89.self_attn.f_b_proj", + "layers.89.self_attn.fused_qkvg_proj", + "layers.89.self_attn.g_proj", + "layers.89.self_attn.in_proj_qkvgfab", + "layers.89.self_attn.k_proj", + "layers.89.self_attn.o_proj", + "layers.89.self_attn.q_proj", + "layers.89.self_attn.v_proj", + "layers.9.self_attn.b_proj", + "layers.9.self_attn.f_a_proj", + "layers.9.self_attn.f_b_proj", + "layers.9.self_attn.fused_qkvg_proj", + "layers.9.self_attn.g_proj", + "layers.9.self_attn.in_proj_qkvgfab", + "layers.9.self_attn.k_proj", + "layers.9.self_attn.o_proj", + "layers.9.self_attn.q_proj", + "layers.9.self_attn.v_proj", + "layers.90.self_attn.b_proj", + "layers.90.self_attn.f_a_proj", + "layers.90.self_attn.f_b_proj", + "layers.90.self_attn.fused_qkvg_proj", + "layers.90.self_attn.g_proj", + "layers.90.self_attn.in_proj_qkvgfab", + "layers.90.self_attn.k_proj", + "layers.90.self_attn.o_proj", + "layers.90.self_attn.q_proj", + "layers.90.self_attn.v_proj", + "layers.91.self_attn.fused_qkv_a_proj", + "layers.91.self_attn.fused_qkv_a_proj_with_mqa", + "layers.91.self_attn.fused_qkvg_proj", + "layers.91.self_attn.g_proj", + "layers.91.self_attn.in_proj_qkvgfab", + "layers.91.self_attn.kv_a_proj_with_mqa", + "layers.91.self_attn.kv_b_proj", + "layers.91.self_attn.o_proj", + "layers.91.self_attn.q_a_proj", + "layers.91.self_attn.q_b_proj", + "layers.92.self_attn.fused_qkv_a_proj", + "layers.92.self_attn.fused_qkv_a_proj_with_mqa", + "layers.92.self_attn.fused_qkvg_proj", + "layers.92.self_attn.g_proj", + "layers.92.self_attn.in_proj_qkvgfab", + "layers.92.self_attn.kv_a_proj_with_mqa", + "layers.92.self_attn.kv_b_proj", + "layers.92.self_attn.o_proj", + "layers.92.self_attn.q_a_proj", + "layers.92.self_attn.q_b_proj", + "model.layers.0.self_attn.b_proj", + "model.layers.0.self_attn.f_a_proj", + "model.layers.0.self_attn.f_b_proj", + "model.layers.0.self_attn.fused_qkvg_proj", + "model.layers.0.self_attn.g_proj", + "model.layers.0.self_attn.in_proj_qkvgfab", + "model.layers.0.self_attn.k_proj", + "model.layers.0.self_attn.o_proj", + "model.layers.0.self_attn.q_proj", + "model.layers.0.self_attn.v_proj", + "model.layers.1.self_attn.b_proj", + "model.layers.1.self_attn.f_a_proj", + "model.layers.1.self_attn.f_b_proj", + "model.layers.1.self_attn.fused_qkvg_proj", + "model.layers.1.self_attn.g_proj", + "model.layers.1.self_attn.in_proj_qkvgfab", + "model.layers.1.self_attn.k_proj", + "model.layers.1.self_attn.o_proj", + "model.layers.1.self_attn.q_proj", + "model.layers.1.self_attn.v_proj", + "model.layers.10.self_attn.b_proj", + "model.layers.10.self_attn.f_a_proj", + "model.layers.10.self_attn.f_b_proj", + "model.layers.10.self_attn.fused_qkvg_proj", + "model.layers.10.self_attn.g_proj", + "model.layers.10.self_attn.in_proj_qkvgfab", + "model.layers.10.self_attn.k_proj", + "model.layers.10.self_attn.o_proj", + "model.layers.10.self_attn.q_proj", + "model.layers.10.self_attn.v_proj", + "model.layers.11.self_attn.fused_qkv_a_proj", + "model.layers.11.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.11.self_attn.fused_qkvg_proj", + "model.layers.11.self_attn.g_proj", + "model.layers.11.self_attn.in_proj_qkvgfab", + "model.layers.11.self_attn.kv_a_proj_with_mqa", + "model.layers.11.self_attn.kv_b_proj", + "model.layers.11.self_attn.o_proj", + "model.layers.11.self_attn.q_a_proj", + "model.layers.11.self_attn.q_b_proj", + "model.layers.12.self_attn.b_proj", + "model.layers.12.self_attn.f_a_proj", + "model.layers.12.self_attn.f_b_proj", + "model.layers.12.self_attn.fused_qkvg_proj", + "model.layers.12.self_attn.g_proj", + "model.layers.12.self_attn.in_proj_qkvgfab", + "model.layers.12.self_attn.k_proj", + "model.layers.12.self_attn.o_proj", + "model.layers.12.self_attn.q_proj", + "model.layers.12.self_attn.v_proj", + "model.layers.13.self_attn.b_proj", + "model.layers.13.self_attn.f_a_proj", + "model.layers.13.self_attn.f_b_proj", + "model.layers.13.self_attn.fused_qkvg_proj", + "model.layers.13.self_attn.g_proj", + "model.layers.13.self_attn.in_proj_qkvgfab", + "model.layers.13.self_attn.k_proj", + "model.layers.13.self_attn.o_proj", + "model.layers.13.self_attn.q_proj", + "model.layers.13.self_attn.v_proj", + "model.layers.14.self_attn.b_proj", + "model.layers.14.self_attn.f_a_proj", + "model.layers.14.self_attn.f_b_proj", + "model.layers.14.self_attn.fused_qkvg_proj", + "model.layers.14.self_attn.g_proj", + "model.layers.14.self_attn.in_proj_qkvgfab", + "model.layers.14.self_attn.k_proj", + "model.layers.14.self_attn.o_proj", + "model.layers.14.self_attn.q_proj", + "model.layers.14.self_attn.v_proj", + "model.layers.15.self_attn.fused_qkv_a_proj", + "model.layers.15.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.15.self_attn.fused_qkvg_proj", + "model.layers.15.self_attn.g_proj", + "model.layers.15.self_attn.in_proj_qkvgfab", + "model.layers.15.self_attn.kv_a_proj_with_mqa", + "model.layers.15.self_attn.kv_b_proj", + "model.layers.15.self_attn.o_proj", + "model.layers.15.self_attn.q_a_proj", + "model.layers.15.self_attn.q_b_proj", + "model.layers.16.self_attn.b_proj", + "model.layers.16.self_attn.f_a_proj", + "model.layers.16.self_attn.f_b_proj", + "model.layers.16.self_attn.fused_qkvg_proj", + "model.layers.16.self_attn.g_proj", + "model.layers.16.self_attn.in_proj_qkvgfab", + "model.layers.16.self_attn.k_proj", + "model.layers.16.self_attn.o_proj", + "model.layers.16.self_attn.q_proj", + "model.layers.16.self_attn.v_proj", + "model.layers.17.self_attn.b_proj", + "model.layers.17.self_attn.f_a_proj", + "model.layers.17.self_attn.f_b_proj", + "model.layers.17.self_attn.fused_qkvg_proj", + "model.layers.17.self_attn.g_proj", + "model.layers.17.self_attn.in_proj_qkvgfab", + "model.layers.17.self_attn.k_proj", + "model.layers.17.self_attn.o_proj", + "model.layers.17.self_attn.q_proj", + "model.layers.17.self_attn.v_proj", + "model.layers.18.self_attn.b_proj", + "model.layers.18.self_attn.f_a_proj", + "model.layers.18.self_attn.f_b_proj", + "model.layers.18.self_attn.fused_qkvg_proj", + "model.layers.18.self_attn.g_proj", + "model.layers.18.self_attn.in_proj_qkvgfab", + "model.layers.18.self_attn.k_proj", + "model.layers.18.self_attn.o_proj", + "model.layers.18.self_attn.q_proj", + "model.layers.18.self_attn.v_proj", + "model.layers.19.self_attn.fused_qkv_a_proj", + "model.layers.19.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.19.self_attn.fused_qkvg_proj", + "model.layers.19.self_attn.g_proj", + "model.layers.19.self_attn.in_proj_qkvgfab", + "model.layers.19.self_attn.kv_a_proj_with_mqa", + "model.layers.19.self_attn.kv_b_proj", + "model.layers.19.self_attn.o_proj", + "model.layers.19.self_attn.q_a_proj", + "model.layers.19.self_attn.q_b_proj", + "model.layers.2.self_attn.b_proj", + "model.layers.2.self_attn.f_a_proj", + "model.layers.2.self_attn.f_b_proj", + "model.layers.2.self_attn.fused_qkvg_proj", + "model.layers.2.self_attn.g_proj", + "model.layers.2.self_attn.in_proj_qkvgfab", + "model.layers.2.self_attn.k_proj", + "model.layers.2.self_attn.o_proj", + "model.layers.2.self_attn.q_proj", + "model.layers.2.self_attn.v_proj", + "model.layers.20.self_attn.b_proj", + "model.layers.20.self_attn.f_a_proj", + "model.layers.20.self_attn.f_b_proj", + "model.layers.20.self_attn.fused_qkvg_proj", + "model.layers.20.self_attn.g_proj", + "model.layers.20.self_attn.in_proj_qkvgfab", + "model.layers.20.self_attn.k_proj", + "model.layers.20.self_attn.o_proj", + "model.layers.20.self_attn.q_proj", + "model.layers.20.self_attn.v_proj", + "model.layers.21.self_attn.b_proj", + "model.layers.21.self_attn.f_a_proj", + "model.layers.21.self_attn.f_b_proj", + "model.layers.21.self_attn.fused_qkvg_proj", + "model.layers.21.self_attn.g_proj", + "model.layers.21.self_attn.in_proj_qkvgfab", + "model.layers.21.self_attn.k_proj", + "model.layers.21.self_attn.o_proj", + "model.layers.21.self_attn.q_proj", + "model.layers.21.self_attn.v_proj", + "model.layers.22.self_attn.b_proj", + "model.layers.22.self_attn.f_a_proj", + "model.layers.22.self_attn.f_b_proj", + "model.layers.22.self_attn.fused_qkvg_proj", + "model.layers.22.self_attn.g_proj", + "model.layers.22.self_attn.in_proj_qkvgfab", + "model.layers.22.self_attn.k_proj", + "model.layers.22.self_attn.o_proj", + "model.layers.22.self_attn.q_proj", + "model.layers.22.self_attn.v_proj", + "model.layers.23.self_attn.fused_qkv_a_proj", + "model.layers.23.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.23.self_attn.fused_qkvg_proj", + "model.layers.23.self_attn.g_proj", + "model.layers.23.self_attn.in_proj_qkvgfab", + "model.layers.23.self_attn.kv_a_proj_with_mqa", + "model.layers.23.self_attn.kv_b_proj", + "model.layers.23.self_attn.o_proj", + "model.layers.23.self_attn.q_a_proj", + "model.layers.23.self_attn.q_b_proj", + "model.layers.24.self_attn.b_proj", + "model.layers.24.self_attn.f_a_proj", + "model.layers.24.self_attn.f_b_proj", + "model.layers.24.self_attn.fused_qkvg_proj", + "model.layers.24.self_attn.g_proj", + "model.layers.24.self_attn.in_proj_qkvgfab", + "model.layers.24.self_attn.k_proj", + "model.layers.24.self_attn.o_proj", + "model.layers.24.self_attn.q_proj", + "model.layers.24.self_attn.v_proj", + "model.layers.25.self_attn.b_proj", + "model.layers.25.self_attn.f_a_proj", + "model.layers.25.self_attn.f_b_proj", + "model.layers.25.self_attn.fused_qkvg_proj", + "model.layers.25.self_attn.g_proj", + "model.layers.25.self_attn.in_proj_qkvgfab", + "model.layers.25.self_attn.k_proj", + "model.layers.25.self_attn.o_proj", + "model.layers.25.self_attn.q_proj", + "model.layers.25.self_attn.v_proj", + "model.layers.26.self_attn.b_proj", + "model.layers.26.self_attn.f_a_proj", + "model.layers.26.self_attn.f_b_proj", + "model.layers.26.self_attn.fused_qkvg_proj", + "model.layers.26.self_attn.g_proj", + "model.layers.26.self_attn.in_proj_qkvgfab", + "model.layers.26.self_attn.k_proj", + "model.layers.26.self_attn.o_proj", + "model.layers.26.self_attn.q_proj", + "model.layers.26.self_attn.v_proj", + "model.layers.27.self_attn.fused_qkv_a_proj", + "model.layers.27.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.27.self_attn.fused_qkvg_proj", + "model.layers.27.self_attn.g_proj", + "model.layers.27.self_attn.in_proj_qkvgfab", + "model.layers.27.self_attn.kv_a_proj_with_mqa", + "model.layers.27.self_attn.kv_b_proj", + "model.layers.27.self_attn.o_proj", + "model.layers.27.self_attn.q_a_proj", + "model.layers.27.self_attn.q_b_proj", + "model.layers.28.self_attn.b_proj", + "model.layers.28.self_attn.f_a_proj", + "model.layers.28.self_attn.f_b_proj", + "model.layers.28.self_attn.fused_qkvg_proj", + "model.layers.28.self_attn.g_proj", + "model.layers.28.self_attn.in_proj_qkvgfab", + "model.layers.28.self_attn.k_proj", + "model.layers.28.self_attn.o_proj", + "model.layers.28.self_attn.q_proj", + "model.layers.28.self_attn.v_proj", + "model.layers.29.self_attn.b_proj", + "model.layers.29.self_attn.f_a_proj", + "model.layers.29.self_attn.f_b_proj", + "model.layers.29.self_attn.fused_qkvg_proj", + "model.layers.29.self_attn.g_proj", + "model.layers.29.self_attn.in_proj_qkvgfab", + "model.layers.29.self_attn.k_proj", + "model.layers.29.self_attn.o_proj", + "model.layers.29.self_attn.q_proj", + "model.layers.29.self_attn.v_proj", + "model.layers.3.self_attn.fused_qkv_a_proj", + "model.layers.3.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.3.self_attn.fused_qkvg_proj", + "model.layers.3.self_attn.g_proj", + "model.layers.3.self_attn.in_proj_qkvgfab", + "model.layers.3.self_attn.kv_a_proj_with_mqa", + "model.layers.3.self_attn.kv_b_proj", + "model.layers.3.self_attn.o_proj", + "model.layers.3.self_attn.q_a_proj", + "model.layers.3.self_attn.q_b_proj", + "model.layers.30.self_attn.b_proj", + "model.layers.30.self_attn.f_a_proj", + "model.layers.30.self_attn.f_b_proj", + "model.layers.30.self_attn.fused_qkvg_proj", + "model.layers.30.self_attn.g_proj", + "model.layers.30.self_attn.in_proj_qkvgfab", + "model.layers.30.self_attn.k_proj", + "model.layers.30.self_attn.o_proj", + "model.layers.30.self_attn.q_proj", + "model.layers.30.self_attn.v_proj", + "model.layers.31.self_attn.fused_qkv_a_proj", + "model.layers.31.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.31.self_attn.fused_qkvg_proj", + "model.layers.31.self_attn.g_proj", + "model.layers.31.self_attn.in_proj_qkvgfab", + "model.layers.31.self_attn.kv_a_proj_with_mqa", + "model.layers.31.self_attn.kv_b_proj", + "model.layers.31.self_attn.o_proj", + "model.layers.31.self_attn.q_a_proj", + "model.layers.31.self_attn.q_b_proj", + "model.layers.32.self_attn.b_proj", + "model.layers.32.self_attn.f_a_proj", + "model.layers.32.self_attn.f_b_proj", + "model.layers.32.self_attn.fused_qkvg_proj", + "model.layers.32.self_attn.g_proj", + "model.layers.32.self_attn.in_proj_qkvgfab", + "model.layers.32.self_attn.k_proj", + "model.layers.32.self_attn.o_proj", + "model.layers.32.self_attn.q_proj", + "model.layers.32.self_attn.v_proj", + "model.layers.33.self_attn.b_proj", + "model.layers.33.self_attn.f_a_proj", + "model.layers.33.self_attn.f_b_proj", + "model.layers.33.self_attn.fused_qkvg_proj", + "model.layers.33.self_attn.g_proj", + "model.layers.33.self_attn.in_proj_qkvgfab", + "model.layers.33.self_attn.k_proj", + "model.layers.33.self_attn.o_proj", + "model.layers.33.self_attn.q_proj", + "model.layers.33.self_attn.v_proj", + "model.layers.34.self_attn.b_proj", + "model.layers.34.self_attn.f_a_proj", + "model.layers.34.self_attn.f_b_proj", + "model.layers.34.self_attn.fused_qkvg_proj", + "model.layers.34.self_attn.g_proj", + "model.layers.34.self_attn.in_proj_qkvgfab", + "model.layers.34.self_attn.k_proj", + "model.layers.34.self_attn.o_proj", + "model.layers.34.self_attn.q_proj", + "model.layers.34.self_attn.v_proj", + "model.layers.35.self_attn.fused_qkv_a_proj", + "model.layers.35.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.35.self_attn.fused_qkvg_proj", + "model.layers.35.self_attn.g_proj", + "model.layers.35.self_attn.in_proj_qkvgfab", + "model.layers.35.self_attn.kv_a_proj_with_mqa", + "model.layers.35.self_attn.kv_b_proj", + "model.layers.35.self_attn.o_proj", + "model.layers.35.self_attn.q_a_proj", + "model.layers.35.self_attn.q_b_proj", + "model.layers.36.self_attn.b_proj", + "model.layers.36.self_attn.f_a_proj", + "model.layers.36.self_attn.f_b_proj", + "model.layers.36.self_attn.fused_qkvg_proj", + "model.layers.36.self_attn.g_proj", + "model.layers.36.self_attn.in_proj_qkvgfab", + "model.layers.36.self_attn.k_proj", + "model.layers.36.self_attn.o_proj", + "model.layers.36.self_attn.q_proj", + "model.layers.36.self_attn.v_proj", + "model.layers.37.self_attn.b_proj", + "model.layers.37.self_attn.f_a_proj", + "model.layers.37.self_attn.f_b_proj", + "model.layers.37.self_attn.fused_qkvg_proj", + "model.layers.37.self_attn.g_proj", + "model.layers.37.self_attn.in_proj_qkvgfab", + "model.layers.37.self_attn.k_proj", + "model.layers.37.self_attn.o_proj", + "model.layers.37.self_attn.q_proj", + "model.layers.37.self_attn.v_proj", + "model.layers.38.self_attn.b_proj", + "model.layers.38.self_attn.f_a_proj", + "model.layers.38.self_attn.f_b_proj", + "model.layers.38.self_attn.fused_qkvg_proj", + "model.layers.38.self_attn.g_proj", + "model.layers.38.self_attn.in_proj_qkvgfab", + "model.layers.38.self_attn.k_proj", + "model.layers.38.self_attn.o_proj", + "model.layers.38.self_attn.q_proj", + "model.layers.38.self_attn.v_proj", + "model.layers.39.self_attn.fused_qkv_a_proj", + "model.layers.39.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.39.self_attn.fused_qkvg_proj", + "model.layers.39.self_attn.g_proj", + "model.layers.39.self_attn.in_proj_qkvgfab", + "model.layers.39.self_attn.kv_a_proj_with_mqa", + "model.layers.39.self_attn.kv_b_proj", + "model.layers.39.self_attn.o_proj", + "model.layers.39.self_attn.q_a_proj", + "model.layers.39.self_attn.q_b_proj", + "model.layers.4.self_attn.b_proj", + "model.layers.4.self_attn.f_a_proj", + "model.layers.4.self_attn.f_b_proj", + "model.layers.4.self_attn.fused_qkvg_proj", + "model.layers.4.self_attn.g_proj", + "model.layers.4.self_attn.in_proj_qkvgfab", + "model.layers.4.self_attn.k_proj", + "model.layers.4.self_attn.o_proj", + "model.layers.4.self_attn.q_proj", + "model.layers.4.self_attn.v_proj", + "model.layers.40.self_attn.b_proj", + "model.layers.40.self_attn.f_a_proj", + "model.layers.40.self_attn.f_b_proj", + "model.layers.40.self_attn.fused_qkvg_proj", + "model.layers.40.self_attn.g_proj", + "model.layers.40.self_attn.in_proj_qkvgfab", + "model.layers.40.self_attn.k_proj", + "model.layers.40.self_attn.o_proj", + "model.layers.40.self_attn.q_proj", + "model.layers.40.self_attn.v_proj", + "model.layers.41.self_attn.b_proj", + "model.layers.41.self_attn.f_a_proj", + "model.layers.41.self_attn.f_b_proj", + "model.layers.41.self_attn.fused_qkvg_proj", + "model.layers.41.self_attn.g_proj", + "model.layers.41.self_attn.in_proj_qkvgfab", + "model.layers.41.self_attn.k_proj", + "model.layers.41.self_attn.o_proj", + "model.layers.41.self_attn.q_proj", + "model.layers.41.self_attn.v_proj", + "model.layers.42.self_attn.b_proj", + "model.layers.42.self_attn.f_a_proj", + "model.layers.42.self_attn.f_b_proj", + "model.layers.42.self_attn.fused_qkvg_proj", + "model.layers.42.self_attn.g_proj", + "model.layers.42.self_attn.in_proj_qkvgfab", + "model.layers.42.self_attn.k_proj", + "model.layers.42.self_attn.o_proj", + "model.layers.42.self_attn.q_proj", + "model.layers.42.self_attn.v_proj", + "model.layers.43.self_attn.fused_qkv_a_proj", + "model.layers.43.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.43.self_attn.fused_qkvg_proj", + "model.layers.43.self_attn.g_proj", + "model.layers.43.self_attn.in_proj_qkvgfab", + "model.layers.43.self_attn.kv_a_proj_with_mqa", + "model.layers.43.self_attn.kv_b_proj", + "model.layers.43.self_attn.o_proj", + "model.layers.43.self_attn.q_a_proj", + "model.layers.43.self_attn.q_b_proj", + "model.layers.44.self_attn.b_proj", + "model.layers.44.self_attn.f_a_proj", + "model.layers.44.self_attn.f_b_proj", + "model.layers.44.self_attn.fused_qkvg_proj", + "model.layers.44.self_attn.g_proj", + "model.layers.44.self_attn.in_proj_qkvgfab", + "model.layers.44.self_attn.k_proj", + "model.layers.44.self_attn.o_proj", + "model.layers.44.self_attn.q_proj", + "model.layers.44.self_attn.v_proj", + "model.layers.45.self_attn.b_proj", + "model.layers.45.self_attn.f_a_proj", + "model.layers.45.self_attn.f_b_proj", + "model.layers.45.self_attn.fused_qkvg_proj", + "model.layers.45.self_attn.g_proj", + "model.layers.45.self_attn.in_proj_qkvgfab", + "model.layers.45.self_attn.k_proj", + "model.layers.45.self_attn.o_proj", + "model.layers.45.self_attn.q_proj", + "model.layers.45.self_attn.v_proj", + "model.layers.46.self_attn.b_proj", + "model.layers.46.self_attn.f_a_proj", + "model.layers.46.self_attn.f_b_proj", + "model.layers.46.self_attn.fused_qkvg_proj", + "model.layers.46.self_attn.g_proj", + "model.layers.46.self_attn.in_proj_qkvgfab", + "model.layers.46.self_attn.k_proj", + "model.layers.46.self_attn.o_proj", + "model.layers.46.self_attn.q_proj", + "model.layers.46.self_attn.v_proj", + "model.layers.47.self_attn.fused_qkv_a_proj", + "model.layers.47.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.47.self_attn.fused_qkvg_proj", + "model.layers.47.self_attn.g_proj", + "model.layers.47.self_attn.in_proj_qkvgfab", + "model.layers.47.self_attn.kv_a_proj_with_mqa", + "model.layers.47.self_attn.kv_b_proj", + "model.layers.47.self_attn.o_proj", + "model.layers.47.self_attn.q_a_proj", + "model.layers.47.self_attn.q_b_proj", + "model.layers.48.self_attn.b_proj", + "model.layers.48.self_attn.f_a_proj", + "model.layers.48.self_attn.f_b_proj", + "model.layers.48.self_attn.fused_qkvg_proj", + "model.layers.48.self_attn.g_proj", + "model.layers.48.self_attn.in_proj_qkvgfab", + "model.layers.48.self_attn.k_proj", + "model.layers.48.self_attn.o_proj", + "model.layers.48.self_attn.q_proj", + "model.layers.48.self_attn.v_proj", + "model.layers.49.self_attn.b_proj", + "model.layers.49.self_attn.f_a_proj", + "model.layers.49.self_attn.f_b_proj", + "model.layers.49.self_attn.fused_qkvg_proj", + "model.layers.49.self_attn.g_proj", + "model.layers.49.self_attn.in_proj_qkvgfab", + "model.layers.49.self_attn.k_proj", + "model.layers.49.self_attn.o_proj", + "model.layers.49.self_attn.q_proj", + "model.layers.49.self_attn.v_proj", + "model.layers.5.self_attn.b_proj", + "model.layers.5.self_attn.f_a_proj", + "model.layers.5.self_attn.f_b_proj", + "model.layers.5.self_attn.fused_qkvg_proj", + "model.layers.5.self_attn.g_proj", + "model.layers.5.self_attn.in_proj_qkvgfab", + "model.layers.5.self_attn.k_proj", + "model.layers.5.self_attn.o_proj", + "model.layers.5.self_attn.q_proj", + "model.layers.5.self_attn.v_proj", + "model.layers.50.self_attn.b_proj", + "model.layers.50.self_attn.f_a_proj", + "model.layers.50.self_attn.f_b_proj", + "model.layers.50.self_attn.fused_qkvg_proj", + "model.layers.50.self_attn.g_proj", + "model.layers.50.self_attn.in_proj_qkvgfab", + "model.layers.50.self_attn.k_proj", + "model.layers.50.self_attn.o_proj", + "model.layers.50.self_attn.q_proj", + "model.layers.50.self_attn.v_proj", + "model.layers.51.self_attn.fused_qkv_a_proj", + "model.layers.51.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.51.self_attn.fused_qkvg_proj", + "model.layers.51.self_attn.g_proj", + "model.layers.51.self_attn.in_proj_qkvgfab", + "model.layers.51.self_attn.kv_a_proj_with_mqa", + "model.layers.51.self_attn.kv_b_proj", + "model.layers.51.self_attn.o_proj", + "model.layers.51.self_attn.q_a_proj", + "model.layers.51.self_attn.q_b_proj", + "model.layers.52.self_attn.b_proj", + "model.layers.52.self_attn.f_a_proj", + "model.layers.52.self_attn.f_b_proj", + "model.layers.52.self_attn.fused_qkvg_proj", + "model.layers.52.self_attn.g_proj", + "model.layers.52.self_attn.in_proj_qkvgfab", + "model.layers.52.self_attn.k_proj", + "model.layers.52.self_attn.o_proj", + "model.layers.52.self_attn.q_proj", + "model.layers.52.self_attn.v_proj", + "model.layers.53.self_attn.b_proj", + "model.layers.53.self_attn.f_a_proj", + "model.layers.53.self_attn.f_b_proj", + "model.layers.53.self_attn.fused_qkvg_proj", + "model.layers.53.self_attn.g_proj", + "model.layers.53.self_attn.in_proj_qkvgfab", + "model.layers.53.self_attn.k_proj", + "model.layers.53.self_attn.o_proj", + "model.layers.53.self_attn.q_proj", + "model.layers.53.self_attn.v_proj", + "model.layers.54.self_attn.b_proj", + "model.layers.54.self_attn.f_a_proj", + "model.layers.54.self_attn.f_b_proj", + "model.layers.54.self_attn.fused_qkvg_proj", + "model.layers.54.self_attn.g_proj", + "model.layers.54.self_attn.in_proj_qkvgfab", + "model.layers.54.self_attn.k_proj", + "model.layers.54.self_attn.o_proj", + "model.layers.54.self_attn.q_proj", + "model.layers.54.self_attn.v_proj", + "model.layers.55.self_attn.fused_qkv_a_proj", + "model.layers.55.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.55.self_attn.fused_qkvg_proj", + "model.layers.55.self_attn.g_proj", + "model.layers.55.self_attn.in_proj_qkvgfab", + "model.layers.55.self_attn.kv_a_proj_with_mqa", + "model.layers.55.self_attn.kv_b_proj", + "model.layers.55.self_attn.o_proj", + "model.layers.55.self_attn.q_a_proj", + "model.layers.55.self_attn.q_b_proj", + "model.layers.56.self_attn.b_proj", + "model.layers.56.self_attn.f_a_proj", + "model.layers.56.self_attn.f_b_proj", + "model.layers.56.self_attn.fused_qkvg_proj", + "model.layers.56.self_attn.g_proj", + "model.layers.56.self_attn.in_proj_qkvgfab", + "model.layers.56.self_attn.k_proj", + "model.layers.56.self_attn.o_proj", + "model.layers.56.self_attn.q_proj", + "model.layers.56.self_attn.v_proj", + "model.layers.57.self_attn.b_proj", + "model.layers.57.self_attn.f_a_proj", + "model.layers.57.self_attn.f_b_proj", + "model.layers.57.self_attn.fused_qkvg_proj", + "model.layers.57.self_attn.g_proj", + "model.layers.57.self_attn.in_proj_qkvgfab", + "model.layers.57.self_attn.k_proj", + "model.layers.57.self_attn.o_proj", + "model.layers.57.self_attn.q_proj", + "model.layers.57.self_attn.v_proj", + "model.layers.58.self_attn.b_proj", + "model.layers.58.self_attn.f_a_proj", + "model.layers.58.self_attn.f_b_proj", + "model.layers.58.self_attn.fused_qkvg_proj", + "model.layers.58.self_attn.g_proj", + "model.layers.58.self_attn.in_proj_qkvgfab", + "model.layers.58.self_attn.k_proj", + "model.layers.58.self_attn.o_proj", + "model.layers.58.self_attn.q_proj", + "model.layers.58.self_attn.v_proj", + "model.layers.59.self_attn.fused_qkv_a_proj", + "model.layers.59.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.59.self_attn.fused_qkvg_proj", + "model.layers.59.self_attn.g_proj", + "model.layers.59.self_attn.in_proj_qkvgfab", + "model.layers.59.self_attn.kv_a_proj_with_mqa", + "model.layers.59.self_attn.kv_b_proj", + "model.layers.59.self_attn.o_proj", + "model.layers.59.self_attn.q_a_proj", + "model.layers.59.self_attn.q_b_proj", + "model.layers.6.self_attn.b_proj", + "model.layers.6.self_attn.f_a_proj", + "model.layers.6.self_attn.f_b_proj", + "model.layers.6.self_attn.fused_qkvg_proj", + "model.layers.6.self_attn.g_proj", + "model.layers.6.self_attn.in_proj_qkvgfab", + "model.layers.6.self_attn.k_proj", + "model.layers.6.self_attn.o_proj", + "model.layers.6.self_attn.q_proj", + "model.layers.6.self_attn.v_proj", + "model.layers.60.self_attn.b_proj", + "model.layers.60.self_attn.f_a_proj", + "model.layers.60.self_attn.f_b_proj", + "model.layers.60.self_attn.fused_qkvg_proj", + "model.layers.60.self_attn.g_proj", + "model.layers.60.self_attn.in_proj_qkvgfab", + "model.layers.60.self_attn.k_proj", + "model.layers.60.self_attn.o_proj", + "model.layers.60.self_attn.q_proj", + "model.layers.60.self_attn.v_proj", + "model.layers.61.self_attn.b_proj", + "model.layers.61.self_attn.f_a_proj", + "model.layers.61.self_attn.f_b_proj", + "model.layers.61.self_attn.fused_qkvg_proj", + "model.layers.61.self_attn.g_proj", + "model.layers.61.self_attn.in_proj_qkvgfab", + "model.layers.61.self_attn.k_proj", + "model.layers.61.self_attn.o_proj", + "model.layers.61.self_attn.q_proj", + "model.layers.61.self_attn.v_proj", + "model.layers.62.self_attn.b_proj", + "model.layers.62.self_attn.f_a_proj", + "model.layers.62.self_attn.f_b_proj", + "model.layers.62.self_attn.fused_qkvg_proj", + "model.layers.62.self_attn.g_proj", + "model.layers.62.self_attn.in_proj_qkvgfab", + "model.layers.62.self_attn.k_proj", + "model.layers.62.self_attn.o_proj", + "model.layers.62.self_attn.q_proj", + "model.layers.62.self_attn.v_proj", + "model.layers.63.self_attn.fused_qkv_a_proj", + "model.layers.63.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.63.self_attn.fused_qkvg_proj", + "model.layers.63.self_attn.g_proj", + "model.layers.63.self_attn.in_proj_qkvgfab", + "model.layers.63.self_attn.kv_a_proj_with_mqa", + "model.layers.63.self_attn.kv_b_proj", + "model.layers.63.self_attn.o_proj", + "model.layers.63.self_attn.q_a_proj", + "model.layers.63.self_attn.q_b_proj", + "model.layers.64.self_attn.b_proj", + "model.layers.64.self_attn.f_a_proj", + "model.layers.64.self_attn.f_b_proj", + "model.layers.64.self_attn.fused_qkvg_proj", + "model.layers.64.self_attn.g_proj", + "model.layers.64.self_attn.in_proj_qkvgfab", + "model.layers.64.self_attn.k_proj", + "model.layers.64.self_attn.o_proj", + "model.layers.64.self_attn.q_proj", + "model.layers.64.self_attn.v_proj", + "model.layers.65.self_attn.b_proj", + "model.layers.65.self_attn.f_a_proj", + "model.layers.65.self_attn.f_b_proj", + "model.layers.65.self_attn.fused_qkvg_proj", + "model.layers.65.self_attn.g_proj", + "model.layers.65.self_attn.in_proj_qkvgfab", + "model.layers.65.self_attn.k_proj", + "model.layers.65.self_attn.o_proj", + "model.layers.65.self_attn.q_proj", + "model.layers.65.self_attn.v_proj", + "model.layers.66.self_attn.b_proj", + "model.layers.66.self_attn.f_a_proj", + "model.layers.66.self_attn.f_b_proj", + "model.layers.66.self_attn.fused_qkvg_proj", + "model.layers.66.self_attn.g_proj", + "model.layers.66.self_attn.in_proj_qkvgfab", + "model.layers.66.self_attn.k_proj", + "model.layers.66.self_attn.o_proj", + "model.layers.66.self_attn.q_proj", + "model.layers.66.self_attn.v_proj", + "model.layers.67.self_attn.fused_qkv_a_proj", + "model.layers.67.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.67.self_attn.fused_qkvg_proj", + "model.layers.67.self_attn.g_proj", + "model.layers.67.self_attn.in_proj_qkvgfab", + "model.layers.67.self_attn.kv_a_proj_with_mqa", + "model.layers.67.self_attn.kv_b_proj", + "model.layers.67.self_attn.o_proj", + "model.layers.67.self_attn.q_a_proj", + "model.layers.67.self_attn.q_b_proj", + "model.layers.68.self_attn.b_proj", + "model.layers.68.self_attn.f_a_proj", + "model.layers.68.self_attn.f_b_proj", + "model.layers.68.self_attn.fused_qkvg_proj", + "model.layers.68.self_attn.g_proj", + "model.layers.68.self_attn.in_proj_qkvgfab", + "model.layers.68.self_attn.k_proj", + "model.layers.68.self_attn.o_proj", + "model.layers.68.self_attn.q_proj", + "model.layers.68.self_attn.v_proj", + "model.layers.69.self_attn.b_proj", + "model.layers.69.self_attn.f_a_proj", + "model.layers.69.self_attn.f_b_proj", + "model.layers.69.self_attn.fused_qkvg_proj", + "model.layers.69.self_attn.g_proj", + "model.layers.69.self_attn.in_proj_qkvgfab", + "model.layers.69.self_attn.k_proj", + "model.layers.69.self_attn.o_proj", + "model.layers.69.self_attn.q_proj", + "model.layers.69.self_attn.v_proj", + "model.layers.7.self_attn.fused_qkv_a_proj", + "model.layers.7.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.7.self_attn.fused_qkvg_proj", + "model.layers.7.self_attn.g_proj", + "model.layers.7.self_attn.in_proj_qkvgfab", + "model.layers.7.self_attn.kv_a_proj_with_mqa", + "model.layers.7.self_attn.kv_b_proj", + "model.layers.7.self_attn.o_proj", + "model.layers.7.self_attn.q_a_proj", + "model.layers.7.self_attn.q_b_proj", + "model.layers.70.self_attn.b_proj", + "model.layers.70.self_attn.f_a_proj", + "model.layers.70.self_attn.f_b_proj", + "model.layers.70.self_attn.fused_qkvg_proj", + "model.layers.70.self_attn.g_proj", + "model.layers.70.self_attn.in_proj_qkvgfab", + "model.layers.70.self_attn.k_proj", + "model.layers.70.self_attn.o_proj", + "model.layers.70.self_attn.q_proj", + "model.layers.70.self_attn.v_proj", + "model.layers.71.self_attn.fused_qkv_a_proj", + "model.layers.71.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.71.self_attn.fused_qkvg_proj", + "model.layers.71.self_attn.g_proj", + "model.layers.71.self_attn.in_proj_qkvgfab", + "model.layers.71.self_attn.kv_a_proj_with_mqa", + "model.layers.71.self_attn.kv_b_proj", + "model.layers.71.self_attn.o_proj", + "model.layers.71.self_attn.q_a_proj", + "model.layers.71.self_attn.q_b_proj", + "model.layers.72.self_attn.b_proj", + "model.layers.72.self_attn.f_a_proj", + "model.layers.72.self_attn.f_b_proj", + "model.layers.72.self_attn.fused_qkvg_proj", + "model.layers.72.self_attn.g_proj", + "model.layers.72.self_attn.in_proj_qkvgfab", + "model.layers.72.self_attn.k_proj", + "model.layers.72.self_attn.o_proj", + "model.layers.72.self_attn.q_proj", + "model.layers.72.self_attn.v_proj", + "model.layers.73.self_attn.b_proj", + "model.layers.73.self_attn.f_a_proj", + "model.layers.73.self_attn.f_b_proj", + "model.layers.73.self_attn.fused_qkvg_proj", + "model.layers.73.self_attn.g_proj", + "model.layers.73.self_attn.in_proj_qkvgfab", + "model.layers.73.self_attn.k_proj", + "model.layers.73.self_attn.o_proj", + "model.layers.73.self_attn.q_proj", + "model.layers.73.self_attn.v_proj", + "model.layers.74.self_attn.b_proj", + "model.layers.74.self_attn.f_a_proj", + "model.layers.74.self_attn.f_b_proj", + "model.layers.74.self_attn.fused_qkvg_proj", + "model.layers.74.self_attn.g_proj", + "model.layers.74.self_attn.in_proj_qkvgfab", + "model.layers.74.self_attn.k_proj", + "model.layers.74.self_attn.o_proj", + "model.layers.74.self_attn.q_proj", + "model.layers.74.self_attn.v_proj", + "model.layers.75.self_attn.fused_qkv_a_proj", + "model.layers.75.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.75.self_attn.fused_qkvg_proj", + "model.layers.75.self_attn.g_proj", + "model.layers.75.self_attn.in_proj_qkvgfab", + "model.layers.75.self_attn.kv_a_proj_with_mqa", + "model.layers.75.self_attn.kv_b_proj", + "model.layers.75.self_attn.o_proj", + "model.layers.75.self_attn.q_a_proj", + "model.layers.75.self_attn.q_b_proj", + "model.layers.76.self_attn.b_proj", + "model.layers.76.self_attn.f_a_proj", + "model.layers.76.self_attn.f_b_proj", + "model.layers.76.self_attn.fused_qkvg_proj", + "model.layers.76.self_attn.g_proj", + "model.layers.76.self_attn.in_proj_qkvgfab", + "model.layers.76.self_attn.k_proj", + "model.layers.76.self_attn.o_proj", + "model.layers.76.self_attn.q_proj", + "model.layers.76.self_attn.v_proj", + "model.layers.77.self_attn.b_proj", + "model.layers.77.self_attn.f_a_proj", + "model.layers.77.self_attn.f_b_proj", + "model.layers.77.self_attn.fused_qkvg_proj", + "model.layers.77.self_attn.g_proj", + "model.layers.77.self_attn.in_proj_qkvgfab", + "model.layers.77.self_attn.k_proj", + "model.layers.77.self_attn.o_proj", + "model.layers.77.self_attn.q_proj", + "model.layers.77.self_attn.v_proj", + "model.layers.78.self_attn.b_proj", + "model.layers.78.self_attn.f_a_proj", + "model.layers.78.self_attn.f_b_proj", + "model.layers.78.self_attn.fused_qkvg_proj", + "model.layers.78.self_attn.g_proj", + "model.layers.78.self_attn.in_proj_qkvgfab", + "model.layers.78.self_attn.k_proj", + "model.layers.78.self_attn.o_proj", + "model.layers.78.self_attn.q_proj", + "model.layers.78.self_attn.v_proj", + "model.layers.79.self_attn.fused_qkv_a_proj", + "model.layers.79.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.79.self_attn.fused_qkvg_proj", + "model.layers.79.self_attn.g_proj", + "model.layers.79.self_attn.in_proj_qkvgfab", + "model.layers.79.self_attn.kv_a_proj_with_mqa", + "model.layers.79.self_attn.kv_b_proj", + "model.layers.79.self_attn.o_proj", + "model.layers.79.self_attn.q_a_proj", + "model.layers.79.self_attn.q_b_proj", + "model.layers.8.self_attn.b_proj", + "model.layers.8.self_attn.f_a_proj", + "model.layers.8.self_attn.f_b_proj", + "model.layers.8.self_attn.fused_qkvg_proj", + "model.layers.8.self_attn.g_proj", + "model.layers.8.self_attn.in_proj_qkvgfab", + "model.layers.8.self_attn.k_proj", + "model.layers.8.self_attn.o_proj", + "model.layers.8.self_attn.q_proj", + "model.layers.8.self_attn.v_proj", + "model.layers.80.self_attn.b_proj", + "model.layers.80.self_attn.f_a_proj", + "model.layers.80.self_attn.f_b_proj", + "model.layers.80.self_attn.fused_qkvg_proj", + "model.layers.80.self_attn.g_proj", + "model.layers.80.self_attn.in_proj_qkvgfab", + "model.layers.80.self_attn.k_proj", + "model.layers.80.self_attn.o_proj", + "model.layers.80.self_attn.q_proj", + "model.layers.80.self_attn.v_proj", + "model.layers.81.self_attn.b_proj", + "model.layers.81.self_attn.f_a_proj", + "model.layers.81.self_attn.f_b_proj", + "model.layers.81.self_attn.fused_qkvg_proj", + "model.layers.81.self_attn.g_proj", + "model.layers.81.self_attn.in_proj_qkvgfab", + "model.layers.81.self_attn.k_proj", + "model.layers.81.self_attn.o_proj", + "model.layers.81.self_attn.q_proj", + "model.layers.81.self_attn.v_proj", + "model.layers.82.self_attn.b_proj", + "model.layers.82.self_attn.f_a_proj", + "model.layers.82.self_attn.f_b_proj", + "model.layers.82.self_attn.fused_qkvg_proj", + "model.layers.82.self_attn.g_proj", + "model.layers.82.self_attn.in_proj_qkvgfab", + "model.layers.82.self_attn.k_proj", + "model.layers.82.self_attn.o_proj", + "model.layers.82.self_attn.q_proj", + "model.layers.82.self_attn.v_proj", + "model.layers.83.self_attn.fused_qkv_a_proj", + "model.layers.83.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.83.self_attn.fused_qkvg_proj", + "model.layers.83.self_attn.g_proj", + "model.layers.83.self_attn.in_proj_qkvgfab", + "model.layers.83.self_attn.kv_a_proj_with_mqa", + "model.layers.83.self_attn.kv_b_proj", + "model.layers.83.self_attn.o_proj", + "model.layers.83.self_attn.q_a_proj", + "model.layers.83.self_attn.q_b_proj", + "model.layers.84.self_attn.b_proj", + "model.layers.84.self_attn.f_a_proj", + "model.layers.84.self_attn.f_b_proj", + "model.layers.84.self_attn.fused_qkvg_proj", + "model.layers.84.self_attn.g_proj", + "model.layers.84.self_attn.in_proj_qkvgfab", + "model.layers.84.self_attn.k_proj", + "model.layers.84.self_attn.o_proj", + "model.layers.84.self_attn.q_proj", + "model.layers.84.self_attn.v_proj", + "model.layers.85.self_attn.b_proj", + "model.layers.85.self_attn.f_a_proj", + "model.layers.85.self_attn.f_b_proj", + "model.layers.85.self_attn.fused_qkvg_proj", + "model.layers.85.self_attn.g_proj", + "model.layers.85.self_attn.in_proj_qkvgfab", + "model.layers.85.self_attn.k_proj", + "model.layers.85.self_attn.o_proj", + "model.layers.85.self_attn.q_proj", + "model.layers.85.self_attn.v_proj", + "model.layers.86.self_attn.b_proj", + "model.layers.86.self_attn.f_a_proj", + "model.layers.86.self_attn.f_b_proj", + "model.layers.86.self_attn.fused_qkvg_proj", + "model.layers.86.self_attn.g_proj", + "model.layers.86.self_attn.in_proj_qkvgfab", + "model.layers.86.self_attn.k_proj", + "model.layers.86.self_attn.o_proj", + "model.layers.86.self_attn.q_proj", + "model.layers.86.self_attn.v_proj", + "model.layers.87.self_attn.fused_qkv_a_proj", + "model.layers.87.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.87.self_attn.fused_qkvg_proj", + "model.layers.87.self_attn.g_proj", + "model.layers.87.self_attn.in_proj_qkvgfab", + "model.layers.87.self_attn.kv_a_proj_with_mqa", + "model.layers.87.self_attn.kv_b_proj", + "model.layers.87.self_attn.o_proj", + "model.layers.87.self_attn.q_a_proj", + "model.layers.87.self_attn.q_b_proj", + "model.layers.88.self_attn.b_proj", + "model.layers.88.self_attn.f_a_proj", + "model.layers.88.self_attn.f_b_proj", + "model.layers.88.self_attn.fused_qkvg_proj", + "model.layers.88.self_attn.g_proj", + "model.layers.88.self_attn.in_proj_qkvgfab", + "model.layers.88.self_attn.k_proj", + "model.layers.88.self_attn.o_proj", + "model.layers.88.self_attn.q_proj", + "model.layers.88.self_attn.v_proj", + "model.layers.89.self_attn.b_proj", + "model.layers.89.self_attn.f_a_proj", + "model.layers.89.self_attn.f_b_proj", + "model.layers.89.self_attn.fused_qkvg_proj", + "model.layers.89.self_attn.g_proj", + "model.layers.89.self_attn.in_proj_qkvgfab", + "model.layers.89.self_attn.k_proj", + "model.layers.89.self_attn.o_proj", + "model.layers.89.self_attn.q_proj", + "model.layers.89.self_attn.v_proj", + "model.layers.9.self_attn.b_proj", + "model.layers.9.self_attn.f_a_proj", + "model.layers.9.self_attn.f_b_proj", + "model.layers.9.self_attn.fused_qkvg_proj", + "model.layers.9.self_attn.g_proj", + "model.layers.9.self_attn.in_proj_qkvgfab", + "model.layers.9.self_attn.k_proj", + "model.layers.9.self_attn.o_proj", + "model.layers.9.self_attn.q_proj", + "model.layers.9.self_attn.v_proj", + "model.layers.90.self_attn.b_proj", + "model.layers.90.self_attn.f_a_proj", + "model.layers.90.self_attn.f_b_proj", + "model.layers.90.self_attn.fused_qkvg_proj", + "model.layers.90.self_attn.g_proj", + "model.layers.90.self_attn.in_proj_qkvgfab", + "model.layers.90.self_attn.k_proj", + "model.layers.90.self_attn.o_proj", + "model.layers.90.self_attn.q_proj", + "model.layers.90.self_attn.v_proj", + "model.layers.91.self_attn.fused_qkv_a_proj", + "model.layers.91.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.91.self_attn.fused_qkvg_proj", + "model.layers.91.self_attn.g_proj", + "model.layers.91.self_attn.in_proj_qkvgfab", + "model.layers.91.self_attn.kv_a_proj_with_mqa", + "model.layers.91.self_attn.kv_b_proj", + "model.layers.91.self_attn.o_proj", + "model.layers.91.self_attn.q_a_proj", + "model.layers.91.self_attn.q_b_proj", + "model.layers.92.self_attn.fused_qkv_a_proj", + "model.layers.92.self_attn.fused_qkv_a_proj_with_mqa", + "model.layers.92.self_attn.fused_qkvg_proj", + "model.layers.92.self_attn.g_proj", + "model.layers.92.self_attn.in_proj_qkvgfab", + "model.layers.92.self_attn.kv_a_proj_with_mqa", + "model.layers.92.self_attn.kv_b_proj", + "model.layers.92.self_attn.o_proj", + "model.layers.92.self_attn.q_a_proj", + "model.layers.92.self_attn.q_b_proj" + ] + } + }, + "ignore": [ + "*block_sparse_moe.gate*", + "*mlp.gate*", + "*block_sparse_moe.routed_expert_up_proj*", + "*mlp.routed_expert_up_proj*", + "*block_sparse_moe.routed_expert_down_proj*", + "*mlp.routed_expert_down_proj*", + "*block_sparse_moe.routed_expert_norm*", + "*mlp.routed_expert_norm*", + "*block_sparse_moe.shared_experts*", + "*mlp.shared_experts*", + "*self_attn.q_conv1d*", + "*self_attn.k_conv1d*", + "*self_attn.v_conv1d*", + "*self_attn.o_norm*", + "*self_attn.q_a_layernorm*", + "*self_attn.kv_a_layernorm*", + "*_res_proj*", + "*_res_norm*", + "*layernorm*", + "language_model.model.layers.0.mlp.*", + "layers.0.mlp.*", + "model.layers.0.mlp.*", + "language_model.lm_head", + "vision_tower*", + "mm_projector*" + ], + "producer": { + "name": "modelopt", + "version": "0.45.0" + }, + "quant_algo": "MIXED_PRECISION", + "quant_method": "modelopt_mixed", + "quantized_layers": { + "language_model.model.layers.0.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.0.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.1.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.1.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.1.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.10.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.10.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.10.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.11.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.11.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.11.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.12.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.12.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.12.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.13.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.13.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.13.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.14.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.14.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.14.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.15.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.15.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.15.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.16.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.16.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.16.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.17.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.17.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.17.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.18.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.18.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.18.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.19.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.19.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.19.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.2.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.2.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.2.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.20.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.20.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.20.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.21.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.21.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.21.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.22.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.22.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.22.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.23.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.23.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.23.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.24.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.24.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.24.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.25.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.25.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.25.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.26.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.26.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.26.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.27.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.27.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.27.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.28.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.28.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.28.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.29.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.29.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.29.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.3.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.3.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.30.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.30.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.30.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.31.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.31.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.31.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.32.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.32.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.32.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.33.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.33.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.33.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.34.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.34.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.34.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.35.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.35.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.35.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.36.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.36.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.36.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.37.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.37.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.37.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.38.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.38.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.38.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.39.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.39.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.39.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.4.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.4.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.40.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.40.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.40.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.41.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.41.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.41.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.42.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.42.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.42.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.43.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.43.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.43.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.44.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.44.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.44.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.45.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.45.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.45.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.46.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.46.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.46.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.47.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.47.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.47.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.48.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.48.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.48.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.49.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.49.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.49.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.5.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.5.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.50.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.50.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.50.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.51.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.51.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.51.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.52.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.52.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.52.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.53.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.53.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.53.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.54.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.54.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.54.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.55.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.55.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.55.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.56.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.56.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.56.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.57.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.57.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.57.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.58.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.58.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.58.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.59.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.59.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.59.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.6.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.6.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.6.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.60.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.60.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.60.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.61.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.61.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.61.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.62.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.62.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.62.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.63.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.63.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.63.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.64.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.64.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.64.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.65.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.65.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.65.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.66.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.66.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.66.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.67.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.67.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.67.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.68.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.68.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.68.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.69.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.69.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.69.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.7.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.7.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.7.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.70.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.70.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.70.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.71.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.71.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.71.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.72.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.72.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.72.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.73.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.73.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.73.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.74.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.74.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.74.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.75.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.75.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.75.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.76.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.76.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.76.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.77.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.77.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.77.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.78.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.78.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.78.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.79.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.79.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.79.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.8.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.8.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.8.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.80.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.80.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.80.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.81.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.81.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.81.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.82.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.82.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.82.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.83.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.83.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.83.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.84.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.84.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.84.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.85.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.85.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.85.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.86.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.86.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.86.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.87.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.87.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.87.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.88.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.88.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.88.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.89.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.89.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.89.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.9.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.9.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.9.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.90.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.90.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.90.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.91.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.91.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.91.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.92.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "language_model.model.layers.92.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "language_model.model.layers.92.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.0.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.1.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.1.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.1.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.10.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.10.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.10.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.11.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.11.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.11.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.12.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.12.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.12.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.13.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.13.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.13.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.14.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.14.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.14.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.15.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.15.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.15.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.16.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.16.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.16.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.17.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.17.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.17.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.18.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.18.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.18.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.19.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.19.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.19.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.2.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.2.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.2.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.20.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.20.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.20.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.21.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.21.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.21.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.22.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.22.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.22.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.23.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.23.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.23.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.24.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.24.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.24.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.25.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.25.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.25.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.26.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.26.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.26.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.27.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.27.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.27.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.28.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.28.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.28.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.29.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.29.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.29.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.3.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.3.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.3.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.30.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.30.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.30.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.31.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.31.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.31.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.32.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.32.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.32.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.33.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.33.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.33.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.34.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.34.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.34.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.35.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.35.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.35.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.36.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.36.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.36.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.37.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.37.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.37.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.38.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.38.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.38.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.39.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.39.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.39.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.4.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.4.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.4.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.40.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.40.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.40.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.41.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.41.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.41.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.42.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.42.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.42.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.43.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.43.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.43.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.44.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.44.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.44.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.45.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.45.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.45.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.46.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.46.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.46.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.47.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.47.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.47.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.48.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.48.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.48.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.49.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.49.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.49.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.5.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.5.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.5.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.50.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.50.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.50.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.51.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.51.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.51.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.52.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.52.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.52.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.53.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.53.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.53.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.54.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.54.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.54.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.55.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.55.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.55.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.56.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.56.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.56.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.57.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.57.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.57.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.58.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.58.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.58.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.59.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.59.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.59.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.6.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.6.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.6.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.60.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.60.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.60.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.61.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.61.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.61.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.62.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.62.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.62.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.63.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.63.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.63.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.64.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.64.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.64.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.65.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.65.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.65.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.66.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.66.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.66.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.67.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.67.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.67.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.68.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.68.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.68.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.69.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.69.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.69.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.7.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.7.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.7.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.70.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.70.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.70.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.71.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.71.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.71.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.72.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.72.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.72.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.73.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.73.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.73.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.74.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.74.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.74.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.75.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.75.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.75.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.76.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.76.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.76.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.77.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.77.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.77.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.78.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.78.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.78.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.79.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.79.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.79.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.8.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.8.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.8.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.80.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.80.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.80.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.81.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.81.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.81.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.82.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.82.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.82.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.83.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.83.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.83.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.84.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.84.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.84.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.85.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.85.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.85.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.86.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.86.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.86.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.87.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.87.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.87.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.88.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.88.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.88.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.89.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.89.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.89.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.9.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.9.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.9.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.90.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.90.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.90.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.91.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.91.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.91.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.92.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "layers.92.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "layers.92.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.0.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.1.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.1.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.1.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.10.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.10.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.10.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.11.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.11.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.11.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.12.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.12.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.12.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.13.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.13.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.13.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.14.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.14.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.14.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.15.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.15.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.15.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.16.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.16.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.16.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.17.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.17.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.17.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.18.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.18.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.18.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.19.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.19.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.19.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.2.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.2.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.2.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.20.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.20.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.20.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.21.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.21.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.21.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.22.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.22.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.22.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.23.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.23.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.23.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.24.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.24.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.24.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.25.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.25.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.25.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.26.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.26.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.26.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.27.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.27.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.27.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.28.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.28.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.28.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.29.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.29.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.29.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.3.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.3.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.3.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.30.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.30.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.30.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.31.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.31.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.31.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.32.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.32.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.32.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.33.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.33.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.33.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.34.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.34.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.34.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.35.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.35.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.35.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.36.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.36.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.36.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.37.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.37.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.37.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.38.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.38.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.38.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.39.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.39.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.39.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.4.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.4.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.4.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.40.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.40.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.40.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.41.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.41.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.41.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.42.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.42.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.42.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.43.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.43.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.43.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.44.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.44.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.44.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.45.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.45.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.45.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.46.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.46.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.46.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.47.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.47.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.47.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.48.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.48.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.48.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.49.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.49.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.49.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.5.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.5.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.5.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.50.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.50.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.50.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.51.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.51.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.51.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.52.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.52.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.52.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.53.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.53.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.53.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.54.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.54.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.54.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.55.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.55.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.55.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.56.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.56.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.56.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.57.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.57.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.57.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.58.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.58.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.58.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.59.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.59.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.59.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.6.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.6.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.6.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.60.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.60.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.60.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.61.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.61.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.61.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.62.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.62.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.62.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.63.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.63.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.63.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.64.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.64.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.64.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.65.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.65.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.65.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.66.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.66.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.66.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.67.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.67.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.67.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.68.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.68.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.68.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.69.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.69.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.69.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.7.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.7.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.7.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.70.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.70.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.70.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.71.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.71.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.71.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.72.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.72.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.72.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.73.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.73.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.73.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.74.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.74.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.74.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.75.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.75.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.75.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.76.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.76.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.76.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.77.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.77.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.77.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.78.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.78.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.78.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.79.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.79.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.79.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.8.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.8.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.8.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.80.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.80.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.80.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.81.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.81.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.81.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.82.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.82.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.82.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.83.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.83.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.83.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.84.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.84.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.84.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.85.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.85.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.85.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.86.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.86.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.86.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.87.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.87.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.87.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.88.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.88.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.88.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.89.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.89.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.89.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.9.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.9.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.9.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.90.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.90.self_attn.b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.f_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.f_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.k_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.q_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.90.self_attn.v_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.91.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.91.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.91.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.block_sparse_moe.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.92.mlp.experts": { + "group_size": 16, + "quant_algo": "NVFP4" + }, + "model.layers.92.self_attn.fused_qkv_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.fused_qkv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.fused_qkvg_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.g_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.in_proj_qkvgfab": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.kv_a_proj_with_mqa": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.kv_b_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.o_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.q_a_proj": { + "quant_algo": "FP8_PB_WO" + }, + "model.layers.92.self_attn.q_b_proj": { + "quant_algo": "FP8_PB_WO" + } + } + }, + "text_config": { + "_name_or_path": "", + "activation_situ_beta": 4.0, + "activation_situ_linear_beta": 25.0, + "add_cross_attention": false, + "architectures": [ + "KimiLinearForCausalLM" + ], + "attn_res_block_size": 12, + "auto_map": { + "AutoConfig": "configuration_kimi_k3.KimiLinearConfig", + "AutoModel": "modeling_kimi_linear.KimiLinearModel", + "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM" + }, + "bad_words_ids": null, + "begin_suppress_tokens": null, + "bos_token_id": 163584, + "chunk_size_feed_forward": 0, + "cross_attention_hidden_size": null, + "decoder_start_token_id": null, + "diversity_penalty": 0.0, + "do_sample": false, + "dtype": "bfloat16", + "early_stopping": false, + "encoder_no_repeat_ngram_size": 0, + "eos_token_id": 163586, + "exponential_decay_length_penalty": null, + "finetuning_task": null, + "first_k_dense_replace": 1, + "forced_bos_token_id": null, + "forced_eos_token_id": null, + "hidden_act": "situ", + "hidden_size": 7168, + "id2label": { + "0": "LABEL_0", + "1": "LABEL_1" + }, + "initializer_range": 0.02, + "intermediate_size": 33792, + "is_decoder": false, + "is_encoder_decoder": false, + "kv_lora_rank": 512, + "label2id": { + "LABEL_0": 0, + "LABEL_1": 1 + }, + "latent_moe_use_norm": true, + "length_penalty": 1.0, + "linear_attn_config": { + "full_attn_layers": [ + 4, + 8, + 12, + 16, + 20, + 24, + 28, + 32, + 36, + 40, + 44, + 48, + 52, + 56, + 60, + 64, + 68, + 72, + 76, + 80, + 84, + 88, + 92, + 93 + ], + "gate_lower_bound": -5.0, + "head_dim": 128, + "kda_layers": [ + 1, + 2, + 3, + 5, + 6, + 7, + 9, + 10, + 11, + 13, + 14, + 15, + 17, + 18, + 19, + 21, + 22, + 23, + 25, + 26, + 27, + 29, + 30, + 31, + 33, + 34, + 35, + 37, + 38, + 39, + 41, + 42, + 43, + 45, + 46, + 47, + 49, + 50, + 51, + 53, + 54, + 55, + 57, + 58, + 59, + 61, + 62, + 63, + 65, + 66, + 67, + 69, + 70, + 71, + 73, + 74, + 75, + 77, + 78, + 79, + 81, + 82, + 83, + 85, + 86, + 87, + 89, + 90, + 91 + ], + "num_heads": 96, + "short_conv_kernel_size": 4, + "use_full_rank_gate": true + }, + "max_length": 20, + "max_position_embeddings": 1048576, + "min_length": 0, + "mla_use_nope": true, + "mla_use_output_gate": true, + "model_type": "kimi_linear", + "moe_intermediate_size": 3072, + "moe_layer_freq": 1, + "moe_renormalize": true, + "moe_router_activation_func": "sigmoid", + "no_repeat_ngram_size": 0, + "num_attention_heads": 96, + "num_beam_groups": 1, + "num_beams": 1, + "num_expert_group": 1, + "num_experts": 896, + "num_experts_per_token": 16, + "num_hidden_layers": 93, + "num_key_value_heads": 96, + "num_nextn_predict_layers": 0, + "num_return_sequences": 1, + "num_shared_experts": 2, + "output_attentions": false, + "output_hidden_states": false, + "output_scores": false, + "pad_token_id": 163839, + "prefix": null, + "problem_type": null, + "pruned_heads": {}, + "q_lora_rank": 1536, + "qk_nope_head_dim": 128, + "qk_rope_head_dim": 64, + "remove_invalid_values": false, + "repetition_penalty": 1.0, + "return_dict": true, + "return_dict_in_generate": false, + "rms_norm_eps": 1e-05, + "routed_expert_hidden_size": 3584, + "routed_scaling_factor": 1.0, + "sep_token_id": null, + "suppress_tokens": null, + "task_specific_params": null, + "temperature": 1.0, + "tf_legacy_loss": false, + "tie_encoder_decoder": false, + "tie_word_embeddings": false, + "tokenizer_class": null, + "top_k": 50, + "top_p": 1.0, + "topk_group": 1, + "topk_method": "noaux_tc", + "torchscript": false, + "transformers_version": "4.56.2", + "typical_p": 1.0, + "use_bfloat16": false, + "use_cache": true, + "use_grouped_topk": true, + "v_head_dim": 128, + "vocab_size": 163840 + }, + "tie_word_embeddings": false, + "vision_config": { + "_attn_implementation": "flash_attention_2", + "activation_func": "gelu_pytorch_tanh", + "attn_bias": false, + "init_pos_emb_height": 64, + "init_pos_emb_time": 4, + "init_pos_emb_width": 64, + "linear_bias": false, + "merge_kernel_size": [ + 2, + 2 + ], + "merge_type": "sd2_tpool", + "mlp_type": "mlp2", + "mm_hidden_size": 1024, + "mm_projector_type": "patchmergerv2", + "norm_type": "rmsnorm", + "patch_embed_proj_bias": false, + "patch_size": 14, + "pos_emb_interpolation_mode": "bilinear", + "pos_emb_type": "divided_fixed", + "projector_hidden_act": "gelu", + "projector_ln_eps": 1e-05, + "qkv_hidden_size": 1536, + "text_hidden_size": 7168, + "vt_hidden_size": 1024, + "vt_intermediate_size": 4096, + "vt_num_attention_heads": 12, + "vt_num_hidden_layers": 27 + } +} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json new file mode 100644 index 0000000000..993b612989 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json @@ -0,0 +1,121 @@ +{ + "architectures": [ + "NemotronHForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "bos_token_id": 1, + "chunk_size": 128, + "conv_kernel": 4, + "dtype": "bfloat16", + "eos_token_id": 2, + "expand": 2, + "head_dim": 128, + "hidden_dropout": 0.0, + "hidden_size": 2688, + "initializer_range": 0.02, + "intermediate_size": 1856, + "layer_norm_epsilon": 1e-05, + "mamba_head_dim": 64, + "mamba_hidden_act": "silu", + "mamba_num_heads": 64, + "mamba_proj_bias": false, + "mamba_ssm_cache_dtype": "float32", + "max_position_embeddings": 262144, + "mlp_bias": false, + "mlp_hidden_act": "relu2", + "model_type": "nemotron_h", + "moe_intermediate_size": 1856, + "moe_latent_size": null, + "moe_shared_expert_intermediate_size": 3712, + "moe_shared_expert_overlap": true, + "n_group": 1, + "n_groups": 8, + "n_routed_experts": 128, + "n_shared_experts": 1, + "norm_eps": 1e-05, + "norm_topk_prob": true, + "num_attention_heads": 32, + "num_experts_per_tok": 6, + "num_hidden_layers": 52, + "num_key_value_heads": 2, + "num_logits_to_keep": 1, + "num_nextn_predict_layers": 1, + "pad_token_id": 0, + "partial_rotary_factor": 1.0, + "rescale_prenorm_residual": true, + "residual_in_fp32": false, + "rope_theta": 10000, + "routed_scaling_factor": 2.5, + "sliding_window": null, + "ssm_state_size": 128, + "tie_word_embeddings": false, + "time_step_floor": 0.0001, + "time_step_max": 0.1, + "time_step_min": 0.001, + "topk_group": 1, + "transformers_version": "4.57.6", + "use_bias": false, + "use_cache": true, + "use_conv_bias": true, + "use_mamba_kernels": true, + "vocab_size": 131072, + "layers_block_type": [ + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "attention", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "moe", + "mamba", + "moe" + ], + "mtp_layers_block_type": [ + "attention", + "moe" + ] +} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json new file mode 100644 index 0000000000..2b5e5dac5b --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json @@ -0,0 +1,2690 @@ +{ + "architectures": [ + "Qwen3_5MoeForCausalLM" + ], + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "base_model_ep_plan": { + "layers.*.mlp.experts": "torch_a2a_experts", + "layers.*.mlp.experts.down_proj": "grouped_gemm", + "layers.*.mlp.experts.gate_up_proj": "grouped_gemm", + "layers.*.mlp.gate": "torch_a2a_router" + }, + "bos_token_id": 248044, + "distributed_config": null, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 8192, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 128, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 2048, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 64, + "num_experts": 512, + "num_experts_per_tok": 10, + "num_hidden_layers": 92, + "num_key_value_heads": 4, + "output_gate_type": "swish", + "output_router_logits": false, + "pad_token_id": null, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.001, + "shared_expert_intermediate_size": 2048, + "tie_word_embeddings": false, + "transformers_version": "5.13.0", + "use_cache": true, + "vocab_size": 248320, + "quantization_config": { + "config_groups": { + "group_0": { + "input_activations": { + "dynamic": false, + "num_bits": 8, + "type": "float" + }, + "weights": { + "dynamic": false, + "num_bits": 8, + "type": "float" + }, + "targets": [ + "model.layers.0.linear_attn.conv1d", + "model.layers.0.linear_attn.in_proj_a", + "model.layers.0.linear_attn.in_proj_b", + "model.layers.0.linear_attn.in_proj_qkv", + "model.layers.0.linear_attn.in_proj_z", + "model.layers.0.linear_attn.out_proj", + "model.layers.1.linear_attn.conv1d", + "model.layers.1.linear_attn.in_proj_a", + "model.layers.1.linear_attn.in_proj_b", + "model.layers.1.linear_attn.in_proj_qkv", + "model.layers.1.linear_attn.in_proj_z", + "model.layers.1.linear_attn.out_proj", + "model.layers.10.linear_attn.conv1d", + "model.layers.10.linear_attn.in_proj_a", + "model.layers.10.linear_attn.in_proj_b", + "model.layers.10.linear_attn.in_proj_qkv", + "model.layers.10.linear_attn.in_proj_z", + "model.layers.10.linear_attn.out_proj", + "model.layers.11.self_attn.k_proj", + "model.layers.11.self_attn.o_proj", + "model.layers.11.self_attn.q_proj", + "model.layers.11.self_attn.v_proj", + "model.layers.12.linear_attn.conv1d", + "model.layers.12.linear_attn.in_proj_a", + "model.layers.12.linear_attn.in_proj_b", + "model.layers.12.linear_attn.in_proj_qkv", + "model.layers.12.linear_attn.in_proj_z", + "model.layers.12.linear_attn.out_proj", + "model.layers.13.linear_attn.conv1d", + "model.layers.13.linear_attn.in_proj_a", + "model.layers.13.linear_attn.in_proj_b", + "model.layers.13.linear_attn.in_proj_qkv", + "model.layers.13.linear_attn.in_proj_z", + "model.layers.13.linear_attn.out_proj", + "model.layers.14.linear_attn.conv1d", + "model.layers.14.linear_attn.in_proj_a", + "model.layers.14.linear_attn.in_proj_b", + "model.layers.14.linear_attn.in_proj_qkv", + "model.layers.14.linear_attn.in_proj_z", + "model.layers.14.linear_attn.out_proj", + "model.layers.15.self_attn.k_proj", + "model.layers.15.self_attn.o_proj", + "model.layers.15.self_attn.q_proj", + "model.layers.15.self_attn.v_proj", + "model.layers.16.linear_attn.conv1d", + "model.layers.16.linear_attn.in_proj_a", + "model.layers.16.linear_attn.in_proj_b", + "model.layers.16.linear_attn.in_proj_qkv", + "model.layers.16.linear_attn.in_proj_z", + "model.layers.16.linear_attn.out_proj", + "model.layers.17.linear_attn.conv1d", + "model.layers.17.linear_attn.in_proj_a", + "model.layers.17.linear_attn.in_proj_b", + "model.layers.17.linear_attn.in_proj_qkv", + "model.layers.17.linear_attn.in_proj_z", + "model.layers.17.linear_attn.out_proj", + "model.layers.18.linear_attn.conv1d", + "model.layers.18.linear_attn.in_proj_a", + "model.layers.18.linear_attn.in_proj_b", + "model.layers.18.linear_attn.in_proj_qkv", + "model.layers.18.linear_attn.in_proj_z", + "model.layers.18.linear_attn.out_proj", + "model.layers.19.self_attn.k_proj", + "model.layers.19.self_attn.o_proj", + "model.layers.19.self_attn.q_proj", + "model.layers.19.self_attn.v_proj", + "model.layers.2.linear_attn.conv1d", + "model.layers.2.linear_attn.in_proj_a", + "model.layers.2.linear_attn.in_proj_b", + "model.layers.2.linear_attn.in_proj_qkv", + "model.layers.2.linear_attn.in_proj_z", + "model.layers.2.linear_attn.out_proj", + "model.layers.20.linear_attn.conv1d", + "model.layers.20.linear_attn.in_proj_a", + "model.layers.20.linear_attn.in_proj_b", + "model.layers.20.linear_attn.in_proj_qkv", + "model.layers.20.linear_attn.in_proj_z", + "model.layers.20.linear_attn.out_proj", + "model.layers.21.linear_attn.conv1d", + "model.layers.21.linear_attn.in_proj_a", + "model.layers.21.linear_attn.in_proj_b", + "model.layers.21.linear_attn.in_proj_qkv", + "model.layers.21.linear_attn.in_proj_z", + "model.layers.21.linear_attn.out_proj", + "model.layers.22.linear_attn.conv1d", + "model.layers.22.linear_attn.in_proj_a", + "model.layers.22.linear_attn.in_proj_b", + "model.layers.22.linear_attn.in_proj_qkv", + "model.layers.22.linear_attn.in_proj_z", + "model.layers.22.linear_attn.out_proj", + "model.layers.23.self_attn.k_proj", + "model.layers.23.self_attn.o_proj", + "model.layers.23.self_attn.q_proj", + "model.layers.23.self_attn.v_proj", + "model.layers.24.linear_attn.conv1d", + "model.layers.24.linear_attn.in_proj_a", + "model.layers.24.linear_attn.in_proj_b", + "model.layers.24.linear_attn.in_proj_qkv", + "model.layers.24.linear_attn.in_proj_z", + "model.layers.24.linear_attn.out_proj", + "model.layers.25.linear_attn.conv1d", + "model.layers.25.linear_attn.in_proj_a", + "model.layers.25.linear_attn.in_proj_b", + "model.layers.25.linear_attn.in_proj_qkv", + "model.layers.25.linear_attn.in_proj_z", + "model.layers.25.linear_attn.out_proj", + "model.layers.26.linear_attn.conv1d", + "model.layers.26.linear_attn.in_proj_a", + "model.layers.26.linear_attn.in_proj_b", + "model.layers.26.linear_attn.in_proj_qkv", + "model.layers.26.linear_attn.in_proj_z", + "model.layers.26.linear_attn.out_proj", + "model.layers.27.self_attn.k_proj", + "model.layers.27.self_attn.o_proj", + "model.layers.27.self_attn.q_proj", + "model.layers.27.self_attn.v_proj", + "model.layers.28.linear_attn.conv1d", + "model.layers.28.linear_attn.in_proj_a", + "model.layers.28.linear_attn.in_proj_b", + "model.layers.28.linear_attn.in_proj_qkv", + "model.layers.28.linear_attn.in_proj_z", + "model.layers.28.linear_attn.out_proj", + "model.layers.29.linear_attn.conv1d", + "model.layers.29.linear_attn.in_proj_a", + "model.layers.29.linear_attn.in_proj_b", + "model.layers.29.linear_attn.in_proj_qkv", + "model.layers.29.linear_attn.in_proj_z", + "model.layers.29.linear_attn.out_proj", + "model.layers.3.self_attn.k_proj", + "model.layers.3.self_attn.o_proj", + "model.layers.3.self_attn.q_proj", + "model.layers.3.self_attn.v_proj", + "model.layers.30.linear_attn.conv1d", + "model.layers.30.linear_attn.in_proj_a", + "model.layers.30.linear_attn.in_proj_b", + "model.layers.30.linear_attn.in_proj_qkv", + "model.layers.30.linear_attn.in_proj_z", + "model.layers.30.linear_attn.out_proj", + "model.layers.31.self_attn.k_proj", + "model.layers.31.self_attn.o_proj", + "model.layers.31.self_attn.q_proj", + "model.layers.31.self_attn.v_proj", + "model.layers.32.linear_attn.conv1d", + "model.layers.32.linear_attn.in_proj_a", + "model.layers.32.linear_attn.in_proj_b", + "model.layers.32.linear_attn.in_proj_qkv", + "model.layers.32.linear_attn.in_proj_z", + "model.layers.32.linear_attn.out_proj", + "model.layers.33.linear_attn.conv1d", + "model.layers.33.linear_attn.in_proj_a", + "model.layers.33.linear_attn.in_proj_b", + "model.layers.33.linear_attn.in_proj_qkv", + "model.layers.33.linear_attn.in_proj_z", + "model.layers.33.linear_attn.out_proj", + "model.layers.34.linear_attn.conv1d", + "model.layers.34.linear_attn.in_proj_a", + "model.layers.34.linear_attn.in_proj_b", + "model.layers.34.linear_attn.in_proj_qkv", + "model.layers.34.linear_attn.in_proj_z", + "model.layers.34.linear_attn.out_proj", + "model.layers.35.self_attn.k_proj", + "model.layers.35.self_attn.o_proj", + "model.layers.35.self_attn.q_proj", + "model.layers.35.self_attn.v_proj", + "model.layers.36.linear_attn.conv1d", + "model.layers.36.linear_attn.in_proj_a", + "model.layers.36.linear_attn.in_proj_b", + "model.layers.36.linear_attn.in_proj_qkv", + "model.layers.36.linear_attn.in_proj_z", + "model.layers.36.linear_attn.out_proj", + "model.layers.37.linear_attn.conv1d", + "model.layers.37.linear_attn.in_proj_a", + "model.layers.37.linear_attn.in_proj_b", + "model.layers.37.linear_attn.in_proj_qkv", + "model.layers.37.linear_attn.in_proj_z", + "model.layers.37.linear_attn.out_proj", + "model.layers.38.linear_attn.conv1d", + "model.layers.38.linear_attn.in_proj_a", + "model.layers.38.linear_attn.in_proj_b", + "model.layers.38.linear_attn.in_proj_qkv", + "model.layers.38.linear_attn.in_proj_z", + "model.layers.38.linear_attn.out_proj", + "model.layers.39.self_attn.k_proj", + "model.layers.39.self_attn.o_proj", + "model.layers.39.self_attn.q_proj", + "model.layers.39.self_attn.v_proj", + "model.layers.4.linear_attn.conv1d", + "model.layers.4.linear_attn.in_proj_a", + "model.layers.4.linear_attn.in_proj_b", + "model.layers.4.linear_attn.in_proj_qkv", + "model.layers.4.linear_attn.in_proj_z", + "model.layers.4.linear_attn.out_proj", + "model.layers.40.linear_attn.conv1d", + "model.layers.40.linear_attn.in_proj_a", + "model.layers.40.linear_attn.in_proj_b", + "model.layers.40.linear_attn.in_proj_qkv", + "model.layers.40.linear_attn.in_proj_z", + "model.layers.40.linear_attn.out_proj", + "model.layers.41.linear_attn.conv1d", + "model.layers.41.linear_attn.in_proj_a", + "model.layers.41.linear_attn.in_proj_b", + "model.layers.41.linear_attn.in_proj_qkv", + "model.layers.41.linear_attn.in_proj_z", + "model.layers.41.linear_attn.out_proj", + "model.layers.42.linear_attn.conv1d", + "model.layers.42.linear_attn.in_proj_a", + "model.layers.42.linear_attn.in_proj_b", + "model.layers.42.linear_attn.in_proj_qkv", + "model.layers.42.linear_attn.in_proj_z", + "model.layers.42.linear_attn.out_proj", + "model.layers.43.self_attn.k_proj", + "model.layers.43.self_attn.o_proj", + "model.layers.43.self_attn.q_proj", + "model.layers.43.self_attn.v_proj", + "model.layers.44.linear_attn.conv1d", + "model.layers.44.linear_attn.in_proj_a", + "model.layers.44.linear_attn.in_proj_b", + "model.layers.44.linear_attn.in_proj_qkv", + "model.layers.44.linear_attn.in_proj_z", + "model.layers.44.linear_attn.out_proj", + "model.layers.45.linear_attn.conv1d", + "model.layers.45.linear_attn.in_proj_a", + "model.layers.45.linear_attn.in_proj_b", + "model.layers.45.linear_attn.in_proj_qkv", + "model.layers.45.linear_attn.in_proj_z", + "model.layers.45.linear_attn.out_proj", + "model.layers.46.linear_attn.conv1d", + "model.layers.46.linear_attn.in_proj_a", + "model.layers.46.linear_attn.in_proj_b", + "model.layers.46.linear_attn.in_proj_qkv", + "model.layers.46.linear_attn.in_proj_z", + "model.layers.46.linear_attn.out_proj", + "model.layers.47.self_attn.k_proj", + "model.layers.47.self_attn.o_proj", + "model.layers.47.self_attn.q_proj", + "model.layers.47.self_attn.v_proj", + "model.layers.48.linear_attn.conv1d", + "model.layers.48.linear_attn.in_proj_a", + "model.layers.48.linear_attn.in_proj_b", + "model.layers.48.linear_attn.in_proj_qkv", + "model.layers.48.linear_attn.in_proj_z", + "model.layers.48.linear_attn.out_proj", + "model.layers.49.linear_attn.conv1d", + "model.layers.49.linear_attn.in_proj_a", + "model.layers.49.linear_attn.in_proj_b", + "model.layers.49.linear_attn.in_proj_qkv", + "model.layers.49.linear_attn.in_proj_z", + "model.layers.49.linear_attn.out_proj", + "model.layers.5.linear_attn.conv1d", + "model.layers.5.linear_attn.in_proj_a", + "model.layers.5.linear_attn.in_proj_b", + "model.layers.5.linear_attn.in_proj_qkv", + "model.layers.5.linear_attn.in_proj_z", + "model.layers.5.linear_attn.out_proj", + "model.layers.50.linear_attn.conv1d", + "model.layers.50.linear_attn.in_proj_a", + "model.layers.50.linear_attn.in_proj_b", + "model.layers.50.linear_attn.in_proj_qkv", + "model.layers.50.linear_attn.in_proj_z", + "model.layers.50.linear_attn.out_proj", + "model.layers.51.self_attn.k_proj", + "model.layers.51.self_attn.o_proj", + "model.layers.51.self_attn.q_proj", + "model.layers.51.self_attn.v_proj", + "model.layers.52.linear_attn.conv1d", + "model.layers.52.linear_attn.in_proj_a", + "model.layers.52.linear_attn.in_proj_b", + "model.layers.52.linear_attn.in_proj_qkv", + "model.layers.52.linear_attn.in_proj_z", + "model.layers.52.linear_attn.out_proj", + "model.layers.53.linear_attn.conv1d", + "model.layers.53.linear_attn.in_proj_a", + "model.layers.53.linear_attn.in_proj_b", + "model.layers.53.linear_attn.in_proj_qkv", + "model.layers.53.linear_attn.in_proj_z", + "model.layers.53.linear_attn.out_proj", + "model.layers.54.linear_attn.conv1d", + "model.layers.54.linear_attn.in_proj_a", + "model.layers.54.linear_attn.in_proj_b", + "model.layers.54.linear_attn.in_proj_qkv", + "model.layers.54.linear_attn.in_proj_z", + "model.layers.54.linear_attn.out_proj", + "model.layers.55.self_attn.k_proj", + "model.layers.55.self_attn.o_proj", + "model.layers.55.self_attn.q_proj", + "model.layers.55.self_attn.v_proj", + "model.layers.56.linear_attn.conv1d", + "model.layers.56.linear_attn.in_proj_a", + "model.layers.56.linear_attn.in_proj_b", + "model.layers.56.linear_attn.in_proj_qkv", + "model.layers.56.linear_attn.in_proj_z", + "model.layers.56.linear_attn.out_proj", + "model.layers.57.linear_attn.conv1d", + "model.layers.57.linear_attn.in_proj_a", + "model.layers.57.linear_attn.in_proj_b", + "model.layers.57.linear_attn.in_proj_qkv", + "model.layers.57.linear_attn.in_proj_z", + "model.layers.57.linear_attn.out_proj", + "model.layers.58.linear_attn.conv1d", + "model.layers.58.linear_attn.in_proj_a", + "model.layers.58.linear_attn.in_proj_b", + "model.layers.58.linear_attn.in_proj_qkv", + "model.layers.58.linear_attn.in_proj_z", + "model.layers.58.linear_attn.out_proj", + "model.layers.59.self_attn.k_proj", + "model.layers.59.self_attn.o_proj", + "model.layers.59.self_attn.q_proj", + "model.layers.59.self_attn.v_proj", + "model.layers.6.linear_attn.conv1d", + "model.layers.6.linear_attn.in_proj_a", + "model.layers.6.linear_attn.in_proj_b", + "model.layers.6.linear_attn.in_proj_qkv", + "model.layers.6.linear_attn.in_proj_z", + "model.layers.6.linear_attn.out_proj", + "model.layers.60.linear_attn.conv1d", + "model.layers.60.linear_attn.in_proj_a", + "model.layers.60.linear_attn.in_proj_b", + "model.layers.60.linear_attn.in_proj_qkv", + "model.layers.60.linear_attn.in_proj_z", + "model.layers.60.linear_attn.out_proj", + "model.layers.61.linear_attn.conv1d", + "model.layers.61.linear_attn.in_proj_a", + "model.layers.61.linear_attn.in_proj_b", + "model.layers.61.linear_attn.in_proj_qkv", + "model.layers.61.linear_attn.in_proj_z", + "model.layers.61.linear_attn.out_proj", + "model.layers.62.linear_attn.conv1d", + "model.layers.62.linear_attn.in_proj_a", + "model.layers.62.linear_attn.in_proj_b", + "model.layers.62.linear_attn.in_proj_qkv", + "model.layers.62.linear_attn.in_proj_z", + "model.layers.62.linear_attn.out_proj", + "model.layers.63.self_attn.k_proj", + "model.layers.63.self_attn.o_proj", + "model.layers.63.self_attn.q_proj", + "model.layers.63.self_attn.v_proj", + "model.layers.64.linear_attn.conv1d", + "model.layers.64.linear_attn.in_proj_a", + "model.layers.64.linear_attn.in_proj_b", + "model.layers.64.linear_attn.in_proj_qkv", + "model.layers.64.linear_attn.in_proj_z", + "model.layers.64.linear_attn.out_proj", + "model.layers.65.linear_attn.conv1d", + "model.layers.65.linear_attn.in_proj_a", + "model.layers.65.linear_attn.in_proj_b", + "model.layers.65.linear_attn.in_proj_qkv", + "model.layers.65.linear_attn.in_proj_z", + "model.layers.65.linear_attn.out_proj", + "model.layers.66.linear_attn.conv1d", + "model.layers.66.linear_attn.in_proj_a", + "model.layers.66.linear_attn.in_proj_b", + "model.layers.66.linear_attn.in_proj_qkv", + "model.layers.66.linear_attn.in_proj_z", + "model.layers.66.linear_attn.out_proj", + "model.layers.67.self_attn.k_proj", + "model.layers.67.self_attn.o_proj", + "model.layers.67.self_attn.q_proj", + "model.layers.67.self_attn.v_proj", + "model.layers.68.linear_attn.conv1d", + "model.layers.68.linear_attn.in_proj_a", + "model.layers.68.linear_attn.in_proj_b", + "model.layers.68.linear_attn.in_proj_qkv", + "model.layers.68.linear_attn.in_proj_z", + "model.layers.68.linear_attn.out_proj", + "model.layers.69.linear_attn.conv1d", + "model.layers.69.linear_attn.in_proj_a", + "model.layers.69.linear_attn.in_proj_b", + "model.layers.69.linear_attn.in_proj_qkv", + "model.layers.69.linear_attn.in_proj_z", + "model.layers.69.linear_attn.out_proj", + "model.layers.7.self_attn.k_proj", + "model.layers.7.self_attn.o_proj", + "model.layers.7.self_attn.q_proj", + "model.layers.7.self_attn.v_proj", + "model.layers.70.linear_attn.conv1d", + "model.layers.70.linear_attn.in_proj_a", + "model.layers.70.linear_attn.in_proj_b", + "model.layers.70.linear_attn.in_proj_qkv", + "model.layers.70.linear_attn.in_proj_z", + "model.layers.70.linear_attn.out_proj", + "model.layers.71.self_attn.k_proj", + "model.layers.71.self_attn.o_proj", + "model.layers.71.self_attn.q_proj", + "model.layers.71.self_attn.v_proj", + "model.layers.72.linear_attn.conv1d", + "model.layers.72.linear_attn.in_proj_a", + "model.layers.72.linear_attn.in_proj_b", + "model.layers.72.linear_attn.in_proj_qkv", + "model.layers.72.linear_attn.in_proj_z", + "model.layers.72.linear_attn.out_proj", + "model.layers.73.linear_attn.conv1d", + "model.layers.73.linear_attn.in_proj_a", + "model.layers.73.linear_attn.in_proj_b", + "model.layers.73.linear_attn.in_proj_qkv", + "model.layers.73.linear_attn.in_proj_z", + "model.layers.73.linear_attn.out_proj", + "model.layers.74.linear_attn.conv1d", + "model.layers.74.linear_attn.in_proj_a", + "model.layers.74.linear_attn.in_proj_b", + "model.layers.74.linear_attn.in_proj_qkv", + "model.layers.74.linear_attn.in_proj_z", + "model.layers.74.linear_attn.out_proj", + "model.layers.75.self_attn.k_proj", + "model.layers.75.self_attn.o_proj", + "model.layers.75.self_attn.q_proj", + "model.layers.75.self_attn.v_proj", + "model.layers.76.linear_attn.conv1d", + "model.layers.76.linear_attn.in_proj_a", + "model.layers.76.linear_attn.in_proj_b", + "model.layers.76.linear_attn.in_proj_qkv", + "model.layers.76.linear_attn.in_proj_z", + "model.layers.76.linear_attn.out_proj", + "model.layers.77.linear_attn.conv1d", + "model.layers.77.linear_attn.in_proj_a", + "model.layers.77.linear_attn.in_proj_b", + "model.layers.77.linear_attn.in_proj_qkv", + "model.layers.77.linear_attn.in_proj_z", + "model.layers.77.linear_attn.out_proj", + "model.layers.78.linear_attn.conv1d", + "model.layers.78.linear_attn.in_proj_a", + "model.layers.78.linear_attn.in_proj_b", + "model.layers.78.linear_attn.in_proj_qkv", + "model.layers.78.linear_attn.in_proj_z", + "model.layers.78.linear_attn.out_proj", + "model.layers.79.self_attn.k_proj", + "model.layers.79.self_attn.o_proj", + "model.layers.79.self_attn.q_proj", + "model.layers.79.self_attn.v_proj", + "model.layers.8.linear_attn.conv1d", + "model.layers.8.linear_attn.in_proj_a", + "model.layers.8.linear_attn.in_proj_b", + "model.layers.8.linear_attn.in_proj_qkv", + "model.layers.8.linear_attn.in_proj_z", + "model.layers.8.linear_attn.out_proj", + "model.layers.80.linear_attn.conv1d", + "model.layers.80.linear_attn.in_proj_a", + "model.layers.80.linear_attn.in_proj_b", + "model.layers.80.linear_attn.in_proj_qkv", + "model.layers.80.linear_attn.in_proj_z", + "model.layers.80.linear_attn.out_proj", + "model.layers.81.linear_attn.conv1d", + "model.layers.81.linear_attn.in_proj_a", + "model.layers.81.linear_attn.in_proj_b", + "model.layers.81.linear_attn.in_proj_qkv", + "model.layers.81.linear_attn.in_proj_z", + "model.layers.81.linear_attn.out_proj", + "model.layers.82.linear_attn.conv1d", + "model.layers.82.linear_attn.in_proj_a", + "model.layers.82.linear_attn.in_proj_b", + "model.layers.82.linear_attn.in_proj_qkv", + "model.layers.82.linear_attn.in_proj_z", + "model.layers.82.linear_attn.out_proj", + "model.layers.83.self_attn.k_proj", + "model.layers.83.self_attn.o_proj", + "model.layers.83.self_attn.q_proj", + "model.layers.83.self_attn.v_proj", + "model.layers.84.linear_attn.conv1d", + "model.layers.84.linear_attn.in_proj_a", + "model.layers.84.linear_attn.in_proj_b", + "model.layers.84.linear_attn.in_proj_qkv", + "model.layers.84.linear_attn.in_proj_z", + "model.layers.84.linear_attn.out_proj", + "model.layers.85.linear_attn.conv1d", + "model.layers.85.linear_attn.in_proj_a", + "model.layers.85.linear_attn.in_proj_b", + "model.layers.85.linear_attn.in_proj_qkv", + "model.layers.85.linear_attn.in_proj_z", + "model.layers.85.linear_attn.out_proj", + "model.layers.86.linear_attn.conv1d", + "model.layers.86.linear_attn.in_proj_a", + "model.layers.86.linear_attn.in_proj_b", + "model.layers.86.linear_attn.in_proj_qkv", + "model.layers.86.linear_attn.in_proj_z", + "model.layers.86.linear_attn.out_proj", + "model.layers.87.self_attn.k_proj", + "model.layers.87.self_attn.o_proj", + "model.layers.87.self_attn.q_proj", + "model.layers.87.self_attn.v_proj", + "model.layers.88.linear_attn.conv1d", + "model.layers.88.linear_attn.in_proj_a", + "model.layers.88.linear_attn.in_proj_b", + "model.layers.88.linear_attn.in_proj_qkv", + "model.layers.88.linear_attn.in_proj_z", + "model.layers.88.linear_attn.out_proj", + "model.layers.89.linear_attn.conv1d", + "model.layers.89.linear_attn.in_proj_a", + "model.layers.89.linear_attn.in_proj_b", + "model.layers.89.linear_attn.in_proj_qkv", + "model.layers.89.linear_attn.in_proj_z", + "model.layers.89.linear_attn.out_proj", + "model.layers.9.linear_attn.conv1d", + "model.layers.9.linear_attn.in_proj_a", + "model.layers.9.linear_attn.in_proj_b", + "model.layers.9.linear_attn.in_proj_qkv", + "model.layers.9.linear_attn.in_proj_z", + "model.layers.9.linear_attn.out_proj", + "model.layers.90.linear_attn.conv1d", + "model.layers.90.linear_attn.in_proj_a", + "model.layers.90.linear_attn.in_proj_b", + "model.layers.90.linear_attn.in_proj_qkv", + "model.layers.90.linear_attn.in_proj_z", + "model.layers.90.linear_attn.out_proj", + "model.layers.91.self_attn.k_proj", + "model.layers.91.self_attn.o_proj", + "model.layers.91.self_attn.q_proj", + "model.layers.91.self_attn.v_proj" + ] + }, + "group_1": { + "input_activations": { + "dynamic": false, + "num_bits": 4, + "type": "float", + "group_size": 16 + }, + "weights": { + "dynamic": false, + "num_bits": 4, + "type": "float", + "group_size": 16 + }, + "targets": [ + "model.layers.0.mlp.experts", + "model.layers.1.mlp.experts", + "model.layers.10.mlp.experts", + "model.layers.11.mlp.experts", + "model.layers.12.mlp.experts", + "model.layers.13.mlp.experts", + "model.layers.14.mlp.experts", + "model.layers.15.mlp.experts", + "model.layers.16.mlp.experts", + "model.layers.17.mlp.experts", + "model.layers.18.mlp.experts", + "model.layers.19.mlp.experts", + "model.layers.2.mlp.experts", + "model.layers.20.mlp.experts", + "model.layers.21.mlp.experts", + "model.layers.22.mlp.experts", + "model.layers.23.mlp.experts", + "model.layers.24.mlp.experts", + "model.layers.25.mlp.experts", + "model.layers.26.mlp.experts", + "model.layers.27.mlp.experts", + "model.layers.28.mlp.experts", + "model.layers.29.mlp.experts", + "model.layers.3.mlp.experts", + "model.layers.30.mlp.experts", + "model.layers.31.mlp.experts", + "model.layers.32.mlp.experts", + "model.layers.33.mlp.experts", + "model.layers.34.mlp.experts", + "model.layers.35.mlp.experts", + "model.layers.36.mlp.experts", + "model.layers.37.mlp.experts", + "model.layers.38.mlp.experts", + "model.layers.39.mlp.experts", + "model.layers.4.mlp.experts", + "model.layers.40.mlp.experts", + "model.layers.41.mlp.experts", + "model.layers.42.mlp.experts", + "model.layers.43.mlp.experts", + "model.layers.44.mlp.experts", + "model.layers.45.mlp.experts", + "model.layers.46.mlp.experts", + "model.layers.47.mlp.experts", + "model.layers.48.mlp.experts", + "model.layers.49.mlp.experts", + "model.layers.5.mlp.experts", + "model.layers.50.mlp.experts", + "model.layers.51.mlp.experts", + "model.layers.52.mlp.experts", + "model.layers.53.mlp.experts", + "model.layers.54.mlp.experts", + "model.layers.55.mlp.experts", + "model.layers.56.mlp.experts", + "model.layers.57.mlp.experts", + "model.layers.58.mlp.experts", + "model.layers.59.mlp.experts", + "model.layers.6.mlp.experts", + "model.layers.60.mlp.experts", + "model.layers.61.mlp.experts", + "model.layers.62.mlp.experts", + "model.layers.63.mlp.experts", + "model.layers.64.mlp.experts", + "model.layers.65.mlp.experts", + "model.layers.66.mlp.experts", + "model.layers.67.mlp.experts", + "model.layers.68.mlp.experts", + "model.layers.69.mlp.experts", + "model.layers.7.mlp.experts", + "model.layers.70.mlp.experts", + "model.layers.71.mlp.experts", + "model.layers.72.mlp.experts", + "model.layers.73.mlp.experts", + "model.layers.74.mlp.experts", + "model.layers.75.mlp.experts", + "model.layers.76.mlp.experts", + "model.layers.77.mlp.experts", + "model.layers.78.mlp.experts", + "model.layers.79.mlp.experts", + "model.layers.8.mlp.experts", + "model.layers.80.mlp.experts", + "model.layers.81.mlp.experts", + "model.layers.82.mlp.experts", + "model.layers.83.mlp.experts", + "model.layers.84.mlp.experts", + "model.layers.85.mlp.experts", + "model.layers.86.mlp.experts", + "model.layers.87.mlp.experts", + "model.layers.88.mlp.experts", + "model.layers.89.mlp.experts", + "model.layers.9.mlp.experts", + "model.layers.90.mlp.experts", + "model.layers.91.mlp.experts" + ] + } + }, + "quantized_layers": { + "model.layers.0.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.0.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.1.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.1.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.2.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.2.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.3.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.4.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.4.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.5.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.5.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.6.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.6.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.7.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.8.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.8.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.9.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.9.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.10.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.10.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.11.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.12.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.12.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.13.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.13.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.14.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.14.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.15.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.16.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.16.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.17.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.17.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.18.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.18.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.19.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.20.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.20.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.21.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.21.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.22.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.22.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.23.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.24.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.24.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.25.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.25.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.26.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.26.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.27.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.28.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.28.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.29.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.29.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.30.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.30.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.31.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.32.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.32.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.33.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.33.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.34.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.34.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.35.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.36.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.36.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.37.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.37.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.38.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.38.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.39.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.40.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.40.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.41.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.41.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.42.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.42.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.43.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.44.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.44.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.45.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.45.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.46.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.46.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.47.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.48.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.48.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.49.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.49.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.50.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.50.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.51.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.52.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.52.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.53.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.53.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.54.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.54.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.55.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.56.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.56.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.57.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.57.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.58.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.58.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.59.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.60.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.60.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.61.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.61.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.62.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.62.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.63.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.64.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.64.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.65.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.65.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.66.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.66.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.67.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.68.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.68.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.69.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.69.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.70.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.70.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.71.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.72.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.72.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.73.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.73.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.74.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.74.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.75.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.76.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.76.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.77.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.77.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.78.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.78.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.79.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.80.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.80.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.81.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.81.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.82.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.82.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.83.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.84.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.84.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.85.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.85.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.86.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.86.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.87.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.88.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.88.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.89.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.89.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.90.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.90.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.91.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + } + }, + "ignore": [ + "mtp*", + "mtp.layers.0*" + ], + "quant_algo": "MIXED_PRECISION", + "kv_cache_scheme": { + "dynamic": false, + "num_bits": 8, + "type": "float" + }, + "producer": { + "name": "modelopt", + "version": "0.42.0" + }, + "quant_method": "modelopt" + } +} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json new file mode 100644 index 0000000000..9a7da44038 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json @@ -0,0 +1,1902 @@ +{ + "producer": { + "name": "modelopt", + "version": "0.42.0" + }, + "quantization": { + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": "FP8", + "quantized_layers": { + "model.layers.0.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.0.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.0.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.1.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.1.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.1.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.2.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.2.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.2.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.3.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.3.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.4.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.4.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.4.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.5.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.5.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.5.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.6.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.6.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.6.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.7.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.7.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.8.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.8.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.8.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.9.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.9.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.9.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.10.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.10.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.10.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.11.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.11.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.12.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.12.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.12.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.13.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.13.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.13.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.14.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.14.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.14.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.15.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.15.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.16.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.16.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.16.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.17.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.17.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.17.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.18.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.18.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.18.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.19.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.19.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.20.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.20.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.20.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.21.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.21.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.21.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.22.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.22.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.22.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.23.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.23.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.24.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.24.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.24.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.25.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.25.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.25.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.26.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.26.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.26.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.27.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.27.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.28.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.28.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.28.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.29.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.29.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.29.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.30.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.30.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.30.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.31.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.31.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.32.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.32.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.32.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.33.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.33.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.33.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.34.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.34.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.34.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.35.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.35.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.36.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.36.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.36.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.37.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.37.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.37.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.38.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.38.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.38.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.39.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.39.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.40.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.40.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.40.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.41.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.41.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.41.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.42.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.42.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.42.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.43.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.43.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.44.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.44.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.44.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.45.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.45.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.45.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.46.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.46.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.46.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.47.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.47.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.48.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.48.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.48.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.49.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.49.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.49.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.50.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.50.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.50.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.51.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.51.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.52.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.52.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.52.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.53.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.53.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.53.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.54.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.54.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.54.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.55.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.55.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.56.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.56.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.56.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.57.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.57.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.57.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.58.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.58.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.58.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.59.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.59.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.60.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.60.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.60.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.61.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.61.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.61.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.62.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.62.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.62.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.63.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.63.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.64.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.64.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.64.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.65.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.65.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.65.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.66.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.66.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.66.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.67.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.67.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.68.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.68.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.68.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.69.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.69.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.69.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.70.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.70.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.70.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.71.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.71.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.72.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.72.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.72.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.73.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.73.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.73.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.74.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.74.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.74.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.75.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.75.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.76.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.76.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.76.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.77.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.77.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.77.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.78.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.78.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.78.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.79.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.79.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.80.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.80.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.80.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.81.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.81.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.81.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.82.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.82.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.82.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.83.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.83.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.84.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.84.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.84.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.85.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.85.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.85.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.86.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.86.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.86.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.87.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.87.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.88.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.88.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.88.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.89.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.89.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.89.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.90.linear_attn.conv1d": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.out_proj": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_qkv": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_z": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_b": { + "quant_algo": "FP8" + }, + "model.layers.90.linear_attn.in_proj_a": { + "quant_algo": "FP8" + }, + "model.layers.90.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + }, + "model.layers.91.self_attn.q_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.self_attn.k_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.self_attn.v_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.self_attn.o_proj": { + "quant_algo": "FP8" + }, + "model.layers.91.mlp.experts": { + "quant_algo": "NVFP4", + "group_size": 16 + } + }, + "exclude_modules": [ + "mtp*", + "mtp.layers.0*" + ] + } +} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json new file mode 100644 index 0000000000..9860d7132a --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json @@ -0,0 +1,27 @@ +{ + "_name_or_path": "openbmb/MiniCPM5-2B", + "architectures": [ + "LlamaForCausalLM" + ], + "bos_token_id": 0, + "eos_token_id": [1, 130073], + "pad_token_id": 1, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "intermediate_size": 6144, + "max_position_embeddings": 131072, + "model_type": "llama", + "num_attention_heads": 16, + "num_hidden_layers": 42, + "num_key_value_heads": 2, + "head_dim": 128, + "rms_norm_eps": 1e-06, + "rope_theta": 5000000, + "rope_scaling": null, + "tie_word_embeddings": false, + "torch_dtype": "bfloat16", + "transformers_version": "5.6.2", + "use_cache": true, + "vocab_size": 130560 +} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json new file mode 100644 index 0000000000..865278c0e0 --- /dev/null +++ b/python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json @@ -0,0 +1,125 @@ +{ + "architectures": [ + "Qwen3_5MoeForConditionalGeneration" + ], + "bos_token_id": null, + "dtype": "bfloat16", + "eos_token_id": 248046, + "hidden_size": 2048, + "image_token_id": 248056, + "model_type": "qwen3_5_moe", + "pad_token_id": 248044, + "text_config": { + "attention_bias": false, + "attention_dropout": 0.0, + "attn_output_gate": true, + "bos_token_id": 248044, + "dtype": "bfloat16", + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2048, + "initializer_range": 0.02, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "model_type": "qwen3_5_moe_text", + "moe_intermediate_size": 512, + "mtp_num_hidden_layers": 1, + "mtp_use_dedicated_embeddings": false, + "num_attention_heads": 16, + "num_experts": 256, + "num_experts_per_tok": 8, + "num_hidden_layers": 40, + "num_key_value_heads": 2, + "output_router_logits": false, + "pad_token_id": 248044, + "partial_rotary_factor": 0.25, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "router_aux_loss_coef": 0.0, + "shared_expert_intermediate_size": 512, + "tie_word_embeddings": false, + "use_cache": false, + "vocab_size": 248320 + }, + "tie_word_embeddings": false, + "transformers_version": "5.8.1", + "video_token_id": 248057, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 27, + "dtype": "bfloat16", + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1152, + "in_channels": 3, + "initializer_range": 0.02, + "intermediate_size": 4304, + "model_type": "qwen3_5_moe_vision", + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2048, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "vision_end_token_id": 248054, + "vision_start_token_id": 248053 +} diff --git a/python/aisimulate/src/aisimulate_core/sdk/common.py b/python/aisimulate/src/aisimulate_core/sdk/common.py index ff135bf849..c804f7f233 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/common.py +++ b/python/aisimulate/src/aisimulate_core/sdk/common.py @@ -678,6 +678,7 @@ def get_default_models() -> set[str]: """ DefaultHFModels = { "deepseek-ai/DeepSeek-V4.1-Flash", + "nvidia/DeepSeek-V4.1-Flash-NVFP4", # Llama 3.1 Models "meta-llama/Meta-Llama-3.1-8B", "meta-llama/Meta-Llama-3.1-70B", @@ -692,6 +693,7 @@ def get_default_models() -> set[str]: "moonshotai/Kimi-K2.5", # Kimi K3 "moonshotai/Kimi-K3", + "nvidia/Kimi-K3-NVFP4", "nvidia/Kimi-K2.5-NVFP4", "nvidia/Kimi-K2.6-NVFP4", "nvidia/Kimi-K2.7-Code-NVFP4", @@ -771,12 +773,14 @@ def get_default_models() -> set[str]: # Qwen3.8-Max Models "Qwen/Qwen3.8-2.4T-A95B", "Qwen/Qwen3.8-2.4T-A95B-FP8", + "nvidia/Qwen3.8-2.4T-A95B-NVFP4", # MiMo Models "XiaomiMiMo/MiMo-V2-Flash", "XiaomiMiMo/MiMo-7B-Base", # NVIDIA Nemotron "nvidia/Llama-3_3-Nemotron-Super-49B-v1", "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", + "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", @@ -789,6 +793,15 @@ def get_default_models() -> set[str]: "google/gemma-4-26B-A4B", # Meta Muse Glimmer "meta-models/Muse-Glimmer-30B", + # Qwen3.8-27B (QWEN35 hybrid GDN dense; onboarding 2026-09-25) + "Qwen/Qwen3.8-27B", + "Qwen/Qwen3.8-27B-FP8", + # Ornith 1.5 (Qwen3.5-MoE architecture) + "ornith-ai/Ornith-1.5-35B-A3B", + # MiniCPM5 (Llama architecture) + "openbmb/MiniCPM5-2B", + # NeoHorse 1 (text-only dense Qwen3.5, Qwen3_5ForCausalLM) + "TokenRhythm/NeoHorse-1-9B", # StepFun Step-3.7 Models "stepfun-ai/Step-3.7-Flash", "stepfun-ai/Step-3.7-Flash-FP8", @@ -906,6 +919,7 @@ def get_default_models() -> set[str]: # Qwen3.8-Max: FLAT config (no text_config nesting) -- do NOT add to # MULTIMODAL_TEXT_CONFIG_KEY below, unlike the two VLM classes above. "Qwen3_5MoeForCausalLM": "QWEN35", + "Qwen3_5ForCausalLM": "QWEN35", # text-only dense Qwen3.5 checkpoint (config at root, no text_config) "Gemma4ForConditionalGeneration": "GEMMA4MIX", "MuseGlimmerForConditionalGeneration": "MUSEGLIMMER", "Mistral3ForConditionalGeneration": "MISTRAL3", @@ -1408,14 +1422,13 @@ class PerfDataFilename(Enum): # DeepSeek-V4 sparse-op family — all share one column schema, served by # the engine table view (``_dsv4_sparse_kernel_data.*`` / # ``_dsv4_csa_topk_calib_data``): - # csa_attn / hca_attn / paged_mqa_logits : FMLA & indexer kernel latency, - # keyed ``num_heads -> tp -> past_kv -> isl -> bs`` (kernel-level Δ data, - # queried by ``_lookup_sparse_kernel``). + # paged_mqa_logits : indexer kernel latency, keyed + # ``num_heads -> tp -> past_kv -> isl -> bs`` (kernel-level Δ data, + # queried by ``_lookup_sparse_kernel``). The csa_attn / hca_attn FMLA + # sidecars were removed 2026-10-15 (retired placeholders, no query path). # csa_topk_calib : two rows/shape (score_mode=flat|top_last); the topK # DELTA (flat-top_last) correction applied to CSA module latency. dsv4_paged_mqa_logits_module = "dsv4_paged_mqa_logits_module_perf.parquet" - dsv4_hca_attn_module = "dsv4_hca_attn_module_perf.parquet" - dsv4_csa_attn_module = "dsv4_csa_attn_module_perf.parquet" dsv4_csa_topk_calib = "dsv4_csa_topk_calib_perf.parquet" dsv4_megamoe_module = "dsv4_megamoe_module_perf.parquet" # Whole-model forward-pass data (forward_model="fpm"). Paired with a diff --git a/python/aisimulate/src/aisimulate_core/sdk/operations/dsv4.py b/python/aisimulate/src/aisimulate_core/sdk/operations/dsv4.py index c0c0dbe77c..6daae8b15e 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/operations/dsv4.py +++ b/python/aisimulate/src/aisimulate_core/sdk/operations/dsv4.py @@ -177,8 +177,10 @@ def _load_sparse(sub_key, filename_enum): # paged_mqa_logits is the only sparse-kernel sidecar with a live # consumer (the CP chunk walk). The hca_attn/csa_attn sidecars - # were design placeholders with no query path and were retired - # (csa_attn never even shipped data). + # were design placeholders with no query path; they were retired + # here first and removed end to end (collector ops, PerfFile / + # PerfDataFilename, shipped tables) on 2026-10-15 — csa_attn never + # even shipped data. sparse_loaded = { "paged_mqa_logits": _load_sparse("paged_mqa_logits", PerfDataFilename.dsv4_paged_mqa_logits_module), } diff --git a/python/aisimulate/src/aisimulate_core/sdk/utils.py b/python/aisimulate/src/aisimulate_core/sdk/utils.py index 5e3fde4354..7ce95dd878 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/utils.py +++ b/python/aisimulate/src/aisimulate_core/sdk/utils.py @@ -1487,6 +1487,9 @@ def positive_int(value, name): # CausalLM architecture string instead of the VLM ConditionalGeneration # classes above. "Qwen3_5MoeForCausalLM", + # text-only dense Qwen3.5 checkpoints (e.g. Altworld/Hemmingway-1, + # TokenRhythm/NeoHorse-1-9B): flat config, no vision_config. + "Qwen3_5ForCausalLM", }: # Qwen3.5 hybrid GDN + full-attention model. layer_types_raw = config.get("layer_types", []) diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml index 43c67faf60..45041ba8f6 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml @@ -27,8 +27,6 @@ tables: status: complete dsv4_csa_generation_module_perf: status: complete - dsv4_hca_attn_module_perf: - status: complete dsv4_hca_context_module_perf: status: complete dsv4_hca_generation_module_perf: diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index f3333ffe059da78533744a164a21a6b4e48dacfb..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14637 zcmd732|U!@7eD@)1vA#MjC~o~WHHit(dFsjc`F{Vu*X#Fs{qCz~?&sWl&-i0LU~df5jjO000L7z{%3m3Jsun(J~6?K~ZoTBbrfI6Oe(^2l&@NIXS_3>x)=i z000I)42nV^gQ5V)mIsafg)Or!05F0xa7wD`O1RA!Gj0s=fB*hB=AvlaFU&dO0DuK# zZtY}FG$&vTRdJglu>3U6|MSDYVd2n*zhLpV0RRt(&)rebV1k)LNX>c5F$f!_6tHKb)B zfH-K42w$pVXn06~HziV;=3YWw5n+0 zFVYJ1000`4mM&ogD4~YLPakRs zIZ{P)GxH#R4D$cg`!@)7H0BowCY_C#^aqCGM2ml4sEdoFR7yu8@fQ{#M4Vi7A0A*UGi=To3to5sejwS0|NVzrs2mK0R{+!lM}_s$+-rHz$7?1#gD+1qc%?HYPSJ9s%38FRRD_2|HGOu3D8D%>6Vd zI?)o1`CC)kc?jSDg>0fIBd9tns60QY{(9q}D}4}DIu86mfqDivD{KlJ{M z`j_4wTL5VU5Y2^Gv+-`y!PRUPA1aY$ym$_k9HxW?h=OLhIBFu0%Fuh{M)bueMah|& zA!;24Lr?sAdY{T_0!n}~@Qgaq&gi^fu1^&>UaBrQ0FxBFB-&TZe}z_m{AizaGNr?_ z9)nx(t)G$Ge_ARyDdmwAS*K+@*Ajz!@)IuKuJFYbFH(4RJ&M{5 z0yuCSzaACP?=!!B<5>3+0nkgx2lE;xRW42DvE`_|+lY~K z+~{89FTV0)0T?ac8$CvTr~W0r#{<|n7qBe&{Wim}7n4QUuOn04c`O0fYhH6hI~CI< z)!X^VS%s;BLAvn+RJa2ZaBWpkezx>->6t5TtSS6ND?WEhr*I)};r)jP2&mMm`KiF# zqSWD({3r1$2v3y8ZfdPwgbm5Y&Q-aUztQ%C>WYa6fWM!>OO(PcRQ+ zoA6Z0I;M293zR@nrusZzDu&VaUgvF+CyFH$!k%~t4X(?za8+isXSZOn2zb?r$x^JE z+)v;+>EFl)zkyC86c+8RK7Rv!o)50l?cgVEUdB_bWbp7jACk`3sYpfR&hRv zq25y-6p7g><&iUn^)SVTv=;?8PZuFRAfY-iy`7(Uxpw7~G=bri5mEVBOs*3xpVX@Y z$|Xx8gIdaHJztsO2bJbQb-L3NRxRPOhlDH4Nh4cJC5pwDAzU64D4p%9{h|KYq8_b&s)pz3 z!PyE=845HoLNt%r<*4|R`Y(phLtnSHwrH%xmY`^;{NbM z9`mEF+G|3I+Ke~k*WIR z!UTScRN=e4_bQhfc>(+RXXUR&#*NeB+5K+tcjEYat{l=?jgJ*vclN`QcQqNXkib@74V#TNg@{RtN3zMooyExJx^ zG1vDu;ccw(e_Bn+AQ$zT-WOWiwJ+X)5&&u5iaOXgZ8Vpi$Pff0!wg}|g+R5bM&#Lx zIBmxII(@5qWP;Y?l38`Dw{=37ROL@xs3(|zFg$Q?nN>w#OG5Nne`yoHelq!kE{=FR z@q>8F>l>l3k9OMXCFt%sY}7$zOj)Rj$z~Ne#!hzD3I!4(Bjg z%q`8%(_X;1h0!j1$a)i2E2kg(=y>?du^Ctgu^0IU=A2@>0+YLtwH2Dt1#xlKSqNcS2U<##xA!+T;r%r4UY@#8AQ3z4rBL7p*o+-ds9d=#Qz5dy8%7>Qxa!)kM zSlZ@l;*L9=A8PxzE@ZUtQl4Xhh&-1*#hlbTW3LIbGUlm0m9cF+?^dVCnZ>jB#I0sa zowq5^@G$Rm9Eqr2R(oC3F)(3=K57wfawuHiP4Lg17J@Q1hyUQl>R`}EmDq!i8GNOd zU9Fs2i1La*SmmJV7A%|NBbog<)DgV`*q#n;x~=D$ux22v6j0cA_u~6tvv)6R)DmK9 z&J}EtW?K$1agY-z4}H5*V~MLi*TPV0)aJ^;6H|kOvNN|ah9rbdCv#W6t%r@&73F%~ zVTy$|ASg*!HANhqCvOjqmW?Y~PZbOZ3LO{qC7mYZL3y*WrG2K8i0Og*L$-=V*P4q* ztMjZ*x~in#Nvaylvf-su4oLwL$t%6xb~ru zTdT&+0}_X9y?w#S>yvkw`kmXSLjyyZ(?cGDn`kFF(5BR&y3Ej)D5O5BfFt%CXPyi< z71buc3wJxDS25W2VKKDVfcIf~=DyxE){ydK`%4G5_qFwQr!$Pt*sCowH1`jO>L_jv zZ)$3o@;^?lOhjB1wsuR3$_UujkG)4B&aBl*FsqAbm1PDY&!{pkFS!?31)TA}`#e6v z(rHbOdm+GAi%-8?ggx_Mf&Nw%V*@O&z^YY#`YhS}(v|iFEspK)?(&*)>l798US}I= zNd1&m)+F(!yDgXbz6(imGEDm=B3#l&FDQVST~;Z}n(?f+q9A7QmfqDVq^k*2L!I%W z`b^SJ_uRf()}*)X!i91w?-IgfCTF@2EV|r3ryFN?|Eaf$DzLSe=k4daGu8FkZ+7!@ z6@CpxPWYy=!r47;y;G=#i7jyR@0oqWl&WS4JNe-Gz&(h*=bcOH&nwJfoR1d}+A?qf zFM|YlKdVadmODNTrp8Cyc|QrO?g0HO_i$?S4tqDY;{|s<`yD9H6c$bR>f+JZt8-&o z{YIp&VSdgP?az^i_cVT%1I-}(xhU50r7fEXI(08w4dG+?IT5D;gq1#rRpH2Z4&$SO z6_v8cjK0Nco}npchVmT+iMqSPW$C_~Hq`7u>c*&b9P5QLSuR#Tm~Xvi;3;z{_IU0e z`Wg=h^6aNWQ0qo}uf<0)JMxZt^Za~JP_(!KJW0Ind1ADY)2vAJjsr$6SIi7Ej^FW4 z8c^VPOmaDrdcF-sM$8_HZ+_`b4{7BuIwQhBBr95+&@Ciob)j;o6(8PD&#H*?q)mBTjG~^^|?Mhz9AUE5tKN!}Ot|p7OqgcoKC~nzH;VuXk zffBKqf}SJF_lj<{_d-T`K)2IQe|X|}AwI#jw%FC{-U|&`rV5LLLq!o0?0hk~tpQ>= zQ;++~GFD9$uxCbiGGihY3Ocj^t|JG8IE`_HTflPY${|7R!o~3A522KCVI5+>=uJao zvv-p#i%xpvzPT6VW7$G!4PVF|nm8gqNnh`5mXFD$xyeQ~WpBq6JxkL@4_&7hlUoh8 zGnJA=c^C2@fDu0_E7mS=ecBw_%WK|^@6+=gmi@*mhS?t*{}^x2$v|3kAIe0scvvBe z!j%hR?5!`qbCV1uJ(Q?TIKWedx+1+e9uH>E%*KXCK52Ei^^kXh9j_FvcE=V~U9TkJ zL>kf(lhRQOhAcgSaumGS)=2klk8A14y%E@J>9ni=6|f+aWAMODjg59UW1uyV>0;cP z8n>kdi)iK}DX&s8yG4%wg*I893zrt_fRE?212N9QUydBuuUGM=i`xZ{m$S*d&26m5 zd5c4FR}07%dZ0V7(bnXdyhB=Gm|3&aR$@toj>Zg0*lrhUpTPAi3^|5PTm9~=h<~Z8 z=y~2sIkb#7!YE(m02U7{|f zAuj11kuv;*?!EX(`s}=}(9ZKGI_^ON0{p1~Bv+GdEb!H>vl2)FzoG{_44M>ivc^pzZ5rEwee*4;7G9{)*4U#7QA#^QtU6s% z+aj8uMo1^Npx2^X#syCV#@-uxlG1mtxBI5<>+E>}joz8n z7>n+zXV!b3?YbHyeznITcoaHzC8(t*sJ=yvbr3gx<@`MQ}@--XY8MN`!zZg+PhDT{uw{%WUQ{14;dRL zJtHyD&*kg)*zt`E-AhyIcWt~TcJ|cJ1Iwo1qEa>(TlFh1%LR4MF~nVTsv&wZ9`2U9 zd2Q11)=Ne(9;b7^*p^(#)R4x%3>)}Rcv9*DX42jal+u<#5LRp7!^z|RBW9j2dLBFMq8>W9 zfH;#bQ{I_lD>}GyIERx(p2^+oZNF5wg>WK&onY`>$wrbFkGm#WU(G*VzsWE|J2wKjt=LiR+D*XV56rK-d1cK8F7N44R-ErY{? zNd2dY3O848YU?eBHnodlEV$Lhuwl5^>(j_%RrQ%-!c+r(z234Mq;y|uTlSt4;O!v7 zEZAIE08jvrSaP9v%dZN;z?g0H+su%n)yV(4>%^gc=7&I z;!1d*+6Q0oI_NyPbSDgsurpCwI9ymzPyptgwcf!k7Y$^uMrXf!@A5j)z}{lL9UJgz zc$_G9f?Ci69zTAXv#U)A$rx+f!Jve-5t(2rmFGZbuoC#qs+PjxgQ{;n?786V6~1P}fJI@vCK;B^n9}Jd5q^^egF}Zsp}ExiGWtl2@}rz&x|~>c^yKA9vrpjW7sa zi)H;&XZ6ch-uCf*YKlv5KYn@r@nFhUp|zonYY^-$Dw@ zHYqR6qO*dg3fOFyMm{^Aq0~Q!N2X79!nx&jD@E*N*GxE){36 zNuN^NG8-}7gcaXWglw3GbPddw>`F)P=}(>YQhhuRMr2n|!ttr+OR*&k8FTz~IlgQ1 zQ%{5Qz_%%nq6S1JTmy1pD4NVoM+XOD%jvS2`jH(0`!b|L8cFA!0vq=)%O!KLTZ0sd z`RmN96Al4m)ebl2@4p&Wx(fR`Ubqu>_ROQ#^b5nV&dE1>M2?obZ{fOeFFEc6c)ph$ zcrl-0oyqHqaKY{iq4m|*S2LvCsZ#CsK9DsbmtR+jG1rwqU{DA+jF<|2&cx;Cu*xJq zDCJ)N@OIy_Mj>R+h?l#2ANwTBoy0h6UEt_SS5-@kY@5-muFK;?Z3laXhsd`s$)Dmo z#Af|^Kfl@TP}ihS>W})9`Da31<~8liQUc0@I3nvNttE}N7Jx-!`O&B4j4EIYKa zFYQT(NZ!1Bra+U?!#5XqTu79x)*|*1MgvpvttY(^n)W9y9Xk8M(eHi216_#L*!<|~ zxX{Y#GLpA!ORA>Qz?roYA+0|tXEpAabQ(4oKe{=tI@z=G$7PM~2VSl|x>{M;+{#EE z%0=>MZ{h31-h+FzpL`jfh=!ldHF}TDaeAaFbu=D}%o}GA5Zmq-H|6;a)4u=l>Zz&8 z48?ba4ixg0m`mRBt#blKAuoJX__JJ^l<%zhq$;|FQ$83RkxRG>k9CuM`(cR0M~snL zX8vkia9{GZSzqs^O61rThhBlfBeQ}2N6w<}?Mr+0{^}hPZ@dsGmF>EK zIx_AX{{@@-%lE5Q4&DiVpi^=+$Q>U8zQnnO6Zq)HT2XcRtRL(s`{0n+%83Uz+p>}X z%0RzrIRoFC@k8{jmgi9Nov#C8-n!YWYIiH*Y#yC)Ys$OQ2phQvzru>Le4*IyBhMak z&Vi#hCwgHSGMajVyY%jjsakvOg-W)u7dx-z83Z2f+&4$ezV_+iCzqhll{Nm_+t_D9 zqq@Qi(-Hjr3rTbCPOE)+j0}=j_@e7YuN7TS$bN>oo)*dqge zYB_JpXU{Q759Z6gdy;$hN?NKVn`>~Zt+nX+l(w6$RohtX#_DTY_IxRhk#Ac&PLd0_ zSLfH#X_dm?_`srLytJ8fWdDfS)wonoHTl~IE|+C>)qk>{d4Eb()_)6Qi>;da7-9F+ zj`u2#JV=4p2RzSVP30g40jc3WlLaWoBhp*BQ@P*z2^=;1R??00SLrN1Sr?k~RCAeH z!oqQ_!O|_&^c-Mb2Kab(p{nJH9-mwmNA!#=}U-G{BqLy0tb$`p-THm^6F|MOMaZ?_)n6TAI z-6lT!Bd5L9@97u3BeRw8tC*aQs|`-mVvX|tx@b(pJIS8E7Q!r zzrwa$^F-oOkvhX5vXUvnqlIY-eYp9xc(dymn;rC z9XR#P$mKpd?ud3K&q_*=jnRau_q3-ok&I9~z&s84&6oRumN7F96I5AKd5O868h3+V0#_`+Xo?W_$ zfhVA4(oHqco-$dP!6A`-r)`uHaM+LL3DJ6ghyYXUUQv4Ugl?M-w<(Q8%Li3~VMdpi zGZVbOw4M#Ld}0pSDzd`*NwdOH<>tA#en~bA(_!1xu=w0&zm7E9lOO%_tra6;E)H52 zKJglpwLThGz}_d+&Cuy`v~)M}_HC}WuFt~i(UM{sFABr>#-5$qwq;U#kI^c>P-x|O zyO$|}y(H(yB3>EQNfWNNK}+S*UR<-XLbiUci7_RF?(^Wwy-GRsHMw72QI|_LuVqG} z@cfA2a>#WCuC#6R*L%NH|9ZXWQGhIV`0;xjmc0P@Dv%LRgYvWjOgI3_8cKun;314Q ztx)Dx8Y8Y1!a>8snNi759vXzJ2!P4plaV$w2s#&Fknw;p?*d=~LjX*(H5o>n02n;* zOfnM?MqDYB%LANXlgtr20mUGa5hf@IQg{O3%EH6VY?8UmXmAl6feDL&a5$90w%`fO z4s}qZQQjt5eN z7fYG+y9pfexiHa18k6c`8wU+Y#URKCqD%^BHZ4s`N*qbBY3C{$f=jC>BP>GOxl2(D zvX;rrj&<$mx?BcDYBJ02#df|XAOn|`%tGRa3G{kosFjMd1vtZmo=jwDH9^?JVpD}@ z@tL|K5cZf>BhlqLCWGZRj=ca816M&9^0cEe@k9xpdW4C3J7*4!C?%zWv~XzWI@(Ia zs@Ef}cb(=wxk}uE=VZ1$n8H(m>y)+0V|J`fL7xxpl=t9dap_OtZLI5549#QNy^zA! zGMBEr2XfE@LE-O^$xz8oKIkPw5$N~JP&)}ZpN1OO2GG<4g5F*gUYSPrnz4O>cM zPylydyA}GGGZ{JJ0oB%~!Lg+PQlV9h%dZt~Ih4HD!2^md0`OFPGIuuyBZ2CMV^I(; zA|5JIglDp}Nk+1sf%41&43-`#e5EuDLvbq{i-V+s4uHmGA?1c3+(o4@9?(myJyQ6E z;o?Z~HaHfOjFd@(?-jv87<7k{*&{(WDaEr{heDYY(!>ReXatfCNTqX1M#_1lu-`9* zY0S~s+!j+Zw5P>U1PH>0mYV7T>KmHHDx41!m~ItU?IsZSpg8y8$Pzr&X2uG3dE&c> zNGpsXs*sj<5SNTFAL{1Tyu_Iu*&!(gL7L&YxbpnOr6k3Xd={Quin-HLSc18Q!zC`& zx--&BaY#$ge8%DqFRXg?KC9rAg5&q$oEbws*0ICfr`F&y1kPTYEIXcZVFp=q+g`h( zVV+9u8Cjcs`yDC`51(_Mk#p|Z@3iOi;q##t@^16JJ6pMU>kp5*H{Cq%$7MQ&$XAm;hLu!97GlQi~?wJXLtQeVJnc zjFxsAlDe3}Ju<j3wfh6KZ#{rCi2Svwir0Qv-|@**oz z@Jt{FmgB9An&2`wk5umC6KbW9ESN60Aa_qRJyK0zCT2RvayG)8sy&{htU zRH$$+Ba`;rK_dw+P7EL}fzL7{O4teBJKm+v+J*?Qi%64nJ|kgjGZ5gMn)k3SL!+k6 zEYSJ+F_|np^TlKX@(6(lyZyATW0D$s2_}~D-d~2&D=UR4@SZ}k;SZZ`ANhB~i7KnXa$lUgI>CRq&2LoZN(u2$G zNM?=1z@^Oyds&_^0sQw%le7`QC!jS|3Up??43rbo2E}4>VP<$JnA{lfuwxnO9S|^! z_~X_iO`02nHDd!fcibMbQ8T4CYW;crFZMnPrHE9)f{+ z7G;*Du3d*>#sgs92AEVqXO{>s%K{)q2CK;&syr!RYQ^y7XAz|&@@P0OMzyM}z`|0v zv{V^V@ChnYw?q6D>`$=pl>xCDwIMU)V{n`Oq6pF2`XlvTOA&o1ROPxTy8UIKQ2 zG<}+{O~yp^80Vn?uX3SD8Pim=wCw0J<%GE`Q~h^&oO|GBFi3wh;yLTQgOO)yBN@!i z?S`F?bezHJ%$wWn&o8VbR>--`TR8XHkuP7CRyurkyW3~Gqbra~+>DVWMZ&(g|8h>^ zLMoUK02p{;aOGm4;7NqUdZ0McFqpG>S^n0Nd1{m0*1P6VQ*es`2oU zlmg}{IH%5lthI1i!SR<4TeglPcf<`Bm=1SGPiEq1mHC$odc!>+J1JHzq|E>17Ydmsr2 zaVT#cO*#$5u-*BVpp#h^L%wHchUTu=fqj*@^hb;H8eQ>+>a(L1yrj3g^pJQPj?L8i z668r?!zm9hmr5)8+18LAo_p9>m#%2$wcT}e%IWI$4~n7bEW4%z`I>vZFGQ5d?|w<* zYkATr8*_ujZ7KiJ^>c<7Gc8k17y=b);=iHz^o5r;csKU!jUS% zN=Dj`v{M^%7=-DUkXBGm0LJeDULl-0^B>?ct)|JS14SLPIF(q7W zNb`a>?tqJfm!{gW2mL-;w_FCD%NkC6Nb%Kw|M(7#SG|nwke{*IvAfwBm(R!s`CA4a zyLXaN17d6yU|&vpPimU-elx51ET4JQx6v49nOEJj>F7|x`M8=tp1s{6p$KdLtH#49(ZmXV&pu!i#8uc2EdWA2>!+qiq>ZXfvD za|N805yGE`J`fkN3*I{tMxVQVy#7&tF?~nkb4mA4{V@j16+4n%QQW_beKc6@+_C3< zx%=0Jm~Crp@_RqCJp6O@(I0xBl%>CY5d&|Jtq`s{8lxTt%3amUpjwyAo```<%CsTv zP+)b5z#6v(z|R;#@$(KMqon{et27iW0t+VxdB=o=2TB8$!T?y)MW-@FdxuB(hXnsj zWAyTl_NRFNOkxTNbq@xY|4d^G^bQa9rn*OjM1}u4n*prD^rrqimz5k&@$-)a>k^~F ze_?_QqLSUcBI81Tp|S@?yHm)Z_`eS!eZ7N%+<#sGrz;m673A*cP40b?#m_y$Ki+#|8q7a}y3u2b4E>1?kBIaR-56k}gan0#dq+gLhm#}yLjW{0 z-D*^Fq<1hS4&WUW6%ZC!MXjTCYfDt9|lcCY9;5hGo2=4_5)&z_Ay>3@P0jxs+ z&`^4P9SZ!?<86u;U=+ZD6};dE{mk^oK5ei(0^z*|!LCBES0Hqzzt`;k&iMXS$mX2C z@Ad}c%MjiX2=+P*y9lK-{<%8$pQXCLLp&LRZVvIodp0m$LU^;G*h&U$H=K^KX$|gw ztjzr#wP+f?IqI+e`7`!qChWuYD(X!OZ~tRC?*9+$jR@GlY=ZKRL9t^P*%U!reAs6_~GArxE6ifv_~Z^GuKwEwxn_IL2v6o$>g|Bmnt@K+F#0Rf9Jor{2%=HAGu-!Bn`q_1;IY#z#ikEgZx}&`#bBGuKz*u zzf#Qx(<%sWKLop<6WflWGyS`&_SXX2-vMqG{39^nKT^eC32FnO9l|>c!S-`uv$=%- zuB-jK#|zpE4`v2>`i@1LJ*{sbEi&<5W5jj~VlN2LkuaMg{YL%Yk$xN-KO3?V zgPjut(SB(2caN^u@6`VtZT(oSKj_5})CJ>B$!DsLZd{(D3-wrY&7_N71=+En-5nYsRl$GfNgxGcBx@7c) z-S0`6mtU6Mon3n&feC1ea5;H>-;yS5b0b^t)o4i z4dY18*74427Jgo))M!v%N9E{sd;-CdKG)D#+2^}Fe(P)3L4J}4RE$8NsM%2|>%#-N2%M()W}=cD9Znm=eLI+Vn@m zb)~I|IvdZ%M(aWv(hWl2h#xHk>=^pHINJq~O?2YP&Y@Ire;t3t2ld&ohatF$^!CsF zSl>ENQ=I^pov}Wi!A_CRs(w_Lop$jAtN35#A;!!9B9GC!lt0N6NS7zdYp042+0@C8 zWVY^#;MT>^_ZPI&Iy7B|4gJgle(*EdT)%aG!K9rkkm7P(49k@Kke(b5UKlmlb`z|3B7)!3MT<)2;W`UCUG_ z##vRv&vU0E!HYg$>xVxYf8U?={rhRZ>7pA3lAKAQ>sS_ae^GX!f6xcM+vlJ9ZNy8E zQydsC(Vo7<^>_oBg2MXJ^BFk4uD2~cd2O`s_@EPk?I-+o{|KT4nM9JDy`m{W5#Psu zEY;Ts_%Wg6@USTFNJUC$s3JK$JR}D5{fA2o{f`ZmaNy5L;Ln}^2mgS3>;M4%VV%nV E2dNwirT_o{ diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml index 5b44fc818f..a9ed5fd7b3 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml @@ -40,22 +40,6 @@ tables: collected_at: '2026-09-15' rows: 768 status: complete - dsv4_hca_attn_module_perf: - rows: 3918 - status: complete - collections: - - collector_ref: c79cc132552deb677becc37f8139b0cca2b08d75 - collector_hash: sha256:ff1a5d288300da98a3c6f38c4894c17ad896f05384afcac3d966971f27130b83 - case_plan_hash: sha256:ff311865487afeebfbf526849e84ff3749335201ae74873e572766a06f91c63e - collected_at: '2026-09-15' - rows: 1960 - status: complete - - collector_ref: c79cc132552deb677becc37f8139b0cca2b08d75 - collector_hash: sha256:ff1a5d288300da98a3c6f38c4894c17ad896f05384afcac3d966971f27130b83 - case_plan_hash: sha256:e6845638359ae7fe7aa5ec93a5aa90bac79287cb3cb80368d0e4a7ac814ea456 - collected_at: '2026-09-15' - rows: 1958 - status: complete dsv4_hca_context_module_perf: rows: 5840 status: complete diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index 5057274a80a35808d2944b0cb7fa0f31cbc72a56..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21704 zcmd431yqz>_c;2@4Ba_|lz>BbNIOHflr&0rgETUvbcmFI(hbrfpfpN@q@aXIDxe}@ zzV{h@%TIsb|99`YcddIC%yV-0-e;eQql3~DqQ;^oqh{5lhEf9*0KnNI@RbD*1prt8 z02pd&YEc8!aB5Co>O5I+8V)rMgE+v6g&Cmz{z^^`&ilTI&;$TrU?Wc!0?CsFK(JKQ zh@Zd`=mG!^I75(MSd3rrcYtwz4AK7i{BOXSs0DulP8tXRgdlKjLp4P;6pOsD;BPSy z{%f4}-;Mu06+!evi0#3z2<0RbJUv(y?>jFYApnqg`Nim z;^W}T=jrX?Y-i&uplE03sc&cJ!e=PTr|fFw{FAMbg#!Q*#DXd4@2?oO{f8OR;{7#5i--Jk5**o z|AB^j2(_gq7>WOOI%8P~um}O5V4%hW$N&~lDrn$G`#Z-3I#9r7Q)r0R1`m`^Di-xT z>|c&Xi}yXigEMLIAperkuf2au$Py1A#s&0;j&f~JC}dDV)6MYQQSA=&}MU+dZ(pS;SmuL zBrgm0>EQ}{VMWXQGq)69G7HYqrNYpiho2_<@@TJ-Y-L^~a}TE~uyfo>gOuhosyhxm3-!P_+0a75)CVChCQ}h5U?q z?wL=g^6GSwJ_?Dcd!uxRN*uNwv7RSrlup*3Y4p;)$fV4}oEwm@3)I6Ne%VM>$3)_KZa$t_#-1>p@do>qIRT+p+MdV#6t_#3Napo%(_;Zyc zBi0QvUuJdn)MGC0;i^!2HheYeWzOIg8*!=6Y(6LvmyK(oOT46Cv$ko7)qd%U(D)aP z#u|Hn!%h|!&IB}aAb&*YlAXzx=gpRn2$$LcN`%|7E+yssRjYX;BeM@MSI3!js`@uL z;SyT!INhCz(qh9;rS1;kYune$KG&@c%HZ%D3afDJ-SwVcQFzy}fh};2?8HDgXDniv z(7KGljX|cW;NzIY`f0n>Qa5I{BdELeBfbp+i?V2h?ShPLE)!a)#-~pWUIl4E$yB~n;S1QykJBC2i-f% z{u`PkPGe3KJ!D?i^!NySOoaPu!9Tr! zTM^5zJ5K?Ox(=dA4JHeJWqwoFp+FQ26C7ATYsUl>jKyPu3=U}R7%M0Hvn>d4eh)b; zVE-R~kbXC5TG0506|?(|C0;+y#TY%eJ>c|f9?I7c*`QZ z>!QH1-k%93b{I=6DqfPaa_~;D;D}mvw?qnZK&YT#z#w?k!jvyhm&6syQp9k3*7{9{ zhC_>>x(pFyBP!ySX-&sPfm*n-(-9t4tiFz3b;I^fKJ_!sw}KEdF9LPvhV5siO{TOL zm=FM~Hp;o-UIDbC6)yCg%;!Btcg9|#2Ik(U`C2B zv0oKAfUhUbljxld#7fMV@MS$HEoIQXlRV+Pp2PscB#=_$o4?!X+b2;acCu3X)R6Wj zJ{m(vg6sl+{#>6#1SBF4L#C|1Q+}8ri3^lvg9^ii#h!hOXXT)25Ceh?qKz^nY}r1j zp?YQoWl;ggonLoo>(n6{_T6e=L6I2sON?yc_byfHNPBZn8 z6*Ltqq1af>5ku-+AEycnPHcp>7A+_;idy&tr;4B=8Wf(ZM2SQVe5l0G`i*1|PokeD zC0Zi70EB!vvs)EC9R-Os;Y&Q8(B(@kZIUCri5~`%IMG*L>zNG9i zn0NXnJM$8OqMapS#K8hhKuxR_RM)%}FUvX=J4P=mhk{RI2x29$OG_6!lUq%8x;QJZ zxj{S= z!iKRq7CeO^F5b$yDW5DuX{+0?To4F=F|CDZ6SpV|Qc66sFkGAvXr=pW5 zMo3gH2W202KvA>3#Y2O#0ZlknKUFop_GHDW;p8?bal&E-oHgWtZUXgfkR)wFn+$s_ z2nndXxw|$PGiKo8#28~d#-jT%%;HxJKI=>#1nqpq#otha4>b4V34-EgNHz+zelCo7 z#+|K1A!I_JU>Z(V9)LJrVmwDj&(5@ETLd&#P@f8*%WYS{mEpk$ttOSx>mXfh1nObW zATX9b(6+7L@L?<@czoLAzF}}OsC!03X(GIvg-tl1Y83OJpjXNc`(!<|vM1)O{6G)c zD7dAu4LXYr4$OTb3WCLjdtvYy9u8Rp#zALeT zO*=0lN6sRuD{ol(WS+sH`f2s495&I(y9jqVrnCnVUqvd~)rwE^0ty=lk~4SkH(eg` zUrMGy#0{EyTfTeQJ%szhM;fKj(}qh>Yz@O_;?#|fXJrM?i=HrQERp$y0hNYFc={Qd=od8 zUgzx0D4LWDSSX=o-#v7Ttn}y@3P}q$8BzAuIuFqWzIm`C`tFfRsSlE~;t+A*V)qu9 zr807ogbs8Z4Ky>R2lk^jg){E&agNwDSB@>H88)#4cE)}5yz5T(j2~EC(D*w1^plPZ1kTy+HdykzlQ!fpZt6(`D^^|ZSQw`_6G>{ zw;)aGS*E{tZA%@Q+7G~;i+rj|WSpdK#FRMF3{++IX45aXTT@5*zC7%|ze{(U$*GH3 zUCe+%Wwg4)j9w$ksgT+LBEdps-g}p8Dp>H=@%h*2x2$B1NXR{`U@R>xMg}t517oh6 z(qaK3Fo>(rnP4F0kP)4*sX^Yyuim>FR64iWoJ$^w-eK#fKHeUEb+e2{fhH=X;}{j5 zL_yE~$lU9~N=5*KM5D6i-UxTAlZPI$rdAsrZ<=D=hR0Kf!2Ln9#POQ{lpw1pvvt69|L{ z2VTFPlr~)iCDc7Ncb9T@EjYh!bT@Brf`nk^Jb9iolsBcT=&*QTlXmBANlJ!?1)1LX z^WMuD9p@zWEw}V{Ti;NYT}+v|!3)KHXiM{fkGm2TMvqM- zHdDYXy$~0n!~5cRmOBMXoQzIq-lDVYk&Kq2h7YrP-YCi(bXHo&>PnRS&?*r@D2O7J_~Jq$%I`uNLXrud)XhTU=d+5b-4aUJ37{vu=2Fti}h|+nK5) zE=dmkukqEVdm)ppbjgK>N3W9*^i}L`_uLAH^C$-go5;or8WmZ;)THp@HbBV&PE;Aw z@!4*z7DBl2y$SAr&5aq#EI%D;l4?9-kJR}#v_4sO=l-WPYI?M@~-3$MCci5J}jucx~X&lkY?g zblkxfCPOEdn!63%26eSHS0PVji3{0`_Nb#-J5%tnx|PIn(zY#s9?r^tC6fvNbH2_q@|0-iJfwcCLmEHw}EtCvyw(E(+>w@9^UQ)Y$o+l})?_>7HN z-t$R2EtQHZysIp;+jWugPNx zQ(z60_B7i`KIOPU1t&st<{5`^v)U8nnJm2;u<)|ldHkM(0x&0zrdxA)z|JlN?fpV| z?gQZ9C2}yNNG_zmHZdXkMLtWpcRpuU(-M`8+HmE~8$HQ&L6+wZd&x1S4A0{^Kftr1 zd!(MPm(OiGNNi_uPrrOxKdk~YU+%hkl6;{sNwhq3Z0}HFDzy=7TC(e!DuMokMUF4# zlA9y8FEZRrCXVB-IIiNXySnTUJyc$IadDB~2nZ;cxV`F+33!&3M@^ZRIP_=*x9pqf z9vjDJT%=sl1k8x!G`cR{>&SuD%CQp8yNhQH&pEf4w!GlAnGlJ%!_S;GG~kw5g(fa{pMX|t3OaWk>x_pc~mNQtx8rhkju!@ucZt*Ol@v_ zS%W9&PO1k)tPZRGuD$Z(_;c!Vq}rg^gyi>auNUozsSKq&=g@d3BqW3_XCF>(c{it7 zW7_Loog{ZzVecz5(tZJ7C*uj*WyQMjs$JG7C5G*#nhb{~#jd_g^|DFR3u_I%P=%sL zri`7h46En%Ubzky(HlXHA?()}5TOzbaILc`m;NNm^FbUW`NpV8n9ySON1Y^e`E#B5 zrH>!mbvW7YI@K$V-Ydud5E2@A;rYHI0c_|oI*5iQ*@up-=N8`il81vWgZ{NLB7LYy zzJ(d7dzYsqZARE%yENar7y9M(P+aOIe@~9E%kWJqxjk7)lSEBKwM0(dmjGT&NR1B` zrh~BKbi%#m>fY~sIM8xPU+N(}$1rtvqf!Z=xImE@+F>ql)d!d)315r(s23L=l_hwY zyN)VII8YqEVdbJrY(S&Wgn)1P3!M);SPQXb#u>s)rYrg;_VU`W4K7}`4QjOzbcp-O zLE1Bo^eZIsW+t_tL7mC>Uqlz=~}njANA} z{mHeR8P2Ia=^S3Wb@SGXAn~sv<5;cP1M-N9C52Z|90^x!7buR~vp#KGos}M(th5*g z3pBpoy}JC$^z|(Zj28{Vd#4b)sAQYo6~UyLHOb7O28= zFnqO8s@@u+{Nz5@jSJPf2-xkp`$m|ZgE)Z6;=wH~avB~^n!4Nc- zfvt=VymU%#;+g2PwH@5|f+|(O#ht%1;A1Q%uF!l*ip+YlDC7b^_ujVj0)rjGFOiKQ^!B|e? z!D7RHgPv`5Hw~P@EN^t`CJHtj|A#2HRHFp^BX6cU?#%?p9GhBWwA#DP?M0DcWl29p zrV`r9XAP1!7)>7|fpgGd9F#(7WHGwfXoZJUY1fw?UH;Tb^vqH8#L;)L!I;SH7Smel zgE{AJ0*hKBp6gPj+T~<&7W`RHN^-#hwbBw^ggm5wvn9Ue?sOcwijhy*=;~?c_dj2P zS2K>HRj;NUMfbLJIjIPuDdKqLx<1#xLD6KvqL@i;_7~o{bBF#;b$*81JwY+Gj4(fY zd;60X$UXzk)aVFU9_NX#-Lgr19LexrkR~RXS8l|k&1`wzXSuZ-blev!E3K1YgtJ{} zdsOi;{*SH>w`CVsmw8_CJ;>BU!e^BFJFPp>#q5mHjR6cwu9 z%J&znCs=)_P_A5j`1v9Dr|{*W#{s2jC_)I@;QRT3&*R)M52@Vom$}O8Cb#e9d^L8M2*!{=4) z^Q~`4+T51s!8lrL2-;n0`kAR)ZV9f$jxXaK8731pgx&hX232;79PmvX3G}AoMqDKI zbURpD_l({0_hKK%Iacu6GUJoK)h{pe7Nrjzj4oswJpg5GaRY-|>BcJW4VQq`=Z0?V zkHNTnnZ_R!)q&S?tJ~*_?3f|=j@!9WH?!(e>GFVuzFOTcBR$?k40|P=BA*d&$L*W& zL-i%VEMsIfSH(j-S96@lL%dSk?rvIA6250cJyJO$q`lZvn800S*U&&;SB2?C%!vH6 z4RUg?_=z-~JC`IM*6EioWs^5<487YqCgY^V4Tw@HE;SA;7LE23r5~Pl1nuKAG%~C> ztQuD$A${6k6xH|jVAjM2vv>L*ca4@3=%N|vR^gKVF3W9hn{K;q%dKx%&;yp+8?A1K z)6cnHv?ynM^Sza6;jHlG{YQuSd~N)pI>C>wD<40Y#gXhu=W`!~9NDLbU(5#{3*KRS z8oH8qGr_zv`o?(dhI0M61f^OUx7$-JbnN2^gVt$-31n08K6Ib7+1+(@VS4nwtZBpL ziuE3Ducl|@nIx`fDnZfJg@T78hSN`2Pi1s^q-9BF^4BEhkYJ{2+w;8n?dXfcTG9`b z{YPP;At5Zl(#P|}@$%atA$$`H=byKRZfpev1o#I)S0?HrmX}&E7OwSNJFv3@Q)Ioo z_CH|KQD)7MZjw<;Tw1M+jqa%%tGyohUmxzSd6JI_jZy&c)R%1BkBP-1X6=k0Ookph=cy>jT+?Syw%yZou_-K%R(Oqj%s^={p}&t0!} zTh@o3Ogi|S*%#%y@o?nD#*C~!yE;Oh{lW7^-RD-t6>6HHWRE1^{TQBfd-t{O)9KF!}^jV*J3OK=XQU=g-oUmz?bWqfFFvExC$k zJ=u7eFsNi=-C)eCeT^3h&qsf{knXJf~rfO>jmRvKzvB+MQ zdUY@vhVLLn#b*9APK+B}q$Y@#yEj1>CY28gvl33^Uf)B;x%fOBX_*WQN3zgAGg<9^ z(0)5)f}x^nGF;uK&5Q=)*UwCA=g+XbN*auBKQ!iLI(rp8=upm|Fu36s#|Q^cu;0c&cQNySPV}YUiXwBX2MOb zp$o=9sFmS8#(-jbe%YwbwMDNI%*fPh^(qaVX@f*5;y~t%5ikSl+h);E6aHk9pj%c@=3mtN8sE!g$Ik-#-26Wj66s*u={2gQ}X8|9W-;$Vit9E z+LQZxTbrA=_&I)*Hc&bdLuVHw2tm>!*K*X<92D3!-Y8YEn>Uu9 zJ@ayN8#m^KPESotjEv0N)7-c>Y3JQx4(?Ph-v-DeOXXALeP;~h&m1v5l6;Ye=P7G& zORFIm3K|rKuY$sutCAskIx{l^4R{ufvhs{mJd2IpkGvXquu=)T-X@pOYk8l>1+%+H zoCbAYy}z*k0el?JJ+bA^c*c{ZU}W7coMZd6czd+Es)~Rc6N0^y?}==G@u#_n6Zz=R z1&jyasuvGFYqY4Bg1m9ty>&buF5bsvQtm148Q|G-DZu8E@=^2Q)2KeI`Y1hqBAjqN z6+q%Jq%{5L>*S7?jZt<8A<-p_O|>7MCY&6ee*Svn(KR`_(}h;U>e=1p<=o>pZ{BEe zXQ-DBK22M_^Mp#J@v9Soule+K@m5(6vzftUllNVnqcnBtulCg|Q+Qm;9F*{Jg z;Hswt5$@?%R$bQK#f+_m)ek{h(f#D)0u5oHupSI7e2-c8L%I!9F#Ceybupt<<_*$1 z&`|iZMqgP^zVd!k+r8n}B1Qa2ijUAKfhDiYRQ4Ws2K>lTlwRywerDNIC*K&&3K2_Z z?-;yR1y*u3t6RQoWN0+H2ie;AIa??kPdPvHr%TTtc1p7d7NdKwpd*b=$S7c#dE^_Pw88v^fT!~ygEpdQ&w+WLi&-+#omYWO|Z{4QVQy6oeG|rycf^B zYtK|)lh55^W>~uydKYfdaM)L^Z}vXK#rG1SL?BxOkL$8rV_1ud+JVUYLDAOwfjrNQlR_?~G7m^1pm^?A3KY$n`%x%pi2)TVXb zvs?WY-8F5nMPk#A;DAV^1pdQ)9!_mw;zQXn(xUlgWkI0ujSE`UTkpFXNmg!kwm8v> z_&nrV-Fa#3<>OU$m)S_Yb9ep$X|}Pz)cvQ|FLq7OtEdc}dt%+Nn=u>S|E*;B?M7t) z!&j@?`=%oIom+8mRFuD*Jp8sS+?ZdLz9Z!7_(gwLcED+EAULu_Y^c?-HE@Eq_FP4L zzD)5j*2z}Xx896vJuFK%wQ5^&GLNyERYh5Sl-{=yPnHcJUbfo|nXT0{6i=c2r|u`N z+e$tEdiTb?+Ib?1&sLF(1XZow>ciPz1IbnnJ!dn+BXRle?za^-l@G0`zO z(Y=Y#NV<8x>P_UP@#jkG%C3y0%Dm+mTYsfNOPnJKqbNB8xt7Y#NZlSB#}!rS@;ANg zK9VcXjh6QX)9q74w(2BgO_Y;H_4(V&<>oEb~ro(`rN*GHIH^hum`qCT;tIYf{ofK~!e> zgYPOWQl5Tg-1%s)M8#A*merJolL+ti9#y-_c=Xxnbyi1VDpSJRcPiRf8R>u6-zLc{di`Rl7d z=iWUJhu{$5`lL6l2ezKfvfDu_riBYTd7*5qxi~KLEA!KD^(PibsdL6NSRxER<_jv(^BCdg25<+7XvBH5o17=k zN%F#dJZoB6ZzXjlPg3QHymKQqd<8&lTAR*HDvj#CNS-}BW~Vtg)}** zB)7{+jWh2=mxSlPwR`q;kvE_~ykuoqLfWf|Y^KYs#(uzrlK781*U21O%w`on)aH2b zN3}KiWP2NR46mhCoxfBqpe-`#89*KyJs1amp63u|vGh<jw&vAJ?Rlg; zfWJv4W_1snYkyKjTXgw-6}%>B+l*68D45Vm&Xq3rDAkl-M1M=r!_38GTYbPZi$8;I zqur9JGjNIFwXwTOW}c9>X){$Yu4^)E>V;w_S?6l2{iwyvE<3xq#HcSG`Y z5|*2!c*9h*#?#PBo5w^|<@L$e8+IBm*-sj&yrMVy@GP%c4vzKInxDo*E?Y<9Tnd!1 zc1y}1-CVI1cCHn99pchEuu*e#k@}{b5Z_UWCStf?QJH61KPK`~{9@i!Yk}Mo&a1Om zpkier!(Rmo;-tAD0v}##zKZ=wdTqQyS2zT=J>;0y(oyOXCqdKpQ3aPLm;5f8Xvf>$ zL3^e>4X5A&SyRk~uGmYg%c?5e8R>czW7=T^cNFD_%y@$?3Xf@x(fBZz5-UCor1*Fk zqqeamy1~ja;bgJ{{cp;9)N1GQrkf z+*oom>np4HZ0`-RgAWa?HcT_z3!&WA%6p*}nv?638@aJuqEWkj>5%RhiPO8V?5~V| z+rn`a%w42RvZ9PbcWTJA+pi7AGT-~u^87Pg+r=(oJjExQj0BoQ z2;bhAgv|x=?Q^#6zT`?7SA=u#!p*kg9R@GUXD&;$u};2?Js*(IljcN*`^NwI=V)p% zwTDuJ+{N{J!qs(nWD=VUme4zs8dKH85O2!kFlF<^Vt^m ztfu$066JqgL9=IU%qq9(7TY3UQlg&PQRMVc(WbA-aFSSx!r4f$HQz;DaE5SCYxxEv>XiL=iRpixRuo^6Tq5qh z`_Y@Vo80DuW291=CG8kwVbDpA5V3BjJqu6v*m?lXaC>LmMZG3zXCjwe5o;9p1bCFe z!9BGwV9qA8ekYBpN5-wv$^O{qG)b(kVw-lY(XyyEx?4~#{j#-_J2qYTQpSas-MOa^ z1B36c1@}?MDuf3XUq+`9F@;_lI5_*mwWEW!B4BieY&kXBZbu_*E-PBhi!^!kO%r=t zuae}m4`Gi4o)T3R-d4W*f}jKkno3azE3W7qr7zqzPcKDtyPR2E0?T3ew{hJDV< z>cLIFuN77f`BmGp>A@U%bb&pvU(FdS2mF8n&oem zMIDoQ#r&}fou~xj<%g91jj2)#pM=`i2`io_h7-G+mf~flb9dJmgxmQtk15@>;1lCZ zx_ncmh7f{Q8(rD5jLg%+_CGRo5lT(t#!8pPh%ZRkO@FpiDu^mmVSZ&kV0>Q9QH2UZ9#bQ0tGb& zU4h2t^Xw(GB!kh7p>f3Odx<^CA{llpabA66GhG@!w%PIgIssc0tCkrKQ4;mq!7Drlurp!MY;6RcS=)}rX6|-P-siNZb zV*9AdJt5qBh4EL;(A2e_gFKex2}X~t;ms8wUtWafoANZBYf1tkZwW226lnWdl!X$# zh-}#v=teD-Me^PfIVvj9KP^xetMMXswN*H`yrz7y>n*Wolmf%cGiC9|y2)PIkqp~N z6{*<<62InR=s^Ik481O>EF^^a3@Ib$8BQ3&LYZUSgV=0{3$%@3qCP`nbLx;1nX^zb zrw(HaH=s2`)?n|9d!U;rD0Ki7>qG*DqwX0Iyz-LitbJJaz$+zqMFIQc)Ue#yG-+6K z0rMX_xJalxEd0z87e*e7#M1*ci*+7f7Y`TSml`9nd7dmTRuLgj?qD#$O4F;Ve7BG& z$u)|N@$F+3Bn3rUVifxYp(koOwqI#Ln#)cur6|; z@Kupevb+K;mL7b3RnLgL0whN!3*>w6>nJP^iC%mi07{Y9gT=D;1Ya8+ao1R$O746# z=Zy`@0;~(1@^l2;v3gX+1+1L6b_R4)DWXa^?YK0p;(6?+smf)rxwYcrDf3t}Ds@V^ z4Lb-78;de(Jxh76ek3sNW6ivs7|qMirJ*tA4R1E?qj1t4HC-%%v^Ed&xjXfmy?|!5 zcMbA+JB6E@3ep5kmGa*$z?FTyL8-U$ilb-|_v(#z>Gx7u1^CJ0uc8}i9v%-0gh4bd zu|#P5sD=c=MBkdCID1gAOei^!$d;8Y=Lz@~Y^#byJO0R=F~=do+;vR{oVA?E036Y~ z6!zBt>u*z;D)GfISE zgRER)NRq_+R&m}d!E(uoKoZ{~w)|b4a;fIdpP9Y<&cK%!I(d<;9mMTJ$d6y|JJv&?MnbbswS2w3$(Tr zi6cc8K~YiwDsrE6CD8`SEci~Uirlx- zk`>uCSh&>a@p{d83rh+ria1dw6zO+1OE^$m>Uwu5GOBD#d91iJOrKF?J=iQ22nFPA79H!@9CXIb~FU9K+8QS-GSr?Au+fMc;9&n0jn@tFbEbrg>8nXNXY`w zuy`=A&=&$kmOa`86=2*D4439YLUPW~I7e6!GP{-Fe6aTjO==D>1?wTPnbP72I18Y6 zVux|n*WmY#oG3%zrC<8CNlI)i5BoR;n5{rL+Z=l+!kv4~cA!)xcmy($tOkWd*Eto> zbL+n24Ql(36&sb3k<%jz>I6K&zrzCNOV?;TZz{9NVbI0Azrin$8a zE8P0|1Xn8c=BkY6xQ)6A3>&;NYV4JGjOMBgTOUx@d6)87yoopJem8d?P0edXoM1dC zJl~L-!E48tU^4DK-&8=&=cJWjI@dVgau8MPDvLBDScKyUdP1;mi=YHLNNip-MB0Ld zlHd%$TY<*oT3VBX;jxX|Gh7p76aH8c0me_wkODL2DJ%kB@_53K9<#S+^H6bJsAn30 zwcet9$3VK*%e^wNKAz9({UsKMcZa2|Y*KOJFZJ!9Zk{rR<(YO7$))h1uB z^?AN;Nz-}$vAbV$b^aUoi=Afc2ZkL!83(Q{(l79lK==FSz>SN*Bwmk5PcY??Jgn;? zgfyUskm-oX6Idrs2^f^ldK=po|8hFX-B;o2yW1*luY~RgP6RU@7jbm&XdU9tR$sNI zu<72n5(s{#F=p91EPL!b8N3R+b~fNp1xrc^c_p&?aVqQKaivG-YnOYUUl`S#_6pp1 zm&^Qh&*9PMb%C&VPglR4EnNEJD=8X6wFclDBVo!`1UjfbNLm3_Ig4O8fjJB8+)fX! zykG=H9t$>405{>)qX^ou0-TGFdr5*)Bbl+R@RTIvDKe*`I25f2uC&X;4Qr$ML(UV~ zaQ4&9)W?XItP^`>D=;2--c)|M{xoQ(pBX9?E6-3w7H6!;#zY>cX0Sn#v#7`+M;@3cQnLV$Rkk`e`f8ae9#MVoVWVby}9H^>@&`V3OwPj=c zsH&>0cMC6@9ZWv~ETkm`Nly=q0UeI3LjfZJLn!|g9))9pC|L!2KFb=Ib^$Olu$NmW zoIn%`CYneDcL5OC3}BTuuw$`+sj4|edKif#7LF8{&l(`1v^q$(u2;31ylYr|EX3B_ zXfT5>qYTx|kh(VcNMzyUSkHtrIW4K^uW9lnj2bEFNXYVrG$4dGA zJ2bAI%w#>nvKNfz8ndEk${vzI2&Y$$*$Fr0OoSjri@C-fwbSoR*OZCfj~lnO*Uw$v zw7>XhYn(4cAAphp60#xygaLqHf!R6u13eh{n-UyB0Lrq4CL{yQ@dJ8bOrT7VU^ZQW zh6=8MB1A%{&(KhQ7Sg+=snwx!qUE(cDe-~G_ME&EpTa?psN{GrojImx2Q(05JuzJ1|9!c5u1Np2Tl zi|xpf#!p3Y`xX+paVju=(uq|zD8l3}7|oGc7J&~ao+mnvq6Z)=a8$xyLrYtL8fr(f z34(e96Vw-<&`Kg+;iR@3MTCzBpz3JSAmcSCjSiT6r^aE2ASsy_6%iY6A&ECEA>vC| zcm@E0PLzE5^-2w8u?N##Kq(dfq6#L0B;#OHD^{AhNbOCh+z^#vA>QRtbWTXSd(d2n$xM@rfNR`2`~hbrVJ;i-)}ORFs_cG9DO|;jD{kg%ay$?ah`W zwkma!stn52=c-IUR8jOg8@8yE+nJZpnZ0s0>i%@4KITddc!G0~p#&yDiIRB&VDby# zN`O@~c)~jZG#rsIo0cdu`6GCHGr-S>&bjac+LYN#Wbau>xGiBsKjD?^lh49qVx&8N zrKYdOR+h(P8svQl&9$z}BfaEwx23VA);*NR{3Z87$4pCIVdK8V+dD?x2O0N6KkQ$F zoHOpd)6Woj!PSa5*m#4Sw825G#+qhyafsx;yXrLMgX1WESxGpTY9OyI8_&`cJwJ!I zXC`)hxfS~*2aVa|HTJblCa*nPo9>QpJD|>)PA6wJmx{e{(#sJnaX%(YjdregLJ~kl`4vix@I>Aj|4Nn zEyK3gyt`dzLzNG^t7$slPGdU zo(qJ1-e3I#%;O>OBQ(CbC5)Li1y8#^f@W+DNBXQ6JTBTijh`!Y2hk6{KqJ$4I1tAoZCoWdhmtOQ{IIP|DELUT(>gr|l@>NZ-*vImmXgJk1Q#rLo( zfKr1e*pu7f1OUc$!Ajgdr$JSzHy#i5H#yJVhuHjIba!?GuD zQ^FZ+83_U(EAls{r6g{$^8`N8bZ_+I+S%k#kQ~vwE1Xen{>shtj0`PI2x?6b$ou~8#+nxR9tDI54ZuK0rCmINk?24)J@+BB zj*8ga1!z|bFqDGfMi@M)+gd|1@dok1US-PyC?#Gwl!|PE&;U&MDipy)9U>BD3E^~H zzWi>TG;~LS*;TUX67UMH0jA`c$R@Lu8EAIt61D)@Y+`i5RxaI$y^DQQ;j`EFs5xF; z4>(nx71{RENKU@b3%vY&*KO%?+R1^jrn{$O+j6H%lY?JG3Se(&Q9f2vQT)w?q)9u9 z9G|A*uZk9*d$XgWAT@LAu4pNb+pfkkZK1JWT9ZXj4ei z{cw-kx~%GJ#QVmbrt{ss<26p9nWi*>Dy!2ej zj}>8j0iz98F0xw?9L+%5cYa)`c(UNmCV6XmE<+6M)~Z=Pv;B21Qhp@4nje z{TqUvPoKH_pT2y4lx;77#-C^)c>5CudPapd9pS$tCbM@o9(?<{{ z&v4SbHSCLLy%}`kQTHmW@r};<=w5ibp1ux)Vtc}fDzR?KUl zU1Ip-P-2Sy%X8ZS!?NGJQYi&DS(!gRR=et*evjc5*X528qltH!onKxptTB$+@W0O) z&D#>WAL!%a{yrB*8X$ptbs`LW_bysl8xQmmiBL3*7P*a!>;Vt6_ucwjO4x2^!QyZ|4&bINIq0($V%v!2hG*qxL)I6v2V5= zw9x(SdVR3$Yis7ui9^X?&;&y?7(4p{!3@k2dQ^agEsQ$3kp4VdinnuVGTCpv%;JrU zL-6wRZ+`8=76(4%!NvRTr=2gm4+4ifU#RIsbf<$>zyet{^JLd(?x7HjG+F(cY{&aY z{wSI>WDWaLKp)MEqnj!r8`O^ihAzH%n{uHjo zEezgPZg&12-YypaO$P9%OW%8O{p`GboIKqB-G^gq=jUW&_wO!T56^4v;PQX>5xdxV zyW6>5^YOUu{qt;W@PZUO*MH9?vhucZbn*pn3%Tz76A*kiSF3BbzJZ=U^^&;wU9+(Q z$^Uf--@(q!?b^Q=U}2bZzwUO;(ay@&2V@^?lY-Y$Sh-&Nk6r?6D_<7?;nV}QiQ!_Cv%&d2APx0SDx2S7~#g2#2W^0jlf2?XGIvUCh| zUnmd&12qwy7=S}Y_b)+H6M^IKcnG`>f>1`_{d%De9WVIPGJqP2x!8gXe8|9$jv;h7 zbO`Wn9Z*C6hWeMHe!$Fxz#l*mXAndy1OxTg+jM?~TyY5fJ>);t{R40_1ilDCbioiu zPz>OI-<$KFm+1V8qPY-kNh78|ADy(0w01PMo17*BpA$p zUuyHy^)|o4U)zQK9{zuXiXTX#An*zZB8ChxLW)5G1Mi~w>z12eLC?1R4=(plsQCf4 z0s~2-V01(&p-@T=U>j_KZBYzx4G<7Y{(Dsi)BaAHqKSZxfB5D(^^GY(9}rG+3q_xCK#$zP$-i>r>LP6bwmUNFyM97zR&p1#CP~VrYK^ds%ZOJ z8_Nfp8fym`i>N!=s<`@r^j;D0`;H%lx`LT2uOwjqCq0he*M3LxFM2@6P$(M_Jy)CW z!zd+;sJ~Z#tQY)0tpCoBt*X4ejj*e@sV++J=k*FM|FQlD)E~GouNa-8tcPKja|~ zDiKruQyO`p#L&MM9x9cr_=11$j`>s;5k*#q9OQ#=@Pre=teQf^F zN2xM(6??#ksrGQobgYku@k%eAn%L17Q8SdI=S-5wOe^RsIO~P*iu&rhja0tXlSq5a z+6P7o8ea_~WCw*i*TfpTe(V1sQG2ezx tr_!yQ2gS!}TMl@h4}zjNRI)f;%6Jdxy#gmCIC=i#kv?xE3GTd9z5uhcIBEa@ diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/vllm-0.25.0-collection-report.json b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/vllm-0.25.0-collection-report.json index 1ed5848afb..4040b035ad 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/vllm-0.25.0-collection-report.json +++ b/python/aisimulate/src/aisimulate_core/systems/data/b200_sxm/vllm-0.25.0-collection-report.json @@ -135,30 +135,6 @@ } ] }, - "dsv4_hca_attn_module_perf.parquet": { - "path": "sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet", - "rows": 3918, - "sha256": "dc93933dbfb7608991654d68500560a565b27976e4f10ff815d766f913566832", - "status": "complete", - "source_jobs": [ - { - "campaign": "campaign-ops-v1", - "shard": "dsv4_hca_attn_module-00", - "job": "job-1968531", - "collector_ref": "c79cc132552deb677becc37f8139b0cca2b08d75", - "source_rows": 1960, - "classified_failures": 0 - }, - { - "campaign": "campaign-ops-v1", - "shard": "dsv4_hca_attn_module-01", - "job": "job-1968532", - "collector_ref": "c79cc132552deb677becc37f8139b0cca2b08d75", - "source_rows": 1958, - "classified_failures": 0 - } - ] - }, "dsv4_hca_context_module_perf.parquet": { "path": "sparse_attention/vllm/0.25.0/dsv4_hca_context_module_perf.parquet", "rows": 5840, @@ -826,7 +802,8 @@ "computescale_perf.parquet": "Fresh smoke measured 5/5 cases but collector finalization rejected the existing undeclared scale_matrix_perf.txt side output. Original data and raw side output retained; do not claim end-to-end smoke passed.", "dsa_context_module_perf.parquet": "Full sweep incomplete; interrupted source snapshots have not finalized.", "dsa_generation_module_perf.parquet": "Full sweep incomplete; interrupted source snapshots have not finalized.", - "msa_generation_module_perf.parquet": "Full sweep incomplete; interrupted source snapshots have not finalized." + "msa_generation_module_perf.parquet": "Full sweep incomplete; interrupted source snapshots have not finalized.", + "dsv4_hca_attn_module_perf.parquet": "Published 2026-09-15 (3918 rows, jobs job-1968531/job-1968532, sha256 dc93933dbfb7608991654d68500560a565b27976e4f10ff815d766f913566832); removed 2026-10-15 together with the dsv4_hca_attn_module op: its SDK reader was a retired placeholder with no query path. The campaign ops/totals/clock entries above still record the run." }, "communication": "Raw CAR/NCCL results exist; not included in this operator-only publication batch.", "reused_kda": { @@ -836,8 +813,8 @@ "new_measurement": false, "sha256": "b21968ecb170a3095725a34b45cb16fe19c7dcd30250a04f5802d16a5d3c202a" }, - "total_published_rows": 419538, - "total_published_tables": 17, + "total_published_rows": 415620, + "total_published_tables": 16, "totals": { "planned": 426888, "successful": 342622, diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.22.0/reuse.yaml b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.22.0/reuse.yaml index 44542fcaab..abbd9ab01a 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.22.0/reuse.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.22.0/reuse.yaml @@ -19,10 +19,6 @@ reuse: from_version: 0.24.0 reason: migrated from SHARED_LAYER_REUSE.txt (legacy newest-first effective donor) approved_by: yimingl -- table: dsv4_hca_attn_module_perf - from_version: 0.24.0 - reason: migrated from SHARED_LAYER_REUSE.txt (legacy newest-first effective donor) - approved_by: yimingl - table: dsv4_hca_context_module_perf from_version: 0.24.0 reason: migrated from SHARED_LAYER_REUSE.txt (legacy newest-first effective donor) diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml index 8f27757676..71654ff7b6 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml @@ -27,8 +27,6 @@ tables: status: complete dsv4_csa_generation_module_perf: status: complete - dsv4_hca_attn_module_perf: - status: complete dsv4_hca_context_module_perf: status: complete dsv4_hca_generation_module_perf: diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index 40559a6601b78c95bf6b38296cd0d72101f2c747..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14614 zcmd6O2UJsA)AmULLJ5SD5Rei;0wN{!B49%A(g{T&KXm!n_y5-V*7~-Zv-j-1=b4#3Gw0-Fv?N)}qv2>2TGR{; zMFU&_AlV}K6^9@J02}}Sdowe0G=RpSC8g0fgurEtXhs1QKoU-$z^;9Aa)RsDHnBPa z01O=55Q0E%2mv5l9yIc5F$ zf_C}E8sb|ZfCy-faPI(_kg#Assz-#J9+etmL#6u5*el5x1W>|#WDG3THj+u?KkmZ* z=OzCptpZx_7iswh000e2OP4Sll+a$uAUI5y5*bbj*fY0>T7KOO1}EK@hO?;^Go(X4r|+ z(h~$0@TNe6qW74gaet{wI|%`7p^yzUWdwC+1GUG3`mZ$(y4G2s*0JCR3RJuYj?RJq zs`&S%e^dN7>Ys|ci2^T~0KB!fJWX{N60RB^f3^h03Dr|a@@2cTu@Z!$k}5R(+@oKQ z8J@d~P_7)8{uX?)EG;ju0;&SYfq(w6QBo+$g25uXbjW$DWQ)k1yf9UCa&u*el?fMW zGn@ERz`EnLM&?PCW8ESplN@SeJ|9LP%b5Ny)ZSC!Wt-&MU$4QI6E!hJRK0@ z86NYhAAG1mpWsGUGXLTvKQ@4I^1acc<9F(x;=A32DRTjEzVFXB4Eu3uqzrY*`rgvr zghAtm%W8?Fv4~kX#nQS_3oCvSlV!T3akdr5W|GmuR9$m-7aOCaH-;CkVBGZ1t?=A< zW+(2&(31oLI7DUT+wYdgdR92Ppcut_#}ln?#X*Fw3n^r>c2(ApcXOIJ$zY#8iS!C3 zVGDWuISh;&71&w!p7F%}R!}FVCN#56Eo;|i@>5CqyBPJe zYMgn-pw1+LeETiddB&0n8UPQn9YWn4kwCgpN>?+1JG2}DRX{w!nI7hR*@<9;8;Oqz zWpyHYNK?wC$3dfE(>B+Rvp_p=FU%Yg2$EZfzLFgXJs`0-nXWZqOa!znI0+gMu&k56 zY-=DESbysMd7C=j9+vzRg`GV+Pmwn76WH_sB6-!&(_2F znUG-q$yU$RD5^=_*DDgyE-wWnpIfPZrX8D}FTdU0eD|cj7PRN(A-;N6(Kg&wudSC> zJ&K41OFDrWaN*Xg8GRzk9TXH3O~>vZqK)?56E}#qA5&Pwj~?A!fAs&N{^_n8MB7vX z1YK)xhHeDY=J$AWJ0omf3bf6VmA9c#H}3M+-@L74)!))qFya~AWTzHwT)A-1&+Ap2 z-xaSpX|H}Z$TP|$1(Vyxl=eoyMH6m8LSKmyU$v5~UDlUVO}kPnxwnbQwOFO@Mdm{$cOs(dvwMM@ zRFQEjgm+cHM%B`{K<$dpoy{X*@i$KgsukslT>R?vz+MD?=&0OsPHJo^=m;NGM6YWvsz3PP$2|H5P4@3jyqmVvAmM72{c_9HJvS^-kurOSSl#6q!VI zy;WmvO$Q`gIO40Qz1eaJQwDFsT37s%a)r6|WEtLIVvDi`#}Sc_z(LLj}XBU}@H^+1^AC(t|60fRvl2%wUS#l#!Fd|K zEu=+y3NJfOzQ~`@2aPpLJABY++W0o~F4ElS-4yHXD{>h(kD*1=pSvf6=>RPhtcYXN zHP4;Bw~v;7vT8a9(@$L-Wb?x4J0`E5)2g1*@tdn6xer%@Dm zT9FkxhQ__WAfrxTkX)Ye-L=nuo8Qo;R{?xRZHetWuD!SuP@28-*dKkowaB(xv;I5y z57??}ZAYd8k9qd)a5&p@Vk(8kzvC=mfubs3DRJkPBs2D|X!$x-@}Fqmc>~j{FPqd2 zT2jek<$a5oK-}I3(8DQDI*wbHBn5AwhlT#+@D&v;wuAC-&O)DNap}oUN~hWvIue%L z4woc3EowhpOh`~Gm*!yF*7HePDU8=e_TIw72Hk_r;*`jek>sJxr%C0Z5+z?%ZR@gb z1JfeLd$#GP>#|M^wFo%fudJ%O=M}gKhQ7vc-+m!7Bdfnt>J-cBO68{$5@J4aB%FX-uZ~t>RlXK=+t7?pAm@nxVk)=(^bl|G|^C zN!0@PzAO!yDZeW&o)K0q^YXYRy(~r}GU)F({&g(sj_=^{OrHmhgr41>h{-Jy*+z#g zcJ$}zZe3XRVKIE1_kMmrU1P5vq0fHV=dD|bs7jQNp|Cjdh1!X^@ohznDU5XuBhK{* zL$!C>eZ6u=gisXzhL5wz*1B@*^JOwJsk`u2ph=hKXF23xC-S-y-@OY1tz;u8E_dj< zn%oE}-BgFTI!lt$EX?}W{_tH`(d`w+H=n3&!g*ekWLz#Y^NX8Q6VBb%=L-Wq)J#aw z6G=Z|dEn&W9pyA*UekdZ`8mbe>CGHAG57txRC}!~PA|o$I%{gyCKlic+E|;{FJw~^ zPm3BY0xF_o^vH3#wuIc-dGInOipzbjfsgx3Z=j8=jYZ7Kkp*!i<<|bXAda!aDLG#RO2=0eSu)oyn(KM28BBe zUUU(A9(M5tb+bZuqV$zIe6n=07GmcvFaG0i{JUe9z|4E6L;m@q#PB! z%p69#w%;*~)DfI<<+vnFV2MU6H|ftCZ)3mzjOH2dz}6V_{>AOv*SovBFI{3?et5!6 z@bbD{f`e_ef{|T>igF#ANOcGe~Z$o6Uiz+R4}i7IHIlC zb~mHAZJfLfgx4EmHd!D z8L2GwyK2Q39UqE=!FZNotGokvi4`E zqzj(=WOm7>V)MObkHZ>O@`((k4e|qwNtC+urArI%r(b`*lD*I78^G;s`bW+u!P)m; zBY6`?o(1EszMep^ve^pw3k++3X_@)@i()zLz5w>Hvn&Wn_C2x{W{(mH$8RWTl`PJN zq&|nu4CKFa;NWg87>JT6K<~g847~U}aLv8xgZFT!8{xy}%b!#e!9;q0$mq55@$)W%6op+abg$ClL`~q%=tjo){r9V`RN?8G$u1bdb7%#lw|fG zMQNO{$N2@jdCDZb^O|rKzHF{?Y~ZtImpA z6NdzlX9#FD{)2}-)TP{%O%|#xhZv$PRUO!4!kt~cPwbV<&M5VEk=fjPQRnty`|6uM z!V1}i+q?Ev-$dnH34Zf>g7!E@;+cTk-K)WO`0ab@E+Rse{8wW;hrJZ}teqWlQV(69 zJ$&f8wnTxcO*}h}GDBU?FBtG0v*yeRnfv%v_Q;2)n>m3<^2t}dr5JjOWTHo{^<{PJ zm#?7&XD<6gvwI1TJA;`LkMHY?2)V^{?L(;7g7@OoCMMpm&R(3)bkpBH9@)AxKq>a? z&f%5i_yO2Xd-#c0u%^dFi(waS>peV9)qtS~1HjBw5EW#2c?U;=L3#Dj!%+1yuKSEt zI?h3U<3=B0W?3(5YYl_^{QR6uq@t|Bb_poUpjtn=kCcYtz(O z>e!1_r)xPP`}NPezJC)M7ItD}U|=Bqlk$C4)qwz^56@~f6v4*bBd_Qxa8fIgOLS5w z+n{=6?(>z`@?OJb!(J=1v&Ss*{TD}_^}O7wdvMB;x4<3-mC=E=Sg9{2OszL7p{5y=* zx2Rkq3t&<&yKlZUdFpBUtMMESt+C}-wOi~K?tjeR*>8A!ySBD=P>Xjhc-U+%+nzXW ze(cAroOED9EL>);CGgawhQ_#C(x4i0k5&83YCEZ2h->EFi=qRep+mgtBdy>Vs(YP} zPuN9XUjEG>_qD?VQ7v(MhB}YiXjWJpzwP@rL)=JPDdOr#@%AGl&l{u_ajM62uX!|) zhwi=wb52+jJqW8brKca$AJwx9|InBUt?Y6dyZq?dOZ#@DzJ46+*t?fem*y#F%x=zi z2W&q*>=kQKFG9rE3yKF6vB1+t%g z+RMuj8)SFo;{!I1un!zxRz5n}>(_PP7)d{@dECP1ix%9hS3G za*4YyKN8QSVkp|LR)a1)2pHQpb{ct{OM!31Yo$1^E=B!KV|LUvjx0T|n8MuNQ|eLI zIJI)q$WQdVf_!}LM|YoR5os;#_%^zY-_ z4lW{5>I)2pcy?6)Lf*o-CuMVN&t$AJUxmhLL*v=|s{zT=77pZM^SRIi%$FxSyA;JP zc)$BYuc;h)pGQzk!rtjsm~!UHszV=F)F`*LnQ}&bD*~6AdJ*jm=DUp!?+%@Ad3r{Y z34Yk|;Oyic|8qr83bq|g&{DiK=#p*ei_ISlk7ixUx{luRu2y z*az#*%xC}*-)fiIS5HwC-rtKDe)T9tIr%~{-)S$xK4fy|+uL13$8gN)kmd&gQHr1V zyZx5LJ=n2}E!_51nUQ7jW^y>$JCf7OD4Dd6f%nu6I8afO#ljbvPCe=5KWkJh9XwMy z`k;~ezz{KDa)~{oKx4S@P+yOVl47#i#J<$&+uZdBf@T)N8_@v+g-MJH&>g3@zkP5Z zBgWv8-wqDmxE{0kO{u%$U7YKls@1VJWu@f!D+g2ys%pAzPh}BHooB2;KC|_1@-7;c z{|e7(iikfyxF^=q-DEPwY+%r{^XsBi&w%1%<;{HW4+r=+tynXU-zjRSw?1`@De_Cs zC}OfEc-K_v*8CScTH;D=J{l98T~xTe>TM2?u2#7%xkTXoTO@BDR}O4 z8jrb=-wrcL%uFU{CEIOpK3n+gP0by}@>bK`diNLwKdlUZM!s*BqCWF188`^ozxR_b zWyy`Y@j?6=f5&wEe6I37|2y7j-BEca=Zok&PXM)j+N<`X+4)4jXQw_%^#peB@@s42 z3jt8>e0qIHI2H%nG3Srs8f)U8?1J5Wk`?K9F4J;vLbwBEQ(HW_X}4#p`f^ard;XVa z>^}OZ`wY2V8uin-AGqSxTj#c0-fw))KqNx&iu$gw>mQf`N2be6hhO(xZ$6zLUBmaS z@f&rs$O+4?)#gF9{IcXDgW{Bgs1_UNoc;`*L&>vCh!CVD{-m@AlJHIFEvDP% z^4NCPAz3BBSdQN~=3b04iF>#0!}P+%&eniQkWxa?OFFS2sZ`0 zELbLclWSDYI3HZ)(@@uX9)j6gt^9e$aYV}|xugDAQH0j92*}R2UZHOH)l?m^I;({{k zo$*WCqO)_I<@O3WYonFjXumaWe-Hw`xkOytlL_?$|Xy8NnHN zqI`e@dx92N)qO7@_MtV=(f7##-+6(|2x3ga)s#CQo6ZZJ;*7SXa#arFPP;NlNl+HZ zlB)_Fx|4tOi-a+0sjz+3WC(jc_Nr0(kBQntq~S@~==2w;X1^^Hf=G;`phc|W!=0hR z31jnW4Y3bpZ*(8g9~Q43XjQ=Ja1~ReC*8%xx+|$_sWXez#?G;nhYiSSyU#~U#xk<3 zlGK|E4&Zuf$Wvnx`IL7P1!*kbgcP5-tXMxdCOXqJ&W_*f^5SxH^EXN7T&t*30i6Ps z+1i-0{OAIO9fMpqy=FL%I`&fcn>TM;Ihqh$YX3oUf6_M^~8pOoKC3S7!w zZnZ>zZ8QmCnO1A|?AgSJ;Ml*th?Zo#tDvGmIl011p!ea2`E#*xfh=fD(|LspqCV*# zV?rPHo$@>C{USsu-|x_f!1065LymXyRSws5W;M>83uTxrkCZ-VzAaR85?8#Z%(B); zYEW3=;cx&0_~Ncs3f(64%bRAu_J{&(5(g2BXsM08wnZahJN?z&@6zIuM+4Z100iv9Q2>+&2S5lk2%JEJ;cyTZG7*AB(ExZQgaOe35g4U0Dpf*| zYD5@nl*UBFLAl(B0GI6NezILM-!Md zI~c$P@LfP6Ru2bND25_;it-r<(x0Ha>ERNiS`Udmyi!+iF@#C;DT!nEl&Ec8=Gsti;{^>MBq9|8>FQw~MfTlPwkZ?ZVFLorQxz0y8btj-Uiycb7%Wn*r#x_*ITJ5havG)R^^b_UyT zl%jC3Gs{bDhHo@2MX8pQ?MI&Bzu%vta=tS=C}w8U6e3lvkCYQuJR>k?l&Ue-nG@C8 zE%*x6v2BWzBX%ee!hi*jxq}8JR08zwoE5<6Qz=9M51)kqV0dV74$wn{IuclHJA}}q z08}l6V^0!Jn7fh*rbOgm9G!vi(+_qOia9q0aMX1$zm3n0i*OSb9VNn8I&z_GL>RJ; z$iNQzm?wCIqr~kNG+=+klrT1?111Ifj^-3(KM)JDSORwYOu?mF4Yd|R6KDue9K0Uf z2Od4>Qm7ar!>iB)?x}7eR2={Z7w{(G7#W&qB+(F560QcuJWoXEj6$$UI5+_juM=`6 zZx1|P1R2_|EARxBrl*Hc&oe?LfJYA(L8up}aY|ZqX1eIHVF%I-RJsn}BWjrKmii5e zT=^23q|J6%W@8^qu7Yy7_!g@HleSTYgXen0<>HOaJQcW0?E;3t$9WAI>s*%wC z$ZVM)T3CfjmeiNeu*yj&tYOpNVoEWwDPQKP)6Th*=z~a#Mr06tF zTzV(xb#njFq~_vTX&>WEr(4aK^XI5CLFt3eQ(U|khmvI@o38I%%ICejm@K#V&2?9R z8{5g2LP(Y$qM!AdIk!wi`l4fOTO#v)xcXx@Hj1L5FVP)Tl#T^ zmcY*7aOgAF)@)r4D1zY!w*ca~%PWOSn2m_L+#muz1~5b&jaA8w1KwUKw5mq(T7blJ z-+{p;i6o+m2kb^U07a>h7^FfHZh&5LIKIOh%7q$MII2v>) z=^4W+5yE1L;=frr#UOWvrenYjxmAdVZ<(VJ11eFQ9s*1%V4w%M!ETo`D%bNR_~D+Q zgw-~Ss*{oa_*?w-u<&0*a)lIZnq2f!ZU1N# zyr&@fF(9GV#m63+;ZZ)*2v;Alx)FA;TcAwIP@NEZBfOGZaNaCc<8IT9$TI~Mi$31l z9?s`PUztIlBp=&0#he?{*Ns^&yr%h_v?K1$S)q65u4%oB&D}l4edI&GxAxom+`X?7 zk9?W)-u`7icmFq7CBW=Mq={I@vw`_V+=#TF)qQl^(rp|Q^AzbdZFCz8F_&W$45&&b zEtU{IFp8k!qX1mgGajKw)WM_yI$Wdi%=&-`j}9VP1(wL((ZR?{#5wp5D4-ZURgfINIxtuQQ#?#Nnjx_Kyz~$gEUcvl}yx^TCU=Vm(&xz zg$PvZ;*=ngRK-NC(hWS1a#!{1UB0Q4VS7%r;7pyqY@Ag{%2@#{MART5?`D=yVJhEj zm4QkdXHHn5py10i!~64Ak$c{Xoe5wta%0ZVOVO(l<(oG$GReQ>;{& zd#A?0Og@d|*Ta*zH*E`Jy)n{fGY-OFM1*izc4+q~)nXTQ&D^=bP$(>wKTvPR8QZTr z6oF5x98l&OAhF-lOA;{BGgo)Z_QzXua;CYNOBHv(@87|zUY1jYtw3O1Fq? zEP;LMtcwQ=YgJmdSOlK1saqBUMyyk$l3o-p{Vl#H_(( z^+dNoirtxvqQE|p z({U{gJ3sn4Tv;wU9o4dRSIfLZhhIVYIFB?1X-W ziNn<|t8z33GdGwRx$_F;Orl7m8?(**E-apwa#>esIiwhP^o9_}MiYWbb^ze*1;SGa zu+hjpIE;LnaezBdHMaM>oqT5C3wP}G;y#tWrkvPC58)YdpVE2xyo+G&57+H~1SlMk zInFBf0y;pxm8@MjoT#RCCeN)08$;FrYmSu^i#;N$(C zcZ!t^=03V&PB*inlxoDyeN8=_ZkN}dJ5G}Evo7Pmd*WRU%<-ik`I6Jvx%0}WB9Z>P zK9}wYVmNB<^Z$zbC6nqtSd|TjRPW0uK$=5Vq&1qzA zOt0djH+hFNXJv9@hr=JORPt)g_qxU1F*RRxMX{)DHA&b-EZ7 zwUoSL&y1Ms$A0fCD^+rP7o3GZk3YZizEf`Ba+&bg1@Eh$r-Lm%-hTY;!@aAhk2_fZ zfVlw-YTz@{D2++a4T^3hGOCQmb0oRJ#nnhG_9(DOgzpQtGJs_a_wb=!1?fdLd(&xqKNU#RTclu3?l2-(UdELbqA~C4w5{5er~$2=NK802)d!kVAnNO-ySXA0rpK5P^ zhaBvMZw&dvo7OQGAebB|evASC0Zzxjme3ko?cov(B^*f*FF~+c@#dC_2->%4C17nEf5# zN&)m&1o)5C@n^zXN9clJoSiUp)+HFCv&J=3f;JN3U;lb+_+k)APO z8X57Q`0-PGbR^t{NWW44d!!$?k)I7Yh{I>$K(rs){MDoD^*i;yM_ap1twX)y#k^z1 z_lV(_ML?+E3nb}Kzf=EvsK4C?f7Y*77C$Egg3{KF2|D7B9v$>Y|1au)24(pk4nJu9G=fj6gEeDPlV4LK0*_g8atxZkP&4u@HM4gqh{cm(D~9RvWKf{kQty~9Cc{n-iI z$|6T^TSSbN+Ly*bi7FhB$78t*L#hoqeCK@(|fukk}>@Xj(=Ri0lf>( zM3YF?^kq8UAV0lua62iIzRWlpoYZlok4@=gODk}@o<1GXhony;>3f4?a0HH(=*&$) zMkJlJt|fiECYqk3oTZ!`eS#FTMqHDOzVUlAqF>z&4G2yS%6`-}Z5-)R8Inkj-1&_w!ggI(I{LKSzfcFZUq(2(p4tfTM$T9LYTH zS9$c}w*DfI?wXW8$>UF#C(_eF-iu;r@8e{&=8E9aMbpn0w9^_iU50i2jQoD^GuT+a zHGV-(4)PIXMQddbL%Qj7g6aB%SgL;a54(Tyj|c^_f`yMGs1I4enm{G%(Cy<$-!K2S z{cDH#lYgzLW+)%-Xs_h8?7612BiY1{Vi*AWxCOzJ zYy#qf&2nvj^xyXX-5m9(DT91c+(3dWsVMKsOpmKma=nL+O z_OXr%YxXjy@Av)NelXa;wr0Au-kNI}sz;L*lzrSC>`0#U^{PMoQUCk-texLa=S>$~ z$KQ$U6tae8LiZOrNBReS;=6tRuHSmR1lq@f@e<|kt+y6$AX89SZ+boh=hyVMq9?EQ z_8lK|BC!30zvdr-9)SiCPGrw0kHGNn^FOv~X#rSV2qi2uk{Ti75fUOp2@4C3#(n>? eiBnLPCCC!M?@i$M(Eq_-V1^w4!2c;H|Nj8^km>IL diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml index 890fb99228..54b7e82d05 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml @@ -29,16 +29,6 @@ tables: collected_at: '2026-09-16' rows: 1544 status: complete - dsv4_hca_attn_module_perf: - rows: 3918 - status: complete - collections: - - collector_ref: cbaf51b64fa460e5ec6146bde407a4c64958212d - collector_hash: sha256:5882781b43c67c93f89e3e238978f461e242c83f681effd9ee6129a4be1637eb - case_plan_hash: sha256:0e844f577823301a0af954d0d20baab4642bb1c0368652998bc28160dfab816d - collected_at: '2026-09-16' - rows: 3918 - status: complete dsv4_hca_context_module_perf: rows: 5840 status: complete diff --git a/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index 59d4ba25b42a7ca8fb204e3ddf66fcaffed29220..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 21277 zcmc({1z1#F`!75*z|hUmA~}R~m&(xHQqtWFo$8R%-6dTD(xHGuhkzhbB1njcNQo`# z*@I8{==+}kcdqYTXNj}dUUC2Kb=SK0#L`9R3&F5qWH44u7z75Q0D-vL1->%lAwVE( z5C~+Xsi_46!Kh(eys*NHz%VEb$|w%v!p3yap1qQj1LMvn5t@QPSinu;MKHMVA_yQ$ z1%v-YmOu{#f&wE1`Gv*!1%F2v`lE~XkL!OU&JUCOi8xI+2t)}G*EUj9R6{V!iwFub z8<=Q`Gb0p!heY|CakRf{{BJx6!sLJAfwmt6q5^oZ_YDxWa<;RvvhnltwDRz>_jh;r z9TQZ)8%X=x*8dYWlr<3ss#=-|D+67GzJZdJqJfd`(;D9u6aB$-@66I5NxAAr6Q`VI{uj}K5xq?B; zr6GVN!Wd!vKik~-|3u@DnY1kbX~16~e=|?w{kN@7Gl8a=zyROEFvXv=NjDAx!2sql z4Eq8M8;L4=`6$@<``Wnw-YsCi9ZCE9=D(Tw)p2NHJU?+lWCH@>0i4)6x_jC935ows ziFki@(*9NZZxAFf=AR&NrG7}n5h?jQ>R3QKE#~FF(WHm*{zQ}51q8we*aoIq*}8k# zx&4kZ{$IVce^>t-3NuXjCo3Zh2Z7`O6ih*#y<*h%F9V{*`?HG{PwtOF)F7_kwEhkK zTO;{@i{1o=S!)9E_%9bTxd;X=fk6-;T;qYrK+J$7P@o_8XR9R8g@Ejy3ysj);Q=xk zgTN*s|8Of>yt6P5jHJZ_|3fOjw*D=ZihXR5ygHl@RXqi+QpgL>3&(2+3@TT)xDv0I z;L9zFbgGOHbjupQ{O*ZO&2C^IWvOC1Ho=xK`%_M=RfF>zk&)+lv6n9wBk8NV?XEPm z^)=(EZ=c?0b}*lQLS>eRJ^_^3z@gOkU5(@@i$N-^b;=`Uoe-_#wL;oeTUt0eXnzcHZgBWCFVI_&I*1`>o zy3$n2YvW*xch3LqoP&v0`s0#hHNx?W&!X-#ZSH%>82OE36rC4c8Aud%E+~d^Mo5ZwH+A~9DOA#7j#syd(g2M`7 zKUwl$6EM5WS-~*>YwKUkTbJQ+@PMc;{*mb6++dDH#7lA24(EpoPN?1Pl_a%|$6K$j zHc{a3=i+R59}|uvRfW1~XjU&pyInDF$rvL!M0J#^o*;%>UfA}8;fU;JE`&+nrTIlk zp{P-)H|-{w0a}5FZ-Ft>`;n^v$ri2t5SD7?aDJqow%LA{NtA*nn6$Qy-d9lFteSYs z);pGaox6>~jYL_>rJd%cH?~lTY{etnseUe(qW7DELoBk234`hhwJ)qQGmJt9$#5wX zGbRAK3XHY{LBlq&CcYT9iH+A4Vz>!+&dqkk0 zP>ES_3=0W2TDF41?ST?LCp|{=D8mrZ5$2teh+Ll-7GmXHN3tzIR!#;txW*(MXg5=B zxuLYr$xg@&nDm(eB_s-$4ja0XDAS#(O;;bIHX@r?1)`vunl?aoP^oU)+ueYeht2A% z(#`5|B!*b3KoyU!^43OC6Ar{T2o4EhX9DWC!3!V|Bqh&+(2Y_tGwE})WDStrq{12q zQsp49O|@g<;RzpRO!Rofvp*~a=#hm)=mrZ3ybAOV&?SP3qbOkzFb!ILpGO$ehyYAu zyH`?Toh6nVSGIy$V@gSirGca2Jydp>r3xjXpV0o?x-cEEB!deq3+9;_1Fl|Zu38UZ zeQrn_JV5g=0q!Gb#{q){Znf?8v|rWr+Ay+4>fIgI_a#RZA~a2WZ@eT0?9PXOV#o+% zBnsyS))cnL*a+l!8b;MM0AuzgR_!HLU{sj!C`kaMEi&VJlevoV2B}-Su5BY={lw+* zF5KGS7=)BUg+BlT)~SrvcOl+PwW+WnK_r8>F@oe0gu_Ou8=xIB2DlRSkYokQ32?b+z&mhcGanN~Qt1-7(#lt2e2MBzvt%GtT@9Gd zk7Rai%sHLvkt2$gysY&cxgf90IsHWE40)T&BN^6+=6I(s2@bEVjC$zN<_70kiE7i9 zn(u0|7)-ORFJES;zze?T&n9So_p!G9TNxQDYpu%+E~ACl-z3V2y%aU^wvKu#s+7ox zpeEG0YV^YSBE!d>_ax!#mR{J{Q%`m-3Kmp}+IIAy;h7wyEVQg_#<}~(3VR-cgn4hR z-!k;Lz3{bM;Fl&~OH8M#4||@8*SAg-#kb__VQZI&E8EG|oxY`r;d0Ngo?kN z%O*-3d2YkGWoAv`R@Op5|A{ESxaE`+ImN4%>_ycfxnir3J|o_+!NZ{xOvrgy(x)bw z(eV)St(3JgNOhN|kYa&AHm-Co?X%j(!qMvc)naw}Z2W8YH3m%jO(IK8avsNXc2HOn zG!63nhn)uXM;!jmPV;Bi-?{A18SJ0^e+BE`>@-bp!LY1QPbiRKDZh?1ltPRLV~uqX6+j-AE@VE!G+m={)5eKnIa0u>Q9?b zy2o~SvnTT_4FkOE7fSK%pS)!ZC6+PLOoIkxrux24=%l}(d=k#MRT^(iQ#n9)>o%>i;f93Gsh;E?#W!e@p2}0uz#%< zOqO=MT{Tr}a49Pz<8C)&0)YoTBfgdTOCKMZg)Thw0J0l@rIrlqwEHPBO^8Pa%)GHb z4M{B!$nW9H{kWg80>_qrbVHxydwDp!Kt1-_I4&v{>VJ=5to_82jwYx>^`Z2M;tHGrKDW zb(v$s9m%<>7@e%#)yFs&*hoTqwCz*tNTwIBDB2fhd2JcNeX37tiG@(@6?ixS;YQ*- z*Wk`eCfASHvk?;g@&YH=#I#44Lv`7SEGGf=Wb`KJkxY)a-KAXX7RLu$1m}|6wenI< zB1>`afR8ACr-6cI4!EGO?L~#G-{v9&#WF` z%}953AHIm$h=q07vRScX_Ix$3W!{^nep}Z3XkNn>A)`>#FAD$fXbAm6I{r;i^V8N@5a|Y z9@RV&3EC2V2i)rR7QY<>H%+RxhV_BZw7IlqoHQgk>Bnrhbxgy~qnj0agoD?1z>JZN zB4T|3?;LQW$-eO2mKzuIal3d)3>QM^O-ylXpdW_>zbWIxF$;?vr|Vc>pB%nW_v5m9J>E`nrCAj#`r%iiqfzBI5zoF92H&C#cc`wyB~fzXt&6Y- zaP+L8h^^q~59;5YwHi8nzZ= zpLJiFz`=8Tx)`HkYHAx#@kPoo7dMux*yKi8$O#+AoYZ3qta_YSz2Mw*XId6Hm&J&t zn^RhK8&^VeAJZKTPn0KS2gM1&Bn?S$L)G2}1u-#`KhLzUU5gtkS}Hu^%d-wONXw^I z;X++|$b7pY@qBo(O2idR%Rz;qXSg`tXu+5$3$#z~o<4n-kPpecl{z|w8ZT16^Mw82 zCJC|BCEN!_BC)E&MY;8}n-@h0Z3Ic3f`euT37L}L2$ssMAYyAok#;2S3Hn~<^N-%1`XB=*+UacW(@ zkZqN4U2EN#t<$W*NTjo+8#dJ-ri`uLn0IeJ9$iUwP)Jdsd9#4XOFYleioacsHd8wN zg$9Rdoh2wbtb0f!kq<3U4A+2OK$^^+-?nxnD%wn{CGBhV)AW?`NI%_Z$~ zyd$)-{8&qo*TS#)pIg0+n?+=i+=e^1dGP`v`}WmvNHA&acyrld(+2oygJWTWD^H#J ztEtMmnyRiA>ah3}h_M{&7`N1|3UVw2rRIY3aoM$6uib-@!@(7?%r$n?jV9RdU2TjB><*D2PgzLt?Y7`A1AMY1%eQ;U_x_G+ZxDq8;d1v=23&I< z4@raeoxqKQsIub-x?pJ+-WTp;C#|zP%b4hGGjN}NIiEjE{PY#aNTRUQ&f@J-L|%#=mzncw{n3mu7Wmk;D@d^G*pO=sr|}XBvzN?pKwBP<&7ry@M0>aC&o_gq9m| zI87Axb~gmHeWfNi4s3Mp7Cj5cm<{|hH`oNkzSnaps|+83pk!y}kKV$Cy#Q}k>fd=b zm34%y#g>|syLXWvM$r{ck`$});5m!1816u&too21YudU#cNC(Q&G}{FsV6p{>f1Q^ z`E**5tB6k&fN(hVy283o z0S6^rS&0j)HRmBMItT#`Ir#Oiffm@q7zqVQW5utKCE1K z&n4!o07Uy`qLS&y9Q7D}6~Faf{Vo$>x01P{#uW#aMj=++H>-_L%}9CZ?)K~z7N9~t zZmc#gO9+gmUO4Q#;Ez4Uc=za@E7Q)_R;zb!Pfrgi9Ee44u31_ZTgL2dZ*TV}rTNxa zdiR`DFSChix%6`6Fy{2&fP@rN4oMz1n)By8=%kdk(}Nv6`vAIscyH|R!X3(lVVma6 zgkdLzLSITl()R(gdZ10zV(;4FhcE9!TC2Cx$L)QhGCmY)!dC)TyF6cCyely-|0I*? z)0*UmkX}oHze6iD6ieQ-KU1rrDjU+59W6L zcIUXM87>HG#ov-IA}Nxg3UOYzC2(;128b4d*acy0ele@#=prX)DnPwfG1qDFvSNF!Iv<9*-Lz|KlLzG702ZB>3-7qv$dAfq@*O` z;WVLSMb_D5^DR~oA!z&7`=^?n=bw)BY0FZljBXxFzTVQEHYlrG^ta#yd~#f(-!0!U zUe6v3?wnh`@6>8`K|@n-3kdw~3-NyLVuQEcEgOs~A8`R`29fYh=^LkUTMu_mmLVQ9 zNdpa+Z)w3Z=(aWBl*#?iB2F6`rhtE1pT#@2j)P?DYAcgyCRNFJk7fzIU3us7i8RnK%*sl@J+~p|LovXU^9-_foQMjf^cah!Ll+ptO0GXjJFyKVaz}3dawGYdGPt0J|)yo z%~tQVKWJe0+_wf2EZd!VAjvmzueWUXiVg@~sx5o2r^rn)D_0ii)Y@&o`APpl&X_~{S)Q9Q^mrj#oo!8TWXSq#xDC0r(D3|t?iFGu}?V>W7NAs#3f}fZGc~AYd zC&#X?q$ORTe$Bp`5pJCMj4sMv&h)f)?sC>i?1Ry_W8wW}@&z|d(_G|n!kwCoJAdQ$C+(2HIEgMeLADJN~ zCH_$e2?_CW&q{`}92@hW#o7ue=bx?pS(upCN)VAy@;Jq&-m`KM6Qj^OyW45w&pGao zKmXEq=xt%K+NJ%utFd3ya?zg{8rtgzl&h|NSNNZ;WAfer)S& z_pL;}U%$VyA8>DV`lN8m7Vwq#A-(#+?XT8fUV&BX*FJXj=i$s!AGZfg51#SkGB@ju zq4HpRVcnT*#LM9UWep#vD1qY0m6!*om}m~!AO zdn{J&?V;t76dM!ZXcRCyeeUADFVh3vhgYW#neKC+_Ia&zct2fpb;)PKM9I5@F8KhR zTMwO{WUgtse?Iw?d9rizDeO{_4#yi$5BFd%w+nANI>bA~#XEp(%UQ0am1Q|rku{A= zm6!K*LU^dR$2|Vy)_YznBu>XZxRZ}PfQRjsj(zl1>4>=#*@tI;ZnZNzbKN~aP?Rq6 zc2{LjU`tbdetbB}_9iyK(Gl7|K7_aZ($l*;bdlQFi1{|(h1=B^;~?)|AAIR}=XpjC zAgT57l8g}TsZ!|Iuh;KC19pb56fIa@F8Rey__ua~i=6~-HHTFlEwAa9KCF&n;D7rB zc?iI1Ih`C$Ilf=sD|H{+nsw%7prm9F@$vWYa7Z(?u($$FA9y1&Y|oC+Wfc}+-wF-* zpw)NyrEmXBpU0P({S|Rho>#35i(*~huHO&-9)W?qir0bl$HHy~IlLh&{d~5_aL3}# zVWS;pK0JdxsVrANo-FlB{Q@$qXB8aMS!U%r1TgO@W9dgm71Cv0rSS&dyCeBv_#Oze&FK%c75iSMZ>mhyQtP2@T96v zbXQtS1Q}B0k`MKH;*5w!%L+WRZCJQKL=Af)D!C3Mx}&!r*QFcRULnQ*I133sy5efs zgb5+n?rP7sc0S~!iec)3Ml2mn0(M+IxBFZhh&3(-p@H9)uQji!nwI^bOb{86aHM_>l4VMo4y;w8&UYUc7H(;7Zf2*61dT~V@QZo%O1v0_8n#qdd3LV+ zB*pv)d1*E>6bgk#g+r8XD)(HYM4>eLZ}?u{>A+XXm*I?g$LLWk#bfIsOEc1$sC(7d zSoZQWgYSf2QoJpvXig$Sp-YeQ$3rLFh9m4tKL|@`k9LMeEK{D7EhFgY^D z#odu$nM!$gk4M7AW}4uH8CB$z|ZPq`OCx?{b)inYL)5d z^)_%i8n)UZTQ0VS4NYT}PtV8f-@kHOEcEcZa-_ypx7&$NxGY6?b{r$C~`@stKw|haH*>etF=2Lv&3? z_{FuN4u7|zJ}sKjP2oe*c8z9dp%=Uw8_CFzOoR%88Z-3d?_0D^8FyOUh$vnBbxa!; zueG%kk;!<;Q<-sId5LXIFL{%pgl|R$9vKqhcFT-%C`o8Xj4@zIXayPIdDr3UjALl^ z#D1iVee-1HL5u(mBP*S~c;RlCjwPRXrkK=9dRod?v3=f`ov|%%R&cTnC-3POU0Q(L zX}lR3NZYFMq*Q!U*rT%3=0^04(wsrV8p4vdA$0{&puSjqIkcF@ecffN+JhJIs*eQsqpw0}zj(--0JVaSOC4&B zL=6;q*jRVMrAKWXtwURCr^H4zcoFXxUpNp3$>}&}qh&>nWb(|0Gliau_Z1l`Qd$t- z4PDB7Z@pq`vqrsjRWS|E)S8~%dsjDeYRWIR=icQ(ZLeoCU)frOUrXsIPq=;Q9{xDj z@j8+;!%IfwaG>dN2qhWtE{|<}>!;!kZgbp`&m;#KKdlN2GLknoWr$b)o5b9OI3 zab_p!Lnl_hLw!1ljuE<0@P*3~tsJYDF9u#TYAg`NlBNB^;5J<;Ma}S0Vqe6Jp%$Nc zkEOPap<&#`PNG_LE1;=;-*gIZ+ftarymG~9ul8!DH2sv*`Z*1Exq8q&^Nqc!is*13 zSzV;wIf~YO4$J*Lk43j2BuaSh9-edfJH!)W&$#{g+hP&|565g~I3Fw!;OdHfdRI$6 zR#Umkt`oU%TT6X`)19rgKg|Bs7KQf%DdMPU7UTB4Nzaee4SSKJG@d=LQ#q}_%k8-0)6?^MWZJNvFv*+P;KFy|LuCABomPsU7j}6PViM>1z)3zA?{qnF@<$t4rUu$O%+yAkKy!?i9Is ziJKASJWQtY3Zs13V_ha*Z5lzXaY25+#!EVNY--u4Rd&o|?(Q){i3eD|YX0(a_ zp~CoyX}O^3-9j5Z+1>k-g~sqag|07*qO_TfO3Ka(y4fypK1J-&I%2JWe(n2xx#&Gz=w5?tXoF zzC*c)e61>BWw{r9ZruCw*M+Zn12v#Z?5j7YqI=Upt*N`Y5M}Z`|A()uISS^#ks#Ta z3o@@DOFk=A@pBN)EoSe!3T59+y&|4_5g~X~>zGq7orIHQKkFIJgu?wgs(kZ-+MDUs zSlp=khD%K6ymdx8&?B#hofxc)3G47at@4Rew&0zob(?fy%`pkg#VJ~@Wq-KkeaXHZGRq^F4b5zVI=zCP*C2lxn3N5 z{)pn<1^@gBE~i4BkV6x$!8#nFdqND4ugvrGD9&xW>YU`k^0;487dW{EJm6t{-{5>l zIrO3ShxltE_MrnFHk!`$j%DYxqAbhJMMyu&$f{`&zeqk9Bhk*j_pB{iM7z=3u|HjW z$3_aJ?7thd^x9TyhAtrKa&*eqhutnA!>UhsJ3V_f(s`-EhOQ5KZ7wGryz+n2vGI5X z-#gMi0k`?V=(fO&e^`3a{>ZZ?0R|kmoL#Rs3eJakQ; zWC=>V`pxQVI8s*nnpc~cMd1?j*L|=Kh?@9IVE+nT3XRCR*qJClCV*Gh++tq^X19hx5 zOZP7o_gr4IPq*0L<9nOLzt7l?rJ`&dPod&8*cPPdpW(YB-l;f^E6-ZUcl>0%e$;-% zt_+`~U@G>iduxM_ncu(@PPAIj_FXnD6)o4Tk4x9)jL&s|E!xh-5jG=iZ{x8^ObgC0 zDDR*5gD*YMP?k>seOaOyS?B(|@uYFJIuGn!DOu8*V|kFcD9x48EChaggZ|v^{AYC? zVmH&b`N9&HN$DE+yc6Eo_vUS+@GY%O{bOyG?!2pt7snoKTu3+J+_7z=WLvdiE5BkkImpO(U^E)? zO?N7mj?{cyfp}EJDn?iMB?0yQtPVQ%P3dn>C5R2*vNu}HhGu>8^fV(K`B>I|X>x+` zp?k`S`{+?{8xA>cp`!=sox{)#i=4$hpFr~`3U!h%V)=XUce;kNDv_|oby`cYwPDTd zkEtGc1@AvGQ;MrL+V0#WFDD-)XAK9rclr$(wRYZms=cqlBP(HiZ{MG7t%!eZ`E^%% z=;wD4FV^IrUc1BmDC9w<=1nEx;I=nR(OUMs&(y5;BsN^%`IbmYlbsvO+DZN}!Fn0&c6|+%EW6h!k);A+;i3n+-uEBB=p^xN zy1PZqa{}q|&{5iV%;xU6cW!b>TYG+CnGdQkk56k0rzDtZ*%&nt>@1t*n^w)QbA%@)?U?0&r72>FRlHbJqVrulhi-l&ztswWR-KzLn+_x!Zaz4?67g=rjUb>c;JJi50 ziPw}UXhhngKaH}SQCY;)lN$kRiVpVYWT zF1^qwX-O4tZ`4$_;4zWjJ z=#^&)w34K(f*o`j70Zw5$rMt%qjXsg*ybN3G`6NwJ|PkYJspycc&S;guu?oL>U&i) zbW`F)QTo~Z>dR_sqpeD~hqDj%l9Q+x*-cgIU5e%n3EsJKUC??qykcsHpX@~HWQLde z^0;iZ?Zq5y!*PMf;?`BtJkg;B>+$wktu)1i{m$nyaF)`lx;tssVB??f#$S^LIm&(K z8Y4Jz-?mIDwP`C2mgY3TmN!xeupzLMcwR9_l(^_GVaZA7r!>|KW1GmoMElJt!;4%^ zxSYGet;K+${>pi|+BeJ`cZj8nlsH5J-&lrTA3Jl3!8{nrb_ zFJnF}|F!k6Ps^>DsE%v?{02MW4G{3%P^cUVLIhkSK@i}JSs*TJFp(SxL=8egCD*}Z zu_$c#3<#eV4I(L#!{M<86PlyJBxn>C<2ne}UJi?49R&u>fJsF`pz|L=c!KLFGRbu; zRwNpiQ8%2xkQs}O8G)|}LQ$45;}C-6u-Mm81d?blYYjl51VUtQjl=64PN>ctP9wUG z#btuVr3WBP&|qRzIF9W)3OZH-p+lme;(}Q8VJHZ32{7OYjB8i|76K@6p~7hmOQ7Oo zXkuhjINftJM7jor!-xXoTZ6DjFc9mgbCM;H3+o6R*OG81XEg5R85C5s1k8+$z*At3 zAVcb6v(O^&B+>oks;DS-E*pGJ=LpJTP&AidIKF-^grWu=#UWTqpwTn{Yf8o8a}Ou9 zI2xcqlf?+c+7Q`uMbZr+V}wgsh+R#v7@n-hNT69re2xYfyX0{BANP|4q7|6lHsQ(~ zm6Bfnh{X(HiA9i=k;O=2vr?4CDhZa6CkJ7(v$Djg>6TGs#}07_rr~S3M^F|W4ROkm z6X+#IP*n;JbL*xNT&a(MHRwiP&_5Baugd-5LkzfU4qwOaf z5scL%vSY-e8`m8X&YdN4lEtE*ja3${5F&Op9Hd`FD~q=^6MOm%GCTvRNDc{+_+$<; z?#QV~KTb>bZz?}`5QHnc$&wO0Q_l3h3-{8S(v)jQ<>$ZAqA!CWsbDG%1WP^p9hQ<{H>`W9576b+eBEh)UCEy<_1VyDXv&cDvaOma2 zp`sw18?YL{5&`qGmSZ8QLCa$gq{cX-Sx8G!Sj?hT91G)R*b(69$IMu~V{%a8)bM$rTJNF-Wsitt>KYLv2u{M{New5m zXT}n%0TT-AMiSYtgLzR&P@Xb++#zKs9iT$jRPfSc6i(AP%3T;4Nz~*h-jt?F;amo&k}hqp((%pt$m?10Lx}?58H|5_gxAD1uI+)|?gDH02^-HR}b>Ou&-;t0QOVI#ET8o9!tAaJjt6>!bdS-6qUZ`Km)JPpVyvQc*^zngSWnY+_?-QNJT*vOTe%(Ib30~2#Sgl9HANnNeEdKD|;!vUX48UU9uSA zic(_l8igB7pUGn7sp!d*VimDyraC)3TAv4RS_2b zv_$&~HWqF*bu)^Ly19x2$z3(w_;mJ)wpew|IQ1(nkFuIFIru!{G%Vgc%0W}XyLv|X zY$)b(pDI)d=k01bt7qgd3{{DDk0ZPy3<{r%a!SvSYX=S(6u)h&#y)lty+)sOOBpMb zH0gyXw;Y&27fFv(Q39s;h{C1^qDd+mhhYhQ3#kC1a7VMSGlzSMH-RZqQR1jq;mYFc ztgFB>$(X_T5RMw+RBK)KmtcIxM_5m4Vsz=bsY;7e$&-$t_=Y7BrKxf_?5bdV!=`~! zGO@xac6B`w6GMF3d9HI(uk=zw3@e!5aVe|0>Sxv&(m%5178Bgj&z%7LIl5NMyUsvq zAhY^XCAV(0x?wphEFG!OV^BnJrB;8w&ZNE0sQcBGrg&Jrz1VHj**YVumiY!`^lkGE zb>n*kBaIPyaLd^;<7>yj*ScZ6HpB@g!@>*AsqK08+-|0muGy{1C%l)Up=PCYTF&wY zz|JKHp>PHf1pu3vF0dOq%gKicMn+AdnP(rRwfTwiozv4$kuEb*V}>=P8{RgXkD$V^ z&SA1=M$=5)(dgfhV6Sm?Wz(qh?v+~g;0qj64KyG!$p{@r^PNAWp5FO=?tEbm!L%Vbpcy-;ejru{xG-MG3q zeMYpSC@0~R^QBA>R*5?pWCCO!K&)YAb|BVWo{$}G$?ksbASh!zX)i94KR@XxF=#sN zql{a$pXg++YW&dMQRLQFB4;_m{FxZ>b7lTFIpt=xXA=WND>$@VG_I<%f0cQqSr|Og( z5u5f-iG0efPcAin#lt7_cHuVv&F)DTPv62l#(vokQZsQJeq{{jcGY(X=ZZ-Dt3()v z^{b`Z8j}4Rd{_6ZX=IkCk^|aXSND-QGOPQ^fjxUghu3j?*1=F)LeO4zo@vrjWQm)+H<^&{v&c zxFQN$gmoMmYXJGON_ykd+_P^VKlFY3ba(?qR06_PMMLP1PVYvwvHo|iy-!FB4&KejC=7Tf-FWiijCa{Un@wSstQPF>}?1wKFZUM=*9?R zj}d`c2eH}F$T$&3Vq#+yguO8ap9>9vq@$pZOITrP!1@5Y4@1XL0shK31HuJz3~Dk2 zjxZ{m$ay`6zFYNd0FX!CplAXqGLk`Xk3dieiz=2YkYqeY<4Lj`VgXSCYR-;B1*9oJ zZUQxw12P04Ph?Idcx;U$)x$!`t`trx2Z*yv4v!a;!J@%r{b(F!+i(KO5-6QuL;{|F z8bK5$yFp=+5V|EaX1^p6YJ%X}mq3NbunXzGh+o6Q5&t=^cI3wQOyouZT z%UXu;5ux6GEH9fPQot-aO zP18|7qu@C$r_AoSPK0n~8Dk}vnyI=$F-2CLf&gKaIo@G=~u3qOhsfk@%`8DH^S6@Swowj6hhjV{*igOtILyobk^i zfdO)0d=tR6fb0^`F%dwF*F+V{nhqv@8G}y)(8ab5hXUCp#9Pjtv1FQF(i(?u3`OkP zq|NG%!oOe*b~S1R!WC3}$DY&Ox;z}Sr_-aR|0 z6~cb^Wz3DPxE(0t8wL(uy<*{XJ^UDvhwy0;7N{EDMM~S$sK;}~A0F!^Ge_&>wDa8h zY|cvaVO)p*{oJkZ*fp2Cm!iCJ+7kGE^W^Nr&(#rUmf(ohP!hT7MpAE<;K6j{iO&>Bn!%i&)KOnM~CgbxvGC`T2{AynSV?g0(;g zF98wCEkh_$L4zFDU>reARth{V04mZ29w&fIK1vKoBoR;(OipPnMhf^7logu>5IbxP zg@v6CUH}rw78`_#X(~9XX&u}Yj4~Oe=b{E4FX?~;bu6BY+Fp*84tP!ka#eaDPBlq6 znTA>~EXaby$z775@u5xi7o`|Rf`#{`7UcL zk5y2K;nqUu`*jJLa5wYq{FL>WX==|FWg&h%v{z`8zjo+^xYEmikE+@U@xUZ3u7c4l+@f#%k3w z!|s`kh%Yp!M7!H_Xb{?G`n6;>)H?|5=}*>)w&t($I?3!UKa4O6DIsofR?&F!Nc~}3 z^)R=qL8#e$V|IIk|Gt|I!`0>K?2h*KeGl(~tE-O!I(vpaysl~VJcCGd4d9F+GMNi|Z+GFx8@>C6SHY#wpa!nF60vYu@ z+aC*SR*rnh1siC^y)niY5syT%VjPbTc#KZhCEwOp#FI4PCowckYlnw-!}^Imf!3#@ zG!py+NxnBu%;nx$V@*^hx#ps@;c~zuxAg!;2_X)gIh)Kez4?ZP@=WX1go=&3i^HA)gEbbq9y#|`-yH<@60RK`Tq)s~$qBk6xt7J7*`rOC z64c0E{3icHPpINy@Z3F>ca1r{uRZRBJ|legabmG0hhsNv_o~&GwZ*=VQ+L$f-Lv}k ze)0ZyoFjKE-gPi#mwgYPU^sD@H8!V7Kj90$NQT$zV9iz^YE$E0*%pg96re^jtaRESG7@|Khu^H#fgadJaFiA9=DJ42`-GBx>$lFGbd6s6CX{MQW6yHOZ%mQjhfBldxh_+8rmA*JdQOix zT!OAf`?|0}DWgNRdT^~ll}}bf*fx(2?1AaRn`{LUdj3Eu(Q+Ua?K$1<=K#VkNW_E^ zv@pn#434>=tu|nueph%xFXir--)$Ujj@VDEb_)>{V`1d(F1IHaHmN(s3n@gZhd3Cr z<223mYt@^viL#_m@*XC>5WT(JIBR-B=tG7p9_VV|5g=mu>?uNW)O6ans-3jq@ zc{&2^Tl|g}uXI0HI_#kJ)_0gEHR9ZqZbHaoJ2Z!Xt=2VHmooaUT9TwX;uCheKy zYAwnSwx&PoapRX+g&V|yUDvf-AKg@t-MkZ4GsLNV z9Dnt6lj=}(M5OUJ$+u^V>AU2Zf~QbgO5t-3iBIEt#Xebg3b*+JKRvKv^38i#_(H7n zQyi0Es;~MG9j+J`UCz~_VdNsEu7 zB~CZUZv_npx*mVll-+t)`*7qzh+*>i`7M_7!Q(o`Z?Za1w}oy6Pk9xe6n!||L7Wen z1%C4h!=^i>6a&#heeIkbJZxa>AQ%w`1o+{;kBx^zpqGyu2S}3lmZT7gb~q+fuLk`{}42c2`A2hSU)OW$~k6?HQ7(?~f zBUyf>TyqHdJ>@^t{e$pxFm)^hK8XcCf?x>$?y!~rbOOt-keLbizlZ$OoPJ=w0#heL z;0rkLkJuQ@-yW^<9}isl6}4&XKcN0>u>ZtdM(Jd+5+((g}0`Ok->{0cr9 zP4Ij0za9Puav7L<1PnhQf!B~=kpDUt<);%;ex<*@i}ic@{|X&HK%&6ZHDGuh8T^10 z1M<&6h03Intlo$PcsuF!c*Ce4QO$#txwU`0(l9>-x3ze~xyxP5pq1y+D0H17F~T z$HD=qUk``Ci163e|2fpZw!wd{uk8~2{Y3y2^+T9|H2w`&-TJ==CHTV~{@iAN zpk+=#sH0}V@Wgvy_|wY(;IE$(V*r0`{qKO(T}4>HHVwz4UP^_4=oo2<5c>K$#?n8s zC)zFo0Zh4oP(&aQdcc27(Lf*q5%Nw5WdyTtFiQw&!s2!!b$ zL(~w8x*`Gs81i~*XCuxQaYp~g5Je1C73~096Zv2>6YUTa5p`#K754yu-zx$EXZQ%j z70g(9B>~4j_;Eg)dj|3ket?A`5OyN^?sjM02qlcD&#FJ>3;u8BpRLDURo>A~*j?OA z4;9iMB67MG=@PiMf?OTeP;K)Q@{5g)=7QF#^FT;?Lp* zRODSuP4r!Dl%+y!Opxw?ey{wD53J{hJmdi-V#>euai%(36)9I!;~+;{Pa{7QVP|(! zWBm|>R>;r%D2814i64bCF8{%g8-^c$dt)I-8x>hCo~N0yke`W&zL=c~Ms)ICnDroaCH}CFEC0nl{De(}wVh3Y^_U3j3p$v{ zW8`CsnJ@IO`DaS}hkczbO-0Dp)JV$BUe!Gi&@13shG+c#AzyiUeaupRk>}Z>O-+)`9CT7SpEOm^%2$*7CFdcu$=ilr5;V&LW!GOO6*g7em`5QnKVAcr} z&w&23b?ae**N^fCK43&Z`3L@)eR$Y;DEpb2*az5o`2NxV=hTZAfwvxzHa_0|4t{)g pNF<+)kB?U%^B<3!%)+Alg8YKOlN0bH`hV~T+9m;kfCCeS{udN+G_L>v diff --git a/python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/collection_meta.yaml index 2bc2e20cb5..902301824b 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/collection_meta.yaml @@ -27,8 +27,6 @@ tables: status: complete dsv4_csa_generation_module_perf: status: complete - dsv4_hca_attn_module_perf: - status: complete dsv4_hca_context_module_perf: status: complete dsv4_hca_generation_module_perf: diff --git a/python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index 1c1dd4d8e6abd042b4f76eb311ba4db1f453f4b7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 14906 zcmd6O2UJr_*Y-&QAwYmo5<-<4I)n~VBy^f|(?m42?qb7@(nOfB^sm zTO_`6AqW6K1^|GSfq@|!Kx5E?qUf8PAR8PFXO{y6$w(8}txtM-kZ)@dwJiX^z`;#U z2;?Rw0E%To<9-uMWeNarkOMC+B_odCO)z|W3j5>x--L6arG68RN&o;PDBQ?OPg9S; zrK2u|-xUJ+D+Bho(Z9jr(dxg!VS539350VGiWx`)Yy-9_{|&qLJ)u(^hTJM zzgTc+kgtbZxP+#MN3gkvho6|0jF`5+OPIHqwu!=SI&uHURoMT`@^9Kop*4TgmSq?K z(4e*?4Z}bUtz@)=LN#0>!d(1!&v*2$oY>vT|E38RE&Q7%NEZM=fSR~^`Ukm$OUmu$ z9>mW{?4P}VgP=uo{RTmyxVr_UjWz<3zAd5XxOFllKX=~@^JrYWtd`4UP7v$V9ZybL8z zgTi56QBhG15DHjYbXr<00#=+>Tx>z9Oa&hH@lbS}0h;SCyW%fG01GH&7qj4?p-9kF zENJXjqa&G^0h$;CexN`r6Uor$VZU4XgY|D#?x6l@r85L@gacC95`yId&%f`Rk%b0N ziAj80t!_Kvv4s238l|s$e=qdwANzcga;~;L3LRzA)mu6$?ycuhK39sdY|fu{APZb~ z#L3Q-<4Og-?Wwi%*EeMIeek%@JkT-pMUH;d;Dh?Hi#3nC-@U6i@p|l9uq-=~TDTog zNJik|!cv@aqZ)cMm_-g4UC2=Qc#|>J4@ys;t|P;9OyqpKq*lbKnmt-&Z;pMRXYBz_ z%2ej6&UoKrVq!prH@;Y#nyGCv&dKxHtcSn=rDURr4Ye-hAMWX&LEmME{G9sTXFI8X z`iyJm9bOJYUxpZ>8l*=NZv2D1Qt2&jnzL|7fJd-bwvaedoI{M@GPc^~X~P`4O%(!cj%R zbAu%Z@gsV5R~3ldP@(1WDvaMN0*k=KKV0#eT!ZWdO!ng{f(uo54Qpq_gi-JhT7{W(8~5q?fg^H_X*LEDfLJezuZXyx z+HT4<5tBxrCZy}t0!uSVQ>4ttCng=q*T!qs7BXSX(MWtE*I-%Fl|5IVNx@QMQMg1X7mNgGwlHGlAOIa?4t2$_WWoujTL*vo63G=D6M{5B1;drHLr~ zO6zJ~{UD2eqBUzGf$~i|hu2$zh44D2`B-;;@}QM(=&~dn6Vbw;)8xXWn>9AEAioqz zUw1t$yBdRZrl-^`e57}0ljl>}+um9s^S9rFnIzA|=3yjYf)2;xY9AkDG~l4R@$H$o zV+wg_S(~Cs#3kDBvo*~Z!bN&H-9oaiLNnivo;{_I6Yct?ZABpAdh7FM>qgHwY`S2Z z+xbZHl88UEaq z&NjSc2+ zd|Mc1hS$cZo?m<=qO>e)1hABR)DR>5m>>*2SzZH`ZU@leTr zp3PR_3aR%z$&WYV>3=P1Qn7Auv8%)bQD6+6wC)x06@5E(tqzw$kLc=Mv0O{q$7Ebs zKcg@213W!^WA3i{l_7rf*LB8$UPP130>yUgI!h@#O&b=fC$!^Gn%-x6W7q7+^X!$& zXCB3#B_vZ)wA~jHawD3I7mOO#1ediwP14+y@x7ZGZ+HTly{ssdQ8DOnzE@A{y|<}4|#9iyBMpJ>&qv?mTM1sIF*TtFstT+-GVDu$zR#-(N{ zlg(;}5xuTsCnYM&B!&4xYbF!O7^B2D9;K_z(Wq(?qr{72Z*>(bv6keXF@#)ZyTh(D zQ>faAJjZ{7>czvpcPndQ^J;PjKQ4O*al>0o%q*@jz8Ir0yQJ0s?1cJ&Oy-kE-CY|g zZb7S?OTfO#G>;`W4{^96`B31%gBxWA2@B2W50cnDqm6w&DH(-IkT{#E0;>o58?HQE zX(oXKA7`cxF4MGpY2p`URz}djZA8R+H9t`-a)l@qJu41Xe0`tiguzv>%A4#;^yw|G z>LKD>PKH!`ZKudi2q3cFav<$X&e&o803 zKQCpYyR2w3i+*3ikrKy7H4XgUx9~LF`xGo4F=S|no8p+C<$K${(BA&G@&?mhqmfO_ zoi?i}o8?CB`WEJDbu-jT%a?FU34?R(Rt0{?uku+so5oo|m0?~E4($5tHg3=Gc?p{A zqa)0Mi0(!=N$r?g-n9`a{DsH&YtBLW{xM$OtpwB*|M9xfuPL>{i>!(0J8-tjSps(IOYB6_}`!)t?B6 zHKtskpqGz2_^qSF&W@E8vLB~wdp=Bn>68yzdgRQ>MwTw_8t7N#nqmJ*1DaWKL&t6> zR>uqLZf<>-kqH|yMa=M*rP3zeqgAH2jvsGlL9=B+FvG_hxt zr)H1(K8x?@`Cu01G}A99FMgcKB^9Z|Cn-G~kK*I4bot|%?gjKD8g%Ex7Ip69qo{b{3_jyh_ALFjK@YY86Pci*pjx#-}$|TqRTdPUS-%kgrL` zQkkPGSJS9;3mpCNO12{{-iYehqOwEuxrs*}Iv3>zFGch+hA|{ReIc&UW3I=-mc*Qq zLq>pK#DCz4`A|OrMIU`LE4~Cve)~uK#ZvIER8-C-Ls_hHZJ`TozrhLPtR196Q6Duyz zO5#IYx)^-1=!~ao`lUPa1)3vwU`#MgfH=n(Hlr9yXjjt7_q6_C-ib=(P1d1PUr1L> zH@-HeHbo$FIHq-2Iv@PJdS9+54GiQCH+91K3a~ls@Z-ebU@_;O>`O zH(m8FaIcy!9<*WTN02>Lh@_=8WDv`_cJFBYs@;0TAzq`5Jr7ncTB)e0_@&<;R@vG^ zj)B6n4_|z&>slAOQ^O zPy71#++;mcDO=2s8IFe$P7S-ubwuBx4?P=lmW8K)EyNXwDYi{56}u)O&CAPJx{jA| z>SP)1P7{PY?(Z9+#YHwyr4EDzR#esA<9_2KlXd#mheMleUq3&7`}vQ@N{63(Q2Fq= z`+e2;!F$4tEklajMIsth4QJ?RadU2Dkcd-sBeLY!W6p;Z9uaj*WiL|DrN+qU!k+ex z_s_3S_B$b`x%)%nQxB=Bf(Gt^6MN@K7r(X!Ma{WWW7dyDzoZ;g=q0~;pyFTv+xVJz zTfoD^k6Rx<4pn;xS%X1c1>A;Ya|1PEFvmP9y+$a`S2#6+H+NX#z-s64D!v|^CCBEk z?;Tm4p_(7hOJK2Udtn>2&dr)Mv3}ve$*j>~!%3iLRN|Q(A{cK( zw)#R*v0&)p9j;XMipb;Xc8iP8I;X}1HN*T3rce9syX>3gK2fvg9O@^pX{;Z%ck(8g z0_#j^aj~r}3-N?)bQtB?WhW1R)cIWXsLM(&b7_yBbxZ-nDY@9_~!S2A=7d^$MiOKX~=)^wQBRVX43<(%v!&jsE;c5pSI;nSbCz z?fuT}%Zfn{qv1E(n{yst*XV<=aC0u6?tgr(DrlujkW@Xnw%B5 zZv4uQ^dQ7X?^IW{EFto}>Fg2^aOv&7f+IKqut_Kjmzzg@(A$@P7$L#Zv6P$Zl%0{m zr*wRO^1(Q;h5IkYx`B&ngiGODUwNgd_9*k zpANPNV*`VCO0IiP8r4WE2Y7F_H1X1LXh!k4=VjZ$(Iu(;6RAeslnZ4jNU1pag&gDQ ziXqwqCvgINqN1XFd>Lz3P<=jX~dpSd>LAS2s7DC%kR1r_!wD=U|6^!H1D@|Pnf~6(#1Fyrir9 z=)T3N9=ug{Yh{Bi>*__$o+lp;^-%XesXTJ+3B~ur*VMn?`{>$Het(sQ?2Cv})aX)Z zheO~@$4qzkzSpat-*ms^e8LF^%3Z)BYxZHk(cz$5L#v&a-oJbI&etz9)oUs0?v)pb zD@*h9R~0!QPr`)mKb?P57;S#Q%gb8FTK-*U=Uzn%ixls76H&M9Jf4oHNSqioJ|KQ# z)cZj983ASt26OnDQ~mwb_wY;YGzbKu_AckiY)T4wxDM{|jT_brad8?2;>yCcd-v>l zlI;dEMqICtj*h-tVO?i&AR_pR@B2gwZr2h-kdDrBSXe?$sXZTzcFOBjIHR}##q#9x zbWww-rs`KU+3+6!xk=<{?Fi*h-p%?Dkv8@=jSAgtjYpA!m-9a;2sZargoJp~GE=Pl z@dx~al{*+S#&c~g{Jy#Q#AD+R%YsDa0s=_S3wjYz(!tlO!QU|f0cM0l_C>3LOQUB;_KdhNmkWk2;N%1XSkVg03su*jicP1ZOot9zE#M3Y*cO~-&FQ(1{vR(g*mc4BGmU3y&W(@*z13m`9h9-NHk zmex3PV&Y?--dmFIzqxz)?9qVu9< zAIHN=KA&!(XsI&To1ulgQ}X_jwbAhFs?WEz`gCK@+FhZPIr-UBV6^eNcdObWZ~x@X zlk+QhUto~QTRCXvs{ggK`D2TR)TXdmS7u&0Hh;=VwWFt993FBx++UdUqMR2?yzHYy z4B#j$TK(c%EadZbGjY~p-$&6)O3S0+6P;%`)5-;OE+4v0?=ItZXJozmqx-UZ4~JV5 zy6Z`{i58Ifl^k>ir0vkn@>tM+js53W`To#M!Rk`iYx zdfA_v(ca?BU4sb04ljxu_aYrNx`Q1FnvlANDd{%Lu9m`Z=||P^kzMrn^xd9!W?cvj z%Ay?idauHDcSv~Xx~;y4T-E4iyKaY@^k6I3<(bS=ow5{>zTpJ*$2Gxb;XA|g%==>ac}8D}hO$y@?A`UxM)`Zn zFqN|rD(=cYQqKOT4He)IuHo!m)eGZo>sUJC9`dz6H=o9{kMU#w8a`J^pC|3=Uo#%b3rT2Xb~|E`80u7 zJ$+YZklSv0TuGlBBPvqO?7l!7c=W?lk&(_f#~;1q?W7i+;9MF=S>QGg3%V)Jg^z*G z8V5-3y(n8bs|&wP>Ek_oJY3)gU4n7>mlX^A~*V?Cc+G0&y zSM~D&J*%XBtnZ&QEZ?Pxl~$rjhU1z-PawP|GgVqX;BPQQzirh}pU5*Ddbrksx4L@b zRXn2qCC9izatv_7`o%E6|2Ua&bX)L8> zZG-HZ<(s}2UQb*iHZLJmAG$L0geXsmzU-D1>g&>4pmo&c!E*4EdalQQPjB|cTnN*b`Ff7$ zw5)0^R@h*kuHlYFQ`Q%y8?BWMu>RS%nfYVuUy=B|vfdTFsd7ONR$6ZH-YRL6=KGv< z)AQwXq@T(ljOAcJmk#ggz=aRi(lRo!pNrV?ayzk?^%FUfM49VNQZu2+a#tF?IB1!* z(WUd1tsfsrv51{>+&qlNt%W8<46MU=b8D#T=#6m?^5yd|qSnc0Q;c%zh*-K)o?;=g z=Q#96wLOPfS8Z_O9V>43RBN3gt-XechID43cOc_IxRI=Jt#gF%*yaHVbvRu*opDNA z)mrUYh^>0p8R|;&UPZa4!=;ZB*%qCKJMtyP*GBp-o{tfGXnA#r~)=w@e zF|_#)&HyHobqAT^i_vhk1LHIbp`I~$;ECa8p~VI&jlyGYE0s;^I3z=(b}4*sk$*ThjaRv{apm!d zue@mTI4hTCB>JQE=aEMvW5ly#8Vmb`$tPb)(wd8auSd@cqdSFvd+A?sPv*6P7Off< z!6q@w2Tw@eM|wfJlltciQs;ciVvC>uoJD;E0AHrU)xdw7?*Yga)F4zQ0E`QOP^!g4 zF+K5cg*gZ+EuIXgmOy3E17+?}gUd8PsQu$%9P9Dqd&?8(6aX@AR4+mlpFo9ohGLun zxJ-EhjlVOL?{*Kx{vHTp89**%(o1EdmOx`uPR1I5(FlZl%UX{XC!@ zegL2$#6wXg@hR-(+_XNr)SQU`rA-fntG0(6mjkAc8VnDHg}niS3|51&48_wJ#S__4 z5FYvm@#LyK02LmLd~E_;^)>|QTu#OZhQ&7rmR{w|!?y`&32#7-**L@4f&sX14#)t& z_ROhK#N`l=#RI%d!)g?&IrK<;JlPR}+wpXAbIhbHAfhKegNFM;8O3}R(`{NU6 z-OJ&!w|gkX`yj}2HIM;9#g_=pL_v7zS^7{ZC>Y9LlhRb`zC_Ee+LTfict$PkBys+}0A0XdHE1&tty& zGFjr#n^7l#5!;JQ!6!?OxuD#N1`yhk$6O5D*xwf2P{T>(6(c=_-HJz@andCZkY0Ni zif_f^WGY!`eAF3BCW@A2YV|UGt@=yuwWn~`+amm3n@}mD1Kfz(cz>h`K&4yG-E+6A z0P|k}7|XaQTJNN3ngy3W0(seo2ymHTX!HGIjYBG;|Y!Q<`#dRq+T(??i?`#^i^n6}iC#b*59!WeZ#a zirD9gp@m_2dH5Pde#!K(Q!tJd%^Qjn#aUss<{X6$1AFiH8bvnepImcKRg!<66*aTY zu^zXgr0~%wdO(_Uqu_?}3V|+m;>@Y{4HPPCA;#v9&oFHc)Cg@9(;a%@c3R{Sh1BM7 z_TlxW(_hzX_I-GB>j=Qa1))h(BbU0JfGXf37fS>4?KY8XsvM!`tO4CfBAqU(=+VZXlfXall zU*t#;aG%TpBLHP+P=g1{Bw^##c(|sAP;wKw*zhV!$su|~NjVwMB7$lUgg_N{hBxyW z4z9$iu9K6Kn6<7;shLJ#jMXIDLm(q#o4=DC-o(o#n?OmhOJpo7C(Dwl&|!a`o0CFD z&x0P;)!NIDm&XL<7faVOoaogod&kFtq}Z>2G_Rl;UrjpaEY9a0*KB}FIZ*1`PD~V@ z@2?h*d9LqFm0#p5g_FuWXW+Sak|R14CsX~L*4K`aW_k)Ie^rkm_-KCFvS!WR2hSO! zcm@U5!)uhE`xwVPH9!63oq!toeiI^(MFldVwvLO+^q`)_0j3r~ZP^ji>_m$yK3Spt zmLp~-`liyQl6eeUyv>WuOwTG=pz)%MM#~9iE4XN-DQB_RXuqU#^<(qT+2x zG`nEIFJenbx3m@1D<~u;3ma5WPZIQwn@EY+S)Q|MJwI_VID(%ytp`E>fN&|iMdW~O zjdl0T@rc7qBF>Oe>)z1*Vh5fJE|jZ#2ZRe7vyv~k3$5CWx>_}rc_e!gMC_jB-fOP4 z6!UYWwtrA^w6!%^EHGl!-tr@+y^sG=NN%0O@?Pf7%3h>rmA>O@9{kG8(xs?gkpmkg z%vWD{NbH}bcKSGRzk7dw{o#+RP5^aL4_q4$WxC)@E)=;KgFB}+e= zN1ad@6uTst;a9f6Nug+@c}d7Lmwf?O!!%-fGqiAZ5-pzEG3FB(R<>HUqDs+uJ3TPG zO625oqtwo+bAb`(+#FXOKJ1f;zOKZ%kyq2TB$OR99`^Wc`G>A&gKN&W^y4*X@oX?YH93kLHH|8Hl^1$+q7!we z-d{?+`qC%p;LHN&ht8F&ufa!_d65&F*LAu#ieJY)(?9)bs7>{cmRpBj6P>;+%J2IW zaq;jUSN43{sJZs_P5qH?pZ5ST!5#=hLp)rW3(5%Iui#@3Z5$UFuV62wS#Tn{Y7Q>v z+)JG|2V=R7fD_8kFd%)%J62G^DD2;YJY!2e=s;)3lX&81ice_@(HfYTlKM+|XQb6Y&|x z@i4V%rU>p2IcjmC5?!A*DC!$>%;BXg^VLl)iA2c^S9V^OQSa;0m6Dmh4!mp+pIw*l zlgtV&bJG=-(*e`ZCj7z~j@M~7lwX8o$GtqW=R0CZjVd%JNwkWaE`3N-m;o$+W%0Ul97p?>^;0XqPewvnh5kshI8K0$%M(%|kMkv?u7 zzmh0|f}H}v<-gKs{5(PfJ^Y=*f+9kH=OzcsQ9b;BFmmoN89Aak$Gm9u#Tm_(^q%uPk z_@#_#k7a?g;J{jDFhIWw{TZiik%bUUCj_?%!L>q2LU&d#?-V@M2H9Qmk7#cT?u1|- zLvYNl`YU!P1@6&SMft83$p2WE{QrTyy$7~2!QVlTptz@$xEEVF@Slr{|GT>5o!Vq= zQ0(rB-}c|OQtJ>*0Tg!@iMvis3c~I+#s6Gtyc2vmk$iXXzxMbxawPq)HXsN1hWjmO)%gJ7})-N>nqiWe zJE{M@nWTe#8>xx{GenL%&x*@oAt7=7bluNt)GeeP)c+po=f&}>BMbN7ra3{hpBR7j zNN78$|2^8)#q`64%$Ob|E{6x#%>_d3tWG6C?WF$qP=CAlezmVl4EIeG1dZQzCRhRf zvqu8`+5e0BpFyd9Y=@s0>o(fAE(m7oFz!m=#EZnVGTm-tEUFY&(?%sfiFY2?T;E_?Oi534|zux)(v4K=2}Hcx}Vc zQcr_G&?9|=t5}UdgC{HsCTuz7}vFSyDlj^plu|8>RVhS$T)FL5z z6SN2fEiZ5kj=-@DNw_{Jh@igJH6e|+RMWJTFp-cTO%S5Dh+CSG7XE0uq;FRpZM>be ztgpwGHMS(JbO;37A1rzVO%rJe36gkIy)BL{BevvkvuKi}>KH}3+Nj6a*%-yzNbmP{ z*YS@8^|h3U+`=aiEJ?iTS`wZ=^zq(Wy9M%#KA>R)f}6CNzuVR{L5t+-t?u@E{C`-# zWskeAx~H3zznq;Z0ss4Y4Zpvw-(XWJj@U=T^|@A23QovV(budQ{o zr)!{9xQ&#zzpb@dEWt4LcYQQtMSjyqV@u0l^zkF<6X9+x>FJ_l#5(qY>^UEiPbw0E~}OJ1OzwPd)Bw3)1%4#{=uK_q*EP2_*%hvmQIN4S)Yl##bB zXpfDQ8Q#N2o#Y=|(t63it=|gbFZs1)nvP_ct(BsmyRLr}7*}v-S#0V1!@uh4W+YQ~ z_;br>Tbup9E;{~TjvL|KZT5q>U^CfTAN9BOf6qs7KZE{JKWC3J<=^8?8p4ir79`_mMhpL wG$g_!T+A&vSj;6fG$@Me$L~unDOqv6I3E0|1pf5=AN&FC(E(A_CRceiwRcgWBnAV{|~BBCH2D%~I*0us`pV4|YlGx(OT ze&7GQ_qq4E>(Mjk>{!3GSM4>-%+`VF@smJE@JSdoNRUWCgdh-G3-4EYbQlN(0f9h9 z8XB4;AQDm%HZGDPX__dH8;v2#ez!8>-wh=HZR`IN zF_bi5`YM_lFe`l>n4Z3(m4d#J&~L@~-9U2U-?jcXG19>#;TnL2|MD1>dN61ni2+cE zpOZVcx38D0gPlLGf`fy%zJr4sw~-LHlDmzcGq;kC7@dx<7vdERQYs4toQ{N=gy(1X zqWe!g{+LP5@Sg_!1@gD>xITZoLvgC z_P?QA#rQwG3=!k901ATOS677D{^daA=zn&Rqs#s=h!n*3o7TUff9oXw|Drd5Nvt)1 z)caS6nMi{{%U}=^keSgz_#k>f6L8Rv=c`p>>L7vacKC9~mk)vN_ zbYLVoI`|)2`L*?Ltz<_)K--B79PpM&u(gpb>Y@RjJUdqH_>)ar+xqOzEl_Y@jX&*n z=X|oz!K-SeBxVc^n~?k2C)o=1B z_i^0rl-Xjdph1y6_1i_kTfQ_QtY~KcB~T^v$oJHPfr~x1Wudbaw$8$5mb=S6)+xZ^ z{cgvx0kI25LgXI$zZC-qa*qKL_&0h%1vG^X=!zWB))nC)bd&<z~FKR}&J_quFBK4*67)xa=soX~-C5^-O>Odft0)f?{em;7t<)$)cd3lxyD z{ChsyuEptwP4E;q`3Vw!+Z7lufSZgifSs!!f;N z?5gUmG~M}_mX5iGBBu5mogQCsK!W912Sb;_NMm0YU1JTE`adu<4hf&UM_ZiqKbw@>!{k>!$dm>+qC6n{0F^xe-J|aUt9kY-?|(yAJNrweo&$GiEXumW?_?m_E$heJwWm zFeh33FgBqOoufca_*L9!35F`^GjLVYP<=J?Gk7HKfD_+-YMn>JR3^H?a<4ru6L_s@Q*%?+8}hxj^%;gL{dHKWh4Erw$3t(H*;~o$LpPww#HAH9%4L)s5>mySR&#g_L#*{k-wthZ;Isd_6iucI<@OlEVb!>kf zi)7EsSVqd_!YzVm@Zf2s6zZK5oz3g7M9oy>r>CbQQn*vZ#jzZ5*=q&vP|ODA-d0W% ziGg=ZMU5A?MTRhuX?=DJ%bF{^cz(?qWoaHh^k_U|`qbNQm;4a{DKkm1;jsL6&{)8% zQs014qPl*SUJ^thsJOXG{d1!Ez6sH;!4&tfMWlztMQ7|mSe#9KGwtM!e%XekhhHUR zyT;LDH|@Nt;57|tqpOexOInSgjJt25r1L9Kj#KPdcex%9_R946kC0Ig2jZ2j7Fs(6 zLgm6NSFKdNBt#;EF)QG;{B)UZ!XQdticAd&N!%s8q4a}mA_8t&aTYeGgw1tEsB0y9 z-x3qeK9_W8wY479d5~T%5*!vfmZ^Ag<1}@(yiKkx+CX#C*pR5KyHwS;{k+KHN4|At zIf{GsY}9{2230VQBe5n2?6c4BuK`pkmZ_78=r1+YX!M^ruQiNL;5_ZR8Gf2j`{=s*2Kue8ygOy? zBZkzsPb=F3m5$`3jL~m0f*9MwJs?)kFkjq# zK`pK>Hun5(oI7JDpzj=+G+`XB&ie}*wNZR+gmz?-hwW*_A8(!ym z5?nqf)-&xbKEb0sPbv1a+6K!HS+gHs?zDYis~H}+`w)eB%xZq+#YnvQP$(GVrVtCR z&OUQr%?3N&F2}U(XgkDr&YV`u*&sYk zlF`C~#peR~OS>Q@{Qli7t|VeP1tumGRhD{iIwM8704)I@bCLw)_HE*7`T&ukIHrMe z0VBoJiUaTtojp5eEcf$Ed!3F0sdF-kuzRpK_(f7KG*Q!`Esp7>4f_zC^2+0Od>Y-e z!4D*On~?@PbY6`<-DDo2MO?9Q!QnR{BkeC%EDVYO^ z2mU(I2IY+RQ&`ce0THgX`tkI=eK7cqxI+W$`>G|DoM!Sr@w`BTk>~(Uxg&bGi^Kc6(**E|G z{r#QS-@kwVSlHq9d?@+Zoi-}i!Sv%uKbXQ9#1q{e&6{v21BfO3Q{_=-g>A57inN%i zT#OZ_nzs>p=$?8QUVZ_VvlXqLH#`-unD@&!{M*FazSYQ*sLch5yiKtjDjYMPeQWpB zxvD5dQHiW>l#LXQ;ht5+if_7%Xt`v^`x18f9B)N(yL-U)?7OO;yL zhiVv>v-2Dr!Pf)f3`RG`yEKETs0F!)bf%wZc9v1flGU|!te-WN7gL25=Bb{`Q=hMP zcGn-=#h_Na5J7p8Qo#SJdvbi`dFQ+KBH5?96G?7*!=FVM@!S^->3v&aCex1MiVs(% zFh7V%5lZpN*TxCiQZSh@uUT4GsD9!cOj9y)HqE&>Z^F_7FfY;~6xkQ6FTS?N4Go=lyd1(R zw`?fe-XF$t(hl2xnKmqhJ+vG)awFvoDZB!*c+4F+LXC|HvDhG+@!fTGAa-5?^x`R$ z9SiIU3W5`zpa){M&_O)HqGN1-tO4}#jv|DCqeH_olqjURk^NYcH!a*AER=apqGRD+ zWCMPqnz~*;l11aAg|a?&Z+Q2TJ2hmXkR1(|53|JeYy?S!?D|7F0a;^eF-MtzDu1S2 z(j7yJESm@W#rFnN+TaAz4-5G39y+OmwB>J9e4pNmtk&9%n9tP}jHi*+_VeL8431_H zmV7o_E_<7w&gWZ=`Ho7CtPSmgvRWR9-xL?cMuDiXO(3W7E zZjRp#spr5-Omd;TaRgb3rRM+eZ9dBYmLr zfDfg-n)r+MTex>H;BBbaojt>oZcrL7(PZ~^B%GHCzKlZ=8&;Etsv^tkgpa|f$kjQ5 zzaDDEl5J8!zX`Dhw)Dw=(_2`NPuEEnZ4BX*-=Dsr&(KP>oj6q3gAh4uJ1no&BbbBlhGLt(bYi)jhzcV9&5A4lc`chSIEqE*E_PREbfdcag ztb>jj6ZeFCUR!|Q1|hO`pM1RT#N2sh3p>K8OLoYD=F5A${1y6pQCK}h)ffg-$vn>$ zM})(jaqJ&EYrD_KurMvvy&tJxd~jWKjS?6wd%N-aNBWRXBJfbll4zPy2NA>51^ z+SPPGBlmrO&AEvhGkw!dRgr(IQ_Ns}<1-UW4;{4{cI1+6IV6#uHX!Y0XuOxHD*OW* zI~yAti?|c!<;)D0h(QKNqQHP`2(x-AAE(SRD>Q*7SPc8yjb4RXczp0diq-CW53*iT3Nhx2@(b$DFp>lHA5` ziPc2izjN*{OSG^LH{LjSL`)-IJhnH-8UtTrep_WhOY^Zcjf3_ka zFqy(DN}VpR_S8kJ+kd>mjdt+dSqPCRaZhLm-`L})*X_(OrF}4U%k;b}bB{T>b6{?3 z`_pskr)tISFWd!kQNwF@)ge!W;8-c9ai1NEIWp5hl~&L*2Z%yvK+oNXj+E<42ThC6 z-7}WBlR0$oH>p?nNKiF6dMI`2lCEglb#-56e3(?Lsr9Y+oP?244Yl|?H199rgkO;^ z367LeuhFI|S8wRAgR4*1&Qrs#!CfW zrx&DT*~t@m5UIFIO|YxGBImIMlrER`b*GaG zjd+LUWF_wP8%d=kq87hRPH=;raq!r@!57bu7)>kWqtX68B5T6}T!D&rJ*Ib) zld4Cd76ot5yMaV7hQnlDj68v`e=(A@#Cl7fv7i9xGH*Jau~3ssa4qwkV!coJOXM0vHjy_#Q%33;;aS(x=yXGb@p zYyG(xu>U;_@3Onjb_Yct~iyn(PVgJz+reR?FF)6%VTvK z^CKrkNmkNGfFb~5d(mdhlRLiq<^@0{pryjM-BJK`KcxRYw%s?InQ@Di+^nkvL9{Pu zHb?CR^0`Qp2W=a3g<3FS-3h(c4T+JbJwq{1H&9;(c9|o5$?9xs7uWUkia^Yn#lxjl zmN%Ylo?YvnZL8#gAo2X@{g>y#QnYEqFC|Ylql;5lSA06mMfQF9i^g~R*!Fh$CM1jt zCmq|4IPgQ@$1Cm@OOp4FK3|x|L42<9NB7BKKF{K z-_AG-b)vc8-A*3#egy1@JRxG#B4J|GxC#B6En%miPLupRzJS%m#UT>ydk?yn=q$;i zOID&yK6HL~^5EE#$krz+6&m*NFl36r4o|CfdVeou>Im2w(Zh8ttF%Vr6k^j)^lRqW zv(<{_%5>P~SNwfDnnHkvJuPOdCl|gQ2EI}JbsDJ;42EC~YpVV5iChK{(>qhYFSDoh zlbSKuZB#cfyK!Dk0%@#%>FHyD>Kp58>j7e*VOam@frc8>YkjtKqce9Jv0bi(J%05* z^tkiZ8=%bXnA``h_whVi7CTx->-EHQv&8|O` zifWY-0<4nbA#01DtuZI&gX-#>oOpl&rHt}P!PN|Yf(U97XKK8hu3X)1D4AO?RpmqR%1|sgfUT1{mP0RJkH>8ClAj5|EtuJ-S zp}*SMfMR;pM!rUMykPzWlhrLcvL|{g)j1lBwbV%pJk|=&@YrUVZmaBMR79Nf*@KLi zpY;_0iQoSkr`$7l81vgFiQD$}GL}#nE+$!BUT%QT-Og3wk`x~G8QskF;l7^k_D8Lh zJrb3hva4;)&F$t`NonB;{5tqmQR&qQrzcHv(}LFty;cLaJ-!4>p0#?QnSS`n0PXNt zZLNo@KZ6%(84?Hk#!|YhN7mY|-S(e51OgB;)Ya5yL<99TLtVCFfI?Q=qsz}`e z$~dib&33(71CD<4;X)ZtoRGkr%axV<855k+<$qpu)*+^8l(UDddF_JYNcZL+N9V03MalYLEMBT&4=38&Hx?@JVDR9S>b88ibo@TB%feF>$Sh4O?wWJ&q@D>bBI3}fZb7X| zZ+B;BXXq*~B@f#*njli#YQ0e*B2qm>fI0nAg)Np37mu?gs`P4g*@!XYmx3Sv;$`k=*UWWv&#k-{Yi$u00Z3R_xJ3xThB9@?)^vF(cWKG!joJ0+-A0pe zK$P5zh^H43Xv`%R~wq?I?9~W#F)Qql6U*K9=lk6 zb+DrWWC$)OFMoa}=g@l|e*R-sI9sAy>!Pm$0Sl3xtW^S4DZ)~|9Z^FTop+sn$UvZP_{^Z`RdyY-w@-&~fUZwIR&urn}(SY6_v8T)!n#;4dwY9Ol zZU&Iab-`?-{O?KkBq%d_3{=t&b|tSA3h4C7>#(~pN}CPxllP%r76?QR{>#bDil8fS zEacR#d6p7f=SnT*IpK(kf$Z7IcS4{Vmx4v#(2{*|o!4%P3H6AW1NIEP+FicvIau*j zx#UK{WOa&Yq3a3omM-K8po>C5f9_8K_Jm0qjJK7mO$uj?E4L>q^72ef(qiJ+Z#$Tp znwl7!3~}6*k*CwcR%K&ucY4=#Zo(Dl2Y3%c%!;Rl{3jsa=$_YMK*BD>v*m)lu>51M_e^f1trVOgkhd9Ao;XbDSkz7n`& z)z#JU@9hPEwP#{WF+bY#KOjQr`%C{c%Spt`UBvP z50KFtJS;46hVkzijb?7v;xTSBl$Vt;Bv4TbblBL~Fl?2=Vo-4?V-*;Sul4&1it~O5 zCP<(9G}xb>l^Z8czxfW1O|3I+Rrnnx-u140gM^F%BtJD)9uq4AR}dda6g}{$XoXD4 z7kEVY$z_pDk$cRUy_WETHWq_IViB{#{tw^l_X(?B++Q7@t;Tt|q~%85h{4aqac=2b z!KHb8)4?;|m{}XCSZ6-zw}@`UNTs!vnaY%jzA_&;^TF^$ zX>xr?S{Jd~ZF$<7->rKixwV2yH}}Q(3&9?vX*R!)+|QN_xPZ5{CStSZrnDU$FDg1% z z>Gj1Jp7l{-Pfxx{wl{4$uV?K&EArU)E9obH zE;=7Y`S#0Uk%h9*xR&;H>iGyPt`(V+tvuhTYTKIE&(6g*cIXt2Wit=FJL;jKcQjUp zli*P@v34st5i6~)*T)ZE$c0%K8ttd|ds2r+xaWIW_8$j^MTw0lkG@S6#4Ql#entd>G|H;BR<8pvd7PAr`t1xmtju&@m*z_gI!>cpuonW zTPP3MsE(bx7e&!hq)gqE4&0hFS5h;ZP~?*)9~Gn_1uwpiAgZlz&zG>HR-*ASZxP5Sh#S} zd3`3ThwF=ucRpS_ca}xD`jo5SyjPzX*~`7RE_}~hrP=B}HJN#K+2_KjQ9WdgKF^9y zVh`Q#N4snZ3|*rs)ZbVedYSdXZnJK^SoUMV_irutWJIvO4ku^iv|gWP7DUP?%WbLM z2vmx5BTp9{)%hB=aZhW?bT;24Ir>&MZehq*`p;$u2iw`3^8rxDs_^quUL)0Rg9pkR zeVO-P9y$jJ3&r1gd|OnR^-Zs_l!Baczu_-?=nM4KHKIvbPun0jmAQ7dQ5CT^mp=5iHZq=}QYM!c;2ZnU-T`3|(i z6!yjRTnO!xbhpshW7q`pS5du;R@`xJlYN4;USvbzY(Mtk!1Rgrc!noSKc~iMa3eS@ z9r@*+%()#}7v!nhuHzh6n5RASpqX&jGjYMQR5Z#G^ql7@u6L<@CfX@pa*vMTNE(@} ze|Dg)oLXyGvC_xOaDomW4PUHr8ArqvH=QusSiD76Kf$)Hk_;bq8=J;?wO`x$T4YQV z+05V#xw1Wfhnv0j1B-XLgXZTckGL0^jc+h}S2=me2`tysp{sGl$8&Q!?CP!3ik z5f^|@<@6mVJ%s1sn>w$+$Hk}9Md`I=TV@_k?OtU?lIto$MrQVo%MPjWjX8!vj#To- zi$g81XM#ezpGDv{rfg`Lct0!oXfWYKR_#!7{~qz%PK?V#o2th%9!VhAT?YN3w46XQ~st`{X|Gb9i75d2pr?&zzL6zORZr~0*wfV+&53eS>hcisO z?`-dTpNMe!$B0=_X_9S1+nA`5O|hUAkJdeECa50Gz(-7dmMoJbZH@2o!;7hO!h_aq z8_q1d^0$Pf&i36BzDc|1NPgkmBSq@^KEq?_)OM=u_6})c!Q;N-t=_Mh+Fol5ZWM)W zU2!xoZlR6@@9!o(ymP$r7CMa|L_jpDSn!%ru(9wBgGXW=cPQ^6V;}9zC%-)At%G^X9b-q>Sn&mxT>tF+BXPCll*Z>tYRNNcRD1z*}G}# zpUa~fA5Wm5vE^=GLXji*QF_nj7cD(OPE=c;{+kqp3i5z## zE@HvCy~dkDaTeDiDAv-$Tf-PzALMgtaEi{VH6qi^T=aWA60MkcE89Hj-FLP+h27TYdT7? zo$xf2-dUVYRrbKqMCx;tW}+Su@rV?RM5rdl?_yB7RK(&@6&;sT3`A8{6*!BdJW zY+o)R35j>-o!A*mPFVRr|6;ogsdMDe7e*^?@HkfWRLIoXNPHF(7-_$M+clU=|82sY zZEksC;!1PpL-oBiQs341n#`yu-j>wn1gYfW)wrV&=`v5&>H!Bn3n6_^WfTj1LV+~X z-jG%-bJuc99yR1}q0LKEmcW-C&JKNocJ%0hdd&-&w2ZK~#Cx`@kHnIwP%OSM*^kro zZ6&mwKcd$6_0oMyktnmbZ$zW0HD!;i^^PaQfWa~;|7iDqz|!Jo_7uuPdNh!09U=vkk*Z|XHW=;~e=b(_2XBx-n6ahh4VAoS*S zJAwIxkc?Gg!6)|5AJiuK2-eozbGZ{Us$=Shj_jAm^TFFdpl0~hqwFTWS)MMcQ-2=f zU}wvU7;-3%5WZHl_S&P4C{w@pM_VF0H8ol0qI~8UVdue9!(myqy>x*^p*v9A3W`xP@h|1jKErP#V_%mOcW%aU_HXTEEV z?qHtwe7?i_N}+=eCigu&KXJynOY_TY`lt;`2CRW%8J|W7?}bFcJVl1{TTs@Qfkz`x zG*640{hwpZ(WVS;+`-))@@jVL%Nt+TNO5bSnz0q+epN?kiazuxk8cl>o~~{!c>1tu&H(PIE6?zjz4o>JsUnD&A#c5-d@n zaG;Ugp40U1m621n$h&EzW#@;hIWVQmkYtVCM*l@}j)wX9v7h=a#_xI_IU+Qzv{>dk zH6wN!ob57sA~r=W)8bnu#~yhV0;EWpPspPZ3D61R@doIyH z!dTXmr?#5HSAR;HOMFfsDOC9hncX$MtL~N(X0do+l*3mOrjrYAW+DR85176mAJ*xx zOC$T?J6`%t$P^F9Olqk{$#ULOa~5#$mJT}9XDGt-=^$4&eKT7~u`!%l@MUybYedbR zi5>k!ezjQ;&(!DALZV{y)41-;foy8)61D)AdT9jq%V1j;2pl``>Dm*#Q5mwy(EKMY zxZ6s|sHF*R9o|zIZ&VzZGWZj(wQZ&@zWp|4^|-gc(}{4zk56~IS--{6d8$a#;af>k z_jO2x&Ru(JCLHS+P_xNx`&%5iBpcZ)K{fG6bT1R$Gk$sUCTkbTkHBZC>#-?eMfET&N|~GD6hrIWBt`G? zuj`A_wC8=|km%90>O_27x7&bZ+Tf=zD@l4!CC<_k%aWs)IW-;mwZvURd}{t{>tCOm zThkJs-~IEu;#iR&;H#MsJ~$Y^6oe`ULL!92k*Q_DXeJ;qW)B=yY!i&12#1ik!hSk7@j>~ zoK1Q(ifLI?p@wJ@0F7)q3N5fF2K!wP8hK(bDi1sg+npYj`l1(;2M`AYLxth@;U`*0 zF_K3iQ|N%PxIkb$XKQ5I5fG|SX%tC=EGn&aFWN0QgpnBrQ6ujole$N42E_%5^)*+Ae)LVrhbte@m&=~4ueKaqvk%6hj2wMdwwjl z`Et_Mb`&1pvSf?1ak#L93zC_g&Thji# zA+bIF^rRGfsC@pI$(}&oXo{mq1=-X=m0-yyY-c&9f|>Mcp=xm)SG|gYg@$V32R@T- z)?R$b&x^J=aww;fr1nIfa<7ErCrhC-AW8q6QqD@`nV&NC>Fg^z(Re5$IUwulI<9Q zkC-y^ykU|8;Qqh|J;kqrYoHro?chG967-Y=sa^KikO}0hx20%o$_(0}^Gzic4S=J& zq=lWG(OwKF%e*bqjyUn7yWCN}_GL>uvUB^+cY74sZy4$D3qg7$k`Y-nIh|wu6d1)wWK8)H?=zMW_NJB|(y$0V7c{x+1&Y%uSPd zMzY7~O8gl!Db5B=RO{%<;(fPo1rAiuemYQ&l*PXB5Q!O`$x%sf6A1miJ<(cNg56UbNSO*P~i|c;%bvj z_T>SncJrvFS9B)nHgk>mOs!U6CsE0p${Oj^IPHixb9X?yxw06pI#771&*M zQ(UtuXi;iaR9*GN%MGfy0@yVi)btgRWN(WYGADW81Vf!c$do;BT*OA>3|FF@hLiZ; z6o>kZB5TM&_V|YN?_}H}$$N=(>!cL5)K=?m-W$JfQ`|R1lQ-!j&S{9Vak9Y_ zB=0|yO{frZ$35{`?x)}5OAw#1!3{GRu%vHHI=pF15|cE@6W^HP;%*liw}~+DKC;qEC@pd zq=;_~N;auPg7%m!Hn$2t2;j!I{6e|h2W@SYsCxQl)fMgXBy>!;c={C`7wyXoi%%!9`Il4C9I6<1D(AoQziVfF zWY8wD*fJIHFvgLC%PFYe?mN$pGHg^DGYysK7zd}R7MbeT!vLvx$- ztAJ-=YhP$D`dFWw-+marCrS5tHVFTHeOkn6SL&A+XiqOjy>7g#yYrQ=6pRg&M^x#( z{(1rs{7MLfyPfXao0_Lz&mTs9`$7joX6peHHo#G(=#hx-BI>Ou-0)3VdU6;BseLqZ z*`_RqGYrcTKZas{6IFZ}hIa!$mVtN+T@|Q{N_3dnG|DhF-@wuCr^ku8lws8{myq@G z#S0WM;J7*WQ*HAlNUSm7g&p+MA@L{5Q8N-GI}b21@+Ya8GZGaZ46w@a-_&Ey>#0^5 zWGdNS3>{$qhxf3o@JCS# z+ajgfmqH>bfelK451AfhJbQz`#>7h2gCQ93Lh7eKzg)6KRt$cl^mrn zb3E4q13^-TLZAZVLa&mMY=uurgwW?!I5RU-KBuahZ-!~5Jqve;s)nOpX5o%KtMn&T zop`;h3T6j(%@8$%YC_L@<}55GD{6v+lg8eDyPP%@>N;b3v`v)`+#Vt7=6ia%z^7ik zXaj9s?`LzLzGLCDAl0zJ&CDAV+T*{Sui?OxnLp{jCs;7tccu+Y3G+?YRU>+Dy! zc2X_+6k`e~0*YQTvx<+qO!<`y79Vl#ORTJDO?SL4K1->Q+E;xR`uuIl#YBzFY5uc_ zH%)iGU9!pos*fUUE=84QlOwt}9VgISh8w}Az{D{Vua#9!(sH1{dvhkm)xU!J^se$X z%-Qst{)|ZEwF;s7m5dHRu3&>JqRY-g*kr+&4Ju$RTLMDsQf6}cUQEQh6rgqyK#xY% zqDLla*o1JQ%i*hJ#8NjaKq(n|F-72DY-Uw4+A~>v`cg372ndM^l!T@thm7AIi_JU@ z#}u0erCP(8M|HufQb3_O0%xWL>d0uFS=K!JI7(mx6)KG;afYMNEyK|O55Y-S5CbYv zRr_gbHmOZ4x-$@tnjYc7W+Mu_kDQ9$HauO(NrHv{mTInb@qX}5CASNP+MO)D;|6b4 z4q*%eqa(r`!d~V`Mi+Jhi9wp&C&bnIH)peKC0O$1T&kJ#Vg&ncUWnx;I$rI#VM@ax@q}rdCc$az{y;Lof5*e3d{8$$dT! zPO}+Fx24WN5h8lP889of#b+h{V*|-PQs3E2}0gQ3o z1VZ_tAyM_^^wj(Itm-QxMvIROy6!u$B(F@k1~wH9T02R9F`bSN1eJ4V zC_n|CGg9<0V(5Ut_~~#6aStP;ArUN)0U}DL7o!ByK`e6A<_JAH^QRtP6-EdJF{{{s z3ocHI{Bt~fm>9Xz{^G-ng~zYsyuzO{Yf^Zw)V7|>4#4eyG6dqu3tGGkLtru&^FrklD}1 z74Wq~6`dPm=rpR(UaH^5H!_ShY>79VAvNS#NYXlC+ltZ@!#A7Q){eZXifQ$zkMv32 z1D+I{G9$76&Nr3u$c;9*RBioXZ@u-exvS%U7Z_-ps7h4Uu)QwVG64Oso}`PtPJBJ6 z{+Z#q$WEs%>3B;$_btw3i%VPb>uf^;CH$#w?k}lcqYR4xpE0IwZBQQv4@V3L7{Ajrfgnho>+m^LKc(}x{h57r*T?tWni9boYpr>J_hTAib3yt!z zWmD?tLGo`^qTlrQX3TbPFd{iot2YzKEW{>ey+$#n8z_)fCFKx5S!QGK!Jf6weV5bD zRKs@cQO;wLd2T-ncZ&~iazz*U`4U(rY-t7a$KAKB3rzDIWc??z8r2zBOFv zb`)t_ne*HuoZIswLqM%&#F0(8#KJwx z7JZ}1qvj%q1s9i=IxC!Gli|6NQcvtT!&W9dTbu4^O4h+7Et}npyPTZ}lI<%&7{H!_nGVWe>|vbAX>D$k zI@Q(-BM86Wns1FcrxyD;YcVG+UrzeS%}#Q)cD^;db@J3R?8|B;A4Pi?p{HkZS1s*_ z)%E~Y&+?ov^LsLFwKK(@0S^n;W8K=iR=;`%4}bA~QxX(&kXjnLDp`or`O)Na?CbF3 zu#L-CZBM`BoWZfeHzB-YN!T)HF-%fhm{wx_RIknwkyngdN!J3dsj~jkP`)hj=OP{3h+x<#DuI*n#3!RRWdFM^= z_&`qd^UUoM|0>?%L(QPj<}3~#Hq zUGgTo?Q)m@PROKJ$=l+O-TT+@K2Im#c$eW`^3&W>Kevbxsk- z7T-gchb=!e)kK_nM24;HTYl`?k9ZyNJ!}*Bi75$$;*v-hM2_lb=j`BNL&6Lq!Dc}M zemC!HuQhkrJqd3jrT0@MH5hU4bo z>*?Tb<>wXP`}1fN;4ljZ_kWJXw(+%dcJT*Jb_nqOi3tYqp2*7HKg9c|R$RA0D?1xN z{9n5;oE$tnto}Ixf>@kqfQOZ{gN?l(uzcVd4>(A`#@*^)t(dkp{&vn*elDR7KZYT@ z___bML-Y6kM?1vN-@*Gw2d>Zc_ zl>!PJg#uXUpIrYm)DNCHVA4)7^a2cR1tYlrdicq&obSFy`aS1A%>9FLCz$jF82SVm zdWM8x{L>*PzZ`k;E6DTY8%5Tpy`Hv@@{EFHzg7JIQ zpI!8y*qvz5wW|YZetUq)e>~RY{|5GtAozh9jZ6whhQ?w-kFK2H|D97xew77YgZFzq z{1kpaL^=bLW+6exu%Sm-2owDN_>up7D#@?l6LFZo2md?7e;}8GNr%AD0bJ-e90c-T z2a)`AB+0M*HxH42&;MV^;s;19nDj0f3de^I;2}T|hm!nxGRd#3=iB~6(EXikesH}D zCY=C7n+c#N_z13lA4T%>sU*K5TvkJV79R3vviK`S{Xkd-lU{PL32n5=%t^YgP)yDJ#Dv^^ko(vifg*Gz-P`@7VfPnh7^?!%@*T(nH^<79p zJ0$^7_z$%Kdi(PZ0s80t->v^=P|QEV;m?ir2im(yB+{%IFm$sW3@w!g0Dt{t7ywgC%UA}`1Y|Kz}()+iNKor#E*f2dkZDWZa^%8kICh)63US3X^0t^Pz1^y#Q9R>@6 z$vMH4U@#|`ywg>ioE!|MiMW%8!Bi2i!1JH4zysnI=m~JnQHJ(-jmEh|VxY7);R#cm|$;XCVZ0b$}5}?&?+t@q8tkf+?>KFE644 z7JSusB^hGkAEbhKwN+N)Gcy);b+}rMDMBh`7|isKA*wJ19YJ1R1bbc8s}Wa=xZ?j~ zhysGEvR0t2iCl=8iB_nIpqjJ2vU?yPuOV;X6+R4Rh!`uU$m{rrJkD2huR#7G53n#8 z%uZ0x-R`Ozrif7W)$Nb@eE)~}SL?A?k#n>Ya2GMtg&}^y`H%VXZvUG9gX<40#1&yv zn2m{+A52*Rm??&Ml)GBAmV)??XGQrdA>|MXK^Wps@&az;Tue>$Ty2!ZLv2jF-2wX= z{)`W-=Z8Mz03#yk-}bmNovpICtEq9YqphcrzlngeyQ#5WC`>c-XL%GtIe(Hz{z}S! z$m52PC&1pA-_b_d$k|Ng$`t|A1tIJU=;;a?A;XXLskr{hPwDsTyW;0*X3XzzBB&>9 zr;JdYoEKs}-a4Xx_=n+t@eh9i69Fw}Q(!$N0(yK7CUOYZHE|d48+urc=!+NMaZY@?{4QI>~0JAf~}xYh^fGpzBCc@ zUH&y62sVJORCjfEMBakd3LS85)Gt#4wp z1A(Bx;LcVs_|8@kz?K({`-Ls5B?truRwyZ`s3|CI#u&aah57#dZ_Kf1m0y^nVnHAz zz}(!%h-5^-8W2^LHiba`ya4md=-=R!(8OQhFl`_ZF97Er9H!>v?dIa-5)u;NVm77zpNOGvO0Y6CGbK1#SrEupdQK!O8?{Zv*lZz&cQfk0iD8U(_{AFA zq`@Fjz#73`RC!uZps$Bph$6|ugJ$L7;U{mSCa+I*3HFxPx6s^7Cei=83-jNX{F}5Y zXwomz3iN?MXh2$qgu#G>Hfs8TLAoxX!7kLz;~l-}O3dcuf0G1*mia{zqzeed3`pYY zNey%fQP$YZJj_2PF@N>`4T1xW{RIM{wUL&dv`vi<1?U*Y$G`CuM9clclidddVgbYk zwmG>{1Ks>KlbhwQSXv;cuY z^{lO}>mf`~8&6NqFJL&-D?B{B9*lrSBqb$9Geh%o^YYADpw;KkS1Yjs$9@b19c_xn z{;gq5Ef};H0^USHIG{2TP#OcMz1}z(%BBIzMgkuwK;L{AdIb8bzTcPrL*L)1f9dPY z4q`%p5{J}f(j^DMf1Z5u=lGvAFDJeI%J2Kaw-`-aJ2>fm#a}Sq)$dXI<=RjlE@L8D z4kQhd0$ux5qfuKbP&qRIt4fNhGO_k0XLQfL(`=cvvD^RAS{;MzUT)%}7&SQm0ropw zomY7V&37b&)l7=`C|FaNDKP7v-X65s$X$&Nt`GIMWeVgP111|S+F(WX{xrls_XAx9 zXe%{$p&7%FMkCzm>w8s<46g7L8@gYez!iY2?c{`e4p9FLH8Y68rlVUjjupf=$uD zUGwAFVJHCtDhHke816;^?)^EQe|EPNNa=f|Gh!72`twiD^#~9EBETgrqX)Ad2I41y zSkMK8ujWB7u>2ASKQ;gv@VznO{&(tM;yaH*_w#_71iwF|Fde}r5mL0I&UfV>RO&aX zxuJ;=qf5<~l=Jy5wn<6Wu0Vuf0$n9pBd&uxG0)1;PHb`5vfTJg`wOjzvcjgPegu>| zZ?FY9RsFMbcGlAuIXz0wFFo^p zynMila@x5MRUw?5#MC7B-RZ4G<9pRfUi_V3yH%W0y=<{1k^3{C0$aCzekq$ofjjfMuW`Lc zx<5R?x&j$#m+_VV`dQ`jn3{-3vJho{Sm=xJw(I8_1!+87T{ER0L_Ve`e3-2OWmfkL zRzvx>B=NH3i$l9bdGTW?wuc%q9-ATdqY3<>#JM-c`pMn=CjT5~lkS8}*0S zSvKzd^*v>7dOAaGCw3Z#R1KZUvp*ZvWXW+TCGQozqo*{FG%v>z$?>R~mXnX#^`Q@L z#HK7s6JBul3@$D`NZDt{j<+zAvagP9=_$Y zbdLMZx6hTYQ~FgqK9_j6K*Y}NKbhc^HD)of>^fB1a9A{|cnp;%csB3JXkNrV{Qf`d zdfxG^DVPyKo`x1DdkFe*I>HhsY|r}EwA(F&hlk+~ho2T>j+e0X3HW>^Mlcc)Dz{ak zai^c*EpdlNRRf&s*ISoQ?I;rXB>Z+?V1HdB2S2*Oxu1Ttr)x`{nUGc|Yio04Y*EF7 zg(bs)nZViS`)QsW)=!qDau_q}-Y)#kl%<-W8Ls_WBWX&|NPk zF|#ZEZ7-){>X-cPyDnX>5p#@h{y3bgcBNNB#Ys@F2Htf#y_vm!Ht)RNl3$e5c)0>? zkKb!(&mkX?+E)qET`Xy52+vw3uYJ?Z&l^SrfFt8iFYZ4Sazp8^LE>#F*&?Tai#IS< z!1Rm)yP3Yt>D9_$HTh3zbTRqU#rH&EIA8kedw6`}4s5YGL9I5E{YJql;;`U$zPfJR z1A11_%5&C@XI$rZtjXQ5L3Hx;pYf;>#y_JUoS1xQTx8C!$QI-?0+Lj{27LnkkoCvR zr;w|Bm?VmHq2@k@sS_US48CCjbqg7sNGuOz8RJMPOQ3Lt+a-LHlWw=>I{Ctr5e#8N zEdpRaj}TIIL>~XmOC_Om6$CrhUcAt%BY01`Lnd#r!g&?8R;y)CDZ3U7#Gr5Vjz&j1 zijH5SCh?Ek<-hSJ?XJmOIhwlsEpNQAN)n~8c45Kegx1~|rtxbNv7O*BnAD96O2l+^ zo(qm)de4$w-o5cKh0FP;N78Pn>=3h-6L_yCOx?RXbSMDc@mzR`3IEg}xnPUOm9F-x z>r^5JBqKe0(gPLi;bVGGSyy_yBR_Hv&Q*Rd_*H+G4AxuRIiaIRWzyA!vXFHzCT7*j zGpa$+pCi`a*uk_1U2QDOQPW&MbrBTQ=ehf2&)mL~c^1Lx; zgO(-unf@&&@P7Aew41jYk0b=k4%d!hnREz2mO-+xfio(JNf(12=g$VbU4RbO%-UXt zUoBtJY&}KDtZFE{X?tX$q_6k=dtd9W8h76$szaY`joz-u{LX31?R<_$>BCp2W49V{)s5^vE^It!aamUx)j;;n)M=`|;WMf=Z&4MmduHK4 z?h-Hh$gV8X@n=iVR+n3oE(a6sdp^7bS*bW|P3QPHS@z|8Jg%&Nd1-NJ@%8J)6Dmpc zMpD?dlFL!TxM*!{hJRyXB8kGrPEF(EOSd3f5fA45Lj&Bt)$@`@>&Bk?L_1`n-~oAG z1O?9&;3&l};nxi4#V70h`yX_-$1qnN>gK<8ftUPif-fif! zjWyVyO!{Nm!={Xri8_LhLe%n89}K*Ne)g^sk$HY}_^>M_q$4X@V(RV--8;;3_g#lq z#B+8#J_6)D8er}cx;)R3R>S3V9L(Zn&3(pUFg1xof;(ch_x7{Khb$$ja`~s(Ia`s} zMI04QCorcp(@{4sAc7m|dF{sVKtw=Z+}X&G4`i|jUt{sY@^V z@Pmk0nb$$@&I%Q~b87R814qRG`EW~bhl^l#l)nEI2ZvW^Uf`e-r+CFFsa{JFj#pi= zsVNlN5`{ygxK9dU5TOZV)kW}Bhj>Zqwjr2Si*YC1zznV+xOEHal)|}@=vQ6K&PuljTlY0_ZF9MB|K>A3O)H94 z=c>HMc{T_3txqx|3ac(%)7Q&P_HpEJa>=e_Zl89L;=;X;p}VsY67+ey<+hxN^UOQ7diDX_86zcA)DsIkD@wyZIcW6n#u2r zm}*CT4nN6vj;u5c|8StL=}zZl+(LsjPCdl>&7kbASQ?=REwRURM5iAMF-aEL$~6s~ zA_=$!_dHvPcI3h3$xR26OY_-y!asXnaa{(P43UKW4_yhYewxW$msLdtU)8P zY4?SUE6DXa!|b7fC-qsN&8CEV(Q1M?oh}y6(*_ATtcP};xp(vCjh#1YpTC(H-Rbmv zHaIvWDEI(X;+tosJmK-gSnm}&@@(_&iRTp8^F^Smlr`IfwTs=YUjpt6idmVgy@BcT z@aj9(+CCgurr|$_+H)&!ZQ$D>k=3t&gUCw@+l7tP%ek4f z7EUb=nrtuFx zo>-SMxTWi9_gj8pwGpu8%lA%l2u{s${Au1ZvL9|01v9%1MgD1~+R66X^lKnMtSK0{ z^&}85bhlp0PP&d&|3_ zLA2}R$Yraq`AvzvW9Ai6c-DThbN6}0n-zS_kXk9Z-HwjXo3$G+!&&+OdO>8<>d^e1UjEN%&5^P!B#oDTPI z`yNfEq9W|=N}_8yk|eX9;cTsB0q@Szqvd<^S!}Sqwl#m)$DNZ7n}^w&Xy-wtL`=*e zz9K#C%z>Ac* z?DmtLho3Z0y{&yh(;S^c$v#!64dl_15Ggj2a&w+4%=Jv-$2=UIzjo=R;LeD=6DY#F zoCmihs)&I)*^#Zl;R01Wy=Q8Cp)2`dD>XzVE2~ez9JlWwb{T}2>^rH{$*}^_od5LZ z;G&1Yf#V(Gkm%c8dy6rAkKQYE)0R>y;lOprX+xu(jBg{c(^M{Lx3slUa=M$~S2+}d zo^xhqcC}5jQfxX^s=l(_SyEh?TzOk*?KQr-bSgu!-&Ijud^U%ASCEVrc@h>|Bl@i3 z?c}u^H76GzG*8O!w>~&*e7D)^Nq}WEOGByj^M~3`d@mQdJeMvK_44zGJv$@Qb~}Ro z@tBPU=VKxf1SR0xE*YP;0zZ#lKWTL&&lL@RZ)Q6})YO57KvYE2-p0nnEgoc?~UF_N=iRBkDSl8K37id2-ToI^w)@zt~>Xj z@TPVKJt5kE%Z`aJL3|Lpwf+kWHJPtW$YHYi4ax;v5Kb!lRu|Kcd_@rXsj z{@k^jw}AoG-X-^?HD%xV|{i)I7_xt8exl@JK2qG`nR>&n%Mv zScmh{=f1XSp+BSc7iKV8!{nPVv%=fYx?62hpV~LpE2+FLyCj~xRbaiHH$QXtxoVz{ z=lgdN$fvXKB6@p{15I>mT|7>_=GD97j{~+do?vGO!mmAQDAUTXwW%5Pwd`~u>z;1*$zAj|{g0O-NjJpB7BtV6 zmZBdDY#}-aP$;YI#P;^~24xr5o9$DpQMI^tWu(!ZgtB|yb&X|xjEo=Id1KC7?aQat z)#$_!YM^~+;v*65lrD4rU}owi_hXB$4J(WUf0NoZBW)hh(jWcpW7G`=jB)F`tKn@o z9{U$;(++kQzd7i8X@dGlDCY{ey!g=q<8F)4DEQhzMwwk&L$igv3QfxDPs2zD=#fZ; zyAJL6Ag#rCqR-h?WPVO5zeMRF`D;^pI;u0PPbI6Hhntl(UlDU$Fmj~7)YTkF9hYyB z^8mK5wU=h~XOQ`n>w_-N(GX)jYf0 zsU96LJ))!cb7trc)z<=+vwk1vMqeKM@<#wexz5!;IM&!d9PGX;a(EZ?-XHC_(67t$#kVQ)5`Mo=&X79hqGRV40A5H)AUW%8M9k#Z-{Kp^5 zudWA%#*9TV!(b%WcQ(7Z@mR4nXlgHO(?nu>z7n@~o=ET3m9B_p3~RwX{=3P| z6v1q8>#0ejFBQ|L7E3c`DpKEFG&&O0k1RX#>0m=}>Gtwu3oW0A4&yo9E-BkjT=BE} zz|z}tLY?))?f{Ox+`}Xy2mj5-BQ_PtdLL0f;|7Fk^(C#S$GyfOmxr?0QYK5>4qGNZ zpExkU5qqe!^3#*JgNH0eD`eyic3g`+5kEFFXy<5qdQWA>YQ}`X9QXvepxVDT_|5Jn z|6GGA?Gef3hMfmJ-j2z|g?k@6g5Qdw91tX;!p|M38G79{H>~=UxluuGy5Sl(Po1&P z1gj*k#xd{L2Ng@2#j0ed_Q3pLtx8|RJRE9)Ggv;^seJ-3@cwd#K`N zK*#G|Sfu}z`xh3&AJezJVaN0ppMP-RaG}edo3>B_F>;@K(g>k0LhnH=cC+x}_#h zW-bJmWrZI)_4IY&>&T?u!vP^xrzm!{j-Cq3^f8a-k(QPNoD&A({AaW)CC=4TE2ya- zR{Ht92I*YTlEuM02@hpF&PMe(NECW$=bm2<9_YMO5Oqi6jF+66t&m>$Zd7yrv62Iv zyoJ+NUqjOuyeG;UTH&syk`g*64B8$f&f1e$O`1L!kg9^*UtZdmg}1&|uzh@_HSW2j zR9a!8kOk`0^XYO4NxV&BXQC0T`Gn=`#jIZx|XkWV?@=8w$_{vr5|!lHyeTkH<}?m9Lx=7H?;bVLVy`zzOik%1ZSY?1WBFPFD zkOTIa;VErc`#kL}ycp3J8YJj< zNdZQ78mP{C&f3j#&XJ>W%DYA=?U+mG@vKp|=LBwv?cz_wyKN}JujjOtF;fCtSM1f3 zBK4kkxl^1uSzboluU%L6`a~&xB)I=nx%#C0@YNUl&n!3G)>yvX7Mt~CZSknO^WCt0 z-X;pl`NI6)4rfi-sxGEV`Q&_(yJB>?_^9o-mudZ=+sV?2j=KD$>b~uIqx){uzsJGD z^0k&fc-a|GkyGG(%I^ElnpQFyw3wZT-^pvy9=1NlYl3Ls{yXYro++AcZZ~8U{Rx}gUizrTw-rscv z{zCJn%uX|99>}=s7Z~`S>^6~1{zCk0Z}t@pwX9lAiF(mn1c@Q`J?U!K*!bsiQ2~#A z%*6wyCsyD2h}jkEb3G-0+JeuVJRYH-Q}k&!Qq#fC=8f}!V4(S22uXfwKEDaE-PrlQ zU42~aJx7c3BERDYqa(&ma~r&P<~3a{{ffNsu(76G!F%^QE4f`=aeh?{b^WA`(WXb;YM=9f2hRde?3aIN(L<{2M5ak+Z<>8fv|^YGLY7s?&HV4ZG(e4#5vSm|D0UB%%0yUE%G zgH76Ry}NJ9c1bwQ4~x=|N6P{)Nh@X0S7m;A^R+!(owHc-_Wt;t7uyjK@Jbbq2Y!K12QewdfRQv141)#3E$9%;5D1|If^enNVNyde ztQ2PmKRt#?r6iW!Jq9Y=8v|7v0&_%6L$SeHx}U)W#7p1Qnr8BP4q1aA1KjC>BZ>ih*mS$6{z$CL#vHl~Ny% z!ni;&bP&QlCYHC;87er$5C_Umn`V|8BCw{X!?wxAvY=w{JcZLoCGoKc%jsC&VmwsP zG#0+6gowUi5ijd3%4RfstqZR~+Dy25ix zh#ezs96?nbTb9xhJ6{txqhG{}d;=xnnO#vwrH+bn#3qrX+SyZ`N#erqm<`q1xiZo_ zC8R7_OdKFQc|)Bz&3i0+q9MGcs4gi(%M^0%Q{F1W1ZmUK6zlr4=(-3dIakqCyP>mu zS6UL}X&G$$U&ry^Sn5)Y;X=9~<1t-I-OA}1NDsMqfk9fgY6%y+k7>N%XnnVOeFnQf zHD2h^QulT`7sr9Lc;RPCJz7HoSd#E?VtO2A4$EwmMC5BvPt=?fMTz1`5>HXb49h|ATthHxIw(c1M2xMf z7lLt)VKc=eP`QBLoWUFxF;Jx;5K^fG%1wsuDEC9M-0h&A0mOc=Z0}v$)4j>3Z$2EhQjs@Zjgs`9;J*nfugjH^11R|G( zGwjHgBm$g9g9&tEC4>#zP$%f%44qOqRsf99r-RwOv2cPW1d&LCv6DbhY0Wqe{UDe` zdbxxNHqI#DnOia!29ca$W;YMY%+iS!$0S3HFQjJ*h7h*Nx)_-_yywpCEN4>@h!^k_ z%aYBF#ko}_3sQr43g<|Y+A~IbqVH#Et-+)ST>TbFWZp7iCg}*n6mrf}Xr)@0^qyGb zz2z$C>b+euwr$2X#L`o>vIhX4zAEkTFR0wi240f^ToVHy98x zEc9dzt_fj2p(`u~I;QC^#)b#!aijy8i3PJs(P2l2V_0D`%!FP#iw0oHPG?On4CDY= z=_oJKi&=N2*Dz%{LlT343ThIODyUfTUM$lN&2|Ga9wsm_t=UV6LufudnnS}wrRoh4 zC+M+r(MOqgU`}u((zu?HuuLQh2UqT}u)vFHW`U&`xEPH~00q`k!EkWx3gSXX&xw&% zVuFj8V>$69EIL@0kh5+~+2y1iQjzUEmAXX>l;=CN>E>Zg;#oQMB(_O`4pYzeBDL}v z)^Q_UL8@5nVijTAp{3;To6B2@V-Qkd%y%sovBf%RUQ%;?x{*1q#mgmA{OaZ0u_}au-c>I z1?Nf-1SukSJ}N=Ry_C&GN;m5s%2T6LfFrb5R|I02L{byuKGjQ-BwD8IJ$e@1V5lx1 zbT8GJS6pB)N&kp?5eC{aW6X+pa9lK57@jKq|79?f>H zY@HGoT_Q0(X{ii1RKj9TgK&CJ!~z4efc6!&sLVHP9p@ivY+ zXq<$!50i;2IqNzWo;To*bbWCmD}SU&a_4z%3wzz{F%Y-3nY8xaYO-(?H$rwli-{4K zl+)C-O(C?8)A9cO({x^(^6_L-50&A(fuohm$HmNwdsFhqI?)=J`^*B#Ood}-b2a;X z%tNZPiWZNa(|P$4AN6q9d4-g_Yt={j2xM;wS)z)-Rc(>DgZC_VMU|eijQ;VUY~QU+ z=M9Z~$(m=gb!E1JgtgouiY1Jk0PYdsqKN=A0jCKY)(64aatE5WHy$nJgD`kV2$s7X zZ}J9jbh?q3A72kOeS@`9$7fdLUxJ$W_*!YV>}A&ol{Qg7W+m^!V$w8}sTi`rTAbf^ z&hsgga+-_wnMNZ)d{-PwO&TWJ(kDpVR&AM4y|4FF z#W?E=x!4-Jd#t9H8%Jwtr)2lPNwK}La4PSyINIqeD}~OQQ=h1J;R~;|Qf z9?Ge9gF!jP#Z0x{Mt=4qSM@Jn2$iRLnCzQ;YSVoEqdYC)w%$`g{#KfmLU7-_!wcin zSI4^QKux#zYqH`iXyvitmLSJ>?QP6+qp>_KF;Ic=m~3kO(MX@8jvw2TZxBNjqfae3 zuHJueb7fSqaK&%0%qlWhIoYIC&l!@JfbYZc=kUzy_C-k*(AeVJJgg@0M zhW5E!45CD0`IM8*(p=G(C|$U)29mR!+X2BzOwW$_hLh3|cTt{+USmRI=BWX9NnN}C zy;oFDolox30MTy|gg3@_XVw z^W=y3fwyIqT5o(TQu9YD_0^B<9rU&Bupg_dR0keF#L#)YV|>mrOqgj1&$Ru6tMivH z#>!J&g@h)EHX4@_UQ@l<>>iF>xuRzp=kGUcG&%L@3UHwY9LSoSdSSEuTGug|@Q!`f zxnid*=ga~R>heALGT(d)tU!ws5}sl8YUvPFIFQ2A{S@<{rCV$9z=?Q=xka(oKFhkG zoNL1KS2J1%d;)?CCmo*aeAXIDFb^pc-~ZB-zYbHx5qjOIXu+oE>R9VyXnpFVYwp{% zAC4~WUsM4>kaTe-i%t+*{T97|W7j5U6vCT>Qx?N|u62LB6MhR?JR2u@eTM5|M7Q?V zm~;a^Ux_`Dx7{3<3qR{n-De|5Vjd5jmAvuN@?zB3wPLsW(>vezpFH?z`SE8u|E_n( zPewmeeX=@qde_ID3zo0koc@^lylbUv&*9}V5%@w67#o6xYGK+Dh4gqFEE$2SfAV$d z^zN`1fk)O}o%!<{f(~M(#Xu#T!R!%u7!D=O>VSm`y`Upx*yFger(u!0ZLIc7ahQP; z`1am5j;N)0EV7i@z_6V=f9R;}cpJ-hYpBGV7$YfDm+-y0?dZn(1o@B|HW!Spz@79& z^`;s21G&0e7SoUI{4m2Aiy?_}WF+aSJ>|;CC2dp4NQU^G z^R!IeH|NTKV?|FX(K5|V!cAZ>LSL1S$w!s80W&vbws2xOqDT`$TzcEQ9(4Q#X2u5ZX_4 zSa663ZDWGNEzqA9v6K#VtV?jDIO*{;jb2caW;L zkj+7U#LouG0+{av1b2Z6cL&Bm*|cK$Kh`n-j+#3M+Z^@R2>luRDgrmPUUt4|neu-u zUjF}qy^#YOm@N>#DG2T<3vOXO2L5~H@&8tw{JS(K-Xk`*#4q``A=DC>FB^iZMB=*H z7*5!{@c2L1BmWLQAIG#g_}@9cfqWLs*A2#%aNr)YGm!r(LH?x}`FH+HAO1rG|KqgS z067Zgs{-SuIB_MM43M8|l7DBtuX%$`GNP( zgYjq77GB&Jp8qSTKOMXqfN?OsPAKj%A8r~A00PCD-xc~%Dauguck2JHCgWh=K)R0L zd(VWsDTpfPMf#2Uza#y)IDR(dhzKr61VH#?ql9YZPu0>SS45+ef1 zLRC?b!QRqneZ{&G>-;yCkQiJI%)?wM#7KLJc@#y}*xTKJ8V1N~tr)hBPas$`))Ms; zJ-^H2y}oxHtFE!eKzbt1f0ld|MbVY>s$@AeC=!_JY559LMSTUR6AR86u~U&S9wTL zvcJfqyDsHV^7t|233azs_H;3@@wPWy4@JOr;SB!*c3OvK$grWGq3;iV`kU*w&M&~; zRyl;CN>+C>V3>{=$k2yovHg2|SpQ3Wgs4zd%)RXZeJCnqB@YUbVIMoje&v7cUw7h9 z@wKj+fpV~&jh3IgAvGNEE09@x*X8|gUm}srP~|s!t}AUvG4^#apaL;&uH;TJ25^C9 zvA#e2AN&6qk3c>H`C+T&XK%aiZ#TwzTOg-4^5H+|%aG3?gzDz6PIU!h!By2J(oSXF zUS^E_KL6McBpcAyO}E}#4=n?&aEgk$x2vr+!JV;w`;U0k`|h80|NZ1|hUi2;dx|}6 z9m|*zFN$`I55~lI`}|YCjePO9i3IW`%+-suo^JqCKv*xvc?Qg{>ut$6yf)f*d?1K` z_7nbkeE7Th>xbA=+{4`bgTK%J*t&Bk2!o}$1RV(V2$6TA(d1o%f&#;_-+#Wus;Da{ aDJTIyDgi%w{vZ4Wjd6fL!2d&3{{H|gE8+0~ diff --git a/python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/collection_meta.yaml index df90df3d9c..e1618e5f12 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/collection_meta.yaml @@ -29,16 +29,6 @@ tables: collected_at: '2026-09-17' rows: 1544 status: complete - dsv4_hca_attn_module_perf: - rows: 3918 - status: complete - collections: - - collector_ref: cbaf51b64fa460e5ec6146bde407a4c64958212d - collector_hash: sha256:5882781b43c67c93f89e3e238978f461e242c83f681effd9ee6129a4be1637eb - case_plan_hash: sha256:0e844f577823301a0af954d0d20baab4642bb1c0368652998bc28160dfab816d - collected_at: '2026-09-17' - rows: 3918 - status: complete dsv4_hca_context_module_perf: rows: 5840 status: complete diff --git a/python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index f04505c645d3d67c3ed8c7d34cf80ae43666533f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 20485 zcmc({1z1#F`!75}44nf*cSv`G4BgTaD&5^60z*nlN+=>Nh#*LZI3OS`-5{x;fFP*Y zXAeF;k3Rao=l`ASJJ(qk&R%=P{kzv)>)y`nrH?cehGW4=;jB7v2pmKX0&zDAeqn|o zK_Dy;2xO|GqYDSYso>mv@O*h-7!DkVUIN68h3P+ZM4${g_dk08{Es6Me{TMVncsZ`0q6OJ6G9sh2nKLs>*(%f<1Z}nCndsubRzz) z{Wl0=IP)(MxYFMx;^-~)C+bk39fA4yZ!~G)e812nash$x0Jec?R<`b5c5Z*7jQ4jh z;-Bh&Lpk;FpR5ey3pl0U=;Nx)3nMzXnl(xc|`l5A+|6fk;8jfFw|$@As#x#Mg&_>^6n_5Oy#?Cf6YFY{*Y;g@Bz# zbYLU`2L4GZzqkG)m8)II)-Y6HnKwZOLsd$pSVrs8-26Y(GI2V1hxc$Q`dg3WUgu+y!(3Q1> z4g4~)Fw2ZHTXL3J2=L)qtnT+$>Uu-JjPj2^{1QOa2kzTffPfSWn{gd*0Cz<}FyV&z zA6cM)aAOPs|3MTufN%%_@gM*JofZ;|Sdsy;ga99;fMoh`;ZfMXO6ISj|B}pat$$0# zdIR(j4-}WJCWPX0A@-B>!xPOY@PWvJREz`^LA*PJvxipS5~rR^2$G zt!?U=2ay9wfh0ipR{q7eRO!X6Mz3OSaA-^@wga%%efSu9IxSRo#`h< zk3)6nGPziGSb%CC3=)z-b$uf;*jT*&ktdftE5C>RW|m53vsD+hW|j{509(oh^U}db z&ydk3Jv}PrH(ZKC%JXE0rartsdm^&QdW-fghE(Hg{p_0i4zXusMLlRf^3yv=p}vvM zO(t8O&u^2(A`$bpiNt>z9`6;!UEsXlNA9rm(ZV#h2K4RQ)vI=MO%n|d8Jfe5t;gLS zL2W19S`1;P;tQc(HsYOYRtKuZ#=`@v@6UGgcq`%Jfn1EFgrw*T-1|N*$o0QVY2a6B zuDjETq6GB$r+njwQT`S<{?W=mHH<(1nD9^Q-}+(xJUGLjBl2fxn0A^*Aoc;uHYFlin!Z-=o0P8)R@KK^)jO*_S zn5ceQFwFnn`Zx2|#jrhE5Igf|F$>#Z#>4y-gcI>X{#D{e`C!zK4#P$Z?FzG0I1d+V&snWSn2 zbDAx&mTTQ}#$?}#%XM%lmfu+$Q)CL9Ssxem)Cwn{A$8(&!GXINtyUPEf~KVO)p_ku$$IU3CfBMwxhaY)qS z&up^;WH@?F;TyKL{4@8HDuor9w@TlhZ#0YW#kY8k;Vezd)Wk)2+B(Lz{|>?yMO*1w zXIMJHV@&IJpn9ecODxOh4u~z;mpvB29HGknf)wChPsvrhi@O&jYIuf{Ia!IMjav(# z`<#-}GTU}f8Uw+c3DjCybXz2+Y;_5NwQ#XC-mvb$z{Y1nyld=)I2uAY1D$lszQD`$N;_W4E2 z`}Gzzeo9fFdwrR^d!)hO56A4u}jNP8z z`;c+tPEJ!ZRTSZd&J#B)b}Pm}^+6nLu}sQc%)#I>NH7;^k;AJ zCoG_denR#EP0N;N@!)i>I8cG}#akZIGwxj@>@!Y_juP3drnZQJuF(aOz*LsCPvhO^ z-n?l+aMm-?rk>>npZl6lQ^z~pqQ<=j8yyvZp-a!^S+7=jT*y zLPue&LW-{ESi?;eN%1-B8uY=PCh}jqdHzEPLjC2Se+WT8y8cNneoZNU^#AR}e+WUS z127yFq60@U{vCv@L9&g9pu3V=>hh$dB5p)ZOmlIl41^OlBB+wd;<*i^BB+ zO~WkCul{LWOp;P<=g2A(YrCLdvxo}rd$hcmJcui2!y;4gc%uv(1F@T+UO2oif=le> zmpSbeldiIJi;Vixx+5v!^xKmSRP?v)xL=MU)eAO-Z?;X+z}d^GImU14;9*wLK1pH) zQPO@STfJVpJsh&{9~H**S)U^Cj?q(WvGWo6OL$wOSm7~y!R9Loy&qVv(j1KP;C+TI zet6R*|qQr@F#_nL&>uNW`!uw z3vXi5G@_}Y$6eQbZ27Vtzmt^Wmr_ta@Cvz#!(yY8`({t9Hl#;s%PV}5&G!UmFlM$e>=GUYl|WwTbMm-c*Do> z5hFZaZojjP19nqUidAQ|iUf$tY5drN8icrAOi9Y%K^}ouc5eRVxx(~^b^O-iVl`)| z?T(&Mm1`f`IF$y;=H8Rr9g3U5dj3U7Ntd0wb`#x8SPL}_Ui(yA%Z>Z^sFCO<6_Z3U z79p0i;w=e!ERu1@mJGFG0dBYfTya>R`^6HaLpn+!jL z_#oH07#|QdUC{0)sFU&@zCa3^we`_}YPtF#0b{*ho9K&-(@kE{L&+!g)Xz2eQEIHq zORrsu^vwlBNg*5O?R6RwWux^P<25vUhCVjYn()}`91j8Jk(M3wRlNA%^QJ!9e061sF- z#80>cMSGF4aZ|r{7rA3JCV)uGjSqW?H%=}>ry`e_05YR@Fj0u@VZXO)jxLe2r*qe= zm(QCh&OF&Wh$L;%G}!E#t7=$!6&H1uJzPvq>s4TMi}SiZ!G$7c3hc2GKw0c$(=wFF zmXLUHFJf`^$j#u7P#(pxRGa|Kubc0%yftGMZ{O&chnQ-A$}I3pYR_&)vzy$%sWbd+?^tt;QYpsAeevl73KY9n(S9ATp`2d%vq_o!liX3^aZ8OK4kXY zK6fWU_LlMrR#lM?oOxUKMH~_*VrhNFo_2?gppI)>r&p9hDl?DzozfJkRa0FZ%MmmG z>m|jbjl(Bwg-j`Gc3>`hu}+3b-;U&DcXxL}LY-=Y$E~2?*%?G5v;NB>FYL>*4c+Bs zezm-mPaZsYvvA9%nX+XXzD`D?;FW7dgvw?kj;$~up_r=b-GSbQHL71J-}{(c&fOE{f8&-^jtG}Is4!a<;#iHI^6JT?_CoE zixw{w^P?3{Qfh5J@7hz>&*OWs(d(>GQ|d> z@v!8uAB0_yOmb0@HpQin5U-xCr(}s&Ti0mwk&Cp&eD3)izhc9Zq}T*=o5@iF#Smc* zGfM=s;YB`GjA=O1>tJ7fkv|uEHS(c!ZVGRL9KXBU*rmdK0;UVk$SKiJOVLy+2IFg& z&YOt04TD+2eebg5h#VQN;ofO?J`qvVOLj-Ndn1i~2C?orr1^;o;x#XH*<&ZLtc_${ zYXM?FI0r}U5Dh|Rya*&Mt*9tb`-NhXIRi^g14kksFHIL0py1t71Uj4J^I7lEGCA7Q z|@6$?|J49QB4 zYP?o>KG_IK%o1SovO#H>l#V>nAk0U_QR2BA*TwRN&zGpERt#W&>Q~yue)*LU(Dj1t z&{2=q%LQOJI1ri}%HS`eDTyF>dQQ)+|JyrFOA4^dTUls0?q<-6>J`V2u)4PM;}0(h z0)bW&M!ElG0kZ?_FO%QkUzhdsiVvkDDxkiyTLy5=_2i3bRpY19>I8J4$B6~*@?Bzq z2LV>)|~rHG9+1&F?H{CG?Od3^9fz*6FW;noQSSx2a<%u(Os+TkJ?4I`!nU79v!W7 zS7o464;~sYZr{BikTbOPAo`iP9%rV>5{>_6LP6$0@Uz$NKYh9U=$Syy;@(Wa!|L|V zWzWVr_nGdxS(zucsq}7N?AjY^YW6?KR2fwo^Dr>nO4OFVz{AktFBcX^K={>I0QW#n zBv0pEXhFh@t+jc}33qi$sy2+rwy*F{yctaqo%Cv6+zV`9cHP@0Um69<;M==LQ#{wT z%F2bA`}Q*fc^b8V4W77a-?-A4_PSD$J}u8?!_~=#`bu@;3<|jdua)5XkhiT>qAzZc zQ=0h}KAFlJZfR+5ezUcGLBMmz2e!VR9T4PaDUfib)qmY@C>@9-w_7re7}o}H6lTKn zO(yLOZdIf{xhhu3P8H9Dkr$tuPo3G7wZ*m>*G7vnx4f9^@@2;NAIdBPw|D&Y&m8C5 zdZfm!7wPj;;>Rl78f|H5VTv#!fRy21`u6VKJJaW0UVLB6*#@$9$7XW0$_y*zWa#K7 z())SfBAl0d(=|%pY1rMO!>jgq6bHm@?24-Ul^kIndv4B7N#Su@PLAf75*gmtd&OwB zZ^JOgG3jLZ_(RszdqYF#{n8-G^u4`-cVRbo0^b3=Qbn|wEqO53>C=Gr#HSlC&VFsV zC~qv>sc-|3nt31C4FMnjot?LYJJ};)r#zxXE=nOx3*wNY1!OC3qdGLw$-oV|R zy8h9d`d6MZ0dlawE_m&eE*^cRYwW}J@?hpb(?$FItszFJ=b}lohww#gHv)c3>ZD%V zo65~IuD1~LLj9{dfpcu zTL^+JX6hCXuI2qGaDry;yvJfws^_9`@^O0XYWuBC3Cc4UM$w911cM)1yRyX=$>Q~h zgQS72YX+h#~aL+l7tUR{;ai^Zs9yU@ajZWbKa}%&ziqi@_B4=qHirUF z3T_yy8-8@@{w(ixt?OzfI?+B?{NvZ$h`EIxrKkQ<1Yh5CBsT%Jx@0-OpOBo1 znwpv`Xmy~eN#e9@&=!e_y%-!1n4_ZP}eQui9!} z$?O<#Q@-2)KJa5Do-JWY_EW(ewUYEPS{fR21Z5V5&Fv2W0nDG#y}fab&9^t*aApH8 z&Wgv)E_lFcW0h{xYuPm)=NpmA9CsIq>8j`Wh(PWHH=f4?QNv1|yOm{SMMU08Hi9Zd zMQhg9R#q1;ytddCeHMhFTXXp7-67o*3;LntH9~-x0=&z5nFQz*Q{Aue=Hb}@c$M|hY4duvdnVc8MHe-$>d&HGFVE?8v z->0Fz9#rt|?bgnQ656esNV!N*zw|<<^~a)T29xValx^ zc(hbun$cR$dM?N$mnWB5%FVy@-adnXR6!zO7fjtFbeklNn%k>T!MVnm3~7@tU_3 zJ@Hj=FhUbqTxhBl-5tc^?}(e)EtsTilq9LO&@ zl5Q3yh%K|Ude8#ws+`yH3v@VN2MU=ZeNXHsMl*HqH=Xz;1Uj0Ungaix@EvY0On1VV zA8d@Jyl!u6Yip^kt(ZW!b+&!|>|$YIXl$&LCH1aHh2or!++|kV;z%Y&T3X`PB9?ig zOqT+cn(AsIVu25S~0RarsCaD7=^xeS48Ht%wYAH{|B|PC?b4*s9i@ER%{tXxX9MQ zgVF$r)+0`Ji43I+H$*qKX1;jI3Kk6833!|jTs3pZU5`6EH0koR(h-LaWiAv*e^-KI z$Oo56Id|wDZ8PGtJN+w%n=X>Qhg|@r0!yd`v zB9uqj`3L&22wbUOuU>2(-H%C6lNXD%lp(~+6E|WScqnb6)ne^solWCc_xVMZ2mR9{ zK@c_M{pyDfQ&V#4u$LFeNq4q6n%5m& z&o8_;o}D;uIu|r)Qq*WSvMWM+EgIDHVJ~%tw0wePJ@o1=wCDax3)=JLz>GeH@K(yf zl%>wObu*?h9_J{j@mmGeZ^`?MTjx7?38Y>OPfO!Js}OqzzpoRtM!6e!78lV+E{3zR zl1n1A;Uy#TV3vLpm`?vWS{nWHYO2U?F zspWl|cd@KRhvMS#yp!Bw^Q;3Un>9$7q#u#>#t-U{T6PkxA*%7!iY)C2LcM9!C+7O^ z&!x<++T1_4KzR}yLlqnLoN-=+LG>(&K-zBQ9-U96a~VTj)fen-`UbJjx~syW##_^- zYE&|5HN?>+yD0u_J3lJ!^Cm4}o%c_kC-L?WjmuB&dWTGrn<|IEM$hjhQW+ta$63N} z1wx-r#A69Y?^bNfdA`{hmL!R$!(zL$=lBUH{igpznLcVtw~3XQWsd?68Mdx7uXTkD zM$V*0*YothY`Cz9rn;VsI;50&&6yUq!RXgSuXV1sk$;M5{v&Bdk7~3Z@9V?v-2Vk*@WebyxVt`E)$MJh0q~b$84+z+Vx9)(M5GEyF|odFY}C;X(e=K=1NxE`me#;5Em!u zF2|whK8w{w*YpuJT;6W6b1wd2yY^#0HzDeuUy5Cw?=|s5La(&#b08;&kRN0~W(_X;S2GNkJh4@t6vlJppE1|)? z>im|O?zmE5Q|MwHRfrTjy4||Eek;pcVgw%O$XsCRUiC6rm?X%$eiXcH=2>(}>C7v! z0LSY{m0+)@vdE#M3jqoi2S#?ON0(=kmv)RN&KfMH79VxQ@6DM86slM=5EGZz)vHju zwZ6TD8p?1ycx@B9CuH7Y5UPl`74#-$u)I^Rg_$vHAg*QqasOqfj82WKF~)l0Wt|nN zqce~hH3^lB49kVLEaJ&Dj<$3DCFyjy2&V($U( zPVMw1Ef}05o8iLnNg|2Glp95w8}F?vk~bNV zf2yxd*u6qlPs7TUxO? ztD`rMn2 zXNj06S!^aQEdE;HuEhLS^HFCCM?HF;dYnsOjx8LMT!;8$~ z>AiTIorbDas6G}k;$5S5_R^Pd+rAf*r$tS8p@G;|$#WS7-$)3t@21#|7ZOisOoLmF* zoC6_OoX+;#I3{Dgqo}WBO=wbWTwu_{Ok`vhd?{V(gJws5z!g%si?p~73CY4^?b_>T z!~;A!3nk8a_PIbc$azJD4V)Y3j1>IjX{-ZUSry%@{FwkKc?h>rLo%c znubd>w0x^GsxIi1>yD81Z5PUSdD~;p>60@>76v@uoMjKlq@u5X%qG(iYuioM>=9^K z*!9JTE~iYwQISUR3fd#ZK8e~%dn?>vf0!x#vb5o7!;GA>HA9nL=F=V5w{Wtun{$2A z#_1#>Rnj|OP?O90Z?6?bRNrGJMA6u$AkMWv9&PE(dTCUfHA0?`V?mO9g+W5rdH1yv zg#}(iV052ZEnIAQ<(x^g_j|3xq>(&3(+?wo8n8F9n|van)olq+eaa##^Pjy=uJ!uF zW|)J(i4Gz{beh?RTN%nx2-dJRwvb)8K)Bc6TSwz*{2}#b-h-Xx*dFrQ8tDCR7Ztnt zrn?G!Kbr6vDZR{G@UoLVw?pL3?vzH=f$_*CoeS~P<1!e7Wld!5%ri* zi;St<^yiYTH?qmMCCz$Lc(ym&F~qMu{>6M_T*rjz75|wB&MDYd%S4?u1C)0*YoFES z#2Yx~R?2>ThjL;yw}hW>^Bi#wOc1$=Co6p8jQY$C3Y+_zl*`U_(}&8F;FcDLkDZsE zkG?+pDKT2Cepxg@$|WFhXsUrBe?mewr)aayp@4Fi!&-nrrFr~A7EFwl(=SG`kLd>K1lTDY6w?9UQg;{wuKLTbO&$n% zmViG~C?a4*PW3PS8e`95zNh}Z_3!Vgtr;l~D}MZLmmmW8H4F$#2n8lB1TKnTm^TPY zSqOq@fWVZsARI9?m;{A_P?Cc2xUC`hg&-)8btJxiA%wC81Qqg*B$QeO)6StFQe%+> z+-N9Mts=J2Y9zkUDufCc!0int1ZK04cEMy6qsX(YAr#giY@O9eib6D$vI325SQrI2 zMnMqPNT}KvgaBZJ&>93)SdGHJf`Txj(bxb565~~fAPPxnUmHoo&5TV|h{ENzj-t2# z{6wq;ge|vcHhmdqT@#>|4JWd@ZsMd65)vJknNLFpD&<0R26#J;&uhL_e? zWmj290{f#G-=VK6oUD-qfmE2WgkegQ-K19)RahzOU}|FBWT_}Dc2<@IP5ol>?D!rI zp>#Z5_b7^62R&ShWcUWjQIusuvD|u*_{OD5@G5<5UW>KF^JrG82f5h%t~N=QtE|** zbJ&8H*OIJ2Y=~}H99P^Lfi2}@#1lm|k*qpGN3m$C(O?|0VwMya{Sw-llOFNAkty!Z zc63YIagv?jYu@p8^qbsQrAK{;{Y#Y@w#{&57uK!?q1hPU)2y1c2r#TToCv#YlWmDFmbAORFf< z=RzFFAx6(&d~Otk8c++jcO*eDFcugVCaMKtx*u-3J^XI2uk{ynFSxf;Hf1gxq^lgx@chQ zuYw4J(O_!8HiQl!S3j?^km4apl*XcQV^A<9>qz1_QZiPoHSk&HC>2hnbR6RzR_ZPd zHKF446!tYd`kj6Xxi?tcQfqAIK$;qNnKRPwMY40?MQiFsWn|^5aJZT!&a5^?`-BiSRE0k^~%+SeOV$Z!$We@HrN@Vi&f+Du}WJgd^k)BfOMGOIo;! zeHYw?Zz@VQRfrNNu26KlG)9VRy^1a5O$57i6cyQlW~m+Ma*fiDCTLm3s%}L=FOlAy zL$6{B21hE}XGT+3I7&1eBFS7Qp^MwhSS$x!B-|qDEK7YbRUwEcP+i(~loUCDd5L7i ztekM-HSWORg{$e;K0k*s?!dgG*z(s%?@0AY6Z>YAG7@o;5(f19|nTZ6pV+-SLKjrQbR_g3yv$C1;t8q&;@k{2An90E1ZB6 zlvbe(Y1~*O@nABYW7MKt53T~ND}iw}nv|3Uiv^fQtW=CEw+dFVM#8*Hz=@PaFttJr zvcSx=WjQQW5e*F}BWrX_DYvS){&=F9SS*(!3m#)D#K&TjHN8WFotK{y_drUsvTECR^MEbSzdN#&AXBoMd<3v}cqL{#EG?Zv30^?N*_H~LiGKD76)3Vq( z3~V)ZT*)uzAG33-YHAr38(zQXU&d>*qh-#Ml-+bpB@i&6Z3QvP?GfV?xjvwSkTHs5 zjwu(b#1nC|HHur@tzw<4Iz8jZ_Wh>~5~VQC*n z;fNiiQOc{}h%4$6k!ReA=?95dAa&KqA3){S@Xq0-gOTLLcjOPxu$FZ;=4*3+*Am$Ip&qHd!$Sl8n>&!M`zZ$mgLkYsoDqeQY)vBWFV z>G2H`45x~dA6KaH4;si{rYvRUsHD8dN>Dl;Nw)B0kQYB`%K%Qc?!Z42=Gm$!(lzVIla z@u_NKRP$_1;>>zQX6huO?_FUZ(i_BA1ih)dN3xV)K4OTfFK_^pC=LPciiQ<)@W<-7 zL8qc%jm(gA7i=E=$P9zR2m1VOM6UW&WRyjO4(V<-yp{_KRLw2Z{3-!emWywwwDgKI zykfx@oMAaFeX0V!v2HS3H?Fmg#@6_iXcJ;7>V6^$Ly@cqE>3fc%H}gh3Iq!$8?Ml5HoM-TF5-oT;>?Hmn zFAYW2@H$!DS>{a7SiGrN3Bw4fx-!W`MtMUCe~OE)-@;_!yM_u)4_Ccni|KoT;`bbM z-7GgOW;@=A*T(9)+p$^B4+ly#l<9i7+0iX5Ja205;`j8uNw@scvAKPX-|NZ5;>yWP z^T^4b7m6T%)wqc-vr@+)nyqW#BQgq#C}hIhRzC}SlGROf&HJi0$vXL+mKy$RJ}G_| zW*Bi>t5pPiGj7sv@i?^(nG^fvS!$b$HA|cM)cBzfmR=}Q%Z#U7^Dm>gwWD{nZK~2U zpvvyzs|)XC=6dx4AKYZvb9HK8ToVXtTV!~B`GxGt7vkV9*~o*KARrk-L#Yv$U`mR` zj4{kuJ)v>d1qN~(3?v}~Mk{Y}XXReVlUyFpTsf{dFaPQyN$C9K%Da|X`PX4y466jU zPx{n5-{if%!mVueajIG2ZQcE_{ZOXQ>s*gMjNcD`=Z01$&mfiK=)vo5eU=y>{pj8~yjP(Slr3m%Jw-6CJ{rvFjl!i@jHI?+1wzrothRo}lZ`>JxvY0_3Iy z*fdB=4`$U^9Utg;ad9%vbOw*bVd4O~n7qECm)}=7=Hf9ctPyyak26TNV=o!9FvWeH zHJDNqkLhzPxm3&HSgpS`XG@f`D1n00z23kk1Ev5_tI0t%6oiWJMGO5Y9UhTidYeS=LKAMr;*s#U#6QXAE z+%Tto(?KvSug+>EMF&k-E_79M*p~Jvw_m4RWV~6&QKUX^^a`g~$(a!s!}_4Yvd0Z;FK`Pb5vd%sZWfao$&8E2Uyi~(;)Vyc8Py+`q zM8lKZR)w-i^j3r#y9U&Cl`Nnp^Ol*jt!$kl{d$WZ7L2Crb}e%?nZ~OGw)h#@deGD0 zLBE3Idw1kn0-kEEiQZmeFx6Bf z#oe)3T3|PLoAam`XCO}IxQH~RLWN6fJkjE~7%@_zCh2N$EpV!Y3EM?ORmLzQd#aR6 z)kVvs%`m%Zs!SkkN6UuRC~q{YysWBH*DG-{|E2#OxxLDB5%^Pu_yH9v{C5pf6Q@dP zO)9k=cm!^~oVueVe%Ek$*|^feq|)N;U9*w9rp;}#HG$JrR@ko?9`nr9Wi;Lc!O);L zz?&VEXwVlxhabmDAP6)F3-Eg2Ap_?M8VrbCnKmkF3gcR!NM%6Q2q|`EV!UEyX9}`< zTM#yvVkDFv6-h6(i7h0>i!;_1Mc-j9A=n?OoSeorg9g&pU@(p~3cP?uNl2qn%FY|4 zg!vlc4ToJShFQ>sO~rWPq7)K;#Tceql!^*7gy2eHG*f8+jMh75%xS=3EmW#(8il07iStiBrIsju8j^b@xr>#bYY1jB| zbbN#ua52m6(%ZK4o$sdaM_{{KW6RFzP>L1B=sMdFg_`wn zn%2e3PutQYjrYmCt4mASwHx7{A7G4raQ)Uj2f?KIVFlCrn_KQq@}JCsu+mVx=i;oc zy)bd3so{=lwTqFg#dJ+z&OMK6H!Fh0xxv7uhB6Nik5J2{=cdi=TOM9vp^MKTk}cf~ zHJk}UsUR#A6Y$mvnAjQ>ejJU318iU#Ks9X@gkP%&Y$>edvCLTaKY$Z-HleI1t6(yq zs&oLQ((j3vIf*2_45XhxYXlz*LKg3x#m9=53Q*%G0mmuO43Sv2k;eceiyxa`^S2ES-{WWU1Vq3^;ov-WJL;9#Rs9kQ5W zohl4>=1vluZH>OXUsxzxvaNO}(Hv`yx~oEEM=z)C<=vWq`$^0G=929T0q2LCK8fzr zHngv(Rv;gRipfXixU9in2j@xjm!vm$JeL=|JXy7pSu^|KWi?;uB7sqMQ^P}^0Kw2_ z1Xd$`=R1$eyuv)(o_$(x?))%)Kisd~YWZbO^XI+$5#I<_K`-n;5JqoMnmCG%e2xjo zQqF4VTa#az1g!P|3;%QE=FBQfs4J*|(J)%Rs2p5j!%IGI0i|)r5=Urysu>zb(^{8G zG=_9j6r{yK1tsx-{8Q~3bL=%^7AbTk#QQFY9E*ahOTd+t>H<4?IyZs<=CTgsE3Wo* zaqA*{p53JPhCfSeuB*Q;U9+<+KT{5IK@#N+x1&0jqiWVoQSKefzx$NnaZxu_fl%C@ zcRnGt&SN^$-MFViYsAzOkC|`|uOj7YNt0Ucv+eWpXq{DXfL2ed* z;9-&ty&cWd+&=N9Iuozt4f@x5Ph|UC&xb78F`aw+M3LB6Ax3T!L^`kc0fZ(p0kWSw z9fu*qyus|P5eH%5e`5+w)3F^k+34;{o%OG>#ErFY{+gyAWx7AW>h05=d z136u!wt<9FLa}>;J>3}|4hL)B%v+!Bxz*>@4GZ3sS|~bb8sr&f+z_E|G;`p!q8T!8 zdXU{byd@C&VaS@eKKF^vbCE|rvf5%tdE@zx;`j4LJOYo5R|sE74VjPndDe4pUvQEQ zS_8fy^DB7U;iT|>{%Hh`|7~cs6O!b{Sggcx5m}gXR=VK0UE(FN61qL-r*duMY1bMC z+2vids&?&%M1U1JpggJ2AnXb;Ogz%gM1gYR=3{x2cXX^;h^1t&rHNJV=q@Ht-rN_H z9r!GI?%Tbg+f?%L8>&OK3UiiBmk zI|$yK9|?R{UqrO$tR1>ARnyd18Rp^cWV!gIf2WBEt)VEl+g!2d^(KS;!EJ zpw71sy~1C$Kl}Ps5`B)&JCY=SRdvgxizxgcimhW6yZKT#ywf{Qa%qj^YH$ySf=}Z4 zrFH5jEhJZ%RThp5aVa$Wl^G{cS69!l(9E9nNw4^%Ht%mR;fD09D)?sPENybBe;6>T zd2_wKV@n`Hde{T^f&-^to-o(#VJE$#8&8*>OFju14d^^lebezm;hXeWw7_wZa^!Xx zPuuNe#{3fH%`OR%1Vj(&xRf_+N5`ve@;33?JL(r-8r``(T^$~9&*|bT%NLhtAASv} zy?Sxijw^J2B>Y`{iTvJ2 z&pz$He*^?XI2O%k3ULqu$Is5$!NUg54uTVMK!A6ceQi7(g1mg)I6yk|AYdG(6*thq z*U!bv^XE1kdxt<5JBOc}aJ{^(Jb~#yw-LEH_|kT> z2P_}BCjnmSwsE)mS1Z1)jlZ3U` z@bk0sweffH0>SYyQggTQckr|e0a4}4)6mm=Cg%jv!wG4`K{%u|KM5L62=r4Wf~ji2 zoNAn~-_N3;;RD|70>L4e*TPAGiyYO#F_;F2h7&lJ0?^RURDUSyJI!n`RU4S|1kBk4 z#!&tJAd259SGH~B35OaPd!XW>BD8w(vL;OyEbqD%q`u_?R-$7!)R25)O z6e(v92?hlE(G2+$F_O{pVPSUr&hm9pQ61*00QC z{RkF+hp6ueb6~2^V9q!S&K7d|f8O5w9~=$w8|Il`!pIN2e+R~&QKR6T4V3>^P=5-% z-vL*#sB*D5Yp6MEsQ|#=Pj#RI?Be%U;A_DD;W9CS{X5b&K2;$u=Q}HZ599xcrrXK|I{#OzqkI+(M~s}?@;l) zRE4CR`5c_1Yyi~nZ|Gy7esBGsL;Y*x`+0r$#5ngw0Z`Q6>-$l|fc~ie)BnE*#sAA4 zer&AY(KdS_R41drJ_qcdQ{n*N@81k#0Do`&?|@V@*-&6(=7dr8#6Una^aw(vq2amn zGT-whL<_zkrtl(FkVvEf@DEe8k;ovVk`q!5iF86LJDs*EDIt-%n3^&YsfoD)_dl+{ z4W2kuKnSUHTGhwgpR%T6E~qaki0MEEpEjOyhMD+R(ZF17)zyS7&WpP`oUXcLTcy)Q>#0b8zgJa0Wl{+v5R4*z z6fdBnb~QgA>}cz0>Tf3E z>~4PE@G?^O@~`};T;~0SALUanf8xgt!%u+yd0|HzbyH^xjZ;$uR2PKNFCeE=XbcbE z*Qeq7gPz)-*LOB^?2(`{$(E~|HVH1Ma)F>oXvstn28t) zIhZM7F6?J+D(z;k;T{C&74R(MQ-1%Fuac4>W+}hP^K{YX zW?HT`>h6Gz>j~MLX#u!E(LbFZ^so8e6<^#?KlvB~2)=o`JT{_52MBLf-yb01CGhXtC zJ*xhtpHuz)q;CxCN^TZr7T%{=S{Qo~G{<~kI{uQ+zt{KOUp!1h0DlRzby7L?H-IR> ztP>`l0sW`zHoye0@8vIiz=(kI6aJ}vc-VQU`CFLT2ike~{nh_ts=Pc1!R&40>l5JM s&u{1L&2QuD>lMWO*8?WAh?szofDrJk1U!5GAN+x4i9sOXv^e4a1$l>lMgRZ+ diff --git a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.22.0/reuse.yaml b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.22.0/reuse.yaml index 44542fcaab..abbd9ab01a 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.22.0/reuse.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.22.0/reuse.yaml @@ -19,10 +19,6 @@ reuse: from_version: 0.24.0 reason: migrated from SHARED_LAYER_REUSE.txt (legacy newest-first effective donor) approved_by: yimingl -- table: dsv4_hca_attn_module_perf - from_version: 0.24.0 - reason: migrated from SHARED_LAYER_REUSE.txt (legacy newest-first effective donor) - approved_by: yimingl - table: dsv4_hca_context_module_perf from_version: 0.24.0 reason: migrated from SHARED_LAYER_REUSE.txt (legacy newest-first effective donor) diff --git a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml index 54bc3ce892..ea336b165f 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/collection_meta.yaml @@ -27,8 +27,6 @@ tables: status: complete dsv4_csa_generation_module_perf: status: complete - dsv4_hca_attn_module_perf: - status: complete dsv4_hca_context_module_perf: status: complete dsv4_hca_generation_module_perf: diff --git a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.24.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index dd3336ef7caf1d764d0091439126e2a12c21fb75..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 15563 zcmd732UL?w6E~ia(0fS;MF=f`(tDTCyL1B5LX#4TbP$!M0-}H*y(%Cbq$tt^q)HJ1 z0RffXL_`rQ>h}cnUcK_(_x-+e{^$HhO?GE@=QlGuJNqOV4bjG;Fer=$#;FG*fPv^h zAif6SuSgO!2m}RzK<0XS`Y;d-4&xJm4YL8$h+sslQXoDkegLumN=pmO+h0U(2?9ZY z&M+GoJj@0H#4^B8zlkL?0)dEt86rYr5<((}2_`xiLi~9DH{oJ1wcms@^@Bh#K)8Xq zwwg8?sU<2Rf|L=_R6%N~=!+kQ1v_*m;&ArAX(9qs`Ari<2MELfXyW7-C}HR7h_%D| z`+M1WdOHQ+oDWlk;joE_L%IJiYG~-8O|@*FOOQiV0+OA=g36wH>;f9hsw%3O1(?H; zBvyNSUVXG40zC`gDQ)MNqj*q)>zCOHu)Ga@N!>8^7yfnnxwxuN8u_gY^N)ub!t=-&bet?-X?Nxsk%!vltuj#~Bied`qFSO`9eli%@Mu3;m(o&6 zD1VP4V!7~%Ek!3Nvg@ZWBMK28z*OykEMtpTg!A^mVuwVt_%k8~+Gxu6*3p)L1aw?<6sCv& zymWXJ#>9=z#1lCig>l!Z&|yB+z!?3&kg$Wz-x3djWDNs!5X5Vkf+5eYw*s(AOL>SQ z;Zee~QR8&yl`m?SEAr&P0D11ZfZPx&VA*ovLzxR29Dw=g#GR2i4OxC;9zrZz2PvQE z6GFf`)01Qjcq7_rmJ3OUc_#42@37*Xa2lPSzQ7hwCx$nX6z?}4BK`ruZ%~~PEMR6s zX{iqZQR6lhIu5LioKKHbmZ1QAiGadsu&io;b4ZoO%f94`R4N7k*TUt4FS92~?(ZjoyGsY}Ip`a8#RZhVvq^xeS>< zH19)e9;}jD?l6cW-h}3{H*=lA_MjDWUwDyZqZq?E-4i70LzJG3 zg|{yoSjJCx6W6Xs$rqFd@Pj8Ci5YTF=!-I&Ri6`6UsIn=Q!dLG5WUG;pyaLC{snxx z&>Z|94q^C@06pXo{v1M({A}T)_-FfX?q5-S$RV670>d^>>cM6i{yK*CK77u1K}SKJ z^2ccq2_BTky2Wy()GEtL)sOauI_l=VdE0GD*d|heVKJ8AH9uMZ77(CaSS+g+r zrF-6jPl&jJ!s=_Nr?NyLA8LD&8!F#81>Xx@(th%-1H1CQ-?`@ipUO85QBYYIq*;U-HyA>*?kgwn2RK5BjGhp1xwpz_4{^m@GVylRf zP}Nk#B>9`pV%?+Y7DeLts5^!6no*8*93reMA;%R#!FLO^zsifaa4Hd7pJ>XTvsm0E;%bQVACLM(ifg2JQ zmubElv#PS&K(X&`yju72a6-;8(@^p&kwB=+HyX@D#9I)AN41}6$x!?0yrC>Oy;DQ1 z0Jn!6mub)%t1NRm_ih*!a0$iu;?|X+)BP2t^z68YgoaRpHCBOb>fxL^$?6;aH3G?8 zI_)HWRejRB^14s%wcE<<$@=;W-67PzD7D>69?5}C8+vbzyFodGxo`mOzYrEVw3iQ%MmTdi0QG=cm}zxlWxzOkO=rW z?={w@Z#~Ci82Nm1Z=PPiny5DTIsGC(qcUycw<4KyceG93R3DcK8XiY}D~~6RYU3~( zZiu>&L-c(7tT$P4f`MkTrluCBgHgaIxP206YsyD{GExDGJL$A{Wtu_lA!_jE^>EsY zeB@_tv(J)y3xYl}5}a4$77TyAD_nwPaA0Ep)?Ltz0F9A;FWWKB)m46Fn-;nmUgGv$ z(vd8KI#R1jou`!L)BDwvE*I@ST4#Qoa7c-f)k0LOyXf2G%H5wZ@zrCXKO-S+grHC7 zTiX`qbTE$m7<~4nmuhm8>R4R~D~OhDjqRl>L@ZyJ-^=;qg-6b@gLYV%!JVVpt!1x7 zNSCGdBu_X3(b+TW-dSH0^0u-w5i9#PWiJzHo8KOFrb>ggr?>n17jq5a>D(@-Pni!C zs*M@4uH{9v6_UK28>(A;B)y?dqamW?K?ArpW&6|v;V2KcJ@=I-!YPwu;w`qzw~iSq z(rO#7&Gj&X17>fdZd5P(6sPVptNKi?99j7RE%{6m~CF%bW7=uGxCCNT37ex zxLe%TNZ4klh4oo4^tA%YC+j&gw|Lk_ZcqNikTp zOf(3yo>Wtfb7)Ji_7JST%*_zeXe1mNtQbzsqluz>v}yac*p5C|=Gl90)Me6Rw;Z!4 zBC&k)QDbhIlzZX51Vdxa+~pK%U2IgmvYxz&%_fE0H3ayJVkzXKCV{L<%F{FHRAbku z$AVnlT{{+J6)}XX4A?3@*N5v$3d4NMP&c}3=M+hn=8z%zBDc|VHu4Ph0ynZ-<=ZBY zVfE=EkJx-5x;Lqta={GLy=+@ZFIRH;>W7VdAu023!IQCuU|FTA*MSdShc-tOgI-I? zgnam-5-zT>A$l|WtgAASzm_EYPPEODPgu5+<+z47g~M-qc8 zOr3Rgv@4@PdfADk6D96<7>AK^(J1rAU3qa5p$y}p@wpG_SC3#PT?&Q+H-iw#qFuEf z!S}gY^cOQysC86cY2NCm;S3;deDfOWY^<43mZ;;8p0>3q$;ad8;k-465P+>^kKP*H z8e4x3J{D!>WAI{TX2xT?RZ&4fp?q;&T{6rIdL4pPfhG(O0i?LR!tpQlbRawz*Mv<05-rc#@G9S|oYX2jkwxr=zU_$GLBVNo<)_xU0vW7Im+t-S2w z^_yb}lG`Qv+6ww({+Ql%@mp@iW4ZL8?Lgmszp2n1 zY~HmhkYK|9wyn*>at3=%>2;LJg=*@nO&>xo<$F1gE%BUk>MS);Y~9|)gvsR?TnSH7 z8HhN;c2T#8+*(I-+kfQhP@O7A>O9!@&?N`%bfDNa`9<&>3o*T=e32xOpV!O><@j zBI>f6O;UfGFjV=AS(2fRDXn1|Z>-1DEE6TA+Q(E2i?A-;%w%C*ryd#O790PWJDzF} zi*?I7W~3uuD4(0TTe&>l(K$TkS>-tsL9(}-*kah#F1M^=lyv8$P%x(h;QXY#geNX# z%;eU#Y`teUci&768#})v+znMDMPOsIJ^Cn|VqB1|;={m=HefF4y{uutDvNJK~ zd2X>=QyDi4VgK?wnD{v&eNb+jQNhK?w}~GL)&)*gRIiNORz-h7ixUS~wqy~L(PtG` z2Ng^*)q%Y}Ttp_Hwc8F3`dpRP*&Nmq860ur^{c=k{545%c9NVS*yq|%vkl?W#~K&* z7q19pm#{J$HhLAQMLMz{M&~Voho0-SQ*|F!LvmMTXQBzN7ka6aWOMX6yX0 z{Rlwp>@SZio%^)7z=?nA*)D0d7D^lN_xB4pA>);lWHmq$UPJfi{zp1sDPN1l6Sn^FH<;zkEF>xc6oE+QPuP+Df&c56YZGHYKu$l04VEGfW(!_`M5_8j* zF6FxFkx8pka6DfTc!3>K?*Yq zXU)q41}Y7`c2Ae@D`ZM~%ec~@$hg#BK=|urKmHiqBI(@d`u4X%*L7ZR)mZnw{(MpS zjbq$ijwumWvhvpJwUsqn5}%$%V#OoRSf5#^TztCoDXgbkP*JF=T<7S~Yme`|0`~Vy z+rTtcvf;O9#dKXWi_u$B8LUbfdTgstPfwRKNIp>ymWy}PI^nn32e)nYbcx4b2 zM38x=(+aaplkUC9oDsBh%S6Z5`ec*y+Xgky9swj={%YnyT0?#0qnG2KnvckndIjX_ z6uDe1ut#c7l#ahHVv|RD+%kS`;!#%ru*h$8ucd0Mxfi@T$*XWG2wslhX;h{r%mnpspB268~N zqTZOqt5tx4R_(cnS3|DfC~l!5(v(H$>B{D(zIY5|QN}h%)J6#>!qz=Ti|&e6CcoAw zn3DuaLjr|%Ie5CW)2*3&B<7l#7^@wZM-4AYu*=+Xce#DDuMW{y<6?EJU2CIz-fz_C z%mw1Mp$VR7)cad4Qj;xx(HvjJ-WVjQXSJiYM~_<#G2GP=Y!zz6-Zr4fd2I12DgZj` z64mb6PBl+wsnjx2LJ&CPx;C?u?Zs(;9!{|NI(>xN4bx@V+S%;O`&=JMZu-7sGGmO{ zbtC7>`-DVX-OVkrhnCl3A}dQw$`$ABFSIr*yE1j8art)L5v$B%Wp<`5KQ1pe(0j7O zEwXq|sDF$@hp#y7`1CwWx?|1LX`aBo+z(dY!F#ap0wdYJ<{CCtgu?@k28p+?nSla_ zy6+^q@YnAS*_SZY7G7ac{~{Rkh*;LlGmDUk9+SsC$OU;eUhnGBDmW8K2q~Z#DZyld zYqi6cn|E>TD$a~zAgLz;JwzF+LNcRt4u=AbrG8D6Lpi z3ujK|E7GSDb19mia>C6z`czH6K3V627g=R0-!R6iNe0mEm3}N1>k@p8)7dIbMJ9)= z$!l4n9BE*?+%43L#ReLx6HD)M%_kb_917Ve16g-`-MVwiDJmK6F1M$d>nthb7hN7bsSgR$sAYOK*Mnj4+E6&M6(LaM7w37i9GVhf zfy4>qOdf^%e>_h*Sj6K0wXx zZa9t(&hj!2SosT|4_3C4w43za%BHyn|9YWLTeN=Ow7o;?gsB`ITYBtW+`2dy{B_hB z`6*5@A+fK@-5W~$7_KMBnq_V~Oss`A+hkQ4+$Ho-zPDFIK-`LEMj7=b@EtqOc_mEZ zMTWOvMW8bxYwNnh>&pJuZETGZQU0kP-7|9opZTuuE!{0Q7pDkEWs!o(wamNF~}y_0)7>%E1giqP-DsbZ88l!Xt$FR(JJ!&uKf2 zI7FH1vg=rHu~I6jU*O7_rKb@CL3}&XlB-Gd&Tm3{uJ^3dE#Zg_4C5`N;D$aF~?db#$Th6_BsIKWftR3XV{(JJO(f<|4xK|Y}) zIfGH$b|-v%Yp;mMhTw#5uO)nd@WrOlE3~B-ZlW_b>&d5C%5R!(g1s*YiZ3k%hMc~b zmpa$=^Xp0Si{`IA|@Pj+9$KMA-SoO<%|==;%$A*m?U z0qxH8yUvC&MVO0$(K~%NMXq^LP0&tX2;n*zSUJQT(dy2~%`Y8%@=~9w-^UX^~+CJ(W*Gn7&B|BqBUad-S-~KM;ymRd}TuzjA`Bd_*w$0hG(t=vT zam^Ohl-4ZrBT=R6+mvXWo!F0He|g=MwU9&Nw~ ztp?+|RG=Pt`5zF~xsONL9X`Y-i~4EMz{izFQ$EkWLl0ycFFGkx<(vYo2;?X+g*ORs z+!WE{I6^un;EtBf^c;|lz(|B+w@UmCbNqFo#EhjPH?O7kN0e=UJ9<|7eCCS(wdTONIwj=R9sh_=$>uCL9D+20;)Y zFqwTg1R4$^1ekRB;e;ajV5(pcloJ_2YG_YDTM|yl>;tC2v=cCmC=-hKL{R$VL)h%w ziIr&xXkwHJ=n-I2DW3>B>vkw(NjuR|Lol7w9F$Khg3P>~0Ip{bMScv2O8J24tn&$( z`jkm^<|4^c<_JY%A}Cx&!f8js2_^F*pxpLRaEW{<4_^nRk1~D&flyxrRbYD*+sAOC z;~*89XhQ-PiVmoZE0{LUFdB~NAkpCKpiNAK@(6U0sPYleUo?cGBsxe>gsaf!ZbzeJ z3P{!Flo?o&7x)be$jloF;I)Yt1YEJ?)@z;cra3}k|9NuTj|7NL%tt7>|g{{Kzh`XO&?-~B{cP!IaT((ZDM7p3QZJJje{cTk}3@WZLDDz zCs{{~hRgzOVoWqQ3LUFs*h!bxu7;Ah9joVagFd&Wi&vSJ%qTt*_^GFx-!O^HtRfOt z>C=78o|fElFcRKS(=F(eL~gSg$=JHxEgViufu)E-bc^(eCMHoh3q&z>VKl_E-%z-! zIU+{Wi6qOgS3K;wSf;0lq?_iic!zVbE+B|yA7Zci<#Qd`z!1w%&tDB{=VE)CPOP|r zO*k{h#r}DU`1t4fg!3S7Bmol%n#>_F3dzkuW=W#TypVWFnVXX?LraCvnD(+Yl#A(c zpN7mRZ9nNNotL6WG8 zMABZYxyCXFB9=yUQpSfvQtP{Tgg-^loXe-XUIIl4ws+`w(a^IrB2nr-9h5kGXlkDm zN+-Xc+MR$teQF7%fgz=Gprc2LM3AaCqV;h0kZdFeuiR786QK_D>DyhX^Cz}RX~R0B z^Y*%U^*}m?VRZBb6c_l^Z!1H@YQW?&9egY@4@qsqQwm#@`DJAyjN>CyibRU|4MNCF z^66ludKZqG-6k`uio9OoN_folDVasn!u9HK9svg?a!j8wye@rNz%@eEYD1Y=u4I`q z5JP@)ZUla}g;3~p(t!0EXIj(L1)-o0UE6mz(puMego7TF+YvlwyuU{%5 z9W%%BQIchrlpceRnI~I_q;4k-;ap8vW@d?`?>!#!3Lay5u}37+K|AadW5T+~y2}1d z*}ylYj&((%SN2hafq$X+kqy&cxycTL!1}r)TmHTBFTf{);hec|5^pK2$Wn)n-pt*t z>E$AFP0*a($pwRuU><&DZQ%G5Ao0yu?Kw!^$G%%ArYU8JP+|m~1d>=o_91z^SRP}` z9O1i+N2Iof>|bVYfw^ls2*nb?^oV7sc0q#CRR@Ob>^VN(_I{)7WcII9`GkV5V5)fg zsIQy6{F>TvMjw+IQa5iM=aq>&{|$o7gT|1kk&Mu1BlE$OTFL~Rh7lxA`250axrzgYDknNU2*s4VqMnKlwUAP2z$qc zBwQC!VZJ61!*H3Cqen|q#rRrEJvU0SU;9L)amomog|F1H%P~Ei@S3Y;751=h!9E@H+LQ(Xmmz1G zAz$;lT*69MZ4ayQvebsc070Cy?#Y=2^Si-ImEIA@tQVTP8y(P9z8NpA)m1E-#v6pj zYRYZ4LK#~Y8hAp6F5B+bPI_!f3WqK0+I?BDxWCtM>-;XMJ(zx~oy1%u>Iemv)MTm? zF(Hx=4OOIqv_C+?=|GLE+_cG4UGVU0u|_yB{Yl`q5{Y28<~e3jiZO2-AtbTfbfD=( z^oXq>iJkQv8KXveWbVK)0TkxdZ>M^d-isz)&~xHoVd_^R5=(qs=7f@GdT4Ae1_En2 zA9J)Eu(6>gLsCl{?I zTi#37T+qika-DoS7${X=r0?nBcyea;y;Nf>%F8#4<=KYI-R5~g@6Z{R7oSn6`(G)2 zBK%H-AbA=)XsdicBH8m)mm0fDxk)76fN5NflaV9^Q0~6)1&Ml@9?z1~aelLl6QFyN zYF?BcKC{eyocAQVI;fDGNil08vJcI+s8X9*mqj#X9~ueyWzMj!%1p{WI!onWK>ut_ zL$hf#W6Qrp+-Acxt!cdSYCxsmvzPXa@=u4Z2A-}f&w1r_sd;wYE2!mV&TFb2`8mQ9 z!4J4>-^2$gEb*KO8E~}S$^M|QrhX!H?56E*MUdjA^NBMvGi)E4dKF)vr4E~?fAO); zLTM+5`s}*Ni_cS&N*`LO&%H^0vDc+~|MQba=RP#Ef7?zA-&;>0{FDVEMNqfx)&Lc5TYLu=5t4h2u=UzZ=NAdu{cH*LVA(qlJSe13f@RIx{A@!L) z3ifX_F=&Pbnplh)7aMJ?w&?<0#*`Yb5^bDO$^t_vM*W!7QXFRR2K;Vcw@~Pt%k~sE z5xrU(VyTbg9rZ}{&nYL1Pj{hI@=5hZ(t9dOjoi&oaf{e2p8be;*(PTV}Q+B)W=*V4M2P?Dqx z`c$`&**FtFxpnQBFuIfihGjl&OC6J3y6dH^kw>r8JfssTfL7F-_Q*}>SSYPs$A&u# zILi)fJ+4i?)50U(tedXNJq@`O*tC&rp%M-0 zhmWs@zPpK}EY;Eko*`1l3whPubj+HJQ|~YUgEqr00;Q~CWO=zhqZI^5ApepOXU%0=ZFQ=|8t1+ zl(VO&-LDIvc;mbRJndYau}*$~`9Pl9O-`UGC1Oy*uV`636qeFpMVH8Y~ zAR-#3UmOji0LI~QV0aZ6rGX;(vwogQ04M_j!3gk`tTe!z0=zMliHHdWRMG<*^sCUH z`*a{O6%20zqdtOBcffd|f7Z_bDR}%Y_;ATTcKbl^HW;2lfEt0Iwh8cp53h*-&xP`T z!mdOT9uE66q7HC3!0@XCs0Bh)G$9`M&=5}?w_Q2C^I%>Q1={=XHw|EbN@x5S5g;*mi=p!Uf3 zH~;^#zWq<||G4>opZWvj8(?@37&SqKN~FRg|5eidTXFlJ^0(jphnW6H{x|@M2E$9h zsB1K+32HpZuQl#}irzSRxZD0tKLy6bZcIE*J!a1qIsJ_=JVr!4njskxUp*$IxnM zG};LGhbLV$ItZ*D){M!<44bv&XgS{;p6KLzxG9?+M-3)ck%p;h*qhWP%z zYHF6khQh-50d(*_b6+$3!XKoCe|ON-5V5k5ba&pj#uBfUCK_$|V~RFf%}`ue7%$#P zdw<5h5&QBFrl{eiY8nJOU{pe^Fb1I*aUEADOUdZ0HxJf}{Fn9n_Bd&&xHyX8q^yk4BEPR!_4vp71E~jC z_*eX%qOlkQKeVPAuv7-$RoOS%Kuz|budceUqzc|4_>K5U-atde%@Skmj@6J2#bSJL z!2X*3jt|&#;13mG6Y=!V{n+0+2TfUbON(F^2QPDfjF>CV(!w|ttsnZkK5C)Ie$z*F zU&~+g@xbd7;AA1{g4Hy4wbI%TMPTcK@cRq+X&)M|!-0KT?my*e9B$vfJTEH?QGblM zv81CW-gPS8czb*drGLbS*}uexzZgc$z||742P0-I;*3$j`^OT$Ui2U9_c!sE_}Vv3 zQ`FDWT-L)$3l{|JD{!(*_x1hZUlkQ&yeWV9bKhu7jE*~269>e&0Z>4u1Kw$?FM$rH)b`ICAQVvc6h5B;C*5)l#s WK0N`SM*koFg4U@(AmBf?ivAx5rz3R$ diff --git a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml index acb9dd93ed..20b19499db 100644 --- a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/collection_meta.yaml @@ -35,16 +35,6 @@ tables: collected_at: '2026-09-17' rows: 1544 status: complete - dsv4_hca_attn_module_perf: - rows: 3918 - status: complete - collections: - - collector_ref: cbaf51b64fa460e5ec6146bde407a4c64958212d - collector_hash: sha256:5882781b43c67c93f89e3e238978f461e242c83f681effd9ee6129a4be1637eb - case_plan_hash: sha256:0e844f577823301a0af954d0d20baab4642bb1c0368652998bc28160dfab816d - collected_at: '2026-09-17' - rows: 3918 - status: complete dsv4_hca_context_module_perf: rows: 5793 status: complete diff --git a/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet b/python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet deleted file mode 100644 index b105008d6a9b327d641bc3de7090779539cb6208..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 23869 zcmc$_WmsIz(k?u>YjAgm;7)LNNC@uk?oN>49^3;2x8M-moe(@Ygy8Pvtl`;m_Sw(- z{`mfUH(WDAuU^&N)m3*_)f!Yp)!4A1vC*+96tE$&L6{&A-PhMoWbmRO5Htt`(oj%P z#0FvGVAC;SX9)q_V6kC|xIuK#;0t&!uNW9Wzn7bkv_K#z;EyaJ2#72p5Fi!{o8~{n zBC3Kwus{!17IrQc*1r=B``0DBf4u*1!r8GU{zEwaFbISV2v^dOm5>!BlVM|JCF5n4 z79*1pQ{?!&v9SNPC*I#T|8Fy4#TNUInc#JUKv=*`OkLbL4J=KJ42)b|?G5Z4OxP*~f15@;1h%gN zu=)QoCYeeI(0533Kp`#`w#<&s4%TKSuCFD`%pBFt%xsu7IGLqvja)35rBrxHRh%8b z?+_rVd~d)WvB|L6|I_kF|0fy$xD${3f9mkBLH<2?^q_xR>;MS}ig17g0+{*#WLe-T zy8u(x;FNN37B_NpF|z%8_rU(!o_K%X{O{@fw*%o}GyI2TAsc}}@PHzX&21fwT-mt) zZX4nMc@ghlpZ>QYP_fDWV+c6DzjR{m$oqHFp@4Qg@XP-ulK`9PKV+gv0VD9`6-)^Kj|t(y|ML=1RRv7XGBCG z1iU5!R9Sj@851HD2r;iL5f%gkfdRt6K*zxN@Rv}`G(ac^J3B`)AXI`wT;lhO(6{dz zz(O_5%`X9=78d3njeyXI@Q4U+c&PM<^mHWzsEmxVFYJhbcLzd(`HAe`Y6}GT=`AG0 z-)IdM&?7RSNjyN8FF*rpFdERH7x01(DDfB`I~w*smH3aY|Bn*?tMy+>G~|KUK>~$k zyyJ3|%|P9MRS03bxm~l#C6>+ci*aj+;%iuT9q!j_K=;2f#O1h!}p;?YD&A4 zj-8+MG9W47XDq!IvxBoBMM>M;cA>gOuj%=+n2(2Jl*Y6o4=+qIEq~sPZ(ZUwu2V|p z_pmT%cul47UZX&h-Uk)~9@R3DmMf@s8pG}Y)T&1BfzN`Y^lI_mSn9FX-wXeotrv|j zXw-OPzpt{M+asMN66eTS?AWcsJ;t<&q#wD zC6Sgs53{ie9V$W~;X({{mtcqpqn?NKg?FL4i#X9WXsI;9ca!fsui*2_A6fzf6%1#QHDS20QJaKWH!3i4Rz( zQFOu}o=LQ=XLQ2&#$PttM}yst@gFw&&l>=j{|CU+|L@j+P2aEp{xb#0_4Pl(JD32n z2+>GBx-b11KCC0MY}WW8^l54!1SD{(>``{}khlgHG?I_Nfv<~EUn$qI>* zGqe<1;0lZCwS7g=O_OI;*rVug43m>IqF%!XM>`Y9nX~PXL_5z|pTUU6t15{PGZn9d zs48hQ!}7qhk66*A_b0>L!iq}qq2Q+`>!3;TmG@r;lS%Q#<3kDfY&L=)Ap*@NBPun*YX#Gd$nN-B4Cq6K&X(+h0W4Tcp=d6cY>2KT-xNpv zm<&90b`d^i=8!Z<7|2(EiW9XW=uKyg5k-(hM24A? zA<;vy&|}F^KC3Fp4>1)J`GU3Ei4SOCK(@T$_GJR#{aeUNMb<_U=_KpN1*_)P03Dj% z0z3#afj?FXHZ19_jsswVoN{H6zI7zA#c7S^N$s$yTFh;bP`vbtL z8D{FEWA07wm*X#ofrt`Sl4XR~h*v$e8mj0+)=`o~u=+(sho2Oz5=tEaZ1VSz>0wz$ zKtDUtbm^bd00Yw&GiSTf{g6085QrC=l^U-V*@Bxdn%02WpX|db2lRA795a^Xqlbrp z#HYbFZ3vNLkK(&BOnx;$FU4nq5S+#;2P|LSx&3&FH&w}}gyMvHKfY^pRTDKqS|xzqHly(IBtY>fFCqVg zpm=Ud^s}5pA;;+ub9oSI6OmxQ``#=rX%4-sE=N$!giv>hN~`$c*!UPf^|9rGnm3uO zCG;|HdK(j+A`Pp;gGhEzWeb8VM{;3gly~IoSzyw;ylIZ6v?t<#&oBu5bXq_7Bmd{}ayg(Nclgs2{V|t#3ncTzUcV%%+cvR*StCS3rfS1h1>zjSYh@Y{136Wjk~q z`N?F_>^cY6tRTPbTnWieI35aL)E?ojq^FQhW75=*+S3SOez8e1Qaxluh+DijygAMy ztPzh2+*$l^D-2|vl??v__Xhrhm;Q!(|9R=(k=%dAZ~wghFY@{u?p-g2#O}-o*!Pg? zU-WBOBU32`5xJ=4$cgS*^+P3RZ!Rw4Y2;%n90PXg(I)lRnD5j&Yl_Tzoxz`uf-&vZ z`4$U!P@{r;NyKq6L~9`6PGKKmkvijTzB4!91xGTKl{>Z0{+Y`%u_U27D50=`N7M#C z8gVzrDg=UVir|~hb+dR#OI4?W)T@srM0uS|A`*!-DO;mdQ>cdYYx+OWgeJ3>UVmyb z%oAOYiT$Kgg0Xk+bat+AklxBgM#sg}*lv`uUZ1I0r4n$wZO?fuIMX*W zfuB6#PMC)w-RoGGJ;+OVE~g%H zaRQua?E&3XGVx@h!F}I!DV!`2If!h##$t1@4gch5yHzHV3#I55+IQogi5=?Y45sg$E+W;MeFDZlH>zszzL`05t z`IP3FQZnBnu}KeCDwK0wS85_w74a#P`e^D7k|Hm>iAW3{f8*_yIJ=slDn*lj#OJPe z3iXe@;w;QeNow1Vr}0XG5mbt=h1(;o5>FcZc_YF)ifM-c)<_Hyjs|qceM7$o* z!+XBJEpxC$RqSd~AN5|Xlh}oEV}p7BJ7PKM&v7wcKoO@skUY<+X}2&!z&?VGV|T2! zZB&YUZ9{8GgPE0xd=68vOEmNa2jvm~8~52;IJTPpiM3bj;QT?f(IS@kd<&!?zg{cb zPT*Bj>caF%LI_(DU2tmn4hDv55lM5d z*ID5s!;&=io(>~1Yrn%m%#y2h=o}hP?2$IL_zdQ@w~Cq!Jcs3$=R2wRHzb~M;e+G) z7+UlKKhI8Q@`gldQ7=7xQ6iH`1&OQ2h|*N3cg`9e-{C!wd$!BcI~(FT2Mlg0#MQ5QOZM0DCTxng`mn{+tRo|rX6$&_ zV@~i+bu!aTZ-t9KaY20UwY22ZT1oFIVzbu|%Nve8!Jnvq%Oru1j`NU$6{=5&&@fn; zt!3C}dQLGu4AvHXs518s=xcZkEUgR1Uf(D3>Ld)r9%_+tLHtA!lMlh?r>{8=6i7?$qAQ;nvVfSXDeObY7_9Uv1!ysJblEc`qo@;0 zNy>0PGBd4fP$cNbX*0<4$cshG)ax75J)fMCsX za?rPA)KRq36jM+4#bfy7QgF@H2!N-VLEmvJ%NdH(!jO)tozpvRotW4veY#2dJ?IbL z$eWPY*DuBU4qx>CmTXs}2U5N}pmHJb*UtFQYA*AlW_7Vgo-f!M!V6Q68Nqz0Uw0<2(waHv z+)ltY4EbTaL+;UbbAjRhlMaYX76tK7=zxz||Ccl$Cu7B57t>zVQY1{j)8g9*?i1ya zMz;t3i2|(?zq)^igam02#D6FS5lu}xe3v0H z!T#~|2)(zRZJ;8O7<$@Htcpy->g1)bpOXa17}wOY_hrc0J7dI?B%_2?pcra$6w29W z5V_9(h5{v$$aUM^!otHM9FCR+z{EI8mEAwk|K;60!{TaSU|!fxT?ZkLsnut?N?-QxPm@agxf=h%tA9EFnbR?DK<>fgr4R`Bz zteJ9%K8nz;TU%S0n_Ece_VJy;bhkC%`#J*qQ-3ehGO4ykc}bmVvQK9NDHq=1@^Wl; z+2znBSpTXKqXj2q>$KcFW^)y+P!{P0Ev* zX&K_5wXDZ@7f0aQdofm$@^K9EKK9Wj?Ts03nO7^(tU0}G+$i2}cFFj<4&;X76EwE& zQNCn4e!g$K=?XN;cDtflj6&%|W&&c&J-q4BW=QWt*~p|Fw5&uh3tI0af-!|N`IL5N zi5SM!wi4fNe4pau)Sq{&1e~;lwQb&wm&unt)rjmCUPEJ{KG^R?ek^vM@z)v!xrA3+noQhN@H2x@4pR^iUqAq9Y!5^H zX$W$uvQ4zaV>Z1w2%)8}$)BwzK}EiE0=c)(fzSU}vEd$G!k?`rpS=|cwsynf%GdB& zd+I_*dk%i@`r`j4fU>=4Z36JZMMwcYK0Y=&I++)+T(W-j?TRSCf=tTBCfAvQR(_sM zexEX;cO-sT#4mb@jwa2%uu=9Czw$6KH!lv`Nh~edH2-9hDe(16X@V1{dTZ?>Omt!T zoo%;BNLS`)r5l*wF1Z`H?S6(1Z%Yzj@&9x)2Z+D8b%KAJT0RD!{#euRJ3Q;fVsOyF z$&3tAbee+ywL+jq3w5@)4m9Br8vd7Mz!uy!#~{MFDeN2N>%XTZY)*`0p#l>OSsCjE z!=>J@-Y#I$rNljDIaS2|q`BL!9tltSO}1vpufA2lI8N0sRp#*wwKjS72h4F<;{}s^ z+3|>Q-xfQoIMdJaS^yHv3wfdW?w8W zpb5iEPa58N@dSZRZ|RLwQrx-1MZiQlzvkef63WL<9mW zef7-DCsDJ%b-sit`t#M2aSWcn*f+UKO01CIQCAb*5n#~^H{Ui`I#_d@xM^pH{`Sa6 zIl97507y(o-v-#~B-7plX2L+?++A5p*8vs9&AFkiogwLE44_nSzjku7EX2H&x%v2B zhx#Ku@pktb_)N2Z_~Q@exgxJ;8|ZL4OQEf`6FeRs-{hc+lN;bXN9NiaNRDT>oA|W9WzPXn*ayqQZ*( z6_giu&DOy`o}#F9K|fhVbu=g~)x6Emz%w=tWX)oYs~s83*@l6$i`e(CY|bl4xs(6p|6|uj*k+?_s_#xGYP@LiTd+#m^(=R&b%WtQ-?6EysR-Yu?YtL(5W7B z)7Hz`*0w>|w8%47Y7cyOCkm?Sls4?vsEW^mA&B6T+c;zF3lhh+h26*qT269TSe%K^ zI*N7TJ;_LKPg<9knoP!~k&*Gsk>a<2>m1VxW9ZMd;_t4EFKB;6@F_%Tg_vi)dhR8P zX~s{Z($c0Md2Hw9U}dd(*)XC*cnzF1)5W&3vBSmui+Vm>z|FK#SL-JvZ!G%rt7xm9C zJ6wMVV+Z|&eN~qASlr^r8!S)7a?ognJV&J;!void@(EPwJZi^WS7&E4WktkAZXplp z|Fe$@a1{0yU7AnBW=N{AANq;f6=;%%e?W$sDxT|Qd_|`9$p8%b{A!MYQic?*UD`)- z*jZZCheNBb>Y%mC=Hw-M1}L_w65@sL)xV9i7!uvNo=-TNCVjtwkGUHZ%Fjm*B!7O= z2E5I%@A2m0$zKwjFCsmUV~Mu@xY__b@e21QHw6(D3!PfvFMQQmeyCW`DI3qSLf1udoe0Z8_=obDT&^1<&6 zyLPrPRmK^n?R&MMp=Q;KHrL3uH7*fb6htR$6}5|X*P1@?^kl0VuaPezpwDqq-DR8-{qfX?3Y<#%yd7%xldkh8!4 zY4u55H#7~4;?~XT>TE#={luEb8Z%9eYrmqes;cVr)RYP*L5x&L4=dQf0`@K@_V#G$ zeq{@I0QrpP+#Vkv4}j2U$1?Wd_8b7jSELU^>5q*)!yW@mpt}kF_SeM6lt#8;*DirlGbduXYIW@$vP#fCTVE4?e!?udfLr zg#24i$z?Ei9s}U;jk9QNVWGLX`4{}xTgb}w$jC@Nuqy~^>#`}xs{`@TSnP0MI`RB> z&iX=&{!hzbW%BSG6zS<1*F5YNQ^CVqFR(PRfk8`4{Q_Fx1K8?W@sMFjvmnqV_TmRa zGQo^dG}3Lzx^Qqs^ZN^3aNcl-zQd5rbaWs3FS}3jw!K+KG@G2{^7%i zv5-0q{=gARo7zGEl$3?@2Up&re`QM3Z4r4dIcL){oz^iOQGs8Z*<`EMs;7>D0V_=L7$s@a^Q^*<2 z1C}XB5;y3U5L-u+_Yh|cZ>`Dl(HmFz%^U9L_=L@m#(R;Gt5pRB1uuzbK^2_@Okfx> z_4~JPp?k1fDI>d^K1P5JZ#l!|;vypG$Y+^WjP}E|ddPH`RK8f$gNfgK!-8@S2=C%= zZ1)4{{dF`ZG0hR`7ez!wy7<9m1}CR3Az>^&AcT^A(AO&lD6Q(`2)jZF+w$OnQ}^CjN0VX?EI+>)cRvuWw(lqrkMK z+>xK351b~l4(cf>DY4l9RmlJzPC>!J-CXYu+`=9+1K~Du_mgzvTZKTk)qx1s%;2LHkM|(edES=*|t`NR=ec{96Yx4 zF5dLWmsrGJT37nxTk-Jzu7l$+YAY!y{Xy9wP9zUcKj0|He6`#ElP!daIyb7{xK>k_ z=SKqLrkNGfF1T*EU*lHj}m_F5>$JE|y z`mt(4wse@3Plki*XCzu)8t>n|>v$;)ZI+eAfXiQVQysNzY|~4HVKQMwF!1k7zwV(F ze3izSUNSlA2A8KEzJr0Fpquf#wWJ^s41k%!!q^$oz*PuwjE?rjR|MXcmUIL$upKUY z7rD8)fLrSn3m$NnYms-{1>h+ z+c#-wYCi0MkC$BYWmxEGzP^OBM!gq}x9H|1gFz%7JGfOI4khkSLR3d`Q*rBz1lr!H zNyfql3S6GNkB)dmsK8@f$LXKn&GnOk$Kvelb+xstJo|oYMMa;1pBwcuob5GfNpW$o zd3bdRadNu4sH<~~?pqOvUu9%u0!Vv3C9(MhZm_SfPmpzc5Mm+K#2Bzlg46FcW`BI0 z0o46fBT-kvp^0&F*&3uhXvGNuXc-UrIk-c=dVcxHy=AphqkN9U8pR|c0YWUXog9pf z$k$CEkwhiU$d;6vF11sS`_5S&N}42GE0ru@b)1plLs~&8!<^mFTpBGZE*6XofRn(y z@K+_6=wb#8;MBup{}7L%^bP|R0R{m90UrT>|JN^9Akiyp0rTPf!BfHff!-yJeUpy( z3wn<%ZW=64!WM0Rj9DPT5{ONbprgmf#VNmj{aSf)bacceBSq=q^X&cuPTR?A0&u+b zZFZmYSFzo4V+;+r=G3vc5~zaC5|o+A!S+r1mpn_Qg)so7AEW_bYT6nU2m?;dwA%8Q za>3cE{rlST(qBMT(=a)WaVB59WWQ^Fifzms0ZsuNU!F3NlN$mQ`}1eYv8%Il#16Yg zCtG+(h>N-jN-G%Cdr z%m5WCQmw_8LSa}Gu#Lj0fCthW?Zg3yjnidm%9IHQVt#+M4q!WRQ2o)Ph=~pfpqRuP z9`gW)$;~y?2Z}J+K>X=9yzB1nNzxi&Scu}t>*VZY?_yr#RR7*1q>ZqVh1<450u>zT z_xvDZM6>44Rw|j@zXaA0V*(CJW=|+~;FRSyfHmoaRtTBONsEe3?p@sZ?h>jOL|&im zUA+ADCii1sHoXFc$rWGgWOllHu^zZiUl+4%^BJo6)(%`IX|zL>sY6%N{MpDc-I%Qn zE{HP>&b#SF$;=cXmg_ThVipp2sdZ|-#);Z^N$E#pBA1U+M|UXR)JA;zT#H95Eg|uU z0j?5;Nkiir3$GK$uH*3GAT7)TX!7WPB^_y=fB(cV2(E<)6ptowl&~D5#M;?0$BcVR zV7=U^QGocOh_A3(K9Mt}`xNjQhqnc}z?wb7DVrs>b8?`7!*r^Rb14Pv6{!39v%m=!DC#YNRhFHQbjj07(kf<->~A z@-=wjFTd&SG9eUjn|1`yKYxLo%uts2g*fbu6NgRNfx=JH@b#6AP54W6 zvGnJefSRcW8Jf~-orHuWhltP3c{|Hdd4iCKkCUyo{&QX*kvxlgN?XQV48gDo@w;WUWaku~-MOaHR#94{+N4^>j-*R~-Dr z1bYL}oL~qz*1HEZ2v+J{J5UtE5S=FoI`oCLK-kuAKt1Bs@rzj5WAXrsDL>njCUDVV z;M>7U^}y(sdJx}jvV5 z5!(D?V_oAC%<}2f=8uTaOS?h{bAbnM-Z#+O(JAOs{4~8uR%u3R^t0$a+1y94y~B5|^x#B1?vB zhp+MZ{lP1VMw@Z&kxogeimzAWSvTS1utALr%bONF1BcsRq7?T(YiYV3{`v6Cux$E3 zqSdFJyE!iCRY9x%sd&W-As40FL4j@^|NNZp4qBRKF`+K@Y$SrtNtNhUg)0IY)d1> zTeetY3KIN{d;iMKt*Xo3eU+0(`8H}If`Ksa!Q4ehbp={K62-`UhsBvK;69?3Nasj` za!5QUxdVUktfPLSP+eX9y^NnR;7us|omG@sLApNLh}XT#DSFe>LenF}+<>w@uk^^->(PvJ|&r zX{Xck2r}#Z414D6j0n9G#PD z+R2T<7*_-xo<^BY?FwPKCI*)akoUJiXOx-@t}y;R_co|*sjMpZ*d*8P;K4(0KWT#I zvaIXclAPr726Q*{B>!IqYYV~ ze(kP5>L!R!sv1;~#r2*!yghoyYbO15Nyquj^PxU6N|91)tB~?2n6IIJ zyLDCW(w8@&pj>CUA6b6(jZS^(LOe1=m}9r0(dkr&bLYO#2u5S?DDh@2!90vhJ=Z8r zjBeUt1mhB7nP+=?@*jktEyKss6ou@5#P%`ctM?EW8?O#+o(i|NrpO#wSI_;mNobB# z9b%WhK+J`?u6dTM7n!d#>96UHS@Q1lE=JJ)*8bo&6JO73YXQ5b(dLk}KRnvuy4or^ zw6p60&l6USmE;_Mv{>SbFu;31M_ zYftZT&A+ow!>ed-m5p@(C4wl%6ma8@;AwKk)=M4-FuB^3Bz7pPO+f(^@AL z9s2Y~V|SRsCTEWP#CWw9T4J8&HjwWIgsoFT*5UVm%=A2vlp--l(WFkM9FS>TUO+XY z+(eJ(wJ&!ypBb%I+-h)+g`AAe5j5O2s-z(^H&geY5^WJccnXl`$ z9_gvmOI^iu2b(`Cr91wJbmG!aLBV+HbrWCy##qxzvDz`%D4UvrG`4$X)ZywVo1Ma+ zrdf+$>(sicr_il1vmP^wa+H#5oCU@77uE*~@$@HmF!URxwduSkrM-55};xK$S=0I*YJoudeOq z(~wfvRHG|A>szl8Od|eN9yBmYQF7FBkg}SETusu=*l4A_M4dsIZ=p!wwR`HbUi-6o zocmrd_s!N^cxXD4JcTj}>iE0c)bu*tQ`i}TZq_U)md}Tcn-2QD?g`^pzf8uDjPMD* zCU6nQ!i^Ifp}Gzb7={?RcqiZeK}?&n#=-H7ZJe+f*l!1EvS?CdbJNy3d#|qApXM0) zapDd)D*Rr06Y~|>y3CbiwF*^|NypzV3|8nyAbI;4lV3n_rtTC`nCD8xkdOO%*D>clcFc81d#L&ToLP`y?_ht&&uV_C|Z+(ndLRDKl^%^6-9Y z9rh@jm7HsYG^fpQx}A_ef8YW3(DP!}YS^!KS2vq{`D^QIk{g~fS;fl&0ih7X&ZdK? z5!gd&mnqu_$vmUixM)0waaY*AYD${dx6IFTRo(usjxxc_hJ}Q+y9>%=I@fF8O6AGRJKW zGxR;FP9{V%W_IvagLODVpe>T*7YD-kKb~>hN>|)DHK0^JhD*UAeL}K}tL&sV*ec9) zXqsJ04=Ft1AD0iSdwQ_74(iAt9}je?XFh5i>zz2O!@idM7N9=5Iyv11nPr;%95v2Go^oE20Cv8cdqYA$iG9hQbQz!94L*I%YZbE9 z+oo31Z?yS?)l={1FG2^>MuRbCb`%+0>iic@voo?tK~g%5xMGXH8(IsyUB#o}8Ar+N zceAee7GBLc2^#38jM;>J2tsN2m`!quA%Kn=3-L3-K`xu5k@zSRYV}G< z;%r~~FHfK!O3hGbFembiKBl)eD6eY0F_~$!pmy%^+2(G567~{)rMKv4Yi|Qp@h+nN z+aAy0!W6FC(z`7~{vvmh%-ZbOpGH;Pl?*Q5`kPaFsTaap$cgr1b2Rfh+x&(~cb`Rt zLr{X>2~W;wklXI0)PHnie#}|Cmwp<{;(kAH$G`FLvr#Gb;68PthLXFv5mLv2rVDLW zw&%K`{ygT$o&Y(fR()jXPrVO*X(;a3x2`31lm)$)@xwVf(=@ggPE6|kH1*vj8|hX7 zHWuY&qP5Ao_5~Q3u&EZy%*LPe>Jy@Byxhm8NmTh(XW4$F7v7-X_Ml|A7z#HWRT_Vz z+0q!eMb5Oj9l5&*yipD2~tqc!GIJ=u)Bs@g53CkNNKcR5EW$D(kRw2GP`jf#q z{So*lt77sY{?4zvBk5oLq(HfTGcg_SRpJGuU+3vJ{dEVreorESp(B#(zY1NssO z+FP7N&C#TDqHR1fkYZ(uegzDi!>{7l80E*hW~$fk=;RrZFCWtJv5sVLA<4VmC+T7S z8s0)HbL9$u6ISsGriGNRmk30V3zK)1l!XyP*dAii)oVj-*zkxj#Yx=F>r*?HWn4s_ zu{6i^bt_?sDq?pS^h-50sNJdBmw8;(cL6s+t1&$3s^MoOesP-oYegHP=yBqCaSV4O z(Bd*Il8wH8(G-nRXNGe1@z5@j!?i!MRnv$3bUK&aR%df7YmH_95=G`6I-cvbrnZXq z{)-j7KKBBdH0Z+7!+B9jHdy=PwI0ESN!>$OBTL}|wrSwGMg@Zokq;RUN%WPyIuwqc ztXem<{Cijx2To(_6M{)6Gz=dbe$*Bdg?x^aR)Jr zg;d$FqbxD=)SwTEqP~d`wsJ-$iAL&)8)Av%pUmn^(tnmlY8D9<&0>a?8;c*UO!XaT z>>$q6Jz2JRH<&siti3Ov85jF)is6*CMo62k*cf~8cF}y;vQ*O@hdJrVyq@7b6BRPf zRHWI5?jIe72f8)BObVU9FP)I&8`(p%j0Y9u2k5vT`pN{WWARd>>Es8J7>$PGWa+0yMf0i(|elt&bUl;InJ<^bTUQwi=O#kk7$0M$>SHnADt_^gsg>kT1 zbj1qIm_1zW;>W*mV^aI26NMR!|^EC{a3HkjCVgVa&Wwacsq;!_76i=!y6Ti$4MDAM(0D5=Qk zCM0tRllHE#_BNyT6210>AG3ZZ&abQ;HV~7WWNJ4ilMAxuHkIa*7t@{qjsr0gr@kMS zs!&n+^Z2{9u({@F>-HMNxbm{KvQ2_Hd*oMKI@tHcZ^~5B_*CHdc>i#!+)R|7b6P)C z+*wEww|!aQPMNH!^E3)Y(@xMaoC(l){I0R3mOYeKe z3?maVAYiq}oRwtBd` zSR@^uA5WZ=(p@uL`IByyJ}#CHH8I!`q|`@8@{~h%h>@D)g}dZlmd6K!h1v9ubNxJ-UPJUdU~d5(gtd~pP&dQ}{~ z4;)=2qDa0?AUkgGIc#v^br6NUL<>v7A$}I!Id8s+BYV^OcI6gT%$7u!XB}mZ-~ml6 zR@!$I!%f^E*_AMa%&?DMVX~iU`W&WA1M(!MgD}0Js4wuqzYg7ru7)3*C5Nx#ijJi> zhd{D=ASzcJstahDpA0fHbTWOkGF3iQZ3rb4kV@VNdn+K?%nwP~dz@>0ZElLvEiRV|e zY_&*rW(HGTYIg{2)33E7=SR!ETO8=Izb0QY?L5lf#i|WdCfhQa@yHG5s1=taZ_@2t zJw8xBIbEER$UNr?|FPd@6nkQcHV8l4 z7wLy#C zt)yh_(giO^zjwCfi7>3B>GC8sXM85=y|T#^}h`CHr6{y&xrN<}B0atY-+$E6qGB*r?)8=!_-w%n>!t{M==7 zKShI-V%y8IFg5+@=a`P4cE@Y1-QS@git(w_?vV|hXH9w^jVlx@y@DBH=%CC3rd5T|M|Q%Yo2*7kSg&WsJGgKC5Kf0BJUc4?kYNbM{Gzdmj>5qx0CvnM2Yu^s?d^4-0Pqc zHb~|CSCa@dByH3r9+=ACG0@V+UNJ-RM0O`guJ@5UB9S|;;vhSOtiD<6%Wjkphz?ek@u`ei*-%9kT%6a~Ab6kGjqQbV*xyDRqB zQyipwO4{`58Y}0WklfujV)b}92=oR-%G?T{{r1{cJIZ(3o)U-$(BI~<2z($tJk$cg zaMty5w}jBtg^<);gs!MMeGf{`E2R66m|xQvVn(^i-$sphw=#*#Y-{i-Rz|fN3=G?71C>#=J=+q z9<5lUw>&JjWcr$4bjZYlLWy8&PAMRfyttW~S2YbdnN-H}yi_fN%%^(JgKDMFO!(kg zuf+y5f&W#^A^VgSIvO38T#ajDPGKU;JYe}ekP20LY7&I_Ar4!gPKDe-GPJ%EQC|~# zDvH`goVmSHY~~{(rw?{2O`ktmytrGSx^BV#7JXEjc+B3%4v>huJ#DIVPgV)-btrAB zDec!GoWcZ?Er;42c&07+`Z$EhRwOwBHfpiGtJ=UCQt}?lKzR8xqc0KpAH&g>g)9E~h3zAKU2+PeDrhk8XhzT2L^NJ?fszQ3xpeO9J>|EBQSBjudov#*k7 zqP)bi(E(-Al{OaFGTG9bb2JvR-#nLt-Tv8M;?NWWYMge+G~0lvjg--SW+}G8So9P- z>ed3upnBs;^R_gBq>lNX>7!Uj+e+IvS@&j-009qiaw7(70WCX&&KnCfDG3xNX}B6( z!mjg~u8D~T{j?zJmTt=>!%<`dVHV$Myaze*YZ$oetDUQqhUWs+0OqE4e@ior^z|#q zQO3!UgXobcxsg%0qp!xt6p(W%Tl6;j-&2CWASodrv3!a8U5fO0T!8Pv9P>js;sU?4&+FXOtxsSd(I3jnq@U zhS4KUcyO-Aa+lqO}M#sP!>|fnD`U%V&6r&v#R*yZ&_C^ffmiP^rZ>nsUNF1kKb*lteyuD0# zO}|9H?a9AyNx4)F7Pe1(BNhKF`Ab#AUdW{J8aL-yOf(T{e;GDF`?qp#iq)4t*_S1S z$rhjg*uizsQT5Z1qtK|9K1fjWoIYhex6BAjKV0SKj$f9f`{TX>xY_$nyzWld0Cija*-`*6pqu%*%|B;J5zLFRedn`al`y-W0HErR-!|1_1OyVJxJMzQ~ zf%W_Q^?~vR3Vf)ma`%e5h;xsnhMG>bSto21f;u>(9AyCmeJbw`%7EK zKqZx==0;vUVpe@mQayV}eTH*6yKQ|&kj|CTyF_bxb5xaf!MD>fZ(;09TQ9zNiSA8#m6#xKRf)}mrm*{Bh@@&I8YvX$|$GQ57xP{cJCogvr4wIgoB;Mnt)OvCMW z*}b`0LB-7TxT2S0%B6yppN!&FUP-Yzy(}F)k@Aur^t&7>KbL||F4YafqSMsU^_sWK z3f4kYTy9iz;;9*t@2{C8gkmL`BiXR$HBKWmHtom0<@KFr%^?|WG(K@xz)7Gv2S*O}r0f8xW9$wCsWBxkBxQJ)WDY3m5$YWacM6B(j&A8nIwr0&W_TV|P9Vo~5I(N@K>8!^ zr#POtXn4#XctX->O(W9}E87vQ!x-`jl!0VlqIoIHVksgW$&3eKMI|}WYsDhq`N$}G zd?Pfiv1;|O@?9|s;335hc}3vya6gdMQc7sYqHFMofyl#3e?-kNq?G z)jmN@6Lsmv!|dADr0FDdWo1RSuwp4l-tPw9?v~N)X3)HW7F{14fWVoN^VKx!g5&Ez zGx8)&J$z%kLz+Rk?cFzgl$R93Mk_6*R8^tKa;RwkMNvsUNcW2Py#%wpbT}^_;igPc z|L5#lCRnnGGN!8EvCIFbmTQ4(;!4Ar00A*PR5B$H6sk^ZQ%DFPQp9#bAUp&D3MA+@ zK5DKCp;=>;j%+p~bD;Dptiu1>IGvMXIfBvE6%T zh$aMEyJzLdRQ66mRYD>yIv#lxh^f7-?mo`DvIFzwMoKexA+S&6Z?UwdE z^QlL@PF0Vn*IiiO;gRn$($iV}+2uiAXXY}h;4pTnw{u63u0s*c=nUOd&y2|FedTa# z%u)Y4uROBoWwh(rz%bz@hIIVe$*rsJz7_E6Pq+VtpV=Dewr9?0JA226?5;QbymwId zEIFSbIv2b295-oDH}k0F^qBVc->ZJwjD0&;Hga_Feyc}sgSYRXYn=E?_Wrs=Wm|FI zX`hjR6Pel#E>S^&zE_Hxto#@ASiVu*ImIdU#Zl{u2k46_c(;hp~ zgeyS{PDfYi=JsxLn=w1BZbk!rZVUa^`vJG-)NhpQUz|0v*E}q`RDIYd*uCX^T{3Oq zh0&wOJqq%v#~PIT@C0h$#pTNIWu~1?`?j^{4*Q&^HM<0E#v&>BGx7B^1D5+Yg{1FE z$>i;Pf>Zm_o4p@>Il63DRX|3g-)q(v!qW6%X>&USVyYhF)JAT7NakLblU9bdJsp5^xF_jlQ2XK(Jw7=D}nb52#o+>Z0N2i}T(?#MfOYa)2) z-tzWe$Lp7IHdp4nQ+Z~4;l{LR{xiHr{)TQ{^eb||X%hF?Z$EYC9ZYGCUp^*eg*rWrCn)JyBTcK^w*eb=V` zAYO2cGODz1xxRlWY~;>f-nN#jw-zsLj#B1F3@>57wP8~$E%nHO&gVoYvbom-!nBMh z^jZl<&t~dExCNFMRx2VIOR5sKt6rnfEBM@Qq-XS!OMKQ`T0;W;GkVzgg|ezdK={TSpA|*%rx_J$l~%@> zM`6(CQvsQiL8et%bcT}ol%*{2yMChA-K;Vgi*%*-eKe)YT%=Ihce(5I(o!I9-{)1L zGL)(`QlqZi;5ge2aIC2`_H*engF;KUIX{yjW^m><=r3x7^-+5?S zp-QWj+6%Zs=Ss`9QngB^Gy?ZQ+Xt{)$u!bQy`FNJNuib+iz-#*G?yZyhOD`p^mhHO zMw3cU4tOhcTD?JKG)fIJQ;`llg#h*L8ktE|s;Ho#TjH23=J-sE!eY>wQ52dl({7;| zbTE#-grMyRmWX*cGxjl`1iT9r1{LyS`GU)0bo3U&q%ko--UlLNuhkYPQs*WF{Q$x4 zAXp~?wK~)FIcskImO8cOiExvOM-g-*6+7mFjZ&fFsY&=A%+2QvJ5cL6HLNYANZhLk z`U(}h=!Vs}LEI@x_Z~>a=Zw0yZ`#zTj!A2eJ?f5aa3`>*V%mEk|K1G``fO4i0gG=b#`|wbcZiV*#f#2#>KLtS7-1E;?gP^wh{qf%i z^_3_=hDG$1OO&5KqMMLOi~Z=;QNoJPp?3aXcAbLECl( z4Oj&u797ul)A)rzpTG#j@p3pV*#ZXn64*|KZBZr=7bHMLHJ*Uu35B2yTA&>Xm8Swl zIG?DBV4E;aAmNBO95{flB034npx{JFf>*gPk(-wrRjeYskwB}2I4+r(l8g&P5gZOw zpP5X|Ae>2Dk;zX9eKhZWfH$Y!P6|v%fz{k^6@2;k$}rIzPr<4QO`+Ln(A z6Y(M9XBTg1I=>`OoTn$SQeeJtB=7?cOsvmj|H%EK&8YzU#Vjur5c>@%1;!S_^9+m= zerLkNi!2lPAc;V+!zc1XtI#Hz^2AEBLTj8Dw~5BZQT&7TGDBIp$`q>5>qBJ*gU%8> e@yZ#@iwxt2alzXYcpLpM{wP7kUL`RU5dwr>1*D0D9(pGr0-={sq&GEGX(9+BO$6zRfC5rgnsfnC z6s&X+REi2H=>1PX@6{{!zVH97^{w^I!esB+d(Sg7duE@Lb4DYKi6op1&Im{8!69%E zGYBM5FaDK_5(5H}fj}S&Jw1Im2#$aYh`@(9foU)}j6)VAKt>us?tC&a0rPei(O7{% zP@pr!2?h^wf&j5BaP%)?sf|G(7%)RZOiEfzVlTn4-67=n?|&1{377aqI7281L<0yn zu)wNgFu z8}Np|D?!xP&&R{rF+g11+1c0B+1XRnLRwUlfcJM3)ijdZODFFCxC;57r~I3?QgHQO zv}NrBf#86)Bn|xm4K1WKef-q$f&O^H-oXyvb0%_c_P=R@gbV$m2`wH3q69Q?a3T2M z10-elau4N?LFAvUe}kZhbNvEAA-|iGF1~v@9}4i1q>q1-$p#nsMJAm)2t);_4J@;B zAow_X?xi=?pQFgXn*WBf*Ab)wppfV{03@Z#&HxDXR|^3W*n|6j46+6cx7P#C(?3sI z+z~Kn91Ma0$1Np@5yS;j0TDsF{T&ObjUXV$S;=msBPHO4JTSZp{L@j4q}(~Cftg53 z@OO`p{O~*XFFzQZfc!Y7$?{AD(AuBsuj|zGAObTCo6!l%VcdyoQ zHpchP3VK91Ii>4~Rm}GDn%l|_`Z{(e!NvJ4lxYPEDZ2hR%4rv_O|8%M?TKSetw=#M`bcH)5* zhzERfS`X4OnGljpFaV-|Co1;AQ>cE4iXRJrEci~O*#DjTr~39|(05FrS>*S-6!Ozt z@t8z;fr@)M;S&AWYq#a1M^c%JlhYqLyw?!mzrG$77tQ3B>UZ9H@Cb6Qc+8eo-3>-( zVqlWgL{XGi_cSSz@_qrPAL}8xVV{w15iZGC$SSQZj}@KgO!gb~5Qm|pqhW&l75DO^ zjSZ|LZ=;{^Gc>a(K7`>EXE>(e8&Y{hxS2!Dod0Gt`hl!J-RAc41&z#X_$4*}q# z4*jZ!AWy@<2<)QBbCRHU{vpYT-ve-uB>ORhr2Nq$Md^?BU)(>Vbgv{U0K?m?^x!cp zf2Nr|BY*u((3kD!3W|)3sh)J~n-nyPbq!Zr_dV6hU$B%YQfL;SAv&bEK4CNEU5ra( zZIn=x9;9P(q88L`x*@E&D8j+WFxLvrg{DCr(<4mzhlQy4oU>U&WVcrH+80(Qm`+s% zW@&RI1rVO!aG`vQi!hnPDIPGYf)Cw@#fdtp3&$KzDJfoA!Bfk*pNh~{;9`2?%+`b% zUY9xIp%6G4TCYPvaWNMPX-$mlrrpngv;7(#_>hiPzU{1Kr<0VTB5gBglxQk{q`57* z8oh53)!em^gI65Lh_@GEZHbI&uC{cv0I@`H#=ZVpvD(BFJUvD;JdDKenj^F8CohgV$H{xI0Qjn>Rxp zyF#DO6^)r8yLC%O$@h1p-dJ0ab7$tyy@s*9hzjN4c=*b@Ms1&TFa2Q%SsE)u>6-_{ZR8@ zvv;>!Rg$f4c9=Ck0eOj!u;5<^6WQzs7eOy$wd1vXX%~-M$iErO7a&88I(7`mIdzV7 zX?gd&DNpi>S`njD>sSBWaa(uMO*@idv8D0j#K;sGczH3A=}g}_<;>TKbtV4&>Posp zTv@zBvcpn`YFHdPKDnnj)5_@;Tp|dyGtPj%*rmYpQkuv8nWmXIqCad_L0Ia#&FYne z2UN^^2XzyVVHmTyYaCbohLyBKDt%+Zs$#u<#49(4hD31t=?RYYDrw!KwdD)1na5NQy{TL}ZAIO#|ss>faUy^ToDBO7fIze)r- z?`b2AkW2LZlsMy_WpmAGj|Oa>DHG^cZ2ZSHb3cA0ub|PC8Q{r6$Gk=G*ZUMse0e-aU2U;0?Y}WP+98Vxp~kdi=2^4G zJOj5_0P!E@#!xlJtD~bk{A4cQ=DNs0TkYaNiI-gvs&!H|OR6D#)>?g7VNFC=)&MvO zd{z}KNWpxk=~Cii0{-DV*fnF`co-!sdPTxGantJviC z!tVCPTTpu0e5=}?7l>(FXJ5{r%~dd7dvzEW_2@xr)GLcq!MW;KcS&U)>6bvxpQu9R zQ$kqfQRIg3x8gqbt~x41x+w?tt!_(X3%7n{@w@f-%xn;ic-}&T z_{LRlADE-LJ>?hCLi4bc9l3VvJc5cXE~DuTH65wf|#PL;#OV@&X~A;)EkPiS61 zSJSvXxHl|3^XbKs1dpgI?YZZJ;o4f9~hjgJ}8Vfm&m-Af@Z-qC+wKBb7zam zXF>F?8wjspVyCCe(zV?H#c$WPy6!tuz+e4V>WbD(Y;@9`oYy@%ysvL?r&$!YX5l`M zH^343bZw%|6{+&=(Pr|=p%Z+XaWWB?UpJ}B7zKtYAIvTiy6f~=$&)FT^^N@HO9HF~ z@5pn8Kg9ch`NpHP?#+M5Gz7N7ecOu*b%fUpwK>?HFxEk{`dUwpp*6&uDYQ7ho{tp6 zt-=cqUVYhMvBtBVvs#z?muttr;FN@9c~lsWcure^C*Q+Q7Yw_&lU;7GZuG22t(oYd zSHCfQIBn?daZlOrTkxl~BS%C;9Ev%%t0-o}!>~_Uw(?)*KiF`IIL8(zIq>Z4+tv4v zP6Y>3M2jWzT6|QLG@+lUsk2-v)xN9@Tv`nTPhFjVXjtZ~e5tvqsfj8(6eD^6<^8i~ z7u~hAj%~L*h=v-5uhiAm9g>v1QKM`UxuTr)wUR)f9q5@PB+NAe!h5q##fGTwDI;-&^Y}w8BY3r${caDTTPGgY-yhdWTklVGN8uWax=Cn ziJa6uChy1Ao_2W3gxA!>r`@UYWWUI;SVmo4JKM<21I>M6Tm2-wI(T(j99%~6nziFK zW10)S;@0!N2J1slJnEl{KCpGFMEb7DJiv9nsK!< z7~gpPK)=NDqD~2p+8p|zf*$bxTjJ$ zXMfh*J6LA;P4SO$13e1-j{7>*v}l#ZPVuKEmeA4bQG`$7x`{Jg?QK_z>pq4!bBc%z zcHSQMcs`VK>EKDByYkd`VRTzJKg{r4J@rW>DvbW?4ca1=6yed;^bP&uGco;8i@WLD zclx&PJlVdpymg0dmfT@5?UmE1R~sk&Ls6Tz1BM33E=H`L$U5I2S3yR`>9V?Vd)jaU zp7?D2!8;`DV>6z28t0Y!0Dd=<_PxE`U)Es0gyq>lgaa^2e!mKkIcBHtVN`AtIZVj@kp!y@U=~zapXYVyk1I8}6zK)w>w>IR7UyFT7 z3{`uYnpuF?&K8(_c+d$q5H9Nz5~^^Yt~xMFbF=-W?~pxjRgfV&8>gSK0N(W;-9~Z& zYG3Nh)K1}t+nZdS3ZarAS|E3fNt--f1DfihGP{BHWe{T7Z@15oo0mkzWh18ABfxEqfh=x6ueOmZiC)joqq$UwU%q@< zTpAl2b9QlYc|7LqutT=iBw8yhwpl$AzHYQOrG~|Z0vCreo-a3xr}+3NY@|%+b)(e8 zsI|1z)UH)l7O(c}46Hdd*@*dvWb2{R`<&K_aQ*ijZs0@^iir$P-tR)f{Iz;~c>r<* zh{qAdQ{-?_7;R81(x~$|KPwA)RJYevSli`EJskMU!4PVt#m!H+P{C7ZrciP zpc*X`ZaK~HzPdJJ+dSl+QwsdOxwYRdv8vTTSMI?0RAgk1!6nm#M-V*g6I9d z-aK{6>Mr5dd#k&ZuReJIcPfwX03#l)@`-z#*00I`1Tvt#@a! ze`_?+5WQbszH51P1|S~f^ErD7BrPnu-Cm+`9Tn7)xeRgPOB=Zo+TfdNB+*G;A|@ap z;1-C%2=Emai`3FkS2YENU$+RqE-QY|iMgx8v--&rJn1%Yz#Q$d{k-`;;@ml(Zr!{@ ztr8P+F8qUHSZ9TqB0w$j!j}hDfkU-PURRi>_W%w_I7xPS%&k( z(#C<=!(6U0&1GvoYbOtX<2Dt2*T4q_t5 zRdC)vJI^zq?fUxg@VPgqPMN0E(>IQJ)K^h5xz8JoTz|~TEW}_o^}?gXWVM`Qxw|41 zW#}5Hpukk-ctU|h&2P+d=+yR!fBkHxaG@JGzv{1t9Lvqg$+=ch$OKR5)s+mDmagOP zYZ7)Qro^oW)YjG3)(V_QM&Di;9#Y;G=@}lnVslQD{Bt1wX+*g67a?Kc``CeF-REJZ z8O?WncyDaox$nD57GpQJEIw=vEw6?sVz`fLhw%A*_0QAiU)3|5ADR6y@MJ&KZxk&-M%O>J85D08{#CdHxB zocSZK*IZ1b_luR4jz|PBo( zc;P!~8~yx=(KrT?jwOOoZqylLC~n`pCF9bT@N(2)RQ??TOH2I7Qf9kz=W*_ngj;2v z1};a*pXqZL2e?&TAyDg{yHP7YZLlDkra##%9cG5vKDnimhEVGW~Jc&E^k8z}$b z)Cr=Hvxe2H14gp$JnhF-c!v-3b6oAIzSvRP&sTeYBhCJu{5ONZN)bIqXj(GdyM*_q zyU(er&qBJsQr^#?|rT6V*I>%+P+^+Xi9}5JXV&k4@WD-|L5H@X=JTTv5 zTcxlkt3@yAe3Qc0n-YR-l)ctn?5`*+CnfU8=2Dz9kb9Fb;dAO(>aEf+R{{D+4wr(M zY}ePF+lNA{6HDvs0|@_k;I{oM?)g)zL$%!JdV3z_E=e^oH!-%lKABW1n^2S1&d-xPH(u70JaMZ{&+i43 zw{7Oq1-B!}7k5o1N_|_s&!X8qOYZu&u(p<+SZk)sN=`0qGJnfS;F|UF6}Zqqe{6E# z)4Qv^bjwV4rU*Un1Zap(4}Xo^NLO}wwKbI6Rwq1!0H3zPL7%iQu+wczRtFu zNIm4zrbfKJ@%HUw(QOw!H9s%EBo0&W9^HQiHzyB2OMSm? z1r@2kg|c&DeKv|(b{R)F9_abnXSgD0OI)8evg zIa)fJ8ESDKjDL5K{c#sf0RlpkDVxn2{eoT3d6e;$z=6-N?ZfFW7hmQke^Obtalbfz z$#}Hz(1b(Rd>)9#$&CVe)v%VERubw%r_Rvn#+{!>4-!qtoRwdAbKgVj!LzyhE(1u4 zk4Lxq2BT)Ub2lX$FT=jjx*rT-mU211COLn&`5^nr^A?E-8ZLf04{T10$ol40$)04g zdvw45Eu>y3yr}CE8I-G4DY3zlyQqfn(el-eJ4@^bFPf=;%LVnbs%@BEY;L$YaA~Qn zXE7zawBvF@>ap~i@LDp*tJ}q#Wj134AH>4;nPl9<=AF226KK+Wyc3j{-R2+wKkR@I zZ46MCBG783OZh}$AH31oyftBCld$?Yh{i*x@nCopBJV6kMC%2cvz0Td4*m;I3xc*E zgbL>ra$5@&i*jW*H<9P1_y^EP(cIlsei`|pjjtfbAV&Z%lVzWDR=)^&*5G2tChh(u z76vPims6}98slj;y;dv?{7M{hSqp4-6-Ii>SFS_lI-Ksmc%03Bl#T7u%ZF{K<9RMT zpP$Yg37XMrNiVhrz^m<|ebR@6BaS0wN($$r_^2)nI7(5QrJi*UQ+(w%_B!qAjmC&&m5PN@)wOl% z$okg${#s$TW1l^OB0}5yhO%Dsd>|k$L{{~?s;re$UC{2Wqsd>0Pk4v=!d=DAR2R3x zd)WOmx8Ioa6~|I9KQ~y32WS%7e&k4;~3hY8fl`2$3hReR)4mg*UIN zy?CvS+3-@gDOpo~4OU<}F`)QFrW!hJ3C^#X?-v(WZE08G7!JL`XYb`3QY~~TRS-$%TL9(URDrUAq8O!#5L7u4CR+ohm9r!k{Zo9Yfq+( zY-b=cl5x{wVDc*%hJX<$ilv=WO9H}tK9!6Q8A++5-NBqSLdLHIrZ!IPU@12u7q$V@ zShcAl>S|&|d?RU&Pe$#(1B#P~$J06qM6(VN+bOg6(_LrbDa$OjDpDgHsMf)>~ zliUnp23i*IiO9{f9mfFQDU`(ZJ1KF27(uJ0L zKxG&KVJT#c70|>anq*D0lxxQdS+plwG)}{7;$uZzFiFeB&KC zW(Cg358>$piq#G**=s32U!@P}QsZ39qBydMKNmj9bMVVJ#nCTQ=T3uoxge~R7;1;q zXf9rEYAZ@Lw&~O}D!eG>Y)TCwhqNSnULLjyN-c%yv~xsWUKCZIHmwRXG-DP-DbWXl z*`FuRBLh=Q^kMaGf-;Jo$c5T5v=&j(>@~%FhsGlrg1DduH|=3claaK3T+oZHWN1k% zDqRPz^b5UCXvN-0hOh!=R4UXbk>}R8VK~du@LknWjWEw;rC>AKAsLDVc=VbL9AtWuc;pNJ|@Z5-?>Quz?-Q zelX}F_mFis0w!I^bl*+elx;L?Ub>t~-@~ky?QvbVOm(C_!RZkD(?{L14ek2g0k!Ne zKXxC!3pVhz=gFI5ER*Y&r}sNwls6|)CO;HPACTgox1g=1@URfLfAl}_)=5k8*_1(W z>xTm?F*|4a1**;M|NM1FeFI5vo!LK{3T3N);kR&+j~t)F6K2 zHY|%TbH)fij2_5+Ss=ab3zYTUb*SE&Lbjflkx3V{pUjB5yuHu^VfO{{-xo~t$n~hS zFBRQldd8RWGUcr%!;_pwm|JljFuvCPqaex(Zrp_yrmS7NrYA)8+5Oy#I zG7_aW8LJ^?-IYWME<*OXX-Z#BJL~Pl!!f6=rI<*2&JUc#`Dp>6?03(JgKJ4`aGV)#V8DmXt)np|xC$R`nt z)pJCdW)B9SrAx5-o(`rtPiIS4^U4$#LidGF&gRRV)G|1Y%1#F_nDHyV=mT+sLHWP| zo1Qhx%P-%DB_-4>)-cRLg_^RCUzyI$5c4Me@;9<_6>z@_kvSt0G{>(Zi8r#PVk}%Y z6Ho=08^tNGTw%N*puyd56mR|L3QLKArcA%_Ilo6mE6@sUK5c#jB9Th3VBfw+To@{y zRH{a<3bd26@bPwPiyEm4tnpQo%x1F^kw8H`a}SfOeinA=dO?H0e$&hSjOD6uA>*@& zW+hVQ$F%vbnx;{kSD1`l#f=GB-ln#=ael1Q&0P4{3u?>ye)#o(dSTo4E0%Y@z-yxT zD)CI$aNP=sn=Tn5&Tvnw6ud=k-X{?^4PEPpS1jr)1F!qo4;-EteS9mU>$>+>nB2?J zo`yEK*om`AHgg{7O+&F&0TrId-+EkZc}ZR!(&uUW5tX2=jyj*3I?MaX0R(*;SRE!= zVEgH+CAk?Rlw&eUc(ecFo%MUwr#>&)fuQ4eA!v!ybOd_{53Zf-BRPd6r7Eq|cspaJ zM64(QPoXy6&eAOrcR+-e&L7w5!5ni2BjSKa#KrQua8qj#99XU)V?}}_lb!V(*=KM) zO6!uTv3gECJgj{>E>h{2Xq^Rgv-{0%NoCgSX^4fh4mzL@W#1#XNEKP#Cm@P)r$k&8 zp0N&x&DUNAi@K?@vpq<3sVh1_=Z-aH;a=p9Z&gXYN zM4G%wn3f!H{q&83+Dy^9Hg+7KKapVid>UDHvs=E(IjLx1#*9oJt|8`kK9^%o$hBzz zFXn$D=NZRqh1X5P$LRwK_q|xqwp4g{i9WEx^u;3XiNcc`?*nU+F1>ZpYI!>GKB#fv z()*yEmY1L32e;AWuf%D!PO~nD^y=oXW%jhr%P)tHB;|iB)w;cCwH!7+0KBE`Ir2WR zI{f9*%Z)ZIA}ppFvX2r>uF804MMfn;^KMg%5rn3|9VFVt6tIcW{g2 zf%1m&)X5c3yY1P-N56~>pZe5t@Ea)o?)I4B>8%;NKenIT{R1o^0cPR45mQv|E9B&IK=qd)3j0torCq zQL1ZiOT?>kpqP@h)p@PMo<+_c5n{u85OKv@i(~{@Hp7R{sTIe~t z&XgL-GRs9}#X{?558W@odzW_Nf~fRVYP?=CYJXp^roviUYH~3zKekWDdhLAHATL^` z32P8EntqwgSwKTxhdb?NMyV2?kZDD~`OTXbsv`JA90dogMsH@`YULC2uNXMKwq}3( zXM$9Ov~1D7E8~*A25HYkX7H^AU$WSZeHd_w!W8l zdhUyl^M~764Njz_=gmpKmVc0KAXV;v=`Ep5@fDR}NVjSJhhol6LJe(HFnq2OtAC!3H1AQHaSo#>`eH1E-l{A(ak*o zD4h-T`$Y(q7XfeQ6cFnB3zyzA$j%WDsQ>#Am8-Ltm)*|`$VkR{2YT7LIpdxD0rP=A z15mh$C)oXsOYMLUaCEctcMo&kod$LHC+xN;0(^hsllccY`|b|VJNkI}`Z@di+xg)G z+%0UzM(?HCF|3~{n@ux>M>K^$;eRv8eCk@Y7>!)bwWL?Reb1x9P4DSt05 zXB7boZa{DdsRWb}_){`YF~-yxokK=y|C5k0#Yi(o_w1YJsw?j<8( z>{%xMkHzD^qh`&L?Tz|teEy7GM}dC4Q|P~EZTLS{jQ@XN?`FU*W+Mdg9D;sMg`VAs zfd5?X{ohrD|E^8S8pYn8_$B*xm0AHKG9c&zT67x?X%qIY_WtK0@ZZ7bqsaFL|0~0H zk*|Oe-C%SMJ^B$H3Hi^u?_Vmwf0w`V@gL&%Z)e3WNDLTJ4n{v_K<6-!Kz=R+|6TOr zv47C~&!c8nXgL@$2u3F|p*t8!LjNlK{Db)`d=Ev` zK+)p}^iwzh2vp8~Hwe7j0Fli6o%`RLNjlbdkp?LcDKK;~8#AMT}BW7vPBPjGYE&%HHqF@r#@7(_$>Tj35 zpY1CUMNf+Wpu}Bg0)hCWMFRcN{)_vcL8-r|!;j1AcM}J|h%2#RbY(mk-75wF{{AqI z1o%7m-vJS=F%aOgjHX0P*MdPnSkPijObjg*cJG#ujnv{K62_=wFc@RtABl7^m|%>m zD@GH8amA>)?(kGqF&KSPOAUj;lD>idk8hwuY62s{7>o&NnyM=xPu(9_jtL}9!v+I` zs#c`FE~#&13@lgIAR)S8G%y$qSD+8{fW9k zWf0_mQw_Do8HC{u>9{#*5rP1H&BcRu@G%&3(p*&yahLDX`Xucs29GoF$7rbo zOXWyi)g7Y^)a7^k8frUAs*)T++K3QTK=SuCrO_`Cre2e zyq1NVwf0UZ0$UeM+F!s=JJ2K@cJ0&l_#sboZ~J!Sd0Sgb2H*~v$T(_|T&LoN=lo|5%aMOa5*B&L;j8UpuC0N%~t^$a^|z6M}(# z1v1NQN8j)MRaG@1nev-IcZ{~e>3HC^2tbS*NI2ni09=5KcGd^~ZT;Wl5y)pCKP=@v ztu1%<+mSTi63D6DeE0`@N&0C85FEW^2o69jI2^JFwUXNLmp*B|``^|B$p+Al>vmc@ zp`|4sjFXaabFehWIFaTb{t=HF-}h%{|9;wUlIp6S);Me59V{JEyog(oK1c)K{quMG zcJsx{A{5A%AO~0VoqPj?0?N9Q&NEjMDLz?~SFSlQ}3GIRXXZG`^! zN$fw5{@W6W7({iam;kDOX#H#H z|HVxHf3IH!f}tl1MBhIhO@kiSY5I;}_`va4u}ADa2k*1t4U%?<%@g!vb&gJ(hlz)(OyP+}lJz(`1l zQ4_&I!AMD2SWCdbz}Q4ZM0Y^J!NkNw)v7=tz|_@@O`bp?!Cah8TKJ&YhTb;U?TcUU%o(KQUwM12X#RL z6$lShAvREnZ-Vryx=5hv+<+g*KrP`hRKfKU)9Po?bG@2qM5)*Obdn zVg%dM;MieyP2D|0uN{1=mmO&~wD#(m|1+xgdZ74=R-)Wd%Sw&a^Ftlmq_=-d7<4R) zR0^mTKoy_?P?=-j^VTCw*hIH;(byB1YaaQ^sSF7j9F%49@tZx@N0ndmXT+@W17>)B z229D35_N|>k%Y@oI~_zv#SO>4-5KT`2GwoNs1fs}@Iy2q@yamIt)D^_1VNXOYzxW^ zu3^SOX_tqdNw`hVOiqeF*+f-@BFUuZvt z^3btSQ&Ay5HCxqXeN-PP+B#*~X1E|7Ovmd}Th?{e#RLc< zH-Yc?!v*k3(>MF}=3Ewxm>Ks9lI4HTvrS`Q_#p<5otpxKl(w!04g z4jyQ6{}ed=+qM1!0{^m`zm8s!>E8qXYW>rKi2jo?vcWKFKx8q11pn`7Z9E7RkPmcG za3B=?0S13q_bWtT0|9~v_$LUU0x$u83;qch(14d0q&L%l2Q+>C$XE%E^b!58g)?j* zf&j@KqniSw1p99e@b3XYAo-WPZvQ`9|J=S_F0>94U{MMiRYGrhCC10_2gy?!eunK~MQT5uX7{(~_t3{P?b}^=CnYJ);4CM&Gu~A{^%HnZn9(J5ZAT zJcI=cGGq`q5F`)LP8Fq$ghrEMF|BX#m3VmH8e=Xr_U-4pAL+SbBj=bC z3J2ZKChSM0k#kFIV}|B}=mz71EUP*}Pj_&iw92GOfOyx(YqZ^vs2iqI1F-QhxM8Q5*_l z65KHt951(}CZ>j*_`99F`<-88DVa>VE(2rZ+c&-{>oT8E5-jc``l>?MaxXQ03w{Fc z0J)$Zpf(=H6vCfa)=9U#9j?wtfrqq&;c#(@HyTW7_?WT=v<<-=OiAnnsnyYoC5 z6{jb(^^$;Wx=tvGWTzfuJrRy9I8Y!>G<$jYk^GKXarY~A4Lo#!LnyVRjZ((+;fS&uJvkVGcs~M9;)Oc8SX-iU zLp=arCgfp8!zSMe=SLr(leW%RW z-M{~lT*}Q=TCMLgq2U$gGj0+m!KOMwrZU=X&PAl#85`KfFSwtftr%TjspTrnf=$Kv zT?$l7ZD{%j8>Q(8rJsYCv-#94om$bY3Ms1dPY1%{qxl4j-Iky+g3YWo(Y6~Hv$-kR zmgLnHlhIP$#U8w1*a&3B_dyU!?k?JFEuj~3any4sRTh_>yVvfm5w^8HsL(0W=IagYmP6nn1PAkC1Mh z+rt?;Cp|#BS%J`4Oi(=Ta95clov|ZV!EvgY*0f=%!BhmXid*ZX*57Tymf-#h?|Eyk zp19ayact=}EMi;J8OLhNr3Gt+oSrT%i#zH>nLBBhQe_W1n^Vt3$(_@S>+1=QO^0#? zFzoOxFPcbpM11#-7$r8AY=8+;@uyJ95g!VZA}v16V*a={WVKHi?S||?zJhkdd-rQu zwwN-{49Sbc$|r6ZE4H;8?C#6ZePfLep5xtJ1~ho~aq>(^x*IC7phiri7vJHn40s{O z&`=%-Y%^kcgx*vuV(=4OyOb@?8kImR9*2-K zQ^8}_+yy6{$!DU+*Jxo`+47%HVZ za6G{r?JMCFJLEiqh6A{IWO2OB5R%3_-Y@*rLUrg5iZ_^1>nx2@xi2OJX^jAMoWSxJ#Sva+##LCABWSOj{S%^a zrX5EaNi;kzD`eH!VhI#4_XQe!A7nr|&HqA~zW<_=f1u2NpZY7T|9de1@AH3h&OcCQ zRW>L_To#ZtgUSA6O+9RhXMT{ynG{=YWaQBq9MQ%4uYR)6a)*L(P~)V>^8WpDP+t6A z%gZ|5%-Bp!*eTesKP9324`l-U7W@Wa0z}}%;LjlUAa@}>QlY)$zjkZn3HJ@bmIc)!SZHiZW;7@HOnzdoOLuG5{=r;ZQKc>ImeS7_fq+Bjt#FeQvCRFOfL~_R&gcwR zO*L9vbHjGLaf$_lV+PWwB~8?P@sxRXNz_=4yki=|i?9+_c=n`TfXt1O#I{H%9G zAc-&91Ie4Ws3RZ(?an}sF{UWjy)l-G1>}qVdczm79m#;0q@@OgUN{VRMc8Fb6}Ej+ z&`oQi{75}0wMH;slq67i|0NanZ8MC6V>VfSX>H5K6b&J_ne(ORPAR0C!K4 z4s9F=uO%cbq@Ks@ujAk*3mGC*H%l#Qt`!8I?J^8iu}vmKQ({bO63y>_W`yAk1}(__C0QsIUjSpjE5F}G z;*h_tZG(x+rtX=`b-P^FTsx|K&}{7{EQc0yzn31869GP254 z)3k#kBpN@bA%Y$aT#KM1RA-THhtrD;?!1VpN#}(yChIk!b+oFK2eskIWD=yp11#$` zkr{@$AS&l>n%38z;+>9T`o)`iYAtd1kR2X~zpKObHVE|*4a1dZw;j0Rx#l#UnG1dG zgJb3#Um=s#?Z1}gvyQp_$x3Yd34s5@u5M6XH@M!zfPI{J2mH#;>`j3os-Sa$RgOJq z9vUKD^DE7XGEPz9ESmw09ma`!6ZpKl@aXUz1d4%ywSo-GU+K*@h})T*P{;ebsN5A|7Q7 ziqqvEd$iwFOuVHBIZ8Oe^8}p>eiHm)_5hTzO8fM6!3R9#JtipB9*V}+Ew^P4#VsfG z(n;!f<-Gchnule)8~yL#{ap_ga0S|}{k8VPp=0hPt()q*qKTx}vgHe%C}q16i3&G? zBLzi0TS&E{c{S;NokPp*?SzJ)*)xGb0SL|HO{K#+W6QSOap=S(x!|7CNzlfzF{9cL zu774AbFyVMG(gY>MN)51y^SU=k zV9iW4Ap>B4rtaWFuG!UK4wyB)IF zu@ItQ`=wX#!Ha9dGKp#-!I6?g0fNBYsxDuz?YIlJt*J&G%N1M0h;%vkV``r(W+`5_ zjl!ji@cK{%#^%c6S@u+})YOCx7uQSO{-Z_;xx1rbFrF+Ru!3HL8cHjt06I!(JV5Ck zdrQppw%&VyPQIkwhHh$u6M|-l!{>4Yz|VD%_2#!`gN|CvI@|ePA)61DetLy{Us}!Q zL4k(2Z%7In!V|stM2F6L2s^ahjMOgbsFMy62R6 z?d6mhPR1WRHx` zRzPKi)uF1AlhGH^f$VXOHDG*FJudhAUZ2spF?3e&bS z9zV%~q@Q0G1PdcmDS`kzpVjURM?qA|#T8p9ImANRZN>>$+)j zoMz@fmv+Y|=SmUIX#!o+tM-zS@(Z&t#`G?Z@A1c4k_{WsX1tARB76F(_toWh!;QnL z7|TTByVXSgJMIwyKge)&YQBusQJsPcmJnkK{)~M+(aK z9SC!L3t_)fC+rmmM|2`M6eP;vYLBi5_eY4tqUnQ``KH9U%#GW?jI1nvR=ousvzC3o zZEj(fQbIkUz}oAcR#XNGMC4Grjw-ge1p6(R=2=L8%Mu)qWR+?ryYh6tZ zr}5|U;o&z|U5QjVvmoqY4E6l5w&S4DLGBaqx4jiX_H+zI6D;wn_{qoCi|8Lrd`~&IOd@r*as4ZFhPYPd4ZGI(gQm=} zV{pjms=Ti}Z0U%gwY;_@ph~dXs%qO&1`^rE&BG%EZ;|i?^V#Vu3lq`2c=M-V>p(!< zVK3|ONrlCxXV&)j!Bb+k+g zi)F&V2C25(DKdX6fbYii z_clG=yUlx>523YW>FOpa-qp13cxlR@ppJsJK1G*+W+Ac8oL8vRYNP#B#&B!YET0uc zk{S;8m&a-h6-JC@WD=mnqD=b04P=|A^pn@12*B|RXhb*QBWz~jt8=MEB`tJ4KN@zL zZf^7gV^9^|t9S%Bu+#`R#`5+tQ}VtegT^vRUx-&ybNU$k?IjqH5b4|Ql45^@hz)s%@UQhl&Z5tk2dH$K=#87YVR%BRM7^xt!8{Fx) znO=td4rCCN648(p8}t)S1(j=Ri0!sYs*JdWjr-&7Ff>Z}^e2v2HZfiE)~zQ;28+3D zMaWvk6xy&YLuV;Sc;{{D?k9>d^XT;!*^sxxa3}kLbwscFe;@_v2z|_;6rEnT?Y0GW z#l{HI zt5WlSo>fT=L7L<$vKu+g4z7aX305#k3P*s(T3CwE(=rDb_r{$8A5Ex`pwHR zRs_!FfXMlrF$MVwh6JfF!IE%CwnrQ8uh2lOX7lk98|VO28&3E$?65tKmn{{#b9NCX zW=z~|6_$lX6-re1PWYs%LgnN?4-cZa>>-2N>cssx@JYtoD)`BmX-Nc~N2^i%L^22% z3gD-zAS4s_+E*mTx`FFy{SIZY-=zvZSh0bvcKfM3*(w4qxVPx<Kz~P^lr|j{h}#9SOF$s~ik~p&U0sCAov*R&VTk@~ zglW`(NWDca_(hsbV%nrygATK_2q;aibPbFFnjf)h410cu@;rsD!LNC5OTJRY>244# zB$ZI?E$JlaBN}g;F6I|dT7B!25U^4|MCQWx7B4lZ%Aq4@nT@Y7U_DUjG>dJix|K}2 zs}4imy#pgtna5gYpC7fnwOc{)I4*W3z`08z<_{z|MotN{+@n|6FN~kq8tC3Tu(7mw z6Z3Cono{dIh2MIr^vGz1dM3(p^pBJgqA6bQeeS?eGh6^ESOnef#7Q8Kyt_wM_3X%K z!>V@-bU*nH(JgpjZ<~G15J12}b9j*Qn3=b)eisGzOCYi-X$~t83CQ!O+fJ3dXcV%^ zO!^x(uQ`aMfCO4RMWy|y!VYPDj<{b#2*Mi&w3n9BzQw4pjjRe9pq-FAIyPpOPJUP) zHt@<|2l@a2@5ZS-T-LzBy_NBn@wlHr+HYDuF>rQYwzGreN%~sF!M9h(HP_s012<=g()M%%LxPIyM6MDlv^D{ zMJ$bP+3B}lDu0AGw{mak6rla|kuGM4m>^>H)@xPI3fB?-rz);lihDxSthXVv-t9N) z3eW|Z6|#`7LZdpMqMEEJso;RD3IucOC4u!cQR-Cn(53aW-7wolv}T?_^ok3*qLWZk z)3cnE24kH$C!M)JGO~6jo#!y6B>Qk`Lc&&vMaNJhFw3j}t?uCLfMKHenHb_2BVmF5fY;K z=`|>LQhW@yhR?ss)UU*h=y|vg?SS5L-1hc%B>39&O6qp>zv?S0Dx!wJb?Cw0H8*oN zH}!V5)zw*jm_?+woJ*{?tY!TiRIsS>HAdJfQ}I|x<#-`BH8nLnJY1Ud&581IXNB1L z%MC3oc({_;U1mQ@KNGDX)8vU`#Z?q0CCL{~nJWQhVq9@tQBiSRK|ula%>+H1o!#BN zz1@Kfz|lWt@MEE^xw*O69QgQ$^3_Z@@CywM!65hpK)J=7zK9|ntUXNypPpuCr<-ut zYrxO@typoS4Q<=MMT*dE+BY)(8wPy3y?7qL%SV-pk9HXT{1|9>d+y3d5Z$v`t$|nI z;06?_5X6i>63=~YbrO98i0kVi8R@}+0jWlX*F;pnJQEl$T&uP%3=f#^uyMZPd~O%_ z>Ep>6X(%dJQT#x|NMbe}otXPl!@kYyr8HW%#>FXn7POe|bW_XK&SwKjJ0U^uElG4m zgbX0>)MMhBJOsu*I(dr2Wu+e9>1uXHc~}u)_OIO^tNQWGky{!M=jZQW`haW^!^y$H z0~8Y-{pLs(7YT{p-rnxAKy-u4KuQV&)Vm;^Jbdb&3G+ z^9|s=X(at970FKHZ5Y)juLNB4qR&ufXz)#Jjkj-QoyLY?XkWbBS8W=#UNbzWGoYo} zG|Etp4rs%k(%5WtSY(SFg=c6f)@U$le^&ae)S-%wlrTt2LSm1Cvh-@CkbNi!2tjXX z);9b3Emg5GGb@_$p2}pB%4n;S4H}u8o30Q1MsqeTuWe~IpVwW_0qCzc9e9Nc8%arx z*&r+F=)>CAY6jbM=18uR9(kEuv}x9e3Dh5w=bEip6Z^QPr?*CUGp2P{mp!=Y8pCU`) za);K*z2zY*%ak03k?;XOIskZEu;C#CI)B;bb6)q)ZX#mc9!yAz4Citi(Bg|WOpSEJZWz$5HnVYSmvqj&oKHo$pJ_W+))K!4TxDaOi{jC&w2 z`6?P38fK0dTMi4C!a_o9`gTTo%&tO0LY<@%X2P5j4D&9U1wTz-j=a6SZ!f2&{>VeV z6TAf?8FN!pt!X9H5~YehE-t3z-D+B(=PoYnG_g$8_7|9v>UKIjqJ=tVz2=q!Xto(B&b$}$V?5Zd9~C6fJi<>uonq52h*n_y}KM@ zWJDxLkcA&&tO&r6>Y6)P;-KG;Re!mPDQ(20R{V|W#^Qj;48d}}V8!t`w~MvqFmT!A z^1hAJ3W$DR`V0@0HZ*bV-l|8hSEahETmGSRX8EHjHO9c<22j7T>>dL0uX*6Mhc@18 zu8-5eS0Ob8&J9XNu)DLfGuzv{lg}((bDLj30Qby-BtYsYOl4f|>rXhl`<8qM)t>22n2EZXP;wJ|tWrqC7vmt&@NS#Qyz5 zQ3i4(EvCxD)7wWcCg}b|z-T*TMcE9*rzhI59^aLug1hMOe8r*<@!q?m1HhP>CKK)+ zV!aDH%%NGE!QeG$Fp${#1@oD7s{AJkQsWY?F=}&yfG00yj65*8%mOn@LZ2Z#%g9S+ zbDzmNP+<2>>H7{P+C2l1W@rT_P9-2fUITK0;x)7N5E=LCZ$*bvI zSNEXHJ;W#;pK}1iV;yym5`GtD6(yy1HtR#2y1ZNt9+FR5+xKrVR?DEuc5xnk*#*#f zyA0MU21(uBVMhar^9C*pfs!wmDH~bgFE1@EP4g-^f5^lB{=&lk!hMJs%^@gu%lM&ZIeYve123@%$K*)-ni@G%~SzC$9#K-+xqBw}@)cgva zmB#sqa}j+=5n*9VQvQ(Q6t{(XgSN?y&FH?(b5|?#f6D?e3>5)UX?y}AIF$nnY;)F& z^DFq!<_*M$P0X~0M}&umhDwNv2BL!BLxAju+Uf|e%bX%1At2NTX$qG@9OE{eBScpX zf*jn*Cj)(N%%og)YTuD3L%#I2o`wie0CM)#-X1V`H3J%0($H&JmZnG-cw`JLWXOI! zH9k*{h~Ls)USPIff&@l~;$l(9l$5W4I=uT@rdkIw-XZ8bk|0*dmHYQkl`iK7JdN3A z_ZxqR!Bm5}rBN$QmHyqWANn#s4VV|6mSC+45C;d~)re+3w^oxHp}(~-wl zj~z&Ul}#C`HyM!ZuV`FynwEw|!qDf=#n{C7HpL<#ngC|{Ge&Z5x*%u;Jp1uTuU>aSi1!;0jWm7v!?>w z2Qcz61Ad9;NsYLKzt%E#?z~E={SB|^?txnhUd6Bo-{Jx?Q~ZESa!JV$3*GZTOT}SJ zVwXY3iREQ1P-0?YUe&*Czl&`FS0A*95a9bE-fNG<6$%_29w^}8Wwiic@OmYEgeC}{YzM>BSYB#HF=;P}A^z^TEb8>e{19kMPvJ!t9#nfUd0;;bw znXelaFGQI4GGCm_n=*gf?}SO z2qPjMPZ^&yfI`$MbSX0v8u0;vUL{TqLf%gs@mxdI@9H&~lA0mlHDg$ zfX{Sfr3hYc<31Y>5QR#S#?^=`flq7l3ld z?}R3g#V$}4ibG^Yj`8gf>;yGpRr&RVmJRj-0)|WyIC9}KH2OQuhE59xbB6Y)bfk2p zbesNwOS3*f0nu>lh`S}F%P`?vB1|cS!TleyifHBfM15JQB8= zR6H&x2%T)XfH(|m<7hYa?$ihjkAYdcR(sY4K9hid(N8&=b{W zafZw>vr=yG3~DdTyK-`GM~z#fZ@Q}>OwO&fqzn+|Lc?LzWXT)Wy3%DqHM2V6X}T@B zemGK+_%+bHtv*rbP{#AK%a`$a{8%e?tRy)m^-O@9Jd|C zZ?=(P%f#^q*m{!2t)(_9_SFXKD#sm^>2TdueVSBr$YF)Gm~BUAnvaVxc1a(Y zqH1aPS8vruKpguPza_r2>dTb3bfj7O(UbC&LGCrX#r z5nR4%tg)Jg*V(96A}PrQ+s}C}y`HOp9}L4edV#OvaTg;CL@qSojHIs*GO!FGpnv*n zv9nZX^1z-2d=uqa&LGul;HVDV&k5wrlKq>#R>PU$&t;#We3`mnnA&oY`n1i&^n6Vw zP(;2~Kwb7Mhnu>n_^#|C<)N3mHq9TLCkkYktUTXgOD#2h$p6w9C^N%4W7Q6-5w7l< zrO>;ycx(-J>@>uWl6nevGw#UbFoSAYxEekx<`5LHApQ6=sZD7OrzYz?n%~%gtw1Yi z^iHLV2yd#>?=P1##)kUbj0HHn$K;JqGd?*>`P$&~JvJld#q#Ou%eEegK7}~k+Qg+! zS{;xXAP+MEaus6z1_JyPmavQc5djX!v{;Qsrqc|1nI`(=78x!FVf70?yWsmB>o`0N zSZCc2F~^oz_sgJzpiDxTP$;I}4WG`ErimMdS{dWPB2CTK5PhC^D}$OvKjuzqNc3<> z$G1P!o5#9cXsFj*D#I0)?ZDuo=>Kvud}Z~Xhl%v)ppFB!R?I1glEFla9;+|+*L!h2 z2@lfUr3OTXZ5Q>&9#>hNt_;fB0F(GCS@xGl|fFc~$zGkpo-if&+W zCY`0-6`9J69rwSMwhJ^n&Mpstc$_qgs<=$UtF^P-q??K{GjyfvYT5A63$KrlzAqfO zKwqDB*<@T6>;AIMxAuU|b#J}pe-!FwRN4FM7k85T1LpRY_vikDCQ&>*6AOv&IiLG8 z`xW9LZDH>g`0TKoW$gjy8=ZQOxAlI+W?Vz#-gO_UTr~10dR-4S?1K3Rnc5N`MTaUw z?sr?ZiQLNa{n+XXn%faS;U*vs%}Qahu7q>m{o(^OEYWMx&j(Wo#`6rdCN2YxCcADr z$K~}oxlgZ=$`ah$cg#qAZ}tKlbsZY6E)rgZ2TojSutygb!6a8D=0|qvaNS6NA}-H233r;N_`ofsqD9eDc;O-OKQ|ez#q*?VW;(Ht~u7YozUvbxq|mmW{HzNI$%0qTNr2l}9^`y+Qwg zulEa6)eYkv3SHjeZ@h*T^sDaL0Pe+NhiIEG!KYWwg?;HCEQ+G%dss;EGby$=V>)wc zXM%Kgz3F!SP8f_pSMHq87uHBB<9C|n0p|?jtP%CGCLCWI!r|TOt58p(XO1OhTL(DW zKaFHvn5+p+l$;kWD}F>;L6aq2sv*9=(P%&jSK0ir8U*5`hb7{|N^pFro|D2Fa5B&^ zcXY(TJ<>lexzJ5!7RBkr`y$dRruSf@^BB9>47v>S*xY*+_!u;zhhU_Am9a%sR$3lt z=t9(j%@>)L&UWcNma^*&ZWFI{pyEj7rBT|2?YV7I5}Z5Yw%h)51deMwCW}`5YZ*v& zgJIoiID#Ws>kHJS^`K0L(A-6&J*GpIfWwlkvHRp!1H7Vb%rR%PMrTHRxN zZMvJbB0Lkk9>}}=m6iT?T=#4CkeGYK4|wYUYjMW4pG|0PPsc6Q&EyT8IuVUf@c@_t_rVk6*r$oqv-1d58m&sD zS!#&;lsj#Dlc~s14;U{7kBR#m|0qFaC{HD z>Hc~WwCK9Lrqe7QO0As&yxWw9mAWDXr z?$L>rt58;ISf?SgF&!>O+^^SKRCQO5B!FM0s_oeJ-Ja70&UI>2T-l448OY@Hlqrq3 z7cl;?XJKvf$zF)d(SfmQpcFP$9q&mFR5JNOx?3^NZoG3NS~}32!YT2DbF8>Tn-R%= zS+Jgi1N9vC!trG#^UJjULWa?Bo6lgAJ#LQUr9#z0>$eIsrzz;np)1;^N6LOS8`ajW zx|4j@Gk<4nf!{n%B;}qmOGr-_tzNk5^Q3a+R%uGTWY)+5SrKFWraVBaX!TT5x z+>}GHYa2|r^U*uQ8**OchLzJ4a*3h*h`ssCL)RbyGjE%W>1UtoxvWRz2m^Ir9mh1= z9}y|}t({KEqfX2O*o|UzE%d7}62TD;+MKmrGx&^KkZapjUzm+hKior^4R4}8@pKmo zTm2j~Z2p2TogCeDoM=a~BPDp;z7C!i5?cG`3S@&hh++DlGDKhYK5jdiB@ zMK{IwFPa~W=;^>j3x*ohW{a%=rKo z;lCdw{{Es|LHlT*xg20IH`~h^NVT-YZgaG%`Ry`h=H+IlTyK}B^fx=Z|NN5P7M|4# zGep;{aC}`d0{1WR(dkdHolGk=ij0e^tPRJu`kdow{oP)$IWxa+8;`zk?XN=eCx{KL&!WaeQylRA z^^L**Z2j|%K|N?pwK_n!9fqbfGBN}|lC=!hS8JH!%D$S&2GQqRJ(`U(RAg8cbQ__4 z0@~9h0T>L}3(XX&aoI377qna^zogBG-lRn<6qh9vh;y3BSsVIq3Cb!EDG-v@%vN8;{`ROXE~O*;#s1}J0)3L=g25SF;w zXT|f9n1Yh(bCcL&*%kT+KWTOOmKj&5mDcmTSmY_x#}uYEsyL+;wQQF-%^zsWvwNu^ zXqBp2IAtlLnHM=RP2;gl^SnFIgs(>oZrKqQQ&AdHESZ&ZVtM`vW_S+w!SnrB zBaLv~#;f*84}H}AlM7=q-IHg+L6;MiUdkV=w@#=9k>s*4Nd><0DuND38Yf_4$s61$ zxwsxx6u=ilAH>+iI0e-BUiHcl+DJ?l>o=bvGX4_G~iv zvlbjubxdV5Gx)Frak|Mo^x}#Ug%J!;e-9nEr}AVfA>>S)%QK#>Lgw9*!(FUl0P4^OhZJhY=SpiXxW#72@IzV&TL-DZ2Y}hM=+bJ zFwTnqJ5g98Xv*aMcs+5k=yz54tVZ~(3wW$@g73tkji6zO(Dr=cKY_6v;j)dh_&p z<8*o*&P=KpOzz=W-OWjY%~)+W5wJ~IOpWNh4On)ML4)2CGup^fP2dI1YhiZ!Y4iie z3|q`tI>SM*JGnmekHV5D-}3`?CIyg`F|?Jd=&8A7V4T%A^r;8Tzrog_o|fg~MzDMm zTPMi%Q&Vb#9}p;^N>)YXi{j{0y5NH4kSN|XbAbP1DTI=dIYMEU1yKY`L4}Q)5DqRz zDl{NDUWWbfA!mW91}6AGLP0o>Qh%b>c2DZVuhkub@yMsX{^iK0fw60iW0+a5T!+ks z6SOuj<=W>Y#h25oReW#yojG29Jr$(i9?ye)(dm1H0u|GoUg>Awey2@G_dM|wzi=F% z=Kof(2PHT^C*8$eX6^J+HyZCF<>fhxE@_5piAvUvyIkfWH-Q|hVnG+i=iB^L^6GF+G$<1N0i)`Fh6u zor7Bb7(4{DmISoBonI^0!U>z=B(m-Ad||V}XigwiXE{}$-hcg!@O4!o=w3dIF~+`0 z5H=f>`UG5g){}Y{n_BlfIw2!@h@ZVH09Fr#>I94G5R=LjlWG%#>OCyFE`)TZ!M#8= zU!c}c1-ey=L`S=!>m~c6hNlMl_Tz73pLQ$;MNp+X*uH56P7F*d&B+2epuS}bgK6A&$%D(SUni4LLPcM19Ob_pZZv*aXk>2kdoD`qjIB|VBnVr~F z;TNcFt$(WAPHFf4z0mjmQ9l}ScVcDg`S|dpz`H1qwXn6dcvqIc@8l4Vzi*+B<5718 zsnETxne~}J>mY=U$5Y&=Z_knOAxV``R5Z+eezhl z=gBTN>j$j~YGC=Fn@~0*GM$OwCkuhaT9u!;1OroF|p|IRdxt9}yb>U1#g|&6&L*^Pwy|U8FFZai zVW)rY2gifgR5jND(SVCpW2G&a`?<5=RG1L}7Xha+p5q(z9h|P}{9}K(K8VzqKSE{@ zfp}047|kA4_D@*Q0uRQ56FM_(S}6+MRc3`dS%@2~k^tJs6*XT?B|ohXAkuyL;1FoU z5VAa5$u}Fx^S_d77G)K!WS7|~_S~hU!pI3pHL@oYZ$#;;Q|OIj>50SW7kufji4!ot zCpda4E*@rg7pM}KaC$fA=Q)Yd zOBBKvp6VDrQhB5v-@4nttvnd>Q_#sz5M&MH`DJ8eWXy30*v0Qr8I_R_iBL~r!WR6y zKMER9RV1fjEE)AYE4WrumKjo(L1BOTj3A=cWyqyZmB!SF6a1h@&aOwu%5TN8#*clL zN<1G9!gm8+2-m5o$TSBN%+fBc`6KYaHv2(~bi38y9$&^g=Yi4XMfj4m4>by2f-4v0 zXRJ;}iIZCsx9<^-$F^qnu9Ix49MnK&CCQw{4ypy55&?A~GDUV5#M{+CfH(o0Bj z12IzCT$EK1h#mwgXcIx9Q&3m~VX9D&9K9PMeJ#m#7)JZD?0!3c5*$?Yg>5Q}ORAqZ zbf|EcYSHNYAYpjKL@Z zM|bcT+Bi|#<#Dt+F5at)(>DF6RE6%iSl*Q zs=`Y9y1(iBS@v~fsBEOhHfb@>!R}m0Xo`I&D4#kx2+76 zE@;-O_UIno<&7xiIbBWQT$q{fnH?I{8GIV!8(Ja>dRj3?R6Dj&p zEVXQ1HW8970wR7UV4y*_#yW{koy{`Xq@MPp<&;b5voLs-`*2oopsvz3meoDN*LI>I z#-fYEoF_|&Cxd}3&l!gThF?CHzqM|CmR^?lp&otW9}c}UZ^_1J*9+|e3EQi)_$c37 z%6d=ASU8ivU+g6`T`pt}Qy@kqe2?ULz;suJaV~|VPpy45LD7)TdiI&%hRuFlrAR26 z5R%_XSk)VcUw4PIG;D_S&`Xg+4-6j(nTPITZu29d`Ctv zv@^YjRJSiAY5XksBn6LZ_d;Z3|0Cr zQ%rDHZV((+A>{=fJWATQYT9tSRUYA%q~VtCC#fQcus2(&g$i(+6>3eE8cTC6I#bqj zG)wHuKP;{-zw+}3rm_f}Us{j4`r@4fnpbzdvmRx3+lS}qM)~%qpZRPv&=dxQ@^{dq z6o<}5hKhNIU}%S^qqohv>{A1|J)qM?8Po9y?JjuM%8EZvYI5n9;kL6yO~IP(BKr^y z{uYp&xdUw73O(GM_4`5*ft;N|0gzl+z2iIy8l3qgkGJbq+!QA-eFJuhVe-IY!FZ0S zvI8Qqun2VM$Vr?55fjAxCq(rd6y|e&79u6GVj)Jb3?vTPyjx91-JhGkf1bDBqzPzF znEy0xN28gWR8PW?&CWobMq6A!AuRlzRPGCuvlz5`(2&|Xd7_W$K?CVQE34DT6wPqi zaUnp|y^haPHXRMC!I z37EW-i%K6D3#!PDfDEF%5aQqt^4%x;Y4#PNG54Le~4&OdYVm2R%}9AflYpDJKD|LiW}@B_sh5R(?ru6#N8{d9Yii%_SUe(M=j=!6qI@ zEpZGrl&%WMSnX%Qjg76(%(YE5>zx>Iq#uk#D{?Wvg|U3l$tpF=T~wQ*8{_eQ7YcO8S47;eiwU`V@Sz*yihV~Ber;H~Ld%&PlijFQ&Hd!jR< z;rr25MRtlHx4j|d6svuQKBD=B4}YNv|012{cM#0qW$*<}Ee%K5z=Fw|CO9;uvzEH1 zrnsoO*k)e!O2z^Px|mQ$*?TjYkRbmcxQb3mroPzF%BsiASW3)T%4te@!eKhDG?Q3e zDPXh6ML)p zx3Hp;q@oUU5*<9ob7~THb5*vl5g993!w7R|qDZjzAdi|b3zk6GRY(kS%A71@9qT-M zM09eiCUvx7)PrQL>GqngNjD8`$ycAV1na%Mst)D63S*XZqfZg<%~S{+$n!Y*MauKD zSAlsFP-6^+4JlcQiyhIYWsq5yP83efp+!#V%RE6X5hU^|0%=EwUcWkV%C9O>aVN=SsZ@( z+P+$zl`-%CQ_GcrHF0g>B<%YlL_liUoT3y0L(CeRn2M1Hsn!zW45zFLSqZ{&Vg* z_fF=^H~+1jeGK`PT*GHF&)j{Vwhld)ma<2E*X@`4F6pn< zytQ%ei^ciTUH|-}viy$a$a2HizE7;{J!=pBZ1EmH|NOJ@-1beo$G7f$7Jgv)m#-ZM zAL)AQ2AxgW`EAZ%%E8ObgZ#QjIo*gMmCfx6HIGis|@uyLrx^CEq@ z-?taAXO}JYx!t6_g=pt^GOH*m%rD7*KTqU!En~B{tyTN&#;DHB9d$$-@tWoLPG^b< z!?RW{@UrEk&+>Bcx?OzX^Y7l{TrG0mKJt`5oH6#%?T(~@-^47-&-UkO4W@e8cpk>q z)oRw?gmx@>nSb~z+h3mKZ%8Q&t!m21(^&C1w6Qr?eMRHQ7yVn9>49C%=3ULEtr{-s z0~r>^^37={EeQktjOL`{+3c#EQ(tATExA7Q8B4!ZZ(H^?T1|QW>Rl~co@ch7d$MD5 zp^28(GtCP>Xoc7RoG>ZF&&`bJ}r2R5^W-%%IA}u;&@}@zJK$+Or;Wb?Vo!4wG1q zvTl5G_8}A?u=N*M`wC{?dTMEG8h)aqS`hi&!@FJk@($L_UBV+)@`T>Jb)LKx5l?f6 zdUCoF!p`H}&$pVKcab$pN1B%BSCw{hsgAw(ISsDV=oQ)NJh~p8-W7_~3#3=v-x>aCzTXk%?~KRJ^EvU^RTn%@$;&z(q#s8oi|NVL$C@@GJ4jahQn#Zu1&kRui zy7>z0^P*-`7rDb;p^sg*rb9xMPt!3q(?pn3wrK~=-67AHLW(<=ZD*F-&rCDfs9|v~ zYf%&bLig5lCc)|UA{^P#p5ip>V`Ze(J!VDOa4N5)7Cv$;4?jpg@~`04x)|rAoQUgp zLo}kY<@(igFg`ptc3ZBIfu4~dKC!J`ey>>m)X&F}GMBMNTW|utQx`klCp~4pFWt6S z8)uhpQN#&Lqq=2!YGXh7(39}9Ud73dm-m{Su0o{tl^Odo_xx;foby4tF6XG)Z`_Y0 zt_}e)v)|JX?AZ1`oSI{IFsDJU&vKSoK(J$H+TO19Z3jkL4xBl2puD%UpeM%r0rBWQ zx`}D@((W>T<6ZSbU(|RV*70*AIuQ%4hWyB?rA@Swr=ihumkq1; zl-JbzSACz}&2{-@x1Dv{7FwRp&Wj1L{k4Vlb-FH*npL$LRc;Xd>Zo1`o99>mv zLN`q5b?ND~-!WpJ>y~2W)O;f=>4?;&b^SVvJ8Qe$%YB@i7H0iKj4oNYD08sGgxAr; z_KvDBFXdBfL`N9jjGDFg_l{C7t!^R7!khlWQD!e!@7co&wfv`xV2~W>@Jz@_iEb*mu87%pIjzU9yphMtsT0j z-l?|wVd~{Ab>qwx8=KOHGx~ne;UyW^_v!_2$lMKMH0-jR>E}gd3wa?;yH;1)EsR>q zX&j4cytS_J%BspE$M)KNJf~5Qk>GW1@9ebsA8a&dJ|8$V+9GE?svXC%UoL+APiNbP zR}B@z+bfFxxHQjbghtY~vCkOOyX`;ZkTECUcx^Dr(L1Y=sefE8ef^56M`PIRj~r}X zEyX0s)HBwa7_aq}?#!_{vSqQ4DNHdsGk3}LJW)~pp0v(B&AX@7PfI!dyS02vPhfhp z<<8)Twn^h6yR>;Cc-Q>9Ev>G}vcorZcYI2yOeTyS4}bmTr;9KCQWtu;#<`N|Tp0#e za`CVGxAXQrv?@7O(syjTPi1EvZ=lEgjJiSo=B$=~+#fi2=i#f&J08&^xV2}_I(wVE z-`ViG!O=tFl=3M1+Lr_V%Le9U%9cg^puxd8+o(w{3Ww_N{}^6-I5>zrJ9L}N{C+uE zdpX9oQQm;vcyhxgVQ&*f4KU|fj?ePISZIpbv0MSu+yP^*KMw~uvqVe*SC%N^&BLs) z25-!xy;@SPNF0}#pxmd);Y#D!T;(pUL}6qC5LfOq;BiF>Tz;fDFN7>`*f?GF4(Y^j1%gOr0d>T=gk(WvESJd< z1NTAO2#|;|`H|Clby-XaJ2p}rm&#R4Q;QSx6*Vo1P^n*CEa3_j1BUEGfl$O1iz7u$ zNn9evTo#@)n7OKS}T2*-uRW+CW zj+k(OD>EEUr#n=Is6M1IGsI;2C@_Yw_-#0-L<4G5M=)mO zF?=h#p(<)+o6gLrQxjAfyIKpniZW5oNM86>O2hvETd@Tcm|1H03^nML4m60S!2gD! zP}P>SQlpvIVan!Hm^6&V@4`WC`p~a>NC;*o9(+3=p(;4|Yhq^bHyd1mT#UuHVxcBO zXxson{*!`m3K^lQ{{G+8X4XILs8E1pVDV*G=#>%FWQ2eya}uhmzH;I%Tkg$KqtIH0 z#kXUjDidhP7}5HMgm5Y!p(?_7iTYII)hCaNKMyDc!c8nb4F_e+f@)2z-!PWE0}-JL z=KLv#$RytMV{rMML9&?7QsvK$vlxpw0LruHwDf@Sj1oguJG)=LXfDLl8hJa4i-}L@H zsP4pen7pkj(1!lN;Rj!UZ9IU5T73ba>VNPEpla`51LBLi)xd2T(#98`#bGS1E%f2A zu#c&W6~C4&PU;en;spDFS3_2zFNnZ!FbvCJQVbjb!!fXL4BATq?_RA$YQ8WWh+IK? z@(LPA6^zJWI1HIaiUIok#6TQQMy3VIz#xf^w1bg$=qe!Y=Z_%9!u~Mq9|PK;1=l1d{6#d64eDX=%3AVu+EI2@Tv@+U-3 z*b|G&q99840LNgMO$_6+(P7vhS!%SZkaztXdDI^cg%r(p=PzQcf?cP|`|_sAE3_)G zkZ&ZWFq0M{hRJ?F)C*~nP^UxuycKPKU(`qvvLHwxCQBlykmBgHuy|&GcPf)69nADCYcX%Jj9hl3j(;Hgrf4YY4X$a5$tEMKd693k&4Ef zjm)QlJ*C(WZ}Atgk1XM{1s;4B$ORTLB8BdbuGdOLK5m*k*leJnONZ9b)FOM!Xzm`d zENVE+LFRi-=A-{ae9-tQ#Tzl5#ADDHLKG_q$rl10`9cOJ)@QnZiv1#pNCEpr%8Kzr z_Zv_OjEzCgGcb<&z3R=OF@X<~2oxoJG(QAvL4bro<4D;8@IDnfJ}J6v8OFj!$P^_d rb0yAfq0pHr5+%xPCLS+s+!wmIy10TTCGh0=Uwkk>8DcPio!sr;D;ttm diff --git a/python/aisimulate/src/aisimulate_core/systems/perf_data_reuse_manifest.yaml b/python/aisimulate/src/aisimulate_core/systems/perf_data_reuse_manifest.yaml index d7fa47380d..446e4f4d45 100644 --- a/python/aisimulate/src/aisimulate_core/systems/perf_data_reuse_manifest.yaml +++ b/python/aisimulate/src/aisimulate_core/systems/perf_data_reuse_manifest.yaml @@ -414,11 +414,6 @@ groups: tier: shared systems: [b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm] frameworks: [sglang] - - op_file: dsv4_hca_attn_module_perf.parquet - kernel_source: 'FLASHMLA_SPARSE_DSV4' - tier: shared - systems: [b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm] - frameworks: [vllm] - op_file: dsv4_hca_context_module_perf.parquet kernel_source: 'DeepseekV4TrtllmAttention' tier: shared diff --git a/python/aisimulate/tests/golden/generator/test_release_1_2_0_backend_contracts.py b/python/aisimulate/tests/golden/generator/test_release_1_2_0_backend_contracts.py index 0ef122af43..abebc367c6 100644 --- a/python/aisimulate/tests/golden/generator/test_release_1_2_0_backend_contracts.py +++ b/python/aisimulate/tests/golden/generator/test_release_1_2_0_backend_contracts.py @@ -313,7 +313,8 @@ def test_vllm_0_20_1_cli_args_golden_contract(): assert _value_after(tokens, "--data-parallel-size") == "1" assert _value_after(tokens, "--kv-cache-dtype") == "bfloat16" assert _value_after(tokens, "--gpu-memory-utilization") == "0.82" - assert _value_after(tokens, "--max-num-batched-tokens") == "4060" + # token budgets round up to a multiple of 64 (backend_config_mapping.yaml): 4060 -> 4096 + assert _value_after(tokens, "--max-num-batched-tokens") == "4096" assert "--enable-expert-parallel" in flags assert "--enforce-eager" in flags assert "--no-enable-prefix-caching" not in flags @@ -362,7 +363,8 @@ def test_sglang_0_5_11_cli_args_golden_contract(): assert _value_after(tokens, "--kv-cache-dtype") == "auto" assert _value_after(tokens, "--mem-fraction-static") == "0.82" assert _value_after(tokens, "--chunked-prefill-size") == "-1" - assert _value_after(tokens, "--max-prefill-tokens") == "3548" + # token budgets round up to a multiple of 64 (backend_config_mapping.yaml): 3548 -> 3584 + assert _value_after(tokens, "--max-prefill-tokens") == "3584" assert _value_after(tokens, "--context-length") == "4096" assert _value_after(tokens, "--max-running-requests") == "512" assert _value_after(tokens, "--speculative-algorithm") == "NEXTN" diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py new file mode 100644 index 0000000000..bc9620893c --- /dev/null +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py @@ -0,0 +1,34 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The encoder collector picks the vision-attention lane the way sglang does: by capability MAJOR +(vision.py:1298-1303 @0.5.21), so SM103 (B300) shares FA4 with SM100 instead of falling to Triton.""" + +import ast +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit +SOURCE = Path(__file__).resolve().parents[4] / "collector" / "sglang" / "collect_attn_encoder.py" + + +def _serving_vision_backend(): + tree = ast.parse(SOURCE.read_text(encoding="utf-8")) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "serving_vision_backend") + ns: dict = {} + exec(compile(ast.Module(body=[fn], type_ignores=[]), str(SOURCE), "exec"), ns) + return ns["serving_vision_backend"] + + +@pytest.mark.parametrize( + "sm, expected", + [(89, "triton_attn"), (90, "fa3"), (100, "fa4"), (103, "fa4"), (120, "triton_attn"), (121, "triton_attn")], +) +def test_vision_backend_follows_capability_major(sm, expected): + assert _serving_vision_backend()(sm) == expected + + +def test_run_dispatch_uses_the_family_helper_not_exact_sm(): + src = SOURCE.read_text(encoding="utf-8") + assert "if sm == 100" not in src and "if sm == 90" not in src + assert "backend = serving_vision_backend(sm)" in src diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py new file mode 100644 index 0000000000..8882c056ac --- /dev/null +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py @@ -0,0 +1,71 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The FlashInfer workspace floor goes through the knob sglang 0.5.21 reads +(envs.SGLANG_FLASHINFER_WORKSPACE_SIZE), not the retired global_config attribute +(ed9109ec wrote the latter and changed nothing: l40s pipeline 72343559).""" + +import ast +import sys +import types +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit +SOURCE = Path(__file__).resolve().parents[4] / "collector" / "sglang" / "collect_attn.py" + + +def _load(): + tree = ast.parse(SOURCE.read_text(encoding="utf-8"), filename=str(SOURCE)) + nodes = [ + n + for n in tree.body + if (isinstance(n, ast.FunctionDef) and n.name == "_ensure_flashinfer_workspace") + or ( + isinstance(n, ast.Assign) + and any(getattr(t, "id", "") == "_FLASHINFER_WORKSPACE_MIN_BYTES" for t in n.targets) + ) + ] + ns = {} + exec(compile(ast.Module(body=nodes, type_ignores=[]), str(SOURCE), "exec"), ns) + return ns["_ensure_flashinfer_workspace"], ns["_FLASHINFER_WORKSPACE_MIN_BYTES"] + + +class _EnvInt: + def __init__(self, v): + self.v = v + + def get(self): + return self.v + + def set(self, v): + self.v = v + + +def test_0521_envs_knob_is_raised_to_the_floor(monkeypatch): + fn, floor = _load() + knob = _EnvInt(384 * 1024 * 1024) + monkeypatch.setitem(sys.modules, "sglang", types.ModuleType("sglang")) + monkeypatch.setitem(sys.modules, "sglang.srt", types.ModuleType("sglang.srt")) + environ = types.ModuleType("sglang.srt.environ") + environ.envs = types.SimpleNamespace(SGLANG_FLASHINFER_WORKSPACE_SIZE=knob) + monkeypatch.setitem(sys.modules, "sglang.srt.environ", environ) + assert fn() == floor and knob.get() == floor == 4 << 30 + + +def test_larger_setting_is_kept(monkeypatch): + fn, floor = _load() + knob = _EnvInt(8 << 30) + monkeypatch.setitem(sys.modules, "sglang", types.ModuleType("sglang")) + monkeypatch.setitem(sys.modules, "sglang.srt", types.ModuleType("sglang.srt")) + environ = types.ModuleType("sglang.srt.environ") + environ.envs = types.SimpleNamespace(SGLANG_FLASHINFER_WORKSPACE_SIZE=knob) + monkeypatch.setitem(sys.modules, "sglang.srt.environ", environ) + assert fn() == 8 << 30 and knob.get() == 8 << 30 + + +def test_no_knob_is_a_reported_noop(monkeypatch): + fn, _ = _load() + for name in ("sglang", "sglang.srt", "sglang.srt.environ", "sglang.global_config"): + monkeypatch.setitem(sys.modules, name, None) # import fails + assert fn() is None diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_gdn_contract.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_gdn_contract.py index 8403ec4386..c3634dbed9 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_gdn_contract.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_gdn_contract.py @@ -324,6 +324,7 @@ def recording_zeros(*args, **kwargs): "fused_recurrent_gated_delta_rule_packed_decode" ), "_resolve_flashinfer_gdn_decode": resolve or (lambda _dtype: (flashinfer_kernel, flashinfer_error)), + "_resolve_fused_gdn_decode_conv": lambda: None, # <0.5.21 split conv lane "benchmark_with_power": lambda *, kernel_func, **_kwargs: _FakeBenchmark(kernel_func), "log_perf": lambda **kwargs: logged.append(kwargs["kernel_source"]) or True, }, diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py index e867243d64..4d4bba49ed 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py @@ -84,7 +84,7 @@ def test_glm5_next_decode_uses_bounded_gate_recurrence_not_packed_decode(): @pytest.mark.parametrize( ("version", "accepted"), - [("0.5.16", True), ("0.5.17", False), ("0.5.19", False), ("0.5.20", True), ("0.5.21", False)], + [("0.5.16", True), ("0.5.17", False), ("0.5.19", False), ("0.5.20", True), ("0.5.21", True), ("0.5.22", False)], ) def test_kda_compat_admits_kimi_branch_and_glm_runtime(version, accepted): from collector.version_resolver import _check_compat @@ -134,22 +134,30 @@ def _gate_namespace(): return namespace +AUDITED_KDA_RELEASES = { + "KimiK3ForConditionalGeneration": ("0.5.16", "0.5.21"), + "Glm5NextForConditionalGeneration": ("0.5.20",), +} + + def test_kda_audited_release_sets_do_not_overlap(): from collector.version_resolver import _check_compat compat = _gate_namespace()["_KDA_ARCHITECTURE_COMPAT"] - audited = {arch: spec.split("==", 1)[1] for arch, spec in compat.items()} - assert set(audited) == {"KimiK3ForConditionalGeneration", "Glm5NextForConditionalGeneration"} - for arch, version in audited.items(): - for other, spec in compat.items(): - assert _check_compat(spec, version) is (other == arch), (arch, other) + assert set(compat) == set(AUDITED_KDA_RELEASES) + for arch, versions in AUDITED_KDA_RELEASES.items(): + for version in versions: + for other, spec in compat.items(): + assert _check_compat(spec, version) is (other == arch), (arch, other, version) @pytest.mark.parametrize( ("model_name", "version", "audited"), [ ("moonshotai/Kimi-K3", "0.5.16", True), + ("moonshotai/Kimi-K3", "0.5.21", True), ("moonshotai/Kimi-K3", "0.5.20", False), + ("zai-org/GLM-5.3-Flash", "0.5.21", False), ("zai-org/GLM-5.3-Flash", "0.5.20", True), ("zai-org/GLM-5.3-Flash", "0.5.16", False), ], diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py new file mode 100644 index 0000000000..b1673b0c16 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py @@ -0,0 +1,195 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""sglang mla_context on SM100 is timed under CUDA-graph replay again. + +Between 2026-10-05 and 2026-10-11 the trtllm_mla PREFILL lane was timed eagerly +because flashinfer 0.6.18 refuses to capture trtllm_ragged_attention_deepseek +without the CPU length mirrors sglang 0.5.21 does not pass; the eager rows +floored at the Python launch loop (B200 0.13 ms, GB200 0.29 ms for every +isl <= 1024 cell). The collector now hands flashinfer the mirrors from +sglang's own extend_seq_lens_cpu and refuses to publish any eager result.""" + +import ast +import contextlib +import inspect +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[4] +SOURCE = REPO_ROOT / "collector" / "sglang" / "collect_mla.py" +FORWARD_MODE = SimpleNamespace(EXTEND="extend", DECODE="decode") + + +def _tree(): + return ast.parse(SOURCE.read_text(encoding="utf-8"), filename=str(SOURCE)) + + +def _load(names, namespace): + selected = [ + node for node in _tree().body if isinstance(node, ast.FunctionDef | ast.ClassDef) and node.name in names + ] + assert len(selected) == len(names) + loaded = dict(namespace) + exec(compile(ast.Module(body=selected, type_ignores=[]), str(SOURCE), "exec"), loaded) + return loaded + + +def _fake_torch(): + return SimpleNamespace( + int32="int32", + tensor=lambda data, dtype=None, device=None: SimpleNamespace(values=tuple(data), dtype=dtype, device=device), + ) + + +def _install_fake_flashinfer(monkeypatch, *, accepts_cpu_lens): + calls = [] + + if accepts_cpu_lens: + + def trtllm_ragged_attention_deepseek( + *, query, cum_seq_lens_q, cum_seq_lens_kv, q_seq_lens_cpu=None, kv_seq_lens_cpu=None + ): + calls.append({"q_seq_lens_cpu": q_seq_lens_cpu, "kv_seq_lens_cpu": kv_seq_lens_cpu}) + return "out" + + else: + + def trtllm_ragged_attention_deepseek(*, query, cum_seq_lens_q, cum_seq_lens_kv): + calls.append({}) + return "out" + + prefill = SimpleNamespace(trtllm_ragged_attention_deepseek=trtllm_ragged_attention_deepseek) + monkeypatch.setitem(sys.modules, "flashinfer", SimpleNamespace(prefill=prefill)) + monkeypatch.setitem(sys.modules, "flashinfer.prefill", prefill) + return prefill, calls, trtllm_ragged_attention_deepseek + + +class _ServingBackend: + """Stand-in for sglang 0.5.21's TRTLLMMLABackend: metadata init + the kernel hook.""" + + def __init__(self): + self.metadata_batches = [] + + def init_forward_metadata(self, forward_batch): + self.metadata_batches.append(forward_batch) + + def _run_prefill_kernel(self, *, q, cum_seq_lens_q, cum_seq_lens_kv, **_rest): + # sglang calls flashinfer through the package attribute at call time + # (trtllm_mla_backend.py:1121 @v0.5.21), without the CPU mirrors. + import flashinfer + + return flashinfer.prefill.trtllm_ragged_attention_deepseek( + query=q, cum_seq_lens_q=cum_seq_lens_q, cum_seq_lens_kv=cum_seq_lens_kv + ) + + +def _backend(): + loaded = _load( + ["_extend_seq_lens_cpu", "_ragged_prefill_with_cpu_seq_lens", "_GraphCapturableTRTLLMMLABackend"], + { + "torch": _fake_torch(), + "ForwardMode": FORWARD_MODE, + "TRTLLMMLABackend": _ServingBackend, + "contextlib": contextlib, + "inspect": inspect, + }, + ) + return loaded["_GraphCapturableTRTLLMMLABackend"]() + + +def _extend_batch(lens): + return SimpleNamespace(forward_mode=FORWARD_MODE.EXTEND, extend_seq_lens_cpu=list(lens)) + + +def test_no_prefix_prefill_call_carries_the_cpu_length_mirrors(monkeypatch): + prefill, calls, original = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=True) + backend = _backend() + backend.init_forward_metadata(_extend_batch([512, 512])) + cum = object() + + assert backend._run_prefill_kernel(q="q", cum_seq_lens_q=cum, cum_seq_lens_kv=cum) == "out" + + (call,) = calls + assert call["q_seq_lens_cpu"].values == (512, 512) + assert call["q_seq_lens_cpu"].dtype == "int32" + assert call["q_seq_lens_cpu"].device == "cpu" + assert call["kv_seq_lens_cpu"] is call["q_seq_lens_cpu"] + # the patch is scoped to the call: serving's function is back afterwards + assert prefill.trtllm_ragged_attention_deepseek is original + + +def test_chunked_prefix_call_goes_through_sglang_unchanged(monkeypatch): + _, calls, _ = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=True) + backend = _backend() + backend.init_forward_metadata(_extend_batch([512])) + + backend._run_prefill_kernel(q="q", cum_seq_lens_q=object(), cum_seq_lens_kv=object()) + + assert calls == [{"q_seq_lens_cpu": None, "kv_seq_lens_cpu": None}] + + +def test_decode_metadata_leaves_the_prefill_hook_inert(monkeypatch): + _, calls, _ = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=True) + backend = _backend() + backend.init_forward_metadata(SimpleNamespace(forward_mode=FORWARD_MODE.DECODE, seq_lens_cpu=[9])) + cum = object() + + backend._run_prefill_kernel(q="q", cum_seq_lens_q=cum, cum_seq_lens_kv=cum) + + assert calls == [{"q_seq_lens_cpu": None, "kv_seq_lens_cpu": None}] + + +def test_flashinfer_without_the_parameters_is_called_as_is(monkeypatch): + prefill, calls, original = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=False) + backend = _backend() + backend.init_forward_metadata(_extend_batch([512])) + cum = object() + + assert backend._run_prefill_kernel(q="q", cum_seq_lens_q=cum, cum_seq_lens_kv=cum) == "out" + assert calls == [{}] + assert prefill.trtllm_ragged_attention_deepseek is original + + +def test_benchmark_layer_refuses_eager_timing(): + benchmark_kwargs = {} + + @contextlib.contextmanager + def benchmark_with_power(**kwargs): + benchmark_kwargs.update(kwargs) + yield {"latency_ms": 0.13, "power_stats": None, "used_cuda_graph": False} + + benchmark_layer = _load( + ["benchmark_layer"], + { + "benchmark_with_power": benchmark_with_power, + "forward_context": lambda _ctx: contextlib.nullcontext(), + "ForwardContext": lambda **_kw: None, + }, + )["benchmark_layer"] + forward_batch = SimpleNamespace(attn_backend=None) + + with pytest.raises(RuntimeError, match="refusing to publish eager timing"): + benchmark_layer(lambda *a, **k: None, forward_batch, SimpleNamespace(device="cuda:0"), None, None, None, None) + + # graph timing is the helper default; the lane must not opt out of it + assert "use_cuda_graph" not in benchmark_kwargs + assert benchmark_kwargs.get("allow_graph_fail") is not True + + +def test_run_mla_has_no_eager_prefill_switch(): + source = SOURCE.read_text(encoding="utf-8") + assert "_eager_prefill" not in source + assert "use_cuda_graph=False" not in source + run_mla = next(node for node in _tree().body if isinstance(node, ast.FunctionDef) and node.name == "run_mla") + constructed = { + ast.unparse(node.func) + for node in ast.walk(run_mla) + if isinstance(node, ast.Call) and ast.unparse(node.func).endswith("TRTLLMMLABackend") + } + assert constructed == {"_GraphCapturableTRTLLMMLABackend"} diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py index 8b2a172c10..5bae25b425 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py @@ -9,6 +9,7 @@ from unittest.mock import patch import pytest +from collector.helper import PhaseClock, render_phase_totals @pytest.fixture(autouse=True) @@ -19,6 +20,8 @@ def _mock_helper_imports(monkeypatch): fake_helper.get_sm_version = lambda: 90 # default Hopper fake_helper.log_perf = lambda **kw: None fake_helper.benchmark_with_power = lambda **kw: None + fake_helper.PhaseClock = PhaseClock + fake_helper.render_phase_totals = render_phase_totals monkeypatch.setitem(__import__("sys").modules, "helper", fake_helper) fake_torch = types.ModuleType("torch") diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py index 2e9300a98f..fe5c2cc1b8 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py @@ -167,9 +167,12 @@ def test_kimi_int4_population_restores_all_hopper_ep_slices(): module_config={"group_size": 32}, ) - assert len(populated) == 3078 - assert sum(case[7] == 64 for case in populated) == 243 - assert sum(case[7] == 128 for case in populated) == 162 + # 114 (quant, shape, tp, ep, distribution) series x 27 token points = the 3078 + # per-token rows the per-token layout used to schedule as separate cases. + assert len(populated) == 114 + assert sum(len(case[1]) for case in populated) == 3078 + assert sum(len(case[1]) for case in populated if case[7] == 64) == 243 + assert sum(len(case[1]) for case in populated if case[7] == 128) == 162 # int4_wo opened to SM100/103 (owner decision 2026-07-11): serving # auto-selects flashinfer_trtllm for Kimi INT4 there (server_args.py:3736), # replacing the old zero-population contract. @@ -179,7 +182,7 @@ def test_kimi_int4_population_restores_all_hopper_ep_slices(): allowed_mode="int4_wo", module_config={"group_size": 32}, ) - assert len(sm100) == 3078 + assert len(sm100) == 114 and sum(len(case[1]) for case in sm100) == 3078 from types import SimpleNamespace from collector.case_generator import get_sglang_moe_backend @@ -291,7 +294,7 @@ def test_dsv4_fp4_population_keeps_runtime_limits_attempted(): allowed_mode="w4a8_mxfp4_mxfp8", ) assert len(attempted) == 1 - assert attempted[0][1:8] == [16384, 7168, 3072, 6, 384, 16, 2] + assert attempted[0][1:8] == [[16384], 7168, 3072, 6, 384, 16, 2] def test_sm120_shared_memory_predictions_do_not_remove_moe_cases(): @@ -504,7 +507,9 @@ def test_quantized_moe_uses_framework_path_and_fails_closed( def framework_benchmark(**kwargs): framework_calls.append(kwargs) - return 1.25, {"power": 100.0}, kernel_source + for num_tokens in kwargs["num_tokens_list"]: + kwargs["on_result"](num_tokens, 1.25, {"power": 100.0}, kernel_source) + return kernel_source run = _load_functions( "run_moe_torch", @@ -781,5 +786,161 @@ def test_framework_moe_source_comes_from_the_constructed_quant_method(): "ModelOptNvFp4FusedMoEMethod", ): assert method_name in framework_bench - assert 'return results["latency_ms"] / len(logits), results["power_stats"], kernel_source' in framework_bench + assert ( + 'on_result(num_tokens, results["latency_ms"] / len(logits), results["power_stats"], kernel_source)' + in framework_bench + ) assert 'model_name == "deepseek-ai/DeepSeek-V4-Pro"' not in run_moe + + +def test_moe_population_emits_one_case_per_token_sweep(): + """2026-10-28: SGLang nvfp4 shards took 4.4-5.8 h against 1-1.4 h for vLLM on the same + shapes because every token point was its own case and rebuilt the framework layer. + Cases now carry the whole sweep (vLLM/TRT-LLM layout) and dedup on the series.""" + case = _gptoss_case(tp=1, ep=1) + case.num_tokens_list = [1, 8, 64, 512] + duplicate = SimpleNamespace(**vars(case)) + duplicate.model_name = "example/equivalent-gptoss" + + populated = _populate_gptoss_cases([case, duplicate]) + + assert len(populated) == 1 + assert populated[0][1] == [1, 8, 64, 512] + assert populated[0][8] == "openai/gpt-oss-120b" + + +def test_run_moe_torch_times_every_token_point_on_one_framework_layer(): + logged = [] + framework_calls = [] + fake_torch = SimpleNamespace( + bfloat16=object(), + device=lambda value: value, + set_default_device=lambda _device: None, + cuda=SimpleNamespace( + set_device=lambda _device: None, + empty_cache=lambda: None, + memory_allocated=lambda _device: 0, + get_device_properties=lambda _device: SimpleNamespace(total_memory=100), + get_device_name=lambda _device: "B200", + ), + ) + + def framework_benchmark(**kwargs): + framework_calls.append(kwargs["num_tokens_list"]) + for num_tokens in kwargs["num_tokens_list"]: + kwargs["on_result"](num_tokens, float(num_tokens), {"power": 1.0}, "sglang_flashinfer_trtllm_moe") + return "sglang_flashinfer_trtllm_moe" + + run = _load_functions( + "run_moe_torch", + namespace={ + "torch": fake_torch, + "_benchmark_framework_quantized_moe": framework_benchmark, + "benchmark": lambda *_args, **_kwargs: pytest.fail("nvfp4 must use the framework layer"), + "get_moe_quantization_module_config": lambda *_args, **_kwargs: {}, + "get_sm_version": lambda: 100, + "_fmoe_kernels_mod": SimpleNamespace(_B_DESC_CACHE=SimpleNamespace(clear=lambda: None)), + "gc": SimpleNamespace(collect=lambda: None), + "log_perf": lambda **kwargs: logged.append(kwargs) or True, + "EXIT_CODE_RESTART": 10, + }, + )["run_moe_torch"] + + assert ( + run( + "nvfp4", + [1, 64, 4096], + 7168, + 2048, + 8, + 256, + 1, + 8, + "nvidia/GLM-5-NVFP4", + distributed="power_law", + power_law_alpha=1.2, + moe_backend="flashinfer_trtllm", + perf_filename="moe.csv", + ) + is None + ) + # One layer build for the whole sweep, one persisted row per token point. + assert framework_calls == [[1, 64, 4096]] + assert [row["item_list"][0]["num_tokens"] for row in logged] == [1, 64, 4096] + assert [row["item_list"][0]["latency"] for row in logged] == [1.0, 64.0, 4096.0] + assert {row["item_list"][0]["distribution"] for row in logged} == {"power_law_1.2"} + assert {row["kernel_source"] for row in logged} == {"sglang_flashinfer_trtllm_moe"} + + # A bare int (fills, direct callers) still measures exactly that point. + logged.clear() + run( + "nvfp4", + 768, + 7168, + 2048, + 8, + 256, + 1, + 8, + "nvidia/GLM-5-NVFP4", + moe_backend="flashinfer_trtllm", + perf_filename="moe.csv", + ) + assert [row["item_list"][0]["num_tokens"] for row in logged] == [768] + + +def test_raw_moe_case_persists_each_token_point_and_cleans_up_once(): + logged = [] + bench_calls = [] + cleared = [] + fake_torch = SimpleNamespace( + bfloat16=object(), + device=lambda value: value, + set_default_device=lambda _device: None, + cuda=SimpleNamespace( + set_device=lambda _device: None, + empty_cache=lambda: None, + memory_allocated=lambda _device: 0, + get_device_properties=lambda _device: SimpleNamespace(total_memory=100), + get_device_name=lambda _device: "H200", + ), + ) + + def fake_benchmark(num_tokens, *_args, **_kwargs): + bench_calls.append(num_tokens) + return 0.5 * num_tokens, {"power": 1.0} + + run = _load_functions( + "run_moe_torch", + namespace={ + "torch": fake_torch, + "benchmark": fake_benchmark, + "build_rank0_workloads": lambda **_kwargs: pytest.fail("EP=1 must not build rank-local workloads"), + "get_moe_quantization_module_config": lambda *_args, **_kwargs: {}, + "_benchmark_framework_quantized_moe": lambda **_kwargs: pytest.fail("BF16 must use the raw benchmark"), + "_fmoe_kernels_mod": SimpleNamespace(_B_DESC_CACHE=SimpleNamespace(clear=lambda: cleared.append(True))), + "gc": SimpleNamespace(collect=lambda: None), + "get_sm_version": lambda: 90, + "log_perf": lambda **kwargs: logged.append(kwargs) or True, + "EXIT_CODE_RESTART": 10, + }, + )["run_moe_torch"] + + run( + "bfloat16", + [16, 32], + 4096, + 14336, + 2, + 8, + 1, + 1, + "mistralai/Mixtral-8x7B-v0.1", + distributed="balanced", + perf_filename="moe.csv", + ) + + assert bench_calls == [16, 32] + assert [row["item_list"][0]["num_tokens"] for row in logged] == [16, 32] + assert [row["kernel_source"] for row in logged] == ["sglang_fused_moe_triton"] * 2 + assert cleared == [True] diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py index 67c911f1d1..26950d79ec 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py @@ -30,12 +30,17 @@ QWEN38MAX_LANES = ["bfloat16", "fp8_block", "nvfp4"] -def _load_guard(installed_version: str): +def _load_guard(installed_version: str, sm_version: int = 90): tree = ast.parse(SOURCE_PATH.read_text(encoding="utf-8"), filename=str(SOURCE_PATH)) function = next( node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_raise_if_unverified_moe_lane" ) - loaded = {"_dist_version": lambda _name: installed_version, "_check_compat": _check_compat} + loaded = { + "_check_compat": _check_compat, + # the guard reads the runtime through the module helpers; stub both + "_dist_version": lambda _name: installed_version, + "get_sm_version": lambda: sm_version, + } exec(compile(ast.Module(body=[function], type_ignores=[]), str(SOURCE_PATH), "exec"), loaded) return loaded["_raise_if_unverified_moe_lane"] @@ -69,3 +74,32 @@ def test_guard_never_fires_for_qwen38max_collected_lanes(moe_type, installed_ver guard = _load_guard(installed_version) assert guard(moe_type) == installed_version + + +@pytest.mark.parametrize("installed_version", ["0.5.21", "0.5.21.post1", "0.5.21+cu130"]) +@pytest.mark.parametrize("sm_version", [89, 90]) +@pytest.mark.parametrize("moe_type", ["int4_wo", "w4a16_mxfp4"]) +def test_guard_accepts_0521_hopper_lanes_reverified_2026_10_04(moe_type, installed_version, sm_version): + """0.5.21 Marlin (int4_wo) and Triton (w4a16_mxfp4) dispatch re-verified on SM89/90 only.""" + guard = _load_guard(installed_version, sm_version) + assert guard(moe_type) == installed_version + + +@pytest.mark.parametrize("moe_type", ["int4_wo", "w4a16_mxfp4", "w4a8_mxfp4_mxfp8"]) +@pytest.mark.parametrize("sm_version", [100, 103]) +def test_guard_accepts_0521_blackwell_mxfp4_lanes(moe_type, sm_version): + """SM100/103 MXFP4 lanes re-verified on 0.5.21 from the B200 identity records (6400345f); + int4_wo re-opened 2026-10-06 from the same records (Kimi-K2.5 -> FLASHINFER_TRTLLM / trtllm_gen_moe).""" + guard = _load_guard("0.5.21", sm_version) + assert guard(moe_type) == "0.5.21" + + +@pytest.mark.parametrize( + "moe_type, sm_version", + [("w4a8_mxfp4_mxfp8", 90), ("int4_wo", 120), ("w4a16_mxfp4", 120)], +) +def test_guard_keeps_0521_unverified_lanes_closed(moe_type, sm_version): + """Every SM120 weight-only lane and the DSV4 FP4 lane on Hopper were not re-verified.""" + guard = _load_guard("0.5.21", sm_version) + with pytest.raises(RuntimeError, match=rf"{moe_type}.*installed: 0.5.21, SM{sm_version}"): + guard(moe_type) diff --git a/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py b/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py index 46edb71f06..19b03ff100 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py @@ -3,6 +3,7 @@ """The SGLang MoE collector records, never chooses, the Triton config source.""" import json +import re from pathlib import Path from types import SimpleNamespace @@ -85,4 +86,7 @@ def test_sglang_collector_wraps_the_timed_call_and_writes_triton_rows_only(): source = (Path(__file__).resolve().parents[4] / "collector/sglang/collect_moe.py").read_text() assert "with observe_triton_config_selection(_triton_config_namespaces())" in source assert "return [sys.modules[get_config_dtype_str.__module__], globals()]" in source - assert 'if moe_backend == "triton":\n append_triton_config_record(' in source + # one task times a num_tokens series: each point's row gets its own counters + assert re.search(r'if moe_backend == "triton":\n\s+append_triton_config_record\(', source) + assert "point_observed = dict(triton_config_observed)" in source + assert "triton_config_observed[key] = 0" in source diff --git a/python/aisimulate/tests/unit/collector/test_b200_025_publication.py b/python/aisimulate/tests/unit/collector/test_b200_025_publication.py index f177bfcbec..3a320a4ebc 100644 --- a/python/aisimulate/tests/unit/collector/test_b200_025_publication.py +++ b/python/aisimulate/tests/unit/collector/test_b200_025_publication.py @@ -38,7 +38,9 @@ def test_published_025_metadata_loads_and_is_discoverable(): assert event["rows"] == source["source_rows"] assert "classified_failures" not in event assert isinstance(source["classified_failures"], int) and source["classified_failures"] >= 0 - assert total_rows == report["total_published_rows"] == 419538 + # 419538 at publication; the retired dsv4_hca_attn_module table (3918 rows) was + # removed 2026-10-15 and is listed under "withheld" with the reason. + assert total_rows == report["total_published_rows"] == 415620 @pytest.mark.parametrize("field", ["unexpected_runtime_field", "classified_failures"]) diff --git a/python/aisimulate/tests/unit/collector/test_benchmark_graph_scope.py b/python/aisimulate/tests/unit/collector/test_benchmark_graph_scope.py index 8f994eb0e1..7811043d58 100644 --- a/python/aisimulate/tests/unit/collector/test_benchmark_graph_scope.py +++ b/python/aisimulate/tests/unit/collector/test_benchmark_graph_scope.py @@ -6,6 +6,7 @@ import ast import logging import sys +import time from contextlib import contextmanager from functools import partial from pathlib import Path @@ -18,7 +19,7 @@ SOURCE = COLLECTOR / "helper.py" -def trace_benchmark(monkeypatch, *, source=SOURCE, failure=None, explicit=None, eager=False, fallback=False): +def trace_benchmark(monkeypatch, *, source=SOURCE, failure=None, explicit=None, eager=False, fallback=False, **extra): """Execute a helper source snapshot; retain graph references like a traceback.""" events, graphs = [], [] state = {"stream": "caller", "replays": 0, "fault_fired": False} @@ -99,7 +100,13 @@ def elapsed_time(self, end): monkeypatch.setitem(sys.modules, "torch", SimpleNamespace(cuda=cuda)) tree = ast.parse(source.read_text()) node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "benchmark_with_power") - namespace = {"contextmanager": contextmanager, "get_device_module": lambda device=None: cuda, "logging": logging} + namespace = { + "contextmanager": contextmanager, + "get_device_module": lambda device=None: cuda, + "logging": logging, + "time": time, + "worker_heartbeat": lambda: events.append("heartbeat"), + } exec(compile(ast.Module(body=[node], type_ignores=[]), str(source), "exec"), namespace) kwargs = { "device": SimpleNamespace(index=0), @@ -113,6 +120,7 @@ def elapsed_time(self, end): } if explicit is not None: kwargs["explicit_graph_cleanup"] = explicit + kwargs.update(extra) result, caught = None, None try: with namespace["benchmark_with_power"](**kwargs) as result: @@ -122,7 +130,8 @@ def elapsed_time(self, end): caught = {"type": type(exc).__name__, "message": str(exc), "original_error": exc is error} return { "events": events, - "result": result, + # phase_seconds are wall-clock readings that differ between otherwise identical runs. + "result": {k: v for k, v in result.items() if k != "phase_seconds"} if result is not None else None, "error": caught, "stream": state["stream"], "resets": [graph.resets for graph in graphs], @@ -179,12 +188,32 @@ def test_default_is_identical_to_explicit_false(monkeypatch): assert trace_benchmark(monkeypatch) == trace_benchmark(monkeypatch, explicit=False) +def test_eager_warmups_can_be_fewer_than_replay_warmups(monkeypatch): + observed = trace_benchmark(monkeypatch, num_eager_warmups=1) + assert observed["events"].count("kernel") == 2 # one eager warmup + capture + assert observed["events"].count("graph:replay") == 5 # replay warmups stay at num_warmups (2) + three runs + assert observed["result"]["used_cuda_graph"] is True + zero = trace_benchmark(monkeypatch, num_eager_warmups=0) + assert zero["events"].count("kernel") == 1 # capture only + default = trace_benchmark(monkeypatch) + assert default["events"].count("kernel") == 3 # num_warmups eager passes + capture, unchanged + + +def test_every_measurement_beats_the_stall_watchdog_before_touching_the_kernel(monkeypatch): + graph = trace_benchmark(monkeypatch) + assert graph["events"].count("heartbeat") == 1 + assert graph["events"].index("heartbeat") < graph["events"].index("kernel") + eager = trace_benchmark(monkeypatch, eager=True) + assert eager["events"].count("heartbeat") == 1 + + def test_eager_does_not_touch_graph_lifetime_in_either_mode(monkeypatch): default = trace_benchmark(monkeypatch, eager=True) opted = trace_benchmark(monkeypatch, eager=True, explicit=True) assert default == opted assert default["resets"] == [] - assert default["events"].count("kernel") == 7 + # eager: num_warmups (2) + num_runs (3); the helper no longer repeats the eager warmup after the capture step + assert default["events"].count("kernel") == 5 def test_only_vllm_mla_module_binds_explicit_graph_cleanup(): diff --git a/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py b/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py new file mode 100644 index 0000000000..5e00ce3506 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""``--ops`` naming an op the collector v2 case plan does not contain exits with the +dedicated EXIT_CODE_OPS_NOT_IN_PLAN, not argparse's 2, so a shard scheduler can mark +the shard terminal (sm100 shard plans 2026-10-05: the registry-only vllm +dsv4_hca_attn_module retried to max_attempts on eight systems). That op was removed +on 2026-10-15, so the probe below uses an op another backend's registry owns: a valid +``--ops`` choice that vLLM's case plan never carries, the shape of a stale or +cross-framework shard plan.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +PKG_ROOT = Path(__file__).resolve().parents[3] + + +def _run_collect(*args: str) -> subprocess.CompletedProcess: + env = dict(os.environ) + env.pop("COLLECTOR_MODEL_PATH", None) + return subprocess.run( + [sys.executable, str(PKG_ROOT / "collector" / "collect.py"), *args], + cwd=PKG_ROOT, + env=env, + capture_output=True, + text=True, + timeout=300, + ) + + +def test_op_outside_the_backend_plan_exits_with_the_dedicated_code(): + from collector.helper import EXIT_CODE_OPS_NOT_IN_PLAN + + # mamba2 is a trtllm registry op: accepted by --ops, never in vLLM's case plan. + result = _run_collect("--backend", "vllm", "--model-cases-full", "--plan-only", "--ops", "mamba2") + + assert result.returncode == EXIT_CODE_OPS_NOT_IN_PLAN, result.stderr[-2000:] + assert EXIT_CODE_OPS_NOT_IN_PLAN not in (0, 1, 2) + assert "Requested ops are not present in the collector v2 case plan: mamba2" in result.stderr + + +def test_planned_op_still_passes_the_ops_check(): + result = _run_collect("--backend", "vllm", "--model-cases-full", "--plan-only", "--ops", "gemm") + + assert result.returncode == 0, result.stderr[-2000:] diff --git a/python/aisimulate/tests/unit/collector/test_collect_provenance_writer.py b/python/aisimulate/tests/unit/collector/test_collect_provenance_writer.py index 38affbdca4..efebea3122 100644 --- a/python/aisimulate/tests/unit/collector/test_collect_provenance_writer.py +++ b/python/aisimulate/tests/unit/collector/test_collect_provenance_writer.py @@ -7237,7 +7237,7 @@ def test_dsa_registry_producers_share_transaction_and_checkpoint_ownership(tmp_p tracker = collect_mod._resume_tracker_for_collection( collection, ctx, backend=BACKEND, checkpoint_dir=str(checkpoint_dir), sm_version=100 ) - assert collect_mod._registered_checkpoint_table(tracker._metadata, backend=BACKEND) == table + assert collect_mod._registered_checkpoint_tables(tracker._metadata, backend=BACKEND) == {table} case_id = f"{collection['type']}-case" tracker.mark_attempted(case_id) tracker.mark_passed(case_id) @@ -7420,7 +7420,7 @@ def test_perf_journal_requires_attempts_for_each_table_in_a_batch(monkeypatch, s journal_path = output_root / collect_mod._PERF_TRANSACTION_FILENAME journal = json.loads(journal_path.read_text(encoding="utf-8")) for record in journal["checkpoints"]: - if record["table"] == "mla_bmm_perf": + if "mla_bmm_perf" in record["tables"]: record["attempted"] = [] assert any(record["attempted"] for record in journal["checkpoints"]) with pytest.raises(RuntimeError, match="tables lack attempted checkpoint case IDs"): diff --git a/python/aisimulate/tests/unit/collector/test_collect_replay.py b/python/aisimulate/tests/unit/collector/test_collect_replay.py new file mode 100644 index 0000000000..bf36467df3 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_collect_replay.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Failure replay (`collect.py --cases-from`): exact-string selection of an earlier run's +failing cases plus a seeded control sample, and the marker that keeps such a run out of +shipped data. Selection semantics only — the executor path is unchanged.""" + +import gzip +import json +import random +import sys +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +_COLLECTOR_DIR = str(Path(__file__).resolve().parents[3] / "collector") +if _COLLECTOR_DIR not in sys.path: + sys.path.insert(0, _COLLECTOR_DIR) + +import collect + +CASES = [ + [32, 255, 128, 1, 128, False, "flashinfer"], + [32, 511, 128, 1, 128, False, "flashinfer"], + [1, 8191, 96, 1, 128, True, "flashinfer"], + [2, 1023, 32, 8, 128, False, "fa3"], + {"id": "sglang.moe:run:[7, 8]", "params": [7, 8]}, # dict-form task +] + + +def _spec(wanted, control=0): + return collect.ReplaySpec("errors.json", set(wanted), control=control) + + +def test_select_matches_exact_case_string_not_substring(): + spec = _spec({"[32, 511, 128, 1, 128, False, 'flashinfer']"}) + out = spec.select("attention_generation", CASES, random.Random(42)) + assert out == [CASES[1]] + # ", 128, 1, 128, " would substring-match two cases; exact identity keeps one + assert spec.matched["attention_generation"] == {"[32, 511, 128, 1, 128, False, 'flashinfer']"} + assert spec.unmatched() == [] + + +def test_select_uses_params_for_dict_form_tasks(): + spec = _spec({"[7, 8]"}) + out = spec.select("moe", CASES, random.Random(42)) + assert out == [CASES[4]] + + +def test_control_sample_is_seeded_and_excludes_requested(): + wanted = {"[32, 255, 128, 1, 128, False, 'flashinfer']"} + a = _spec(wanted, control=2).select("op", CASES, random.Random(42)) + b = _spec(wanted, control=2).select("op", CASES, random.Random(42)) + assert a == b + assert a[0] == CASES[0] + assert len(a) == 3 and CASES[0] not in a[1:] + big = _spec(wanted, control=100) + assert len(big.select("op", CASES, random.Random(1))) == len(CASES) # capped at the rest + + +def test_report_lists_unmatched_requests_across_ops(): + spec = _spec({"[32, 255, 128, 1, 128, False, 'flashinfer']", "[9, 9, 9]"}) + spec.select("attention_generation", CASES, random.Random(42)) + spec.select("gemm", [], random.Random(42)) + rep = spec.report() + assert rep["requested"] == 2 and rep["matched"] == 1 + assert rep["unmatched"] == ["[9, 9, 9]"] + assert rep["matched_by_op"] == {"attention_generation": 1, "gemm": 0} + + +def test_load_replay_cases_from_errors_json_summary_gz_and_plain_list(tmp_path: Path): + errors = [ + {"task_params": "[32, 255, 128, 1, 128, False, 'flashinfer']", "error_type": "RuntimeError"}, + {"task_params": None, "error_type": "UnresolvedFailures"}, # checkpoint summary entry, no case + ] + ej = tmp_path / "errors_sglang.attention_generation.json" + ej.write_text(json.dumps(errors)) + assert collect.load_replay_cases(ej) == {"[32, 255, 128, 1, 128, False, 'flashinfer']"} + + summary = tmp_path / "collection_summary_sglang.json.gz" + with gzip.open(summary, "wt") as f: + json.dump({"summary": {"total_errors": 1}, "errors": errors}, f) + assert collect.load_replay_cases(summary) == {"[32, 255, 128, 1, 128, False, 'flashinfer']"} + + plain = tmp_path / "cases.json" + plain.write_text(json.dumps(["[1, 2]", "[3, 4]"])) + assert collect.load_replay_cases(plain) == {"[1, 2]", "[3, 4]"} + + lines = tmp_path / "cases.txt" + lines.write_text("[1, 2]\n\n[3, 4]\n") + assert collect.load_replay_cases(lines) == {"[1, 2]", "[3, 4]"} + + (tmp_path / "empty.json").write_text("[]") + with pytest.raises(ValueError): + collect.load_replay_cases(tmp_path / "empty.json") + + +def test_finish_replay_writes_marker_and_warns_on_unmatched(tmp_path: Path, caplog): + spec = _spec({"[32, 255, 128, 1, 128, False, 'flashinfer']", "[9, 9, 9]"}) + spec.select("attention_generation", CASES, random.Random(42)) + import logging + + log = logging.getLogger("replay-test") + with caplog.at_level(logging.INFO, logger="replay-test"): + collect._finish_replay(spec, tmp_path, log) + marker = json.loads((tmp_path / collect.ReplaySpec.MARKER).read_text()) + assert marker["requested"] == 2 and marker["matched"] == 1 and marker["unmatched"] == ["[9, 9, 9]"] + assert any("not in this run's case plan" in r.message for r in caplog.records) + assert collect._finish_replay(None, tmp_path, log) is None diff --git a/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py b/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py index 5d25bb7bec..5f6ecbf7c5 100644 --- a/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py +++ b/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py @@ -534,3 +534,94 @@ def test_no_fabricated_node_num_derivation_remains(): # (the docstrings cite the legacy expression, so match its code shape). assert "node_num = max(" not in SOURCE_TEXT assert "world_size // gpus_per_node" in SOURCE_TEXT + + +# --------------------------------------------------------------------------- +# CUDA-graph timing — no eager fallback, capture outcome agreed across ranks +# --------------------------------------------------------------------------- + + +def test_benchmark_op_agrees_capture_before_timing_and_never_falls_back(): + import ast + + tree = ast.parse(SOURCE_TEXT) + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_benchmark_op") + callees = [ast.unparse(node.func) for node in ast.walk(function) if isinstance(node, ast.Call)] + assert callees.index("_agree_graph_capturable") < callees.index("benchmark_with_power") + bench = next( + node + for node in ast.walk(function) + if isinstance(node, ast.Call) and ast.unparse(node.func) == "benchmark_with_power" + ) + assert all(keyword.arg != "allow_graph_fail" for keyword in bench.keywords) + assert "allow_graph_fail=True" not in SOURCE_TEXT + + +class _Flag: + def __init__(self, value): + self.value = value + + def item(self): + return self.value + + +def _fake_runtime(monkeypatch, *, local_capture_fails, peer_fails, world_size): + import sys + from contextlib import contextmanager + from types import SimpleNamespace + + events = [] + + @contextmanager + def graph(_graph): + events.append("capture") + if local_capture_fails: + raise RuntimeError("operation not permitted when stream is capturing") + yield + + def all_reduce(flag, op=None): + events.append("all_reduce") + flag.value = max(flag.value, 1 if peer_fails else 0) + + dist = SimpleNamespace( + is_available=lambda: True, + is_initialized=lambda: world_size > 1, + get_world_size=lambda: world_size, + all_reduce=all_reduce, + ReduceOp=SimpleNamespace(MAX="max"), + ) + torch = SimpleNamespace( + cuda=SimpleNamespace(CUDAGraph=lambda: object(), graph=graph, synchronize=lambda: events.append("sync")), + tensor=lambda data, **_kwargs: _Flag(data[0]), + int32="int32", + distributed=dist, + ) + monkeypatch.setitem(sys.modules, "torch", torch) + monkeypatch.setitem(sys.modules, "torch.distributed", dist) + return events + + +def test_capture_probe_warms_eagerly_then_captures_and_agrees(monkeypatch): + events = _fake_runtime(monkeypatch, local_capture_fails=False, peer_fails=False, world_size=8) + ata._agree_graph_capturable(lambda: events.append("func"), "dispatch", "cuda:0") + # one eager call (lazy init is not capturable), the probe capture, then the agreement + assert events == ["func", "sync", "capture", "func", "sync", "all_reduce"] + + +def test_capture_probe_skips_the_collective_in_a_single_rank_world(monkeypatch): + events = _fake_runtime(monkeypatch, local_capture_fails=False, peer_fails=False, world_size=1) + ata._agree_graph_capturable(lambda: None, "dispatch", "cuda:0") + assert "all_reduce" not in events + + +def test_local_capture_failure_is_agreed_then_raises_classified(monkeypatch): + events = _fake_runtime(monkeypatch, local_capture_fails=True, peer_fails=False, world_size=8) + with pytest.raises(ata.TrtllmAlltoallBenchmarkError, match=r"CUDA graph capture of combine failed \(RuntimeError"): + ata._agree_graph_capturable(lambda: None, "combine", "cuda:0") + assert "all_reduce" in events # the peers learn before any of them replays + + +def test_peer_capture_failure_raises_on_this_rank_too(monkeypatch): + _fake_runtime(monkeypatch, local_capture_fails=False, peer_fails=True, world_size=8) + with pytest.raises(ata.TrtllmAlltoallBenchmarkError, match="peer rank"): + ata._agree_graph_capturable(lambda: None, "prepare", "cuda:0") diff --git a/python/aisimulate/tests/unit/collector/test_dsv4_common_test_cases.py b/python/aisimulate/tests/unit/collector/test_dsv4_common_test_cases.py index b786a15261..fea3f3b580 100644 --- a/python/aisimulate/tests/unit/collector/test_dsv4_common_test_cases.py +++ b/python/aisimulate/tests/unit/collector/test_dsv4_common_test_cases.py @@ -127,7 +127,7 @@ def test_dsv4_cases_skip_unrelated_model_filter(monkeypatch): monkeypatch.setattr(sys, "argv", ["pytest"]) assert common_test_cases.get_dsv4_csa_context_test_cases() == [] - assert common_test_cases.get_dsv4_hca_attn_test_cases() == [] + assert common_test_cases.get_dsv4_paged_mqa_logits_test_cases() == [] def test_sglang_sparse_modules_topk_calib_uses_canonical_model(monkeypatch, capsys): diff --git a/python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py b/python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py new file mode 100644 index 0000000000..f3f6617e7f --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The DeepSeek-V4 FMLA sidecar ops ``dsv4_hca_attn_module`` / ``dsv4_csa_attn_module`` +were removed end to end on 2026-10-15: collector registries and case plan, ``PerfFile``, +the SDK ``PerfDataFilename``, the op catalog, the capability floors and the shipped +tables. Their SDK readers had been retired placeholders with no query path (csa_attn +never shipped data), and as registry-only ops every sm100 shard plan generated from +the registry since 2026-10-05 carried them and failed one or two shards per cell. +This guard keeps them out of every layer a shard plan or the SDK reads.""" + +from __future__ import annotations + +import importlib +from pathlib import Path + +import pytest +from collector.capabilities import _load_capabilities +from collector.op_catalog import load_family_map +from collector.registry_types import PerfFile + +pytestmark = pytest.mark.unit + +RETIRED_OPS = frozenset({"dsv4_hca_attn_module", "dsv4_csa_attn_module"}) +PKG_ROOT = Path(__file__).resolve().parents[3] + + +@pytest.mark.parametrize("backend", ("vllm", "sglang", "trtllm")) +def test_retired_sidecar_ops_are_not_registered(backend): + registered = {entry.op for entry in importlib.import_module(f"collector.{backend}.registry").REGISTRY} + assert not registered & RETIRED_OPS + + +def test_retired_sidecar_tables_have_no_perf_file_catalog_entry_or_floor(): + assert not {member.name for member in PerfFile} & {"DSV4_HCA_ATTN_MODULE", "DSV4_CSA_ATTN_MODULE"} + family_map = load_family_map() + assert family_map is not None + assert not {f"{op}_perf" for op in RETIRED_OPS} & set(family_map) + _, op_min_sm = _load_capabilities() + assert not RETIRED_OPS & set(op_min_sm) + + +def test_retired_sidecar_tables_have_no_sdk_filename_or_shipped_data(): + from aisimulate_core.sdk.common import PerfDataFilename + + assert not {member.name for member in PerfDataFilename} & RETIRED_OPS + data_root = PKG_ROOT / "src" / "aisimulate_core" / "systems" / "data" + shipped = sorted(str(p.relative_to(data_root)) for op in RETIRED_OPS for p in data_root.rglob(f"{op}_perf.parquet")) + assert shipped == [] diff --git a/python/aisimulate/tests/unit/collector/test_error_classification.py b/python/aisimulate/tests/unit/collector/test_error_classification.py new file mode 100644 index 0000000000..ba4c4ae91a --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_error_classification.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A collector-raised FIXME(kernel-limit) failure is reported as ``kernel_limit``, +everything else as ``unexpected`` (pipeline summaries classified by hand until +2026-10-08). Runs with PYTHONPATH=collector like test_registry_table_producers.""" + +import sys + +import pytest + +pytestmark = pytest.mark.unit + + +def _collect(): + if "torch" not in sys.modules: + from unittest.mock import MagicMock + + _torch = MagicMock() + _torch.AcceleratorError = type("AcceleratorError", (Exception,), {}) + sys.modules["torch"] = _torch + import collect + + return collect + + +def test_kernel_limit_prefix_is_classified(): + collect = _collect() + assert collect._classify_exception(RuntimeError("FIXME(kernel-limit): no head_dim=192 kernel")) == "kernel_limit" + assert collect._classify_exception(ValueError(" FIXME(kernel-limit): divisibility")) == "kernel_limit" + assert collect._classify_exception(RuntimeError("CUDA error: an illegal memory access")) == "unexpected" + assert collect._classify_exception(AssertionError()) == "unexpected" + + +@pytest.mark.parametrize( + "path, needle", + [ + ("collector/sglang/collect_moe.py", "FIXME(kernel-limit): SGLang Triton gated BF16 GELU"), + ("collector/trtllm/collect_moe.py", "FIXME(kernel-limit): TRT-LLM fused MoE requires"), + ("collector/trtllm/collect_attn.py", "FIXME(kernel-limit): TRT-LLM trtllm-gen FMHA has no head_dim=192"), + ("collector/vllm/collect_gemm.py", "FIXME(kernel-limit): vLLM fp8_block GEMM"), + # 2026-10-11 fix list: bare errors that were deterministic kernel limits + ("collector/trtllm/collect_gemm.py", "FIXME(kernel-limit): TRT-LLM fp8 GEMM (cublas/cutlass scaled_mm)"), + ("collector/sglang/collect_moe.py", "FIXME(kernel-limit): SGLang Triton gated BF16 GELU/SiLU"), + ("collector/sglang/collect_attn.py", "FIXME(kernel-limit): SGLang flashinfer has no FP8 prefill compute path"), + ( + "collector/sglang/collect_attn.py", + "FIXME(kernel-limit): SGLang {attn_backend_name} quantizes Q to FP8 internally", + ), + ], +) +def test_deterministic_guards_carry_the_prefix(path, needle): + from pathlib import Path + + root = Path(__file__).resolve().parents[3] + assert needle in (root / path).read_text(encoding="utf-8") diff --git a/python/aisimulate/tests/unit/collector/test_framework_manifest.py b/python/aisimulate/tests/unit/collector/test_framework_manifest.py index d37fff566c..b8dd9a740c 100644 --- a/python/aisimulate/tests/unit/collector/test_framework_manifest.py +++ b/python/aisimulate/tests/unit/collector/test_framework_manifest.py @@ -33,14 +33,16 @@ def test_manifest_exposes_current_framework_versions_and_images(): trtllm = get_collector_runtime("trtllm") vllm = get_collector_runtime("vllm") - assert sglang.version == "0.5.14" - assert sglang.image().startswith("lmsysorg/sglang:v0.5.14@sha256:") - assert sglang.image("cu130").startswith("lmsysorg/sglang:v0.5.14-cu130@sha256:") - assert trtllm.version == "1.3.0rc20" - assert trtllm.image().startswith("nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20@sha256:") - assert vllm.version == "0.24.0" - assert vllm.image().startswith("vllm/vllm-openai:v0.24.0@sha256:") - assert vllm.image("cu129").startswith("vllm/vllm-openai:v0.24.0-cu129@sha256:") + assert sglang.version == "0.5.21" + assert sglang.image().startswith("lmsysorg/sglang:v0.5.21@sha256:") + assert sglang.image("cu130").startswith("lmsysorg/sglang:v0.5.21-cu130@sha256:") + assert trtllm.version == "1.3.0rc29" + assert trtllm.image().startswith("nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc29@sha256:") + assert vllm.version == "0.30.0" + assert vllm.image().startswith("vllm/vllm-openai:v0.30.0@sha256:") + # cu129 variant intentionally dropped at the 0.29.0 bump (no cu129 build); + # unknown variants fall back to the pinned default image below. + assert vllm.image("cu129").startswith("vllm/vllm-openai:v0.30.0@sha256:") # Unknown variants intentionally fall back to the pinned default image. assert vllm.image("cu130") == vllm.image() @@ -81,10 +83,10 @@ def test_active_cuda_vllm_collectors_are_exactly_pinned_to_manifest_version(): ["collector.vllm.collect_moe", "collector.vllm.collect_gdn", "collector.vllm.collect_gemm"], ) def test_vllm_target_lane_collectors_declare_the_exact_bumped_compat_range(module): - expected = '__compat__ = "vllm>=0.24.0,<=0.27.1,!=0.25.1,!=0.26.0,!=0.27.0"' + expected = '__compat__ = "vllm>=0.24.0,<=0.30.0,!=0.25.1,!=0.26.0,!=0.27.0"' if module in {"collector.vllm.collect_gemm", "collector.vllm.collect_moe"}: - # GLM-5.3-Flash model pin: 0.31.0 is admitted; 0.28.0-0.30.0 are not. - expected = '__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0,!=0.28.0,!=0.29.0,!=0.30.0"' + # GLM-5.3-Flash model pin: 0.31.0 is admitted on top of the 0.30.0 default lanes. + expected = '__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0"' source = (REPO_ROOT / f"{module.replace('.', '/')}.py").read_text(encoding="utf-8") declarations = [line.strip() for line in source.splitlines() if line.startswith("__compat__")] assert declarations == [expected], module @@ -303,11 +305,11 @@ def test_wideep_entry_missing_data_backend_is_rejected(tmp_path): @pytest.mark.parametrize( ("installed_version", "requested_ops", "workload", "version"), [ - # "all ops" is no longer resolvable in one container for sglang — the - # kda family pins the kimi-k3 branch runtime (0.5.16), so the default - # expectation is asserted on an explicit default-family op instead. - ("0.5.14+cu130", {"gemm"}, "default", "0.5.14"), - ("0.5.16", {"kda"}, "default", "0.5.16"), + # Since the 0.5.21 default (2026-10-01) kda and msa resolve on the + # stock runtime again (the kimi-k3 / v0.5.16 family pins are retired). + ("0.5.21+cu130", {"gemm"}, "default", "0.5.21"), + ("0.5.21", {"kda"}, "default", "0.5.21"), + ("0.5.21", {"msa_context_module"}, "default", "0.5.21"), ("0.5.10", {"moe_ep"}, "wideep", "0.5.10"), ], ) @@ -332,14 +334,11 @@ def test_vllm_xpu_runtime_selection_rejects_version_mismatch(): @pytest.mark.parametrize( ("installed_version", "requested_ops", "match"), [ - ("0.5.13", {"gemm"}, r"stock collector requires exactly 0\.5\.14"), - ("0.5.14rc1", {"gemm"}, r"stock collector requires exactly 0\.5\.14"), - ("0.5.14.post1", {"gemm"}, r"stock collector requires exactly 0\.5\.14"), - ("0.5.14", {"moe_ep"}, r"WideEP collector requires exactly 0\.5\.10"), - ("0.5.14", {"gemm", "moe_ep"}, r"0\.5\.14 != 0\.5\.10.*separate containers"), - # kda runs only on the kimi-k3 branch runtime (families.kda pin): - # mixing it with a default-family op must fail closed. - ("0.5.14", {"gemm", "kda"}, r"multiple runtime versions"), + ("0.5.14", {"gemm"}, r"stock collector requires exactly 0\.5\.21"), + ("0.5.21rc1", {"gemm"}, r"stock collector requires exactly 0\.5\.21"), + ("0.5.21.post1", {"gemm"}, r"stock collector requires exactly 0\.5\.21"), + ("0.5.21", {"moe_ep"}, r"WideEP collector requires exactly 0\.5\.10"), + ("0.5.21", {"gemm", "moe_ep"}, r"0\.5\.21 != 0\.5\.10.*separate containers"), ], ) def test_runtime_selection_rejects_mismatched_or_mixed_pins(installed_version, requested_ops, match): @@ -353,8 +352,8 @@ def test_runtime_selection_rejects_mismatched_or_mixed_pins(installed_version, r @pytest.mark.parametrize( ("installed_version", "requested_ops", "workload", "version"), [ - ("0.5.14+cu130", {"gemm"}, "default", "0.5.14"), - ("0.5.16", {"kda"}, "default", "0.5.16"), + ("0.5.21+cu130", {"gemm"}, "default", "0.5.21"), + ("0.5.21", {"kda"}, "default", "0.5.21"), ("0.5.10", {"moe_ep"}, "wideep", "0.5.10"), ], ) @@ -374,10 +373,10 @@ def test_no_model_identity_resolves_exactly_like_today(installed_version, reques def test_unknown_model_id_falls_back_to_default_resolution(): - baseline = require_collector_runtime("sglang", "0.5.14", requested_ops={"gemm"}, wideep_ops=WIDEEP_OPS) + baseline = require_collector_runtime("sglang", "0.5.21", requested_ops={"gemm"}, wideep_ops=WIDEEP_OPS) unmatched = require_collector_runtime( "sglang", - "0.5.14", + "0.5.21", requested_ops={"gemm"}, wideep_ops=WIDEEP_OPS, model_path="some-org/not-a-pinned-model", @@ -410,7 +409,7 @@ def test_model_pin_mismatch_error_names_the_model_scoped_image(): with pytest.raises(RuntimeError) as excinfo: require_collector_runtime( "sglang", - "0.5.14", + "0.5.21", requested_ops={"gemm"}, wideep_ops=WIDEEP_OPS, model_path="Qwen/Qwen3.8-2.4T-A95B", @@ -418,7 +417,7 @@ def test_model_pin_mismatch_error_names_the_model_scoped_image(): message = str(excinfo.value) # Same template as the pre-4b guard ("~:249"), but naming the # model-scoped runtime/image instead of the framework default. - assert "sglang stock collector requires exactly 0.5.17, found 0.5.14" in message + assert "sglang stock collector requires exactly 0.5.17, found 0.5.21" in message assert "use lmsysorg/sglang:v0.5.17@sha256:" in message @@ -455,10 +454,10 @@ def test_vllm_model_pin_mismatch_error_names_the_model_scoped_image(): def test_vllm_unknown_model_id_falls_back_to_default_resolution(): - baseline = require_collector_runtime("vllm", "0.24.0", requested_ops={"gemm"}, wideep_ops=set()) + baseline = require_collector_runtime("vllm", "0.30.0", requested_ops={"gemm"}, wideep_ops=set()) unmatched = require_collector_runtime( "vllm", - "0.24.0", + "0.30.0", requested_ops={"gemm"}, wideep_ops=set(), model_path="some-org/not-a-pinned-model", @@ -476,7 +475,7 @@ def test_real_manifest_models_section_does_not_break_validate_resolution(): def test_unknown_requested_op_fails_with_key_error(): with pytest.raises(KeyError, match=r"has no op\(s\): \['not_a_real_op'\]"): - require_collector_runtime("sglang", "0.5.14", requested_ops={"not_a_real_op"}, wideep_ops=set()) + require_collector_runtime("sglang", "0.5.21", requested_ops={"not_a_real_op"}, wideep_ops=set()) def test_vllm_xpu_unknown_requested_op_fails_with_key_error(): @@ -487,7 +486,7 @@ def test_vllm_xpu_unknown_requested_op_fails_with_key_error(): def test_typo_mixed_with_real_op_fails_closed(): # A typo must not be silently dropped just because another requested op is valid. with pytest.raises(KeyError, match=r"has no op\(s\): \['not_a_real_op'\]"): - require_collector_runtime("sglang", "0.5.14", requested_ops={"gemm", "not_a_real_op"}, wideep_ops=set()) + require_collector_runtime("sglang", "0.5.21", requested_ops={"gemm", "not_a_real_op"}, wideep_ops=set()) def test_wideep_registry_entries_are_separate_from_stock_backend_registries(): @@ -796,9 +795,9 @@ def test_model_pin_images_must_be_digest_pinned(tmp_path): ("0.26.0", False), ("0.27.0", False), ("0.27.1", True), - ("0.28.0", False), - ("0.29.0", False), - ("0.30.0", False), + ("0.28.0", True), + ("0.29.0", True), + ("0.30.0", True), ("0.31.0", True), ("0.31.1", False), ], @@ -896,4 +895,4 @@ def test_explicit_nondefault_manifest_path_is_not_overridden(tmp_path, monkeypat path.write_bytes((COLLECTOR_ROOT / "framework_manifest.yaml").read_bytes()) monkeypatch.setenv(RUNTIME_MANIFEST_ENV, "/missing/ambient/manifest") monkeypatch.setenv(RUNTIME_MANIFEST_SHA256_ENV, "0" * 64) - assert get_collector_runtime("vllm", path=path).version == "0.24.0" + assert get_collector_runtime("vllm", path=path).version == "0.30.0" diff --git a/python/aisimulate/tests/unit/collector/test_getter_deduplication.py b/python/aisimulate/tests/unit/collector/test_getter_deduplication.py index a3c9d87ffc..c37078e5bf 100644 --- a/python/aisimulate/tests/unit/collector/test_getter_deduplication.py +++ b/python/aisimulate/tests/unit/collector/test_getter_deduplication.py @@ -137,6 +137,7 @@ def _install_vllm_stubs(monkeypatch): "collector.helper", balanced_logits=_noop, benchmark_with_power=_noop, + flashinfer_autotuned=_noop, get_sm_version=lambda: 100, log_perf=_noop, power_law_logits_v3=_noop, diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py index 6c98432a6a..dcb41bad5c 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py @@ -170,9 +170,9 @@ def test_vllm_nvfp4_uses_modelopt_only_for_the_glm53flash_checkpoint(monkeypatch "version,accepted", [ ("0.31.0", True), - ("0.30.0", False), - ("0.28.0", False), - ("0.29.0", False), + ("0.30.0", True), # the collector default lanes + ("0.28.0", True), + ("0.29.0", True), ("0.31.1", False), ("0.32.0", False), ("0.24.0", True), @@ -198,6 +198,8 @@ def test_sglang_0_5_20_runs_delegate_to_the_serving_built_fork(path, fork, names and "(0, 5, 20)" in ast.unparse(node.test) ) body = ast.unparse(gate) + # exactly 0.5.20: the 0.5.21 default keeps the module's own implementation + assert "==" in ast.unparse(gate.test) assert f"from collector.sglang import {fork} as _fork_0520" in body for name in names: assert f"{name} = _fork_0520.{name}" in body @@ -219,6 +221,7 @@ def test_sglang_moe_admits_0_5_20_but_not_unaudited_releases(): ("0.5.18", False), ("0.5.19", False), ("0.5.20", True), + ("0.5.21", True), ]: assert _check_compat(compat, version) is accepted, version diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py b/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py index 1e5c52fee3..45db41e170 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py @@ -49,11 +49,11 @@ def test_glm_pin_rejects_other_runtimes(backend, wrong_version, expected): @pytest.mark.parametrize( "backend,version,model_path", [ - ("vllm", "0.24.0", None), - ("vllm", "0.24.0", "zai-org/GLM-5.2-FP8"), + ("vllm", "0.30.0", None), + ("vllm", "0.30.0", "zai-org/GLM-5.2-FP8"), ("vllm", "0.27.1", "Qwen/Qwen3.8-2.4T-A95B"), - ("sglang", "0.5.14", None), - ("sglang", "0.5.14", "zai-org/GLM-5.2-FP8"), + ("sglang", "0.5.21", None), + ("sglang", "0.5.21", "zai-org/GLM-5.2-FP8"), ("sglang", "0.5.17", "Qwen/Qwen3.8-2.4T-A95B"), ], ) diff --git a/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py b/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py new file mode 100644 index 0000000000..fd404e34d3 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py @@ -0,0 +1,447 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""``helper.flashinfer_autotuned`` times FlashInfer-backed kernels on the autotuned +tactic (serving truth) instead of FlashInfer's untuned fallback, whose default kernel +moved between FlashInfer 0.6.12 and 0.6.18 and broke the SM100 nvfp4 MoE EP sweeps of +the vLLM 0.30.0 / SGLang 0.5.21 campaign tables (2026-10-27).""" + +from __future__ import annotations + +import sys +import types +from contextlib import contextmanager +from typing import ClassVar + +import pytest +from collector import helper +from collector.helper import ( + FLASHINFER_AUTOTUNE_BUDGET_ENV, + FLASHINFER_AUTOTUNE_ENV, + FlashInferTuneBudget, + flashinfer_autotuned, +) + +pytestmark = pytest.mark.unit + +_REAL_DEVICE_TAG = helper._flashinfer_device_tag + + +class _FakeTuner: + instance = None + + def __init__(self): + self.profiling_cache = {} + # FlashInfer's own per-tactic profiling budget (AutoTuner.__init__ defaults). + self.warmup = 3 + self.repeat = 10 + self.stream_delay_micro_secs = 5000 + + def budget(self): + return (self.warmup, self.repeat, self.stream_delay_micro_secs) + + @classmethod + def get(cls): + if cls.instance is None: + cls.instance = cls() + return cls.instance + + +@pytest.fixture +def fake_flashinfer(monkeypatch): + """A stand-in ``flashinfer.autotuner`` that records every ``autotune`` context.""" + _FakeTuner.instance = None + events: list[tuple] = [] + active: list[tuple] = [] + + @contextmanager + def autotune(tune_mode=True, cache=None, tuning_buckets=None, round_up=None, skip_ops=None): + entry = (tune_mode, tuning_buckets) + events.append(("enter", *entry)) + active.append(entry) + try: + yield + finally: + active.pop() + events.append(("exit", *entry)) + + module = types.ModuleType("flashinfer.autotuner") + module.autotune = autotune + module.AutoTuner = _FakeTuner + package = types.ModuleType("flashinfer") + package.autotuner = module + monkeypatch.setitem(sys.modules, "flashinfer", package) + monkeypatch.setitem(sys.modules, "flashinfer.autotuner", module) + monkeypatch.delenv(FLASHINFER_AUTOTUNE_ENV, raising=False) + monkeypatch.delenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, raising=False) + # Healthy CUDA context unless a test says otherwise; no device tag (other test modules + # leave a stub ``torch`` in sys.modules, so the tag must not depend on the import). + monkeypatch.setattr(helper, "cuda_context_error", lambda device=None: None) + monkeypatch.setattr(helper, "_flashinfer_device_tag", lambda: "") + return types.SimpleNamespace(events=events, active=active, tuner=_FakeTuner) + + +def test_tunes_once_at_the_exact_token_count_then_times_under_lookup_only(fake_flashinfer, capsys): + seen: list[tuple] = [] + + # A key tuned by an earlier case on this worker is not reported again. + fake_flashinfer.tuner.get().profiling_cache[("moe", 64)] = ((64, 1), None) + + def tune_fn(): + # The tuning forward runs inside tune_mode=True with the exact bucket and + # stands in for the autotuner filling its cache. + seen.append((tuple(fake_flashinfer.active), fake_flashinfer.tuner.get().budget())) + fake_flashinfer.tuner.get().profiling_cache[("moe", 320)] = ("tactic", "profile") + + with flashinfer_autotuned(320, tune_fn, label="vllm_trtllm_nvfp4") as tuned: + assert tuned is True + assert fake_flashinfer.active == [(False, (320,))] + # The reduced budget applies to the tuning pass only; the timed body sees + # FlashInfer's own values again. + assert fake_flashinfer.tuner.get().budget() == (3, 10, 5000) + + # Tuned under the collector's default budget: 1 warm-up, 3 timed, 1 ms delay. + assert seen == [(((True, (320,)),), (1, 3, 1000))] + assert fake_flashinfer.tuner.get().budget() == (3, 10, 5000) + assert fake_flashinfer.events == [ + ("enter", True, (320,)), + ("exit", True, (320,)), + ("enter", False, (320,)), + ("exit", False, (320,)), + ] + out = capsys.readouterr().out + assert "FlashInfer autotune: 1 new tactic entries at num_tokens=320 vllm_trtllm_nvfp4 [" in out + assert "s, warmup=1 repeat=3 delay_us=1000]" in out + assert "FlashInfer autotune chosen at num_tokens=320 vllm_trtllm_nvfp4: ('moe', 320) -> tactic 'tactic'" in out + assert "WARNING" not in out + + +def test_budget_env_overrides_or_keeps_flashinfer_defaults(fake_flashinfer, monkeypatch, capsys): + tuner = fake_flashinfer.tuner.get() + seen = [] + + monkeypatch.setenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, "2, 5, 2000") + with flashinfer_autotuned(64, lambda: seen.append(tuner.budget())): + pass + assert seen == [(2, 5, 2000)] + assert tuner.budget() == (3, 10, 5000) + assert "warmup=2 repeat=5 delay_us=2000]" in capsys.readouterr().out + + monkeypatch.setenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, "flashinfer") + with flashinfer_autotuned(64, lambda: seen.append(tuner.budget())): + pass + assert seen[-1] == (3, 10, 5000) + assert "flashinfer defaults]" in capsys.readouterr().out + + +@pytest.mark.parametrize("raw", ["3,10", "a,b,c", "1,0,1000", "-1,3,1000", "1,3,-5"]) +def test_budget_env_rejects_malformed_values(monkeypatch, raw): + monkeypatch.setenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, raw) + with pytest.raises(ValueError, match=FLASHINFER_AUTOTUNE_BUDGET_ENV): + helper.flashinfer_autotune_budget() + + +def test_budget_default_and_restore_when_tune_fn_raises(fake_flashinfer, monkeypatch): + monkeypatch.delenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, raising=False) + assert helper.flashinfer_autotune_budget() == FlashInferTuneBudget(1, 3, 1000) + tuner = fake_flashinfer.tuner.get() + + def boom(): + raise RuntimeError("tuning forward failed") + + with pytest.raises(RuntimeError, match="tuning forward failed"), flashinfer_autotuned(8, boom): + pass + assert tuner.budget() == (3, 10, 5000) + assert fake_flashinfer.active == [] + + +def test_env_switch_restores_the_untuned_measurement(fake_flashinfer, monkeypatch): + monkeypatch.setenv(FLASHINFER_AUTOTUNE_ENV, "0") + calls = [] + + with flashinfer_autotuned(1024, lambda: calls.append("tuned")) as tuned: + assert tuned is False + assert fake_flashinfer.active == [] + + assert calls == [] + assert fake_flashinfer.events == [] + + +def test_without_flashinfer_the_body_runs_untouched(monkeypatch): + monkeypatch.delenv(FLASHINFER_AUTOTUNE_ENV, raising=False) + monkeypatch.setattr(helper, "_flashinfer_autotune_api", lambda: (None, None)) + calls = [] + + with flashinfer_autotuned(64, lambda: calls.append("tuned")) as tuned: + assert tuned is False + + assert calls == [] + + +def test_default_is_on_and_accepts_the_shared_bool_grammar(monkeypatch): + monkeypatch.delenv(FLASHINFER_AUTOTUNE_ENV, raising=False) + assert helper.flashinfer_autotune_enabled() is True + for value, expected in (("false", False), ("no", False), ("1", True), ("TRUE", True)): + monkeypatch.setenv(FLASHINFER_AUTOTUNE_ENV, value) + assert helper.flashinfer_autotune_enabled() is expected + + +def test_both_moe_collectors_wrap_their_timing_in_the_tuning_context(): + """The call sites are GPU-only; pin the wiring statically so a refactor cannot + silently drop the tuning pass from either collector.""" + from pathlib import Path + + collector_root = Path(helper.__file__).resolve().parent + for backend in ("vllm", "sglang"): + source = (collector_root / backend / "collect_moe.py").read_text(encoding="utf-8") + assert "flashinfer_autotuned," in source, backend + assert source.count("flashinfer_autotuned(num_tokens, ") == 1, backend + tuned_at = source.index("flashinfer_autotuned(num_tokens, ") + timed_at = source.index("benchmark_with_power(", tuned_at) + assert 0 < timed_at - tuned_at < 400, f"{backend}: tuning context must directly wrap the timing" + + +def test_poisoned_context_after_the_tuning_pass_is_reported_as_the_autotune_failure(fake_flashinfer, monkeypatch): + """gb200 job 512476645 (2026-10-27): a Llama-4 (topk=1) nvfp4 case faulted inside a + candidate tactic; FlashInfer swallowed it, the chosen tactic then failed with an opaque + 'Got CUDA error', and the next case on the worker inherited the dead context.""" + tuner = fake_flashinfer.tuner.get() + tuner.stats = types.SimpleNamespace(failed_tactics={"flashinfer::trtllm_fp4_block_scale_moe": {(16, 3), (32, 7)}}) + dead = "AcceleratorError: CUDA error: an illegal memory access was encountered" + monkeypatch.setattr(helper, "cuda_context_error", lambda device=None: dead) + timed = [] + + with pytest.raises(RuntimeError) as info, flashinfer_autotuned(1, lambda: None, label="llama4_nvfp4"): + timed.append("timed") + + msg = str(info.value) + assert "FlashInfer autotune pass at num_tokens=1 llama4_nvfp4 left the CUDA context in error state" in msg + assert "illegal memory access" in msg + assert "2 failed tactics" in msg and "(16, 3)" in msg + assert timed == [] # the measurement never ran on the dead context + assert fake_flashinfer.events == [("enter", True, (1,)), ("exit", True, (1,))] + + +def test_tune_fn_error_on_a_healthy_context_propagates_unchanged(fake_flashinfer): + def tune_fn(): + raise ValueError("shape rejected") + + with pytest.raises(ValueError, match="shape rejected"), flashinfer_autotuned(8, tune_fn): + pass + + +def test_cuda_context_error_probe_reports_a_dead_context(monkeypatch): + class _Cuda: + @staticmethod + def current_device(): + return 0 + + @staticmethod + def synchronize(device): + raise RuntimeError("CUDA error: an illegal memory access was encountered") + + fake_torch = types.SimpleNamespace(cuda=_Cuda, ones=lambda *a, **k: None) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + assert helper.cuda_context_error() == "RuntimeError: CUDA error: an illegal memory access was encountered" + + class _HealthyCuda(_Cuda): + @staticmethod + def synchronize(device): + return None + + class _Tensor: + def sum(self): + return self + + def item(self): + return 1.0 + + fake_torch = types.SimpleNamespace(cuda=_HealthyCuda, ones=lambda *a, **k: _Tensor()) + monkeypatch.setitem(sys.modules, "torch", fake_torch) + assert helper.cuda_context_error(0) is None + + +def test_autotuner_fallback_messages_are_surfaced_and_logging_is_restored(fake_flashinfer, capsys): + """b300 sglang 0.5.21 (2026-10-28): the Nemotron-3-Nano nvfp4 relu2 shape came back + identical to the untuned table while every other shape moved -- FlashInfer's + ``MoERunner.get_valid_tactics`` swallows the C++ exception, logs it at DEBUG and + returns ``[]``, so the op is timed on the fallback with no visible trace. The tuning + pass captures that log and prints the explanation next to the case.""" + import logging + + fi_logger = logging.getLogger("flashinfer") + # Other test modules leave this logger at an explicit level (seen at ERROR in + # the full collector run); the scenario needs the production default, NOTSET. + inherited_level = fi_logger.level + fi_logger.setLevel(logging.NOTSET) + # The threshold the capture pins pre-existing handlers at is the logger's EFFECTIVE level, + # inherited from the root logger — WARNING by default, INFO when another test module + # configured logging in this process; read it instead of assuming. + effective_before = fi_logger.getEffectiveLevel() + stream = logging.StreamHandler() + fi_logger.addHandler(stream) + try: + seen_levels = [] + + def tune_fn(): + seen_levels.append((fi_logger.level, stream.level)) + fi_logger.debug( + "[Autotuner]: Failed to get valid tactics for (E2m1, 6, 2688, 1920, 128, Relu2). " + "Error occurred: No kernel found for the given options: mActType 6 tile 192" + ) + fi_logger.debug("[Autotuner]: profiling runner MoERunner tactic (128, 3)") + fi_logger.debug("[AutoTuner]: Using fallback tactic for trtllm_fp4_block_scale_moe") + fi_logger.debug("[AutoTuner]: Using fallback tactic for trtllm_fp4_block_scale_moe") # duplicate + + with flashinfer_autotuned(768, tune_fn, label="sglang_flashinfer_trtllm_moe"): + pass + + # Captured at DEBUG, while the pre-existing stream handler was pinned at the old + # effective threshold so FlashInfer's stderr output did not change. + assert seen_levels == [(logging.DEBUG, effective_before)] + assert fi_logger.level == logging.NOTSET + assert stream.level == logging.NOTSET + assert all(h is not fi_logger for h in fi_logger.handlers) and stream in fi_logger.handlers + finally: + fi_logger.removeHandler(stream) + fi_logger.setLevel(inherited_level) + + out = capsys.readouterr().out + assert "FlashInfer autotune: 0 new tactic entries at num_tokens=768 sglang_flashinfer_trtllm_moe [" in out + warning = [line for line in out.splitlines() if line.startswith("FlashInfer autotune WARNING")] + assert len(warning) == 1 + assert "num_tokens=768 sglang_flashinfer_trtllm_moe: 2 autotuner messages" in warning[0] + assert "No kernel found for the given options: mActType 6 tile 192" in warning[0] + assert warning[0].count("Using fallback tactic") == 1 + assert "profiling runner" not in warning[0] + assert "tactics failed while profiling" not in warning[0] + + +def test_per_tactic_profiling_failures_are_counted_from_the_tuner_statistics(fake_flashinfer, capsys): + """Every candidate tactic runs the whole op on synthetic inputs; when e.g. the routing + kernel faults on them FlashInfer marks the tactic inf at DEBUG, records it in + ``stats.failed_tactics`` and -- if none survives -- serves the fallback. The pass + reports the new failures from the statistics even when no log line matched.""" + import logging + + tuner = fake_flashinfer.tuner.get() + op = "flashinfer::trtllm_fp4_block_scale_moe::MoERunner" + tuner.stats = types.SimpleNamespace(failed_tactics={op: {(8, 1)}}) + fi_logger = logging.getLogger("flashinfer") + + def tune_fn(): + tuner.stats.failed_tactics[op].update({(16, 2), (32, 5)}) + fi_logger.debug( + "[Autotuner]: Failed when profiling MoERunner (16, 2), shapes=[...]. Error occurred: " + "CUDA error in trtllm_fused_moe_routing_custom.cu:1045: an illegal memory access" + ) + fi_logger.info("[Autotuner]: Skipped 2 unsupported tactic(s) for trtllm_fp4_block_scale_moe") + + with flashinfer_autotuned(3072, tune_fn, label="sglang_flashinfer_trtllm_moe"): + pass + + out = capsys.readouterr().out + warning = [line for line in out.splitlines() if line.startswith("FlashInfer autotune WARNING")] + assert len(warning) == 1 + assert f"tactics failed while profiling: {op}=2" in warning[0] + assert "routing_custom.cu:1045" in warning[0] + assert "Skipped 2 unsupported tactic(s)" in warning[0] + + +def test_newly_enumerated_tactic_tables_are_printed_per_instance_key(fake_flashinfer, monkeypatch, capsys): + """Probe 77139257 (b200 sglang 0.5.21, 2026-10-28): the Nemotron relu2 nvfp4 cases wrote one + cache entry each after a 0.02-0.9 s pass and still measured the untuned latency -- the + autotuner may simply have had one candidate. Print the launcher-enumerated tactic list + per instance key (intermediate size, activation, ... included) for every key the pass + added, reading FlashInfer's already-built module only (no JIT build from here).""" + + class _Runner: + valid_tactics_dict: ClassVar[dict] = {("E2m1", 8, 7168, 2048, 256, "Swiglu", 768): [(64, 0), (64, 1), (128, 0)]} + + calls = [] + + class _Impl: + currsize = 0 + + @staticmethod + def cache_info(): + return types.SimpleNamespace(currsize=_Impl.currsize) + + def get_module(): + calls.append("get") + return types.SimpleNamespace(MoERunner=_Runner) + + core = types.ModuleType("flashinfer.fused_moe.core") + core._get_trtllm_moe_sm100_module_impl = _Impl + core.get_trtllm_moe_sm100_module = get_module + fused = types.ModuleType("flashinfer.fused_moe") + fused.core = core + sys.modules["flashinfer"].fused_moe = fused + monkeypatch.setitem(sys.modules, "flashinfer.fused_moe", fused) + monkeypatch.setitem(sys.modules, "flashinfer.fused_moe.core", core) + + # Module not built yet: nothing is read and the factory is not called. + with flashinfer_autotuned(8, lambda: None, label="cold"): + pass + assert calls == [] + assert "FlashInfer autotune tactics" not in capsys.readouterr().out + + def tune_fn(): + _Impl.currsize = 1 # the forward built the module ... + _Runner.valid_tactics_dict[("E2m1", 6, 2688, 1920, 128, "Relu2", 768)] = [(64, 3)] # ... and enumerated + + with flashinfer_autotuned(768, tune_fn, label="sglang_flashinfer_trtllm_moe"): + pass + out = capsys.readouterr().out + lines = [line for line in out.splitlines() if line.startswith("FlashInfer autotune tactics")] + # The module came up during this pass, so every key it holds is new to the snapshot: + # the control shape's three tactics and the single Relu2 candidate. + assert len(lines) == 2 + assert ( + "FlashInfer autotune tactics at num_tokens=768 sglang_flashinfer_trtllm_moe: trtllm_sm100 " + "('E2m1', 8, 7168, 2048, 256, 'Swiglu', 768) -> 3 tactics [(64, 0), (64, 1), (128, 0)]" + ) in lines + assert ( + "FlashInfer autotune tactics at num_tokens=768 sglang_flashinfer_trtllm_moe: trtllm_sm100 " + "('E2m1', 6, 2688, 1920, 128, 'Relu2', 768) -> 1 tactics [(64, 3)]" + ) in lines + + # Next pass: only keys added since the snapshot are reported. + def tune_fn_ep4(): + _Runner.valid_tactics_dict[("E2m1", 6, 2688, 1920, 32, "Relu2", 768)] = [(64, 3)] + + with flashinfer_autotuned(768, tune_fn_ep4, label="sglang_flashinfer_trtllm_moe"): + pass + lines = [line for line in capsys.readouterr().out.splitlines() if line.startswith("FlashInfer autotune tactics")] + assert lines == [ + "FlashInfer autotune tactics at num_tokens=768 sglang_flashinfer_trtllm_moe: trtllm_sm100 " + "('E2m1', 6, 2688, 1920, 32, 'Relu2', 768) -> 1 tactics [(64, 3)]" + ] + + +def test_autotune_lines_carry_the_cuda_device_tag(fake_flashinfer, monkeypatch, capsys): + """Parallel workers share one log and tune the same key independently; the device tag + is what lets a per-worker tactic choice be reconstructed afterwards (gb200 sglang part07 + could not be: no worker tag on the autotune lines).""" + + class _Cuda: + @staticmethod + def is_available(): + return True + + @staticmethod + def current_device(): + return 3 + + monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(cuda=_Cuda)) + monkeypatch.setattr(helper, "_flashinfer_device_tag", _REAL_DEVICE_TAG) # fixture blanks it + assert helper._flashinfer_device_tag() == " dev3" + + def tune_fn(): + fake_flashinfer.tuner.get().profiling_cache[("moe", 96)] = ((128, 2), None) + + with flashinfer_autotuned(96, tune_fn, label="sglang_flashinfer_trtllm_moe"): + pass + out = capsys.readouterr().out + assert "FlashInfer autotune: 1 new tactic entries at num_tokens=96 sglang_flashinfer_trtllm_moe [" in out + assert "delay_us=1000] dev3\n" in out + assert "-> tactic (128, 2) dev3\n" in out diff --git a/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py b/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py new file mode 100644 index 0000000000..e7fdd2a2c8 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py @@ -0,0 +1,107 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +import time + +import pytest +from collector.helper import PhaseClock, render_phase_totals + +pytestmark = pytest.mark.unit + + +def test_phase_clock_marks_absorbs_and_renders(): + clock = PhaseClock() + time.sleep(0.01) + clock.mark("build") + clock.mark("build") # a second slice accumulates into the same phase + clock.absorb({"warmup": 0.5, "timed": 0.25}, prefix="bench_") + clock.absorb(None) + assert clock.phases["build"] >= 0.01 + assert clock.phases["bench_warmup"] == 0.5 + assert clock.phases["bench_timed"] == 0.25 + rendered = clock.render() + assert rendered.startswith("build=") + assert "bench_warmup=0.50s" in rendered + assert rendered.split()[-1].startswith("total=") and rendered.endswith("s") + assert clock.total() >= clock.phases["build"] + + +def test_phase_totals_accumulate_points_and_render_without_the_counter(): + clock = PhaseClock() + clock.mark("build") + clock.absorb({"timed": 0.25}, prefix="bench_") + totals: dict[str, float] = {} + clock.add_to(totals) + clock.add_to(totals) + assert totals["points"] == 2 + assert totals["bench_timed"] == 0.5 + assert totals["total"] >= 2 * totals["build"] + text = render_phase_totals(totals) + assert text.startswith("2 point(s): ") + assert "bench_timed=0.5s" in text + assert "points=" not in text + assert render_phase_totals({}) == "0 point(s): " + + +def _fake_torch(monkeypatch, allocated): + import sys + import types + + state = {"allocated": allocated} + fake = types.ModuleType("torch") + fake.cuda = types.SimpleNamespace( + is_available=lambda: True, + memory_allocated=lambda: state["allocated"], + ) + monkeypatch.setitem(sys.modules, "torch", fake) + return state + + +def test_periodic_gc_collects_every_n_calls(monkeypatch): + import gc + + from collector.helper import PeriodicGarbageCollector + + _fake_torch(monkeypatch, allocated=1000) + calls = [] + monkeypatch.setattr(gc, "collect", lambda: calls.append(1) or 0) + pgc = PeriodicGarbageCollector(every=4, growth_bytes=10**9) + results = [pgc.maybe_collect() for _ in range(9)] + assert results == [False, False, False, True, False, False, False, True, False] + assert len(calls) == 2 + + +def test_periodic_gc_collects_early_on_memory_growth(monkeypatch): + import gc + + from collector.helper import PeriodicGarbageCollector + + state = _fake_torch(monkeypatch, allocated=1000) + calls = [] + monkeypatch.setattr(gc, "collect", lambda: calls.append(1) or 0) + pgc = PeriodicGarbageCollector(every=1000, growth_bytes=500) + assert pgc.maybe_collect() is False # first call sets the floor + state["allocated"] = 1400 + assert pgc.maybe_collect() is False # within the growth budget + state["allocated"] = 1600 + assert pgc.maybe_collect() is True # grew past floor + growth_bytes + assert len(calls) == 1 + assert pgc.calls == 0 + assert pgc.floor == 1000 # the floor never rises + + +def test_periodic_gc_without_cuda_counts_calls_only(monkeypatch): + import gc + import sys + import types + + from collector.helper import PeriodicGarbageCollector + + fake = types.ModuleType("torch") + fake.cuda = types.SimpleNamespace(is_available=lambda: False) + monkeypatch.setitem(sys.modules, "torch", fake) + calls = [] + monkeypatch.setattr(gc, "collect", lambda: calls.append(1) or 0) + pgc = PeriodicGarbageCollector(every=2) + assert [pgc.maybe_collect() for _ in range(4)] == [False, True, False, True] + assert len(calls) == 2 diff --git a/python/aisimulate/tests/unit/collector/test_kv_block_size.py b/python/aisimulate/tests/unit/collector/test_kv_block_size.py new file mode 100644 index 0000000000..0996cf02d0 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_kv_block_size.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""KV page size is a per-SM branch (owner decision 2026-09-30 after the B200 +campaign caught the 64 literal running a different FMHA cubin on SM100).""" + +import ast +from pathlib import Path + +import pytest + +SRC = Path(__file__).resolve().parents[3] / "collector" / "vllm" / "utils.py" + + +def _kv_block_size(): + """The module imports vllm at import time; lift just the pure function.""" + tree = ast.parse(SRC.read_text()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "kv_block_size") + ns = {} + exec(compile(ast.Module(body=[fn], type_ignores=[]), str(SRC), "exec"), ns) + return ns["kv_block_size"] + + +kv_block_size = _kv_block_size() + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize( + "sm,op,expected", + [ + (90, "attention", 64), + (90, "mla", 64), + (100, "attention", 16), + (103, "attention", 16), + (100, "mla", 32), + (103, "mla", 32), + (120, "attention", 64), + (120, "mla", 64), + ], +) +def test_page_size_per_sm(sm, op, expected, monkeypatch): + monkeypatch.delenv("AIS_KV_BLOCK_SIZE", raising=False) + assert kv_block_size(sm, op) == expected + + +def test_ab_override_hook(monkeypatch): + monkeypatch.setenv("AIS_KV_BLOCK_SIZE", "16") + assert kv_block_size(90, "attention") == 16 diff --git a/python/aisimulate/tests/unit/collector/test_model_cases.py b/python/aisimulate/tests/unit/collector/test_model_cases.py index fd81adb5d4..7fbf01c4fa 100644 --- a/python/aisimulate/tests/unit/collector/test_model_cases.py +++ b/python/aisimulate/tests/unit/collector/test_model_cases.py @@ -522,6 +522,7 @@ def test_moe_model_quantization_policy_is_yaml_backed(): # -BF16 suffix (there is no zai-org/GLM-5.3-FP8 repository). assert moe_model_allows_quantization("sglang", "zai-org/GLM-5.3", "fp8_block") assert not moe_model_allows_quantization("sglang", "zai-org/GLM-5.3", "bfloat16") + assert not moe_model_allows_quantization("sglang", "zai-org/GLM-5.3", "nvfp4") assert moe_model_allows_quantization("sglang", "zai-org/GLM-5.3-BF16", "bfloat16") assert not moe_model_allows_quantization("sglang", "zai-org/GLM-5.3-BF16", "fp8_block") @@ -1422,8 +1423,8 @@ def test_mla_module_metadata_and_micro_sweeps_are_yaml_backed(): "zai-org/GLM-5.2", "zai-org/GLM-5.2-FP8", "nvidia/GLM-5.2-NVFP4", - "zai-org/GLM-5.3", "zai-org/GLM-5.3-BF16", + "zai-org/GLM-5.3", "nvidia/GLM-5.3-NVFP4", } assert {spec.native_num_heads for spec in dsa_specs if spec.architecture == "GlmMoeDsaForCausalLM"} == {64} @@ -1585,11 +1586,10 @@ def test_model_cases_path_can_infer_model_path(): assert "dsv4_csa_context_module" in plan.selected_ops assert "dsv4_csa_topk_calib" in plan.selected_ops assert "mhc_module" in plan.selected_ops - assert { - "dsv4_paged_mqa_logits_module", - "dsv4_hca_attn_module", - "dsv4_csa_attn_module", - }.isdisjoint(plan.selected_ops) + # the SDK's only live sparse sidecar (CP chunk walk) is planned; the FMLA sidecars + # (hca_attn / csa_attn) were removed from the registry on 2026-10-15 + assert "dsv4_paged_mqa_logits_module" in plan.selected_ops + assert {"dsv4_hca_attn_module", "dsv4_csa_attn_module"}.isdisjoint(plan.selected_ops) def test_plan_rejects_model_declared_ops_unknown_to_backend_registry(tmp_path): @@ -1643,7 +1643,7 @@ def test_vllm_024_schedules_consumed_dsv4_modules_only(): "dsv4_csa_generation_module", "dsv4_hca_generation_module", } - registry_only_ops = {"dsv4_paged_mqa_logits_module", "dsv4_hca_attn_module", "mhc_module"} + removed_sidecar_ops = {"dsv4_hca_attn_module", "dsv4_csa_attn_module"} # removed 2026-10-15 plan = build_collection_case_plan(backend="vllm", model_path="sgl-project/DeepSeek-V4-Pro-FP8") assert plan.ops == [ @@ -1651,12 +1651,15 @@ def test_vllm_024_schedules_consumed_dsv4_modules_only(): "dsv4_csa_generation_module", "dsv4_hca_context_module", "dsv4_hca_generation_module", + "dsv4_paged_mqa_logits_module", # consumed by the SDK CP chunk walk; shipped for vllm 0.24/0.25 "gemm", + "mhc_module", # SDK queries mhc pre/post on every backend; scheduled again 2026-10-11 "moe", ] + registered_ops = {entry.op for entry in REGISTRY} assert consumed_dsv4_ops <= plan.selected_ops - assert registry_only_ops.isdisjoint(plan.selected_ops) - assert consumed_dsv4_ops | registry_only_ops <= {entry.op for entry in REGISTRY} + assert consumed_dsv4_ops <= registered_ops + assert removed_sidecar_ops.isdisjoint(registered_ops) def test_model_architecture_can_select_case_file(): @@ -1727,8 +1730,7 @@ def test_vllm_024_model_plans_only_schedule_representable_attention_paths(): "mla_context", "mla_generation", "moe", - "moe_ep", - ] + ] # moe_ep left the default trtllm plan with the separate WideEP pin (2026-10-04) assert build_collection_case_plan(backend="vllm_xpu", model_path=kimi_path).ops == ["gemm", "moe"] models_with_unrepresentable_vllm_attention = ( @@ -1848,6 +1850,7 @@ def get_model_specs(**kwargs): "get_context_test_cases": lambda _attention_type: [[128, 1, 8, "bfloat16", "bfloat16", "bfloat16"]], "get_generation_test_cases": lambda _attention_type: [], "get_mla_module_model_specs": get_model_specs, + "get_sm_version": lambda: 90, } exec(compile(ast.Module(body=[function], type_ignores=[]), str(source_path), "exec"), namespace) @@ -1857,6 +1860,167 @@ def get_model_specs(**kwargs): assert cases == [[128, 1, 8, "bfloat16", "bfloat16", "bfloat16", "deepseek-ai/DeepSeek-V3", "mla"]] +def _load_module_case_builder(backend: str, namespace: dict): + from types import SimpleNamespace + + source_path = REPO_ROOT / f"collector/{backend}/collect_mla_module.py" + tree = ast.parse(source_path.read_text(), filename=str(source_path)) + wanted = {"_build_module_test_cases"} + if backend == "vllm": + # the DSA context grouping expands each series through the plan predicate + wanted |= {"_dsa_context_point_allowed", "_dsa_context_budgets"} + namespace.setdefault( + "get_mla_module_sweep_spec", + lambda _backend=None: SimpleNamespace( + context_dsa_chunk_prefill_size=16384, + context_max_tokens=131072, + context_dsa_max_full_sequence_length=1048576, + ), + ) + functions = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name in wanted] + exec(compile(ast.Module(body=functions, type_ignores=[]), str(source_path), "exec"), namespace) + return namespace["_build_module_test_cases"] + + +@pytest.mark.parametrize("backend", ["vllm", "trtllm"]) +def test_dsa_module_plan_drops_sub_8_head_cells_on_sm100_only(backend): + """b200 2026-10-06 (jobs 471043976 / 469988228): per-rank heads 1/2/4 have no sparse-MLA + decode kernel on SM100/103 (trtllm-gen tileSizeQ < 8; TRT-LLM FMHA rows % 8) and failed + 68,435 + 15,408 cases after a full module load each. The planners drop them for DSA on + SM100/103 and keep them for MLA and on other SMs.""" + from types import SimpleNamespace + + heads = [128, 64, 8, 4, 2, 1] + base = [[1, 1, h, "bfloat16", "bfloat16", "bfloat16", 0] for h in heads] + for sm, expected in ((100, [128, 64, 8]), (103, [128, 64, 8]), (90, heads)): + ns = { + "get_context_test_cases": lambda _t, base=base: base, + "get_generation_test_cases": lambda _t: [], + "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], + "get_sm_version": lambda sm=sm: sm, + "_DSA_SPARSE_DECODE_MIN_HEADS": 8, + "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), + "_model_max_position_embeddings": lambda _m: None, + } + build = _load_module_case_builder(backend, ns) + assert [c[2] for c in build("dsa", "context")] == expected, (backend, sm) + assert [c[2] for c in build("mla", "context")] == heads, (backend, sm) # dense MLA untouched + + +@pytest.mark.parametrize("backend", ["vllm", "trtllm"]) +def test_module_plan_keeps_only_the_heads_of_declared_tensor_parallel_shards(backend): + """A model row that declares tensor_parallel_sizes (case_authoring model-correlated + narrowing, as the MSA rows do) measures only native_num_heads // tp; the shared head + grid's other entries (96/48/24/12 for the 96-head Kimi-K3 MLA module, 128 for a 64-head + model) were 40-50% of the b200 vllm dsa_context rows (2026-10-29). Rows without the + declaration keep the full grid.""" + from types import SimpleNamespace + + heads = [128, 96, 64, 48, 32, 24, 16, 12, 8] + base = [[1, 1, h, "bfloat16", "bfloat16", "bfloat16", 0] for h in heads] + glm = SimpleNamespace(model_path="zai-org/GLM-5.2", native_num_heads=64, tensor_parallel_sizes=(1, 2, 4, 8)) + kimi = SimpleNamespace(model_path="moonshotai/Kimi-K3", native_num_heads=96, tensor_parallel_sizes=None) + ns = { + "get_context_test_cases": lambda _t: base, + "get_generation_test_cases": lambda _t: [], + "get_mla_module_model_specs": lambda **_k: [glm, kimi], + "get_sm_version": lambda: 100, + "_DSA_SPARSE_DECODE_MIN_HEADS": 8, + "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), + "_model_max_position_embeddings": lambda _m: None, + } + build = _load_module_case_builder(backend, ns) + by_model = {} + for case in build("dsa", "context"): + by_model.setdefault(case[6], []).append(case[2]) + assert by_model["zai-org/GLM-5.2"] == [64, 32, 16, 8] + assert by_model["moonshotai/Kimi-K3"] == heads + + +def test_vllm_module_plan_groups_the_seq_len_points_of_one_module_into_one_case(): + """One case per (batch, heads, dtypes, gemm, model, attn, prefix) with its seq_len points + in slot 0, largest first: run_mla_module builds the module once per case (the module + build dominated vllm dsa_context_module in the 2026-10 campaign).""" + from types import SimpleNamespace + + base = [ + [128, 1, 64, "bfloat16", "bfloat16", "bfloat16", 0], + [16, 1, 64, "bfloat16", "bfloat16", "bfloat16", 0], + [1024, 1, 64, "bfloat16", "bfloat16", "bfloat16", 0], + [128, 1, 64, "bfloat16", "bfloat16", "bfloat16", 4096], # other prefix -> same module series + [16, 1, 64, "bfloat16", "bfloat16", "bfloat16", 4096], + [1024, 1, 64, "bfloat16", "bfloat16", "bfloat16", 4096], + [16, 8, 64, "bfloat16", "bfloat16", "bfloat16", 0], # other batch -> other case + [16, 1, 64, "fp8", "bfloat16", "bfloat16", 0], # other kv dtype -> other case + ] + ns = { + "get_context_test_cases": lambda _t: base, + "get_generation_test_cases": lambda _t: [], + "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], + "get_sm_version": lambda: 100, + "_DSA_SPARSE_DECODE_MIN_HEADS": 8, + "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), + "_model_max_position_embeddings": lambda _m: None, + } + build = _load_module_case_builder("vllm", ns) + cases = build("dsa", "context") + # DSA context: the module series carries its isl list AND its prefix list (the plan's + # isl x prefix grid); the runner expands the grid through the same predicate. + assert cases == [ + [[1024, 128, 16], 1, 64, "bfloat16", "bfloat16", "bfloat16", "deepseek-ai/DeepSeek-V3.2", "dsa", [4096, 0]], + [[16], 8, 64, "bfloat16", "bfloat16", "bfloat16", "deepseek-ai/DeepSeek-V3.2", "dsa", [0]], + [[16], 1, 64, "fp8", "bfloat16", "bfloat16", "deepseek-ai/DeepSeek-V3.2", "dsa", [0]], + ] + # the trtllm planner keeps one case per point + trt = _load_module_case_builder("trtllm", dict(ns)) + assert [c[0] for c in trt("dsa", "context")] == [128, 16, 1024, 128, 16, 1024, 16, 16] + + +def test_vllm_dsa_module_plan_refuses_a_grid_the_runner_would_not_reproduce(): + """The plan generator and run_mla_module share _dsa_context_point_allowed; a base list whose + (isl, prefix) points are not that predicate's product cannot be expressed as a series.""" + from types import SimpleNamespace + + base = [ + [1024, 1, 64, "bfloat16", "bfloat16", "bfloat16", 0], + [16, 1, 64, "bfloat16", "bfloat16", "bfloat16", 4096], # (1024, 4096) and (16, 0) missing + ] + ns = { + "get_context_test_cases": lambda _t: base, + "get_generation_test_cases": lambda _t: [], + "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], + "get_sm_version": lambda: 100, + "_DSA_SPARSE_DECODE_MIN_HEADS": 8, + "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), + "_model_max_position_embeddings": lambda _m: None, + } + build = _load_module_case_builder("vllm", ns) + with pytest.raises(RuntimeError, match="expands to 4 points but the plan holds 2"): + build("dsa", "context") + + +def test_vllm_dsa_module_plan_caps_prefix_at_the_model_max_position(): + """7,050 b200 vllm DSA cases (job 471043976) asked vLLM for max_model_len > the model's + max_position_embeddings (prefix 262144+ on DeepSeek-V3.2 / GLM-5): the planner now + applies the same per-model cap the sglang module collector applies.""" + from types import SimpleNamespace + + base = [[16, 1, 64, "bfloat16", "bfloat16", "bfloat16", p] for p in (0, 4096, 163824, 163825, 262144)] + ns = { + "get_context_test_cases": lambda _t: base, + "get_generation_test_cases": lambda _t: [], + "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], + "get_sm_version": lambda: 100, + "_DSA_SPARSE_DECODE_MIN_HEADS": 8, + "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), + "_model_max_position_embeddings": lambda _m: 163840, + } + build = _load_module_case_builder("vllm", ns) + assert [c[-1] for c in build("dsa", "context")] == [[163824, 4096, 0]] + ns["_model_max_position_embeddings"] = lambda _m: None # unknown ceiling -> no cap + assert [c[-1] for c in build("dsa", "context")] == [[262144, 163825, 163824, 4096, 0]] + + def test_support_matrix_models_have_model_case_aliases(): case_aliases = set() for path in (REPO_ROOT / "collector" / "cases" / "models").glob("*_cases.yaml"): @@ -2228,3 +2392,22 @@ def test_nvfp4_checkpoint_targets_msa_module_specs(monkeypatch): assert all(s.model_path == "MiniMaxAI/MiniMax-M3" for s in specs), ( "alias rows must stay keyed to the canonical model path" ) + + +def test_msa_head_axis_derives_from_declared_shards(): + """MSA is one fixed geometry (MiniMax-M3 64q/4kv/4idx): the collectors sweep + native // tp for the model row's tensor_parallel_sizes, never the shared + GQA-style head grid (owner decision 2026-09-29; TP 16 is the ceiling).""" + from collector.case_generator import get_mla_module_model_specs, get_msa_head_counts + + (spec,) = [ + s + for s in get_mla_module_model_specs(attention_type="msa", apply_model_filter=False) + if s.model_path == "MiniMaxAI/MiniMax-M3" + ] + assert spec.tensor_parallel_sizes == (1, 2, 4, 8, 16) + for backend in ("vllm", "trtllm", "sglang"): + heads = get_msa_head_counts(backend) + assert heads == [64, 32, 16, 8, 4], (backend, heads) + # every derived shard keeps the index-head count a power of two + assert all((max(1, h // 16) & (max(1, h // 16) - 1)) == 0 for h in heads) diff --git a/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py b/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py index b96d46e5e6..916fcfd24e 100644 --- a/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py +++ b/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py @@ -60,10 +60,16 @@ def _stub_torch(monkeypatch): ) +class _PeriodicGCStub: + def maybe_collect(self): + return False + + def _install_helper_stub(monkeypatch): _stub_module( monkeypatch, "collector.helper", + PeriodicGarbageCollector=_PeriodicGCStub, _resolve_local_model_path=lambda path: path, benchmark_with_power=_noop, get_sm_version=lambda: 100, @@ -237,6 +243,29 @@ def test_targeted_msa_getters_keep_prefixes_and_worker_tuple_compatibility(monke assert captured[1]["perf_filename"] == "generation.txt" +@pytest.mark.parametrize( + "sm_version,expected_max_tokens", + [(89, 32768), (90, 32768), (120, 32768), (100, 131072), (103, 131072)], +) +def test_trtllm_context_getter_caps_only_the_triton_reference_implementation( + monkeypatch, capsys, sm_version, expected_max_tokens +): + """mla_module_trtllm.context.msa_triton_max_tokens binds where the collector + resolves implementation="triton" (off SM100/103); fmha_sm100 systems keep + the shared context budget. isl 32768 itself stays at b=1 on both.""" + monkeypatch.setenv("COLLECTOR_MODEL_PATH", MODEL_PATH) + module = _load_backend(monkeypatch, "trtllm") + monkeypatch.setattr(module, "get_sm_version", lambda: sm_version) + + cases = module.get_msa_context_module_test_cases() + shapes = {(case[0], case[1]) for case in cases} + + assert max(s * b for s, b in shapes) == expected_max_tokens + assert (32768, 1) in shapes + dropped = "[trtllm-msa-module-cases] context: dropped" in capsys.readouterr().out + assert dropped == (expected_max_tokens == 32768) + + def test_vllm_public_generation_getter_applies_live_memory_filter(monkeypatch, capsys): monkeypatch.setenv("COLLECTOR_MODEL_PATH", MODEL_PATH) module = _load_backend(monkeypatch, "vllm") diff --git a/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py b/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py new file mode 100644 index 0000000000..3a024cd2ca --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py @@ -0,0 +1,45 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""The M3 index-head kernel limit is the DECODE scorer's: a guard keyed on the +head count alone dropped 1808 prefill rows on the 0.30 sweep (2026-09-28) +that the 0.29 sweep had collected.""" + +import ast +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +SRC = Path(__file__).resolve().parents[3] / "collector" / "vllm" / "collect_msa_module.py" + + +def _predicate(): + """Load only the pure predicate — the module itself imports vllm.""" + tree = ast.parse(SRC.read_text()) + fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "msa_decode_scorer_kernel_limit") + ns = {} + exec(compile(ast.Module(body=[fn], type_ignores=[]), str(SRC), "exec"), ns) + return ns["msa_decode_scorer_kernel_limit"] + + +@pytest.mark.parametrize("idx_heads,num_heads", [(3, 48), (6, 96)]) +def test_non_power_of_two_index_heads_fail_only_where_the_decode_scorer_runs(idx_heads, num_heads): + limit = _predicate() + assert limit(idx_heads, num_heads, is_context=False, query_len=1) # generation + assert limit(idx_heads, num_heads, is_context=True, query_len=1) # one-token context query + assert limit(idx_heads, num_heads, is_context=True, query_len=16) is None # prefill path + assert limit(idx_heads, num_heads, is_context=True, query_len=32768) is None + + +@pytest.mark.parametrize("idx_heads", [1, 2, 4, 8]) +def test_power_of_two_index_heads_never_limited(idx_heads): + limit = _predicate() + for is_context, q in ((False, 1), (True, 1), (True, 4096)): + assert limit(idx_heads, idx_heads * 16, is_context=is_context, query_len=q) is None + + +def test_builder_passes_the_query_length(): + text = SRC.read_text() + assert "query_len=seq_len if is_context else 1," in text + assert "msa_decode_scorer_kernel_limit(" in text diff --git a/python/aisimulate/tests/unit/collector/test_network_layout.py b/python/aisimulate/tests/unit/collector/test_network_layout.py index 50531e3d00..4f94621153 100644 --- a/python/aisimulate/tests/unit/collector/test_network_layout.py +++ b/python/aisimulate/tests/unit/collector/test_network_layout.py @@ -83,10 +83,13 @@ def test_submit_moe_a2a_default_image_matches_manifest(): def test_submit_trtllm_alltoall_default_image_matches_manifest(): - # Same sync requirement against the trtllm pin. The collector gates the - # installed tensorrt_llm version against this pin, so drift here would - # fail every job loudly. - manifest_image = _manifest_frameworks()["trtllm"]["default"]["images"]["default"] + # Same sync requirement, against the WideEP all-to-all build base + # (trtllm_a2a: the rc20 image is the immutable base the rc11 source wheel is + # built over), NOT the stock trtllm pin — stock moved to rc29 on 2026-10-02 + # while WideEP stays on its own pins. The collector gates the installed + # tensorrt_llm version against this pin, so drift here would fail every + # job loudly. + manifest_image = _manifest_frameworks()["trtllm_a2a"]["default"]["images"]["default"] assert _default_container_image(SUBMIT_TRTLLM_ALLTOALL) == manifest_image diff --git a/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py b/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py new file mode 100644 index 0000000000..42c2be3bc2 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py @@ -0,0 +1,65 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""No collector may time a kernel eagerly behind the CUDA-graph fallback. + +``benchmark_with_power(allow_graph_fail=True)`` replaces a failed capture with +the Python launch loop and ships that floor as kernel latency; no perf table +records the helper's ``used_cuda_graph`` flag, and layer_permissions.md allows +a measurement-method degradation only when it is recorded in the output row. +The sglang trtllm_mla prefill (B1) and the trtllm GDN lanes were closed on +2026-10-11, the last four sites — trtllm mamba2 and moe, wideep moe_compute, +network trtllm alltoall — on 2026-10-12. This test keeps the count at zero for +every file under ``collector/``.""" + +import ast +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + +COLLECTOR_ROOT = Path(__file__).resolve().parents[3] / "collector" + +# Former fallback sites: number of benchmark_with_power calls each still makes. +FORMER_SITES = { + "network/slurm/collect_trtllm_alltoall.py": 1, + "trtllm/collect_mamba2.py": 8, + "trtllm/collect_moe.py": 1, + "vllm/collect_moe_xpu.py": 1, + "wideep/trtllm/collect_moe_compute.py": 1, +} + + +def _benchmark_calls(tree): + return [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "benchmark_with_power" + ] + + +def test_no_collector_passes_allow_graph_fail_true(): + offenders = [] + for path in sorted(COLLECTOR_ROOT.rglob("*.py")): + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + for node in ast.walk(tree): + if not isinstance(node, ast.Call): + continue + for keyword in node.keywords: + if ( + keyword.arg == "allow_graph_fail" + and isinstance(keyword.value, ast.Constant) + and keyword.value.value is True + ): + offenders.append(f"{path.relative_to(COLLECTOR_ROOT)}:{node.lineno}") + assert offenders == [], f"eager-timed rows would ship from: {offenders}" + + +@pytest.mark.parametrize(("relative_path", "expected_calls"), sorted(FORMER_SITES.items())) +def test_former_fallback_sites_time_under_cuda_graphs_only(relative_path, expected_calls): + tree = ast.parse((COLLECTOR_ROOT / relative_path).read_text(encoding="utf-8")) + calls = _benchmark_calls(tree) + assert len(calls) == expected_calls + for call in calls: + assert all(keyword.arg != "allow_graph_fail" for keyword in call.keywords), relative_path diff --git a/python/aisimulate/tests/unit/collector/test_parallel_run.py b/python/aisimulate/tests/unit/collector/test_parallel_run.py index 8ff0be5b7e..b287b555fd 100644 --- a/python/aisimulate/tests/unit/collector/test_parallel_run.py +++ b/python/aisimulate/tests/unit/collector/test_parallel_run.py @@ -10,6 +10,10 @@ - Sentinel balance under repeated restarts (the core bug-fix scenario) - Lost progress tick: worker dies after recording done but before its progress_value tick (the 99/100 hang, H20 2026-07-19) +- Stall watchdog: wedged workers are killed and restarted, the queue keeps + draining; only consecutive rounds without progress close the op, and the + never-dispatched tasks are not written into the failed ledger + (b200 vllm dsa_context_module, 2026-10-11) """ import json @@ -18,6 +22,8 @@ import os import signal import sys +import threading +import time from pathlib import Path from unittest.mock import MagicMock @@ -40,9 +46,9 @@ if _COLLECTOR_DIR not in sys.path: sys.path.insert(0, _COLLECTOR_DIR) +_torch = MagicMock() +_torch.AcceleratorError = type("AcceleratorError", (Exception,), {}) if "torch" not in sys.modules: - _torch = MagicMock() - _torch.AcceleratorError = type("AcceleratorError", (Exception,), {}) sys.modules["torch"] = _torch import collect as _collect_mod @@ -76,6 +82,26 @@ def _task_fn(label, behavior, device): os.kill(os.getpid(), signal.SIGABRT) elif behavior == "error": raise ValueError(f"simulated: {label}") + elif behavior == "hang": + # a wedged kernel: never returns (Event.wait is immune to the shortened + # time.sleep the parent patched before forking, and survives SIGUSR1) + threading.Event().wait() + elif behavior.startswith("hang_unless:"): + if not os.path.exists(behavior.split(":", 1)[1]): + threading.Event().wait() + elif behavior.startswith(("beat:", "beat_then_hang:")): + # a long series task: one heartbeat per measured point for N seconds + # (benchmark_with_power beats the same way); "beat_then_hang" then + # wedges like "hang", so the watchdog timer runs from its last beat + from helper import worker_heartbeat + + kind, seconds = behavior.split(":", 1) + end = time.monotonic() + float(seconds) + while time.monotonic() < end: + worker_heartbeat() + time.sleep(0.1) + if kind == "beat_then_hang": + threading.Event().wait() elif behavior == "oom": raise sys.modules["torch"].OutOfMemoryError(f"simulated: {label}") # "normal": return silently @@ -94,6 +120,7 @@ def _worker_dying_after_done( module_name="unknown", current_task_ids=None, consumed_sentinel=None, + heartbeat=None, ): """Scripted stand-in for ``collect.worker`` reproducing the 2026-07-19 H20 death window: the task SUCCEEDS and lands in ``done_tasks``, but the @@ -161,6 +188,12 @@ def _fork_mp(monkeypatch): warnings.filterwarnings("ignore", message=".*fork.*", category=DeprecationWarning) ctx = mp.get_context("fork") monkeypatch.setattr(_collect_mod, "mp", ctx) + # In a full run other test modules import collect before this one without a + # torch stub (collect.torch stays None) or swap the stub in sys.modules; the + # fork'd workers would then die in _require_torch and be restarted until the + # restart cap, which looks like a hang. Pin the stub in both places per test. + monkeypatch.setitem(sys.modules, "torch", _torch) + monkeypatch.setattr(_collect_mod, "torch", _torch) @pytest.fixture(autouse=True) @@ -274,6 +307,10 @@ def test_torch_out_of_memory_error_is_fatal(self): "CUBLAS_STATUS_EXECUTION_FAILED", "CUBLAS_STATUS_INTERNAL_ERROR", "CUBLAS_STATUS_ALLOC_FAILED", + # FlashInfer TVM-FFI launcher wrapper (error text only on stderr) + "Error in function 'run' at /workspace/csrc/fused_moe/trtllm_backend/" + "trtllm_fused_moe_routing_llama4.cu:550: Got CUDA error. See above for details.", + "CUDA error: cudaErrorIllegalAddress", ], ) def test_cuda_fatal_markers_are_fatal(self, message): @@ -564,6 +601,87 @@ def _on_alarm(signum, frame): assert len(_crash_errors(errors)) >= 1 +class TestStallWatchdog: + """STALL_TIMEOUT_SEC of zero progress kills and restarts the workers; only + STALL_MAX_ROUNDS consecutive rounds without progress close the op, and the + tasks nobody dispatched stay out of the failed ledger.""" + + @pytest.fixture(autouse=True) + def _fast_stall(self, monkeypatch): + monkeypatch.setattr(_collect_mod, "STALL_THRESHOLD", 1) + monkeypatch.setattr(_collect_mod, "STALL_DIAG_SEC", 0) + monkeypatch.setattr(_collect_mod, "STALL_TIMEOUT_SEC", 2) + + @staticmethod + def _ids(errors, error_type): + return {e["task_id"] for e in errors if e.get("error_type") == error_type} + + def test_stall_kill_restarts_workers_and_the_queue_keeps_draining(self, tmp_path, monkeypatch): + monkeypatch.setattr(_collect_mod, "STALL_MAX_ROUNDS", 3) + module_name = "stall_restart" + # both workers take a wedged task first; the six normal tasks queue behind them + tasks = _tasks([("h0", "hang"), ("h1", "hang")] + [(f"n{i}", "normal") for i in range(6)]) + + errors = _run(tasks, 2, tmp_path, module_name=module_name) + + assert self._ids(errors, "WorkerStallTimeout") == {"h0", "h1"} + assert self._ids(errors, "WorkerOrphanedTask") == set() + assert _crash_errors(errors) == [] # a watchdog kill is not a crash + assert _load_done_ids(tmp_path, module_name) == {f"n{i}" for i in range(6)} + assert _load_failed_ids(tmp_path, module_name) == {"h0", "h1"} + + def test_heartbeats_hold_the_watchdog_until_the_measuring_task_finishes(self, tmp_path, monkeypatch): + monkeypatch.setattr(_collect_mod, "STALL_MAX_ROUNDS", 3) + module_name = "stall_heartbeat" + # "beat" measures for longer than STALL_TIMEOUT_SEC with one heartbeat per + # point and must finish; "quiet" beats briefly, then wedges: it is killed + # once nothing else beats, STALL_TIMEOUT_SEC after the last progress + tasks = _tasks([("beat", "beat:3.5"), ("quiet", "beat_then_hang:0.5")]) + + errors = _run(tasks, 2, tmp_path, module_name=module_name) + + assert self._ids(errors, "WorkerStallTimeout") == {"quiet"} + assert self._ids(errors, "WorkerOrphanedTask") == set() + assert _crash_errors(errors) == [] + assert _load_done_ids(tmp_path, module_name) == {"beat"} + assert _load_failed_ids(tmp_path, module_name) == {"quiet"} + + def test_consecutive_stall_rounds_close_the_op_and_orphans_are_not_failed(self, tmp_path, monkeypatch): + monkeypatch.setattr(_collect_mod, "STALL_MAX_ROUNDS", 2) + module_name = "stall_close" + release = tmp_path / "release" + tasks = _tasks([(f"h{i}", f"hang_unless:{release}") for i in range(6)]) + + errors = _run(tasks, 2, tmp_path, module_name=module_name) + + # round 1: h0/h1 wedged -> killed, workers restarted; round 2: h2/h3 + # wedged with nothing finished in between -> op closed. h4/h5 were + # never dispatched: reported, but neither failed nor attempted. + stalled = self._ids(errors, "WorkerStallTimeout") + orphaned = self._ids(errors, "WorkerOrphanedTask") + assert stalled == {"h0", "h1", "h2", "h3"} + assert orphaned == {"h4", "h5"} + assert _load_done_ids(tmp_path, module_name) == set() + assert _load_failed_ids(tmp_path, module_name) == stalled + assert _load_attempted_ids(tmp_path, module_name) & orphaned == set() + + # a plain --resume (no retry-failed) runs exactly the orphans + release.touch() + resumed = parallel_run( + tasks, + _task_fn, + num_processes=2, + module_name=module_name, + resume_options={"checkpoint_dir": str(tmp_path / ".checkpoint"), "resume": True}, + ) + assert _load_done_ids(tmp_path, module_name) == {"h4", "h5"} + assert _load_failed_ids(tmp_path, module_name) == stalled + assert self._ids(resumed, "WorkerOrphanedTask") == set() + assert self._ids(resumed, "WorkerStallTimeout") == set() + unresolved = [e for e in resumed if e.get("error_type") == "UnresolvedFailures"] + assert len(unresolved) == 1 and "4 unresolved" in unresolved[0]["error_message"] + + class TestResumeIntegrity: """Checkpoint binds runtime identity; resume surfaces unresolved failures.""" @@ -665,3 +783,26 @@ def test_fresh_zero_work_run_replaces_stale_checkpoint(self, tmp_path): assert _load_done_ids(tmp_path, module_name) == set() assert _load_failed_ids(tmp_path, module_name) == set() assert _load_attempted_ids(tmp_path, module_name) == set() + + +def test_worker_heartbeat_finds_the_sink_registered_through_the_other_module_name(monkeypatch): + """collect.py registers the sink on ``helper``; the op modules beat through + ``collector.helper`` (a second module object) and must still reach it.""" + import types + + import helper + + beats = [] + twin = types.ModuleType("collector.helper") + twin._WORKER_HEARTBEAT = lambda: beats.append(1) + monkeypatch.setattr(helper, "_WORKER_HEARTBEAT", None) + monkeypatch.setitem(sys.modules, "collector.helper", twin) + + helper.worker_heartbeat() + helper.worker_heartbeat() + + assert beats == [1, 1] + monkeypatch.setattr(helper, "_WORKER_HEARTBEAT", None) + monkeypatch.delitem(sys.modules, "collector.helper") + helper.worker_heartbeat() # no sink anywhere: a silent no-op + assert beats == [1, 1] diff --git a/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py b/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py index 3205f270ab..e6a0b4708f 100644 --- a/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py +++ b/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py @@ -880,6 +880,84 @@ def test_convert_without_merge_overwrites(tmp_path): assert _rows_by_key(parquet) == {"s2": 2.0} # legacy overwrite preserved when opted out +def test_finalize_third_merge_survives_large_string_round_trip(tmp_path): + """run -> resume -> retry-failed: the second merge writes string columns back as + `large_string` (pandas round trip); the third finalize must still merge, not + overwrite (sm120 attention_generation fp8kv lost 27,303 rows this way, 2026-10-01).""" + import pyarrow as pa + + perf = tmp_path / "gemm_perf.txt" + parquet = perf.with_suffix(".parquet") + _write_keyed_perf_csv(perf, [("s1", 1.0)]) + finalize_perf_files([perf]) + _write_keyed_perf_csv(perf, [("s2", 2.0)]) + finalize_perf_files([perf]) + # force the state the pandas round trip produces: identity column stored as large_string + t = pq.read_table(parquet) + idx = t.schema.get_field_index("shape") + pq.write_table(t.set_column(idx, "shape", t.column("shape").cast(pa.large_string())), parquet) + assert str(pq.read_table(parquet).schema.field("shape").type) == "large_string" + _write_keyed_perf_csv(perf, [("s3", 3.0)]) + finalize_perf_files([perf]) + assert _rows_by_key(parquet) == {"s1": 1.0, "s2": 2.0, "s3": 3.0} + + +@contextmanager +def _pandas_unavailable(monkeypatch): + """The collector images ship pyarrow (harness ``pyarrow-site``) but no pandas.""" + import builtins + + real_import = builtins.__import__ + + def _import(name, *args, **kwargs): + if name == "pandas" or name.startswith("pandas."): + raise ModuleNotFoundError("No module named 'pandas'") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _import) + try: + yield + finally: + monkeypatch.setattr(builtins, "__import__", real_import) + + +def test_finalize_merge_does_not_need_pandas(tmp_path, monkeypatch): + """The second finalize of a shard (resume / retry-failed) merges into the + existing parquet; it died with ``No module named 'pandas'`` on the vllm + image (b200 dsa_context_module job 481670367 window 2, 2026-10-12).""" + perf = tmp_path / "gemm_perf.txt" + parquet = perf.with_suffix(".parquet") + _write_keyed_perf_csv(perf, [("s1", 1.0), ("s2", 2.0)]) + finalize_perf_files([perf]) + + with _pandas_unavailable(monkeypatch): + _write_keyed_perf_csv(perf, [("s1", 9.0), ("s3", 3.0)]) + finalize_perf_files([perf]) + + assert _rows_by_key(parquet) == {"s1": 9.0, "s2": 2.0, "s3": 3.0} + table = pq.read_table(parquet) + assert [r["shape"] for r in table.to_pylist()] == ["s2", "s1", "s3"] # kept rows keep their order + assert not table.schema.metadata # no pandas index metadata in the published parquet + + +def test_finalize_moe_legacy_merge_does_not_need_pandas(tmp_path, monkeypatch): + perf = tmp_path / "moe_perf.txt" + parquet = perf.with_suffix(".parquet") + perf.write_text("kernel_source,shape,latency,default_eligible\nexact,s1,1.0,false\nexact,s2,2.0,false\n") + finalize_perf_files([perf]) + + with _pandas_unavailable(monkeypatch): + perf.write_text("kernel_source,shape,latency\nexact,s1,3.0\nexact,s3,4.0\n") # legacy recollection + finalize_perf_files([perf]) + + rows = {r["shape"]: r for r in pq.read_table(parquet).to_pylist()} + assert {k: (v["latency"], v["default_eligible"]) for k, v in rows.items()} == { + "s1": (3.0, False), # re-measured, keeps its annotation + "s2": (2.0, False), + "s3": (4.0, True), # new key: eligible by default + } + + def test_finalize_merge_falls_back_to_overwrite_on_schema_mismatch(tmp_path): perf = tmp_path / "gemm_perf.txt" parquet = perf.with_suffix(".parquet") @@ -1044,3 +1122,58 @@ def test_finalize_keeps_power_disabled_schema_column_free(tmp_path): [parquet] = finalize_perf_files([perf]) assert pq.read_table(parquet).column_names == ["shape", "latency"] + + +def test_finalize_drops_rows_re_appended_by_a_retried_task_keeping_the_newest(tmp_path, caplog): + """The staging CSV is append-only across resumed windows and a task retried with + --resume-retry-failed logs its successful rows again (rtxpro msa_context: 10,342 of + 13,972 rows were such re-appends). Finalization keeps the newest row per identity + key, in the original order, like the merge-with-existing path already did.""" + perf = tmp_path / "gemm_perf.txt" + _write_keyed_perf_csv(perf, [("s1", 1.0), ("s2", 2.0), ("s1", 1.5), ("s3", 3.0), ("s2", 2.5)]) + + with caplog.at_level("INFO", logger="collector.helper"): + [parquet] = finalize_perf_files([perf], delete_source=False) + + table = pq.read_table(parquet) + assert table.column("shape").to_pylist() == ["s1", "s3", "s2"] + assert table.column("latency").to_pylist() == [1.5, 3.0, 2.5] + assert "dropped 2 re-appended duplicate rows" in caplog.text + + +def test_finalize_keeps_rows_that_differ_in_a_non_metric_column(tmp_path): + perf = tmp_path / "gemm_perf.txt" + perf.write_text("op,shape,power,latency\nmatmul,s1,100.0,1.0\nmatmul,s1,120.0,1.1\nmatmul,s2,100.0,2.0\n") + + [parquet] = finalize_perf_files([perf], delete_source=False) + + # power is a metric, so the two s1 rows share one identity: the newer one wins + assert pq.read_table(parquet).column("latency").to_pylist() == [1.1, 2.0] + + +def test_finalize_dedupe_ignores_moe_eligibility_metadata_and_tolerates_empty_columns(tmp_path): + perf = tmp_path / "moe_perf.txt" + perf.write_text( + "op,shape,kernel_source,note,default_eligible,latency\n" + "moe,s1,cutlass,,true,1.0\n" + "moe,s1,cutlass,,false,1.2\n" + "moe,s2,cutlass,,true,2.0\n" + ) + + [parquet] = finalize_perf_files([perf], delete_source=False) + + table = pq.read_table(parquet) + assert table.column("shape").to_pylist() == ["s1", "s2"] + assert table.column("default_eligible").to_pylist() == [False, True] + assert table.column("latency").to_pylist() == [1.2, 2.0] + + +def test_finalize_reports_deduplicated_row_count(tmp_path): + perf = tmp_path / "gemm_perf.txt" + _write_keyed_perf_csv(perf, [("s1", 1.0), ("s1", 1.5)]) + finalization_info = {} + + finalize_perf_files([perf], delete_source=False, finalization_info=finalization_info) + + [info] = finalization_info.values() + assert info.new_rows == 1 diff --git a/python/aisimulate/tests/unit/collector/test_provenance.py b/python/aisimulate/tests/unit/collector/test_provenance.py index bdbe13ead0..48e248a2ce 100644 --- a/python/aisimulate/tests/unit/collector/test_provenance.py +++ b/python/aisimulate/tests/unit/collector/test_provenance.py @@ -162,7 +162,6 @@ def test_dsa_collector_hash_covers_worker_binding(tmp_path, backend, binding_fil "collect_gemm", "collect_mhc_module", "collect_mla_module", - "collect_mla_module_027", "collect_moe", "collect_msa_module", ], @@ -179,18 +178,6 @@ def test_vllm_collector_hash_covers_native_setup(tmp_path, module_name): assert provenance.collector_hash(module, tmp_path, closures) != before -def test_vllm_mla_shim_hash_covers_executed_module(tmp_path): - module = "collector.vllm.collect_mla_module_027" - closures = provenance.load_closures(HASH_CLOSURES_PATH) - paths = {module.replace(".", "/") + ".py", *provenance.SHARED_CORE, *closures[module]} - for relative in paths - {provenance.MODEL_CASES_GROUP}: - _write(tmp_path / relative, (REPO_ROOT / relative).read_text(encoding="utf-8")) - before = provenance.collector_hash(module, tmp_path, closures) - implementation = tmp_path / "collector/vllm/collect_mla_module.py" - _write(implementation, implementation.read_text(encoding="utf-8") + "\n# native forward changed\n") - assert provenance.collector_hash(module, tmp_path, closures) != before - - def test_collector_hash_changes_when_model_cases_group_changes(tmp_path): _build_fake_repo(tmp_path) before = provenance.collector_hash("collector.sglang.collect_gemm", tmp_path, FAKE_CLOSURES) diff --git a/python/aisimulate/tests/unit/collector/test_registry_table_producers.py b/python/aisimulate/tests/unit/collector/test_registry_table_producers.py new file mode 100644 index 0000000000..d4a51c3ca2 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_registry_table_producers.py @@ -0,0 +1,166 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A checkpoint producer owns every table its registry entry declares. + +collect.py's fail-closed finalize binds each staged ``*_perf.txt`` to the +registry entries that declare it. A collector that writes a second table +its op did not declare cannot be finalized (first full vllm compute_scale +run, 2026-09-27: ``scale_matrix_perf.txt has no selected producer``; the +PerfFile.SCALE_MATRIX enum alone never fixed that). Owner decision +2026-09-28: the executor supports multi-table producers — ``OpEntry. +extra_perf_filenames`` — instead of splitting the op. +""" + +import importlib +import sys +from pathlib import Path + +import pytest +from collector.registry_types import OpEntry, PerfFile + +pytestmark = pytest.mark.unit + +BACKENDS = ("vllm", "sglang", "trtllm") + + +def _registry(backend): + return importlib.import_module(f"collector.{backend}.registry").REGISTRY + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_compute_scale_declares_both_tables_on_one_producer(backend): + entries = {entry.op: entry for entry in _registry(backend)} + assert "scale_matrix" not in entries # one measurement, one op + cs = entries["compute_scale"] + assert cs.perf_filename == PerfFile.COMPUTESCALE + assert cs.extra_perf_filenames == (PerfFile.SCALE_MATRIX,) + assert cs.perf_filenames == (PerfFile.COMPUTESCALE, PerfFile.SCALE_MATRIX) + + +def test_op_entry_rejects_duplicate_tables(): + with pytest.raises(ValueError, match="duplicate perf tables"): + OpEntry( + op="x", + module="m", + get_func="g", + run_func="r", + perf_filename=PerfFile.GEMM, + extra_perf_filenames=(PerfFile.GEMM,), + ) + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_provenance_collections_carry_extra_tables(backend): + from collector.version_resolver import build_collections + + version = {"vllm": "0.30.0", "sglang": "0.5.14", "trtllm": "1.3.0rc23"}[backend] + (cs,) = [ + c + for c in build_collections(_registry(backend), backend, version, ops=["compute_scale"]) + if c["type"] == "compute_scale" + ] + assert cs["perf_filename"] == PerfFile.COMPUTESCALE + assert tuple(cs["extra_perf_filenames"]) == (PerfFile.SCALE_MATRIX,) + + +def test_executor_binds_every_declared_table_to_the_producer(): + if "torch" not in sys.modules: # collect.py imports torch at module level (see test_collect_provenance_writer) + from unittest.mock import MagicMock + + _torch = MagicMock() + _torch.AcceleratorError = type("AcceleratorError", (Exception,), {}) + sys.modules["torch"] = _torch + collector_dir = str(Path(__file__).resolve().parents[3] / "collector") + if collector_dir not in sys.path: + sys.path.insert(0, collector_dir) + import collect + + collection = { + "name": "vllm", + "type": "compute_scale", + "module": "collector.vllm.collect_computescale", + "get_func": "get_computescale_test_cases", + "run_func": "run_computescale", + "perf_filename": PerfFile.COMPUTESCALE, + "extra_perf_filenames": (PerfFile.SCALE_MATRIX,), + } + assert collect._collection_perf_filenames(collection) == ["computescale_perf.txt", "scale_matrix_perf.txt"] + assert collect._collection_tables(collection) == ("computescale_perf", "scale_matrix_perf") + identity = { + "schema": collect.RESUME_SCHEMA_VERSION, + "backend": "vllm", + "module": "vllm.compute_scale", + "run_func": "run_computescale", + "framework_version": "0.30.0", + "sm_version": 90, + } + assert set(identity) == set(collect._CHECKPOINT_IDENTITY_FIELDS) + assert collect._registered_checkpoint_tables(identity, backend="vllm") == {"computescale_perf", "scale_matrix_perf"} + gemm_identity = {**identity, "module": "vllm.gemm", "run_func": "run_gemm"} + assert collect._registered_checkpoint_tables(gemm_identity, backend="vllm") == {"gemm_perf"} + with pytest.raises(RuntimeError, match="no unambiguous registered table set"): + collect._registered_checkpoint_tables({**identity, "module": "vllm.nope"}, backend="vllm") + + +@pytest.mark.parametrize("backend", BACKENDS) +def test_compute_scale_collector_writes_the_handed_in_tables(backend): + src = importlib.util.find_spec(f"collector.{backend}.collect_computescale").origin + text = Path(src).read_text() + assert "def run_computescale(m, k, *, perf_filename, extra_perf_filenames" in text + assert "def run_scale_matrix" not in text + # no table name is hard-coded any more: the executor hands both in + assert 'perf_filename="scale_matrix_perf.txt"' not in text + assert "perf_filename=scale_matrix_filename" in text + + +# ── shared-table producers: sglang dsa_*_module_skip_indexer ───────────────── +# The skip-indexer rows live in the full-module table (op_name "_skip_indexer" +# suffix; the SDK reads them from there), so the skip ops declare that table +# and select the mode with OpEntry.run_kwargs. Until 2026-10-06 they declared a +# dsa_*_skip_indexer_perf.txt nobody wrote and --resume finalization raised +# "open checkpoint event has no regular staging table" on every skip shard +# (b200_sxm sglang 0.5.21, GitLab job 469988017). + + +def test_sglang_skip_indexer_ops_share_the_full_module_table_and_select_by_run_kwargs(): + entries = {entry.op: entry for entry in _registry("sglang")} + for phase in ("context", "generation"): + full = entries[f"dsa_{phase}_module"] + skip = entries[f"dsa_{phase}_module_skip_indexer"] + assert skip.perf_filename == full.perf_filename == getattr(PerfFile, f"DSA_{phase.upper()}_MODULE") + assert skip.run_func == full.run_func == "run_mla_module_worker" + assert skip.run_kwargs_dict == {"skip_indexer": True} + assert full.run_kwargs == () + assert not any("SKIP_INDEXER" in member.name for member in PerfFile) + + +def test_provenance_collections_carry_run_kwargs(): + from collector.version_resolver import build_collections + + by_type = { + c["type"]: c + for c in build_collections( + _registry("sglang"), + "sglang", + "0.5.21", + ops=["dsa_context_module", "dsa_context_module_skip_indexer"], + ) + } + assert dict(by_type["dsa_context_module_skip_indexer"]["run_kwargs"]) == {"skip_indexer": True} + assert tuple(by_type["dsa_context_module"]["run_kwargs"]) == () + assert by_type["dsa_context_module_skip_indexer"]["perf_filename"] == by_type["dsa_context_module"]["perf_filename"] + + +@pytest.mark.parametrize( + "bad", [(("perf_filename", "x.txt"),), (("device", "cuda:1"),), (("", True),), (("a", 1), ("a", 2))] +) +def test_op_entry_rejects_executor_owned_or_malformed_run_kwargs(bad): + with pytest.raises(ValueError, match="run_kwargs"): + OpEntry(op="x", module="m", get_func="g", run_func="r", perf_filename=PerfFile.GEMM, run_kwargs=bad) + + +def test_skip_indexer_worker_takes_the_explicit_argument_and_does_not_sniff_the_table_name(): + src = importlib.util.find_spec("collector.sglang.collect_mla_module").origin + text = Path(src).read_text(encoding="utf-8") + assert 'skip_indexer: bool = False,\n):\n """Worker-compatible wrapper' in text + assert '"skip_indexer" in os.path.basename(perf_filename)' not in text diff --git a/python/aisimulate/tests/unit/collector/test_resume_finalize_zero_passed.py b/python/aisimulate/tests/unit/collector/test_resume_finalize_zero_passed.py new file mode 100644 index 0000000000..545ffbd70d --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_resume_finalize_zero_passed.py @@ -0,0 +1,90 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""--resume finalization of a producer whose every attempted case failed. + +b200_sxm sglang 0.5.21 (GitLab job 469988017, 2026-10-05): moe/int4_wo had +3,078 attempted / 0 passed (lane guard) and therefore no staging table; +``_pending_resume_perf_outputs`` raised "open checkpoint event has no regular +staging table" and the run died in finalization although the failures were +already in the error report. An open event with completed cases and no table +is still a hard error (rows were produced and are missing). + +Runs with ``PYTHONPATH=collector`` like test_registry_table_producers (collect.py +is a script-style module importing torch at module level; torch is stubbed). +""" + +import sys +import types +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit + + +def _collect(): + if "torch" not in sys.modules: + from unittest.mock import MagicMock + + _torch = MagicMock() + _torch.AcceleratorError = type("AcceleratorError", (Exception,), {}) + sys.modules["torch"] = _torch + import collect + + return collect + + +def _fake_tracker(attempted, done): + return types.SimpleNamespace(_attempted=set(attempted), _done=set(done), _failed=set(attempted) - set(done)) + + +@pytest.fixture +def provenance_ctx(): + return { + "collections": [ + { + "name": "sglang", + "type": "moe", + "module": "collector.sglang.collect_moe", + "get_func": "g", + "run_func": "run_moe_torch", + "perf_filename": "moe_perf.txt", + "extra_perf_filenames": (), + }, + ] + } + + +def test_all_failed_producer_without_staging_table_is_skipped_not_fatal(tmp_path, monkeypatch, provenance_ctx): + collect = _collect() + monkeypatch.setattr( + collect, "_load_selected_producer_checkpoint", lambda *a, **k: _fake_tracker(attempted={"a", "b"}, done=set()) + ) + pending = collect._pending_resume_perf_outputs( + tmp_path, provenance_ctx, backend="sglang", checkpoint_dir="ckpt", sm_version=100 + ) + assert pending == [] + + +def test_completed_cases_without_staging_table_still_fail_closed(tmp_path, monkeypatch, provenance_ctx): + collect = _collect() + monkeypatch.setattr( + collect, "_load_selected_producer_checkpoint", lambda *a, **k: _fake_tracker(attempted={"a", "b"}, done={"a"}) + ) + with pytest.raises(RuntimeError, match="open checkpoint event has no regular staging table"): + collect._pending_resume_perf_outputs( + tmp_path, provenance_ctx, backend="sglang", checkpoint_dir="ckpt", sm_version=100 + ) + + +def test_present_staging_table_with_open_event_is_selected(tmp_path, monkeypatch, provenance_ctx): + collect = _collect() + staging = tmp_path / "moe_perf.txt" + staging.write_text("row\n") + monkeypatch.setattr( + collect, "_load_selected_producer_checkpoint", lambda *a, **k: _fake_tracker(attempted={"a"}, done={"a"}) + ) + pending = collect._pending_resume_perf_outputs( + tmp_path, provenance_ctx, backend="sglang", checkpoint_dir="ckpt", sm_version=100 + ) + assert pending == [Path(staging)] diff --git a/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py b/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py index 856195015d..5160213b10 100644 --- a/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py +++ b/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py @@ -61,7 +61,8 @@ def test_glm5_flash_mla_keeps_the_production_topk_width(): def test_glm5_sparse_collector_is_pinned_to_sglang_0514_dsa_api(): source = SOURCE_PATH.read_text(encoding="utf-8") - assert '__compat__ = "sglang==0.5.14"' in source + # ported to the 0.5.21 pin on 2026-10-01 (same DSA module API, verified in collector/sglang/runtime_compat.py) + assert '__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20"' in source assert "SGLANG_DSA_FUSE_TOPK" in source assert "SGLANG_NSA_FUSE_TOPK" not in source @@ -120,9 +121,10 @@ def test_glm5_sparse_platform_gating_lives_outside_the_getter(): def test_dsv4_sparse_platform_gating_lives_outside_the_getter(): - """Same contract for the DSV4 sparse sub-kernel family; the flash_mla-less - HCA/CSA collectors are parked whole-op unverified instead of silently - enumerating zero cases.""" + """Same contract for the DSV4 sparse sub-kernel family. The flash_mla-less + HCA/CSA FMLA sidecar ops were parked unverified until 2026-10-15 and are now + removed outright (retired SDK placeholders that failed every registry-generated + shard plan), so only the indexer scoring kernel and the topk calibration remain.""" from collector.capabilities import _load_capabilities from collector.sglang.registry import REGISTRY @@ -131,11 +133,10 @@ def test_dsv4_sparse_platform_gating_lives_outside_the_getter(): assert "_dsv4_topk_kernel_supported" not in dsv4_source _, op_min_sm = _load_capabilities() - floors = ("dsv4_paged_mqa_logits_module", "dsv4_hca_attn_module", "dsv4_csa_attn_module", "dsv4_csa_topk_calib") + floors = ("dsv4_paged_mqa_logits_module", "dsv4_csa_topk_calib") for op in floors: assert op_min_sm.get(op) == 90 by_op = {entry.op: entry for entry in REGISTRY} assert by_op["dsv4_paged_mqa_logits_module"].unverified_sms == (120,) assert by_op["dsv4_csa_topk_calib"].unverified_sms == (120,) - assert by_op["dsv4_hca_attn_module"].unverified is True - assert by_op["dsv4_csa_attn_module"].unverified is True + assert not {"dsv4_hca_attn_module", "dsv4_csa_attn_module"} & set(by_op) diff --git a/python/aisimulate/tests/unit/collector/test_version_resolver.py b/python/aisimulate/tests/unit/collector/test_version_resolver.py index 10c3e714e4..ee7cc3b073 100644 --- a/python/aisimulate/tests/unit/collector/test_version_resolver.py +++ b/python/aisimulate/tests/unit/collector/test_version_resolver.py @@ -443,32 +443,21 @@ def test_output_dict_shape(self): "get_func", "run_func", "perf_filename", - "worker_perf_filename", + "extra_perf_filenames", + "run_kwargs", "unverified", "unverified_sms", } + assert c["extra_perf_filenames"] == () + assert c["run_kwargs"] == () assert c["name"] == "vllm" assert c["unverified"] is False assert c["unverified_sms"] == () - assert c["worker_perf_filename"] is None def test_perf_filename_propagated(self): colls = build_collections(self.SAMPLE_REGISTRY, "vllm", "0.17.0", ops=["gemm"]) assert colls[0]["perf_filename"] == PerfFile.GEMM - def test_worker_filename_keeps_canonical_output_filename(self): - entry = OpEntry( - op="dsa_context_module_skip_indexer", - module="collector.sglang.collect_mla_module", - get_func="get_dsa_context_module_skip_indexer_test_cases", - run_func="run_mla_module_worker", - perf_filename=PerfFile.DSA_CONTEXT_MODULE, - worker_perf_filename=PerfFile.DSA_CONTEXT_MODULE_SKIP_INDEXER, - ) - collection = build_collections([entry], "sglang", "0.5.14")[0] - assert collection["perf_filename"] == PerfFile.DSA_CONTEXT_MODULE - assert collection["worker_perf_filename"] == PerfFile.DSA_CONTEXT_MODULE_SKIP_INDEXER - # --------------------------------------------------------------------------- # OpEntry validation diff --git a/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py b/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py new file mode 100644 index 0000000000..b87e0318a2 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py @@ -0,0 +1,67 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""collect_attn.kv_cache_in_resolved_layout: the collector's paged KV cache must be laid out +in the layout the framework RESOLVED for the backend under test (RFC #42082), not a fixed HND. + +Origin: SM120 collection 2026-10-01 — FlashInfer resolves NHD outside SM100 and its XQA +decode asserted on the collector's contiguous [B, H, N, 2D] memory for every fp8-KV GQA +generation case (20,458 failures); SM100 (HND) and FLASH_ATTN never noticed. +The helper is pure torch, so the test imports it through ast, without vLLM.""" + +import ast +from pathlib import Path + +import pytest + +torch = pytest.importorskip("torch") +if not isinstance(getattr(torch, "__version__", None), str): # collector conftest stubs torch with a Mock + pytest.skip("real torch required (the helper is exercised on real tensors)", allow_module_level=True) +pytestmark = pytest.mark.unit + +_SRC = Path(__file__).resolve().parents[3] / "collector" / "vllm" / "collect_attn.py" + + +def _helper(): + tree = ast.parse(_SRC.read_text()) + fn = next(n for n in ast.walk(tree) if isinstance(n, ast.FunctionDef) and n.name == "kv_cache_in_resolved_layout") + ns = {"torch": torch} + exec(compile(ast.Module(body=[fn], type_ignores=[]), str(_SRC), "exec"), ns) + return ns["kv_cache_in_resolved_layout"] + + +# KVCacheLayout.layer_view_order values (vllm/v1/kv_cache_layout.py @v0.30.0): logical [B, H, N, C] +LBHNC = (0, 1, 2, 3) # HND, identity +LBNHC = (0, 2, 1, 3) # NHD + + +@pytest.mark.parametrize("order", [LBHNC, LBNHC, (1, 0, 2, 3)]) +def test_logical_view_is_unchanged_and_physical_order_is_contiguous(order): + f = _helper() + kv = torch.arange(2 * 3 * 4 * 6, dtype=torch.float32).reshape(2, 3, 4, 6) # [B, H, N, C] + out = f(kv, order) + assert out.shape == kv.shape and torch.equal(out, kv) # same logical [B, H, N, C] values + assert out.permute(*order).is_contiguous() # what the backend's permute(*layer_view_order) needs + + +def test_identity_layout_returns_contiguous_input(): + f = _helper() + kv = torch.zeros(2, 3, 4, 6) + assert f(kv, LBHNC).data_ptr() == kv.data_ptr() + + +def test_collector_uses_the_resolved_layout(): + src = _SRC.read_text() + assert "kv_cache_in_resolved_layout(" in src + assert "get_resolved_kv_cache_layout().layer_view_order" in src + + +def test_flashinfer_decode_label_names_the_kernel(): + """FlashInferTrtllmAPIDecode carries XQA and trtllm-gen in one class; the label must keep the kernel.""" + src = _SRC.read_text() + assert 'kernel = getattr(phase_metadata, "kernel", None)' in src + vocab = (Path(__file__).resolve().parents[3] / "collector" / "kernel_source_backends.yaml").read_text() + for label in ( + "vllm_flashinfer_flashinfertrtllmapidecode_xqa", + "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", + ): + assert label in vocab diff --git a/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py b/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py index 74510ddcb8..952cbf97bd 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py @@ -125,7 +125,12 @@ def __init__(self, _device): self._q_scale_float = self._k_scale_float = self._v_scale_float = 1.0 config = SimpleNamespace( - cache_config=SimpleNamespace(block_size=64, num_gpu_blocks=8192), + cache_config=SimpleNamespace( + block_size=64, + num_gpu_blocks=8192, + # resolved layout (772728f3): identity order = HND + get_resolved_kv_cache_layout=lambda: SimpleNamespace(name="LBHNC", layer_view_order=(0, 1, 2, 3)), + ), model_config=SimpleNamespace(get_sliding_window=lambda: None), ) @@ -164,6 +169,17 @@ def benchmark(**kwargs): "benchmark_with_power": benchmark, "log_perf": lambda **kwargs: calls.setdefault("log", kwargs), "vllm_version": "0.24.0", + # per-SM page size (collect_attn:125 -> utils.kv_block_size); the test pins SM90's 16 + "get_sm_version": lambda: 90, + "kv_block_size": lambda _sm, _op: 16, + # 1d0492eb: the page is asked from the framework per backend (collect_attn:201 + # -> utils.framework_kv_block_size); the test pins the same 16 + "framework_kv_block_size": lambda _backend_cls: 16, + # framework-resolved KV layout (collect_attn:248-270, 772728f3): identity order = HND + "get_supported_kv_cache_layouts": lambda _backends: [SimpleNamespace(name="LBHNC")], + "resolve_kv_cache_layout": lambda _cfg, _layouts, _specs: None, + "get_kv_cache_layout": lambda: SimpleNamespace(name="LBHNC", layer_view_order=(0, 1, 2, 3)), + "kv_cache_in_resolved_layout": lambda kv, _order: kv, } _load_function(ATTN_SOURCE, "_dense_kernel_source", namespace) run = _load_function(ATTN_SOURCE, "run_attention_torch", namespace) diff --git a/python/aisimulate/tests/unit/collector/test_vllm_dsv4_attn.py b/python/aisimulate/tests/unit/collector/test_vllm_dsv4_attn.py index f06ae0c006..c57ddea11a 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_dsv4_attn.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_dsv4_attn.py @@ -147,7 +147,7 @@ def test_smoke_has_zero_and_nonzero_prefix_without_endpoint_sweep(dsv4_module, m assert {(case[0], case[9]) for case in cases} == {(64, 0), (64, 128)} -@pytest.mark.parametrize("kernel", ["paged_mqa_logits", "hca_attn"]) +@pytest.mark.parametrize("kernel", ["paged_mqa_logits"]) def test_sparse_cases_queue_short_shapes_for_runtime_observation(dsv4_module, monkeypatch, kernel): mod = dsv4_module monkeypatch.setattr(sys, "argv", ["pytest"]) @@ -155,7 +155,6 @@ def test_sparse_cases_queue_short_shapes_for_runtime_observation(dsv4_module, mo monkeypatch.setattr(mod, "_DSV4_SPARSE_BS_LIST", [1]) monkeypatch.setattr(mod, "_DSV4_SPARSE_ISL_LIST", [1]) monkeypatch.setattr(mod, "_DSV4_SPARSE_PAST_KV_LIST", [0]) - monkeypatch.setattr(mod, "_DSV4_SPARSE_TP_LIST_ATTN", [1]) monkeypatch.setattr(mod, "_DSV4_SPARSE_TP_LIST_INDEXER", [1]) assert mod._build_vllm_dsv4_sparse_test_cases(kernel) == [[1, 1, 0, 1, kernel, "model"]] @@ -188,7 +187,7 @@ def test_dsv4_worker_rejects_ignored_runtime_inputs(dsv4_module, override, messa dsv4_module.run_dsv4_attn_worker(**args) -@pytest.mark.parametrize("kernel", ["paged_mqa_logits", "hca_attn"]) +@pytest.mark.parametrize("kernel", ["paged_mqa_logits"]) def test_sparse_worker_observes_short_shapes(dsv4_module, monkeypatch, kernel): calls = {} monkeypatch.setattr(dsv4_module, "_init_cuda", lambda device: calls.setdefault("device", device)) diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py index f4e28658ed..b36a45384d 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py @@ -73,6 +73,8 @@ def measure(**kwargs): names = { "_supported_model_map", "_get_precision_combos", + "_dsa_context_point_allowed", + "_dsa_context_budgets", "get_context_test_cases", "get_generation_test_cases", "run_mla_module_worker", diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py index f0880846cb..7f21920022 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py @@ -5,7 +5,9 @@ import ast import math +import os import sys +import time from contextlib import ExitStack, contextmanager, nullcontext from functools import wraps from pathlib import Path @@ -24,8 +26,9 @@ def _load_function(path, name, namespace): return namespace[name] -def _module_runner(*, graph_flag=True, failure=None, reuse_frequency=1, reuse_pattern=None): +def _module_runner(*, graph_flag=True, failure=None, reuse_frequency=1, reuse_pattern=None, point_failures=None): events, published, benchmark_calls = [], [], [] + point_failures = dict(point_failures or {}) error = RuntimeError(f"{failure} failed") active_contexts = [] active_slot_mapping = {} @@ -102,6 +105,18 @@ def benchmark(**kwargs): yield {"used_cuda_graph": graph_flag, "latency_ms": 0.015, "power_stats": None} events.append("graph teardown") + def create_kv(**kwargs): + events.append(("kv", kwargs["seq_len"])) + if kwargs["seq_len"] in point_failures: + raise point_failures[kwargs["seq_len"]] + return ( + Tensor(), + object(), + SimpleNamespace(slot_mapping=query_slots), + Tensor() if kwargs["attn_type"] == "dsa" else None, + object() if kwargs["attn_type"] == "dsa" else None, + ) + def cleanup(): assert active_contexts == [] assert module.mla_attn.skip_topk is False @@ -121,20 +136,17 @@ def cleanup(): set_device=lambda value: None, get_device_name=lambda value: "B200", OutOfMemoryError=MemoryError, + empty_cache=lambda: events.append("empty_cache"), ), ), + "gc": SimpleNamespace(collect=lambda: None), "setup_distributed": lambda device: None, "enable_engine_fused_ops": lambda: None, "init_workspace_manager": lambda device: events.append("workspace init"), "_create_attention_module": create_module, + "_install_rope_missing_key_shims": lambda module: ([], []), "_process_module_weights": lambda *args: None, - "_create_kv_cache_and_metadata": lambda **kwargs: ( - Tensor(), - object(), - SimpleNamespace(slot_mapping=query_slots), - Tensor() if kwargs["attn_type"] == "dsa" else None, - object() if kwargs["attn_type"] == "dsa" else None, - ), + "_create_kv_cache_and_metadata": create_kv, "set_current_vllm_config": lambda config: context("config"), "set_forward_context": forward_context, "benchmark_with_power": benchmark, @@ -143,20 +155,35 @@ def cleanup(): "_cleanup": cleanup, "vllm_version": "0.25.0", "traceback": SimpleNamespace(print_exc=lambda: None), + "time": time, } _load_function(COLLECTOR / "vllm/utils.py", "with_exit_stack", namespace) + # A prefix list expands through the plan predicate: budgets from the sweep spec, + # ceiling from the model config (both faked here). + namespace["get_mla_module_sweep_spec"] = lambda _backend=None: SimpleNamespace( + context_dsa_chunk_prefill_size=16384, context_max_tokens=131072, context_dsa_max_full_sequence_length=1048576 + ) + namespace["_model_max_position_embeddings"] = lambda _model: None + for helper in ( + "_series_seq_lens", + "_dsa_context_point_allowed", + "_dsa_context_budgets", + "_series_points", + "_is_fatal_cuda_error", + ): + _load_function(COLLECTOR / "vllm/collect_mla_module.py", helper, namespace) run = _load_function(COLLECTOR / "vllm/collect_mla_module.py", "run_mla_module", namespace) - def execute(attn_type="dsa", phase="context"): + def execute(attn_type="dsa", phase="context", seq_len=16, prefix_len=128): return run( - seq_len=16, + seq_len=seq_len, batch_size=2, num_heads=8, kv_cache_dtype="fp8", compute_dtype="bfloat16", gemm_type="bfloat16", perf_filename=f"{attn_type}_{phase}_module_perf.txt", - prefix_len=128, + prefix_len=prefix_len, model_path="fixture/model", attn_type=attn_type, ) @@ -169,7 +196,7 @@ def execute(attn_type="dsa", phase="context"): @pytest.mark.parametrize("reuse_frequency", [1, None]) def test_graph_publication_and_workspace_teardown_after_context_exit(attn_type, phase, reuse_frequency): execute, events, published, calls, _ = _module_runner(reuse_frequency=reuse_frequency) - assert execute(attn_type, phase) == 0.015 + assert execute(attn_type, phase) == [0.015] assert len(calls) == len(published) == 1 assert events[-4:] == ["graph teardown", "exit forward", "exit config", "cleanup"] row = published[0]["item_list"][0] @@ -185,7 +212,7 @@ def test_graph_publication_and_workspace_teardown_after_context_exit(attn_type, @pytest.mark.parametrize("reuse", [{"reuse_frequency": 4}, {"reuse_frequency": None, "reuse_pattern": "FFSS"}]) def test_reuse_checkpoint_publishes_native_full_and_skip_measurements(phase, reuse): execute, events, published, calls, _ = _module_runner(**reuse) - assert execute(phase=phase) == 0.015 + assert execute(phase=phase) == [0.015] assert [call["skip_indexer"] for call in calls] == [False, True] assert [row["op_name"] for row in published] == [ f"dsa_{phase}_module", @@ -218,6 +245,107 @@ def test_ordinary_mla_does_not_publish_dsa_reuse_rows(): assert [row["op_name"] for row in published] == ["mla_context_module"] +@pytest.mark.parametrize("reuse_frequency", [1, 4]) +def test_each_case_prints_one_phase_timing_line(capsys, reuse_frequency): + execute, _, _, calls, _ = _module_runner(reuse_frequency=reuse_frequency) + execute() + lines = [line for line in capsys.readouterr().out.splitlines() if "module timing" in line] + assert len(lines) == 1 + (line,) = lines + assert line.startswith(" [DSA module timing] context b=2, s=16, prefix=128, heads=8, gemm=bfloat16, kv=fp8: ") + phases = ["module", "weights", "kv_metadata", "dry_run", "measure_full"] + if reuse_frequency == 4: + phases.append("measure_skip") + phases += ["indexer_fill", "total"] + assert [field.split("=")[0] for field in line.split(": ", 1)[1].split()] == phases + assert len(calls) == len([p for p in phases if p.startswith("measure")]) + + +def test_series_builds_the_module_once_and_measures_every_point_largest_first(capsys): + execute, events, published, calls, _ = _module_runner(reuse_frequency=4) + assert execute(seq_len=[8, 16, 8]) == [0.015, 0.015] + assert events.count("workspace init") == 1 + assert [e for e in events if isinstance(e, tuple) and e[0] == "kv"] == [("kv", 16), ("kv", 8)] + assert [(row["op_name"], row["item_list"][0]["isl"]) for row in published] == [ + ("dsa_context_module", 16), + ("dsa_context_module_skip_indexer", 16), + ("dsa_context_module", 8), + ("dsa_context_module_skip_indexer", 8), + ] + assert len(calls) == 4 and events[-1] == "cleanup" and events.count("cleanup") == 1 + out = capsys.readouterr().out + timing = [line for line in out.splitlines() if "module timing" in line] + assert len(timing) == 2 + assert "module=" in timing[0] and "weights=" in timing[0], "setup phases are reported once, on the first point" + assert "module=" not in timing[1] and "kv_metadata=" in timing[1] + series_lines = [line for line in out.splitlines() if "[DSA module series] context b=2, prefix=128" in line] + assert len(series_lines) == 1 and "2/2 points" in series_lines[0] + + +def test_series_sweeps_the_isl_x_prefix_grid_of_one_module_largest_total_context_first(capsys): + """A DSA context case carries its prefix list: one module, every (isl, prefix) point of the + plan grid measured on it, largest total context first (module build was 0.9-1.2 s/pt under + the campaign's 8 workers when it was amortised over one prefix only, 2026-10-29).""" + execute, events, published, calls, _ = _module_runner(reuse_frequency=4) + assert execute(seq_len=[8, 16], prefix_len=[0, 128]) == [0.015] * 4 + assert events.count("workspace init") == 1 + kv_events = [e for e in events if isinstance(e, tuple) and e[0] == "kv"] + assert kv_events == [("kv", 16), ("kv", 8), ("kv", 16), ("kv", 8)] + rows = [ + (row["item_list"][0]["isl"], row["item_list"][0]["step"]) + for row in published + if row["op_name"] == "dsa_context_module" + ] + assert rows == [(16, 128), (8, 128), (16, 0), (8, 0)] + out = capsys.readouterr().out + assert "[DSA module] context b=2, s=[16, 8], prefix=[128, 0]," in out + series_lines = [line for line in out.splitlines() if "[DSA module series] context b=2, prefix=[128, 0]" in line] + assert len(series_lines) == 1 and "4/4 points" in series_lines[0] + assert len(calls) == 8 and events.count("cleanup") == 1 + + +def test_series_keeps_measuring_after_a_non_fatal_point_failure_and_raises_it_afterwards(): + oom = MemoryError("CUDA out of memory (fixture)") + execute, events, published, calls, _ = _module_runner(point_failures={16: oom}) + with pytest.raises(MemoryError) as caught: + execute(seq_len=[16, 8]) + assert caught.value is oom + assert [e for e in events if isinstance(e, tuple) and e[0] == "kv"] == [("kv", 16), ("kv", 8)] + assert "empty_cache" in events, "the failed point's memory is released before the next point" + assert [row["item_list"][0]["isl"] for row in published] == [8] + assert len(calls) == 1 and events[-1] == "cleanup" + + +def test_series_summarises_several_point_failures_in_one_classified_error(): + failures = {16: MemoryError("oom at 16"), 4: RuntimeError("Missing TRTLLM-GEN kernel (decode)")} + execute, _, published, _, _ = _module_runner(point_failures=failures) + with pytest.raises(RuntimeError, match=r"2/3 point\(s\) of the module series failed") as caught: + execute(seq_len=[16, 8, 4]) + assert "s=16 prefix=128: MemoryError: oom at 16" in str(caught.value) + assert "s=4 prefix=128: RuntimeError: Missing TRTLLM-GEN kernel" in str(caught.value) + assert [row["item_list"][0]["isl"] for row in published] == [8] + + +def test_series_aborts_at_once_on_a_sticky_cuda_error(): + sticky = RuntimeError("CUDA error: an illegal memory access was encountered") + execute, events, published, _, _ = _module_runner(point_failures={16: sticky}) + with pytest.raises(RuntimeError) as caught: + execute(seq_len=[16, 8]) + assert caught.value is sticky + kv_points = [e for e in events if isinstance(e, tuple) and e[0] == "kv"] + assert kv_points == [("kv", 16)], "no further point on a dead context" + assert published == [] and events[-1] == "cleanup" + + +def test_single_point_failures_keep_their_original_exception_and_timing(): + """A one-point case (CLI, single-point fills) behaves exactly like before the series layout.""" + oom = MemoryError("oom") + execute, events, published, _, _ = _module_runner(point_failures={16: oom}) + with pytest.raises(MemoryError) as caught: + execute(seq_len=16) + assert caught.value is oom and published == [] and events[-1] == "cleanup" + + def test_explicit_full_pattern_overrides_reuse_frequency(): execute, _, published, calls, _ = _module_runner(reuse_frequency=4, reuse_pattern="FFFF") execute() @@ -309,8 +437,13 @@ def empty(*shape, **kwargs): monkeypatch.setitem( sys.modules, "vllm.utils.torch_utils", SimpleNamespace(set_default_torch_dtype=lambda dtype: nullcontext()) ) + # The generic serving-parity construction resolves the model class through + # vLLM's ModelRegistry; the stubs below cover the explicit-class path, so + # select it (AB-only switch) for this allocation check. + monkeypatch.setenv("AIS_DSA_LEGACY_MODULE", "1") namespace = { "math": math, + "os": os, "torch": SimpleNamespace(empty=empty, int32="int32", no_grad=nullcontext), "_resolve_model_path": lambda model: model, "create_vllm_config": create_config, @@ -332,4 +465,7 @@ def empty(*shape, **kwargs): ) assert allocations == [(expected_tokens, 2048)] assert configs[0]["max_model_len"] == max(prefix + query, 4096) - assert configs[0]["num_gpu_blocks"] == max(1 + math.ceil((prefix + query + 1) / 64) * batch, 8192) + # The module is built before the backend (and its page) is known: the KV + # capacity is an upper bound at the smallest page (16) and is recomputed + # per selected page in _create_kv_cache_and_metadata. + assert configs[0]["num_gpu_blocks"] == max(1 + math.ceil((prefix + query + 1) / 16) * batch, 8192) diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py index ceb8c96599..378379dbb3 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py @@ -58,6 +58,87 @@ def test_materialization_rejects_meta_runtime_buffer_without_a_constructor_value move(module, "cpu") +class _Rotary: + """Stand-in for vLLM's RotaryEmbeddingBase: a non-persistent cos/sin table built by a + device-free recipe in __init__ (rotary_embedding/base.py:60-63 @ v0.30.0).""" + + def __new__(cls, torch, scale, dtype): + class Rotary(torch.nn.Module): + def __init__(self): + super().__init__() + self.scale = scale + self.register_buffer("cos_sin_cache", self._compute_cos_sin_cache().to(dtype), persistent=False) + + def _compute_cos_sin_cache(self): + return torch.arange(6, dtype=torch.float32).reshape(3, 2) * self.scale + + return Rotary() + + +def _meta_attention_module(torch): + # Serving-parity construction instantiates the model under torch.device("meta"): the + # two rotary tables (rotary_emb, indexer_rope_emb) and the KV-quant scale buffers that + # MLAAttention registers (attention.py:129-135,184 @ v0.30.0) are all meta. + with torch.device("meta"): + module = torch.nn.Module() + module.projection = torch.nn.Linear(4, 4) + module.rotary_emb = _Rotary(torch, 1.0, torch.bfloat16) + module.indexer_rope_emb = _Rotary(torch, 0.5, torch.float32) + module.mla_attn = torch.nn.Module() + for scale in ("_k_scale", "_v_scale", "_q_scale", "_prob_scale"): + module.mla_attn.register_buffer(scale, torch.tensor(1.0, dtype=torch.float32)) + module.mla_attn._k_scale_cpu = torch.tensor(1.0, dtype=torch.float32) + module.mla_attn._v_scale_cpu = torch.tensor(1.0, dtype=torch.float32) + return module + + +def test_materialization_rebuilds_meta_rotary_tables_and_quant_scales_from_the_owner_recipe(torch): + module = _meta_attention_module(torch) + assert module.rotary_emb.cos_sin_cache.is_meta and module.mla_attn._k_scale.is_meta + + move = _load_function("_move_module_preserving_buffers", torch) + moved = move(module, "cpu") + + recipe = torch.arange(6, dtype=torch.float32).reshape(3, 2) + assert moved is module + assert not moved.projection.weight.is_meta + assert moved.rotary_emb.cos_sin_cache.dtype == torch.bfloat16 + assert torch.equal(moved.rotary_emb.cos_sin_cache, recipe.to(torch.bfloat16)) + assert moved.indexer_rope_emb.cos_sin_cache.dtype == torch.float32 + assert torch.equal(moved.indexer_rope_emb.cos_sin_cache, recipe * 0.5) + assert "rotary_emb.cos_sin_cache" not in moved.state_dict() + for scale in ("_k_scale", "_v_scale", "_q_scale", "_prob_scale"): + tensor = getattr(moved.mla_attn, scale) + assert not tensor.is_meta and tensor.dtype == torch.float32 and tensor.item() == 1.0 + assert f"mla_attn.{scale}" in moved.state_dict() + assert moved.mla_attn._k_scale_cpu.device.type == "cpu" and moved.mla_attn._k_scale_cpu.item() == 1.0 + assert moved.mla_attn._v_scale_cpu.item() == 1.0 + + +def test_materialization_handles_a_rotary_shared_between_attention_and_indexer(torch): + # get_rope() caches instances per argument tuple, so both names can hold one module. + with torch.device("meta"): + module = torch.nn.Module() + module.projection = torch.nn.Linear(4, 4) + shared = _Rotary(torch, 2.0, torch.bfloat16) + module.rotary_emb = shared + module.indexer_rope_emb = shared + move = _load_function("_move_module_preserving_buffers", torch) + moved = move(module, "cpu") + recipe = (torch.arange(6, dtype=torch.float32).reshape(3, 2) * 2.0).to(torch.bfloat16) + assert moved.rotary_emb is moved.indexer_rope_emb + assert torch.equal(moved.indexer_rope_emb.cos_sin_cache, recipe) + + +def test_materialization_refuses_a_meta_buffer_whose_owner_has_no_recipe(torch): + module = _meta_attention_module(torch) + with torch.device("meta"): + module.register_buffer("block_table_cache", torch.zeros(4), persistent=False) + move = _load_function("_move_module_preserving_buffers", torch) + with pytest.raises(RuntimeError, match="uninitialized meta buffer.*block_table_cache"): + move(module, "cpu") + + @pytest.mark.parametrize("dtype_name", ["float8_e4m3fn", "float8_e5m2"]) def test_synthetic_fp8_weights_produce_nonzero_queries_without_changing_buffers_or_rng(dtype_name, torch): def make_module(): @@ -179,9 +260,15 @@ def test_nvfp4_config_preserves_native_modelopt_without_projection_exclusions(mo } -def _nvfp4_fixture(monkeypatch, torch, *, invalid_scales=False): +def _nvfp4_fixture(monkeypatch, torch, *, invalid_scales=False, w4a16=False): + # vLLM 0.30: ModelOptLinearMethod carries a (weight, activation) QuantSpec and a + # CkptCtx; NVFP4 is recognized by spec.weight == kNvfp4Static, W4A16 has no + # activation scheme and therefore no input_scale (modelopt.py:1913-2028,2448-2476). + nvfp4_static = object() + class NativeMethod: - quant_config = SimpleNamespace(group_size=16) + spec = SimpleNamespace(weight=nvfp4_static, activation=None if w4a16 else object()) + ctx = SimpleNamespace(group_size=16) calls = [] @@ -207,16 +294,19 @@ def native_quantize(values, inverse_scale, *, is_sf_swizzled_layout): ) monkeypatch.setitem( sys.modules, - "vllm.model_executor.layers.quantization.modelopt", - SimpleNamespace(ModelOptNvFp4LinearMethod=NativeMethod), + "vllm.model_executor.layers.quantization.utils.quant_utils", + SimpleNamespace(kNvfp4Static=nvfp4_static), ) module = torch.nn.Module() layer = module.projection = torch.nn.Module() layer.quant_method = NativeMethod() layer.weight = torch.nn.Parameter(torch.zeros(1025, 512, dtype=torch.uint8), requires_grad=False) - layer.weight_scale = torch.nn.Parameter(torch.zeros(1025, 64, dtype=torch.float8_e4m3fn), requires_grad=False) + layer.weight_scale = torch.nn.Parameter( + torch.full((1025, 64), float("nan")).to(torch.float8_e4m3fn), requires_grad=False + ) layer.weight_scale_2 = torch.nn.Parameter(torch.zeros(2), requires_grad=False) - layer.input_scale = torch.nn.Parameter(torch.zeros(2), requires_grad=False) + if not w4a16: + layer.input_scale = torch.nn.Parameter(torch.zeros(2), requires_grad=False) module.other_weight = torch.nn.Parameter(torch.full((2, 2), 17.0), requires_grad=False) module.register_buffer("rotary", torch.tensor([0.3, 0.7]), persistent=False) return module, calls @@ -257,3 +347,14 @@ def test_nvfp4_cannot_be_claimed_without_a_native_nvfp4_projection(monkeypatch, module.projection.quant_method = object() with pytest.raises(RuntimeError, match="no native ModelOpt NVFP4 linear"): _load_function("_initialize_nvfp4_parameters", torch)(module) + + +def test_nvfp4_w4a16_projection_is_populated_without_an_input_scale(monkeypatch, torch): + module, calls = _nvfp4_fixture(monkeypatch, torch, w4a16=True) + _load_function("_initialize_nvfp4_parameters", torch)(module) + layer = module.projection + assert not hasattr(layer, "input_scale") + assert [v.shape[0] for v, _ in calls] == [1024, 1] + assert torch.all(layer.weight == 0x12) + assert not torch.isnan(layer.weight_scale.float()).any() + assert torch.unique(layer.weight_scale_2).numel() == 1 and torch.all(layer.weight_scale_2 > 0) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py index 7c5cf83e6f..cf0d286df9 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py @@ -36,6 +36,8 @@ def _load_module_with_torch_stub(monkeypatch): assert spec and spec.loader module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) + # no GPU under unit tests: the population / serving-KV-dtype paths read the SM + monkeypatch.setattr(module, "get_sm_version", lambda: 90, raising=False) return module @@ -172,7 +174,7 @@ def test_module_cache_reuses_same_geometry_and_evicts_on_change(monkeypatch): builds = [] - def fake_build(*, model_path, attn_kind, tp_size, device): + def fake_build(*, model_path, attn_kind, tp_size, device, kv_cache_dtype="fp8"): builds.append((model_path, attn_kind, tp_size)) return (object(), object(), {"local_heads": 8, "native_heads": 64}) @@ -186,7 +188,7 @@ def fake_build(*, model_path, attn_kind, tp_size, device): b1 = module._cached_dsv4_attention_module("m/flash", "hca", 4, "cuda:0") assert len(builds) == 2 - assert list(module._MODULE_CACHE) == [("m/flash", "hca", 4, "cuda:0")] + assert list(module._MODULE_CACHE) == [("m/flash", "hca", 4, "cuda:0", "fp8")] # key carries the KV dtype (189589d2) assert b1 is not a1 module._MODULE_CACHE.clear() @@ -271,6 +273,8 @@ def _kv_cache_config(**kw): return SimpleNamespace(**kw) llm_args.KvCacheConfig = _kv_cache_config + # no GPU under unit tests: pin the Hopper branch (fp8_ds_mla pool) deterministically + monkeypatch.setattr(module, "get_sm_version", lambda: 90) # the create function re-imports this locally (relative-or-plain fallback) mla_stub = _mod("collect_mla_module") mla_stub.get_kv_cache_manager_cls = lambda mc, cfg: _CapturingManager @@ -460,3 +464,50 @@ def test_constructor_args_match_the_serving_oracle(monkeypatch): # position == past_seen (the runner builds the tensor from the # single-sourced geometry helper) assert module.generation_request_geometry(sl)["position"] == oracle["first_position"] + + +@pytest.mark.parametrize( + "bs, sl, is_ctx, prefix", + [(1, 1, True, 0), (16, 10240, True, 4096), (1024, 1, True, 0), (16, 1, False, 0), (128, 65536, False, 0)], +) +def test_kv_quota_is_capped_by_this_batch_s_tokens(monkeypatch, bs, sl, is_ctx, prefix): + """Per-case KV pool: the byte quota stays the serving half-of-free bound, + AND a max_tokens cap sized to this batch (doubled, page-rounded) is handed + to the manager's own DSV4 arithmetic — never collector-computed bytes.""" + module = _load_module_with_torch_stub(monkeypatch) + _, _, captured = _run_create(module, monkeypatch, batch_size=bs, seq_len=sl, is_context=is_ctx, prefix_len=prefix) + (kw,) = captured["kv_cache_config"] + assert kw["max_gpu_total_bytes"] == 50 * 2**30 # mem_get_info stub: free=100 GiB + tpb = kw["tokens_per_block"] + envelope = (prefix + sl + 1) if is_ctx else (sl + 1) + envelope = max(envelope, 512) + needed = bs * envelope + assert kw["max_tokens"] % tpb == 0 + floor = module._KV_POOL_MIN_TOKENS + expected = max(2 * needed, floor) + assert expected <= kw["max_tokens"] <= max(2 * (needed + bs * tpb) + tpb, floor) + # the floor keeps the footer-scale pool past inductor's int64 threshold + assert floor > 1_572_864 # H20 calibration: 1,048,576 tokens -> i32, 1,572,864 -> i64 + + +@pytest.mark.parametrize( + ("sm", "expected"), [(100, "bfloat16"), (103, "bfloat16"), (90, "fp8"), (120, "fp8"), (121, "fp8")] +) +def test_serving_kv_cache_dtype_is_an_sdk_enum_member_name(monkeypatch, sm, expected): + # engine_table_view rehydrates kv_cache_dtype through KVCacheQuantMode; "bf16" raised + # KeyError on every SM100 DSV4 table in Full CI (2026-10-29). + from aisimulate_core.quantization import KVCacheQuantMode + + module = _load_module_with_torch_stub(monkeypatch) + monkeypatch.delenv("AIS_DSV4_KV_CACHE_DTYPE", raising=False) + monkeypatch.setattr(module, "get_sm_version", lambda: sm) + label = module._serving_kv_cache_dtype() + assert label == expected + assert label in KVCacheQuantMode.__members__ + + +@pytest.mark.parametrize(("override", "expected"), [("bf16", "bfloat16"), ("bfloat16", "bfloat16"), ("fp8", "fp8")]) +def test_serving_kv_cache_dtype_override_normalises_the_bf16_alias(monkeypatch, override, expected): + module = _load_module_with_torch_stub(monkeypatch) + monkeypatch.setenv("AIS_DSV4_KV_CACHE_DTYPE", override) + assert module._serving_kv_cache_dtype() == expected diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py new file mode 100644 index 0000000000..cb179bfb6e --- /dev/null +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The trtllm GDN collector publishes CUDA-graph-timed rows only. + +Until 2026-10-11 every benchmark_with_power call in collector/trtllm/collect_gdn.py +passed allow_graph_fail=True: a failed capture logged one WARNING, timed the +Python launch loop eagerly and shipped that floor as the kernel latency with no +trace in the row. Capture failures now raise, the failed cell is counted, and +the case ends as a classified strict-completeness failure (the sglang GDN +collector's contract).""" + +import ast +from contextlib import contextmanager, nullcontext +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[4] +SOURCE = REPO_ROOT / "collector" / "trtllm" / "collect_gdn.py" + + +def _tree(): + return ast.parse(SOURCE.read_text(encoding="utf-8"), filename=str(SOURCE)) + + +def _function(name): + return next(node for node in _tree().body if isinstance(node, ast.FunctionDef) and node.name == name) + + +def test_trtllm_gdn_benchmarks_never_fall_back_to_eager_timing(): + calls = [ + node + for node in ast.walk(_tree()) + if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "benchmark_with_power" + ] + assert len(calls) == 9 + for call in calls: + keyword = next((kw for kw in call.keywords if kw.arg == "allow_graph_fail"), None) + assert keyword is None or not (isinstance(keyword.value, ast.Constant) and keyword.value.value is True) + assert all(kw.arg != "use_cuda_graph" for kw in call.keywords) + + +@pytest.mark.parametrize("function_name", ["run_gdn_context_benchmark", "run_gdn_generation_benchmark"]) +def test_trtllm_gdn_phase_runners_end_in_a_strict_completeness_raise(function_name): + function = _function(function_name) + # A per-cell `continue` is only acceptable because the function re-raises at the end. + raises = [ + node + for node in function.body + if isinstance(node, ast.If) and any(isinstance(statement, ast.Raise) for statement in node.body) + ] + assert raises, f"{function_name} swallows cell failures without a final raise" + assert "strict completeness" in ast.unparse(raises[-1]) + + +class _Tensor: + def __init__(self, shape): + self.shape = tuple(shape) + + def reshape(self, *shape): + return _Tensor(shape) + + def float(self): + return self + + +def _fake_torch(): + def factory(*shape, **_kwargs): + return _Tensor(shape) + + return SimpleNamespace( + bfloat16="bf16", + float32="fp32", + int64="i64", + device=lambda name: name, + cuda=SimpleNamespace( + set_device=lambda _d: None, + synchronize=lambda: None, + empty_cache=lambda: None, + get_device_name=lambda _d: "fake-gpu", + ), + set_default_device=lambda _d: None, + randn=factory, + zeros=factory, + arange=lambda *a, **k: _Tensor((1,)), + sigmoid=lambda t: t, + nn=SimpleNamespace(functional=SimpleNamespace(logsigmoid=lambda t: t)), + ) + + +def _load_context_runner(benchmark_with_power, log_perf): + namespace = { + "torch": _fake_torch(), + "gc": SimpleNamespace(collect=lambda: None), + "aic_debug": 0, + "aic_cached_inputs": 1, + "benchmark_with_power": benchmark_with_power, + "log_perf": log_perf, + "causal_conv1d_fn": lambda *a, **k: None, + "chunk_gated_delta_rule": lambda *a, **k: None, + "_CHUNK_KERNEL_SOURCE": "chunk_gated_delta_rule_flashinfer", + "_make_input_pool": None, + } + module = ast.Module(body=[_function("run_gdn_context_benchmark")], type_ignores=[]) + exec(compile(module, str(SOURCE), "exec"), namespace) + return namespace["run_gdn_context_benchmark"] + + +def test_trtllm_gdn_context_capture_failure_fails_the_case_but_keeps_the_other_cells(): + calls = {"bench": 0} + logged = [] + + @contextmanager + def benchmark_with_power(**_kwargs): + calls["bench"] += 1 + if calls["bench"] == 3: # second (batch, seq) cell: the conv capture fails + raise RuntimeError("CUDA graph capture failed: operation not permitted when stream is capturing") + yield {"latency_ms": 0.01, "power_stats": None, "used_cuda_graph": True} + + def log_perf(**kwargs): + logged.append(kwargs["kernel_source"]) + return True + + run = _load_context_runner(benchmark_with_power, log_perf) + expected = r"strict completeness: ok=1 error=1 skip=0.*seq_len=64: RuntimeError" + with pytest.raises(RuntimeError, match=expected) as info: + run( + d_model=2048, + d_conv=4, + num_k_heads=16, + head_k_dim=128, + num_v_heads=32, + head_v_dim=128, + batch_size_list=[1], + seq_len_list=[32, 64], + model_name="Qwen/Qwen3.5-test", + perf_filename="gdn_perf.txt", + trtllm_version="1.3.0rc29", + device="cuda:0", + ) + assert "CUDA graph capture failed" in str(info.value) + # the first cell shipped both of its rows; the failed cell shipped none + assert logged == ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"] + + +def test_trtllm_gdn_context_all_cells_captured_returns_quietly(): + @contextmanager + def benchmark_with_power(**_kwargs): + yield {"latency_ms": 0.01, "power_stats": None, "used_cuda_graph": True} + + run = _load_context_runner(benchmark_with_power, lambda **_k: True) + with nullcontext(): + run( + d_model=2048, + d_conv=4, + num_k_heads=16, + head_k_dim=128, + num_v_heads=32, + head_v_dim=128, + batch_size_list=[1, 2], + seq_len_list=[32], + model_name="Qwen/Qwen3.5-test", + perf_filename="gdn_perf.txt", + trtllm_version="1.3.0rc29", + ) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py new file mode 100644 index 0000000000..eafe60a80f --- /dev/null +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py @@ -0,0 +1,178 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU-free tests for the TRT-LLM GEMM collector's per-(gemm_type, n, k) module cache. + +``collector/trtllm/collect_gemm.py`` imports torch and tensorrt_llm at module +scope, so ``_gemm_ops`` / ``run_gemm`` are AST-extracted and executed against +fakes (same technique as ``test_collect_moe_ep.py``). +""" + +from __future__ import annotations + +import ast +import math +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace +from typing import ClassVar + +import pytest + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[4] +SOURCE_PATH = REPO_ROOT / "collector" / "trtllm" / "collect_gemm.py" +NAMES = ("_gemm_op_cache", "_gemm_ops", "run_gemm") +L2_BYTES = 126 * 2**20 + + +class _Tensor: + def __init__(self, shape): + self.shape = shape + + def to(self, *args, **kwargs): + return self + + +class _Linear: + instances: ClassVar[list] = [] + + def __init__(self, in_features, out_features, **kwargs): + self.in_features = in_features + self.out_features = out_features + self.kwargs = kwargs + self.loaded = [] + self.post_loaded = 0 + self.forwards: list = [] + _Linear.instances.append(self) + + def load_weights(self, weights): + self.loaded.append(weights) + + def post_load_weights(self): + self.post_loaded += 1 + + def to(self, device): + self.device = device + return self + + def forward(self, x): + self.forwards.append(x) + + +def _load(): + tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) + + def wanted(node) -> bool: + if isinstance(node, ast.FunctionDef): + return node.name in NAMES + if isinstance(node, ast.AnnAssign): + return isinstance(node.target, ast.Name) and node.target.id in NAMES + return False + + selected = [node for node in tree.body if wanted(node)] + assert len(selected) == len(NAMES) + _Linear.instances = [] + built: list = [] + empty_cache_calls: list = [] + benchmarks: list = [] + published: list = [] + + @contextmanager + def benchmark_with_power(*, device, kernel_func, repeat_n): + benchmarks.append({"forwards_before": [len(op.forwards) for op in _Linear.instances], "repeat_n": repeat_n}) + kernel_func() + yield {"latency_ms": 1.0, "power_stats": None} + + namespace = { + "math": math, + "torch": SimpleNamespace( + device=lambda value: value, + bfloat16="bf16", + randn=lambda shape, dtype: _Tensor(shape), + set_default_device=lambda device: None, + cuda=SimpleNamespace( + set_device=lambda device: None, + empty_cache=lambda: empty_cache_calls.append(True), + get_device_name=lambda device: "B200", + ), + ), + "Linear": _Linear, + "_build_weights": lambda gemm_type, n, k, device, dtype, group_size, x: ( + built.append((gemm_type, n, k, group_size)) or {"weight": (n, k)} + ), + "_get_l2_cache_bytes": lambda index: L2_BYTES, + "get_sm_version": lambda: 100, + "is_sm_100f": lambda sm: sm in (100, 103), + "QuantConfig": lambda **kwargs: kwargs, + "QuantAlgo": SimpleNamespace(FP8="fp8", FP8_BLOCK_SCALES="fp8_block", NVFP4="nvfp4"), + "benchmark_with_power": benchmark_with_power, + "log_perf": lambda **kwargs: published.append(kwargs), + "tensorrt_llm": SimpleNamespace(__version__="1.3.0rc29"), + } + exec(compile(ast.Module(body=selected, type_ignores=[]), str(SOURCE_PATH), "exec"), namespace) + return namespace, built, empty_cache_calls, benchmarks, published + + +def _ops(namespace, gemm_type, n, k, *, copies): + return namespace["_gemm_ops"]( + gemm_type, + n, + k, + quant_config=None, + group_size=None, + outside_loop_count=copies, + device=SimpleNamespace(index=0), + dtype="bf16", + x=_Tensor((1, k)), + ) + + +def test_same_key_reuses_the_built_modules_and_a_new_key_replaces_them(): + namespace, built, empty_cache_calls, _, _ = _load() + first = _ops(namespace, "nvfp4", 4096, 4096, copies=5) + assert len(first) == 5 and len(built) == 5 and len(_Linear.instances) == 5 + assert empty_cache_calls == [True], "the previous key's modules are dropped before building" + assert _ops(namespace, "nvfp4", 4096, 4096, copies=5) is first + assert len(built) == 5, "a second m for the same (gemm_type, n, k) builds nothing" + second = _ops(namespace, "nvfp4", 8192, 4096, copies=2) + assert second is not first and len(second) == 2 + assert len(built) == 7 and len(empty_cache_calls) == 2 + assert list(namespace["_gemm_op_cache"]) == [("nvfp4", 8192, 4096, 2)], "one key is held at a time" + + +def test_run_gemm_builds_once_per_key_dry_runs_every_copy_and_divides_the_latency(): + namespace, built, _, benchmarks, published = _load() + n = k = 1024 # 2 MiB bf16 weight -> 5 copies to span the 126 MiB L2 + for m in (8192, 4096, 1): + namespace["run_gemm"]("bfloat16", m, n, k, perf_filename="gemm_perf.txt", device="cuda:0") + assert len(built) == 5 and len(_Linear.instances) == 5 + expected_copies = min(5, math.ceil(L2_BYTES / (n * k * 2))) + assert expected_copies == 5 + # Before each capture every cached copy saw its dry run at the current m (JIT / lazy init). + assert [b["forwards_before"] for b in benchmarks] == [[1] * 5, [3] * 5, [5] * 5] + assert all(b["repeat_n"] == 1 for b in benchmarks) + assert [row["item_list"][0]["m"] for row in published] == [8192, 4096, 1] + assert all(row["item_list"][0]["latency"] == pytest.approx(1.0 / 5) for row in published) + assert all(row["kernel_source"] == "torch_flow" for row in published) + + +def test_fp8_block_post_load_runs_once_per_copy_and_is_tagged_deepgemm_on_sm100(): + namespace, built, _, _, published = _load() + n, k = 16384, 7168 # 112 MiB fp8 weight -> 2 copies + namespace["run_gemm"]("fp8_block", 64, n, k, perf_filename="gemm_perf.txt", device="cuda:0") + namespace["run_gemm"]("fp8_block", 32, n, k, perf_filename="gemm_perf.txt", device="cuda:0") + assert built == [("fp8_block", n, k, 128)] * 2 + assert [op.post_loaded for op in _Linear.instances] == [1, 1] + expected_quant = {"quant_algo": "fp8_block", "group_size": 128} + assert [op.kwargs["quant_config"] for op in _Linear.instances] == [expected_quant] * 2 + assert [row["kernel_source"] for row in published] == ["deepgemm", "deepgemm"] + assert all(row["item_list"][0]["latency"] == pytest.approx(0.5) for row in published) + + +def test_fp8_alignment_limit_still_raises_before_any_module_is_built(): + namespace, built, _, _, _ = _load() + with pytest.raises(RuntimeError, match="FIXME\\(kernel-limit\\)"): + namespace["run_gemm"]("fp8", 16, 24, 4096, perf_filename="gemm_perf.txt", device="cuda:0") + assert built == [] and _Linear.instances == [] diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py new file mode 100644 index 0000000000..7f56e2797e --- /dev/null +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py @@ -0,0 +1,336 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""The trtllm Mamba2 collector publishes CUDA-graph-timed rows only, never +drops a kernel lane silently, and runs the SSM kernels the way serving does. + +Until 2026-10-12 every benchmark_with_power call in collector/trtllm/collect_mamba2.py +passed allow_graph_fail=True, each (batch, seq) cell sat in a bare +`except Exception: continue`, and neither SSM kernel received the caller-owned +`out` buffer TRT-LLM requires (ssd_chunk_scan.py:667, selective_state_update.py +`out.dim()` @1.3.0rc20/rc29). The shipped rc20 mamba2 tables of all eight +systems and the rc29 b200 package contain only the causal_conv1d lanes: the +mamba_chunk_scan_combined / selective_state_update lanes vanished with zero +failure records. Capture failures now raise, every failed cell is counted, the +case ends as a classified strict-completeness failure (the GDN contract), both +SSM calls carry `out=`, and the decode state update is the kernel serving's +Mamba2Mixer gate selects (FlashInfer or TRT-LLM's Triton), labelled as such.""" + +import ast +from contextlib import contextmanager +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[4] +SOURCE = REPO_ROOT / "collector" / "trtllm" / "collect_mamba2.py" + + +def _tree(): + return ast.parse(SOURCE.read_text(encoding="utf-8"), filename=str(SOURCE)) + + +def _function(name): + return next(node for node in _tree().body if isinstance(node, ast.FunctionDef) and node.name == name) + + +def _callee(node): + return ast.unparse(node.func) + + +def test_trtllm_mamba2_benchmarks_never_fall_back_to_eager_timing(): + calls = [ + node for node in ast.walk(_tree()) if isinstance(node, ast.Call) and _callee(node) == "benchmark_with_power" + ] + assert len(calls) == 8 + for call in calls: + assert all(keyword.arg not in ("allow_graph_fail", "use_cuda_graph") for keyword in call.keywords) + + +@pytest.mark.parametrize("function_name", ["run_mamba2_context_benchmark", "run_mamba2_generation_benchmark"]) +def test_trtllm_mamba2_phase_runners_end_in_a_strict_completeness_raise(function_name): + function = _function(function_name) + # A per-cell `continue` is only acceptable because the function re-raises at the end. + raises = [ + node + for node in function.body + if isinstance(node, ast.If) and any(isinstance(statement, ast.Raise) for statement in node.body) + ] + assert raises, f"{function_name} swallows cell failures without a final raise" + assert "strict completeness" in ast.unparse(raises[-1]) + + +def test_every_ssm_kernel_call_hands_over_the_caller_owned_out_buffer(): + # TRT-LLM's chunk scan asserts on `out` and its state update calls + # `out.dim()`; the missing keyword is what emptied the SSM lanes. + scan_calls = [ + node + for node in ast.walk(_function("run_mamba2_context_benchmark")) + if isinstance(node, ast.Call) and _callee(node) == "mamba_chunk_scan_combined" + ] + update_calls = [ + node + for node in ast.walk(_function("run_mamba2_generation_benchmark")) + if isinstance(node, ast.Call) and _callee(node) == "ssu_func" + ] + assert len(scan_calls) == 4 # cached + pooled, warm-up call + timed closure + assert len(update_calls) == 4 + for call in scan_calls + update_calls: + assert any(keyword.arg == "out" for keyword in call.keywords), ast.unparse(call) + for call in update_calls: + assert any(keyword.arg == "state_batch_indices" for keyword in call.keywords), ast.unparse(call) + + +def test_generation_runner_never_calls_the_triton_state_update_directly(): + # The decode kernel comes from the serving gate, never from a hard-coded name. + direct = [ + node + for node in ast.walk(_function("run_mamba2_generation_benchmark")) + if isinstance(node, ast.Call) and _callee(node) == "selective_state_update" + ] + assert direct == [] + for literal in ("selective_state_update", "selective_state_update_flashinfer", "selective_state_update_triton"): + assert all( + not (isinstance(node, ast.Constant) and node.value == literal) + for node in ast.walk(_function("run_mamba2_generation_benchmark")) + ), f"{literal!r} is hard-coded in the generation runner" + + +def _load_gate(): + tree = _tree() + body = [ + node + for node in tree.body + if (isinstance(node, ast.Assign) and ast.unparse(node.targets[0]).startswith("_FLASHINFER_SSU_")) + or (isinstance(node, ast.FunctionDef) and node.name == "_serving_selects_flashinfer_ssu") + ] + namespace = {} + exec(compile(ast.Module(body=body, type_ignores=[]), str(SOURCE), "exec"), namespace) + return namespace["_serving_selects_flashinfer_ssu"] + + +@pytest.mark.parametrize( + ("nheads", "n_groups", "head_dim", "d_state", "flashinfer"), + [ + (64, 8, 64, 128, True), # Nemotron-3 Nano / generic 4K: ratio 8 + (256, 8, 64, 256, True), # Nemotron-H 56B: ratio 32, d_state 256 + (16, 4, 64, 64, True), # generic 1K: ratio 4 (rc29 widened the list; rc20 had [1, 8, 16]) + (64, 8, 80, 128, False), # Nemotron-v2-Nano head_dim 80: no FlashInfer kernel + (48, 16, 64, 128, False), # ratio 3 + (64, 8, 64, 32, False), # d_state 32 + (64, 0, 64, 128, False), # no groups + ], +) +def test_decode_kernel_gate_mirrors_the_rc29_mixer(nheads, n_groups, head_dim, d_state, flashinfer): + assert _load_gate()(nheads, n_groups, head_dim, d_state) is flashinfer + + +class _Tensor: + """Shape-only stand-in; arithmetic and method calls return another stand-in.""" + + def __init__(self, shape=()): + self.shape = tuple(shape) + + def __getattr__(self, name): + if name.startswith("__"): + raise AttributeError(name) + return lambda *args, **kwargs: _Tensor(self.shape) + + def __neg__(self): + return _Tensor(self.shape) + + def __add__(self, other): + return _Tensor(self.shape) + + __radd__ = __add__ + __sub__ = __add__ + __rsub__ = __add__ + __mul__ = __add__ + __rmul__ = __add__ + + +def _factory(*shape, **_kwargs): + return _Tensor(shape) + + +def _fake_torch(): + return SimpleNamespace( + bfloat16="bf16", + float32="fp32", + int32="i32", + device=lambda name: name, + set_default_device=lambda _device: None, + cuda=SimpleNamespace( + set_device=lambda _device: None, + synchronize=lambda: None, + empty_cache=lambda: None, + get_device_name=lambda _device: "fake-gpu", + ), + randn=_factory, + rand=_factory, + zeros=_factory, + arange=lambda n, **_kwargs: _Tensor((n,)), + empty_like=lambda tensor: _Tensor(tensor.shape), + ) + + +def _fake_input_pool(shapes, count, _dtype, _device): + return {name: [_Tensor(shape) for _ in range(count)] for name, shape in shapes.items()} + + +def _load_runner( + name, benchmark_with_power, log_perf, *, cached_inputs=1, ssu_label="selective_state_update_flashinfer" +): + ssu_calls = [] + + def ssu(*args, **kwargs): + ssu_calls.append(kwargs) + + namespace = { + "torch": _fake_torch(), + "gc": SimpleNamespace(collect=lambda: None), + "aic_debug": 0, + "aic_cached_inputs": cached_inputs, + "benchmark_with_power": benchmark_with_power, + "log_perf": log_perf, + "repeat": lambda tensor, _pattern, **_axes: tensor, + "tensorrt_llm": SimpleNamespace(__version__="1.3.0rc29"), + "causal_conv1d_fn": lambda *args, **kwargs: None, + "causal_conv1d_update": lambda *args, **kwargs: None, + "mamba_chunk_scan_combined": lambda *args, **kwargs: None, + "selective_state_update": lambda *args, **kwargs: None, + "_decode_state_update_kernel": lambda *geometry: (ssu, ssu_label), + "_decode_dt": lambda batch_size, nheads, head_dim, _device: _Tensor((batch_size, nheads, head_dim)), + "_token_major_prefill_conv": lambda: False, + "_prefill_conv_token_major_inputs": None, + "_run_prefill_conv_token_major": None, + "_make_input_pool": _fake_input_pool, + } + module = ast.Module(body=[_function(name)], type_ignores=[]) + exec(compile(module, str(SOURCE), "exec"), namespace) + return namespace[name], ssu_calls + + +def _failing_on(call_index): + """A fake benchmark_with_power whose ``call_index``-th capture fails (0 = never).""" + calls = {"bench": 0} + logged = [] + + @contextmanager + def benchmark_with_power(**kwargs): + calls["bench"] += 1 + if calls["bench"] == call_index: + raise RuntimeError("CUDA graph capture failed: operation not permitted when stream is capturing") + kwargs["kernel_func"]() # the real helper runs the closure for its warmups and the capture + yield {"latency_ms": 0.01, "power_stats": None, "used_cuda_graph": True} + + def log_perf(**kwargs): + logged.append(kwargs["kernel_source"]) + return True + + return benchmark_with_power, log_perf, logged + + +_LAYER = { + "d_model": 4096, + "d_state": 128, + "d_conv": 4, + "nheads": 128, + "head_dim": 64, + "n_groups": 8, + "chunk_size": 256, + "model_name": "nvidia/Nemotron-test", + "perf_filename": "mamba2_perf.txt", + "device": "cuda:0", +} + + +def test_trtllm_mamba2_context_scan_capture_failure_fails_the_case_instead_of_dropping_the_lane(): + # The shipped-data failure mode: a cell's conv row ships, then its scan lane raises. + benchmark_with_power, log_perf, logged = _failing_on(2) + run, _ = _load_runner("run_mamba2_context_benchmark", benchmark_with_power, log_perf) + expected = r"strict completeness: ok=1 error=1 skip=0.*seq_len=32: RuntimeError" + with pytest.raises(RuntimeError, match=expected) as info: + run(batch_size_list=[1], seq_len_list=[32, 64], **_LAYER) + assert "CUDA graph capture failed" in str(info.value) + # cell (1, 32) shipped its conv row and lost the scan; cell (1, 64) shipped both + assert logged == ["causal_conv1d_fn", "causal_conv1d_fn", "mamba_chunk_scan_combined"] + + +@pytest.mark.parametrize("cached_inputs", [1, 0]) +def test_trtllm_mamba2_context_all_cells_captured_returns_quietly(cached_inputs): + benchmark_with_power, log_perf, logged = _failing_on(0) + run, _ = _load_runner("run_mamba2_context_benchmark", benchmark_with_power, log_perf, cached_inputs=cached_inputs) + assert run(batch_size_list=[1, 2], seq_len_list=[32], **_LAYER) is None + assert logged == ["causal_conv1d_fn", "mamba_chunk_scan_combined"] * 2 + + +def test_trtllm_mamba2_generation_state_update_capture_failure_fails_the_case(): + benchmark_with_power, log_perf, logged = _failing_on(4) + run, _ = _load_runner("run_mamba2_generation_benchmark", benchmark_with_power, log_perf) + expected = r"strict completeness: ok=1 error=1 skip=0.*batch_size=2: RuntimeError" + with pytest.raises(RuntimeError, match=expected): + run(batch_size_list=[1, 2], **_LAYER) + assert logged == ["causal_conv1d_update", "selective_state_update_flashinfer", "causal_conv1d_update"] + + +@pytest.mark.parametrize("cached_inputs", [1, 0]) +@pytest.mark.parametrize("ssu_label", ["selective_state_update_flashinfer", "selective_state_update_triton"]) +def test_trtllm_mamba2_generation_labels_the_rows_with_the_kernel_serving_selected(cached_inputs, ssu_label): + benchmark_with_power, log_perf, logged = _failing_on(0) + run, ssu_calls = _load_runner( + "run_mamba2_generation_benchmark", + benchmark_with_power, + log_perf, + cached_inputs=cached_inputs, + ssu_label=ssu_label, + ) + assert run(batch_size_list=[1], **_LAYER) is None + assert logged == ["causal_conv1d_update", ssu_label] + # warm-up call + the timed closure; every call carries the serving-shaped extras + assert len(ssu_calls) == 2 + for kwargs in ssu_calls: + assert kwargs["out"].shape == (1, 128, 64) + assert kwargs["state_batch_indices"].shape == (1,) + assert kwargs["dt_softplus"] is True and kwargs["z"] is None + + +def test_generation_runner_builds_dt_only_through_the_serving_layout_helper(): + # Both input modes take dt from _decode_dt; no dense (batch, nheads, head_dim) dt is made. + runner = _function("run_mamba2_generation_benchmark") + helper_calls = [node for node in ast.walk(runner) if isinstance(node, ast.Call) and _callee(node) == "_decode_dt"] + assert len(helper_calls) == 2 + dt_assignments = [ + node + for node in ast.walk(runner) + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "dt" for t in node.targets) + ] + assert dt_assignments, "cached-input mode no longer binds dt" + for assignment in dt_assignments: + assert isinstance(assignment.value, ast.Call) and _callee(assignment.value) == "_decode_dt" + + +def test_decode_dt_is_an_fp32_per_head_tensor_broadcast_over_head_dim(): + # FlashInfer's binding requires dt.stride(1) == 1 and dt.stride(2) == 0 + # (csrc/selective_state_update.cu:187-188 @v0.6.18); serving gets there with + # fp32 (batch, nheads) + einops.repeat "b h -> b h p" (mamba2_mixer.py:764 @rc29). + seen = {} + + def randn(*shape, dtype=None, device=None): + seen["randn"] = (shape, dtype) + return _Tensor(shape) + + def repeat(tensor, pattern, **axes): + seen["repeat"] = (tensor.shape, pattern, axes) + return _Tensor(tensor.shape + (axes["p"],)) + + namespace = {"torch": SimpleNamespace(randn=randn, float32="fp32", Tensor=object), "repeat": repeat} + module = ast.Module(body=[_function("_decode_dt")], type_ignores=[]) + exec(compile(module, str(SOURCE), "exec"), namespace) + dt = namespace["_decode_dt"](4, 128, 64, "cuda:0") + assert seen["randn"] == ((4, 128), "fp32") + assert seen["repeat"] == ((4, 128), "b h -> b h p", {"p": 64}) + assert dt.shape == (4, 128, 64) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py new file mode 100644 index 0000000000..f8769b6e4c --- /dev/null +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU-free tests for the per-worker attention-layer caches of the TRT-LLM MLA/DSA and +MSA module collectors (same pattern as collect_dsv4_attn._cached_dsv4_attention_module). +The modules import tensorrt_llm at module scope, so the helpers are AST-extracted.""" + +from __future__ import annotations + +import ast +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit +COLLECTOR = Path(__file__).resolve().parents[4] / "collector" / "trtllm" + + +def _load(filename: str, helper: str, builder: str): + tree = ast.parse((COLLECTOR / filename).read_text()) + selected = [ + n + for n in tree.body + if (isinstance(n, ast.FunctionDef) and n.name == helper) + or (isinstance(n, ast.AnnAssign) and isinstance(n.target, ast.Name) and n.target.id == "_LAYER_CACHE") + ] + assert len(selected) == 2 + builds, evictions = [], [] + namespace = { + builder: lambda **kwargs: (builds.append(kwargs) or ("module", "config", "arch", len(builds))), + "gc": SimpleNamespace(collect=lambda: evictions.append("gc")), + "torch": SimpleNamespace(cuda=SimpleNamespace(empty_cache=lambda: evictions.append("empty_cache"))), + } + exec(compile(ast.Module(body=selected, type_ignores=[]), filename, "exec"), namespace) + return namespace[helper], builds, evictions + + +def test_mla_layer_is_built_once_per_key_and_evicted_on_a_new_key(): + cached, builds, evictions = _load("collect_mla_module.py", "_cached_attention_layer", "create_attention_layer") + first = cached("deepseek-ai/DeepSeek-V3.2", 128, True, "fp8_block", "cuda:0") + assert cached("deepseek-ai/DeepSeek-V3.2", 128, True, "fp8_block", "cuda:0") is first + assert len(builds) == 1 and evictions == [] + assert builds[0] == { + "model_path": "deepseek-ai/DeepSeek-V3.2", + "num_heads": 128, + "use_fp8_kv_cache": True, + "gemm_type": "fp8_block", + "device": "cuda:0", + } + second = cached("deepseek-ai/DeepSeek-V3.2", 16, True, "fp8_block", "cuda:0") + assert second is not first and len(builds) == 2 + assert evictions == ["gc", "empty_cache"], "the previous layer is released before the next build" + third = cached("deepseek-ai/DeepSeek-V3.2", 128, True, "fp8_block", "cuda:0") + assert third is not first, "one entry at a time" + + +def test_msa_layer_cache_keys_on_model_heads_gemm_and_device(): + cached, builds, evictions = _load( + "collect_msa_module.py", "_cached_msa_attention_layer", "create_msa_attention_layer" + ) + a = cached("MiniMaxAI/MiniMax-M3", 64, "bfloat16", "cuda:0") + assert cached("MiniMaxAI/MiniMax-M3", 64, "bfloat16", "cuda:0") is a and len(builds) == 1 + expected = {"model_path": "MiniMaxAI/MiniMax-M3", "num_heads": 64, "gemm_type": "bfloat16", "device": "cuda:0"} + assert builds[0] == expected + cached("MiniMaxAI/MiniMax-M3", 64, "fp8_block", "cuda:0") + assert len(builds) == 2 and evictions == ["gc", "empty_cache"] + + +@pytest.mark.parametrize( + "filename,runner,helper", + [ + ("collect_mla_module.py", "run_mla_module", "_cached_attention_layer"), + ("collect_msa_module.py", "run_msa_module", "_cached_msa_attention_layer"), + ], +) +def test_runner_builds_its_layer_through_the_cache(filename, runner, helper): + tree = ast.parse((COLLECTOR / filename).read_text()) + body = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == runner) + calls = [ + n.func.id + for n in ast.walk(body) + if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id.startswith(("create_", "_cached_")) + ] + assert helper in calls + assert not any(c in ("create_attention_layer", "create_msa_attention_layer") for c in calls) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py index fd6e800818..919f82bdb9 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py @@ -415,30 +415,29 @@ def test_moe_ep_alone_resolves_to_the_wideep_trtllm_pin(): assert runtime.version == "1.3.0rc20" -def test_same_pin_mixing_with_stock_trtllm_ops_is_accepted(): - # Unlike wideep_sglang (0.5.10 vs stock 0.5.14 — mixing fail-closes), - # wideep_trtllm pins the SAME version AND image digest as stock trtllm - # (framework_manifest.yaml), so require_collector_runtime accepts a mixed - # stock+wideep request — the reason moe_ep can stay in the default trtllm - # model plans. The stock op here must live on the DEFAULT trtllm runtime: - # moe/sparse_attention carry per-family rc23 overrides, and mixing those - # with default-runtime ops correctly fail-closes as a multi-runtime - # request (a separate behavior from wideep/stock mixing). +def test_mixing_stock_and_wideep_trtllm_ops_fail_closes_on_different_pins(): + # Stock trtllm (1.3.0rc29 default) and wideep_trtllm (1.3.0rc20) no longer + # share a pin (framework_manifest.yaml, 2026-10-04): a mixed stock+wideep + # request is a multi-runtime request and must be split, exactly like + # wideep_sglang (0.5.10) vs stock sglang. moe_ep alone still resolves to + # the WideEP pin (test above); the stock op alone resolves to rc29. from collector.framework_manifest import require_collector_runtime - mixed = require_collector_runtime("trtllm", "1.3.0rc20", requested_ops={"gemm", "moe_ep"}, wideep_ops={"moe_ep"}) - stock = require_collector_runtime("trtllm", "1.3.0rc20", requested_ops={"gemm"}, wideep_ops={"moe_ep"}) - assert mixed.version == stock.version == "1.3.0rc20" - assert mixed.images == stock.images + with pytest.raises(RuntimeError, match=r"different runtime versions \(1\.3\.0rc29 != 1\.3\.0rc20\)"): + require_collector_runtime("trtllm", "1.3.0rc29", requested_ops={"gemm", "moe_ep"}, wideep_ops={"moe_ep"}) + stock = require_collector_runtime("trtllm", "1.3.0rc29", requested_ops={"gemm"}, wideep_ops={"moe_ep"}) + assert stock.framework == "trtllm" and stock.version == "1.3.0rc29" -def test_moe_ep_is_in_the_default_trtllm_plan_for_wideep_models(): +def test_moe_ep_is_requested_independently_of_the_default_trtllm_plan(): + # Mirrors sglang: with separate WideEP pins, moe_ep stays out of the default + # model plans on both backends so a default run resolves to one runtime. from collector.model_cases import build_collection_case_plan - trtllm_plan = build_collection_case_plan(backend="trtllm", model_path="deepseek-ai/DeepSeek-V3") - assert "moe_ep" in trtllm_plan.selected_ops - assert "trtllm_moe_wideep" not in trtllm_plan.selected_ops - # sglang keeps it out of the default plan (separate 0.5.10 runtime). + for model in ("deepseek-ai/DeepSeek-V3", "moonshotai/Kimi-K2.5", "zai-org/GLM-5", "MiniMaxAI/MiniMax-M2.5"): + plan = build_collection_case_plan(backend="trtllm", model_path=model) + assert "moe_ep" not in plan.selected_ops, model + assert "trtllm_moe_wideep" not in plan.selected_ops sglang_plan = build_collection_case_plan(backend="sglang", model_path="deepseek-ai/DeepSeek-V3") assert "moe_ep" not in sglang_plan.selected_ops diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py new file mode 100644 index 0000000000..492f9df7f1 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU-free tests for the TRT-LLM MoE worker-recycling policy. + +``collector/trtllm/collect_moe.py`` imports torch and tensorrt_llm at module +scope, so the policy helpers and the ``run_moe_torch`` wrapper are AST-extracted +and executed against fakes (same technique as ``test_collect_moe_ep.py``). +""" + +from __future__ import annotations + +import ast +import os +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit + +REPO_ROOT = Path(__file__).resolve().parents[4] +SOURCE_PATH = REPO_ROOT / "collector" / "trtllm" / "collect_moe.py" +NAMES = ("_cases_since_worker_start", "_worker_restart_policy", "_recycle_worker_if_due", "run_moe_torch") +RELEASE = "_release_trtllm_memory_buffers" +EXIT_CODE_RESTART = 10 + +CASE_ARGS = ("nvfp4", [1, 2, 4], 7168, 2048, 8, 256, 1, 8, False, "deepseek-ai/DeepSeek-V3") +CASE_KWARGS = {"distributed": "power_law", "power_law_alpha": 1.2, "perf_filename": "moe_perf.txt", "device": "cuda:3"} + + +def _load(*, allocated_mb: float = 0.0): + tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) + + def wanted(node) -> bool: + if isinstance(node, ast.FunctionDef): + return node.name in NAMES + if isinstance(node, ast.Assign): + return any(isinstance(target, ast.Name) and target.id in NAMES for target in node.targets) + return False + + selected = [node for node in tree.body if wanted(node)] + assert len(selected) == len(NAMES) + cases: list = [] + gc_calls: list = [] + namespace = { + "os": os, + "sys": sys, + "EXIT_CODE_RESTART": EXIT_CODE_RESTART, + "torch": SimpleNamespace( + device=lambda value: f"device:{value}", + cuda=SimpleNamespace( + memory_allocated=lambda device: allocated_mb * 2**20, + memory_reserved=lambda device: allocated_mb * 2**20, + ), + ), + "_print_live_cuda_tensors": lambda device: None, + "_release_trtllm_memory_buffers": lambda: 0, + "gc_collect": lambda: gc_calls.append(True), + "_run_moe_case": lambda *args, **kwargs: cases.append((args, kwargs)), + } + exec(compile(ast.Module(body=selected, type_ignores=[]), str(SOURCE_PATH), "exec"), namespace) + return namespace, cases, gc_calls + + +def _run_cases(namespace, count: int) -> None: + for _ in range(count): + namespace["run_moe_torch"](*CASE_ARGS, **CASE_KWARGS) + + +@pytest.fixture +def clean_env(monkeypatch): + for name in ("TRTLLM_MOE_RESTART_WORKER", "TRTLLM_MOE_RESTART_EVERY", "TRTLLM_MOE_RESTART_LEAK_MB"): + monkeypatch.delenv(name, raising=False) + + +def test_default_policy_recycles_after_sixteen_cases(clean_env, capsys): + namespace, cases, gc_calls = _load() + _run_cases(namespace, 15) + assert len(cases) == 15 and len(gc_calls) == 15 + with pytest.raises(SystemExit) as caught: + _run_cases(namespace, 1) + assert caught.value.code == EXIT_CODE_RESTART + assert len(cases) == 16, "the 16th case must still run (and persist) before the worker exits" + out = capsys.readouterr().out + assert "trtllm.moe: recycling worker after 16 case(s)" in out + assert "every 16" in out + assert out.count("MiB allocated, ") == 16, "one memory line per case, recycle or not" + + +def test_leak_guard_recycles_immediately_when_memory_stays_allocated(clean_env, capsys): + namespace, cases, _ = _load(allocated_mb=2048) + with pytest.raises(SystemExit) as caught: + _run_cases(namespace, 1) + assert caught.value.code == EXIT_CODE_RESTART + assert len(cases) == 1 + out = capsys.readouterr().out + assert "2048 MiB still allocated, leak guard" in out + + +def test_every_one_restores_the_legacy_per_case_recycle(clean_env, monkeypatch): + monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "1") + namespace, cases, _ = _load() + with pytest.raises(SystemExit) as caught: + _run_cases(namespace, 1) + assert caught.value.code == EXIT_CODE_RESTART and len(cases) == 1 + + +def test_legacy_switch_disables_recycling(clean_env, monkeypatch): + monkeypatch.setenv("TRTLLM_MOE_RESTART_WORKER", "0") + monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "1") + namespace, cases, _ = _load(allocated_mb=8192) + _run_cases(namespace, 40) + assert len(cases) == 40 + + +def test_zero_disables_the_count_and_the_guard(clean_env, monkeypatch): + monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "0") + monkeypatch.setenv("TRTLLM_MOE_RESTART_LEAK_MB", "0") + namespace, cases, gc_calls = _load(allocated_mb=8192) + _run_cases(namespace, 50) + assert len(cases) == 50 and len(gc_calls) == 50 + + +@pytest.mark.parametrize("name", ["TRTLLM_MOE_RESTART_EVERY", "TRTLLM_MOE_RESTART_LEAK_MB"]) +def test_negative_values_are_rejected(clean_env, monkeypatch, name): + monkeypatch.setenv(name, "-1") + namespace, _, _ = _load() + with pytest.raises(ValueError, match="must be >= 0"): + _run_cases(namespace, 1) + + +def test_wrapper_forwards_the_case_unchanged_and_recycles_after_it(clean_env, monkeypatch): + monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "2") + namespace, cases, gc_calls = _load() + _run_cases(namespace, 1) + assert cases == [(CASE_ARGS, CASE_KWARGS)] + assert gc_calls == [True], "cleanup runs after the case body returned" + + +def test_case_body_no_longer_exits_on_its_own(): + tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) + body = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_run_moe_case") + exits = [ + node + for node in ast.walk(body) + if isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "exit" + and isinstance(node.func.value, ast.Name) + and node.func.value.id == "sys" + ] + assert exits == [], "worker recycling belongs to run_moe_torch's policy, not the case body" + assert "TRTLLM_MOE_RESTART_WORKER" not in ast.get_source_segment(SOURCE_PATH.read_text(), body) + + +def test_release_drops_every_trtllm_workspace_block_including_reserved(monkeypatch): + import types + + tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) + node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == RELEASE) + namespace: dict = {} + exec(compile(ast.Module(body=[node], type_ignores=[]), str(SOURCE_PATH), "exec"), namespace) + + class _Tensor: + def __init__(self, mib): + self.mib = mib + + def numel(self): + return self.mib * 2**20 + + def element_size(self): + return 1 + + class _Block: + def __init__(self, mib, reserved): + self.buffer = _Tensor(mib) + self.is_reserved = reserved + + blocks = {"moe_workspace": [_Block(7168, True), _Block(3584, False)], "other": [_Block(28, False)]} + pool = SimpleNamespace(buffers=blocks) + utils = types.ModuleType("tensorrt_llm._torch.memory_buffer_utils") + utils.get_memory_buffers = lambda: pool + pkg = types.ModuleType("tensorrt_llm._torch") + pkg.memory_buffer_utils = utils + monkeypatch.setitem(sys.modules, "tensorrt_llm", types.ModuleType("tensorrt_llm")) + monkeypatch.setitem(sys.modules, "tensorrt_llm._torch", pkg) + monkeypatch.setitem(sys.modules, "tensorrt_llm._torch.memory_buffer_utils", utils) + assert namespace[RELEASE]() == 7168 + 3584 + 28 + assert pool.buffers == {} + + +def test_release_is_a_no_op_without_the_trtllm_module(monkeypatch): + tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) + node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == RELEASE) + namespace: dict = {} + exec(compile(ast.Module(body=[node], type_ignores=[]), str(SOURCE_PATH), "exec"), namespace) + monkeypatch.setitem(sys.modules, "tensorrt_llm", None) + assert namespace[RELEASE]() == 0 diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_mla_precision_mxfp4_padding.py b/python/aisimulate/tests/unit/collector/trtllm/test_mla_precision_mxfp4_padding.py index 01b4f66440..2f1968679e 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_mla_precision_mxfp4_padding.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_mla_precision_mxfp4_padding.py @@ -56,7 +56,7 @@ class ReachedNativeBuilder(Exception): @pytest.mark.parametrize("model_name", ["openai/gpt-oss-120b", "openai/gpt-oss-20b", "other-mxfp4-model"]) def test_unaligned_mxfp4_honors_native_padding_window(sm, tp, quant, model_name): tree = ast.parse((COLLECTOR / "collect_moe.py").read_text()) - function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "run_moe_torch") + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_run_moe_case") native_builder = MagicMock(side_effect=ReachedNativeBuilder) namespace = { "torch": MagicMock(), @@ -72,17 +72,21 @@ def test_unaligned_mxfp4_honors_native_padding_window(sm, tp, quant, model_name) "get_sm_version": lambda: sm, "_MXFP4_MOE_TYPES": {"w4a16_mxfp4", "w4a8_mxfp4_mxfp8"}, "RenormalizeMoeRoutingMethod": MagicMock(), + # Declared-routing lookup (DeepSeek-V3-family gate) is not exercised by + # these MXFP4 checkpoints; keep the softmax-top-k path under test. + "_declared_moe_routing": lambda _model_name: None, + "DeepseekV3Gate": MagicMock(), "create_moe": native_builder, "inspect": SimpleNamespace(signature=lambda _: SimpleNamespace(parameters={})), } exec(compile(ast.Module(body=[function], type_ignores=[]), "collect_moe.py", "exec"), namespace) if sm in (90, 120): with pytest.raises(ValueError, match="weight-layout alignment"): - namespace["run_moe_torch"](quant, [1], 2880, 2880, 4, 128, tp, 1, False, model_name, perf_filename="unused") + namespace["_run_moe_case"](quant, [1], 2880, 2880, 4, 128, tp, 1, False, model_name, perf_filename="unused") native_builder.assert_not_called() return with pytest.raises(ReachedNativeBuilder): - namespace["run_moe_torch"](quant, [1], 2880, 2880, 4, 128, tp, 1, False, model_name, perf_filename="unused") + namespace["_run_moe_case"](quant, [1], 2880, 2880, 4, 128, tp, 1, False, model_name, perf_filename="unused") assert native_builder.call_args.kwargs["intermediate_size"] == 2880 assert native_builder.call_args.kwargs["model_config"].mapping.moe_tp_size == tp assert native_builder.call_args.kwargs["bias"] == model_name.startswith("openai/gpt-oss-") @@ -95,7 +99,7 @@ def test_unaligned_mxfp4_honors_native_padding_window(sm, tp, quant, model_name) @pytest.mark.parametrize("outcomes", ["fail,fail", "fail,pass", "pass,fail", "pass,pass", "fatal", "cached"]) def test_moe_autotuning_requires_success_or_loaded_cache(outcomes): tree = ast.parse((COLLECTOR / "collect_moe.py").read_text()) - function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "run_moe_torch") + function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_run_moe_case") tuning = next( node for node in ast.walk(function) diff --git a/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py b/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py index 9250f6bb21..147e5a5f64 100644 --- a/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py +++ b/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py @@ -270,12 +270,14 @@ def test_glm5_next_decode_mirrors_serving_decode(): @pytest.mark.parametrize( ("version", "accepted"), [ - ("0.1.dev19262+gb6bbf29dd", True), + ("0.1.dev19262+gb6bbf29dd", False), # Kimi-K3 preview pin retired 2026-09-28 + ("0.29.0", False), + ("0.30.0", True), ("0.31.0", True), ("0.31.1", False), ], ) -def test_kda_compat_admits_kimi_preview_and_glm_runtime(version, accepted): +def test_kda_compat_admits_kimi_and_glm_runtimes(version, accepted): from collector.version_resolver import _check_compat declaration = next( @@ -327,7 +329,8 @@ def test_kda_audited_release_sets_do_not_overlap(): @pytest.mark.parametrize( ("model_name", "version", "audited"), [ - ("moonshotai/Kimi-K3", "0.1.dev19262+gb6bbf29dd", True), + ("moonshotai/Kimi-K3", "0.30.0", True), + ("moonshotai/Kimi-K3", "0.1.dev19262+gb6bbf29dd", False), ("moonshotai/Kimi-K3", "0.27.0", False), ("moonshotai/Kimi-K3", "0.31.0", False), ("zai-org/GLM-5.3-Flash", "0.31.0", True), diff --git a/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py b/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py new file mode 100644 index 0000000000..b9c37c545e --- /dev/null +++ b/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py @@ -0,0 +1,261 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""vLLM 0.30/0.31 rope q_pe alone (key=None) in the DSV3.2 dense-MHA FP8 branch +(models/deepseek_v32/attention.py:534 @0.30.0). Only DeepseekScalingRotaryEmbedding +rejects that; the collector wraps THAT class's forward so the call ropes the query +against a throwaway key that shares the query's strides and returns (query, None). +Rotary classes that accept key=None (GLM-5.x's RotaryEmbedding) stay on serving's call: +wrapping them handed vLLM's own rope kernel — one head stride for query and key — a +contiguous key with the wrong head stride, i.e. out-of-bounds in-place writes (the +GLM-5.2 fp8-KV dsa_context illegal-address / stall family, 2026-10-14).""" + +import ast +import math +from pathlib import Path + +import pytest + +pytestmark = pytest.mark.unit +SOURCE = Path(__file__).resolve().parents[4] / "collector" / "vllm" / "collect_mla_module.py" +_FUNCTIONS = ( + "_strided_throwaway_copy", + "_rope_tolerating_missing_key", + "_rope_module_rejects_missing_key", + "_install_rope_missing_key_shims", +) + + +def _load_all(): + tree = ast.parse(SOURCE.read_text(encoding="utf-8"), filename=str(SOURCE)) + fns = [n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in _FUNCTIONS] + assert {f.name for f in fns} == set(_FUNCTIONS) + ns: dict = {} + exec(compile(ast.Module(body=fns, type_ignores=[]), str(SOURCE), "exec"), ns) + return ns + + +def _load(): + return _load_all()["_rope_tolerating_missing_key"] + + +def _contiguous_strides(shape): + strides = [] + acc = 1 + for s in reversed(shape): + strides.append(acc) + acc *= s + return tuple(reversed(strides)) + + +class _Q: + """Just enough of torch.Tensor for the shim: shape/stride and the four methods it calls.""" + + def __init__(self, tag, shape=(4, 8, 64), stride=None, storage_numel=None): + self.tag = tag + self.shape = tuple(shape) + self._stride = tuple(stride) if stride is not None else _contiguous_strides(self.shape) + self.storage_numel = storage_numel if storage_numel is not None else math.prod(self.shape) + self.made_by = "fresh" + + def stride(self): + return self._stride + + def clone(self): + q = _Q(self.tag + "'", self.shape) + q.made_by = "clone" + return q + + def new_empty(self, numel): + return _Q("empty", (numel,), storage_numel=numel) + + def as_strided(self, shape, stride): + q = _Q("view", shape, stride, storage_numel=self.storage_numel) + q.made_by = "as_strided" + return q + + def copy_(self, src): + self.tag = src.tag + "'" + return self + + +def _q_pe_slice(tokens=3, heads=8, qk_head_dim=256, rope_dim=64): + # q_pe = q.split([nope, rope], dim=-1)[1]: a last-dim slice of [tokens, heads, qk_head_dim]. + return _Q("q", (tokens, heads, rope_dim), (heads * qk_head_dim, qk_head_dim, 1)) + + +def test_missing_key_is_replaced_by_a_throwaway_copy_and_dropped(): + calls = [] + + def forward(positions, query, key=None, offsets=None): + assert key is not None # the real impls' contract + calls.append((positions, query.tag, key.tag, offsets)) + return (_Q(query.tag + "+rope"), _Q(key.tag + "+rope")) + + wrapped = _load()(forward) + q, k = wrapped("pos", _Q("q")) + assert (q.tag, k) == ("q+rope", None) + assert calls == [("pos", "q", "q'", None)] + + +def test_throwaway_key_shares_the_query_strides_not_a_contiguous_clone(): + # vLLM's rope kernel addresses the key with the QUERY's head stride + # (csrc/libtorch_stable/pos_encoding_kernels.cu:57,68,164-165 @0.30.0). A contiguous clone of + # the q_pe slice (head stride 64 vs 256) is written past its end by 192*(heads-1) elements. + seen = [] + + def forward(positions, query, key=None): + seen.append(key) + return (query, key) + + query = _q_pe_slice(tokens=3, heads=8) + _load()(forward)("pos", query) + (key,) = seen + assert key.made_by == "as_strided" + assert key.shape == query.shape + assert key.stride() == query.stride() == (8 * 256, 256, 1) + # Storage covers the last addressable element of the strided layout, nothing less. + assert key.storage_numel == 1 + 2 * (8 * 256) + 7 * 256 + 63 + assert key.tag == "q'" # the data was copied from the query + + +def test_throwaway_copy_of_an_empty_query_falls_back_to_clone(): + copy = _load_all()["_strided_throwaway_copy"] + key = copy(_Q("q", (0, 8, 64), (8 * 256, 256, 1))) + assert key.made_by == "clone" + + +def test_present_key_passes_through_unchanged(): + def forward(positions, query, key=None, offsets=None): + return (query, key) + + wrapped = _load()(forward) + assert wrapped("pos", "q", "k", "off") == ("q", "k") + + +def test_shim_forwards_only_the_arguments_it_received(): + # vLLM 0.30 forward_cuda(positions, query, key=None): no offsets parameter. The first shim + # passed offsets=None positionally -> TypeError "takes from 3 to 4 positional arguments but + # 5 were given" (508 GLM-5 cases, pipelines 72387095/72387091). + def forward_cuda(positions, query, key=None): + return (query, key) + + wrapped = _load()(forward_cuda) + assert wrapped("pos", "q", "k") == ("q", "k") + q, k = wrapped("pos", _Q("q")) + assert (q.tag, k) == ("q", None) + + +def test_shim_is_idempotent(): + # Installed per case on a possibly reused module: re-wrapping stacked closures until + # RecursionError (2,286 cases per run at e598d380). + def forward(positions, query, key=None): + return (query, key) + + shim = _load() + once = shim(forward) + assert shim(once) is once and shim(shim(once)) is once + assert getattr(once, "_aisim_rope_shim", False) is True + + +# --- which rotary classes get the shim ------------------------------------------------- + + +class RotaryEmbedding: + """vLLM's base class: forward_cuda -> ops.rotary_embedding, key may be None.""" + + def forward_cuda(self, positions, query, key=None): + return (query, key) + + def forward(self, positions, query, key=None): + return self.forward_cuda(positions, query, key) + + +class Llama3RotaryEmbedding(RotaryEmbedding): + pass + + +class DeepseekScalingRotaryEmbedding(RotaryEmbedding): + """DeepSeek-V3.2's yarn rope: FlashInfer op key.view / native assert key is not None.""" + + def forward_cuda(self, positions, query, key=None): + assert key is not None, "the real impls reject key=None" + return (query, key) + + +class DeepseekV4ScalingRotaryEmbedding(DeepseekScalingRotaryEmbedding): + """Overrides forward_cuda back onto ops.rotary_embedding: accepts key=None again.""" + + def forward_cuda(self, positions, query, key=None): + return (query, key) + + +class Linear: + def forward(self, x): + return x + + +class _Attn: + def __init__(self, **modules): + self._modules = modules + + def named_modules(self): + return list(self._modules.items()) + + +def test_rejects_missing_key_only_for_deepseek_scaling_rope_itself(): + rejects = _load_all()["_rope_module_rejects_missing_key"] + assert rejects(DeepseekScalingRotaryEmbedding()) is True + assert rejects(RotaryEmbedding()) is False # GLM-5.x: rope_type "default" + assert rejects(Llama3RotaryEmbedding()) is False + assert rejects(DeepseekV4ScalingRotaryEmbedding()) is False + assert rejects(Linear()) is False + + +def test_installer_shims_deepseek_scaling_rope_and_leaves_glm_rope_on_servings_call(): + ns = _load_all() + glm_rope = RotaryEmbedding() + dsv32_rope = DeepseekScalingRotaryEmbedding() + attn = _Attn( + **{ + "rotary_emb": glm_rope, + "indexer_rope_emb": Llama3RotaryEmbedding(), + "mla_attn.rotary_emb": dsv32_rope, + "q_b_proj": Linear(), + } + ) + patched, left = ns["_install_rope_missing_key_shims"](attn) + assert patched == ["DeepseekScalingRotaryEmbedding"] + assert left == ["RotaryEmbedding", "Llama3RotaryEmbedding"] + + # GLM's rope is untouched: serving's `rotary_emb(positions, q_pe)` reaches the class's own + # forward with key=None, exactly as in vLLM. + assert "forward" not in vars(glm_rope) + assert glm_rope.forward("pos", "q") == ("q", None) + + # DeepSeek-V3.2's rope now accepts the same call and returns (query, None). + assert getattr(dsv32_rope.forward, "_aisim_rope_shim", False) is True + q, k = dsv32_rope.forward("pos", _q_pe_slice()) + assert (q.tag, k) == ("q", None) + + +def test_installer_is_idempotent_across_cases(): + ns = _load_all() + dsv32_rope = DeepseekScalingRotaryEmbedding() + attn = _Attn(rotary_emb=dsv32_rope) + ns["_install_rope_missing_key_shims"](attn) + once = dsv32_rope.forward + ns["_install_rope_missing_key_shims"](attn) + assert dsv32_rope.forward is once + + +def test_run_mla_module_installs_shims_through_the_class_filter_only(): + text = SOURCE.read_text(encoding="utf-8") + assert "_install_rope_missing_key_shims(attn_module)" in text + # The old unconditional wrap of every *RotaryEmbedding submodule must not come back. + assert "_m.forward = _rope_tolerating_missing_key(_m.forward)" not in text + + +def test_fp8_block_guard_present_before_layer_construction(): + text = (Path(__file__).resolve().parents[4] / "collector" / "vllm" / "collect_gemm.py").read_text() + guard = text.index("FIXME(kernel-limit): vLLM fp8_block GEMM needs n and k multiples of 16") + assert guard < text.index("def create_gemm():") diff --git a/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py b/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py index 62f94c2ee9..60c1c4e2bf 100644 --- a/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py +++ b/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py @@ -78,7 +78,7 @@ def test_externally_evaluated_engine_limits_are_preserved(backend: str, rule: st def test_default_vllm_rules_still_resize_unpinned_requests(): result = apply_rule_plugins(_params(preserve=False), "vllm") - assert result["params"]["prefill"]["max_num_tokens"] == 1628 + assert result["params"]["prefill"]["max_num_tokens"] == 1664 # 128 + 1500 = 1628, aligned up to 64 assert result["params"]["decode"]["max_batch_size"] == 512 assert result["params"]["decode"]["max_num_tokens"] == 512 diff --git a/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py b/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py new file mode 100644 index 0000000000..c7b1167ef7 --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Ada (sm89) hardware facts: the l40s profile, and the quantization-keyed trtllm MoE backend. + +Without the profile resolve_facts() raised "Unknown hardware profile 'l40s'" and the pipeline swallowed +it (no model default reached l40s renders). On SM89 trtllm 1.3.0rc29 serves NVFP4 MoE only through +MARLIN, which rejects bf16/fp8 MoE layers, so the profile keys the choice by quantization family +(op-probe harness L40 campaign, 2026-10-04, findings sm89_trtllm_rc29_probe_2026_10_04).""" + +import pytest + +from aisimulate.generator.facts.apply import apply_moe_backend, quant_family_of +from aisimulate.generator.facts.request_resolution import hardware_key_for_system +from aisimulate.generator.facts.resolve import _FACTS_DIR, load_backend_version_matrix, resolve_facts + +pytestmark = pytest.mark.unit + + +def _latest_dynamo() -> str: + matrix = load_backend_version_matrix(str(_FACTS_DIR / "runtimes" / "dynamo.yaml")) + return next(v for v, backends in matrix.items() if "trtllm" in backends) + + +def _l40s(): + return resolve_facts( + model_profile_id=None, hardware="l40s", transport="nvlink", dynamo_version=_latest_dynamo(), backend="trtllm" + ).hardware + + +def test_l40s_is_its_own_profile_and_no_phantom_l40_system(): + assert hardware_key_for_system("l40s") == "l40s" + assert hardware_key_for_system("l40") == "l40" # not an SDK system: falls through, resolve_facts raises clearly + hw = _l40s() + assert hw["node_selector"]["nvidia.com/gpu.product"] == "NVIDIA-L40S" + assert hw["moe_backend"] == {"trtllm": {"nvfp4": "MARLIN"}} + + +@pytest.mark.parametrize( + "quant, family", + [ + (None, None), + ({"quant_algo": "NVFP4", "kv_cache_quant_algo": "FP8"}, "nvfp4"), # modelopt hf_quant_config + ({"quant_method": "mxfp4"}, "mxfp4"), # gpt-oss + ({"quant_method": "fp8", "weight_block_size": [128, 128]}, "fp8_block"), # DeepSeek-V3 class + ({"quant_method": "fp8"}, "fp8"), + ({"quant_algo": "FP8"}, "fp8"), + # modelopt MIXED_PRECISION: the experts' algo decides (nvidia/Qwen3.8-2.4T-A95B-NVFP4 layout) + ( + { + "quant_algo": "MIXED_PRECISION", + "quant_method": "modelopt", + "quantized_layers": { + "model.layers.0.linear_attn.in_proj_qkv": {"quant_algo": "FP8"}, + "model.layers.0.linear_attn.out_proj": {"quant_algo": "FP8"}, + "model.layers.0.mlp.experts": {"quant_algo": "NVFP4"}, + }, + }, + "nvfp4", + ), + ( + { + "quant_algo": "MIXED_PRECISION", + "quant_method": "modelopt", + "quantized_layers": { + "model.layers.0.self_attn.q_proj": {"quant_algo": "FP8"}, + "model.layers.0.mlp.down_proj": {"quant_algo": "FP8"}, + }, + }, + "fp8", + ), + ], +) +def test_quant_family_of(quant, family): + assert quant_family_of(quant) == family + + +def test_l40s_fills_marlin_for_nvfp4_moe_only(): + hw = _l40s() + ctx = {"is_moe": True, "moe_config": {}} + apply_moe_backend(ctx, hw, backend="trtllm", quantization={"quant_algo": "NVFP4"}) + assert ctx["moe_config"]["backend"] == "MARLIN" + for quant in (None, {"quant_method": "fp8", "weight_block_size": [128, 128]}, {"quant_method": "mxfp4"}): + ctx = {"is_moe": True, "moe_config": {}} + apply_moe_backend(ctx, hw, backend="trtllm", quantization=quant) + assert "backend" not in ctx["moe_config"], quant # the template default (CUTLASS) applies + dense = {"is_moe": False, "moe_config": {}} + apply_moe_backend(dense, hw, backend="trtllm", quantization={"quant_algo": "NVFP4"}) + assert "backend" not in dense["moe_config"] + + +def test_user_moe_backend_param_beats_the_hardware_fill_on_trtllm(): + """`--generator-set params.agg.moe_backend=...` used to be silently dropped on trtllm: the hardware fact + filled moe_config.backend regardless and the template never read the generic param.""" + hw = {"moe_backend": {"trtllm": "CUTLASS"}} + ctx = {"is_moe": True, "moe_config": {}, "agg_moe_backend": "marlin", "moe_backend": "marlin"} + apply_moe_backend(ctx, hw, backend="trtllm") + assert "backend" not in ctx["moe_config"] + ctx = {"is_moe": True, "moe_config": {}} + apply_moe_backend(ctx, hw, backend="trtllm") + assert ctx["moe_config"]["backend"] == "CUTLASS" diff --git a/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py b/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py new file mode 100644 index 0000000000..78c2b2a1a7 --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py @@ -0,0 +1,53 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""RTX PRO (sm120) hardware facts profile. Without it resolve_facts() raised +"Unknown hardware profile 'rtx_pro_6000_server'", the pipeline swallowed the +error and every sm120 render lost its model defaults (op-probe harness, +2026-10-01: V4.1's block-size 64 never reached sm120 engine args).""" + +import pytest + +from aisimulate.generator.facts.apply import apply_model_default_args +from aisimulate.generator.facts.request_resolution import hardware_key_for_system +from aisimulate.generator.facts.resolve import _FACTS_DIR, load_backend_version_matrix, resolve_facts + +pytestmark = pytest.mark.unit + + +def _latest_dynamo() -> str: + matrix = load_backend_version_matrix(str(_FACTS_DIR / "runtimes" / "dynamo.yaml")) + return next(v for v, backends in matrix.items() if "vllm" in backends) + + +def test_rtx_pro_6000_server_maps_to_the_sm120_profile(): + assert hardware_key_for_system("rtx_pro_6000_server") == "sm120" + # unknown systems still fall through to the raw name (clear error downstream) + assert hardware_key_for_system("not_a_system") == "not_a_system" + + +def test_sm120_profile_resolves_and_carries_no_wideep_moe_backend(): + facts = resolve_facts( + model_profile_id="deepseek-v4.1", + hardware="sm120", + transport="nvlink", + dynamo_version=_latest_dynamo(), + backend="vllm", + ) + assert facts.hardware["arch"] == "x86_64" + assert facts.hardware["node_selector"]["nvidia.com/gpu.product"].startswith("NVIDIA-RTX-PRO-6000") + # PCIe Blackwell: no wide-EP MoE backend is asserted for any backend + assert "moe_backend" not in facts.hardware + + +def test_model_defaults_reach_sm120_engine_args(): + facts = resolve_facts( + model_profile_id="deepseek-v4.1", + hardware="sm120", + transport="nvlink", + dynamo_version=_latest_dynamo(), + backend="vllm", + ) + tokens: list[str] = [] + apply_model_default_args(tokens, facts.model, backend="vllm", system="sm120", role="agg", variant=None) + assert tokens[tokens.index("--block-size") + 1] == "64" + assert "--trust-remote-code" in tokens diff --git a/python/aisimulate/tests/unit/generator/test_llmd_deployment.py b/python/aisimulate/tests/unit/generator/test_llmd_deployment.py index 017781f36a..d6a511883c 100644 --- a/python/aisimulate/tests/unit/generator/test_llmd_deployment.py +++ b/python/aisimulate/tests/unit/generator/test_llmd_deployment.py @@ -174,7 +174,8 @@ def test_llmd_kustomize_vllm_disagg_mode(): assert prefill["spec"]["replicas"] == 14 assert prefill_container["resources"]["limits"]["nvidia.com/gpu"] == "1" assert "--max-num-batched-tokens" in prefill_container["args"] - assert "5500" in prefill_container["args"] + # every token budget renders 64-aligned (owner decision 2026-09-24): 5500 -> 5504 + assert "5504" in prefill_container["args"] @pytest.mark.unit diff --git a/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py b/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py new file mode 100644 index 0000000000..f800497751 --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py @@ -0,0 +1,55 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""DeepSeek-V4.1 has its own facts profile: V4's `DeepSeek-V4*` glob used to +hand it block-size 256, which vllm 0.30.0 rejects for V4.1's 64-token kernel +blocks (op-probe harness A/B, 2026-10-01).""" + +import pytest + +from aisimulate.generator.facts.apply import apply_model_default_args +from aisimulate.generator.facts.request_resolution import model_profile_for_path +from aisimulate.generator.facts.resolve import resolve_facts + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize( + "path,profile", + [ + ("deepseek-ai/DeepSeek-V4-Flash-0731", "deepseek-v4"), + ("deepseek-ai/DeepSeek-V4-Pro", "deepseek-v4"), + ("sgl-project/DeepSeek-V4-Flash-FP8", "deepseek-v4"), + ("deepseek-ai/DeepSeek-V4.1-Flash", "deepseek-v4.1"), + ("nvidia/DeepSeek-V4.1-Flash-NVFP4", "deepseek-v4.1"), + ], +) +def test_v4_and_v41_resolve_to_their_own_profiles(path, profile): + assert model_profile_for_path(path) == profile + + +def _vllm_defaults(profile_id: str) -> list[str]: + facts = resolve_facts( + model_profile_id=profile_id, + hardware="h200", + transport="nvlink", + dynamo_version=_latest_dynamo(), + backend="vllm", + ) + tokens: list[str] = [] + apply_model_default_args(tokens, facts.model, backend="vllm", system=None, role="agg", variant=None) + return tokens + + +def _latest_dynamo() -> str: + from aisimulate.generator.facts.resolve import _FACTS_DIR, load_backend_version_matrix + + matrix = load_backend_version_matrix(str(_FACTS_DIR / "runtimes" / "dynamo.yaml")) + return next(v for v, backends in matrix.items() if "vllm" in backends) + + +def test_v41_vllm_block_size_is_64_not_v4s_256(): + v41 = _vllm_defaults("deepseek-v4.1") + v4 = _vllm_defaults("deepseek-v4") + assert v41[v41.index("--block-size") + 1] == "64" + assert v4[v4.index("--block-size") + 1] == "256" + assert "--trust-remote-code" in v41 and "--no-enable-flashinfer-autotune" in v41 diff --git a/python/aisimulate/tests/unit/generator/test_model_profile_minimax_m3.py b/python/aisimulate/tests/unit/generator/test_model_profile_minimax_m3.py new file mode 100644 index 0000000000..afd734b6df --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_model_profile_minimax_m3.py @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""MiniMax-M3 model facts: vllm needs block-size 128 because the sparse layers' backend accepts only that KV page +size while the dense head's FlashAttention accepts MultipleOf(16) (op-probe harness, L40, 2026-10-05).""" + +import pytest + +from aisimulate.generator.facts.apply import apply_model_default_args +from aisimulate.generator.facts.request_resolution import model_profile_for_path +from aisimulate.generator.facts.resolve import _FACTS_DIR, load_backend_version_matrix, resolve_facts + +pytestmark = pytest.mark.unit + + +def _latest_dynamo(backend: str) -> str: + matrix = load_backend_version_matrix(str(_FACTS_DIR / "runtimes" / "dynamo.yaml")) + return next(v for v, backends in matrix.items() if backend in backends) + + +@pytest.mark.parametrize("path", ["MiniMaxAI/MiniMax-M3", "MiniMaxAI/MiniMax-M3-MXFP8", "nvidia/MiniMax-M3-NVFP4"]) +def test_minimax_m3_paths_resolve_to_the_profile(path): + assert model_profile_for_path(path) == "minimax-m3" + + +def test_minimax_m3_vllm_default_is_block_size_128_only(): + def tokens_for(backend: str) -> list[str]: + facts = resolve_facts( + model_profile_id="minimax-m3", + hardware="h200", + transport="nvlink", + dynamo_version=_latest_dynamo(backend), + backend=backend, + ) + out: list[str] = [] + apply_model_default_args(out, facts.model, backend=backend, system=None, role="agg", variant=None) + return out + + assert tokens_for("vllm") == ["--block-size", "128"] + # sglang / trtllm carry no M3 default from this profile + assert tokens_for("sglang") == [] + assert tokens_for("trtllm") == [] diff --git a/python/aisimulate/tests/unit/generator/test_sglang_cuda_graph_flags_0520.py b/python/aisimulate/tests/unit/generator/test_sglang_cuda_graph_flags_0520.py new file mode 100644 index 0000000000..beb4332fdc --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_sglang_cuda_graph_flags_0520.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""SGLang >= 0.5.20 renamed `--cuda-graph-bs` / `--cuda-graph-max-bs` to the +`*-decode` spellings (ServerArgs is a msgspec Struct; the old aliases are gone, +0.5.21 rejects `--cuda-graph-bs` as ambiguous). cli_args.0.5.20.j2 renders the +new spelling; older versions keep the old one (op-probe harness, 2026-10-01).""" + +import copy + +import pytest +import yaml + +from aisimulate.generator.api import generate_backend_artifacts + +pytestmark = pytest.mark.unit + +_PARAMS = { + "ServiceConfig": { + "model_path": "deepseek-ai/DeepSeek-V3", + "served_model_path": "deepseek-ai/DeepSeek-V3", + "served_model_name": "DeepSeek-V3", + "include_frontend": True, + }, + "K8sConfig": {"name_prefix": "test", "k8s_namespace": "default"}, + "DynConfig": {"mode": "agg"}, + "WorkerConfig": {"agg_workers": 1, "agg_gpus_per_worker": 8, "prefill_workers": 0, "decode_workers": 0}, + "NodeConfig": {"num_gpus_per_node": 8}, + "SlaConfig": {"isl": 1024, "osl": 256}, + "ModelConfig": {"is_moe": True, "prefix": 0, "nextn": 0}, + "BenchConfig": {}, + "params": { + "agg": { + "tensor_parallel_size": 8, + "pipeline_parallel_size": 1, + "data_parallel_size": 1, + "moe_tensor_parallel_size": 1, + "moe_expert_parallel_size": 8, + "max_batch_size": 64, + "max_num_tokens": 4096, + "max_seq_len": 4096, + } + }, +} + + +def _worker_args(version: str) -> str: + artifacts = generate_backend_artifacts( + copy.deepcopy(_PARAMS), "sglang", backend_version=version, deployment_target="dynamo-j2" + ) + k8s = yaml.safe_load(artifacts["k8s_deploy.yaml"]) + worker = next(svc for name, svc in k8s["spec"]["services"].items() if name != "Frontend") + return " ".join(worker["extraPodSpec"]["mainContainer"]["args"]) + + +@pytest.mark.parametrize("version", ["0.5.20", "0.5.21"]) +def test_new_versions_render_decode_spelling(version): + args = _worker_args(version) + assert "--cuda-graph-bs-decode " in args + assert "--cuda-graph-bs " not in args + assert "--cuda-graph-max-bs " not in args + + +@pytest.mark.parametrize("version", ["0.5.16", "0.5.19"]) +def test_old_versions_keep_old_spelling(version): + args = _worker_args(version) + assert "--cuda-graph-bs " in args + assert "--cuda-graph-bs-decode" not in args + + +def test_prefill_graph_backend_param_renders_only_on_new_versions(): + """params.agg.cuda_graph_backend_prefill -> --cuda-graph-backend-prefill (sglang >= 0.5.20 template).""" + params = copy.deepcopy(_PARAMS) + params["params"]["agg"]["cuda_graph_backend_prefill"] = "breakable" + new = generate_backend_artifacts(params, "sglang", backend_version="0.5.21", deployment_target="dynamo-j2") + k8s = yaml.safe_load(new["k8s_deploy.yaml"]) + worker = next(s for n, s in k8s["spec"]["services"].items() if n != "Frontend") + assert '--cuda-graph-backend-prefill "breakable"' in " ".join(worker["extraPodSpec"]["mainContainer"]["args"]) + old = generate_backend_artifacts( + copy.deepcopy(params), "sglang", backend_version="0.5.16", deployment_target="dynamo-j2" + ) + k8s = yaml.safe_load(old["k8s_deploy.yaml"]) + worker = next(s for n, s in k8s["spec"]["services"].items() if n != "Frontend") + assert "--cuda-graph-backend-prefill" not in " ".join(worker["extraPodSpec"]["mainContainer"]["args"]) diff --git a/python/aisimulate/tests/unit/generator/test_trtllm_msa_implementation.py b/python/aisimulate/tests/unit/generator/test_trtllm_msa_implementation.py index bb2ef369c5..2b73242847 100644 --- a/python/aisimulate/tests/unit/generator/test_trtllm_msa_implementation.py +++ b/python/aisimulate/tests/unit/generator/test_trtllm_msa_implementation.py @@ -128,3 +128,60 @@ def test_naive_generator_carries_the_msa_prescription(): mode="agg", ) assert "msa_sparse_implementation" not in off_family["ModelConfig"] + + +# --- rc29: sparse_attention_config.algorithm (required on every SM) --------------------- +_RC29 = "1.3.0rc29" + + +def test_rc29_template_renders_algorithm_for_m3_without_implementation(): + """SM90 M3 (no fmha_sm100 prescription) still needs algorithm=minimax_m3 at rc29.""" + artifacts = generate_backend_artifacts( + backend="trtllm", backend_version=_RC29, params=_params({"msa_sparse_algorithm": "minimax_m3"}) + ) + assert _engine_yaml(artifacts).get("sparse_attention_config") == {"algorithm": "minimax_m3"} + + +def test_rc29_template_renders_both_keys_on_sm100(): + artifacts = generate_backend_artifacts( + backend="trtllm", + backend_version=_RC29, + params=_params({"msa_sparse_algorithm": "minimax_m3", "msa_sparse_implementation": "msa"}), + ) + assert _engine_yaml(artifacts).get("sparse_attention_config") == { + "algorithm": "minimax_m3", + "implementation": "msa", + } + + +def test_rc23_template_ignores_the_algorithm_key(): + """Prior template untouched: rc23 never read `algorithm`, so it must not appear.""" + artifacts = generate_backend_artifacts( + backend="trtllm", backend_version=_BACKEND_VERSION, params=_params({"msa_sparse_algorithm": "minimax_m3"}) + ) + assert "sparse_attention_config" not in _engine_yaml(artifacts) + + +@pytest.mark.parametrize( + ("backend", "model", "expected"), + [ + ("trtllm", "MiniMaxAI/MiniMax-M3", "minimax_m3"), + ("trtllm", "nvidia/MiniMax-M3-NVFP4", "minimax_m3"), + ("trtllm", "deepseek-ai/DeepSeek-V3", None), + ("sglang", "MiniMaxAI/MiniMax-M3", None), + ], +) +def test_algorithm_prescription_is_architecture_keyed_and_trtllm_only(backend, model, expected): + from aisimulate.generator.module_bridge import _msa_sparse_algorithm + + assert _msa_sparse_algorithm(_FakeTask(backend, model, "h200_sxm")) == expected + + +def test_naive_generator_carries_the_algorithm_on_every_system(): + from aisimulate.generator.naive import build_naive_generator_params + + for system in ("b200_sxm", "h200_sxm"): + params = build_naive_generator_params( + model_name="MiniMaxAI/MiniMax-M3", total_gpus=8, system_name=system, backend_name="trtllm", mode="agg" + ) + assert params["ModelConfig"]["msa_sparse_algorithm"] == "minimax_m3" diff --git a/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py b/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py new file mode 100644 index 0000000000..f277654b91 --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""KDA (Kimi Delta Attention) models on vLLM: the decode batch is capped at 256. + +vLLM 0.29.0 / SM90 faults at the replay of the 512-batch decode CUDA graph +for Kimi-K3 (64/256/384/448/480/496 pass, 512 fails, graphs off at 512 +passes); owner decision 2026-09-25: cap max_batch_size at 256 for models +declaring linear_attn_config.kda_layers. The fact enters the rules as +ModelConfig.has_kda from both render paths (module_bridge, naive). +""" + +import pytest + +from aisimulate.generator.rendering.rule_engine import apply_rule_plugins +from aisimulate.generator.utils import model_has_kda + +pytestmark = pytest.mark.unit + + +def _pv(has_kda, batch=512, preserve=False): + return { + "SlaConfig": {"isl": 4000, "osl": 500}, + "DynConfig": {"mode": "agg"}, + "ModelConfig": {"is_moe": True, "has_kda": has_kda}, + "params": { + "agg": { + "max_batch_size": batch, + "tokens_per_block": 32, + **({"preserve_engine_limits": True} if preserve else {}), + } + }, + } + + +def test_kda_model_decode_batch_capped_at_256_and_budgets_follow(): + pv = _pv(True) + apply_rule_plugins(pv, backend="vllm") + agg = pv["params"]["agg"] + assert agg["max_batch_size"] == 256 + # token budgets derive from the CAPPED batch (256 + 4000 + 1500 = 5756 -> 5760) + assert agg["max_num_tokens"] == 5760 + + +def test_non_kda_model_keeps_the_512_floor(): + pv = _pv(False, batch=256) + apply_rule_plugins(pv, backend="vllm") + assert pv["params"]["agg"]["max_batch_size"] == 512 # the pre-existing floor, untouched + pv = _pv(None, batch=256) # fact absent (older callers): no cap + apply_rule_plugins(pv, backend="vllm") + assert pv["params"]["agg"]["max_batch_size"] == 512 + + +def test_small_requested_batch_is_not_raised_by_the_cap(): + pv = _pv(True, batch=64) + apply_rule_plugins(pv, backend="vllm") + # the 512 floor lifts 64 -> 512, the KDA cap brings it to 256 (never above the cap) + assert pv["params"]["agg"]["max_batch_size"] == 256 + + +def test_preserve_engine_limits_disables_the_cap(): + pv = _pv(True, batch=512, preserve=True) + apply_rule_plugins(pv, backend="vllm") + assert pv["params"]["agg"]["max_batch_size"] == 512 + + +def test_model_has_kda_reads_the_bundled_config(): + assert model_has_kda("moonshotai/Kimi-K3") is True + assert model_has_kda("Qwen/Qwen3-8B") is False + assert model_has_kda("org/does-not-exist-anywhere") is False diff --git a/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py b/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py new file mode 100644 index 0000000000..a6fcc9723d --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py @@ -0,0 +1,117 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""NVFP4 DSA checkpoints on vLLM render an explicit fp8 kv-cache dtype. + +vLLM 0.29.0 resolves `--kv-cache-dtype auto` for the modelopt NVFP4 DSA +artifacts (hf_quant kv_cache_quant_algo FP8) to the literal fp8_e4m3 and its +sparse-MLA selector rejects that spelling; the same engine with an explicit +`fp8` loads FlashMLASparseImpl. The prescription is keyed on the checkpoint +architecture and the task's nvfp4 GEMM quant mode — never on model names. +""" + +from __future__ import annotations + +import pytest + +from aisimulate.generator import utils +from aisimulate.generator.rendering.engine import render_backend_parameters + +pytestmark = pytest.mark.unit + +_ARCH = { + "nvidia/GLM-5.3-NVFP4": "GlmMoeDsaForCausalLM", + "nvidia/GLM-5.2-NVFP4": "GlmMoeDsaForCausalLM", + "nvidia/DeepSeek-V3.2-NVFP4": "DeepseekV32ForCausalLM", + "zai-org/GLM-5.3": "GlmMoeDsaForCausalLM", + "nvidia/Llama-3.3-70B-Instruct-FP4": "LlamaForCausalLM", + "nvidia/DeepSeek-R1-0528-FP4": "DeepseekV3ForCausalLM", + "local/unbundled": None, +} + + +_MODELOPT_FP8_KV = { + "quant_method": "modelopt", + "quant_algo": "NVFP4", + "kv_cache_scheme": {"dynamic": False, "num_bits": 8, "type": "float"}, +} +# what utils._bundled_quantization returns: config.json quantization block +# merged with the SDK-attached hf_quant_config (kv_cache_quant_algo lives there) +_QUANT = { + "nvidia/GLM-5.2-NVFP4": _MODELOPT_FP8_KV, # config.json carries the KV scheme + "nvidia/GLM-5.3-NVFP4": { + "quant_method": "modelopt", + "quant_algo": "MIXED_PRECISION", + "kv_cache_quant_algo": "FP8", + }, # hf_quant only + "nvidia/DeepSeek-V3.2-NVFP4": { + "quant_method": "nvfp4", + "quant_algo": "NVFP4", + "kv_cache_quant_algo": "FP8", + }, # hf_quant only + "nvidia/GLM-5-NVFP4-unquant-bundle": {"quant_method": "nvfp4"}, # neither spelling present + "zai-org/GLM-5.3": {"quant_method": "fp8", "fmt": "e4m3", "weight_block_size": [128, 128]}, + "nvidia/Llama-3.3-70B-Instruct-FP4": _MODELOPT_FP8_KV, +} +_ARCH["nvidia/GLM-5-NVFP4-unquant-bundle"] = "GlmMoeDsaForCausalLM" + + +@pytest.fixture(autouse=True) +def _stub_sdk_lookups(monkeypatch): + # the helpers take an optional frozen config (naive path); the stubs ignore it + monkeypatch.setattr(utils, "_model_architecture", lambda model_path, model_config=None: _ARCH.get(model_path)) + monkeypatch.setattr(utils, "_bundled_quantization", lambda model_path, raw=None: _QUANT.get(model_path)) + + +@pytest.mark.parametrize( + ("backend", "model", "gemm", "expected"), + [ + # optimized path: the task's nvfp4 GEMM quant mode is the artifact fact + ("vllm", "nvidia/GLM-5.3-NVFP4", "nvfp4", "fp8"), + ("vllm", "nvidia/DeepSeek-V3.2-NVFP4", "nvfp4", "fp8"), + # naive path (no quant mode): the artifact's fp8 KV fact, either spelling + ("vllm", "nvidia/GLM-5.2-NVFP4", None, "fp8"), # config.json kv_cache_scheme + ("vllm", "nvidia/GLM-5.3-NVFP4", None, "fp8"), # hf_quant kv_cache_quant_algo + ("vllm", "nvidia/DeepSeek-V3.2-NVFP4", None, "fp8"), + ("vllm", "nvidia/GLM-5-NVFP4-unquant-bundle", None, None), # no KV fact anywhere: no prescription + ("vllm", "zai-org/GLM-5.3", "fp8_block", None), # fp8 DSA artifact: auto is fine + ("vllm", "zai-org/GLM-5.3", None, None), + ("vllm", "nvidia/Llama-3.3-70B-Instruct-FP4", "nvfp4", None), # NVFP4 but not DSA + ("vllm", "nvidia/Llama-3.3-70B-Instruct-FP4", None, None), + ("vllm", "nvidia/DeepSeek-R1-0528-FP4", "nvfp4", None), # plain MLA, not DSA + ("vllm", "local/unbundled", "nvfp4", None), # unresolvable config: no prescription + ("sglang", "nvidia/GLM-5.2-NVFP4", "nvfp4", None), # vllm-only spelling gap + ("trtllm", "nvidia/GLM-5.2-NVFP4", None, None), + ], +) +def test_prescription_keyed_on_dsa_architecture_and_an_artifact_fact(backend, model, gemm, expected): + assert utils.vllm_dsa_kv_cache_dtype(backend, model, gemm) == expected + + +def test_artifact_kv_scheme_detection(): + assert utils._artifact_pins_fp8_kv(_MODELOPT_FP8_KV) + assert utils._artifact_pins_fp8_kv({"kv_cache_quant_algo": "FP8"}) + assert not utils._artifact_pins_fp8_kv({"quant_method": "fp8", "fmt": "e4m3"}) + assert not utils._artifact_pins_fp8_kv(None) + + +class _FakeTask: + def __init__(self, backend, model): + self.primary_backend_name = backend + self.primary_model_path = model + + +def test_bridge_wrapper_passes_task_facts_through(): + # module_bridge imports the SDK (native runtime); skip where it is not built + bridge = pytest.importorskip("aisimulate.generator.module_bridge", exc_type=ImportError) + _vllm_dsa_kv_cache_dtype = bridge._vllm_dsa_kv_cache_dtype + + assert _vllm_dsa_kv_cache_dtype(_FakeTask("vllm", "nvidia/GLM-5.3-NVFP4"), "nvfp4") == "fp8" + assert _vllm_dsa_kv_cache_dtype(_FakeTask("vllm", "zai-org/GLM-5.3"), "fp8_block") is None + + +def test_fp8_kv_dtype_renders_as_the_explicit_vllm_flag(): + out = render_backend_parameters({"kv_cache_dtype": "fp8"}, "vllm") + assert out["kv_cache_dtype"]["kv-cache-dtype"] == "fp8" + # and BF16 renders literally: vLLM >= 0.11 accepts it (backend_config_mapping.yaml guard, #487) + out = render_backend_parameters({"kv_cache_dtype": "bfloat16"}, "vllm") + assert out["kv_cache_dtype"]["kv-cache-dtype"] == "bfloat16" diff --git a/python/aisimulate/tests/unit/generator/test_vllm_token_budget_alignment.py b/python/aisimulate/tests/unit/generator/test_vllm_token_budget_alignment.py new file mode 100644 index 0000000000..a28a4d5899 --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_vllm_token_budget_alignment.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Every rendered token budget is a multiple of 64, on every backend. + +Engines size per-cache-group slot-mapping rows and scheduler chunks by these +budgets; a non-idiomatic width misaligns every row after the first (vLLM +0.29 DeepSeek-V4: 6012 crashes the cutedsl compress kernel at first forward, +6016 passes). The alignment lives in ONE place — backend_config_mapping.yaml — +so rule formulas stay plain and no backend is a special case (owner decision +2026-09-24). +""" + +import pytest + +from aisimulate.generator.rendering.engine import render_backend_parameters + +pytestmark = pytest.mark.unit + + +def _render(params, backend): + return render_backend_parameters(params, backend) + + +@pytest.mark.parametrize( + ("backend", "param", "key"), + [ + ("vllm", "max_num_tokens", "max-num-batched-tokens"), + ("trtllm", "max_num_tokens", "max_num_tokens"), + ("sglang", "max_prefill_tokens", "max-prefill-tokens"), + ], +) +def test_odd_budget_rounds_up_to_64_on_every_backend(backend, param, key): + assert _render({param: 6012}, backend)[param][key] == 6016 + assert _render({param: 6017}, backend)[param][key] == 6080 # up, never down + assert _render({param: 6016}, backend)[param][key] == 6016 + assert _render({param: 2048}, backend)[param][key] == 2048 + + +def test_sglang_chunked_prefill_size_follows_the_aligned_budget(): + out = _render({"enable_chunked_prefill": True, "max_num_tokens": 6012}, "sglang") + assert out["enable_chunked_prefill"]["chunked-prefill-size"] == 6016 + + +def test_absent_budget_stays_omitted(): + for backend in ("vllm", "trtllm"): + assert "max_num_tokens" not in _render({}, backend) + assert "max_prefill_tokens" not in _render({}, "sglang") + + +@pytest.mark.parametrize("backend", ["vllm", "sglang", "trtllm"]) +def test_rule_engine_aligns_every_budget_after_the_plain_formulas(backend): + """The rule formulas stay plain (vllm agg: 512+4000+1500 = 6012, the DSV4 + crash trigger); the rule engine's single alignment pass rounds every + budget key in every role up to 64 — including trtllm, whose engine-yaml + template bypasses the mapping transforms.""" + from aisimulate.generator.rendering.rule_engine import apply_rule_plugins + + pv = { + "SlaConfig": {"isl": 4000, "osl": 500}, + "DynConfig": {}, + "params": {"agg": {"max_batch_size": 512, "tokens_per_block": 32}}, + } + apply_rule_plugins(pv, backend=backend) + for role, values in pv["params"].items(): + for key in ("max_num_tokens", "max_prefill_tokens"): + if isinstance(values.get(key), int): + assert values[key] % 64 == 0, (backend, role, key, values[key]) + agg = pv["params"]["agg"] + budget_key = "max_prefill_tokens" if backend == "sglang" else "max_num_tokens" + assert agg[budget_key] >= 4000 + 500 # never rounded down (formulas add isl + 500/1500) + + +def test_alignment_pass_applies_without_a_rule_file(): + from aisimulate.generator.rendering.rule_engine import apply_rule_plugins + + pv = {"params": {"agg": {"max_num_tokens": 6012, "max_prefill_tokens": 5500}}} + apply_rule_plugins(pv, backend="no-such-backend") + assert pv["params"]["agg"] == {"max_num_tokens": 6016, "max_prefill_tokens": 5504} diff --git a/python/aisimulate/tests/unit/sdk/database/test_dsv4_sparse.py b/python/aisimulate/tests/unit/sdk/database/test_dsv4_sparse.py index 07041562d6..887d20f8cc 100644 --- a/python/aisimulate/tests/unit/sdk/database/test_dsv4_sparse.py +++ b/python/aisimulate/tests/unit/sdk/database/test_dsv4_sparse.py @@ -5,7 +5,7 @@ Covers: * the per-(attn_kind, mode) module loaders (test-only parsers) - * the sparse-kernel CSV loader (paged_mqa_logits / hca_attn) + * the sparse-kernel CSV loader (paged_mqa_logits) * ``_lookup_sparse_kernel`` (exact + engine resolve + tp fallback) """ @@ -128,14 +128,12 @@ def test_dsv4_test_cases_skipped_under_other_model(monkeypatch): from collector.case_generator import ( get_dsv4_csa_context_test_cases, get_dsv4_csa_generation_test_cases, - get_dsv4_hca_attn_test_cases, get_dsv4_paged_mqa_logits_test_cases, ) assert get_dsv4_csa_context_test_cases() == [] assert get_dsv4_csa_generation_test_cases() == [] assert get_dsv4_paged_mqa_logits_test_cases() == [] - assert get_dsv4_hca_attn_test_cases() == [] @pytest.mark.parametrize( @@ -169,17 +167,11 @@ def test_dsv4_sparse_test_cases_emit_one_kernel_case_per_model(monkeypatch, mode TP is no longer a case axis — the worker fixes tp=1 internally because the kernel is TP-invariant.""" monkeypatch.setenv("COLLECTOR_MODEL_PATH", model_path) - from collector.case_generator import ( - get_dsv4_hca_attn_test_cases, - get_dsv4_paged_mqa_logits_test_cases, - ) + from collector.case_generator import get_dsv4_paged_mqa_logits_test_cases paged = get_dsv4_paged_mqa_logits_test_cases() - hca = get_dsv4_hca_attn_test_cases() assert {c[1] for c in paged} == {"paged_mqa_logits"} - assert {c[1] for c in hca} == {"hca_attn"} assert {c[0] for c in paged} == {model_path} - assert {c[0] for c in hca} == {model_path} # ─────────────────────────────────────────────────────────────────────── diff --git a/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py b/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py index d508e3d4f4..293c643586 100644 --- a/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py +++ b/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py @@ -558,6 +558,8 @@ def test_packaged_targets_only_checkpoints_are_reviewed(): _PACKAGED: "DEEPSEEKV32", "nvidia/MiniMax-M3-NVFP4": "MINIMAXM3", "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4": "NEMOTRONH", + # QWEN35 family: outside the changed DEEPSEEK/KIMIK25/DEEPSEEKV32 resolvers. + "nvidia/Qwen3.8-2.4T-A95B-NVFP4": "QWEN35", "nvidia/Qwen3.6-27B-NVFP4": "QWEN35", "nvidia/Qwen3.6-35B-A3B-NVFP4": "QWEN35", } diff --git a/python/aisimulate/tests/unit/sdk/test_utils.py b/python/aisimulate/tests/unit/sdk/test_utils.py index f2a721ac27..e53edd1591 100644 --- a/python/aisimulate/tests/unit/sdk/test_utils.py +++ b/python/aisimulate/tests/unit/sdk/test_utils.py @@ -497,6 +497,51 @@ def test_parse_qwen35_layer_types_length_mismatch_raises(self): with pytest.raises(ValueError, match="layer_types length"): _parse_hf_config_json(config) + def test_parse_qwen35_dense_causal_lm_flat_config(self): + """Text-only dense Qwen3.5 checkpoints (Altworld/Hemmingway-1, + TokenRhythm/NeoHorse-1-9B) ship Qwen3_5ForCausalLM with a FLAT config + and no vision_config; they must build the same Qwen35Config the VLM + class does (shape values from Altworld/Hemmingway-1).""" + layer_types = (["linear_attention"] * 3 + ["full_attention"]) * 16 # 64 layers + config = { + "architectures": ["Qwen3_5ForCausalLM"], + "hidden_size": 5120, + "num_hidden_layers": 64, + "layer_types": layer_types, + "num_attention_heads": 24, + "num_key_value_heads": 4, + "head_dim": 256, + "intermediate_size": 17408, + "linear_num_key_heads": 16, + "linear_key_head_dim": 128, + "linear_num_value_heads": 48, + "linear_value_head_dim": 128, + "linear_conv_kernel_dim": 4, + "full_attention_interval": 4, + "vocab_size": 248320, + "max_position_embeddings": 262144, + "rms_norm_eps": 1e-6, + "hidden_act": "silu", + "dtype": "bfloat16", + "tie_word_embeddings": False, + } + + result = _parse_hf_config_json(config) + + assert result["architecture"] == "Qwen3_5ForCausalLM" + assert result["layers"] == 64 + assert result["hidden_size"] == 5120 + assert result["n"] == 24 + assert result["n_kv"] == 4 + assert result["d"] == 256 + extra_params = result["extra_params"] + assert isinstance(extra_params, common.Qwen35Config) + assert extra_params.layer_types.count("linear_attention") == 48 + assert extra_params.layer_types.count("full_attention") == 16 + assert extra_params.linear_num_value_heads == 48 + assert extra_params.num_experts == 0 + assert extra_params.vision_config is None + def test_parse_qwen38_max_flat_config(self): """Test parsing Qwen3.8-Max (Qwen/Qwen3.8-2.4T-A95B) → Qwen35Config from a FLAT config. diff --git a/python/aisimulate/tests/unit/test_support_config_profile.py b/python/aisimulate/tests/unit/test_support_config_profile.py index 97cec392bb..ea368c5461 100644 --- a/python/aisimulate/tests/unit/test_support_config_profile.py +++ b/python/aisimulate/tests/unit/test_support_config_profile.py @@ -135,6 +135,7 @@ def test_glm53_flash_config_is_runtime_resolved_not_rejected(): "nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16_config.json", "nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8_config.json", "nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4_config.json", + "nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json", "nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4_config.json", "nvidia--Nemotron-H-56B-Base-8K_config.json", } @@ -216,6 +217,7 @@ def test_packaged_unrecognized_architectures_are_pending_and_mla_keeps_explicit_ assert unloadable == { "nvidia--DeepSeek-V4-Flash-NVFP4", "nvidia--DeepSeek-V4-Pro-NVFP4", + "nvidia--DeepSeek-V4.1-Flash-NVFP4", "nvidia--Gemma-4-31B-IT-NVFP4", } assert unrecognized == {name: [f"unrecognized architecture {arch}"] for name, arch in _UNRECOGNIZED.items()} diff --git a/python/aisimulate/tests/unit/tools/test_collect_campaign.py b/python/aisimulate/tests/unit/tools/test_collect_campaign.py new file mode 100644 index 0000000000..5e66800bcd --- /dev/null +++ b/python/aisimulate/tests/unit/tools/test_collect_campaign.py @@ -0,0 +1,81 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""tools/perf_database/collect_campaign.py — the invariants a hand-rolled launcher drifted on (sm120, 2026-09-30).""" + +import importlib.util +import sys +from pathlib import Path + +import pytest +import yaml + +pytestmark = pytest.mark.unit + +_TOOL = Path(__file__).resolve().parents[3] / "tools" / "perf_database" / "collect_campaign.py" + + +def _load(): + spec = importlib.util.spec_from_file_location("collect_campaign", _TOOL) + mod = importlib.util.module_from_spec(spec) + sys.modules["collect_campaign"] = mod + spec.loader.exec_module(mod) + return mod + + +def _plan(tmp_path, shards): + p = tmp_path / "plan.yaml" + p.write_text(yaml.safe_dump({"backend": "vllm", "sm": 120, "shards": shards})) + return p + + +def test_collector_argv_always_resumes_and_never_keeps_csv(tmp_path): + c = _load() + plan = c.load_plan(_plan(tmp_path, [{"op": "gemm", "name": "bf16", "case_filter": "['bfloat16', "}, {"op": "kda"}])) + for shard in plan.shards: + argv = c.collector_argv(plan, shard) + assert "--resume" in argv and "--keep-csv" not in argv + assert argv[argv.index("--sm") + 1] == "120" + assert argv[argv.index("--ops") + 1] == shard.op + assert "--case-filter" in c.collector_argv(plan, plan.shards[0]) + assert "--case-filter" not in c.collector_argv(plan, plan.shards[1]) + assert plan.shards[1].name == "all" + + +def test_one_namespace_per_shard_and_readonly_checkout(tmp_path): + c = _load() + plan = c.load_plan(_plan(tmp_path, [{"op": "gemm", "name": "bf16"}, {"op": "gemm", "name": "fp8"}])) + dirs = {c.shard_dir(tmp_path, s) for s in plan.shards} + assert len(dirs) == 2 + argv = c.docker_argv( + plan, plan.shards[0], image="img", gpu=3, checkout=Path("/ck"), shard_dir=dirs.pop(), container_name="x" + ) + assert '"device=3"' in argv and "/ck:/ais:ro" in argv and argv[argv.index("-w") + 1] == "/out" + + +def test_duplicate_shard_keys_are_rejected(tmp_path): + c = _load() + with pytest.raises(ValueError): + c.load_plan(_plan(tmp_path, [{"op": "gemm", "name": "a"}, {"op": "gemm", "name": "a"}])) + + +def test_image_is_the_manifest_pin_plus_finalize_deps(): + c = _load() + base = c.pinned_image("vllm", "gemm") + assert base.startswith("vllm/vllm-openai:") # framework_manifest default pin, never a hand-typed tag + df = c.dockerfile(base) + assert df.startswith(f"FROM {base}\n") and "pyarrow" in df and "pandas" in df + assert c.collect_image_tag(base).startswith("aisim-collect:") + + +def test_shard_state_reads_the_namespace(tmp_path): + c = _load() + plan = c.load_plan(_plan(tmp_path, [{"op": "gemm", "name": "bf16"}])) + s = plan.shards[0] + assert c.shard_state(tmp_path, s) == "pending" + d = c.shard_dir(tmp_path, s) + (d / ".ckpt").mkdir(parents=True) + assert c.shard_state(tmp_path, s) == "started" + (d / "DONE").touch() + assert c.shard_state(tmp_path, s) == "done-no-parquet" # the --keep-csv failure mode, picked up by `finalize` + (d / "gemm_perf.parquet").write_bytes(b"") + assert c.shard_state(tmp_path, s) == "finalized" diff --git a/python/aisimulate/tools/perf_database/collect_campaign.py b/python/aisimulate/tools/perf_database/collect_campaign.py new file mode 100644 index 0000000000..fde9087df1 --- /dev/null +++ b/python/aisimulate/tools/perf_database/collect_campaign.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Run a perf-data collection campaign as declared shards, one container per shard. + +Why this exists (sm120 campaign, 2026-09-30): every box so far re-invented the +same launcher by hand — an image with the finalize deps added, ``--case-filter`` +shards, one cwd per shard so staging CSVs never collide, a GPU pool, and the +finalize step. Each re-invention drifted somewhere (the sm120 run passed +``--keep-csv`` and silently produced no parquet/provenance for 13 shards; the +official image lacks pyarrow so the smoke run died at finalize). This tool fixes +the invariants once: + +* the image is the framework_manifest pin for the (framework, op) being run — + never a hand-typed tag — plus the finalize deps baked on top (``build-image``); +* one output/checkpoint namespace per shard (``///``), exactly + as the collector-upgrade playbook §5 asks; +* ``collect.py`` is always invoked with ``--resume`` and never ``--keep-csv``, + so a finished shard always ends with parquet + collection_meta.yaml; +* every container launch is appended to ``/campaign.jsonl`` (image digest, + argv, exit code, parquet outputs) — the work log the playbook §4 wants. + +Shard plan (YAML):: + + backend: vllm + sm: 120 + shards: + - {op: gemm, name: bf16, case_filter: "['bfloat16', "} + - {op: gemm, name: fp8, case_filter: "['fp8', "} + - {op: kda, name: all} + +Usage:: + + python3 tools/perf_database/collect_campaign.py build-image --backend vllm --op gemm + python3 tools/perf_database/collect_campaign.py run --plan shards.yaml --root /ws/collect --gpus 0,1,2,3 + python3 tools/perf_database/collect_campaign.py status --plan shards.yaml --root /ws/collect + python3 tools/perf_database/collect_campaign.py finalize --plan shards.yaml --root /ws/collect --gpus 0 + +``run`` is re-entrant: shards with a ``DONE`` marker are skipped, unfinished +shards resume from their checkpoint. ``finalize`` re-enters finished shards that +have no parquet (e.g. runs made by hand with ``--keep-csv``) — every task is +already checkpointed, so collect.py only finalizes. Merging the shards of one op +into one table is the delivery step (playbook §8) and is out of scope here. +""" + +from __future__ import annotations + +import argparse +import json +import shlex +import subprocess +import sys +import time +from dataclasses import dataclass +from pathlib import Path + +import yaml + +PKG_ROOT = Path(__file__).resolve().parents[2] # python/aisimulate +if str(PKG_ROOT) not in sys.path: + sys.path.insert(0, str(PKG_ROOT)) + +FINALIZE_DEPS = ("pyarrow", "pandas") # collector/helper.py finalize_perf_files imports +CONTAINER_CHECKOUT = "/ais" +CONTAINER_OUT = "/out" + + +@dataclass(frozen=True) +class Shard: + op: str + name: str + case_filter: str | None = None + + @property + def key(self) -> str: + return f"{self.op}/{self.name}" + + +@dataclass(frozen=True) +class Plan: + backend: str + sm: int + shards: tuple[Shard, ...] + + +def load_plan(path: Path) -> Plan: + doc = yaml.safe_load(path.read_text()) + shards = [] + seen: set[str] = set() + for raw in doc["shards"]: + s = Shard(op=str(raw["op"]), name=str(raw.get("name", "all")), case_filter=raw.get("case_filter") or None) + if s.key in seen: + raise ValueError(f"duplicate shard {s.key}") + seen.add(s.key) + shards.append(s) + return Plan(backend=str(doc["backend"]), sm=int(doc["sm"]), shards=tuple(shards)) + + +def pinned_image(backend: str, op: str) -> str: + """The manifest-pinned image for (backend, op) — family pins included.""" + from collector.framework_manifest import resolve_op_runtime + + return resolve_op_runtime(backend, op).image() + + +def collect_image_tag(base_image: str) -> str: + """Deterministic local tag for + finalize deps.""" + name = base_image.split("@", 1)[0].replace("/", "-").replace(":", "-") + return f"aisim-collect:{name}" + + +def dockerfile(base_image: str) -> str: + return ( + f"FROM {base_image}\n" + "# collector finalize deps the framework image does not ship (collector/helper.py finalize_perf_files)\n" + f"RUN pip install --no-cache-dir {' '.join(FINALIZE_DEPS)}\n" + ) + + +def build_image(base_image: str, *, network: str = "host", extra_build_args: list[str] | None = None) -> str: + tag = collect_image_tag(base_image) + cmd = ["docker", "build", "--network", network, "-t", tag, "-f", "-", "."] + (extra_build_args or []) + subprocess.run(cmd, input=dockerfile(base_image), text=True, check=True) + return tag + + +def image_digest(image: str) -> str | None: + try: + out = subprocess.run( + # a locally built collect image has NO RepoDigests — `index .RepoDigests 0` fails and the launcher + # refused its own build-image output (B200 2026-10-03); fall back to the image Id + ["docker", "inspect", "--format", "{{if .RepoDigests}}{{index .RepoDigests 0}}{{end}}|{{.Id}}", image], + capture_output=True, + text=True, + check=True, + ).stdout.strip() + return out + except Exception: + return None + + +def collector_argv(plan: Plan, shard: Shard) -> list[str]: + """The exact collect.py argv for one shard. ``--resume`` always, ``--keep-csv`` never.""" + argv = [ + f"{CONTAINER_CHECKOUT}/collector/collect.py", + "--backend", + plan.backend, + "--model-cases-full", + "--sm", + str(plan.sm), + "--ops", + shard.op, + "--checkpoint-dir", + f"{CONTAINER_OUT}/.ckpt", + "--resume", + ] + if shard.case_filter: + argv += ["--case-filter", shard.case_filter] + assert "--keep-csv" not in argv + return argv + + +def docker_argv( + plan: Plan, shard: Shard, *, image: str, gpu: int, checkout: Path, shard_dir: Path, container_name: str +) -> list[str]: + return [ + "docker", + "run", + "--rm", + "--name", + container_name, + "--gpus", + f'"device={gpu}"', + "--shm-size", + "16g", + "-e", + "CUDA_MPS_PIPE_DIRECTORY=/nonexistent-no-mps", + "-e", + "HF_HUB_OFFLINE=1", + "-e", + f"PYTHONPATH={CONTAINER_CHECKOUT}", + "-e", + f"TRITON_CACHE_DIR={CONTAINER_OUT}/.cache/triton", + "-e", + f"DG_JIT_CACHE_DIR={CONTAINER_OUT}/.cache/deep_gemm", + "-e", + f"FLASHINFER_WORKSPACE_BASE={CONTAINER_OUT}/.cache", + "-v", + f"{checkout}:{CONTAINER_CHECKOUT}:ro", + "-v", + f"{shard_dir}:{CONTAINER_OUT}", + "-w", + CONTAINER_OUT, + "--entrypoint", + "python3", + image, + *collector_argv(plan, shard), + ] + + +def shard_dir(root: Path, shard: Shard) -> Path: + return root / shard.op / shard.name + + +def shard_state(root: Path, shard: Shard) -> str: + d = shard_dir(root, shard) + if not d.exists(): + return "pending" + if list(d.glob("*.parquet")): + return "finalized" + if (d / "DONE").exists(): + return "done-no-parquet" + if (d / ".ckpt").exists(): + return "started" + return "pending" + + +def _log(root: Path, record: dict) -> None: + with (root / "campaign.jsonl").open("a") as fh: + fh.write(json.dumps({"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), **record}) + "\n") + + +def run_shard(plan: Plan, shard: Shard, *, root: Path, gpu: int, checkout: Path, image: str, mode: str = "run") -> int: + d = shard_dir(root, shard) + (d / ".cache").mkdir(parents=True, exist_ok=True) + name = f"{'f' if mode == 'finalize' else 'c'}_{plan.backend}_{shard.op}_{shard.name}" + cmd = docker_argv(plan, shard, image=image, gpu=gpu, checkout=checkout, shard_dir=d, container_name=name) + _log( + root, + { + "event": "start", + "mode": mode, + "shard": shard.key, + "gpu": gpu, + "image": image, + "image_digest": image_digest(image), + "argv": collector_argv(plan, shard), + }, + ) + with (d / "run.log").open("a") as log: + log.write(f"### {time.strftime('%FT%T')} {mode} gpu{gpu} {shlex.join(cmd)}\n") + log.flush() + rc = subprocess.run(cmd, stdout=log, stderr=subprocess.STDOUT).returncode + log.write(f"### {time.strftime('%FT%T')} exit={rc}\n") + parquets = sorted(p.name for p in d.glob("*.parquet")) + if rc == 0: + (d / "DONE").touch() + _log(root, {"event": "end", "mode": mode, "shard": shard.key, "gpu": gpu, "exit": rc, "parquet": parquets}) + return rc + + +def _free_gpu(busy: dict[int, subprocess.Popen | None], gpus: list[int]) -> int | None: + for g in gpus: + p = busy.get(g) + if p is None or p.poll() is not None: + return g + return None + + +def run_campaign( + plan: Plan, *, plan_path: Path, root: Path, gpus: list[int], checkout: Path, image_override: str | None, mode: str +) -> int: + """GPU pool over the plan. mode=run: skip DONE shards; mode=finalize: only done-no-parquet shards.""" + root.mkdir(parents=True, exist_ok=True) + todo = [] + for s in plan.shards: + st = shard_state(root, s) + if mode == "run" and st in ("finalized", "done-no-parquet") and (shard_dir(root, s) / "DONE").exists(): + continue + if mode == "finalize" and st != "done-no-parquet": + continue + todo.append(s) + print(f"{mode}: {len(todo)}/{len(plan.shards)} shards on gpus {gpus}") + images = {s.op: (image_override or collect_image_tag(pinned_image(plan.backend, s.op))) for s in todo} + for img in sorted(set(images.values())): + if image_digest(img) is None: + raise SystemExit(f"image {img} not present — run `build-image` for its base first") + busy: dict[int, subprocess.Popen | None] = {} + failures = 0 + while todo: + g = _free_gpu(busy, gpus) + if g is None: + time.sleep(15) + for p in busy.values(): + if p is not None and p.poll() not in (None, 0): + failures += 1 + busy[[k for k, v in busy.items() if v is p][0]] = None + continue + s = todo.pop(0) + print(f"{time.strftime('%T')} gpu{g} <- {s.key}") + busy[g] = subprocess.Popen( + [ + sys.executable, + __file__, + "_shard", + "--plan", + str(plan_path), + "--root", + str(root), + "--gpu", + str(g), + "--shard", + s.key, + "--checkout", + str(checkout), + "--image", + images[s.op], + "--mode", + mode, + ] + ) + for p in busy.values(): + if p is not None and p.wait() != 0: + failures += 1 + print(f"{mode} finished, {failures} shard(s) exited non-zero") + return 1 if failures else 0 + + +def status(plan: Plan, root: Path) -> None: + for s in plan.shards: + d = shard_dir(root, s) + last = "" + log = d / "run.log" + if log.exists(): + tail = log.read_text(errors="replace").replace("\r", "\n").splitlines() + last = next( + (ln for ln in reversed(tail) if "it/s" in ln or "s/it" in ln or "Completed" in ln or "exit=" in ln), "" + ) + print(f"{s.op:28s} {s.name:20s} {shard_state(root, s):16s} {last[:100]}") + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + b = sub.add_parser("build-image", help="build + finalize deps") + b.add_argument("--backend", required=True) + b.add_argument("--op", required=True, help="any op of the backend; family pins may select a different image") + b.add_argument("--build-arg", action="append", default=[], help="extra docker build args, e.g. http_proxy=...") + for name in ("run", "finalize", "status"): + p = sub.add_parser(name) + p.add_argument("--plan", required=True, type=Path) + p.add_argument("--root", required=True, type=Path, help="campaign root: one // namespace per shard") + if name != "status": + p.add_argument("--gpus", required=True, help="comma list of GPU indices for the pool") + p.add_argument("--checkout", type=Path, default=PKG_ROOT, help="python/aisimulate checkout to mount") + p.add_argument("--image", default=None, help="override the manifest-derived collect image (A/B only)") + h = sub.add_parser("_shard") # internal: one shard in one container (spawned by run/finalize) + for flag in ("--plan", "--root", "--checkout"): + h.add_argument(flag, required=True, type=Path) + h.add_argument("--gpu", required=True, type=int) + h.add_argument("--shard", required=True) + h.add_argument("--image", required=True) + h.add_argument("--mode", required=True) + args = ap.parse_args() + if args.cmd == "_shard": + plan = load_plan(args.plan) + shard = next(s for s in plan.shards if s.key == args.shard) + return run_shard( + plan, shard, root=args.root, gpu=args.gpu, checkout=args.checkout, image=args.image, mode=args.mode + ) + if args.cmd == "build-image": + base = pinned_image(args.backend, args.op) + tag = build_image(base, extra_build_args=[f"--build-arg={a}" for a in args.build_arg]) + print(f"built {tag} from {base}") + return 0 + plan = load_plan(args.plan.resolve()) + if args.cmd == "status": + status(plan, args.root.resolve()) + return 0 + gpus = [int(x) for x in args.gpus.split(",") if x.strip()] + return run_campaign( + plan, + plan_path=args.plan.resolve(), + root=args.root.resolve(), + gpus=gpus, + checkout=args.checkout.resolve(), + image_override=args.image, + mode=args.cmd, + ) + + +if __name__ == "__main__": + raise SystemExit(main()) From 117aa556f0fecabb524be824d2134bd2c5edd45a Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 14:19:38 +0800 Subject: [PATCH 02/13] chore: leave the probe-roster model configs to the harness PR The nine bundled configs (eleven files) were added on 2026-09-25 so the op-probe harness could render its model roster offline; no collector case, collection run or shipped table uses them. They move to the harness PR together with what exists only for them: their DefaultHFModels entries, the Qwen3_5ForCausalLM -> QWEN35 mapping and flat-config handling, the THIRD_PARTY_NOTICES sections (both copies), the NeoHorse SDK test and the merge-time entries in the packaged-config inventory tests. The generator facts that cite these checkpoints stay: they match by model path or quantization layout and load no bundled config. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- THIRD_PARTY_NOTICES.md | 43 - python/aisimulate/THIRD_PARTY_NOTICES.md | 43 - .../Qwen--Qwen3.8-27B-FP8_config.json | 1033 -- .../Qwen--Qwen3.8-27B_config.json | 140 - .../TokenRhythm--NeoHorse-1-9B_config.json | 81 - ...dia--DeepSeek-V4.1-Flash-NVFP4_config.json | 365 - ...Seek-V4.1-Flash-NVFP4_hf_quant_config.json | 179 - .../nvidia--Kimi-K3-NVFP4_config.json | 14241 ---------------- ...ron-3.5-Lightning-30B-A3B-BF16_config.json | 121 - ...vidia--Qwen3.8-2.4T-A95B-NVFP4_config.json | 2690 --- ...en3.8-2.4T-A95B-NVFP4_hf_quant_config.json | 1902 --- .../openbmb--MiniCPM5-2B_config.json | 27 - .../ornith-ai--Ornith-1.5-35B-A3B_config.json | 125 - .../src/aisimulate_core/sdk/common.py | 14 - .../src/aisimulate_core/sdk/utils.py | 3 - .../test_modelopt_mixed_precision_targets.py | 2 - .../aisimulate/tests/unit/sdk/test_utils.py | 45 - .../tests/unit/test_support_config_profile.py | 2 - 18 files changed, 21056 deletions(-) delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json delete mode 100644 python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md index 1f2c508bd2..2fd2b58b6b 100644 --- a/THIRD_PARTY_NOTICES.md +++ b/THIRD_PARTY_NOTICES.md @@ -851,49 +851,6 @@ THE SOFTWARE AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED "AS IS", WITHOUT For any questions regarding this license, please contact model-business@notice.qwencloud.com. ``` -## Qwen3.8-27B model configuration files - -The following model configuration files are copied byte-for-byte from the -named repositories at the immutable revisions shown (onboarding 2026-09-25): - -| Packaged file | Upstream revision | -| --- | --- | -| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json` | `Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | -| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json` | `Qwen/Qwen3.8-27B-FP8@017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` | - -Copyright owner: Qwen. Both upstream revisions are licensed under the Apache -License 2.0 (`LICENSE` in each repository); the Apache License 2.0 terms are -reproduced in this distribution's `LICENSE` file. - -## Ornith 1.5 model configuration - -`aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json` is -an unmodified copy of `config.json` from -https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B at immutable revision -`10fbf86fed7ecee4a061f8b499a618f46001cac1`. The repository declares the MIT -License in its model card (no separate LICENSE file is shipped); copyright -owner: ornith-ai. The MIT License terms are reproduced in the DeepSeek -section above. - -## MiniCPM5 model configuration - -`aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json` is an -unmodified copy of `config.json` from -https://huggingface.co/openbmb/MiniCPM5-2B at immutable revision -`12a3808a956f869c767195e9266b59c4d21d92e2`. The repository declares the -Apache License 2.0 in its model card (no separate LICENSE file is shipped); -copyright owner: OpenBMB. The Apache License 2.0 terms are reproduced in this -distribution's `LICENSE` file. - -## NeoHorse 1 model configuration - -`aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json` is an -unmodified copy of `config.json` from -https://huggingface.co/TokenRhythm/NeoHorse-1-9B at immutable revision -`ba5b6e40d88a6ddf4591e176738254a3bc715765`. Copyright owner: TokenRhythm. -License: Apache License 2.0 (`LICENSE` in the repository), reproduced in this -distribution's `LICENSE` file. - ## Meta Llama 4 The following bundled model configs are modified copies of Meta Llama 4 diff --git a/python/aisimulate/THIRD_PARTY_NOTICES.md b/python/aisimulate/THIRD_PARTY_NOTICES.md index 1f2c508bd2..2fd2b58b6b 100644 --- a/python/aisimulate/THIRD_PARTY_NOTICES.md +++ b/python/aisimulate/THIRD_PARTY_NOTICES.md @@ -851,49 +851,6 @@ THE SOFTWARE AND ANY OUTPUT AND RESULTS THEREFROM ARE PROVIDED "AS IS", WITHOUT For any questions regarding this license, please contact model-business@notice.qwencloud.com. ``` -## Qwen3.8-27B model configuration files - -The following model configuration files are copied byte-for-byte from the -named repositories at the immutable revisions shown (onboarding 2026-09-25): - -| Packaged file | Upstream revision | -| --- | --- | -| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json` | `Qwen/Qwen3.8-27B@1d4bf0f2ff6012fd82039f2fa52739d0dd7c60c0` | -| `aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json` | `Qwen/Qwen3.8-27B-FP8@017b9c7af6b5689d5dd426a76e0bc077eb5ca20a` | - -Copyright owner: Qwen. Both upstream revisions are licensed under the Apache -License 2.0 (`LICENSE` in each repository); the Apache License 2.0 terms are -reproduced in this distribution's `LICENSE` file. - -## Ornith 1.5 model configuration - -`aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json` is -an unmodified copy of `config.json` from -https://huggingface.co/ornith-ai/Ornith-1.5-35B-A3B at immutable revision -`10fbf86fed7ecee4a061f8b499a618f46001cac1`. The repository declares the MIT -License in its model card (no separate LICENSE file is shipped); copyright -owner: ornith-ai. The MIT License terms are reproduced in the DeepSeek -section above. - -## MiniCPM5 model configuration - -`aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json` is an -unmodified copy of `config.json` from -https://huggingface.co/openbmb/MiniCPM5-2B at immutable revision -`12a3808a956f869c767195e9266b59c4d21d92e2`. The repository declares the -Apache License 2.0 in its model card (no separate LICENSE file is shipped); -copyright owner: OpenBMB. The Apache License 2.0 terms are reproduced in this -distribution's `LICENSE` file. - -## NeoHorse 1 model configuration - -`aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json` is an -unmodified copy of `config.json` from -https://huggingface.co/TokenRhythm/NeoHorse-1-9B at immutable revision -`ba5b6e40d88a6ddf4591e176738254a3bc715765`. Copyright owner: TokenRhythm. -License: Apache License 2.0 (`LICENSE` in the repository), reproduced in this -distribution's `LICENSE` file. - ## Meta Llama 4 The following bundled model configs are modified copies of Meta Llama 4 diff --git a/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json b/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json deleted file mode 100644 index e66d0dcb2d..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B-FP8_config.json +++ /dev/null @@ -1,1033 +0,0 @@ -{ - "architectures": [ - "Qwen3_5ForConditionalGeneration" - ], - "image_token_id": 248056, - "language_model_only": false, - "model_type": "qwen3_5", - "text_config": { - "attention_bias": false, - "attention_dropout": 0.0, - "attn_output_gate": true, - "bos_token_id": 248044, - "dtype": "bfloat16", - "eos_token_id": 248044, - "full_attention_interval": 4, - "head_dim": 256, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 17408, - "layer_types": [ - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention" - ], - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 48, - "linear_value_head_dim": 128, - "mamba_ssm_dtype": "float32", - "max_position_embeddings": 262144, - "model_type": "qwen3_5_text", - "mtp_num_hidden_layers": 1, - "mtp_use_dedicated_embeddings": false, - "num_attention_heads": 24, - "num_hidden_layers": 64, - "num_key_value_heads": 4, - "output_gate_type": "swish", - "pad_token_id": null, - "partial_rotary_factor": 0.25, - "rms_norm_eps": 1e-06, - "rope_parameters": { - "mrope_interleaved": true, - "mrope_section": [ - 11, - 11, - 10 - ], - "partial_rotary_factor": 0.25, - "rope_theta": 10000000, - "rope_type": "default" - }, - "tie_word_embeddings": false, - "use_cache": true, - "vocab_size": 248320 - }, - "tie_word_embeddings": false, - "transformers_version": "5.8.0.dev0", - "video_token_id": 248057, - "vision_config": { - "deepstack_visual_indexes": [], - "depth": 27, - "hidden_act": "gelu_pytorch_tanh", - "hidden_size": 1152, - "in_channels": 3, - "initializer_range": 0.02, - "intermediate_size": 4304, - "model_type": "qwen3_5", - "num_heads": 16, - "num_position_embeddings": 2304, - "out_hidden_size": 5120, - "patch_size": 16, - "spatial_merge_size": 2, - "temporal_patch_size": 2 - }, - "vision_end_token_id": 248054, - "vision_start_token_id": 248053, - "quantization_config": { - "activation_scheme": "dynamic", - "fmt": "e4m3", - "quant_method": "fp8", - "modules_to_not_convert": [ - "model.visual.blocks.0.attn.proj", - "model.visual.blocks.0.attn.qkv", - "model.visual.blocks.0.mlp.linear_fc1", - "model.visual.blocks.0.mlp.linear_fc2", - "visual.blocks.0.attn.proj", - "visual.blocks.0.attn.qkv_proj", - "visual.blocks.0.mlp.linear_fc1", - "visual.blocks.0.mlp.linear_fc2", - "model.visual.blocks.1.attn.proj", - "model.visual.blocks.1.attn.qkv", - "model.visual.blocks.1.mlp.linear_fc1", - "model.visual.blocks.1.mlp.linear_fc2", - "visual.blocks.1.attn.proj", - "visual.blocks.1.attn.qkv_proj", - "visual.blocks.1.mlp.linear_fc1", - "visual.blocks.1.mlp.linear_fc2", - "model.visual.blocks.2.attn.proj", - "model.visual.blocks.2.attn.qkv", - "model.visual.blocks.2.mlp.linear_fc1", - "model.visual.blocks.2.mlp.linear_fc2", - "visual.blocks.2.attn.proj", - "visual.blocks.2.attn.qkv_proj", - "visual.blocks.2.mlp.linear_fc1", - "visual.blocks.2.mlp.linear_fc2", - "model.visual.blocks.3.attn.proj", - "model.visual.blocks.3.attn.qkv", - "model.visual.blocks.3.mlp.linear_fc1", - "model.visual.blocks.3.mlp.linear_fc2", - "visual.blocks.3.attn.proj", - "visual.blocks.3.attn.qkv_proj", - "visual.blocks.3.mlp.linear_fc1", - "visual.blocks.3.mlp.linear_fc2", - "model.visual.blocks.4.attn.proj", - "model.visual.blocks.4.attn.qkv", - "model.visual.blocks.4.mlp.linear_fc1", - "model.visual.blocks.4.mlp.linear_fc2", - "visual.blocks.4.attn.proj", - "visual.blocks.4.attn.qkv_proj", - "visual.blocks.4.mlp.linear_fc1", - "visual.blocks.4.mlp.linear_fc2", - "model.visual.blocks.5.attn.proj", - "model.visual.blocks.5.attn.qkv", - "model.visual.blocks.5.mlp.linear_fc1", - "model.visual.blocks.5.mlp.linear_fc2", - "visual.blocks.5.attn.proj", - "visual.blocks.5.attn.qkv_proj", - "visual.blocks.5.mlp.linear_fc1", - "visual.blocks.5.mlp.linear_fc2", - "model.visual.blocks.6.attn.proj", - "model.visual.blocks.6.attn.qkv", - "model.visual.blocks.6.mlp.linear_fc1", - "model.visual.blocks.6.mlp.linear_fc2", - "visual.blocks.6.attn.proj", - "visual.blocks.6.attn.qkv_proj", - "visual.blocks.6.mlp.linear_fc1", - "visual.blocks.6.mlp.linear_fc2", - "model.visual.blocks.7.attn.proj", - "model.visual.blocks.7.attn.qkv", - "model.visual.blocks.7.mlp.linear_fc1", - "model.visual.blocks.7.mlp.linear_fc2", - "visual.blocks.7.attn.proj", - "visual.blocks.7.attn.qkv_proj", - "visual.blocks.7.mlp.linear_fc1", - "visual.blocks.7.mlp.linear_fc2", - "model.visual.blocks.8.attn.proj", - "model.visual.blocks.8.attn.qkv", - "model.visual.blocks.8.mlp.linear_fc1", - "model.visual.blocks.8.mlp.linear_fc2", - "visual.blocks.8.attn.proj", - "visual.blocks.8.attn.qkv_proj", - "visual.blocks.8.mlp.linear_fc1", - "visual.blocks.8.mlp.linear_fc2", - "model.visual.blocks.9.attn.proj", - "model.visual.blocks.9.attn.qkv", - "model.visual.blocks.9.mlp.linear_fc1", - "model.visual.blocks.9.mlp.linear_fc2", - "visual.blocks.9.attn.proj", - "visual.blocks.9.attn.qkv_proj", - "visual.blocks.9.mlp.linear_fc1", - "visual.blocks.9.mlp.linear_fc2", - "model.visual.blocks.10.attn.proj", - "model.visual.blocks.10.attn.qkv", - "model.visual.blocks.10.mlp.linear_fc1", - "model.visual.blocks.10.mlp.linear_fc2", - "visual.blocks.10.attn.proj", - "visual.blocks.10.attn.qkv_proj", - "visual.blocks.10.mlp.linear_fc1", - "visual.blocks.10.mlp.linear_fc2", - "model.visual.blocks.11.attn.proj", - "model.visual.blocks.11.attn.qkv", - "model.visual.blocks.11.mlp.linear_fc1", - "model.visual.blocks.11.mlp.linear_fc2", - "visual.blocks.11.attn.proj", - "visual.blocks.11.attn.qkv_proj", - "visual.blocks.11.mlp.linear_fc1", - "visual.blocks.11.mlp.linear_fc2", - "model.visual.blocks.12.attn.proj", - "model.visual.blocks.12.attn.qkv", - "model.visual.blocks.12.mlp.linear_fc1", - "model.visual.blocks.12.mlp.linear_fc2", - "visual.blocks.12.attn.proj", - "visual.blocks.12.attn.qkv_proj", - "visual.blocks.12.mlp.linear_fc1", - "visual.blocks.12.mlp.linear_fc2", - "model.visual.blocks.13.attn.proj", - "model.visual.blocks.13.attn.qkv", - "model.visual.blocks.13.mlp.linear_fc1", - "model.visual.blocks.13.mlp.linear_fc2", - "visual.blocks.13.attn.proj", - "visual.blocks.13.attn.qkv_proj", - "visual.blocks.13.mlp.linear_fc1", - "visual.blocks.13.mlp.linear_fc2", - "model.visual.blocks.14.attn.proj", - "model.visual.blocks.14.attn.qkv", - "model.visual.blocks.14.mlp.linear_fc1", - "model.visual.blocks.14.mlp.linear_fc2", - "visual.blocks.14.attn.proj", - "visual.blocks.14.attn.qkv_proj", - "visual.blocks.14.mlp.linear_fc1", - "visual.blocks.14.mlp.linear_fc2", - "model.visual.blocks.15.attn.proj", - "model.visual.blocks.15.attn.qkv", - "model.visual.blocks.15.mlp.linear_fc1", - "model.visual.blocks.15.mlp.linear_fc2", - "visual.blocks.15.attn.proj", - "visual.blocks.15.attn.qkv_proj", - "visual.blocks.15.mlp.linear_fc1", - "visual.blocks.15.mlp.linear_fc2", - "model.visual.blocks.16.attn.proj", - "model.visual.blocks.16.attn.qkv", - "model.visual.blocks.16.mlp.linear_fc1", - "model.visual.blocks.16.mlp.linear_fc2", - "visual.blocks.16.attn.proj", - "visual.blocks.16.attn.qkv_proj", - "visual.blocks.16.mlp.linear_fc1", - "visual.blocks.16.mlp.linear_fc2", - "model.visual.blocks.17.attn.proj", - "model.visual.blocks.17.attn.qkv", - "model.visual.blocks.17.mlp.linear_fc1", - "model.visual.blocks.17.mlp.linear_fc2", - "visual.blocks.17.attn.proj", - "visual.blocks.17.attn.qkv_proj", - "visual.blocks.17.mlp.linear_fc1", - "visual.blocks.17.mlp.linear_fc2", - "model.visual.blocks.18.attn.proj", - "model.visual.blocks.18.attn.qkv", - "model.visual.blocks.18.mlp.linear_fc1", - "model.visual.blocks.18.mlp.linear_fc2", - "visual.blocks.18.attn.proj", - "visual.blocks.18.attn.qkv_proj", - "visual.blocks.18.mlp.linear_fc1", - "visual.blocks.18.mlp.linear_fc2", - "model.visual.blocks.19.attn.proj", - "model.visual.blocks.19.attn.qkv", - "model.visual.blocks.19.mlp.linear_fc1", - "model.visual.blocks.19.mlp.linear_fc2", - "visual.blocks.19.attn.proj", - "visual.blocks.19.attn.qkv_proj", - "visual.blocks.19.mlp.linear_fc1", - "visual.blocks.19.mlp.linear_fc2", - "model.visual.blocks.20.attn.proj", - "model.visual.blocks.20.attn.qkv", - "model.visual.blocks.20.mlp.linear_fc1", - "model.visual.blocks.20.mlp.linear_fc2", - "visual.blocks.20.attn.proj", - "visual.blocks.20.attn.qkv_proj", - "visual.blocks.20.mlp.linear_fc1", - "visual.blocks.20.mlp.linear_fc2", - "model.visual.blocks.21.attn.proj", - "model.visual.blocks.21.attn.qkv", - "model.visual.blocks.21.mlp.linear_fc1", - "model.visual.blocks.21.mlp.linear_fc2", - "visual.blocks.21.attn.proj", - "visual.blocks.21.attn.qkv_proj", - "visual.blocks.21.mlp.linear_fc1", - "visual.blocks.21.mlp.linear_fc2", - "model.visual.blocks.22.attn.proj", - "model.visual.blocks.22.attn.qkv", - "model.visual.blocks.22.mlp.linear_fc1", - "model.visual.blocks.22.mlp.linear_fc2", - "visual.blocks.22.attn.proj", - "visual.blocks.22.attn.qkv_proj", - "visual.blocks.22.mlp.linear_fc1", - "visual.blocks.22.mlp.linear_fc2", - "model.visual.blocks.23.attn.proj", - "model.visual.blocks.23.attn.qkv", - "model.visual.blocks.23.mlp.linear_fc1", - "model.visual.blocks.23.mlp.linear_fc2", - "visual.blocks.23.attn.proj", - "visual.blocks.23.attn.qkv_proj", - "visual.blocks.23.mlp.linear_fc1", - "visual.blocks.23.mlp.linear_fc2", - "model.visual.blocks.24.attn.proj", - "model.visual.blocks.24.attn.qkv", - "model.visual.blocks.24.mlp.linear_fc1", - "model.visual.blocks.24.mlp.linear_fc2", - "visual.blocks.24.attn.proj", - "visual.blocks.24.attn.qkv_proj", - "visual.blocks.24.mlp.linear_fc1", - "visual.blocks.24.mlp.linear_fc2", - "model.visual.blocks.25.attn.proj", - "model.visual.blocks.25.attn.qkv", - "model.visual.blocks.25.mlp.linear_fc1", - "model.visual.blocks.25.mlp.linear_fc2", - "visual.blocks.25.attn.proj", - "visual.blocks.25.attn.qkv_proj", - "visual.blocks.25.mlp.linear_fc1", - "visual.blocks.25.mlp.linear_fc2", - "model.visual.blocks.26.attn.proj", - "model.visual.blocks.26.attn.qkv", - "model.visual.blocks.26.mlp.linear_fc1", - "model.visual.blocks.26.mlp.linear_fc2", - "visual.blocks.26.attn.proj", - "visual.blocks.26.attn.qkv_proj", - "visual.blocks.26.mlp.linear_fc1", - "visual.blocks.26.mlp.linear_fc2", - "model.visual.deepstack_merger_list.0.linear_fc1", - "model.visual.deepstack_merger_list.0.linear_fc2", - "model.visual.deepstack_merger_list.0.norm", - "visual.deepstack_merger_list.0.linear_fc1", - "visual.deepstack_merger_list.0.linear_fc2", - "visual.deepstack_merger_list.0.norm", - "model.visual.deepstack_merger_list.1.linear_fc1", - "model.visual.deepstack_merger_list.1.linear_fc2", - "model.visual.deepstack_merger_list.1.norm", - "visual.deepstack_merger_list.1.linear_fc1", - "visual.deepstack_merger_list.1.linear_fc2", - "visual.deepstack_merger_list.1.norm", - "model.visual.deepstack_merger_list.2.linear_fc1", - "model.visual.deepstack_merger_list.2.linear_fc2", - "model.visual.deepstack_merger_list.2.norm", - "visual.deepstack_merger_list.2.linear_fc1", - "visual.deepstack_merger_list.2.linear_fc2", - "visual.deepstack_merger_list.2.norm", - "model.visual.merger.linear_fc1", - "model.visual.merger.linear_fc2", - "model.visual.merger.norm", - "model.visual.patch_embed.proj", - "model.visual.pos_embed", - "visual.merger.linear_fc1", - "visual.merger.linear_fc2", - "visual.merger.norm", - "visual.patch_embed.proj", - "visual.pos_embed", - "visual", - "model.visual", - "lm_head", - "model.embed_tokens", - "model.language_model.layers.0.input_layernorm", - "model.language_model.layers.0.mlp.shared_expert_gate", - "model.language_model.layers.0.post_attention_layernorm", - "model.language_model.layers.0.mlp.gate", - "model.language_model.layers.0.linear_attn.A_log", - "model.language_model.layers.0.linear_attn.conv1d", - "model.language_model.layers.0.linear_attn.dt_bias", - "model.language_model.layers.0.linear_attn.in_proj_ba", - "model.language_model.layers.0.linear_attn.in_proj_b", - "model.language_model.layers.0.linear_attn.in_proj_a", - "model.language_model.layers.0.linear_attn.norm", - "model.language_model.layers.1.input_layernorm", - "model.language_model.layers.1.mlp.shared_expert_gate", - "model.language_model.layers.1.post_attention_layernorm", - "model.language_model.layers.1.mlp.gate", - "model.language_model.layers.1.linear_attn.A_log", - "model.language_model.layers.1.linear_attn.conv1d", - "model.language_model.layers.1.linear_attn.dt_bias", - "model.language_model.layers.1.linear_attn.in_proj_ba", - "model.language_model.layers.1.linear_attn.in_proj_b", - "model.language_model.layers.1.linear_attn.in_proj_a", - "model.language_model.layers.1.linear_attn.norm", - "model.language_model.layers.2.input_layernorm", - "model.language_model.layers.2.mlp.shared_expert_gate", - "model.language_model.layers.2.post_attention_layernorm", - "model.language_model.layers.2.mlp.gate", - "model.language_model.layers.2.linear_attn.A_log", - "model.language_model.layers.2.linear_attn.conv1d", - "model.language_model.layers.2.linear_attn.dt_bias", - "model.language_model.layers.2.linear_attn.in_proj_ba", - "model.language_model.layers.2.linear_attn.in_proj_b", - "model.language_model.layers.2.linear_attn.in_proj_a", - "model.language_model.layers.2.linear_attn.norm", - "model.language_model.layers.3.input_layernorm", - "model.language_model.layers.3.mlp.shared_expert_gate", - "model.language_model.layers.3.post_attention_layernorm", - "model.language_model.layers.3.mlp.gate", - "model.language_model.layers.3.self_attn.k_norm", - "model.language_model.layers.3.self_attn.q_norm", - "model.language_model.layers.4.input_layernorm", - "model.language_model.layers.4.mlp.shared_expert_gate", - "model.language_model.layers.4.post_attention_layernorm", - "model.language_model.layers.4.mlp.gate", - "model.language_model.layers.4.linear_attn.A_log", - "model.language_model.layers.4.linear_attn.conv1d", - "model.language_model.layers.4.linear_attn.dt_bias", - "model.language_model.layers.4.linear_attn.in_proj_ba", - "model.language_model.layers.4.linear_attn.in_proj_b", - "model.language_model.layers.4.linear_attn.in_proj_a", - "model.language_model.layers.4.linear_attn.norm", - "model.language_model.layers.5.input_layernorm", - "model.language_model.layers.5.mlp.shared_expert_gate", - "model.language_model.layers.5.post_attention_layernorm", - "model.language_model.layers.5.mlp.gate", - "model.language_model.layers.5.linear_attn.A_log", - "model.language_model.layers.5.linear_attn.conv1d", - "model.language_model.layers.5.linear_attn.dt_bias", - "model.language_model.layers.5.linear_attn.in_proj_ba", - "model.language_model.layers.5.linear_attn.in_proj_b", - "model.language_model.layers.5.linear_attn.in_proj_a", - "model.language_model.layers.5.linear_attn.norm", - "model.language_model.layers.6.input_layernorm", - "model.language_model.layers.6.mlp.shared_expert_gate", - "model.language_model.layers.6.post_attention_layernorm", - "model.language_model.layers.6.mlp.gate", - "model.language_model.layers.6.linear_attn.A_log", - "model.language_model.layers.6.linear_attn.conv1d", - "model.language_model.layers.6.linear_attn.dt_bias", - "model.language_model.layers.6.linear_attn.in_proj_ba", - "model.language_model.layers.6.linear_attn.in_proj_b", - "model.language_model.layers.6.linear_attn.in_proj_a", - "model.language_model.layers.6.linear_attn.norm", - "model.language_model.layers.7.input_layernorm", - "model.language_model.layers.7.mlp.shared_expert_gate", - "model.language_model.layers.7.post_attention_layernorm", - "model.language_model.layers.7.mlp.gate", - "model.language_model.layers.7.self_attn.k_norm", - "model.language_model.layers.7.self_attn.q_norm", - "model.language_model.layers.8.input_layernorm", - "model.language_model.layers.8.mlp.shared_expert_gate", - "model.language_model.layers.8.post_attention_layernorm", - "model.language_model.layers.8.mlp.gate", - "model.language_model.layers.8.linear_attn.A_log", - "model.language_model.layers.8.linear_attn.conv1d", - "model.language_model.layers.8.linear_attn.dt_bias", - "model.language_model.layers.8.linear_attn.in_proj_ba", - "model.language_model.layers.8.linear_attn.in_proj_b", - "model.language_model.layers.8.linear_attn.in_proj_a", - "model.language_model.layers.8.linear_attn.norm", - "model.language_model.layers.9.input_layernorm", - "model.language_model.layers.9.mlp.shared_expert_gate", - "model.language_model.layers.9.post_attention_layernorm", - "model.language_model.layers.9.mlp.gate", - "model.language_model.layers.9.linear_attn.A_log", - "model.language_model.layers.9.linear_attn.conv1d", - "model.language_model.layers.9.linear_attn.dt_bias", - "model.language_model.layers.9.linear_attn.in_proj_ba", - "model.language_model.layers.9.linear_attn.in_proj_b", - "model.language_model.layers.9.linear_attn.in_proj_a", - "model.language_model.layers.9.linear_attn.norm", - "model.language_model.layers.10.input_layernorm", - "model.language_model.layers.10.mlp.shared_expert_gate", - "model.language_model.layers.10.post_attention_layernorm", - "model.language_model.layers.10.mlp.gate", - "model.language_model.layers.10.linear_attn.A_log", - "model.language_model.layers.10.linear_attn.conv1d", - "model.language_model.layers.10.linear_attn.dt_bias", - "model.language_model.layers.10.linear_attn.in_proj_ba", - "model.language_model.layers.10.linear_attn.in_proj_b", - "model.language_model.layers.10.linear_attn.in_proj_a", - "model.language_model.layers.10.linear_attn.norm", - "model.language_model.layers.11.input_layernorm", - "model.language_model.layers.11.mlp.shared_expert_gate", - "model.language_model.layers.11.post_attention_layernorm", - "model.language_model.layers.11.mlp.gate", - "model.language_model.layers.11.self_attn.k_norm", - "model.language_model.layers.11.self_attn.q_norm", - "model.language_model.layers.12.input_layernorm", - "model.language_model.layers.12.mlp.shared_expert_gate", - "model.language_model.layers.12.post_attention_layernorm", - "model.language_model.layers.12.mlp.gate", - "model.language_model.layers.12.linear_attn.A_log", - "model.language_model.layers.12.linear_attn.conv1d", - "model.language_model.layers.12.linear_attn.dt_bias", - "model.language_model.layers.12.linear_attn.in_proj_ba", - "model.language_model.layers.12.linear_attn.in_proj_b", - "model.language_model.layers.12.linear_attn.in_proj_a", - "model.language_model.layers.12.linear_attn.norm", - "model.language_model.layers.13.input_layernorm", - "model.language_model.layers.13.mlp.shared_expert_gate", - "model.language_model.layers.13.post_attention_layernorm", - "model.language_model.layers.13.mlp.gate", - "model.language_model.layers.13.linear_attn.A_log", - "model.language_model.layers.13.linear_attn.conv1d", - "model.language_model.layers.13.linear_attn.dt_bias", - "model.language_model.layers.13.linear_attn.in_proj_ba", - "model.language_model.layers.13.linear_attn.in_proj_b", - "model.language_model.layers.13.linear_attn.in_proj_a", - "model.language_model.layers.13.linear_attn.norm", - "model.language_model.layers.14.input_layernorm", - "model.language_model.layers.14.mlp.shared_expert_gate", - "model.language_model.layers.14.post_attention_layernorm", - "model.language_model.layers.14.mlp.gate", - "model.language_model.layers.14.linear_attn.A_log", - "model.language_model.layers.14.linear_attn.conv1d", - "model.language_model.layers.14.linear_attn.dt_bias", - "model.language_model.layers.14.linear_attn.in_proj_ba", - "model.language_model.layers.14.linear_attn.in_proj_b", - "model.language_model.layers.14.linear_attn.in_proj_a", - "model.language_model.layers.14.linear_attn.norm", - "model.language_model.layers.15.input_layernorm", - "model.language_model.layers.15.mlp.shared_expert_gate", - "model.language_model.layers.15.post_attention_layernorm", - "model.language_model.layers.15.mlp.gate", - "model.language_model.layers.15.self_attn.k_norm", - "model.language_model.layers.15.self_attn.q_norm", - "model.language_model.layers.16.input_layernorm", - "model.language_model.layers.16.mlp.shared_expert_gate", - "model.language_model.layers.16.post_attention_layernorm", - "model.language_model.layers.16.mlp.gate", - "model.language_model.layers.16.linear_attn.A_log", - "model.language_model.layers.16.linear_attn.conv1d", - "model.language_model.layers.16.linear_attn.dt_bias", - "model.language_model.layers.16.linear_attn.in_proj_ba", - "model.language_model.layers.16.linear_attn.in_proj_b", - "model.language_model.layers.16.linear_attn.in_proj_a", - "model.language_model.layers.16.linear_attn.norm", - "model.language_model.layers.17.input_layernorm", - "model.language_model.layers.17.mlp.shared_expert_gate", - "model.language_model.layers.17.post_attention_layernorm", - "model.language_model.layers.17.mlp.gate", - "model.language_model.layers.17.linear_attn.A_log", - "model.language_model.layers.17.linear_attn.conv1d", - "model.language_model.layers.17.linear_attn.dt_bias", - "model.language_model.layers.17.linear_attn.in_proj_ba", - "model.language_model.layers.17.linear_attn.in_proj_b", - "model.language_model.layers.17.linear_attn.in_proj_a", - "model.language_model.layers.17.linear_attn.norm", - "model.language_model.layers.18.input_layernorm", - "model.language_model.layers.18.mlp.shared_expert_gate", - "model.language_model.layers.18.post_attention_layernorm", - "model.language_model.layers.18.mlp.gate", - "model.language_model.layers.18.linear_attn.A_log", - "model.language_model.layers.18.linear_attn.conv1d", - "model.language_model.layers.18.linear_attn.dt_bias", - "model.language_model.layers.18.linear_attn.in_proj_ba", - "model.language_model.layers.18.linear_attn.in_proj_b", - "model.language_model.layers.18.linear_attn.in_proj_a", - "model.language_model.layers.18.linear_attn.norm", - "model.language_model.layers.19.input_layernorm", - "model.language_model.layers.19.mlp.shared_expert_gate", - "model.language_model.layers.19.post_attention_layernorm", - "model.language_model.layers.19.mlp.gate", - "model.language_model.layers.19.self_attn.k_norm", - "model.language_model.layers.19.self_attn.q_norm", - "model.language_model.layers.20.input_layernorm", - "model.language_model.layers.20.mlp.shared_expert_gate", - "model.language_model.layers.20.post_attention_layernorm", - "model.language_model.layers.20.mlp.gate", - "model.language_model.layers.20.linear_attn.A_log", - "model.language_model.layers.20.linear_attn.conv1d", - "model.language_model.layers.20.linear_attn.dt_bias", - "model.language_model.layers.20.linear_attn.in_proj_ba", - "model.language_model.layers.20.linear_attn.in_proj_b", - "model.language_model.layers.20.linear_attn.in_proj_a", - "model.language_model.layers.20.linear_attn.norm", - "model.language_model.layers.21.input_layernorm", - "model.language_model.layers.21.mlp.shared_expert_gate", - "model.language_model.layers.21.post_attention_layernorm", - "model.language_model.layers.21.mlp.gate", - "model.language_model.layers.21.linear_attn.A_log", - "model.language_model.layers.21.linear_attn.conv1d", - "model.language_model.layers.21.linear_attn.dt_bias", - "model.language_model.layers.21.linear_attn.in_proj_ba", - "model.language_model.layers.21.linear_attn.in_proj_b", - "model.language_model.layers.21.linear_attn.in_proj_a", - "model.language_model.layers.21.linear_attn.norm", - "model.language_model.layers.22.input_layernorm", - "model.language_model.layers.22.mlp.shared_expert_gate", - "model.language_model.layers.22.post_attention_layernorm", - "model.language_model.layers.22.mlp.gate", - "model.language_model.layers.22.linear_attn.A_log", - "model.language_model.layers.22.linear_attn.conv1d", - "model.language_model.layers.22.linear_attn.dt_bias", - "model.language_model.layers.22.linear_attn.in_proj_ba", - "model.language_model.layers.22.linear_attn.in_proj_b", - "model.language_model.layers.22.linear_attn.in_proj_a", - "model.language_model.layers.22.linear_attn.norm", - "model.language_model.layers.23.input_layernorm", - "model.language_model.layers.23.mlp.shared_expert_gate", - "model.language_model.layers.23.post_attention_layernorm", - "model.language_model.layers.23.mlp.gate", - "model.language_model.layers.23.self_attn.k_norm", - "model.language_model.layers.23.self_attn.q_norm", - "model.language_model.layers.24.input_layernorm", - "model.language_model.layers.24.mlp.shared_expert_gate", - "model.language_model.layers.24.post_attention_layernorm", - "model.language_model.layers.24.mlp.gate", - "model.language_model.layers.24.linear_attn.A_log", - "model.language_model.layers.24.linear_attn.conv1d", - "model.language_model.layers.24.linear_attn.dt_bias", - "model.language_model.layers.24.linear_attn.in_proj_ba", - "model.language_model.layers.24.linear_attn.in_proj_b", - "model.language_model.layers.24.linear_attn.in_proj_a", - "model.language_model.layers.24.linear_attn.norm", - "model.language_model.layers.25.input_layernorm", - "model.language_model.layers.25.mlp.shared_expert_gate", - "model.language_model.layers.25.post_attention_layernorm", - "model.language_model.layers.25.mlp.gate", - "model.language_model.layers.25.linear_attn.A_log", - "model.language_model.layers.25.linear_attn.conv1d", - "model.language_model.layers.25.linear_attn.dt_bias", - "model.language_model.layers.25.linear_attn.in_proj_ba", - "model.language_model.layers.25.linear_attn.in_proj_b", - "model.language_model.layers.25.linear_attn.in_proj_a", - "model.language_model.layers.25.linear_attn.norm", - "model.language_model.layers.26.input_layernorm", - "model.language_model.layers.26.mlp.shared_expert_gate", - "model.language_model.layers.26.post_attention_layernorm", - "model.language_model.layers.26.mlp.gate", - "model.language_model.layers.26.linear_attn.A_log", - "model.language_model.layers.26.linear_attn.conv1d", - "model.language_model.layers.26.linear_attn.dt_bias", - "model.language_model.layers.26.linear_attn.in_proj_ba", - "model.language_model.layers.26.linear_attn.in_proj_b", - "model.language_model.layers.26.linear_attn.in_proj_a", - "model.language_model.layers.26.linear_attn.norm", - "model.language_model.layers.27.input_layernorm", - "model.language_model.layers.27.mlp.shared_expert_gate", - "model.language_model.layers.27.post_attention_layernorm", - "model.language_model.layers.27.mlp.gate", - "model.language_model.layers.27.self_attn.k_norm", - "model.language_model.layers.27.self_attn.q_norm", - "model.language_model.layers.28.input_layernorm", - "model.language_model.layers.28.mlp.shared_expert_gate", - "model.language_model.layers.28.post_attention_layernorm", - "model.language_model.layers.28.mlp.gate", - "model.language_model.layers.28.linear_attn.A_log", - "model.language_model.layers.28.linear_attn.conv1d", - "model.language_model.layers.28.linear_attn.dt_bias", - "model.language_model.layers.28.linear_attn.in_proj_ba", - "model.language_model.layers.28.linear_attn.in_proj_b", - "model.language_model.layers.28.linear_attn.in_proj_a", - "model.language_model.layers.28.linear_attn.norm", - "model.language_model.layers.29.input_layernorm", - "model.language_model.layers.29.mlp.shared_expert_gate", - "model.language_model.layers.29.post_attention_layernorm", - "model.language_model.layers.29.mlp.gate", - "model.language_model.layers.29.linear_attn.A_log", - "model.language_model.layers.29.linear_attn.conv1d", - "model.language_model.layers.29.linear_attn.dt_bias", - "model.language_model.layers.29.linear_attn.in_proj_ba", - "model.language_model.layers.29.linear_attn.in_proj_b", - "model.language_model.layers.29.linear_attn.in_proj_a", - "model.language_model.layers.29.linear_attn.norm", - "model.language_model.layers.30.input_layernorm", - "model.language_model.layers.30.mlp.shared_expert_gate", - "model.language_model.layers.30.post_attention_layernorm", - "model.language_model.layers.30.mlp.gate", - "model.language_model.layers.30.linear_attn.A_log", - "model.language_model.layers.30.linear_attn.conv1d", - "model.language_model.layers.30.linear_attn.dt_bias", - "model.language_model.layers.30.linear_attn.in_proj_ba", - "model.language_model.layers.30.linear_attn.in_proj_b", - "model.language_model.layers.30.linear_attn.in_proj_a", - "model.language_model.layers.30.linear_attn.norm", - "model.language_model.layers.31.input_layernorm", - "model.language_model.layers.31.mlp.shared_expert_gate", - "model.language_model.layers.31.post_attention_layernorm", - "model.language_model.layers.31.mlp.gate", - "model.language_model.layers.31.self_attn.k_norm", - "model.language_model.layers.31.self_attn.q_norm", - "model.language_model.layers.32.input_layernorm", - "model.language_model.layers.32.mlp.shared_expert_gate", - "model.language_model.layers.32.post_attention_layernorm", - "model.language_model.layers.32.mlp.gate", - "model.language_model.layers.32.linear_attn.A_log", - "model.language_model.layers.32.linear_attn.conv1d", - "model.language_model.layers.32.linear_attn.dt_bias", - "model.language_model.layers.32.linear_attn.in_proj_ba", - "model.language_model.layers.32.linear_attn.in_proj_b", - "model.language_model.layers.32.linear_attn.in_proj_a", - "model.language_model.layers.32.linear_attn.norm", - "model.language_model.layers.33.input_layernorm", - "model.language_model.layers.33.mlp.shared_expert_gate", - "model.language_model.layers.33.post_attention_layernorm", - "model.language_model.layers.33.mlp.gate", - "model.language_model.layers.33.linear_attn.A_log", - "model.language_model.layers.33.linear_attn.conv1d", - "model.language_model.layers.33.linear_attn.dt_bias", - "model.language_model.layers.33.linear_attn.in_proj_ba", - "model.language_model.layers.33.linear_attn.in_proj_b", - "model.language_model.layers.33.linear_attn.in_proj_a", - "model.language_model.layers.33.linear_attn.norm", - "model.language_model.layers.34.input_layernorm", - "model.language_model.layers.34.mlp.shared_expert_gate", - "model.language_model.layers.34.post_attention_layernorm", - "model.language_model.layers.34.mlp.gate", - "model.language_model.layers.34.linear_attn.A_log", - "model.language_model.layers.34.linear_attn.conv1d", - "model.language_model.layers.34.linear_attn.dt_bias", - "model.language_model.layers.34.linear_attn.in_proj_ba", - "model.language_model.layers.34.linear_attn.in_proj_b", - "model.language_model.layers.34.linear_attn.in_proj_a", - "model.language_model.layers.34.linear_attn.norm", - "model.language_model.layers.35.input_layernorm", - "model.language_model.layers.35.mlp.shared_expert_gate", - "model.language_model.layers.35.post_attention_layernorm", - "model.language_model.layers.35.mlp.gate", - "model.language_model.layers.35.self_attn.k_norm", - "model.language_model.layers.35.self_attn.q_norm", - "model.language_model.layers.36.input_layernorm", - "model.language_model.layers.36.mlp.shared_expert_gate", - "model.language_model.layers.36.post_attention_layernorm", - "model.language_model.layers.36.mlp.gate", - "model.language_model.layers.36.linear_attn.A_log", - "model.language_model.layers.36.linear_attn.conv1d", - "model.language_model.layers.36.linear_attn.dt_bias", - "model.language_model.layers.36.linear_attn.in_proj_ba", - "model.language_model.layers.36.linear_attn.in_proj_b", - "model.language_model.layers.36.linear_attn.in_proj_a", - "model.language_model.layers.36.linear_attn.norm", - "model.language_model.layers.37.input_layernorm", - "model.language_model.layers.37.mlp.shared_expert_gate", - "model.language_model.layers.37.post_attention_layernorm", - "model.language_model.layers.37.mlp.gate", - "model.language_model.layers.37.linear_attn.A_log", - "model.language_model.layers.37.linear_attn.conv1d", - "model.language_model.layers.37.linear_attn.dt_bias", - "model.language_model.layers.37.linear_attn.in_proj_ba", - "model.language_model.layers.37.linear_attn.in_proj_b", - "model.language_model.layers.37.linear_attn.in_proj_a", - "model.language_model.layers.37.linear_attn.norm", - "model.language_model.layers.38.input_layernorm", - "model.language_model.layers.38.mlp.shared_expert_gate", - "model.language_model.layers.38.post_attention_layernorm", - "model.language_model.layers.38.mlp.gate", - "model.language_model.layers.38.linear_attn.A_log", - "model.language_model.layers.38.linear_attn.conv1d", - "model.language_model.layers.38.linear_attn.dt_bias", - "model.language_model.layers.38.linear_attn.in_proj_ba", - "model.language_model.layers.38.linear_attn.in_proj_b", - "model.language_model.layers.38.linear_attn.in_proj_a", - "model.language_model.layers.38.linear_attn.norm", - "model.language_model.layers.39.input_layernorm", - "model.language_model.layers.39.mlp.shared_expert_gate", - "model.language_model.layers.39.post_attention_layernorm", - "model.language_model.layers.39.mlp.gate", - "model.language_model.layers.39.self_attn.k_norm", - "model.language_model.layers.39.self_attn.q_norm", - "model.language_model.layers.40.input_layernorm", - "model.language_model.layers.40.mlp.shared_expert_gate", - "model.language_model.layers.40.post_attention_layernorm", - "model.language_model.layers.40.mlp.gate", - "model.language_model.layers.40.linear_attn.A_log", - "model.language_model.layers.40.linear_attn.conv1d", - "model.language_model.layers.40.linear_attn.dt_bias", - "model.language_model.layers.40.linear_attn.in_proj_ba", - "model.language_model.layers.40.linear_attn.in_proj_b", - "model.language_model.layers.40.linear_attn.in_proj_a", - "model.language_model.layers.40.linear_attn.norm", - "model.language_model.layers.41.input_layernorm", - "model.language_model.layers.41.mlp.shared_expert_gate", - "model.language_model.layers.41.post_attention_layernorm", - "model.language_model.layers.41.mlp.gate", - "model.language_model.layers.41.linear_attn.A_log", - "model.language_model.layers.41.linear_attn.conv1d", - "model.language_model.layers.41.linear_attn.dt_bias", - "model.language_model.layers.41.linear_attn.in_proj_ba", - "model.language_model.layers.41.linear_attn.in_proj_b", - "model.language_model.layers.41.linear_attn.in_proj_a", - "model.language_model.layers.41.linear_attn.norm", - "model.language_model.layers.42.input_layernorm", - "model.language_model.layers.42.mlp.shared_expert_gate", - "model.language_model.layers.42.post_attention_layernorm", - "model.language_model.layers.42.mlp.gate", - "model.language_model.layers.42.linear_attn.A_log", - "model.language_model.layers.42.linear_attn.conv1d", - "model.language_model.layers.42.linear_attn.dt_bias", - "model.language_model.layers.42.linear_attn.in_proj_ba", - "model.language_model.layers.42.linear_attn.in_proj_b", - "model.language_model.layers.42.linear_attn.in_proj_a", - "model.language_model.layers.42.linear_attn.norm", - "model.language_model.layers.43.input_layernorm", - "model.language_model.layers.43.mlp.shared_expert_gate", - "model.language_model.layers.43.post_attention_layernorm", - "model.language_model.layers.43.mlp.gate", - "model.language_model.layers.43.self_attn.k_norm", - "model.language_model.layers.43.self_attn.q_norm", - "model.language_model.layers.44.input_layernorm", - "model.language_model.layers.44.mlp.shared_expert_gate", - "model.language_model.layers.44.post_attention_layernorm", - "model.language_model.layers.44.mlp.gate", - "model.language_model.layers.44.linear_attn.A_log", - "model.language_model.layers.44.linear_attn.conv1d", - "model.language_model.layers.44.linear_attn.dt_bias", - "model.language_model.layers.44.linear_attn.in_proj_ba", - "model.language_model.layers.44.linear_attn.in_proj_b", - "model.language_model.layers.44.linear_attn.in_proj_a", - "model.language_model.layers.44.linear_attn.norm", - "model.language_model.layers.45.input_layernorm", - "model.language_model.layers.45.mlp.shared_expert_gate", - "model.language_model.layers.45.post_attention_layernorm", - "model.language_model.layers.45.mlp.gate", - "model.language_model.layers.45.linear_attn.A_log", - "model.language_model.layers.45.linear_attn.conv1d", - "model.language_model.layers.45.linear_attn.dt_bias", - "model.language_model.layers.45.linear_attn.in_proj_ba", - "model.language_model.layers.45.linear_attn.in_proj_b", - "model.language_model.layers.45.linear_attn.in_proj_a", - "model.language_model.layers.45.linear_attn.norm", - "model.language_model.layers.46.input_layernorm", - "model.language_model.layers.46.mlp.shared_expert_gate", - "model.language_model.layers.46.post_attention_layernorm", - "model.language_model.layers.46.mlp.gate", - "model.language_model.layers.46.linear_attn.A_log", - "model.language_model.layers.46.linear_attn.conv1d", - "model.language_model.layers.46.linear_attn.dt_bias", - "model.language_model.layers.46.linear_attn.in_proj_ba", - "model.language_model.layers.46.linear_attn.in_proj_b", - "model.language_model.layers.46.linear_attn.in_proj_a", - "model.language_model.layers.46.linear_attn.norm", - "model.language_model.layers.47.input_layernorm", - "model.language_model.layers.47.mlp.shared_expert_gate", - "model.language_model.layers.47.post_attention_layernorm", - "model.language_model.layers.47.mlp.gate", - "model.language_model.layers.47.self_attn.k_norm", - "model.language_model.layers.47.self_attn.q_norm", - "model.language_model.layers.48.input_layernorm", - "model.language_model.layers.48.mlp.shared_expert_gate", - "model.language_model.layers.48.post_attention_layernorm", - "model.language_model.layers.48.mlp.gate", - "model.language_model.layers.48.linear_attn.A_log", - "model.language_model.layers.48.linear_attn.conv1d", - "model.language_model.layers.48.linear_attn.dt_bias", - "model.language_model.layers.48.linear_attn.in_proj_ba", - "model.language_model.layers.48.linear_attn.in_proj_b", - "model.language_model.layers.48.linear_attn.in_proj_a", - "model.language_model.layers.48.linear_attn.norm", - "model.language_model.layers.49.input_layernorm", - "model.language_model.layers.49.mlp.shared_expert_gate", - "model.language_model.layers.49.post_attention_layernorm", - "model.language_model.layers.49.mlp.gate", - "model.language_model.layers.49.linear_attn.A_log", - "model.language_model.layers.49.linear_attn.conv1d", - "model.language_model.layers.49.linear_attn.dt_bias", - "model.language_model.layers.49.linear_attn.in_proj_ba", - "model.language_model.layers.49.linear_attn.in_proj_b", - "model.language_model.layers.49.linear_attn.in_proj_a", - "model.language_model.layers.49.linear_attn.norm", - "model.language_model.layers.50.input_layernorm", - "model.language_model.layers.50.mlp.shared_expert_gate", - "model.language_model.layers.50.post_attention_layernorm", - "model.language_model.layers.50.mlp.gate", - "model.language_model.layers.50.linear_attn.A_log", - "model.language_model.layers.50.linear_attn.conv1d", - "model.language_model.layers.50.linear_attn.dt_bias", - "model.language_model.layers.50.linear_attn.in_proj_ba", - "model.language_model.layers.50.linear_attn.in_proj_b", - "model.language_model.layers.50.linear_attn.in_proj_a", - "model.language_model.layers.50.linear_attn.norm", - "model.language_model.layers.51.input_layernorm", - "model.language_model.layers.51.mlp.shared_expert_gate", - "model.language_model.layers.51.post_attention_layernorm", - "model.language_model.layers.51.mlp.gate", - "model.language_model.layers.51.self_attn.k_norm", - "model.language_model.layers.51.self_attn.q_norm", - "model.language_model.layers.52.input_layernorm", - "model.language_model.layers.52.mlp.shared_expert_gate", - "model.language_model.layers.52.post_attention_layernorm", - "model.language_model.layers.52.mlp.gate", - "model.language_model.layers.52.linear_attn.A_log", - "model.language_model.layers.52.linear_attn.conv1d", - "model.language_model.layers.52.linear_attn.dt_bias", - "model.language_model.layers.52.linear_attn.in_proj_ba", - "model.language_model.layers.52.linear_attn.in_proj_b", - "model.language_model.layers.52.linear_attn.in_proj_a", - "model.language_model.layers.52.linear_attn.norm", - "model.language_model.layers.53.input_layernorm", - "model.language_model.layers.53.mlp.shared_expert_gate", - "model.language_model.layers.53.post_attention_layernorm", - "model.language_model.layers.53.mlp.gate", - "model.language_model.layers.53.linear_attn.A_log", - "model.language_model.layers.53.linear_attn.conv1d", - "model.language_model.layers.53.linear_attn.dt_bias", - "model.language_model.layers.53.linear_attn.in_proj_ba", - "model.language_model.layers.53.linear_attn.in_proj_b", - "model.language_model.layers.53.linear_attn.in_proj_a", - "model.language_model.layers.53.linear_attn.norm", - "model.language_model.layers.54.input_layernorm", - "model.language_model.layers.54.mlp.shared_expert_gate", - "model.language_model.layers.54.post_attention_layernorm", - "model.language_model.layers.54.mlp.gate", - "model.language_model.layers.54.linear_attn.A_log", - "model.language_model.layers.54.linear_attn.conv1d", - "model.language_model.layers.54.linear_attn.dt_bias", - "model.language_model.layers.54.linear_attn.in_proj_ba", - "model.language_model.layers.54.linear_attn.in_proj_b", - "model.language_model.layers.54.linear_attn.in_proj_a", - "model.language_model.layers.54.linear_attn.norm", - "model.language_model.layers.55.input_layernorm", - "model.language_model.layers.55.mlp.shared_expert_gate", - "model.language_model.layers.55.post_attention_layernorm", - "model.language_model.layers.55.mlp.gate", - "model.language_model.layers.55.self_attn.k_norm", - "model.language_model.layers.55.self_attn.q_norm", - "model.language_model.layers.56.input_layernorm", - "model.language_model.layers.56.mlp.shared_expert_gate", - "model.language_model.layers.56.post_attention_layernorm", - "model.language_model.layers.56.mlp.gate", - "model.language_model.layers.56.linear_attn.A_log", - "model.language_model.layers.56.linear_attn.conv1d", - "model.language_model.layers.56.linear_attn.dt_bias", - "model.language_model.layers.56.linear_attn.in_proj_ba", - "model.language_model.layers.56.linear_attn.in_proj_b", - "model.language_model.layers.56.linear_attn.in_proj_a", - "model.language_model.layers.56.linear_attn.norm", - "model.language_model.layers.57.input_layernorm", - "model.language_model.layers.57.mlp.shared_expert_gate", - "model.language_model.layers.57.post_attention_layernorm", - "model.language_model.layers.57.mlp.gate", - "model.language_model.layers.57.linear_attn.A_log", - "model.language_model.layers.57.linear_attn.conv1d", - "model.language_model.layers.57.linear_attn.dt_bias", - "model.language_model.layers.57.linear_attn.in_proj_ba", - "model.language_model.layers.57.linear_attn.in_proj_b", - "model.language_model.layers.57.linear_attn.in_proj_a", - "model.language_model.layers.57.linear_attn.norm", - "model.language_model.layers.58.input_layernorm", - "model.language_model.layers.58.mlp.shared_expert_gate", - "model.language_model.layers.58.post_attention_layernorm", - "model.language_model.layers.58.mlp.gate", - "model.language_model.layers.58.linear_attn.A_log", - "model.language_model.layers.58.linear_attn.conv1d", - "model.language_model.layers.58.linear_attn.dt_bias", - "model.language_model.layers.58.linear_attn.in_proj_ba", - "model.language_model.layers.58.linear_attn.in_proj_b", - "model.language_model.layers.58.linear_attn.in_proj_a", - "model.language_model.layers.58.linear_attn.norm", - "model.language_model.layers.59.input_layernorm", - "model.language_model.layers.59.mlp.shared_expert_gate", - "model.language_model.layers.59.post_attention_layernorm", - "model.language_model.layers.59.mlp.gate", - "model.language_model.layers.59.self_attn.k_norm", - "model.language_model.layers.59.self_attn.q_norm", - "model.language_model.layers.60.input_layernorm", - "model.language_model.layers.60.mlp.shared_expert_gate", - "model.language_model.layers.60.post_attention_layernorm", - "model.language_model.layers.60.mlp.gate", - "model.language_model.layers.60.linear_attn.A_log", - "model.language_model.layers.60.linear_attn.conv1d", - "model.language_model.layers.60.linear_attn.dt_bias", - "model.language_model.layers.60.linear_attn.in_proj_ba", - "model.language_model.layers.60.linear_attn.in_proj_b", - "model.language_model.layers.60.linear_attn.in_proj_a", - "model.language_model.layers.60.linear_attn.norm", - "model.language_model.layers.61.input_layernorm", - "model.language_model.layers.61.mlp.shared_expert_gate", - "model.language_model.layers.61.post_attention_layernorm", - "model.language_model.layers.61.mlp.gate", - "model.language_model.layers.61.linear_attn.A_log", - "model.language_model.layers.61.linear_attn.conv1d", - "model.language_model.layers.61.linear_attn.dt_bias", - "model.language_model.layers.61.linear_attn.in_proj_ba", - "model.language_model.layers.61.linear_attn.in_proj_b", - "model.language_model.layers.61.linear_attn.in_proj_a", - "model.language_model.layers.61.linear_attn.norm", - "model.language_model.layers.62.input_layernorm", - "model.language_model.layers.62.mlp.shared_expert_gate", - "model.language_model.layers.62.post_attention_layernorm", - "model.language_model.layers.62.mlp.gate", - "model.language_model.layers.62.linear_attn.A_log", - "model.language_model.layers.62.linear_attn.conv1d", - "model.language_model.layers.62.linear_attn.dt_bias", - "model.language_model.layers.62.linear_attn.in_proj_ba", - "model.language_model.layers.62.linear_attn.in_proj_b", - "model.language_model.layers.62.linear_attn.in_proj_a", - "model.language_model.layers.62.linear_attn.norm", - "model.language_model.layers.63.input_layernorm", - "model.language_model.layers.63.mlp.shared_expert_gate", - "model.language_model.layers.63.post_attention_layernorm", - "model.language_model.layers.63.mlp.gate", - "model.language_model.layers.63.self_attn.k_norm", - "model.language_model.layers.63.self_attn.q_norm", - "mtp.layers.0.input_layernorm", - "mtp.layers.0.mlp.gate", - "mtp.layers.0.mlp.shared_expert_gate", - "mtp.layers.0.post_attention_layernorm", - "mtp.layers.0.self_attn.k_norm", - "mtp.layers.0.self_attn.q_norm", - "mtp.fc", - "mtp.norm", - "mtp.pre_fc_norm_embedding", - "mtp.pre_fc_norm_hidden" - ], - "weight_block_size": [ - 128, - 128 - ] - } -} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json deleted file mode 100644 index 706cebd746..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/Qwen--Qwen3.8-27B_config.json +++ /dev/null @@ -1,140 +0,0 @@ -{ - "architectures": [ - "Qwen3_5ForConditionalGeneration" - ], - "image_token_id": 248056, - "language_model_only": false, - "model_type": "qwen3_5", - "text_config": { - "attention_bias": false, - "attention_dropout": 0.0, - "attn_output_gate": true, - "bos_token_id": 248044, - "dtype": "bfloat16", - "eos_token_id": 248044, - "full_attention_interval": 4, - "head_dim": 256, - "hidden_act": "silu", - "hidden_size": 5120, - "initializer_range": 0.02, - "intermediate_size": 17408, - "layer_types": [ - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention" - ], - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 48, - "linear_value_head_dim": 128, - "mamba_ssm_dtype": "float32", - "max_position_embeddings": 262144, - "model_type": "qwen3_5_text", - "mtp_num_hidden_layers": 1, - "mtp_use_dedicated_embeddings": false, - "num_attention_heads": 24, - "num_hidden_layers": 64, - "num_key_value_heads": 4, - "output_gate_type": "swish", - "pad_token_id": null, - "partial_rotary_factor": 0.25, - "rms_norm_eps": 1e-06, - "rope_parameters": { - "mrope_interleaved": true, - "mrope_section": [ - 11, - 11, - 10 - ], - "partial_rotary_factor": 0.25, - "rope_theta": 10000000, - "rope_type": "default" - }, - "tie_word_embeddings": false, - "use_cache": true, - "vocab_size": 248320 - }, - "tie_word_embeddings": false, - "transformers_version": "5.8.0.dev0", - "video_token_id": 248057, - "vision_config": { - "deepstack_visual_indexes": [], - "depth": 27, - "hidden_act": "gelu_pytorch_tanh", - "hidden_size": 1152, - "in_channels": 3, - "initializer_range": 0.02, - "intermediate_size": 4304, - "model_type": "qwen3_5", - "num_heads": 16, - "num_position_embeddings": 2304, - "out_hidden_size": 5120, - "patch_size": 16, - "spatial_merge_size": 2, - "temporal_patch_size": 2 - }, - "vision_end_token_id": 248054, - "vision_start_token_id": 248053 -} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json deleted file mode 100644 index 109d798e5d..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/TokenRhythm--NeoHorse-1-9B_config.json +++ /dev/null @@ -1,81 +0,0 @@ -{ - "attention_bias": false, - "attention_dropout": 0.0, - "attn_output_gate": true, - "dtype": "bfloat16", - "eos_token_id": 248044, - "full_attention_interval": 4, - "head_dim": 256, - "hidden_act": "silu", - "hidden_size": 4096, - "initializer_range": 0.02, - "intermediate_size": 12288, - "layer_types": [ - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention" - ], - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "linear_value_head_dim": 128, - "max_position_embeddings": 262144, - "mlp_only_layers": [], - "model_type": "qwen3_5_text", - "mtp_num_hidden_layers": 1, - "mtp_use_dedicated_embeddings": false, - "num_attention_heads": 16, - "num_hidden_layers": 32, - "num_key_value_heads": 4, - "rms_norm_eps": 1e-06, - "use_cache": true, - "vocab_size": 248320, - "mamba_ssm_dtype": "float32", - "rope_parameters": { - "mrope_interleaved": true, - "mrope_section": [ - 11, - 11, - 10 - ], - "rope_type": "default", - "rope_theta": 10000000, - "partial_rotary_factor": 0.25 - }, - "architectures": [ - "Qwen3_5ForCausalLM" - ], - "tie_word_embeddings": false, - "transformers_version": "5.16.1", - "modification_notice": "Modified by TokenRhythm: language-model weights fine-tuned from Qwen/Qwen3.5-9B; repackaged for text-only inference by changing config and tensor key prefixes. Tensor values are unchanged by repackaging. Original model: Copyright 2026 Alibaba Cloud." -} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json deleted file mode 100644 index b42385da33..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_config.json +++ /dev/null @@ -1,365 +0,0 @@ -{ - "architectures": [ - "DeepseekV41ForCausalLM" - ], - "bos_token_id": 0, - "dtype": "bfloat16", - "eos_token_id": 1, - "image_token_id": 129264, - "model_type": "deepseek_v41", - "pad_token_id": 2, - "quantization_config": { - "activation_scheme": "dynamic", - "config_groups": { - "group_0": { - "input_activations": { - "dynamic": false, - "group_size": 16, - "num_bits": 4, - "type": "float" - }, - "targets": [ - "Linear" - ], - "weights": { - "dynamic": false, - "group_size": 16, - "num_bits": 4, - "type": "float" - } - } - }, - "expert_dtype": "fp4", - "group_size": 16, - "ignore": [ - "*.attn.*", - "*.ffn.shared_experts.*", - "head", - "mtp.*" - ], - "kv_cache_quant_algo": null, - "moe_quant_algo": "NVFP4", - "producer": { - "name": "modelopt", - "version": "dsv4-nvfp4-experts" - }, - "quant_algo": "MIXED_PRECISION", - "quant_method": "fp8", - "quantized_layers": { - "layers.0.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.1.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.10.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.11.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.12.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.13.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.14.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.15.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.16.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.17.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.18.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.19.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.2.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.20.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.21.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.22.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.23.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.24.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.25.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.26.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.27.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.28.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.29.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.3.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.30.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.31.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.32.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.33.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.34.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.35.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.36.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.37.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.38.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.39.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.4.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.5.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.6.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.7.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.8.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.9.ffn.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - } - }, - "scale_fmt": "ue8m0", - "weight_block_size": [ - 32, - 32 - ] - }, - "text_config": { - "attention_bias": false, - "attention_dropout": 0.0, - "candidate_block_size": 8, - "candidate_source_layer_id": 20, - "candidate_topk_blocks": 2048, - "compress_ratios": [ - 0, - 0, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 2, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 1, - 0, - 0, - 0 - ], - "compress_rope_theta": 160000, - "dspark_block_size": 5, - "dspark_markov_rank": 256, - "dspark_n_routed_experts": 128, - "dspark_noise_token_id": 128799, - "dspark_num_experts_per_tok": 3, - "dspark_target_layer_ids": [ - 37, - 38, - 39 - ], - "engram_compressed_vocab_size": 99092, - "engram_head_dim": 256, - "engram_layer_ids": [ - 1, - 14 - ], - "engram_max_ngram_size": 4, - "engram_n_heads": 8, - "engram_num_embeddings": [ - 384006168, - 384016682 - ], - "engram_pad_token_id": 2, - "engram_vocab_size": 16000000, - "hc_eps": 1e-06, - "hc_mult": 4, - "hc_sinkhorn_iters": 20, - "head_dim": 512, - "hidden_act": "silu", - "hidden_size": 5120, - "index_head_dim": 128, - "index_n_heads": 32, - "index_source_layer_ids": [ - 2, - 8, - 14, - 20, - 24, - 28, - 32, - 36 - ], - "index_topk": 512, - "initializer_range": 0.02, - "kv_source_layer_ids": [ - 2, - 8, - 14, - 20 - ], - "max_position_embeddings": 1048576, - "model_type": "deepseek_v41_text", - "moe_intermediate_size": 2304, - "n_routed_experts": 384, - "n_shared_experts": 1, - "norm_topk_prob": true, - "num_attention_heads": 64, - "num_experts_per_tok": 6, - "num_hidden_layers": 40, - "num_key_value_heads": 1, - "num_nextn_predict_layers": 3, - "o_groups": 8, - "o_lora_rank": 1024, - "q_lora_rank": 1280, - "qk_rope_head_dim": 64, - "rms_norm_eps": 1e-20, - "rope_scaling": { - "beta_fast": 32, - "beta_slow": 1, - "factor": 16, - "original_max_position_embeddings": 65536, - "rope_type": "yarn" - }, - "rope_theta": 10000, - "routed_scaling_factor": 1.5, - "scoring_func": "sqrtsoftplus", - "sliding_window": 128, - "swiglu_limit": 10.0, - "tie_word_embeddings": false, - "topk_method": "noaux_tc", - "use_cache": true, - "vocab_size": 129280 - }, - "transformers_version": "5.6.0", - "vision_config": { - "downsample_ratio": 3, - "hidden_size": 1024, - "intermediate_size": 2816, - "max_image_tokens": 1024, - "max_wh_ratio": null, - "min_pixels": 295936, - "model_type": "deepseek_v41_vision", - "num_attention_heads": 16, - "num_hidden_layers": 32, - "patch_size": 14, - "rope_theta": 10000 - } -} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json deleted file mode 100644 index 91a7f2a4f3..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--DeepSeek-V4.1-Flash-NVFP4_hf_quant_config.json +++ /dev/null @@ -1,179 +0,0 @@ -{ - "producer": { - "name": "modelopt", - "version": "dsv4-nvfp4-experts" - }, - "quantization": { - "quant_algo": "MIXED_PRECISION", - "kv_cache_quant_algo": null, - "group_size": 16, - "quantized_layers": { - "layers.0.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.1.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.10.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.11.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.12.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.13.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.14.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.15.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.16.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.17.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.18.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.19.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.2.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.20.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.21.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.22.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.23.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.24.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.25.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.26.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.27.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.28.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.29.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.3.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.30.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.31.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.32.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.33.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.34.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.35.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.36.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.37.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.38.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.39.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.4.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.5.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.6.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.7.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.8.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "layers.9.ffn.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - } - }, - "exclude_modules": [ - "*.attn.*", - "*.ffn.shared_experts.*", - "head", - "mtp.*" - ] - } -} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json deleted file mode 100644 index 0e4b41bbe8..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Kimi-K3-NVFP4_config.json +++ /dev/null @@ -1,14241 +0,0 @@ -{ - "architectures": [ - "KimiK3ForConditionalGeneration" - ], - "auto_map": { - "AutoConfig": "configuration_kimi_k3.KimiK3Config", - "AutoModel": "modeling_kimi_k3.KimiK3ForConditionalGeneration", - "AutoModelForCausalLM": "modeling_kimi_k3.KimiK3ForConditionalGeneration" - }, - "bos_token_id": 163584, - "dtype": "bfloat16", - "eos_token_id": 163586, - "ignore_index": -100, - "image_placeholder": "<|kimi_image_placeholder|>", - "media_placeholder_token_id": 163605, - "model_type": "kimi_k3", - "pad_token_id": 163839, - "quantization_config": { - "config_groups": { - "group_0": { - "input_activations": { - "dynamic": false, - "group_size": 16, - "num_bits": 4, - "type": "float" - }, - "targets": [ - "language_model.model.layers.1.block_sparse_moe.experts", - "language_model.model.layers.1.mlp.experts", - "language_model.model.layers.10.block_sparse_moe.experts", - "language_model.model.layers.10.mlp.experts", - "language_model.model.layers.11.block_sparse_moe.experts", - "language_model.model.layers.11.mlp.experts", - "language_model.model.layers.12.block_sparse_moe.experts", - "language_model.model.layers.12.mlp.experts", - "language_model.model.layers.13.block_sparse_moe.experts", - "language_model.model.layers.13.mlp.experts", - "language_model.model.layers.14.block_sparse_moe.experts", - "language_model.model.layers.14.mlp.experts", - "language_model.model.layers.15.block_sparse_moe.experts", - "language_model.model.layers.15.mlp.experts", - "language_model.model.layers.16.block_sparse_moe.experts", - "language_model.model.layers.16.mlp.experts", - "language_model.model.layers.17.block_sparse_moe.experts", - "language_model.model.layers.17.mlp.experts", - "language_model.model.layers.18.block_sparse_moe.experts", - "language_model.model.layers.18.mlp.experts", - "language_model.model.layers.19.block_sparse_moe.experts", - "language_model.model.layers.19.mlp.experts", - "language_model.model.layers.2.block_sparse_moe.experts", - "language_model.model.layers.2.mlp.experts", - "language_model.model.layers.20.block_sparse_moe.experts", - "language_model.model.layers.20.mlp.experts", - "language_model.model.layers.21.block_sparse_moe.experts", - "language_model.model.layers.21.mlp.experts", - "language_model.model.layers.22.block_sparse_moe.experts", - "language_model.model.layers.22.mlp.experts", - "language_model.model.layers.23.block_sparse_moe.experts", - "language_model.model.layers.23.mlp.experts", - "language_model.model.layers.24.block_sparse_moe.experts", - "language_model.model.layers.24.mlp.experts", - "language_model.model.layers.25.block_sparse_moe.experts", - "language_model.model.layers.25.mlp.experts", - "language_model.model.layers.26.block_sparse_moe.experts", - "language_model.model.layers.26.mlp.experts", - "language_model.model.layers.27.block_sparse_moe.experts", - "language_model.model.layers.27.mlp.experts", - "language_model.model.layers.28.block_sparse_moe.experts", - "language_model.model.layers.28.mlp.experts", - "language_model.model.layers.29.block_sparse_moe.experts", - "language_model.model.layers.29.mlp.experts", - "language_model.model.layers.3.block_sparse_moe.experts", - "language_model.model.layers.3.mlp.experts", - "language_model.model.layers.30.block_sparse_moe.experts", - "language_model.model.layers.30.mlp.experts", - "language_model.model.layers.31.block_sparse_moe.experts", - "language_model.model.layers.31.mlp.experts", - "language_model.model.layers.32.block_sparse_moe.experts", - "language_model.model.layers.32.mlp.experts", - "language_model.model.layers.33.block_sparse_moe.experts", - "language_model.model.layers.33.mlp.experts", - "language_model.model.layers.34.block_sparse_moe.experts", - "language_model.model.layers.34.mlp.experts", - "language_model.model.layers.35.block_sparse_moe.experts", - "language_model.model.layers.35.mlp.experts", - "language_model.model.layers.36.block_sparse_moe.experts", - "language_model.model.layers.36.mlp.experts", - "language_model.model.layers.37.block_sparse_moe.experts", - "language_model.model.layers.37.mlp.experts", - "language_model.model.layers.38.block_sparse_moe.experts", - "language_model.model.layers.38.mlp.experts", - "language_model.model.layers.39.block_sparse_moe.experts", - "language_model.model.layers.39.mlp.experts", - "language_model.model.layers.4.block_sparse_moe.experts", - "language_model.model.layers.4.mlp.experts", - "language_model.model.layers.40.block_sparse_moe.experts", - "language_model.model.layers.40.mlp.experts", - "language_model.model.layers.41.block_sparse_moe.experts", - "language_model.model.layers.41.mlp.experts", - "language_model.model.layers.42.block_sparse_moe.experts", - "language_model.model.layers.42.mlp.experts", - "language_model.model.layers.43.block_sparse_moe.experts", - "language_model.model.layers.43.mlp.experts", - "language_model.model.layers.44.block_sparse_moe.experts", - "language_model.model.layers.44.mlp.experts", - "language_model.model.layers.45.block_sparse_moe.experts", - "language_model.model.layers.45.mlp.experts", - "language_model.model.layers.46.block_sparse_moe.experts", - "language_model.model.layers.46.mlp.experts", - "language_model.model.layers.47.block_sparse_moe.experts", - "language_model.model.layers.47.mlp.experts", - "language_model.model.layers.48.block_sparse_moe.experts", - "language_model.model.layers.48.mlp.experts", - "language_model.model.layers.49.block_sparse_moe.experts", - "language_model.model.layers.49.mlp.experts", - "language_model.model.layers.5.block_sparse_moe.experts", - "language_model.model.layers.5.mlp.experts", - "language_model.model.layers.50.block_sparse_moe.experts", - "language_model.model.layers.50.mlp.experts", - "language_model.model.layers.51.block_sparse_moe.experts", - "language_model.model.layers.51.mlp.experts", - "language_model.model.layers.52.block_sparse_moe.experts", - "language_model.model.layers.52.mlp.experts", - "language_model.model.layers.53.block_sparse_moe.experts", - "language_model.model.layers.53.mlp.experts", - "language_model.model.layers.54.block_sparse_moe.experts", - "language_model.model.layers.54.mlp.experts", - "language_model.model.layers.55.block_sparse_moe.experts", - "language_model.model.layers.55.mlp.experts", - "language_model.model.layers.56.block_sparse_moe.experts", - "language_model.model.layers.56.mlp.experts", - "language_model.model.layers.57.block_sparse_moe.experts", - "language_model.model.layers.57.mlp.experts", - "language_model.model.layers.58.block_sparse_moe.experts", - "language_model.model.layers.58.mlp.experts", - "language_model.model.layers.59.block_sparse_moe.experts", - "language_model.model.layers.59.mlp.experts", - "language_model.model.layers.6.block_sparse_moe.experts", - "language_model.model.layers.6.mlp.experts", - "language_model.model.layers.60.block_sparse_moe.experts", - "language_model.model.layers.60.mlp.experts", - "language_model.model.layers.61.block_sparse_moe.experts", - "language_model.model.layers.61.mlp.experts", - "language_model.model.layers.62.block_sparse_moe.experts", - "language_model.model.layers.62.mlp.experts", - "language_model.model.layers.63.block_sparse_moe.experts", - "language_model.model.layers.63.mlp.experts", - "language_model.model.layers.64.block_sparse_moe.experts", - "language_model.model.layers.64.mlp.experts", - "language_model.model.layers.65.block_sparse_moe.experts", - "language_model.model.layers.65.mlp.experts", - "language_model.model.layers.66.block_sparse_moe.experts", - "language_model.model.layers.66.mlp.experts", - "language_model.model.layers.67.block_sparse_moe.experts", - "language_model.model.layers.67.mlp.experts", - "language_model.model.layers.68.block_sparse_moe.experts", - "language_model.model.layers.68.mlp.experts", - "language_model.model.layers.69.block_sparse_moe.experts", - "language_model.model.layers.69.mlp.experts", - "language_model.model.layers.7.block_sparse_moe.experts", - "language_model.model.layers.7.mlp.experts", - "language_model.model.layers.70.block_sparse_moe.experts", - "language_model.model.layers.70.mlp.experts", - "language_model.model.layers.71.block_sparse_moe.experts", - "language_model.model.layers.71.mlp.experts", - "language_model.model.layers.72.block_sparse_moe.experts", - "language_model.model.layers.72.mlp.experts", - "language_model.model.layers.73.block_sparse_moe.experts", - "language_model.model.layers.73.mlp.experts", - "language_model.model.layers.74.block_sparse_moe.experts", - "language_model.model.layers.74.mlp.experts", - "language_model.model.layers.75.block_sparse_moe.experts", - "language_model.model.layers.75.mlp.experts", - "language_model.model.layers.76.block_sparse_moe.experts", - "language_model.model.layers.76.mlp.experts", - "language_model.model.layers.77.block_sparse_moe.experts", - "language_model.model.layers.77.mlp.experts", - "language_model.model.layers.78.block_sparse_moe.experts", - "language_model.model.layers.78.mlp.experts", - "language_model.model.layers.79.block_sparse_moe.experts", - "language_model.model.layers.79.mlp.experts", - "language_model.model.layers.8.block_sparse_moe.experts", - "language_model.model.layers.8.mlp.experts", - "language_model.model.layers.80.block_sparse_moe.experts", - "language_model.model.layers.80.mlp.experts", - "language_model.model.layers.81.block_sparse_moe.experts", - "language_model.model.layers.81.mlp.experts", - "language_model.model.layers.82.block_sparse_moe.experts", - "language_model.model.layers.82.mlp.experts", - "language_model.model.layers.83.block_sparse_moe.experts", - "language_model.model.layers.83.mlp.experts", - "language_model.model.layers.84.block_sparse_moe.experts", - "language_model.model.layers.84.mlp.experts", - "language_model.model.layers.85.block_sparse_moe.experts", - "language_model.model.layers.85.mlp.experts", - "language_model.model.layers.86.block_sparse_moe.experts", - "language_model.model.layers.86.mlp.experts", - "language_model.model.layers.87.block_sparse_moe.experts", - "language_model.model.layers.87.mlp.experts", - "language_model.model.layers.88.block_sparse_moe.experts", - "language_model.model.layers.88.mlp.experts", - "language_model.model.layers.89.block_sparse_moe.experts", - "language_model.model.layers.89.mlp.experts", - "language_model.model.layers.9.block_sparse_moe.experts", - "language_model.model.layers.9.mlp.experts", - "language_model.model.layers.90.block_sparse_moe.experts", - "language_model.model.layers.90.mlp.experts", - "language_model.model.layers.91.block_sparse_moe.experts", - "language_model.model.layers.91.mlp.experts", - "language_model.model.layers.92.block_sparse_moe.experts", - "language_model.model.layers.92.mlp.experts", - "layers.1.block_sparse_moe.experts", - "layers.1.mlp.experts", - "layers.10.block_sparse_moe.experts", - "layers.10.mlp.experts", - "layers.11.block_sparse_moe.experts", - "layers.11.mlp.experts", - "layers.12.block_sparse_moe.experts", - "layers.12.mlp.experts", - "layers.13.block_sparse_moe.experts", - "layers.13.mlp.experts", - "layers.14.block_sparse_moe.experts", - "layers.14.mlp.experts", - "layers.15.block_sparse_moe.experts", - "layers.15.mlp.experts", - "layers.16.block_sparse_moe.experts", - "layers.16.mlp.experts", - "layers.17.block_sparse_moe.experts", - "layers.17.mlp.experts", - "layers.18.block_sparse_moe.experts", - "layers.18.mlp.experts", - "layers.19.block_sparse_moe.experts", - "layers.19.mlp.experts", - "layers.2.block_sparse_moe.experts", - "layers.2.mlp.experts", - "layers.20.block_sparse_moe.experts", - "layers.20.mlp.experts", - "layers.21.block_sparse_moe.experts", - "layers.21.mlp.experts", - "layers.22.block_sparse_moe.experts", - "layers.22.mlp.experts", - "layers.23.block_sparse_moe.experts", - "layers.23.mlp.experts", - "layers.24.block_sparse_moe.experts", - "layers.24.mlp.experts", - "layers.25.block_sparse_moe.experts", - "layers.25.mlp.experts", - "layers.26.block_sparse_moe.experts", - "layers.26.mlp.experts", - "layers.27.block_sparse_moe.experts", - "layers.27.mlp.experts", - "layers.28.block_sparse_moe.experts", - "layers.28.mlp.experts", - "layers.29.block_sparse_moe.experts", - "layers.29.mlp.experts", - "layers.3.block_sparse_moe.experts", - "layers.3.mlp.experts", - "layers.30.block_sparse_moe.experts", - "layers.30.mlp.experts", - "layers.31.block_sparse_moe.experts", - "layers.31.mlp.experts", - "layers.32.block_sparse_moe.experts", - "layers.32.mlp.experts", - "layers.33.block_sparse_moe.experts", - "layers.33.mlp.experts", - "layers.34.block_sparse_moe.experts", - "layers.34.mlp.experts", - "layers.35.block_sparse_moe.experts", - "layers.35.mlp.experts", - "layers.36.block_sparse_moe.experts", - "layers.36.mlp.experts", - "layers.37.block_sparse_moe.experts", - "layers.37.mlp.experts", - "layers.38.block_sparse_moe.experts", - "layers.38.mlp.experts", - "layers.39.block_sparse_moe.experts", - "layers.39.mlp.experts", - "layers.4.block_sparse_moe.experts", - "layers.4.mlp.experts", - "layers.40.block_sparse_moe.experts", - "layers.40.mlp.experts", - "layers.41.block_sparse_moe.experts", - "layers.41.mlp.experts", - "layers.42.block_sparse_moe.experts", - "layers.42.mlp.experts", - "layers.43.block_sparse_moe.experts", - "layers.43.mlp.experts", - "layers.44.block_sparse_moe.experts", - "layers.44.mlp.experts", - "layers.45.block_sparse_moe.experts", - "layers.45.mlp.experts", - "layers.46.block_sparse_moe.experts", - "layers.46.mlp.experts", - "layers.47.block_sparse_moe.experts", - "layers.47.mlp.experts", - "layers.48.block_sparse_moe.experts", - "layers.48.mlp.experts", - "layers.49.block_sparse_moe.experts", - "layers.49.mlp.experts", - "layers.5.block_sparse_moe.experts", - "layers.5.mlp.experts", - "layers.50.block_sparse_moe.experts", - "layers.50.mlp.experts", - "layers.51.block_sparse_moe.experts", - "layers.51.mlp.experts", - "layers.52.block_sparse_moe.experts", - "layers.52.mlp.experts", - "layers.53.block_sparse_moe.experts", - "layers.53.mlp.experts", - "layers.54.block_sparse_moe.experts", - "layers.54.mlp.experts", - "layers.55.block_sparse_moe.experts", - "layers.55.mlp.experts", - "layers.56.block_sparse_moe.experts", - "layers.56.mlp.experts", - "layers.57.block_sparse_moe.experts", - "layers.57.mlp.experts", - "layers.58.block_sparse_moe.experts", - "layers.58.mlp.experts", - "layers.59.block_sparse_moe.experts", - "layers.59.mlp.experts", - "layers.6.block_sparse_moe.experts", - "layers.6.mlp.experts", - "layers.60.block_sparse_moe.experts", - "layers.60.mlp.experts", - "layers.61.block_sparse_moe.experts", - "layers.61.mlp.experts", - "layers.62.block_sparse_moe.experts", - "layers.62.mlp.experts", - "layers.63.block_sparse_moe.experts", - "layers.63.mlp.experts", - "layers.64.block_sparse_moe.experts", - "layers.64.mlp.experts", - "layers.65.block_sparse_moe.experts", - "layers.65.mlp.experts", - "layers.66.block_sparse_moe.experts", - "layers.66.mlp.experts", - "layers.67.block_sparse_moe.experts", - "layers.67.mlp.experts", - "layers.68.block_sparse_moe.experts", - "layers.68.mlp.experts", - "layers.69.block_sparse_moe.experts", - "layers.69.mlp.experts", - "layers.7.block_sparse_moe.experts", - "layers.7.mlp.experts", - "layers.70.block_sparse_moe.experts", - "layers.70.mlp.experts", - "layers.71.block_sparse_moe.experts", - "layers.71.mlp.experts", - "layers.72.block_sparse_moe.experts", - "layers.72.mlp.experts", - "layers.73.block_sparse_moe.experts", - "layers.73.mlp.experts", - "layers.74.block_sparse_moe.experts", - "layers.74.mlp.experts", - "layers.75.block_sparse_moe.experts", - "layers.75.mlp.experts", - "layers.76.block_sparse_moe.experts", - "layers.76.mlp.experts", - "layers.77.block_sparse_moe.experts", - "layers.77.mlp.experts", - "layers.78.block_sparse_moe.experts", - "layers.78.mlp.experts", - "layers.79.block_sparse_moe.experts", - "layers.79.mlp.experts", - "layers.8.block_sparse_moe.experts", - "layers.8.mlp.experts", - "layers.80.block_sparse_moe.experts", - "layers.80.mlp.experts", - "layers.81.block_sparse_moe.experts", - "layers.81.mlp.experts", - "layers.82.block_sparse_moe.experts", - "layers.82.mlp.experts", - "layers.83.block_sparse_moe.experts", - "layers.83.mlp.experts", - "layers.84.block_sparse_moe.experts", - "layers.84.mlp.experts", - "layers.85.block_sparse_moe.experts", - "layers.85.mlp.experts", - "layers.86.block_sparse_moe.experts", - "layers.86.mlp.experts", - "layers.87.block_sparse_moe.experts", - "layers.87.mlp.experts", - "layers.88.block_sparse_moe.experts", - "layers.88.mlp.experts", - "layers.89.block_sparse_moe.experts", - "layers.89.mlp.experts", - "layers.9.block_sparse_moe.experts", - "layers.9.mlp.experts", - "layers.90.block_sparse_moe.experts", - "layers.90.mlp.experts", - "layers.91.block_sparse_moe.experts", - "layers.91.mlp.experts", - "layers.92.block_sparse_moe.experts", - "layers.92.mlp.experts", - "model.layers.1.block_sparse_moe.experts", - "model.layers.1.mlp.experts", - "model.layers.10.block_sparse_moe.experts", - "model.layers.10.mlp.experts", - "model.layers.11.block_sparse_moe.experts", - "model.layers.11.mlp.experts", - "model.layers.12.block_sparse_moe.experts", - "model.layers.12.mlp.experts", - "model.layers.13.block_sparse_moe.experts", - "model.layers.13.mlp.experts", - "model.layers.14.block_sparse_moe.experts", - "model.layers.14.mlp.experts", - "model.layers.15.block_sparse_moe.experts", - "model.layers.15.mlp.experts", - "model.layers.16.block_sparse_moe.experts", - "model.layers.16.mlp.experts", - "model.layers.17.block_sparse_moe.experts", - "model.layers.17.mlp.experts", - "model.layers.18.block_sparse_moe.experts", - "model.layers.18.mlp.experts", - "model.layers.19.block_sparse_moe.experts", - "model.layers.19.mlp.experts", - "model.layers.2.block_sparse_moe.experts", - "model.layers.2.mlp.experts", - "model.layers.20.block_sparse_moe.experts", - "model.layers.20.mlp.experts", - "model.layers.21.block_sparse_moe.experts", - "model.layers.21.mlp.experts", - "model.layers.22.block_sparse_moe.experts", - "model.layers.22.mlp.experts", - "model.layers.23.block_sparse_moe.experts", - "model.layers.23.mlp.experts", - "model.layers.24.block_sparse_moe.experts", - "model.layers.24.mlp.experts", - "model.layers.25.block_sparse_moe.experts", - "model.layers.25.mlp.experts", - "model.layers.26.block_sparse_moe.experts", - "model.layers.26.mlp.experts", - "model.layers.27.block_sparse_moe.experts", - "model.layers.27.mlp.experts", - "model.layers.28.block_sparse_moe.experts", - "model.layers.28.mlp.experts", - "model.layers.29.block_sparse_moe.experts", - "model.layers.29.mlp.experts", - "model.layers.3.block_sparse_moe.experts", - "model.layers.3.mlp.experts", - "model.layers.30.block_sparse_moe.experts", - "model.layers.30.mlp.experts", - "model.layers.31.block_sparse_moe.experts", - "model.layers.31.mlp.experts", - "model.layers.32.block_sparse_moe.experts", - "model.layers.32.mlp.experts", - "model.layers.33.block_sparse_moe.experts", - "model.layers.33.mlp.experts", - "model.layers.34.block_sparse_moe.experts", - "model.layers.34.mlp.experts", - "model.layers.35.block_sparse_moe.experts", - "model.layers.35.mlp.experts", - "model.layers.36.block_sparse_moe.experts", - "model.layers.36.mlp.experts", - "model.layers.37.block_sparse_moe.experts", - "model.layers.37.mlp.experts", - "model.layers.38.block_sparse_moe.experts", - "model.layers.38.mlp.experts", - "model.layers.39.block_sparse_moe.experts", - "model.layers.39.mlp.experts", - "model.layers.4.block_sparse_moe.experts", - "model.layers.4.mlp.experts", - "model.layers.40.block_sparse_moe.experts", - "model.layers.40.mlp.experts", - "model.layers.41.block_sparse_moe.experts", - "model.layers.41.mlp.experts", - "model.layers.42.block_sparse_moe.experts", - "model.layers.42.mlp.experts", - "model.layers.43.block_sparse_moe.experts", - "model.layers.43.mlp.experts", - "model.layers.44.block_sparse_moe.experts", - "model.layers.44.mlp.experts", - "model.layers.45.block_sparse_moe.experts", - "model.layers.45.mlp.experts", - "model.layers.46.block_sparse_moe.experts", - "model.layers.46.mlp.experts", - "model.layers.47.block_sparse_moe.experts", - "model.layers.47.mlp.experts", - "model.layers.48.block_sparse_moe.experts", - "model.layers.48.mlp.experts", - "model.layers.49.block_sparse_moe.experts", - "model.layers.49.mlp.experts", - "model.layers.5.block_sparse_moe.experts", - "model.layers.5.mlp.experts", - "model.layers.50.block_sparse_moe.experts", - "model.layers.50.mlp.experts", - "model.layers.51.block_sparse_moe.experts", - "model.layers.51.mlp.experts", - "model.layers.52.block_sparse_moe.experts", - "model.layers.52.mlp.experts", - "model.layers.53.block_sparse_moe.experts", - "model.layers.53.mlp.experts", - "model.layers.54.block_sparse_moe.experts", - "model.layers.54.mlp.experts", - "model.layers.55.block_sparse_moe.experts", - "model.layers.55.mlp.experts", - "model.layers.56.block_sparse_moe.experts", - "model.layers.56.mlp.experts", - "model.layers.57.block_sparse_moe.experts", - "model.layers.57.mlp.experts", - "model.layers.58.block_sparse_moe.experts", - "model.layers.58.mlp.experts", - "model.layers.59.block_sparse_moe.experts", - "model.layers.59.mlp.experts", - "model.layers.6.block_sparse_moe.experts", - "model.layers.6.mlp.experts", - "model.layers.60.block_sparse_moe.experts", - "model.layers.60.mlp.experts", - "model.layers.61.block_sparse_moe.experts", - "model.layers.61.mlp.experts", - "model.layers.62.block_sparse_moe.experts", - "model.layers.62.mlp.experts", - "model.layers.63.block_sparse_moe.experts", - "model.layers.63.mlp.experts", - "model.layers.64.block_sparse_moe.experts", - "model.layers.64.mlp.experts", - "model.layers.65.block_sparse_moe.experts", - "model.layers.65.mlp.experts", - "model.layers.66.block_sparse_moe.experts", - "model.layers.66.mlp.experts", - "model.layers.67.block_sparse_moe.experts", - "model.layers.67.mlp.experts", - "model.layers.68.block_sparse_moe.experts", - "model.layers.68.mlp.experts", - "model.layers.69.block_sparse_moe.experts", - "model.layers.69.mlp.experts", - "model.layers.7.block_sparse_moe.experts", - "model.layers.7.mlp.experts", - "model.layers.70.block_sparse_moe.experts", - "model.layers.70.mlp.experts", - "model.layers.71.block_sparse_moe.experts", - "model.layers.71.mlp.experts", - "model.layers.72.block_sparse_moe.experts", - "model.layers.72.mlp.experts", - "model.layers.73.block_sparse_moe.experts", - "model.layers.73.mlp.experts", - "model.layers.74.block_sparse_moe.experts", - "model.layers.74.mlp.experts", - "model.layers.75.block_sparse_moe.experts", - "model.layers.75.mlp.experts", - "model.layers.76.block_sparse_moe.experts", - "model.layers.76.mlp.experts", - "model.layers.77.block_sparse_moe.experts", - "model.layers.77.mlp.experts", - "model.layers.78.block_sparse_moe.experts", - "model.layers.78.mlp.experts", - "model.layers.79.block_sparse_moe.experts", - "model.layers.79.mlp.experts", - "model.layers.8.block_sparse_moe.experts", - "model.layers.8.mlp.experts", - "model.layers.80.block_sparse_moe.experts", - "model.layers.80.mlp.experts", - "model.layers.81.block_sparse_moe.experts", - "model.layers.81.mlp.experts", - "model.layers.82.block_sparse_moe.experts", - "model.layers.82.mlp.experts", - "model.layers.83.block_sparse_moe.experts", - "model.layers.83.mlp.experts", - "model.layers.84.block_sparse_moe.experts", - "model.layers.84.mlp.experts", - "model.layers.85.block_sparse_moe.experts", - "model.layers.85.mlp.experts", - "model.layers.86.block_sparse_moe.experts", - "model.layers.86.mlp.experts", - "model.layers.87.block_sparse_moe.experts", - "model.layers.87.mlp.experts", - "model.layers.88.block_sparse_moe.experts", - "model.layers.88.mlp.experts", - "model.layers.89.block_sparse_moe.experts", - "model.layers.89.mlp.experts", - "model.layers.9.block_sparse_moe.experts", - "model.layers.9.mlp.experts", - "model.layers.90.block_sparse_moe.experts", - "model.layers.90.mlp.experts", - "model.layers.91.block_sparse_moe.experts", - "model.layers.91.mlp.experts", - "model.layers.92.block_sparse_moe.experts", - "model.layers.92.mlp.experts" - ], - "weights": { - "dynamic": false, - "group_size": 16, - "num_bits": 4, - "type": "float" - } - }, - "group_1": { - "quant_algo": "FP8_PB_WO", - "targets": [ - "language_model.model.layers.0.self_attn.b_proj", - "language_model.model.layers.0.self_attn.f_a_proj", - "language_model.model.layers.0.self_attn.f_b_proj", - "language_model.model.layers.0.self_attn.fused_qkvg_proj", - "language_model.model.layers.0.self_attn.g_proj", - "language_model.model.layers.0.self_attn.in_proj_qkvgfab", - "language_model.model.layers.0.self_attn.k_proj", - "language_model.model.layers.0.self_attn.o_proj", - "language_model.model.layers.0.self_attn.q_proj", - "language_model.model.layers.0.self_attn.v_proj", - "language_model.model.layers.1.self_attn.b_proj", - "language_model.model.layers.1.self_attn.f_a_proj", - "language_model.model.layers.1.self_attn.f_b_proj", - "language_model.model.layers.1.self_attn.fused_qkvg_proj", - "language_model.model.layers.1.self_attn.g_proj", - "language_model.model.layers.1.self_attn.in_proj_qkvgfab", - "language_model.model.layers.1.self_attn.k_proj", - "language_model.model.layers.1.self_attn.o_proj", - "language_model.model.layers.1.self_attn.q_proj", - "language_model.model.layers.1.self_attn.v_proj", - "language_model.model.layers.10.self_attn.b_proj", - "language_model.model.layers.10.self_attn.f_a_proj", - "language_model.model.layers.10.self_attn.f_b_proj", - "language_model.model.layers.10.self_attn.fused_qkvg_proj", - "language_model.model.layers.10.self_attn.g_proj", - "language_model.model.layers.10.self_attn.in_proj_qkvgfab", - "language_model.model.layers.10.self_attn.k_proj", - "language_model.model.layers.10.self_attn.o_proj", - "language_model.model.layers.10.self_attn.q_proj", - "language_model.model.layers.10.self_attn.v_proj", - "language_model.model.layers.11.self_attn.fused_qkv_a_proj", - "language_model.model.layers.11.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.11.self_attn.fused_qkvg_proj", - "language_model.model.layers.11.self_attn.g_proj", - "language_model.model.layers.11.self_attn.in_proj_qkvgfab", - "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.11.self_attn.kv_b_proj", - "language_model.model.layers.11.self_attn.o_proj", - "language_model.model.layers.11.self_attn.q_a_proj", - "language_model.model.layers.11.self_attn.q_b_proj", - "language_model.model.layers.12.self_attn.b_proj", - "language_model.model.layers.12.self_attn.f_a_proj", - "language_model.model.layers.12.self_attn.f_b_proj", - "language_model.model.layers.12.self_attn.fused_qkvg_proj", - "language_model.model.layers.12.self_attn.g_proj", - "language_model.model.layers.12.self_attn.in_proj_qkvgfab", - "language_model.model.layers.12.self_attn.k_proj", - "language_model.model.layers.12.self_attn.o_proj", - "language_model.model.layers.12.self_attn.q_proj", - "language_model.model.layers.12.self_attn.v_proj", - "language_model.model.layers.13.self_attn.b_proj", - "language_model.model.layers.13.self_attn.f_a_proj", - "language_model.model.layers.13.self_attn.f_b_proj", - "language_model.model.layers.13.self_attn.fused_qkvg_proj", - "language_model.model.layers.13.self_attn.g_proj", - "language_model.model.layers.13.self_attn.in_proj_qkvgfab", - "language_model.model.layers.13.self_attn.k_proj", - "language_model.model.layers.13.self_attn.o_proj", - "language_model.model.layers.13.self_attn.q_proj", - "language_model.model.layers.13.self_attn.v_proj", - "language_model.model.layers.14.self_attn.b_proj", - "language_model.model.layers.14.self_attn.f_a_proj", - "language_model.model.layers.14.self_attn.f_b_proj", - "language_model.model.layers.14.self_attn.fused_qkvg_proj", - "language_model.model.layers.14.self_attn.g_proj", - "language_model.model.layers.14.self_attn.in_proj_qkvgfab", - "language_model.model.layers.14.self_attn.k_proj", - "language_model.model.layers.14.self_attn.o_proj", - "language_model.model.layers.14.self_attn.q_proj", - "language_model.model.layers.14.self_attn.v_proj", - "language_model.model.layers.15.self_attn.fused_qkv_a_proj", - "language_model.model.layers.15.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.15.self_attn.fused_qkvg_proj", - "language_model.model.layers.15.self_attn.g_proj", - "language_model.model.layers.15.self_attn.in_proj_qkvgfab", - "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.15.self_attn.kv_b_proj", - "language_model.model.layers.15.self_attn.o_proj", - "language_model.model.layers.15.self_attn.q_a_proj", - "language_model.model.layers.15.self_attn.q_b_proj", - "language_model.model.layers.16.self_attn.b_proj", - "language_model.model.layers.16.self_attn.f_a_proj", - "language_model.model.layers.16.self_attn.f_b_proj", - "language_model.model.layers.16.self_attn.fused_qkvg_proj", - "language_model.model.layers.16.self_attn.g_proj", - "language_model.model.layers.16.self_attn.in_proj_qkvgfab", - "language_model.model.layers.16.self_attn.k_proj", - "language_model.model.layers.16.self_attn.o_proj", - "language_model.model.layers.16.self_attn.q_proj", - "language_model.model.layers.16.self_attn.v_proj", - "language_model.model.layers.17.self_attn.b_proj", - "language_model.model.layers.17.self_attn.f_a_proj", - "language_model.model.layers.17.self_attn.f_b_proj", - "language_model.model.layers.17.self_attn.fused_qkvg_proj", - "language_model.model.layers.17.self_attn.g_proj", - "language_model.model.layers.17.self_attn.in_proj_qkvgfab", - "language_model.model.layers.17.self_attn.k_proj", - "language_model.model.layers.17.self_attn.o_proj", - "language_model.model.layers.17.self_attn.q_proj", - "language_model.model.layers.17.self_attn.v_proj", - "language_model.model.layers.18.self_attn.b_proj", - "language_model.model.layers.18.self_attn.f_a_proj", - "language_model.model.layers.18.self_attn.f_b_proj", - "language_model.model.layers.18.self_attn.fused_qkvg_proj", - "language_model.model.layers.18.self_attn.g_proj", - "language_model.model.layers.18.self_attn.in_proj_qkvgfab", - "language_model.model.layers.18.self_attn.k_proj", - "language_model.model.layers.18.self_attn.o_proj", - "language_model.model.layers.18.self_attn.q_proj", - "language_model.model.layers.18.self_attn.v_proj", - "language_model.model.layers.19.self_attn.fused_qkv_a_proj", - "language_model.model.layers.19.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.19.self_attn.fused_qkvg_proj", - "language_model.model.layers.19.self_attn.g_proj", - "language_model.model.layers.19.self_attn.in_proj_qkvgfab", - "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.19.self_attn.kv_b_proj", - "language_model.model.layers.19.self_attn.o_proj", - "language_model.model.layers.19.self_attn.q_a_proj", - "language_model.model.layers.19.self_attn.q_b_proj", - "language_model.model.layers.2.self_attn.b_proj", - "language_model.model.layers.2.self_attn.f_a_proj", - "language_model.model.layers.2.self_attn.f_b_proj", - "language_model.model.layers.2.self_attn.fused_qkvg_proj", - "language_model.model.layers.2.self_attn.g_proj", - "language_model.model.layers.2.self_attn.in_proj_qkvgfab", - "language_model.model.layers.2.self_attn.k_proj", - "language_model.model.layers.2.self_attn.o_proj", - "language_model.model.layers.2.self_attn.q_proj", - "language_model.model.layers.2.self_attn.v_proj", - "language_model.model.layers.20.self_attn.b_proj", - "language_model.model.layers.20.self_attn.f_a_proj", - "language_model.model.layers.20.self_attn.f_b_proj", - "language_model.model.layers.20.self_attn.fused_qkvg_proj", - "language_model.model.layers.20.self_attn.g_proj", - "language_model.model.layers.20.self_attn.in_proj_qkvgfab", - "language_model.model.layers.20.self_attn.k_proj", - "language_model.model.layers.20.self_attn.o_proj", - "language_model.model.layers.20.self_attn.q_proj", - "language_model.model.layers.20.self_attn.v_proj", - "language_model.model.layers.21.self_attn.b_proj", - "language_model.model.layers.21.self_attn.f_a_proj", - "language_model.model.layers.21.self_attn.f_b_proj", - "language_model.model.layers.21.self_attn.fused_qkvg_proj", - "language_model.model.layers.21.self_attn.g_proj", - "language_model.model.layers.21.self_attn.in_proj_qkvgfab", - "language_model.model.layers.21.self_attn.k_proj", - "language_model.model.layers.21.self_attn.o_proj", - "language_model.model.layers.21.self_attn.q_proj", - "language_model.model.layers.21.self_attn.v_proj", - "language_model.model.layers.22.self_attn.b_proj", - "language_model.model.layers.22.self_attn.f_a_proj", - "language_model.model.layers.22.self_attn.f_b_proj", - "language_model.model.layers.22.self_attn.fused_qkvg_proj", - "language_model.model.layers.22.self_attn.g_proj", - "language_model.model.layers.22.self_attn.in_proj_qkvgfab", - "language_model.model.layers.22.self_attn.k_proj", - "language_model.model.layers.22.self_attn.o_proj", - "language_model.model.layers.22.self_attn.q_proj", - "language_model.model.layers.22.self_attn.v_proj", - "language_model.model.layers.23.self_attn.fused_qkv_a_proj", - "language_model.model.layers.23.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.23.self_attn.fused_qkvg_proj", - "language_model.model.layers.23.self_attn.g_proj", - "language_model.model.layers.23.self_attn.in_proj_qkvgfab", - "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.23.self_attn.kv_b_proj", - "language_model.model.layers.23.self_attn.o_proj", - "language_model.model.layers.23.self_attn.q_a_proj", - "language_model.model.layers.23.self_attn.q_b_proj", - "language_model.model.layers.24.self_attn.b_proj", - "language_model.model.layers.24.self_attn.f_a_proj", - "language_model.model.layers.24.self_attn.f_b_proj", - "language_model.model.layers.24.self_attn.fused_qkvg_proj", - "language_model.model.layers.24.self_attn.g_proj", - "language_model.model.layers.24.self_attn.in_proj_qkvgfab", - "language_model.model.layers.24.self_attn.k_proj", - "language_model.model.layers.24.self_attn.o_proj", - "language_model.model.layers.24.self_attn.q_proj", - "language_model.model.layers.24.self_attn.v_proj", - "language_model.model.layers.25.self_attn.b_proj", - "language_model.model.layers.25.self_attn.f_a_proj", - "language_model.model.layers.25.self_attn.f_b_proj", - "language_model.model.layers.25.self_attn.fused_qkvg_proj", - "language_model.model.layers.25.self_attn.g_proj", - "language_model.model.layers.25.self_attn.in_proj_qkvgfab", - "language_model.model.layers.25.self_attn.k_proj", - "language_model.model.layers.25.self_attn.o_proj", - "language_model.model.layers.25.self_attn.q_proj", - "language_model.model.layers.25.self_attn.v_proj", - "language_model.model.layers.26.self_attn.b_proj", - "language_model.model.layers.26.self_attn.f_a_proj", - "language_model.model.layers.26.self_attn.f_b_proj", - "language_model.model.layers.26.self_attn.fused_qkvg_proj", - "language_model.model.layers.26.self_attn.g_proj", - "language_model.model.layers.26.self_attn.in_proj_qkvgfab", - "language_model.model.layers.26.self_attn.k_proj", - "language_model.model.layers.26.self_attn.o_proj", - "language_model.model.layers.26.self_attn.q_proj", - "language_model.model.layers.26.self_attn.v_proj", - "language_model.model.layers.27.self_attn.fused_qkv_a_proj", - "language_model.model.layers.27.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.27.self_attn.fused_qkvg_proj", - "language_model.model.layers.27.self_attn.g_proj", - "language_model.model.layers.27.self_attn.in_proj_qkvgfab", - "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.27.self_attn.kv_b_proj", - "language_model.model.layers.27.self_attn.o_proj", - "language_model.model.layers.27.self_attn.q_a_proj", - "language_model.model.layers.27.self_attn.q_b_proj", - "language_model.model.layers.28.self_attn.b_proj", - "language_model.model.layers.28.self_attn.f_a_proj", - "language_model.model.layers.28.self_attn.f_b_proj", - "language_model.model.layers.28.self_attn.fused_qkvg_proj", - "language_model.model.layers.28.self_attn.g_proj", - "language_model.model.layers.28.self_attn.in_proj_qkvgfab", - "language_model.model.layers.28.self_attn.k_proj", - "language_model.model.layers.28.self_attn.o_proj", - "language_model.model.layers.28.self_attn.q_proj", - "language_model.model.layers.28.self_attn.v_proj", - "language_model.model.layers.29.self_attn.b_proj", - "language_model.model.layers.29.self_attn.f_a_proj", - "language_model.model.layers.29.self_attn.f_b_proj", - "language_model.model.layers.29.self_attn.fused_qkvg_proj", - "language_model.model.layers.29.self_attn.g_proj", - "language_model.model.layers.29.self_attn.in_proj_qkvgfab", - "language_model.model.layers.29.self_attn.k_proj", - "language_model.model.layers.29.self_attn.o_proj", - "language_model.model.layers.29.self_attn.q_proj", - "language_model.model.layers.29.self_attn.v_proj", - "language_model.model.layers.3.self_attn.fused_qkv_a_proj", - "language_model.model.layers.3.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.3.self_attn.fused_qkvg_proj", - "language_model.model.layers.3.self_attn.g_proj", - "language_model.model.layers.3.self_attn.in_proj_qkvgfab", - "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.3.self_attn.kv_b_proj", - "language_model.model.layers.3.self_attn.o_proj", - "language_model.model.layers.3.self_attn.q_a_proj", - "language_model.model.layers.3.self_attn.q_b_proj", - "language_model.model.layers.30.self_attn.b_proj", - "language_model.model.layers.30.self_attn.f_a_proj", - "language_model.model.layers.30.self_attn.f_b_proj", - "language_model.model.layers.30.self_attn.fused_qkvg_proj", - "language_model.model.layers.30.self_attn.g_proj", - "language_model.model.layers.30.self_attn.in_proj_qkvgfab", - "language_model.model.layers.30.self_attn.k_proj", - "language_model.model.layers.30.self_attn.o_proj", - "language_model.model.layers.30.self_attn.q_proj", - "language_model.model.layers.30.self_attn.v_proj", - "language_model.model.layers.31.self_attn.fused_qkv_a_proj", - "language_model.model.layers.31.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.31.self_attn.fused_qkvg_proj", - "language_model.model.layers.31.self_attn.g_proj", - "language_model.model.layers.31.self_attn.in_proj_qkvgfab", - "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.31.self_attn.kv_b_proj", - "language_model.model.layers.31.self_attn.o_proj", - "language_model.model.layers.31.self_attn.q_a_proj", - "language_model.model.layers.31.self_attn.q_b_proj", - "language_model.model.layers.32.self_attn.b_proj", - "language_model.model.layers.32.self_attn.f_a_proj", - "language_model.model.layers.32.self_attn.f_b_proj", - "language_model.model.layers.32.self_attn.fused_qkvg_proj", - "language_model.model.layers.32.self_attn.g_proj", - "language_model.model.layers.32.self_attn.in_proj_qkvgfab", - "language_model.model.layers.32.self_attn.k_proj", - "language_model.model.layers.32.self_attn.o_proj", - "language_model.model.layers.32.self_attn.q_proj", - "language_model.model.layers.32.self_attn.v_proj", - "language_model.model.layers.33.self_attn.b_proj", - "language_model.model.layers.33.self_attn.f_a_proj", - "language_model.model.layers.33.self_attn.f_b_proj", - "language_model.model.layers.33.self_attn.fused_qkvg_proj", - "language_model.model.layers.33.self_attn.g_proj", - "language_model.model.layers.33.self_attn.in_proj_qkvgfab", - "language_model.model.layers.33.self_attn.k_proj", - "language_model.model.layers.33.self_attn.o_proj", - "language_model.model.layers.33.self_attn.q_proj", - "language_model.model.layers.33.self_attn.v_proj", - "language_model.model.layers.34.self_attn.b_proj", - "language_model.model.layers.34.self_attn.f_a_proj", - "language_model.model.layers.34.self_attn.f_b_proj", - "language_model.model.layers.34.self_attn.fused_qkvg_proj", - "language_model.model.layers.34.self_attn.g_proj", - "language_model.model.layers.34.self_attn.in_proj_qkvgfab", - "language_model.model.layers.34.self_attn.k_proj", - "language_model.model.layers.34.self_attn.o_proj", - "language_model.model.layers.34.self_attn.q_proj", - "language_model.model.layers.34.self_attn.v_proj", - "language_model.model.layers.35.self_attn.fused_qkv_a_proj", - "language_model.model.layers.35.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.35.self_attn.fused_qkvg_proj", - "language_model.model.layers.35.self_attn.g_proj", - "language_model.model.layers.35.self_attn.in_proj_qkvgfab", - "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.35.self_attn.kv_b_proj", - "language_model.model.layers.35.self_attn.o_proj", - "language_model.model.layers.35.self_attn.q_a_proj", - "language_model.model.layers.35.self_attn.q_b_proj", - "language_model.model.layers.36.self_attn.b_proj", - "language_model.model.layers.36.self_attn.f_a_proj", - "language_model.model.layers.36.self_attn.f_b_proj", - "language_model.model.layers.36.self_attn.fused_qkvg_proj", - "language_model.model.layers.36.self_attn.g_proj", - "language_model.model.layers.36.self_attn.in_proj_qkvgfab", - "language_model.model.layers.36.self_attn.k_proj", - "language_model.model.layers.36.self_attn.o_proj", - "language_model.model.layers.36.self_attn.q_proj", - "language_model.model.layers.36.self_attn.v_proj", - "language_model.model.layers.37.self_attn.b_proj", - "language_model.model.layers.37.self_attn.f_a_proj", - "language_model.model.layers.37.self_attn.f_b_proj", - "language_model.model.layers.37.self_attn.fused_qkvg_proj", - "language_model.model.layers.37.self_attn.g_proj", - "language_model.model.layers.37.self_attn.in_proj_qkvgfab", - "language_model.model.layers.37.self_attn.k_proj", - "language_model.model.layers.37.self_attn.o_proj", - "language_model.model.layers.37.self_attn.q_proj", - "language_model.model.layers.37.self_attn.v_proj", - "language_model.model.layers.38.self_attn.b_proj", - "language_model.model.layers.38.self_attn.f_a_proj", - "language_model.model.layers.38.self_attn.f_b_proj", - "language_model.model.layers.38.self_attn.fused_qkvg_proj", - "language_model.model.layers.38.self_attn.g_proj", - "language_model.model.layers.38.self_attn.in_proj_qkvgfab", - "language_model.model.layers.38.self_attn.k_proj", - "language_model.model.layers.38.self_attn.o_proj", - "language_model.model.layers.38.self_attn.q_proj", - "language_model.model.layers.38.self_attn.v_proj", - "language_model.model.layers.39.self_attn.fused_qkv_a_proj", - "language_model.model.layers.39.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.39.self_attn.fused_qkvg_proj", - "language_model.model.layers.39.self_attn.g_proj", - "language_model.model.layers.39.self_attn.in_proj_qkvgfab", - "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.39.self_attn.kv_b_proj", - "language_model.model.layers.39.self_attn.o_proj", - "language_model.model.layers.39.self_attn.q_a_proj", - "language_model.model.layers.39.self_attn.q_b_proj", - "language_model.model.layers.4.self_attn.b_proj", - "language_model.model.layers.4.self_attn.f_a_proj", - "language_model.model.layers.4.self_attn.f_b_proj", - "language_model.model.layers.4.self_attn.fused_qkvg_proj", - "language_model.model.layers.4.self_attn.g_proj", - "language_model.model.layers.4.self_attn.in_proj_qkvgfab", - "language_model.model.layers.4.self_attn.k_proj", - "language_model.model.layers.4.self_attn.o_proj", - "language_model.model.layers.4.self_attn.q_proj", - "language_model.model.layers.4.self_attn.v_proj", - "language_model.model.layers.40.self_attn.b_proj", - "language_model.model.layers.40.self_attn.f_a_proj", - "language_model.model.layers.40.self_attn.f_b_proj", - "language_model.model.layers.40.self_attn.fused_qkvg_proj", - "language_model.model.layers.40.self_attn.g_proj", - "language_model.model.layers.40.self_attn.in_proj_qkvgfab", - "language_model.model.layers.40.self_attn.k_proj", - "language_model.model.layers.40.self_attn.o_proj", - "language_model.model.layers.40.self_attn.q_proj", - "language_model.model.layers.40.self_attn.v_proj", - "language_model.model.layers.41.self_attn.b_proj", - "language_model.model.layers.41.self_attn.f_a_proj", - "language_model.model.layers.41.self_attn.f_b_proj", - "language_model.model.layers.41.self_attn.fused_qkvg_proj", - "language_model.model.layers.41.self_attn.g_proj", - "language_model.model.layers.41.self_attn.in_proj_qkvgfab", - "language_model.model.layers.41.self_attn.k_proj", - "language_model.model.layers.41.self_attn.o_proj", - "language_model.model.layers.41.self_attn.q_proj", - "language_model.model.layers.41.self_attn.v_proj", - "language_model.model.layers.42.self_attn.b_proj", - "language_model.model.layers.42.self_attn.f_a_proj", - "language_model.model.layers.42.self_attn.f_b_proj", - "language_model.model.layers.42.self_attn.fused_qkvg_proj", - "language_model.model.layers.42.self_attn.g_proj", - "language_model.model.layers.42.self_attn.in_proj_qkvgfab", - "language_model.model.layers.42.self_attn.k_proj", - "language_model.model.layers.42.self_attn.o_proj", - "language_model.model.layers.42.self_attn.q_proj", - "language_model.model.layers.42.self_attn.v_proj", - "language_model.model.layers.43.self_attn.fused_qkv_a_proj", - "language_model.model.layers.43.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.43.self_attn.fused_qkvg_proj", - "language_model.model.layers.43.self_attn.g_proj", - "language_model.model.layers.43.self_attn.in_proj_qkvgfab", - "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.43.self_attn.kv_b_proj", - "language_model.model.layers.43.self_attn.o_proj", - "language_model.model.layers.43.self_attn.q_a_proj", - "language_model.model.layers.43.self_attn.q_b_proj", - "language_model.model.layers.44.self_attn.b_proj", - "language_model.model.layers.44.self_attn.f_a_proj", - "language_model.model.layers.44.self_attn.f_b_proj", - "language_model.model.layers.44.self_attn.fused_qkvg_proj", - "language_model.model.layers.44.self_attn.g_proj", - "language_model.model.layers.44.self_attn.in_proj_qkvgfab", - "language_model.model.layers.44.self_attn.k_proj", - "language_model.model.layers.44.self_attn.o_proj", - "language_model.model.layers.44.self_attn.q_proj", - "language_model.model.layers.44.self_attn.v_proj", - "language_model.model.layers.45.self_attn.b_proj", - "language_model.model.layers.45.self_attn.f_a_proj", - "language_model.model.layers.45.self_attn.f_b_proj", - "language_model.model.layers.45.self_attn.fused_qkvg_proj", - "language_model.model.layers.45.self_attn.g_proj", - "language_model.model.layers.45.self_attn.in_proj_qkvgfab", - "language_model.model.layers.45.self_attn.k_proj", - "language_model.model.layers.45.self_attn.o_proj", - "language_model.model.layers.45.self_attn.q_proj", - "language_model.model.layers.45.self_attn.v_proj", - "language_model.model.layers.46.self_attn.b_proj", - "language_model.model.layers.46.self_attn.f_a_proj", - "language_model.model.layers.46.self_attn.f_b_proj", - "language_model.model.layers.46.self_attn.fused_qkvg_proj", - "language_model.model.layers.46.self_attn.g_proj", - "language_model.model.layers.46.self_attn.in_proj_qkvgfab", - "language_model.model.layers.46.self_attn.k_proj", - "language_model.model.layers.46.self_attn.o_proj", - "language_model.model.layers.46.self_attn.q_proj", - "language_model.model.layers.46.self_attn.v_proj", - "language_model.model.layers.47.self_attn.fused_qkv_a_proj", - "language_model.model.layers.47.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.47.self_attn.fused_qkvg_proj", - "language_model.model.layers.47.self_attn.g_proj", - "language_model.model.layers.47.self_attn.in_proj_qkvgfab", - "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.47.self_attn.kv_b_proj", - "language_model.model.layers.47.self_attn.o_proj", - "language_model.model.layers.47.self_attn.q_a_proj", - "language_model.model.layers.47.self_attn.q_b_proj", - "language_model.model.layers.48.self_attn.b_proj", - "language_model.model.layers.48.self_attn.f_a_proj", - "language_model.model.layers.48.self_attn.f_b_proj", - "language_model.model.layers.48.self_attn.fused_qkvg_proj", - "language_model.model.layers.48.self_attn.g_proj", - "language_model.model.layers.48.self_attn.in_proj_qkvgfab", - "language_model.model.layers.48.self_attn.k_proj", - "language_model.model.layers.48.self_attn.o_proj", - "language_model.model.layers.48.self_attn.q_proj", - "language_model.model.layers.48.self_attn.v_proj", - "language_model.model.layers.49.self_attn.b_proj", - "language_model.model.layers.49.self_attn.f_a_proj", - "language_model.model.layers.49.self_attn.f_b_proj", - "language_model.model.layers.49.self_attn.fused_qkvg_proj", - "language_model.model.layers.49.self_attn.g_proj", - "language_model.model.layers.49.self_attn.in_proj_qkvgfab", - "language_model.model.layers.49.self_attn.k_proj", - "language_model.model.layers.49.self_attn.o_proj", - "language_model.model.layers.49.self_attn.q_proj", - "language_model.model.layers.49.self_attn.v_proj", - "language_model.model.layers.5.self_attn.b_proj", - "language_model.model.layers.5.self_attn.f_a_proj", - "language_model.model.layers.5.self_attn.f_b_proj", - "language_model.model.layers.5.self_attn.fused_qkvg_proj", - "language_model.model.layers.5.self_attn.g_proj", - "language_model.model.layers.5.self_attn.in_proj_qkvgfab", - "language_model.model.layers.5.self_attn.k_proj", - "language_model.model.layers.5.self_attn.o_proj", - "language_model.model.layers.5.self_attn.q_proj", - "language_model.model.layers.5.self_attn.v_proj", - "language_model.model.layers.50.self_attn.b_proj", - "language_model.model.layers.50.self_attn.f_a_proj", - "language_model.model.layers.50.self_attn.f_b_proj", - "language_model.model.layers.50.self_attn.fused_qkvg_proj", - "language_model.model.layers.50.self_attn.g_proj", - "language_model.model.layers.50.self_attn.in_proj_qkvgfab", - "language_model.model.layers.50.self_attn.k_proj", - "language_model.model.layers.50.self_attn.o_proj", - "language_model.model.layers.50.self_attn.q_proj", - "language_model.model.layers.50.self_attn.v_proj", - "language_model.model.layers.51.self_attn.fused_qkv_a_proj", - "language_model.model.layers.51.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.51.self_attn.fused_qkvg_proj", - "language_model.model.layers.51.self_attn.g_proj", - "language_model.model.layers.51.self_attn.in_proj_qkvgfab", - "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.51.self_attn.kv_b_proj", - "language_model.model.layers.51.self_attn.o_proj", - "language_model.model.layers.51.self_attn.q_a_proj", - "language_model.model.layers.51.self_attn.q_b_proj", - "language_model.model.layers.52.self_attn.b_proj", - "language_model.model.layers.52.self_attn.f_a_proj", - "language_model.model.layers.52.self_attn.f_b_proj", - "language_model.model.layers.52.self_attn.fused_qkvg_proj", - "language_model.model.layers.52.self_attn.g_proj", - "language_model.model.layers.52.self_attn.in_proj_qkvgfab", - "language_model.model.layers.52.self_attn.k_proj", - "language_model.model.layers.52.self_attn.o_proj", - "language_model.model.layers.52.self_attn.q_proj", - "language_model.model.layers.52.self_attn.v_proj", - "language_model.model.layers.53.self_attn.b_proj", - "language_model.model.layers.53.self_attn.f_a_proj", - "language_model.model.layers.53.self_attn.f_b_proj", - "language_model.model.layers.53.self_attn.fused_qkvg_proj", - "language_model.model.layers.53.self_attn.g_proj", - "language_model.model.layers.53.self_attn.in_proj_qkvgfab", - "language_model.model.layers.53.self_attn.k_proj", - "language_model.model.layers.53.self_attn.o_proj", - "language_model.model.layers.53.self_attn.q_proj", - "language_model.model.layers.53.self_attn.v_proj", - "language_model.model.layers.54.self_attn.b_proj", - "language_model.model.layers.54.self_attn.f_a_proj", - "language_model.model.layers.54.self_attn.f_b_proj", - "language_model.model.layers.54.self_attn.fused_qkvg_proj", - "language_model.model.layers.54.self_attn.g_proj", - "language_model.model.layers.54.self_attn.in_proj_qkvgfab", - "language_model.model.layers.54.self_attn.k_proj", - "language_model.model.layers.54.self_attn.o_proj", - "language_model.model.layers.54.self_attn.q_proj", - "language_model.model.layers.54.self_attn.v_proj", - "language_model.model.layers.55.self_attn.fused_qkv_a_proj", - "language_model.model.layers.55.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.55.self_attn.fused_qkvg_proj", - "language_model.model.layers.55.self_attn.g_proj", - "language_model.model.layers.55.self_attn.in_proj_qkvgfab", - "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.55.self_attn.kv_b_proj", - "language_model.model.layers.55.self_attn.o_proj", - "language_model.model.layers.55.self_attn.q_a_proj", - "language_model.model.layers.55.self_attn.q_b_proj", - "language_model.model.layers.56.self_attn.b_proj", - "language_model.model.layers.56.self_attn.f_a_proj", - "language_model.model.layers.56.self_attn.f_b_proj", - "language_model.model.layers.56.self_attn.fused_qkvg_proj", - "language_model.model.layers.56.self_attn.g_proj", - "language_model.model.layers.56.self_attn.in_proj_qkvgfab", - "language_model.model.layers.56.self_attn.k_proj", - "language_model.model.layers.56.self_attn.o_proj", - "language_model.model.layers.56.self_attn.q_proj", - "language_model.model.layers.56.self_attn.v_proj", - "language_model.model.layers.57.self_attn.b_proj", - "language_model.model.layers.57.self_attn.f_a_proj", - "language_model.model.layers.57.self_attn.f_b_proj", - "language_model.model.layers.57.self_attn.fused_qkvg_proj", - "language_model.model.layers.57.self_attn.g_proj", - "language_model.model.layers.57.self_attn.in_proj_qkvgfab", - "language_model.model.layers.57.self_attn.k_proj", - "language_model.model.layers.57.self_attn.o_proj", - "language_model.model.layers.57.self_attn.q_proj", - "language_model.model.layers.57.self_attn.v_proj", - "language_model.model.layers.58.self_attn.b_proj", - "language_model.model.layers.58.self_attn.f_a_proj", - "language_model.model.layers.58.self_attn.f_b_proj", - "language_model.model.layers.58.self_attn.fused_qkvg_proj", - "language_model.model.layers.58.self_attn.g_proj", - "language_model.model.layers.58.self_attn.in_proj_qkvgfab", - "language_model.model.layers.58.self_attn.k_proj", - "language_model.model.layers.58.self_attn.o_proj", - "language_model.model.layers.58.self_attn.q_proj", - "language_model.model.layers.58.self_attn.v_proj", - "language_model.model.layers.59.self_attn.fused_qkv_a_proj", - "language_model.model.layers.59.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.59.self_attn.fused_qkvg_proj", - "language_model.model.layers.59.self_attn.g_proj", - "language_model.model.layers.59.self_attn.in_proj_qkvgfab", - "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.59.self_attn.kv_b_proj", - "language_model.model.layers.59.self_attn.o_proj", - "language_model.model.layers.59.self_attn.q_a_proj", - "language_model.model.layers.59.self_attn.q_b_proj", - "language_model.model.layers.6.self_attn.b_proj", - "language_model.model.layers.6.self_attn.f_a_proj", - "language_model.model.layers.6.self_attn.f_b_proj", - "language_model.model.layers.6.self_attn.fused_qkvg_proj", - "language_model.model.layers.6.self_attn.g_proj", - "language_model.model.layers.6.self_attn.in_proj_qkvgfab", - "language_model.model.layers.6.self_attn.k_proj", - "language_model.model.layers.6.self_attn.o_proj", - "language_model.model.layers.6.self_attn.q_proj", - "language_model.model.layers.6.self_attn.v_proj", - "language_model.model.layers.60.self_attn.b_proj", - "language_model.model.layers.60.self_attn.f_a_proj", - "language_model.model.layers.60.self_attn.f_b_proj", - "language_model.model.layers.60.self_attn.fused_qkvg_proj", - "language_model.model.layers.60.self_attn.g_proj", - "language_model.model.layers.60.self_attn.in_proj_qkvgfab", - "language_model.model.layers.60.self_attn.k_proj", - "language_model.model.layers.60.self_attn.o_proj", - "language_model.model.layers.60.self_attn.q_proj", - "language_model.model.layers.60.self_attn.v_proj", - "language_model.model.layers.61.self_attn.b_proj", - "language_model.model.layers.61.self_attn.f_a_proj", - "language_model.model.layers.61.self_attn.f_b_proj", - "language_model.model.layers.61.self_attn.fused_qkvg_proj", - "language_model.model.layers.61.self_attn.g_proj", - "language_model.model.layers.61.self_attn.in_proj_qkvgfab", - "language_model.model.layers.61.self_attn.k_proj", - "language_model.model.layers.61.self_attn.o_proj", - "language_model.model.layers.61.self_attn.q_proj", - "language_model.model.layers.61.self_attn.v_proj", - "language_model.model.layers.62.self_attn.b_proj", - "language_model.model.layers.62.self_attn.f_a_proj", - "language_model.model.layers.62.self_attn.f_b_proj", - "language_model.model.layers.62.self_attn.fused_qkvg_proj", - "language_model.model.layers.62.self_attn.g_proj", - "language_model.model.layers.62.self_attn.in_proj_qkvgfab", - "language_model.model.layers.62.self_attn.k_proj", - "language_model.model.layers.62.self_attn.o_proj", - "language_model.model.layers.62.self_attn.q_proj", - "language_model.model.layers.62.self_attn.v_proj", - "language_model.model.layers.63.self_attn.fused_qkv_a_proj", - "language_model.model.layers.63.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.63.self_attn.fused_qkvg_proj", - "language_model.model.layers.63.self_attn.g_proj", - "language_model.model.layers.63.self_attn.in_proj_qkvgfab", - "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.63.self_attn.kv_b_proj", - "language_model.model.layers.63.self_attn.o_proj", - "language_model.model.layers.63.self_attn.q_a_proj", - "language_model.model.layers.63.self_attn.q_b_proj", - "language_model.model.layers.64.self_attn.b_proj", - "language_model.model.layers.64.self_attn.f_a_proj", - "language_model.model.layers.64.self_attn.f_b_proj", - "language_model.model.layers.64.self_attn.fused_qkvg_proj", - "language_model.model.layers.64.self_attn.g_proj", - "language_model.model.layers.64.self_attn.in_proj_qkvgfab", - "language_model.model.layers.64.self_attn.k_proj", - "language_model.model.layers.64.self_attn.o_proj", - "language_model.model.layers.64.self_attn.q_proj", - "language_model.model.layers.64.self_attn.v_proj", - "language_model.model.layers.65.self_attn.b_proj", - "language_model.model.layers.65.self_attn.f_a_proj", - "language_model.model.layers.65.self_attn.f_b_proj", - "language_model.model.layers.65.self_attn.fused_qkvg_proj", - "language_model.model.layers.65.self_attn.g_proj", - "language_model.model.layers.65.self_attn.in_proj_qkvgfab", - "language_model.model.layers.65.self_attn.k_proj", - "language_model.model.layers.65.self_attn.o_proj", - "language_model.model.layers.65.self_attn.q_proj", - "language_model.model.layers.65.self_attn.v_proj", - "language_model.model.layers.66.self_attn.b_proj", - "language_model.model.layers.66.self_attn.f_a_proj", - "language_model.model.layers.66.self_attn.f_b_proj", - "language_model.model.layers.66.self_attn.fused_qkvg_proj", - "language_model.model.layers.66.self_attn.g_proj", - "language_model.model.layers.66.self_attn.in_proj_qkvgfab", - "language_model.model.layers.66.self_attn.k_proj", - "language_model.model.layers.66.self_attn.o_proj", - "language_model.model.layers.66.self_attn.q_proj", - "language_model.model.layers.66.self_attn.v_proj", - "language_model.model.layers.67.self_attn.fused_qkv_a_proj", - "language_model.model.layers.67.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.67.self_attn.fused_qkvg_proj", - "language_model.model.layers.67.self_attn.g_proj", - "language_model.model.layers.67.self_attn.in_proj_qkvgfab", - "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.67.self_attn.kv_b_proj", - "language_model.model.layers.67.self_attn.o_proj", - "language_model.model.layers.67.self_attn.q_a_proj", - "language_model.model.layers.67.self_attn.q_b_proj", - "language_model.model.layers.68.self_attn.b_proj", - "language_model.model.layers.68.self_attn.f_a_proj", - "language_model.model.layers.68.self_attn.f_b_proj", - "language_model.model.layers.68.self_attn.fused_qkvg_proj", - "language_model.model.layers.68.self_attn.g_proj", - "language_model.model.layers.68.self_attn.in_proj_qkvgfab", - "language_model.model.layers.68.self_attn.k_proj", - "language_model.model.layers.68.self_attn.o_proj", - "language_model.model.layers.68.self_attn.q_proj", - "language_model.model.layers.68.self_attn.v_proj", - "language_model.model.layers.69.self_attn.b_proj", - "language_model.model.layers.69.self_attn.f_a_proj", - "language_model.model.layers.69.self_attn.f_b_proj", - "language_model.model.layers.69.self_attn.fused_qkvg_proj", - "language_model.model.layers.69.self_attn.g_proj", - "language_model.model.layers.69.self_attn.in_proj_qkvgfab", - "language_model.model.layers.69.self_attn.k_proj", - "language_model.model.layers.69.self_attn.o_proj", - "language_model.model.layers.69.self_attn.q_proj", - "language_model.model.layers.69.self_attn.v_proj", - "language_model.model.layers.7.self_attn.fused_qkv_a_proj", - "language_model.model.layers.7.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.7.self_attn.fused_qkvg_proj", - "language_model.model.layers.7.self_attn.g_proj", - "language_model.model.layers.7.self_attn.in_proj_qkvgfab", - "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.7.self_attn.kv_b_proj", - "language_model.model.layers.7.self_attn.o_proj", - "language_model.model.layers.7.self_attn.q_a_proj", - "language_model.model.layers.7.self_attn.q_b_proj", - "language_model.model.layers.70.self_attn.b_proj", - "language_model.model.layers.70.self_attn.f_a_proj", - "language_model.model.layers.70.self_attn.f_b_proj", - "language_model.model.layers.70.self_attn.fused_qkvg_proj", - "language_model.model.layers.70.self_attn.g_proj", - "language_model.model.layers.70.self_attn.in_proj_qkvgfab", - "language_model.model.layers.70.self_attn.k_proj", - "language_model.model.layers.70.self_attn.o_proj", - "language_model.model.layers.70.self_attn.q_proj", - "language_model.model.layers.70.self_attn.v_proj", - "language_model.model.layers.71.self_attn.fused_qkv_a_proj", - "language_model.model.layers.71.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.71.self_attn.fused_qkvg_proj", - "language_model.model.layers.71.self_attn.g_proj", - "language_model.model.layers.71.self_attn.in_proj_qkvgfab", - "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.71.self_attn.kv_b_proj", - "language_model.model.layers.71.self_attn.o_proj", - "language_model.model.layers.71.self_attn.q_a_proj", - "language_model.model.layers.71.self_attn.q_b_proj", - "language_model.model.layers.72.self_attn.b_proj", - "language_model.model.layers.72.self_attn.f_a_proj", - "language_model.model.layers.72.self_attn.f_b_proj", - "language_model.model.layers.72.self_attn.fused_qkvg_proj", - "language_model.model.layers.72.self_attn.g_proj", - "language_model.model.layers.72.self_attn.in_proj_qkvgfab", - "language_model.model.layers.72.self_attn.k_proj", - "language_model.model.layers.72.self_attn.o_proj", - "language_model.model.layers.72.self_attn.q_proj", - "language_model.model.layers.72.self_attn.v_proj", - "language_model.model.layers.73.self_attn.b_proj", - "language_model.model.layers.73.self_attn.f_a_proj", - "language_model.model.layers.73.self_attn.f_b_proj", - "language_model.model.layers.73.self_attn.fused_qkvg_proj", - "language_model.model.layers.73.self_attn.g_proj", - "language_model.model.layers.73.self_attn.in_proj_qkvgfab", - "language_model.model.layers.73.self_attn.k_proj", - "language_model.model.layers.73.self_attn.o_proj", - "language_model.model.layers.73.self_attn.q_proj", - "language_model.model.layers.73.self_attn.v_proj", - "language_model.model.layers.74.self_attn.b_proj", - "language_model.model.layers.74.self_attn.f_a_proj", - "language_model.model.layers.74.self_attn.f_b_proj", - "language_model.model.layers.74.self_attn.fused_qkvg_proj", - "language_model.model.layers.74.self_attn.g_proj", - "language_model.model.layers.74.self_attn.in_proj_qkvgfab", - "language_model.model.layers.74.self_attn.k_proj", - "language_model.model.layers.74.self_attn.o_proj", - "language_model.model.layers.74.self_attn.q_proj", - "language_model.model.layers.74.self_attn.v_proj", - "language_model.model.layers.75.self_attn.fused_qkv_a_proj", - "language_model.model.layers.75.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.75.self_attn.fused_qkvg_proj", - "language_model.model.layers.75.self_attn.g_proj", - "language_model.model.layers.75.self_attn.in_proj_qkvgfab", - "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.75.self_attn.kv_b_proj", - "language_model.model.layers.75.self_attn.o_proj", - "language_model.model.layers.75.self_attn.q_a_proj", - "language_model.model.layers.75.self_attn.q_b_proj", - "language_model.model.layers.76.self_attn.b_proj", - "language_model.model.layers.76.self_attn.f_a_proj", - "language_model.model.layers.76.self_attn.f_b_proj", - "language_model.model.layers.76.self_attn.fused_qkvg_proj", - "language_model.model.layers.76.self_attn.g_proj", - "language_model.model.layers.76.self_attn.in_proj_qkvgfab", - "language_model.model.layers.76.self_attn.k_proj", - "language_model.model.layers.76.self_attn.o_proj", - "language_model.model.layers.76.self_attn.q_proj", - "language_model.model.layers.76.self_attn.v_proj", - "language_model.model.layers.77.self_attn.b_proj", - "language_model.model.layers.77.self_attn.f_a_proj", - "language_model.model.layers.77.self_attn.f_b_proj", - "language_model.model.layers.77.self_attn.fused_qkvg_proj", - "language_model.model.layers.77.self_attn.g_proj", - "language_model.model.layers.77.self_attn.in_proj_qkvgfab", - "language_model.model.layers.77.self_attn.k_proj", - "language_model.model.layers.77.self_attn.o_proj", - "language_model.model.layers.77.self_attn.q_proj", - "language_model.model.layers.77.self_attn.v_proj", - "language_model.model.layers.78.self_attn.b_proj", - "language_model.model.layers.78.self_attn.f_a_proj", - "language_model.model.layers.78.self_attn.f_b_proj", - "language_model.model.layers.78.self_attn.fused_qkvg_proj", - "language_model.model.layers.78.self_attn.g_proj", - "language_model.model.layers.78.self_attn.in_proj_qkvgfab", - "language_model.model.layers.78.self_attn.k_proj", - "language_model.model.layers.78.self_attn.o_proj", - "language_model.model.layers.78.self_attn.q_proj", - "language_model.model.layers.78.self_attn.v_proj", - "language_model.model.layers.79.self_attn.fused_qkv_a_proj", - "language_model.model.layers.79.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.79.self_attn.fused_qkvg_proj", - "language_model.model.layers.79.self_attn.g_proj", - "language_model.model.layers.79.self_attn.in_proj_qkvgfab", - "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.79.self_attn.kv_b_proj", - "language_model.model.layers.79.self_attn.o_proj", - "language_model.model.layers.79.self_attn.q_a_proj", - "language_model.model.layers.79.self_attn.q_b_proj", - "language_model.model.layers.8.self_attn.b_proj", - "language_model.model.layers.8.self_attn.f_a_proj", - "language_model.model.layers.8.self_attn.f_b_proj", - "language_model.model.layers.8.self_attn.fused_qkvg_proj", - "language_model.model.layers.8.self_attn.g_proj", - "language_model.model.layers.8.self_attn.in_proj_qkvgfab", - "language_model.model.layers.8.self_attn.k_proj", - "language_model.model.layers.8.self_attn.o_proj", - "language_model.model.layers.8.self_attn.q_proj", - "language_model.model.layers.8.self_attn.v_proj", - "language_model.model.layers.80.self_attn.b_proj", - "language_model.model.layers.80.self_attn.f_a_proj", - "language_model.model.layers.80.self_attn.f_b_proj", - "language_model.model.layers.80.self_attn.fused_qkvg_proj", - "language_model.model.layers.80.self_attn.g_proj", - "language_model.model.layers.80.self_attn.in_proj_qkvgfab", - "language_model.model.layers.80.self_attn.k_proj", - "language_model.model.layers.80.self_attn.o_proj", - "language_model.model.layers.80.self_attn.q_proj", - "language_model.model.layers.80.self_attn.v_proj", - "language_model.model.layers.81.self_attn.b_proj", - "language_model.model.layers.81.self_attn.f_a_proj", - "language_model.model.layers.81.self_attn.f_b_proj", - "language_model.model.layers.81.self_attn.fused_qkvg_proj", - "language_model.model.layers.81.self_attn.g_proj", - "language_model.model.layers.81.self_attn.in_proj_qkvgfab", - "language_model.model.layers.81.self_attn.k_proj", - "language_model.model.layers.81.self_attn.o_proj", - "language_model.model.layers.81.self_attn.q_proj", - "language_model.model.layers.81.self_attn.v_proj", - "language_model.model.layers.82.self_attn.b_proj", - "language_model.model.layers.82.self_attn.f_a_proj", - "language_model.model.layers.82.self_attn.f_b_proj", - "language_model.model.layers.82.self_attn.fused_qkvg_proj", - "language_model.model.layers.82.self_attn.g_proj", - "language_model.model.layers.82.self_attn.in_proj_qkvgfab", - "language_model.model.layers.82.self_attn.k_proj", - "language_model.model.layers.82.self_attn.o_proj", - "language_model.model.layers.82.self_attn.q_proj", - "language_model.model.layers.82.self_attn.v_proj", - "language_model.model.layers.83.self_attn.fused_qkv_a_proj", - "language_model.model.layers.83.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.83.self_attn.fused_qkvg_proj", - "language_model.model.layers.83.self_attn.g_proj", - "language_model.model.layers.83.self_attn.in_proj_qkvgfab", - "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.83.self_attn.kv_b_proj", - "language_model.model.layers.83.self_attn.o_proj", - "language_model.model.layers.83.self_attn.q_a_proj", - "language_model.model.layers.83.self_attn.q_b_proj", - "language_model.model.layers.84.self_attn.b_proj", - "language_model.model.layers.84.self_attn.f_a_proj", - "language_model.model.layers.84.self_attn.f_b_proj", - "language_model.model.layers.84.self_attn.fused_qkvg_proj", - "language_model.model.layers.84.self_attn.g_proj", - "language_model.model.layers.84.self_attn.in_proj_qkvgfab", - "language_model.model.layers.84.self_attn.k_proj", - "language_model.model.layers.84.self_attn.o_proj", - "language_model.model.layers.84.self_attn.q_proj", - "language_model.model.layers.84.self_attn.v_proj", - "language_model.model.layers.85.self_attn.b_proj", - "language_model.model.layers.85.self_attn.f_a_proj", - "language_model.model.layers.85.self_attn.f_b_proj", - "language_model.model.layers.85.self_attn.fused_qkvg_proj", - "language_model.model.layers.85.self_attn.g_proj", - "language_model.model.layers.85.self_attn.in_proj_qkvgfab", - "language_model.model.layers.85.self_attn.k_proj", - "language_model.model.layers.85.self_attn.o_proj", - "language_model.model.layers.85.self_attn.q_proj", - "language_model.model.layers.85.self_attn.v_proj", - "language_model.model.layers.86.self_attn.b_proj", - "language_model.model.layers.86.self_attn.f_a_proj", - "language_model.model.layers.86.self_attn.f_b_proj", - "language_model.model.layers.86.self_attn.fused_qkvg_proj", - "language_model.model.layers.86.self_attn.g_proj", - "language_model.model.layers.86.self_attn.in_proj_qkvgfab", - "language_model.model.layers.86.self_attn.k_proj", - "language_model.model.layers.86.self_attn.o_proj", - "language_model.model.layers.86.self_attn.q_proj", - "language_model.model.layers.86.self_attn.v_proj", - "language_model.model.layers.87.self_attn.fused_qkv_a_proj", - "language_model.model.layers.87.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.87.self_attn.fused_qkvg_proj", - "language_model.model.layers.87.self_attn.g_proj", - "language_model.model.layers.87.self_attn.in_proj_qkvgfab", - "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.87.self_attn.kv_b_proj", - "language_model.model.layers.87.self_attn.o_proj", - "language_model.model.layers.87.self_attn.q_a_proj", - "language_model.model.layers.87.self_attn.q_b_proj", - "language_model.model.layers.88.self_attn.b_proj", - "language_model.model.layers.88.self_attn.f_a_proj", - "language_model.model.layers.88.self_attn.f_b_proj", - "language_model.model.layers.88.self_attn.fused_qkvg_proj", - "language_model.model.layers.88.self_attn.g_proj", - "language_model.model.layers.88.self_attn.in_proj_qkvgfab", - "language_model.model.layers.88.self_attn.k_proj", - "language_model.model.layers.88.self_attn.o_proj", - "language_model.model.layers.88.self_attn.q_proj", - "language_model.model.layers.88.self_attn.v_proj", - "language_model.model.layers.89.self_attn.b_proj", - "language_model.model.layers.89.self_attn.f_a_proj", - "language_model.model.layers.89.self_attn.f_b_proj", - "language_model.model.layers.89.self_attn.fused_qkvg_proj", - "language_model.model.layers.89.self_attn.g_proj", - "language_model.model.layers.89.self_attn.in_proj_qkvgfab", - "language_model.model.layers.89.self_attn.k_proj", - "language_model.model.layers.89.self_attn.o_proj", - "language_model.model.layers.89.self_attn.q_proj", - "language_model.model.layers.89.self_attn.v_proj", - "language_model.model.layers.9.self_attn.b_proj", - "language_model.model.layers.9.self_attn.f_a_proj", - "language_model.model.layers.9.self_attn.f_b_proj", - "language_model.model.layers.9.self_attn.fused_qkvg_proj", - "language_model.model.layers.9.self_attn.g_proj", - "language_model.model.layers.9.self_attn.in_proj_qkvgfab", - "language_model.model.layers.9.self_attn.k_proj", - "language_model.model.layers.9.self_attn.o_proj", - "language_model.model.layers.9.self_attn.q_proj", - "language_model.model.layers.9.self_attn.v_proj", - "language_model.model.layers.90.self_attn.b_proj", - "language_model.model.layers.90.self_attn.f_a_proj", - "language_model.model.layers.90.self_attn.f_b_proj", - "language_model.model.layers.90.self_attn.fused_qkvg_proj", - "language_model.model.layers.90.self_attn.g_proj", - "language_model.model.layers.90.self_attn.in_proj_qkvgfab", - "language_model.model.layers.90.self_attn.k_proj", - "language_model.model.layers.90.self_attn.o_proj", - "language_model.model.layers.90.self_attn.q_proj", - "language_model.model.layers.90.self_attn.v_proj", - "language_model.model.layers.91.self_attn.fused_qkv_a_proj", - "language_model.model.layers.91.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.91.self_attn.fused_qkvg_proj", - "language_model.model.layers.91.self_attn.g_proj", - "language_model.model.layers.91.self_attn.in_proj_qkvgfab", - "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.91.self_attn.kv_b_proj", - "language_model.model.layers.91.self_attn.o_proj", - "language_model.model.layers.91.self_attn.q_a_proj", - "language_model.model.layers.91.self_attn.q_b_proj", - "language_model.model.layers.92.self_attn.fused_qkv_a_proj", - "language_model.model.layers.92.self_attn.fused_qkv_a_proj_with_mqa", - "language_model.model.layers.92.self_attn.fused_qkvg_proj", - "language_model.model.layers.92.self_attn.g_proj", - "language_model.model.layers.92.self_attn.in_proj_qkvgfab", - "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa", - "language_model.model.layers.92.self_attn.kv_b_proj", - "language_model.model.layers.92.self_attn.o_proj", - "language_model.model.layers.92.self_attn.q_a_proj", - "language_model.model.layers.92.self_attn.q_b_proj", - "layers.0.self_attn.b_proj", - "layers.0.self_attn.f_a_proj", - "layers.0.self_attn.f_b_proj", - "layers.0.self_attn.fused_qkvg_proj", - "layers.0.self_attn.g_proj", - "layers.0.self_attn.in_proj_qkvgfab", - "layers.0.self_attn.k_proj", - "layers.0.self_attn.o_proj", - "layers.0.self_attn.q_proj", - "layers.0.self_attn.v_proj", - "layers.1.self_attn.b_proj", - "layers.1.self_attn.f_a_proj", - "layers.1.self_attn.f_b_proj", - "layers.1.self_attn.fused_qkvg_proj", - "layers.1.self_attn.g_proj", - "layers.1.self_attn.in_proj_qkvgfab", - "layers.1.self_attn.k_proj", - "layers.1.self_attn.o_proj", - "layers.1.self_attn.q_proj", - "layers.1.self_attn.v_proj", - "layers.10.self_attn.b_proj", - "layers.10.self_attn.f_a_proj", - "layers.10.self_attn.f_b_proj", - "layers.10.self_attn.fused_qkvg_proj", - "layers.10.self_attn.g_proj", - "layers.10.self_attn.in_proj_qkvgfab", - "layers.10.self_attn.k_proj", - "layers.10.self_attn.o_proj", - "layers.10.self_attn.q_proj", - "layers.10.self_attn.v_proj", - "layers.11.self_attn.fused_qkv_a_proj", - "layers.11.self_attn.fused_qkv_a_proj_with_mqa", - "layers.11.self_attn.fused_qkvg_proj", - "layers.11.self_attn.g_proj", - "layers.11.self_attn.in_proj_qkvgfab", - "layers.11.self_attn.kv_a_proj_with_mqa", - "layers.11.self_attn.kv_b_proj", - "layers.11.self_attn.o_proj", - "layers.11.self_attn.q_a_proj", - "layers.11.self_attn.q_b_proj", - "layers.12.self_attn.b_proj", - "layers.12.self_attn.f_a_proj", - "layers.12.self_attn.f_b_proj", - "layers.12.self_attn.fused_qkvg_proj", - "layers.12.self_attn.g_proj", - "layers.12.self_attn.in_proj_qkvgfab", - "layers.12.self_attn.k_proj", - "layers.12.self_attn.o_proj", - "layers.12.self_attn.q_proj", - "layers.12.self_attn.v_proj", - "layers.13.self_attn.b_proj", - "layers.13.self_attn.f_a_proj", - "layers.13.self_attn.f_b_proj", - "layers.13.self_attn.fused_qkvg_proj", - "layers.13.self_attn.g_proj", - "layers.13.self_attn.in_proj_qkvgfab", - "layers.13.self_attn.k_proj", - "layers.13.self_attn.o_proj", - "layers.13.self_attn.q_proj", - "layers.13.self_attn.v_proj", - "layers.14.self_attn.b_proj", - "layers.14.self_attn.f_a_proj", - "layers.14.self_attn.f_b_proj", - "layers.14.self_attn.fused_qkvg_proj", - "layers.14.self_attn.g_proj", - "layers.14.self_attn.in_proj_qkvgfab", - "layers.14.self_attn.k_proj", - "layers.14.self_attn.o_proj", - "layers.14.self_attn.q_proj", - "layers.14.self_attn.v_proj", - "layers.15.self_attn.fused_qkv_a_proj", - "layers.15.self_attn.fused_qkv_a_proj_with_mqa", - "layers.15.self_attn.fused_qkvg_proj", - "layers.15.self_attn.g_proj", - "layers.15.self_attn.in_proj_qkvgfab", - "layers.15.self_attn.kv_a_proj_with_mqa", - "layers.15.self_attn.kv_b_proj", - "layers.15.self_attn.o_proj", - "layers.15.self_attn.q_a_proj", - "layers.15.self_attn.q_b_proj", - "layers.16.self_attn.b_proj", - "layers.16.self_attn.f_a_proj", - "layers.16.self_attn.f_b_proj", - "layers.16.self_attn.fused_qkvg_proj", - "layers.16.self_attn.g_proj", - "layers.16.self_attn.in_proj_qkvgfab", - "layers.16.self_attn.k_proj", - "layers.16.self_attn.o_proj", - "layers.16.self_attn.q_proj", - "layers.16.self_attn.v_proj", - "layers.17.self_attn.b_proj", - "layers.17.self_attn.f_a_proj", - "layers.17.self_attn.f_b_proj", - "layers.17.self_attn.fused_qkvg_proj", - "layers.17.self_attn.g_proj", - "layers.17.self_attn.in_proj_qkvgfab", - "layers.17.self_attn.k_proj", - "layers.17.self_attn.o_proj", - "layers.17.self_attn.q_proj", - "layers.17.self_attn.v_proj", - "layers.18.self_attn.b_proj", - "layers.18.self_attn.f_a_proj", - "layers.18.self_attn.f_b_proj", - "layers.18.self_attn.fused_qkvg_proj", - "layers.18.self_attn.g_proj", - "layers.18.self_attn.in_proj_qkvgfab", - "layers.18.self_attn.k_proj", - "layers.18.self_attn.o_proj", - "layers.18.self_attn.q_proj", - "layers.18.self_attn.v_proj", - "layers.19.self_attn.fused_qkv_a_proj", - "layers.19.self_attn.fused_qkv_a_proj_with_mqa", - "layers.19.self_attn.fused_qkvg_proj", - "layers.19.self_attn.g_proj", - "layers.19.self_attn.in_proj_qkvgfab", - "layers.19.self_attn.kv_a_proj_with_mqa", - "layers.19.self_attn.kv_b_proj", - "layers.19.self_attn.o_proj", - "layers.19.self_attn.q_a_proj", - "layers.19.self_attn.q_b_proj", - "layers.2.self_attn.b_proj", - "layers.2.self_attn.f_a_proj", - "layers.2.self_attn.f_b_proj", - "layers.2.self_attn.fused_qkvg_proj", - "layers.2.self_attn.g_proj", - "layers.2.self_attn.in_proj_qkvgfab", - "layers.2.self_attn.k_proj", - "layers.2.self_attn.o_proj", - "layers.2.self_attn.q_proj", - "layers.2.self_attn.v_proj", - "layers.20.self_attn.b_proj", - "layers.20.self_attn.f_a_proj", - "layers.20.self_attn.f_b_proj", - "layers.20.self_attn.fused_qkvg_proj", - "layers.20.self_attn.g_proj", - "layers.20.self_attn.in_proj_qkvgfab", - "layers.20.self_attn.k_proj", - "layers.20.self_attn.o_proj", - "layers.20.self_attn.q_proj", - "layers.20.self_attn.v_proj", - "layers.21.self_attn.b_proj", - "layers.21.self_attn.f_a_proj", - "layers.21.self_attn.f_b_proj", - "layers.21.self_attn.fused_qkvg_proj", - "layers.21.self_attn.g_proj", - "layers.21.self_attn.in_proj_qkvgfab", - "layers.21.self_attn.k_proj", - "layers.21.self_attn.o_proj", - "layers.21.self_attn.q_proj", - "layers.21.self_attn.v_proj", - "layers.22.self_attn.b_proj", - "layers.22.self_attn.f_a_proj", - "layers.22.self_attn.f_b_proj", - "layers.22.self_attn.fused_qkvg_proj", - "layers.22.self_attn.g_proj", - "layers.22.self_attn.in_proj_qkvgfab", - "layers.22.self_attn.k_proj", - "layers.22.self_attn.o_proj", - "layers.22.self_attn.q_proj", - "layers.22.self_attn.v_proj", - "layers.23.self_attn.fused_qkv_a_proj", - "layers.23.self_attn.fused_qkv_a_proj_with_mqa", - "layers.23.self_attn.fused_qkvg_proj", - "layers.23.self_attn.g_proj", - "layers.23.self_attn.in_proj_qkvgfab", - "layers.23.self_attn.kv_a_proj_with_mqa", - "layers.23.self_attn.kv_b_proj", - "layers.23.self_attn.o_proj", - "layers.23.self_attn.q_a_proj", - "layers.23.self_attn.q_b_proj", - "layers.24.self_attn.b_proj", - "layers.24.self_attn.f_a_proj", - "layers.24.self_attn.f_b_proj", - "layers.24.self_attn.fused_qkvg_proj", - "layers.24.self_attn.g_proj", - "layers.24.self_attn.in_proj_qkvgfab", - "layers.24.self_attn.k_proj", - "layers.24.self_attn.o_proj", - "layers.24.self_attn.q_proj", - "layers.24.self_attn.v_proj", - "layers.25.self_attn.b_proj", - "layers.25.self_attn.f_a_proj", - "layers.25.self_attn.f_b_proj", - "layers.25.self_attn.fused_qkvg_proj", - "layers.25.self_attn.g_proj", - "layers.25.self_attn.in_proj_qkvgfab", - "layers.25.self_attn.k_proj", - "layers.25.self_attn.o_proj", - "layers.25.self_attn.q_proj", - "layers.25.self_attn.v_proj", - "layers.26.self_attn.b_proj", - "layers.26.self_attn.f_a_proj", - "layers.26.self_attn.f_b_proj", - "layers.26.self_attn.fused_qkvg_proj", - "layers.26.self_attn.g_proj", - "layers.26.self_attn.in_proj_qkvgfab", - "layers.26.self_attn.k_proj", - "layers.26.self_attn.o_proj", - "layers.26.self_attn.q_proj", - "layers.26.self_attn.v_proj", - "layers.27.self_attn.fused_qkv_a_proj", - "layers.27.self_attn.fused_qkv_a_proj_with_mqa", - "layers.27.self_attn.fused_qkvg_proj", - "layers.27.self_attn.g_proj", - "layers.27.self_attn.in_proj_qkvgfab", - "layers.27.self_attn.kv_a_proj_with_mqa", - "layers.27.self_attn.kv_b_proj", - "layers.27.self_attn.o_proj", - "layers.27.self_attn.q_a_proj", - "layers.27.self_attn.q_b_proj", - "layers.28.self_attn.b_proj", - "layers.28.self_attn.f_a_proj", - "layers.28.self_attn.f_b_proj", - "layers.28.self_attn.fused_qkvg_proj", - "layers.28.self_attn.g_proj", - "layers.28.self_attn.in_proj_qkvgfab", - "layers.28.self_attn.k_proj", - "layers.28.self_attn.o_proj", - "layers.28.self_attn.q_proj", - "layers.28.self_attn.v_proj", - "layers.29.self_attn.b_proj", - "layers.29.self_attn.f_a_proj", - "layers.29.self_attn.f_b_proj", - "layers.29.self_attn.fused_qkvg_proj", - "layers.29.self_attn.g_proj", - "layers.29.self_attn.in_proj_qkvgfab", - "layers.29.self_attn.k_proj", - "layers.29.self_attn.o_proj", - "layers.29.self_attn.q_proj", - "layers.29.self_attn.v_proj", - "layers.3.self_attn.fused_qkv_a_proj", - "layers.3.self_attn.fused_qkv_a_proj_with_mqa", - "layers.3.self_attn.fused_qkvg_proj", - "layers.3.self_attn.g_proj", - "layers.3.self_attn.in_proj_qkvgfab", - "layers.3.self_attn.kv_a_proj_with_mqa", - "layers.3.self_attn.kv_b_proj", - "layers.3.self_attn.o_proj", - "layers.3.self_attn.q_a_proj", - "layers.3.self_attn.q_b_proj", - "layers.30.self_attn.b_proj", - "layers.30.self_attn.f_a_proj", - "layers.30.self_attn.f_b_proj", - "layers.30.self_attn.fused_qkvg_proj", - "layers.30.self_attn.g_proj", - "layers.30.self_attn.in_proj_qkvgfab", - "layers.30.self_attn.k_proj", - "layers.30.self_attn.o_proj", - "layers.30.self_attn.q_proj", - "layers.30.self_attn.v_proj", - "layers.31.self_attn.fused_qkv_a_proj", - "layers.31.self_attn.fused_qkv_a_proj_with_mqa", - "layers.31.self_attn.fused_qkvg_proj", - "layers.31.self_attn.g_proj", - "layers.31.self_attn.in_proj_qkvgfab", - "layers.31.self_attn.kv_a_proj_with_mqa", - "layers.31.self_attn.kv_b_proj", - "layers.31.self_attn.o_proj", - "layers.31.self_attn.q_a_proj", - "layers.31.self_attn.q_b_proj", - "layers.32.self_attn.b_proj", - "layers.32.self_attn.f_a_proj", - "layers.32.self_attn.f_b_proj", - "layers.32.self_attn.fused_qkvg_proj", - "layers.32.self_attn.g_proj", - "layers.32.self_attn.in_proj_qkvgfab", - "layers.32.self_attn.k_proj", - "layers.32.self_attn.o_proj", - "layers.32.self_attn.q_proj", - "layers.32.self_attn.v_proj", - "layers.33.self_attn.b_proj", - "layers.33.self_attn.f_a_proj", - "layers.33.self_attn.f_b_proj", - "layers.33.self_attn.fused_qkvg_proj", - "layers.33.self_attn.g_proj", - "layers.33.self_attn.in_proj_qkvgfab", - "layers.33.self_attn.k_proj", - "layers.33.self_attn.o_proj", - "layers.33.self_attn.q_proj", - "layers.33.self_attn.v_proj", - "layers.34.self_attn.b_proj", - "layers.34.self_attn.f_a_proj", - "layers.34.self_attn.f_b_proj", - "layers.34.self_attn.fused_qkvg_proj", - "layers.34.self_attn.g_proj", - "layers.34.self_attn.in_proj_qkvgfab", - "layers.34.self_attn.k_proj", - "layers.34.self_attn.o_proj", - "layers.34.self_attn.q_proj", - "layers.34.self_attn.v_proj", - "layers.35.self_attn.fused_qkv_a_proj", - "layers.35.self_attn.fused_qkv_a_proj_with_mqa", - "layers.35.self_attn.fused_qkvg_proj", - "layers.35.self_attn.g_proj", - "layers.35.self_attn.in_proj_qkvgfab", - "layers.35.self_attn.kv_a_proj_with_mqa", - "layers.35.self_attn.kv_b_proj", - "layers.35.self_attn.o_proj", - "layers.35.self_attn.q_a_proj", - "layers.35.self_attn.q_b_proj", - "layers.36.self_attn.b_proj", - "layers.36.self_attn.f_a_proj", - "layers.36.self_attn.f_b_proj", - "layers.36.self_attn.fused_qkvg_proj", - "layers.36.self_attn.g_proj", - "layers.36.self_attn.in_proj_qkvgfab", - "layers.36.self_attn.k_proj", - "layers.36.self_attn.o_proj", - "layers.36.self_attn.q_proj", - "layers.36.self_attn.v_proj", - "layers.37.self_attn.b_proj", - "layers.37.self_attn.f_a_proj", - "layers.37.self_attn.f_b_proj", - "layers.37.self_attn.fused_qkvg_proj", - "layers.37.self_attn.g_proj", - "layers.37.self_attn.in_proj_qkvgfab", - "layers.37.self_attn.k_proj", - "layers.37.self_attn.o_proj", - "layers.37.self_attn.q_proj", - "layers.37.self_attn.v_proj", - "layers.38.self_attn.b_proj", - "layers.38.self_attn.f_a_proj", - "layers.38.self_attn.f_b_proj", - "layers.38.self_attn.fused_qkvg_proj", - "layers.38.self_attn.g_proj", - "layers.38.self_attn.in_proj_qkvgfab", - "layers.38.self_attn.k_proj", - "layers.38.self_attn.o_proj", - "layers.38.self_attn.q_proj", - "layers.38.self_attn.v_proj", - "layers.39.self_attn.fused_qkv_a_proj", - "layers.39.self_attn.fused_qkv_a_proj_with_mqa", - "layers.39.self_attn.fused_qkvg_proj", - "layers.39.self_attn.g_proj", - "layers.39.self_attn.in_proj_qkvgfab", - "layers.39.self_attn.kv_a_proj_with_mqa", - "layers.39.self_attn.kv_b_proj", - "layers.39.self_attn.o_proj", - "layers.39.self_attn.q_a_proj", - "layers.39.self_attn.q_b_proj", - "layers.4.self_attn.b_proj", - "layers.4.self_attn.f_a_proj", - "layers.4.self_attn.f_b_proj", - "layers.4.self_attn.fused_qkvg_proj", - "layers.4.self_attn.g_proj", - "layers.4.self_attn.in_proj_qkvgfab", - "layers.4.self_attn.k_proj", - "layers.4.self_attn.o_proj", - "layers.4.self_attn.q_proj", - "layers.4.self_attn.v_proj", - "layers.40.self_attn.b_proj", - "layers.40.self_attn.f_a_proj", - "layers.40.self_attn.f_b_proj", - "layers.40.self_attn.fused_qkvg_proj", - "layers.40.self_attn.g_proj", - "layers.40.self_attn.in_proj_qkvgfab", - "layers.40.self_attn.k_proj", - "layers.40.self_attn.o_proj", - "layers.40.self_attn.q_proj", - "layers.40.self_attn.v_proj", - "layers.41.self_attn.b_proj", - "layers.41.self_attn.f_a_proj", - "layers.41.self_attn.f_b_proj", - "layers.41.self_attn.fused_qkvg_proj", - "layers.41.self_attn.g_proj", - "layers.41.self_attn.in_proj_qkvgfab", - "layers.41.self_attn.k_proj", - "layers.41.self_attn.o_proj", - "layers.41.self_attn.q_proj", - "layers.41.self_attn.v_proj", - "layers.42.self_attn.b_proj", - "layers.42.self_attn.f_a_proj", - "layers.42.self_attn.f_b_proj", - "layers.42.self_attn.fused_qkvg_proj", - "layers.42.self_attn.g_proj", - "layers.42.self_attn.in_proj_qkvgfab", - "layers.42.self_attn.k_proj", - "layers.42.self_attn.o_proj", - "layers.42.self_attn.q_proj", - "layers.42.self_attn.v_proj", - "layers.43.self_attn.fused_qkv_a_proj", - "layers.43.self_attn.fused_qkv_a_proj_with_mqa", - "layers.43.self_attn.fused_qkvg_proj", - "layers.43.self_attn.g_proj", - "layers.43.self_attn.in_proj_qkvgfab", - "layers.43.self_attn.kv_a_proj_with_mqa", - "layers.43.self_attn.kv_b_proj", - "layers.43.self_attn.o_proj", - "layers.43.self_attn.q_a_proj", - "layers.43.self_attn.q_b_proj", - "layers.44.self_attn.b_proj", - "layers.44.self_attn.f_a_proj", - "layers.44.self_attn.f_b_proj", - "layers.44.self_attn.fused_qkvg_proj", - "layers.44.self_attn.g_proj", - "layers.44.self_attn.in_proj_qkvgfab", - "layers.44.self_attn.k_proj", - "layers.44.self_attn.o_proj", - "layers.44.self_attn.q_proj", - "layers.44.self_attn.v_proj", - "layers.45.self_attn.b_proj", - "layers.45.self_attn.f_a_proj", - "layers.45.self_attn.f_b_proj", - "layers.45.self_attn.fused_qkvg_proj", - "layers.45.self_attn.g_proj", - "layers.45.self_attn.in_proj_qkvgfab", - "layers.45.self_attn.k_proj", - "layers.45.self_attn.o_proj", - "layers.45.self_attn.q_proj", - "layers.45.self_attn.v_proj", - "layers.46.self_attn.b_proj", - "layers.46.self_attn.f_a_proj", - "layers.46.self_attn.f_b_proj", - "layers.46.self_attn.fused_qkvg_proj", - "layers.46.self_attn.g_proj", - "layers.46.self_attn.in_proj_qkvgfab", - "layers.46.self_attn.k_proj", - "layers.46.self_attn.o_proj", - "layers.46.self_attn.q_proj", - "layers.46.self_attn.v_proj", - "layers.47.self_attn.fused_qkv_a_proj", - "layers.47.self_attn.fused_qkv_a_proj_with_mqa", - "layers.47.self_attn.fused_qkvg_proj", - "layers.47.self_attn.g_proj", - "layers.47.self_attn.in_proj_qkvgfab", - "layers.47.self_attn.kv_a_proj_with_mqa", - "layers.47.self_attn.kv_b_proj", - "layers.47.self_attn.o_proj", - "layers.47.self_attn.q_a_proj", - "layers.47.self_attn.q_b_proj", - "layers.48.self_attn.b_proj", - "layers.48.self_attn.f_a_proj", - "layers.48.self_attn.f_b_proj", - "layers.48.self_attn.fused_qkvg_proj", - "layers.48.self_attn.g_proj", - "layers.48.self_attn.in_proj_qkvgfab", - "layers.48.self_attn.k_proj", - "layers.48.self_attn.o_proj", - "layers.48.self_attn.q_proj", - "layers.48.self_attn.v_proj", - "layers.49.self_attn.b_proj", - "layers.49.self_attn.f_a_proj", - "layers.49.self_attn.f_b_proj", - "layers.49.self_attn.fused_qkvg_proj", - "layers.49.self_attn.g_proj", - "layers.49.self_attn.in_proj_qkvgfab", - "layers.49.self_attn.k_proj", - "layers.49.self_attn.o_proj", - "layers.49.self_attn.q_proj", - "layers.49.self_attn.v_proj", - "layers.5.self_attn.b_proj", - "layers.5.self_attn.f_a_proj", - "layers.5.self_attn.f_b_proj", - "layers.5.self_attn.fused_qkvg_proj", - "layers.5.self_attn.g_proj", - "layers.5.self_attn.in_proj_qkvgfab", - "layers.5.self_attn.k_proj", - "layers.5.self_attn.o_proj", - "layers.5.self_attn.q_proj", - "layers.5.self_attn.v_proj", - "layers.50.self_attn.b_proj", - "layers.50.self_attn.f_a_proj", - "layers.50.self_attn.f_b_proj", - "layers.50.self_attn.fused_qkvg_proj", - "layers.50.self_attn.g_proj", - "layers.50.self_attn.in_proj_qkvgfab", - "layers.50.self_attn.k_proj", - "layers.50.self_attn.o_proj", - "layers.50.self_attn.q_proj", - "layers.50.self_attn.v_proj", - "layers.51.self_attn.fused_qkv_a_proj", - "layers.51.self_attn.fused_qkv_a_proj_with_mqa", - "layers.51.self_attn.fused_qkvg_proj", - "layers.51.self_attn.g_proj", - "layers.51.self_attn.in_proj_qkvgfab", - "layers.51.self_attn.kv_a_proj_with_mqa", - "layers.51.self_attn.kv_b_proj", - "layers.51.self_attn.o_proj", - "layers.51.self_attn.q_a_proj", - "layers.51.self_attn.q_b_proj", - "layers.52.self_attn.b_proj", - "layers.52.self_attn.f_a_proj", - "layers.52.self_attn.f_b_proj", - "layers.52.self_attn.fused_qkvg_proj", - "layers.52.self_attn.g_proj", - "layers.52.self_attn.in_proj_qkvgfab", - "layers.52.self_attn.k_proj", - "layers.52.self_attn.o_proj", - "layers.52.self_attn.q_proj", - "layers.52.self_attn.v_proj", - "layers.53.self_attn.b_proj", - "layers.53.self_attn.f_a_proj", - "layers.53.self_attn.f_b_proj", - "layers.53.self_attn.fused_qkvg_proj", - "layers.53.self_attn.g_proj", - "layers.53.self_attn.in_proj_qkvgfab", - "layers.53.self_attn.k_proj", - "layers.53.self_attn.o_proj", - "layers.53.self_attn.q_proj", - "layers.53.self_attn.v_proj", - "layers.54.self_attn.b_proj", - "layers.54.self_attn.f_a_proj", - "layers.54.self_attn.f_b_proj", - "layers.54.self_attn.fused_qkvg_proj", - "layers.54.self_attn.g_proj", - "layers.54.self_attn.in_proj_qkvgfab", - "layers.54.self_attn.k_proj", - "layers.54.self_attn.o_proj", - "layers.54.self_attn.q_proj", - "layers.54.self_attn.v_proj", - "layers.55.self_attn.fused_qkv_a_proj", - "layers.55.self_attn.fused_qkv_a_proj_with_mqa", - "layers.55.self_attn.fused_qkvg_proj", - "layers.55.self_attn.g_proj", - "layers.55.self_attn.in_proj_qkvgfab", - "layers.55.self_attn.kv_a_proj_with_mqa", - "layers.55.self_attn.kv_b_proj", - "layers.55.self_attn.o_proj", - "layers.55.self_attn.q_a_proj", - "layers.55.self_attn.q_b_proj", - "layers.56.self_attn.b_proj", - "layers.56.self_attn.f_a_proj", - "layers.56.self_attn.f_b_proj", - "layers.56.self_attn.fused_qkvg_proj", - "layers.56.self_attn.g_proj", - "layers.56.self_attn.in_proj_qkvgfab", - "layers.56.self_attn.k_proj", - "layers.56.self_attn.o_proj", - "layers.56.self_attn.q_proj", - "layers.56.self_attn.v_proj", - "layers.57.self_attn.b_proj", - "layers.57.self_attn.f_a_proj", - "layers.57.self_attn.f_b_proj", - "layers.57.self_attn.fused_qkvg_proj", - "layers.57.self_attn.g_proj", - "layers.57.self_attn.in_proj_qkvgfab", - "layers.57.self_attn.k_proj", - "layers.57.self_attn.o_proj", - "layers.57.self_attn.q_proj", - "layers.57.self_attn.v_proj", - "layers.58.self_attn.b_proj", - "layers.58.self_attn.f_a_proj", - "layers.58.self_attn.f_b_proj", - "layers.58.self_attn.fused_qkvg_proj", - "layers.58.self_attn.g_proj", - "layers.58.self_attn.in_proj_qkvgfab", - "layers.58.self_attn.k_proj", - "layers.58.self_attn.o_proj", - "layers.58.self_attn.q_proj", - "layers.58.self_attn.v_proj", - "layers.59.self_attn.fused_qkv_a_proj", - "layers.59.self_attn.fused_qkv_a_proj_with_mqa", - "layers.59.self_attn.fused_qkvg_proj", - "layers.59.self_attn.g_proj", - "layers.59.self_attn.in_proj_qkvgfab", - "layers.59.self_attn.kv_a_proj_with_mqa", - "layers.59.self_attn.kv_b_proj", - "layers.59.self_attn.o_proj", - "layers.59.self_attn.q_a_proj", - "layers.59.self_attn.q_b_proj", - "layers.6.self_attn.b_proj", - "layers.6.self_attn.f_a_proj", - "layers.6.self_attn.f_b_proj", - "layers.6.self_attn.fused_qkvg_proj", - "layers.6.self_attn.g_proj", - "layers.6.self_attn.in_proj_qkvgfab", - "layers.6.self_attn.k_proj", - "layers.6.self_attn.o_proj", - "layers.6.self_attn.q_proj", - "layers.6.self_attn.v_proj", - "layers.60.self_attn.b_proj", - "layers.60.self_attn.f_a_proj", - "layers.60.self_attn.f_b_proj", - "layers.60.self_attn.fused_qkvg_proj", - "layers.60.self_attn.g_proj", - "layers.60.self_attn.in_proj_qkvgfab", - "layers.60.self_attn.k_proj", - "layers.60.self_attn.o_proj", - "layers.60.self_attn.q_proj", - "layers.60.self_attn.v_proj", - "layers.61.self_attn.b_proj", - "layers.61.self_attn.f_a_proj", - "layers.61.self_attn.f_b_proj", - "layers.61.self_attn.fused_qkvg_proj", - "layers.61.self_attn.g_proj", - "layers.61.self_attn.in_proj_qkvgfab", - "layers.61.self_attn.k_proj", - "layers.61.self_attn.o_proj", - "layers.61.self_attn.q_proj", - "layers.61.self_attn.v_proj", - "layers.62.self_attn.b_proj", - "layers.62.self_attn.f_a_proj", - "layers.62.self_attn.f_b_proj", - "layers.62.self_attn.fused_qkvg_proj", - "layers.62.self_attn.g_proj", - "layers.62.self_attn.in_proj_qkvgfab", - "layers.62.self_attn.k_proj", - "layers.62.self_attn.o_proj", - "layers.62.self_attn.q_proj", - "layers.62.self_attn.v_proj", - "layers.63.self_attn.fused_qkv_a_proj", - "layers.63.self_attn.fused_qkv_a_proj_with_mqa", - "layers.63.self_attn.fused_qkvg_proj", - "layers.63.self_attn.g_proj", - "layers.63.self_attn.in_proj_qkvgfab", - "layers.63.self_attn.kv_a_proj_with_mqa", - "layers.63.self_attn.kv_b_proj", - "layers.63.self_attn.o_proj", - "layers.63.self_attn.q_a_proj", - "layers.63.self_attn.q_b_proj", - "layers.64.self_attn.b_proj", - "layers.64.self_attn.f_a_proj", - "layers.64.self_attn.f_b_proj", - "layers.64.self_attn.fused_qkvg_proj", - "layers.64.self_attn.g_proj", - "layers.64.self_attn.in_proj_qkvgfab", - "layers.64.self_attn.k_proj", - "layers.64.self_attn.o_proj", - "layers.64.self_attn.q_proj", - "layers.64.self_attn.v_proj", - "layers.65.self_attn.b_proj", - "layers.65.self_attn.f_a_proj", - "layers.65.self_attn.f_b_proj", - "layers.65.self_attn.fused_qkvg_proj", - "layers.65.self_attn.g_proj", - "layers.65.self_attn.in_proj_qkvgfab", - "layers.65.self_attn.k_proj", - "layers.65.self_attn.o_proj", - "layers.65.self_attn.q_proj", - "layers.65.self_attn.v_proj", - "layers.66.self_attn.b_proj", - "layers.66.self_attn.f_a_proj", - "layers.66.self_attn.f_b_proj", - "layers.66.self_attn.fused_qkvg_proj", - "layers.66.self_attn.g_proj", - "layers.66.self_attn.in_proj_qkvgfab", - "layers.66.self_attn.k_proj", - "layers.66.self_attn.o_proj", - "layers.66.self_attn.q_proj", - "layers.66.self_attn.v_proj", - "layers.67.self_attn.fused_qkv_a_proj", - "layers.67.self_attn.fused_qkv_a_proj_with_mqa", - "layers.67.self_attn.fused_qkvg_proj", - "layers.67.self_attn.g_proj", - "layers.67.self_attn.in_proj_qkvgfab", - "layers.67.self_attn.kv_a_proj_with_mqa", - "layers.67.self_attn.kv_b_proj", - "layers.67.self_attn.o_proj", - "layers.67.self_attn.q_a_proj", - "layers.67.self_attn.q_b_proj", - "layers.68.self_attn.b_proj", - "layers.68.self_attn.f_a_proj", - "layers.68.self_attn.f_b_proj", - "layers.68.self_attn.fused_qkvg_proj", - "layers.68.self_attn.g_proj", - "layers.68.self_attn.in_proj_qkvgfab", - "layers.68.self_attn.k_proj", - "layers.68.self_attn.o_proj", - "layers.68.self_attn.q_proj", - "layers.68.self_attn.v_proj", - "layers.69.self_attn.b_proj", - "layers.69.self_attn.f_a_proj", - "layers.69.self_attn.f_b_proj", - "layers.69.self_attn.fused_qkvg_proj", - "layers.69.self_attn.g_proj", - "layers.69.self_attn.in_proj_qkvgfab", - "layers.69.self_attn.k_proj", - "layers.69.self_attn.o_proj", - "layers.69.self_attn.q_proj", - "layers.69.self_attn.v_proj", - "layers.7.self_attn.fused_qkv_a_proj", - "layers.7.self_attn.fused_qkv_a_proj_with_mqa", - "layers.7.self_attn.fused_qkvg_proj", - "layers.7.self_attn.g_proj", - "layers.7.self_attn.in_proj_qkvgfab", - "layers.7.self_attn.kv_a_proj_with_mqa", - "layers.7.self_attn.kv_b_proj", - "layers.7.self_attn.o_proj", - "layers.7.self_attn.q_a_proj", - "layers.7.self_attn.q_b_proj", - "layers.70.self_attn.b_proj", - "layers.70.self_attn.f_a_proj", - "layers.70.self_attn.f_b_proj", - "layers.70.self_attn.fused_qkvg_proj", - "layers.70.self_attn.g_proj", - "layers.70.self_attn.in_proj_qkvgfab", - "layers.70.self_attn.k_proj", - "layers.70.self_attn.o_proj", - "layers.70.self_attn.q_proj", - "layers.70.self_attn.v_proj", - "layers.71.self_attn.fused_qkv_a_proj", - "layers.71.self_attn.fused_qkv_a_proj_with_mqa", - "layers.71.self_attn.fused_qkvg_proj", - "layers.71.self_attn.g_proj", - "layers.71.self_attn.in_proj_qkvgfab", - "layers.71.self_attn.kv_a_proj_with_mqa", - "layers.71.self_attn.kv_b_proj", - "layers.71.self_attn.o_proj", - "layers.71.self_attn.q_a_proj", - "layers.71.self_attn.q_b_proj", - "layers.72.self_attn.b_proj", - "layers.72.self_attn.f_a_proj", - "layers.72.self_attn.f_b_proj", - "layers.72.self_attn.fused_qkvg_proj", - "layers.72.self_attn.g_proj", - "layers.72.self_attn.in_proj_qkvgfab", - "layers.72.self_attn.k_proj", - "layers.72.self_attn.o_proj", - "layers.72.self_attn.q_proj", - "layers.72.self_attn.v_proj", - "layers.73.self_attn.b_proj", - "layers.73.self_attn.f_a_proj", - "layers.73.self_attn.f_b_proj", - "layers.73.self_attn.fused_qkvg_proj", - "layers.73.self_attn.g_proj", - "layers.73.self_attn.in_proj_qkvgfab", - "layers.73.self_attn.k_proj", - "layers.73.self_attn.o_proj", - "layers.73.self_attn.q_proj", - "layers.73.self_attn.v_proj", - "layers.74.self_attn.b_proj", - "layers.74.self_attn.f_a_proj", - "layers.74.self_attn.f_b_proj", - "layers.74.self_attn.fused_qkvg_proj", - "layers.74.self_attn.g_proj", - "layers.74.self_attn.in_proj_qkvgfab", - "layers.74.self_attn.k_proj", - "layers.74.self_attn.o_proj", - "layers.74.self_attn.q_proj", - "layers.74.self_attn.v_proj", - "layers.75.self_attn.fused_qkv_a_proj", - "layers.75.self_attn.fused_qkv_a_proj_with_mqa", - "layers.75.self_attn.fused_qkvg_proj", - "layers.75.self_attn.g_proj", - "layers.75.self_attn.in_proj_qkvgfab", - "layers.75.self_attn.kv_a_proj_with_mqa", - "layers.75.self_attn.kv_b_proj", - "layers.75.self_attn.o_proj", - "layers.75.self_attn.q_a_proj", - "layers.75.self_attn.q_b_proj", - "layers.76.self_attn.b_proj", - "layers.76.self_attn.f_a_proj", - "layers.76.self_attn.f_b_proj", - "layers.76.self_attn.fused_qkvg_proj", - "layers.76.self_attn.g_proj", - "layers.76.self_attn.in_proj_qkvgfab", - "layers.76.self_attn.k_proj", - "layers.76.self_attn.o_proj", - "layers.76.self_attn.q_proj", - "layers.76.self_attn.v_proj", - "layers.77.self_attn.b_proj", - "layers.77.self_attn.f_a_proj", - "layers.77.self_attn.f_b_proj", - "layers.77.self_attn.fused_qkvg_proj", - "layers.77.self_attn.g_proj", - "layers.77.self_attn.in_proj_qkvgfab", - "layers.77.self_attn.k_proj", - "layers.77.self_attn.o_proj", - "layers.77.self_attn.q_proj", - "layers.77.self_attn.v_proj", - "layers.78.self_attn.b_proj", - "layers.78.self_attn.f_a_proj", - "layers.78.self_attn.f_b_proj", - "layers.78.self_attn.fused_qkvg_proj", - "layers.78.self_attn.g_proj", - "layers.78.self_attn.in_proj_qkvgfab", - "layers.78.self_attn.k_proj", - "layers.78.self_attn.o_proj", - "layers.78.self_attn.q_proj", - "layers.78.self_attn.v_proj", - "layers.79.self_attn.fused_qkv_a_proj", - "layers.79.self_attn.fused_qkv_a_proj_with_mqa", - "layers.79.self_attn.fused_qkvg_proj", - "layers.79.self_attn.g_proj", - "layers.79.self_attn.in_proj_qkvgfab", - "layers.79.self_attn.kv_a_proj_with_mqa", - "layers.79.self_attn.kv_b_proj", - "layers.79.self_attn.o_proj", - "layers.79.self_attn.q_a_proj", - "layers.79.self_attn.q_b_proj", - "layers.8.self_attn.b_proj", - "layers.8.self_attn.f_a_proj", - "layers.8.self_attn.f_b_proj", - "layers.8.self_attn.fused_qkvg_proj", - "layers.8.self_attn.g_proj", - "layers.8.self_attn.in_proj_qkvgfab", - "layers.8.self_attn.k_proj", - "layers.8.self_attn.o_proj", - "layers.8.self_attn.q_proj", - "layers.8.self_attn.v_proj", - "layers.80.self_attn.b_proj", - "layers.80.self_attn.f_a_proj", - "layers.80.self_attn.f_b_proj", - "layers.80.self_attn.fused_qkvg_proj", - "layers.80.self_attn.g_proj", - "layers.80.self_attn.in_proj_qkvgfab", - "layers.80.self_attn.k_proj", - "layers.80.self_attn.o_proj", - "layers.80.self_attn.q_proj", - "layers.80.self_attn.v_proj", - "layers.81.self_attn.b_proj", - "layers.81.self_attn.f_a_proj", - "layers.81.self_attn.f_b_proj", - "layers.81.self_attn.fused_qkvg_proj", - "layers.81.self_attn.g_proj", - "layers.81.self_attn.in_proj_qkvgfab", - "layers.81.self_attn.k_proj", - "layers.81.self_attn.o_proj", - "layers.81.self_attn.q_proj", - "layers.81.self_attn.v_proj", - "layers.82.self_attn.b_proj", - "layers.82.self_attn.f_a_proj", - "layers.82.self_attn.f_b_proj", - "layers.82.self_attn.fused_qkvg_proj", - "layers.82.self_attn.g_proj", - "layers.82.self_attn.in_proj_qkvgfab", - "layers.82.self_attn.k_proj", - "layers.82.self_attn.o_proj", - "layers.82.self_attn.q_proj", - "layers.82.self_attn.v_proj", - "layers.83.self_attn.fused_qkv_a_proj", - "layers.83.self_attn.fused_qkv_a_proj_with_mqa", - "layers.83.self_attn.fused_qkvg_proj", - "layers.83.self_attn.g_proj", - "layers.83.self_attn.in_proj_qkvgfab", - "layers.83.self_attn.kv_a_proj_with_mqa", - "layers.83.self_attn.kv_b_proj", - "layers.83.self_attn.o_proj", - "layers.83.self_attn.q_a_proj", - "layers.83.self_attn.q_b_proj", - "layers.84.self_attn.b_proj", - "layers.84.self_attn.f_a_proj", - "layers.84.self_attn.f_b_proj", - "layers.84.self_attn.fused_qkvg_proj", - "layers.84.self_attn.g_proj", - "layers.84.self_attn.in_proj_qkvgfab", - "layers.84.self_attn.k_proj", - "layers.84.self_attn.o_proj", - "layers.84.self_attn.q_proj", - "layers.84.self_attn.v_proj", - "layers.85.self_attn.b_proj", - "layers.85.self_attn.f_a_proj", - "layers.85.self_attn.f_b_proj", - "layers.85.self_attn.fused_qkvg_proj", - "layers.85.self_attn.g_proj", - "layers.85.self_attn.in_proj_qkvgfab", - "layers.85.self_attn.k_proj", - "layers.85.self_attn.o_proj", - "layers.85.self_attn.q_proj", - "layers.85.self_attn.v_proj", - "layers.86.self_attn.b_proj", - "layers.86.self_attn.f_a_proj", - "layers.86.self_attn.f_b_proj", - "layers.86.self_attn.fused_qkvg_proj", - "layers.86.self_attn.g_proj", - "layers.86.self_attn.in_proj_qkvgfab", - "layers.86.self_attn.k_proj", - "layers.86.self_attn.o_proj", - "layers.86.self_attn.q_proj", - "layers.86.self_attn.v_proj", - "layers.87.self_attn.fused_qkv_a_proj", - "layers.87.self_attn.fused_qkv_a_proj_with_mqa", - "layers.87.self_attn.fused_qkvg_proj", - "layers.87.self_attn.g_proj", - "layers.87.self_attn.in_proj_qkvgfab", - "layers.87.self_attn.kv_a_proj_with_mqa", - "layers.87.self_attn.kv_b_proj", - "layers.87.self_attn.o_proj", - "layers.87.self_attn.q_a_proj", - "layers.87.self_attn.q_b_proj", - "layers.88.self_attn.b_proj", - "layers.88.self_attn.f_a_proj", - "layers.88.self_attn.f_b_proj", - "layers.88.self_attn.fused_qkvg_proj", - "layers.88.self_attn.g_proj", - "layers.88.self_attn.in_proj_qkvgfab", - "layers.88.self_attn.k_proj", - "layers.88.self_attn.o_proj", - "layers.88.self_attn.q_proj", - "layers.88.self_attn.v_proj", - "layers.89.self_attn.b_proj", - "layers.89.self_attn.f_a_proj", - "layers.89.self_attn.f_b_proj", - "layers.89.self_attn.fused_qkvg_proj", - "layers.89.self_attn.g_proj", - "layers.89.self_attn.in_proj_qkvgfab", - "layers.89.self_attn.k_proj", - "layers.89.self_attn.o_proj", - "layers.89.self_attn.q_proj", - "layers.89.self_attn.v_proj", - "layers.9.self_attn.b_proj", - "layers.9.self_attn.f_a_proj", - "layers.9.self_attn.f_b_proj", - "layers.9.self_attn.fused_qkvg_proj", - "layers.9.self_attn.g_proj", - "layers.9.self_attn.in_proj_qkvgfab", - "layers.9.self_attn.k_proj", - "layers.9.self_attn.o_proj", - "layers.9.self_attn.q_proj", - "layers.9.self_attn.v_proj", - "layers.90.self_attn.b_proj", - "layers.90.self_attn.f_a_proj", - "layers.90.self_attn.f_b_proj", - "layers.90.self_attn.fused_qkvg_proj", - "layers.90.self_attn.g_proj", - "layers.90.self_attn.in_proj_qkvgfab", - "layers.90.self_attn.k_proj", - "layers.90.self_attn.o_proj", - "layers.90.self_attn.q_proj", - "layers.90.self_attn.v_proj", - "layers.91.self_attn.fused_qkv_a_proj", - "layers.91.self_attn.fused_qkv_a_proj_with_mqa", - "layers.91.self_attn.fused_qkvg_proj", - "layers.91.self_attn.g_proj", - "layers.91.self_attn.in_proj_qkvgfab", - "layers.91.self_attn.kv_a_proj_with_mqa", - "layers.91.self_attn.kv_b_proj", - "layers.91.self_attn.o_proj", - "layers.91.self_attn.q_a_proj", - "layers.91.self_attn.q_b_proj", - "layers.92.self_attn.fused_qkv_a_proj", - "layers.92.self_attn.fused_qkv_a_proj_with_mqa", - "layers.92.self_attn.fused_qkvg_proj", - "layers.92.self_attn.g_proj", - "layers.92.self_attn.in_proj_qkvgfab", - "layers.92.self_attn.kv_a_proj_with_mqa", - "layers.92.self_attn.kv_b_proj", - "layers.92.self_attn.o_proj", - "layers.92.self_attn.q_a_proj", - "layers.92.self_attn.q_b_proj", - "model.layers.0.self_attn.b_proj", - "model.layers.0.self_attn.f_a_proj", - "model.layers.0.self_attn.f_b_proj", - "model.layers.0.self_attn.fused_qkvg_proj", - "model.layers.0.self_attn.g_proj", - "model.layers.0.self_attn.in_proj_qkvgfab", - "model.layers.0.self_attn.k_proj", - "model.layers.0.self_attn.o_proj", - "model.layers.0.self_attn.q_proj", - "model.layers.0.self_attn.v_proj", - "model.layers.1.self_attn.b_proj", - "model.layers.1.self_attn.f_a_proj", - "model.layers.1.self_attn.f_b_proj", - "model.layers.1.self_attn.fused_qkvg_proj", - "model.layers.1.self_attn.g_proj", - "model.layers.1.self_attn.in_proj_qkvgfab", - "model.layers.1.self_attn.k_proj", - "model.layers.1.self_attn.o_proj", - "model.layers.1.self_attn.q_proj", - "model.layers.1.self_attn.v_proj", - "model.layers.10.self_attn.b_proj", - "model.layers.10.self_attn.f_a_proj", - "model.layers.10.self_attn.f_b_proj", - "model.layers.10.self_attn.fused_qkvg_proj", - "model.layers.10.self_attn.g_proj", - "model.layers.10.self_attn.in_proj_qkvgfab", - "model.layers.10.self_attn.k_proj", - "model.layers.10.self_attn.o_proj", - "model.layers.10.self_attn.q_proj", - "model.layers.10.self_attn.v_proj", - "model.layers.11.self_attn.fused_qkv_a_proj", - "model.layers.11.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.11.self_attn.fused_qkvg_proj", - "model.layers.11.self_attn.g_proj", - "model.layers.11.self_attn.in_proj_qkvgfab", - "model.layers.11.self_attn.kv_a_proj_with_mqa", - "model.layers.11.self_attn.kv_b_proj", - "model.layers.11.self_attn.o_proj", - "model.layers.11.self_attn.q_a_proj", - "model.layers.11.self_attn.q_b_proj", - "model.layers.12.self_attn.b_proj", - "model.layers.12.self_attn.f_a_proj", - "model.layers.12.self_attn.f_b_proj", - "model.layers.12.self_attn.fused_qkvg_proj", - "model.layers.12.self_attn.g_proj", - "model.layers.12.self_attn.in_proj_qkvgfab", - "model.layers.12.self_attn.k_proj", - "model.layers.12.self_attn.o_proj", - "model.layers.12.self_attn.q_proj", - "model.layers.12.self_attn.v_proj", - "model.layers.13.self_attn.b_proj", - "model.layers.13.self_attn.f_a_proj", - "model.layers.13.self_attn.f_b_proj", - "model.layers.13.self_attn.fused_qkvg_proj", - "model.layers.13.self_attn.g_proj", - "model.layers.13.self_attn.in_proj_qkvgfab", - "model.layers.13.self_attn.k_proj", - "model.layers.13.self_attn.o_proj", - "model.layers.13.self_attn.q_proj", - "model.layers.13.self_attn.v_proj", - "model.layers.14.self_attn.b_proj", - "model.layers.14.self_attn.f_a_proj", - "model.layers.14.self_attn.f_b_proj", - "model.layers.14.self_attn.fused_qkvg_proj", - "model.layers.14.self_attn.g_proj", - "model.layers.14.self_attn.in_proj_qkvgfab", - "model.layers.14.self_attn.k_proj", - "model.layers.14.self_attn.o_proj", - "model.layers.14.self_attn.q_proj", - "model.layers.14.self_attn.v_proj", - "model.layers.15.self_attn.fused_qkv_a_proj", - "model.layers.15.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.15.self_attn.fused_qkvg_proj", - "model.layers.15.self_attn.g_proj", - "model.layers.15.self_attn.in_proj_qkvgfab", - "model.layers.15.self_attn.kv_a_proj_with_mqa", - "model.layers.15.self_attn.kv_b_proj", - "model.layers.15.self_attn.o_proj", - "model.layers.15.self_attn.q_a_proj", - "model.layers.15.self_attn.q_b_proj", - "model.layers.16.self_attn.b_proj", - "model.layers.16.self_attn.f_a_proj", - "model.layers.16.self_attn.f_b_proj", - "model.layers.16.self_attn.fused_qkvg_proj", - "model.layers.16.self_attn.g_proj", - "model.layers.16.self_attn.in_proj_qkvgfab", - "model.layers.16.self_attn.k_proj", - "model.layers.16.self_attn.o_proj", - "model.layers.16.self_attn.q_proj", - "model.layers.16.self_attn.v_proj", - "model.layers.17.self_attn.b_proj", - "model.layers.17.self_attn.f_a_proj", - "model.layers.17.self_attn.f_b_proj", - "model.layers.17.self_attn.fused_qkvg_proj", - "model.layers.17.self_attn.g_proj", - "model.layers.17.self_attn.in_proj_qkvgfab", - "model.layers.17.self_attn.k_proj", - "model.layers.17.self_attn.o_proj", - "model.layers.17.self_attn.q_proj", - "model.layers.17.self_attn.v_proj", - "model.layers.18.self_attn.b_proj", - "model.layers.18.self_attn.f_a_proj", - "model.layers.18.self_attn.f_b_proj", - "model.layers.18.self_attn.fused_qkvg_proj", - "model.layers.18.self_attn.g_proj", - "model.layers.18.self_attn.in_proj_qkvgfab", - "model.layers.18.self_attn.k_proj", - "model.layers.18.self_attn.o_proj", - "model.layers.18.self_attn.q_proj", - "model.layers.18.self_attn.v_proj", - "model.layers.19.self_attn.fused_qkv_a_proj", - "model.layers.19.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.19.self_attn.fused_qkvg_proj", - "model.layers.19.self_attn.g_proj", - "model.layers.19.self_attn.in_proj_qkvgfab", - "model.layers.19.self_attn.kv_a_proj_with_mqa", - "model.layers.19.self_attn.kv_b_proj", - "model.layers.19.self_attn.o_proj", - "model.layers.19.self_attn.q_a_proj", - "model.layers.19.self_attn.q_b_proj", - "model.layers.2.self_attn.b_proj", - "model.layers.2.self_attn.f_a_proj", - "model.layers.2.self_attn.f_b_proj", - "model.layers.2.self_attn.fused_qkvg_proj", - "model.layers.2.self_attn.g_proj", - "model.layers.2.self_attn.in_proj_qkvgfab", - "model.layers.2.self_attn.k_proj", - "model.layers.2.self_attn.o_proj", - "model.layers.2.self_attn.q_proj", - "model.layers.2.self_attn.v_proj", - "model.layers.20.self_attn.b_proj", - "model.layers.20.self_attn.f_a_proj", - "model.layers.20.self_attn.f_b_proj", - "model.layers.20.self_attn.fused_qkvg_proj", - "model.layers.20.self_attn.g_proj", - "model.layers.20.self_attn.in_proj_qkvgfab", - "model.layers.20.self_attn.k_proj", - "model.layers.20.self_attn.o_proj", - "model.layers.20.self_attn.q_proj", - "model.layers.20.self_attn.v_proj", - "model.layers.21.self_attn.b_proj", - "model.layers.21.self_attn.f_a_proj", - "model.layers.21.self_attn.f_b_proj", - "model.layers.21.self_attn.fused_qkvg_proj", - "model.layers.21.self_attn.g_proj", - "model.layers.21.self_attn.in_proj_qkvgfab", - "model.layers.21.self_attn.k_proj", - "model.layers.21.self_attn.o_proj", - "model.layers.21.self_attn.q_proj", - "model.layers.21.self_attn.v_proj", - "model.layers.22.self_attn.b_proj", - "model.layers.22.self_attn.f_a_proj", - "model.layers.22.self_attn.f_b_proj", - "model.layers.22.self_attn.fused_qkvg_proj", - "model.layers.22.self_attn.g_proj", - "model.layers.22.self_attn.in_proj_qkvgfab", - "model.layers.22.self_attn.k_proj", - "model.layers.22.self_attn.o_proj", - "model.layers.22.self_attn.q_proj", - "model.layers.22.self_attn.v_proj", - "model.layers.23.self_attn.fused_qkv_a_proj", - "model.layers.23.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.23.self_attn.fused_qkvg_proj", - "model.layers.23.self_attn.g_proj", - "model.layers.23.self_attn.in_proj_qkvgfab", - "model.layers.23.self_attn.kv_a_proj_with_mqa", - "model.layers.23.self_attn.kv_b_proj", - "model.layers.23.self_attn.o_proj", - "model.layers.23.self_attn.q_a_proj", - "model.layers.23.self_attn.q_b_proj", - "model.layers.24.self_attn.b_proj", - "model.layers.24.self_attn.f_a_proj", - "model.layers.24.self_attn.f_b_proj", - "model.layers.24.self_attn.fused_qkvg_proj", - "model.layers.24.self_attn.g_proj", - "model.layers.24.self_attn.in_proj_qkvgfab", - "model.layers.24.self_attn.k_proj", - "model.layers.24.self_attn.o_proj", - "model.layers.24.self_attn.q_proj", - "model.layers.24.self_attn.v_proj", - "model.layers.25.self_attn.b_proj", - "model.layers.25.self_attn.f_a_proj", - "model.layers.25.self_attn.f_b_proj", - "model.layers.25.self_attn.fused_qkvg_proj", - "model.layers.25.self_attn.g_proj", - "model.layers.25.self_attn.in_proj_qkvgfab", - "model.layers.25.self_attn.k_proj", - "model.layers.25.self_attn.o_proj", - "model.layers.25.self_attn.q_proj", - "model.layers.25.self_attn.v_proj", - "model.layers.26.self_attn.b_proj", - "model.layers.26.self_attn.f_a_proj", - "model.layers.26.self_attn.f_b_proj", - "model.layers.26.self_attn.fused_qkvg_proj", - "model.layers.26.self_attn.g_proj", - "model.layers.26.self_attn.in_proj_qkvgfab", - "model.layers.26.self_attn.k_proj", - "model.layers.26.self_attn.o_proj", - "model.layers.26.self_attn.q_proj", - "model.layers.26.self_attn.v_proj", - "model.layers.27.self_attn.fused_qkv_a_proj", - "model.layers.27.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.27.self_attn.fused_qkvg_proj", - "model.layers.27.self_attn.g_proj", - "model.layers.27.self_attn.in_proj_qkvgfab", - "model.layers.27.self_attn.kv_a_proj_with_mqa", - "model.layers.27.self_attn.kv_b_proj", - "model.layers.27.self_attn.o_proj", - "model.layers.27.self_attn.q_a_proj", - "model.layers.27.self_attn.q_b_proj", - "model.layers.28.self_attn.b_proj", - "model.layers.28.self_attn.f_a_proj", - "model.layers.28.self_attn.f_b_proj", - "model.layers.28.self_attn.fused_qkvg_proj", - "model.layers.28.self_attn.g_proj", - "model.layers.28.self_attn.in_proj_qkvgfab", - "model.layers.28.self_attn.k_proj", - "model.layers.28.self_attn.o_proj", - "model.layers.28.self_attn.q_proj", - "model.layers.28.self_attn.v_proj", - "model.layers.29.self_attn.b_proj", - "model.layers.29.self_attn.f_a_proj", - "model.layers.29.self_attn.f_b_proj", - "model.layers.29.self_attn.fused_qkvg_proj", - "model.layers.29.self_attn.g_proj", - "model.layers.29.self_attn.in_proj_qkvgfab", - "model.layers.29.self_attn.k_proj", - "model.layers.29.self_attn.o_proj", - "model.layers.29.self_attn.q_proj", - "model.layers.29.self_attn.v_proj", - "model.layers.3.self_attn.fused_qkv_a_proj", - "model.layers.3.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.3.self_attn.fused_qkvg_proj", - "model.layers.3.self_attn.g_proj", - "model.layers.3.self_attn.in_proj_qkvgfab", - "model.layers.3.self_attn.kv_a_proj_with_mqa", - "model.layers.3.self_attn.kv_b_proj", - "model.layers.3.self_attn.o_proj", - "model.layers.3.self_attn.q_a_proj", - "model.layers.3.self_attn.q_b_proj", - "model.layers.30.self_attn.b_proj", - "model.layers.30.self_attn.f_a_proj", - "model.layers.30.self_attn.f_b_proj", - "model.layers.30.self_attn.fused_qkvg_proj", - "model.layers.30.self_attn.g_proj", - "model.layers.30.self_attn.in_proj_qkvgfab", - "model.layers.30.self_attn.k_proj", - "model.layers.30.self_attn.o_proj", - "model.layers.30.self_attn.q_proj", - "model.layers.30.self_attn.v_proj", - "model.layers.31.self_attn.fused_qkv_a_proj", - "model.layers.31.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.31.self_attn.fused_qkvg_proj", - "model.layers.31.self_attn.g_proj", - "model.layers.31.self_attn.in_proj_qkvgfab", - "model.layers.31.self_attn.kv_a_proj_with_mqa", - "model.layers.31.self_attn.kv_b_proj", - "model.layers.31.self_attn.o_proj", - "model.layers.31.self_attn.q_a_proj", - "model.layers.31.self_attn.q_b_proj", - "model.layers.32.self_attn.b_proj", - "model.layers.32.self_attn.f_a_proj", - "model.layers.32.self_attn.f_b_proj", - "model.layers.32.self_attn.fused_qkvg_proj", - "model.layers.32.self_attn.g_proj", - "model.layers.32.self_attn.in_proj_qkvgfab", - "model.layers.32.self_attn.k_proj", - "model.layers.32.self_attn.o_proj", - "model.layers.32.self_attn.q_proj", - "model.layers.32.self_attn.v_proj", - "model.layers.33.self_attn.b_proj", - "model.layers.33.self_attn.f_a_proj", - "model.layers.33.self_attn.f_b_proj", - "model.layers.33.self_attn.fused_qkvg_proj", - "model.layers.33.self_attn.g_proj", - "model.layers.33.self_attn.in_proj_qkvgfab", - "model.layers.33.self_attn.k_proj", - "model.layers.33.self_attn.o_proj", - "model.layers.33.self_attn.q_proj", - "model.layers.33.self_attn.v_proj", - "model.layers.34.self_attn.b_proj", - "model.layers.34.self_attn.f_a_proj", - "model.layers.34.self_attn.f_b_proj", - "model.layers.34.self_attn.fused_qkvg_proj", - "model.layers.34.self_attn.g_proj", - "model.layers.34.self_attn.in_proj_qkvgfab", - "model.layers.34.self_attn.k_proj", - "model.layers.34.self_attn.o_proj", - "model.layers.34.self_attn.q_proj", - "model.layers.34.self_attn.v_proj", - "model.layers.35.self_attn.fused_qkv_a_proj", - "model.layers.35.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.35.self_attn.fused_qkvg_proj", - "model.layers.35.self_attn.g_proj", - "model.layers.35.self_attn.in_proj_qkvgfab", - "model.layers.35.self_attn.kv_a_proj_with_mqa", - "model.layers.35.self_attn.kv_b_proj", - "model.layers.35.self_attn.o_proj", - "model.layers.35.self_attn.q_a_proj", - "model.layers.35.self_attn.q_b_proj", - "model.layers.36.self_attn.b_proj", - "model.layers.36.self_attn.f_a_proj", - "model.layers.36.self_attn.f_b_proj", - "model.layers.36.self_attn.fused_qkvg_proj", - "model.layers.36.self_attn.g_proj", - "model.layers.36.self_attn.in_proj_qkvgfab", - "model.layers.36.self_attn.k_proj", - "model.layers.36.self_attn.o_proj", - "model.layers.36.self_attn.q_proj", - "model.layers.36.self_attn.v_proj", - "model.layers.37.self_attn.b_proj", - "model.layers.37.self_attn.f_a_proj", - "model.layers.37.self_attn.f_b_proj", - "model.layers.37.self_attn.fused_qkvg_proj", - "model.layers.37.self_attn.g_proj", - "model.layers.37.self_attn.in_proj_qkvgfab", - "model.layers.37.self_attn.k_proj", - "model.layers.37.self_attn.o_proj", - "model.layers.37.self_attn.q_proj", - "model.layers.37.self_attn.v_proj", - "model.layers.38.self_attn.b_proj", - "model.layers.38.self_attn.f_a_proj", - "model.layers.38.self_attn.f_b_proj", - "model.layers.38.self_attn.fused_qkvg_proj", - "model.layers.38.self_attn.g_proj", - "model.layers.38.self_attn.in_proj_qkvgfab", - "model.layers.38.self_attn.k_proj", - "model.layers.38.self_attn.o_proj", - "model.layers.38.self_attn.q_proj", - "model.layers.38.self_attn.v_proj", - "model.layers.39.self_attn.fused_qkv_a_proj", - "model.layers.39.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.39.self_attn.fused_qkvg_proj", - "model.layers.39.self_attn.g_proj", - "model.layers.39.self_attn.in_proj_qkvgfab", - "model.layers.39.self_attn.kv_a_proj_with_mqa", - "model.layers.39.self_attn.kv_b_proj", - "model.layers.39.self_attn.o_proj", - "model.layers.39.self_attn.q_a_proj", - "model.layers.39.self_attn.q_b_proj", - "model.layers.4.self_attn.b_proj", - "model.layers.4.self_attn.f_a_proj", - "model.layers.4.self_attn.f_b_proj", - "model.layers.4.self_attn.fused_qkvg_proj", - "model.layers.4.self_attn.g_proj", - "model.layers.4.self_attn.in_proj_qkvgfab", - "model.layers.4.self_attn.k_proj", - "model.layers.4.self_attn.o_proj", - "model.layers.4.self_attn.q_proj", - "model.layers.4.self_attn.v_proj", - "model.layers.40.self_attn.b_proj", - "model.layers.40.self_attn.f_a_proj", - "model.layers.40.self_attn.f_b_proj", - "model.layers.40.self_attn.fused_qkvg_proj", - "model.layers.40.self_attn.g_proj", - "model.layers.40.self_attn.in_proj_qkvgfab", - "model.layers.40.self_attn.k_proj", - "model.layers.40.self_attn.o_proj", - "model.layers.40.self_attn.q_proj", - "model.layers.40.self_attn.v_proj", - "model.layers.41.self_attn.b_proj", - "model.layers.41.self_attn.f_a_proj", - "model.layers.41.self_attn.f_b_proj", - "model.layers.41.self_attn.fused_qkvg_proj", - "model.layers.41.self_attn.g_proj", - "model.layers.41.self_attn.in_proj_qkvgfab", - "model.layers.41.self_attn.k_proj", - "model.layers.41.self_attn.o_proj", - "model.layers.41.self_attn.q_proj", - "model.layers.41.self_attn.v_proj", - "model.layers.42.self_attn.b_proj", - "model.layers.42.self_attn.f_a_proj", - "model.layers.42.self_attn.f_b_proj", - "model.layers.42.self_attn.fused_qkvg_proj", - "model.layers.42.self_attn.g_proj", - "model.layers.42.self_attn.in_proj_qkvgfab", - "model.layers.42.self_attn.k_proj", - "model.layers.42.self_attn.o_proj", - "model.layers.42.self_attn.q_proj", - "model.layers.42.self_attn.v_proj", - "model.layers.43.self_attn.fused_qkv_a_proj", - "model.layers.43.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.43.self_attn.fused_qkvg_proj", - "model.layers.43.self_attn.g_proj", - "model.layers.43.self_attn.in_proj_qkvgfab", - "model.layers.43.self_attn.kv_a_proj_with_mqa", - "model.layers.43.self_attn.kv_b_proj", - "model.layers.43.self_attn.o_proj", - "model.layers.43.self_attn.q_a_proj", - "model.layers.43.self_attn.q_b_proj", - "model.layers.44.self_attn.b_proj", - "model.layers.44.self_attn.f_a_proj", - "model.layers.44.self_attn.f_b_proj", - "model.layers.44.self_attn.fused_qkvg_proj", - "model.layers.44.self_attn.g_proj", - "model.layers.44.self_attn.in_proj_qkvgfab", - "model.layers.44.self_attn.k_proj", - "model.layers.44.self_attn.o_proj", - "model.layers.44.self_attn.q_proj", - "model.layers.44.self_attn.v_proj", - "model.layers.45.self_attn.b_proj", - "model.layers.45.self_attn.f_a_proj", - "model.layers.45.self_attn.f_b_proj", - "model.layers.45.self_attn.fused_qkvg_proj", - "model.layers.45.self_attn.g_proj", - "model.layers.45.self_attn.in_proj_qkvgfab", - "model.layers.45.self_attn.k_proj", - "model.layers.45.self_attn.o_proj", - "model.layers.45.self_attn.q_proj", - "model.layers.45.self_attn.v_proj", - "model.layers.46.self_attn.b_proj", - "model.layers.46.self_attn.f_a_proj", - "model.layers.46.self_attn.f_b_proj", - "model.layers.46.self_attn.fused_qkvg_proj", - "model.layers.46.self_attn.g_proj", - "model.layers.46.self_attn.in_proj_qkvgfab", - "model.layers.46.self_attn.k_proj", - "model.layers.46.self_attn.o_proj", - "model.layers.46.self_attn.q_proj", - "model.layers.46.self_attn.v_proj", - "model.layers.47.self_attn.fused_qkv_a_proj", - "model.layers.47.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.47.self_attn.fused_qkvg_proj", - "model.layers.47.self_attn.g_proj", - "model.layers.47.self_attn.in_proj_qkvgfab", - "model.layers.47.self_attn.kv_a_proj_with_mqa", - "model.layers.47.self_attn.kv_b_proj", - "model.layers.47.self_attn.o_proj", - "model.layers.47.self_attn.q_a_proj", - "model.layers.47.self_attn.q_b_proj", - "model.layers.48.self_attn.b_proj", - "model.layers.48.self_attn.f_a_proj", - "model.layers.48.self_attn.f_b_proj", - "model.layers.48.self_attn.fused_qkvg_proj", - "model.layers.48.self_attn.g_proj", - "model.layers.48.self_attn.in_proj_qkvgfab", - "model.layers.48.self_attn.k_proj", - "model.layers.48.self_attn.o_proj", - "model.layers.48.self_attn.q_proj", - "model.layers.48.self_attn.v_proj", - "model.layers.49.self_attn.b_proj", - "model.layers.49.self_attn.f_a_proj", - "model.layers.49.self_attn.f_b_proj", - "model.layers.49.self_attn.fused_qkvg_proj", - "model.layers.49.self_attn.g_proj", - "model.layers.49.self_attn.in_proj_qkvgfab", - "model.layers.49.self_attn.k_proj", - "model.layers.49.self_attn.o_proj", - "model.layers.49.self_attn.q_proj", - "model.layers.49.self_attn.v_proj", - "model.layers.5.self_attn.b_proj", - "model.layers.5.self_attn.f_a_proj", - "model.layers.5.self_attn.f_b_proj", - "model.layers.5.self_attn.fused_qkvg_proj", - "model.layers.5.self_attn.g_proj", - "model.layers.5.self_attn.in_proj_qkvgfab", - "model.layers.5.self_attn.k_proj", - "model.layers.5.self_attn.o_proj", - "model.layers.5.self_attn.q_proj", - "model.layers.5.self_attn.v_proj", - "model.layers.50.self_attn.b_proj", - "model.layers.50.self_attn.f_a_proj", - "model.layers.50.self_attn.f_b_proj", - "model.layers.50.self_attn.fused_qkvg_proj", - "model.layers.50.self_attn.g_proj", - "model.layers.50.self_attn.in_proj_qkvgfab", - "model.layers.50.self_attn.k_proj", - "model.layers.50.self_attn.o_proj", - "model.layers.50.self_attn.q_proj", - "model.layers.50.self_attn.v_proj", - "model.layers.51.self_attn.fused_qkv_a_proj", - "model.layers.51.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.51.self_attn.fused_qkvg_proj", - "model.layers.51.self_attn.g_proj", - "model.layers.51.self_attn.in_proj_qkvgfab", - "model.layers.51.self_attn.kv_a_proj_with_mqa", - "model.layers.51.self_attn.kv_b_proj", - "model.layers.51.self_attn.o_proj", - "model.layers.51.self_attn.q_a_proj", - "model.layers.51.self_attn.q_b_proj", - "model.layers.52.self_attn.b_proj", - "model.layers.52.self_attn.f_a_proj", - "model.layers.52.self_attn.f_b_proj", - "model.layers.52.self_attn.fused_qkvg_proj", - "model.layers.52.self_attn.g_proj", - "model.layers.52.self_attn.in_proj_qkvgfab", - "model.layers.52.self_attn.k_proj", - "model.layers.52.self_attn.o_proj", - "model.layers.52.self_attn.q_proj", - "model.layers.52.self_attn.v_proj", - "model.layers.53.self_attn.b_proj", - "model.layers.53.self_attn.f_a_proj", - "model.layers.53.self_attn.f_b_proj", - "model.layers.53.self_attn.fused_qkvg_proj", - "model.layers.53.self_attn.g_proj", - "model.layers.53.self_attn.in_proj_qkvgfab", - "model.layers.53.self_attn.k_proj", - "model.layers.53.self_attn.o_proj", - "model.layers.53.self_attn.q_proj", - "model.layers.53.self_attn.v_proj", - "model.layers.54.self_attn.b_proj", - "model.layers.54.self_attn.f_a_proj", - "model.layers.54.self_attn.f_b_proj", - "model.layers.54.self_attn.fused_qkvg_proj", - "model.layers.54.self_attn.g_proj", - "model.layers.54.self_attn.in_proj_qkvgfab", - "model.layers.54.self_attn.k_proj", - "model.layers.54.self_attn.o_proj", - "model.layers.54.self_attn.q_proj", - "model.layers.54.self_attn.v_proj", - "model.layers.55.self_attn.fused_qkv_a_proj", - "model.layers.55.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.55.self_attn.fused_qkvg_proj", - "model.layers.55.self_attn.g_proj", - "model.layers.55.self_attn.in_proj_qkvgfab", - "model.layers.55.self_attn.kv_a_proj_with_mqa", - "model.layers.55.self_attn.kv_b_proj", - "model.layers.55.self_attn.o_proj", - "model.layers.55.self_attn.q_a_proj", - "model.layers.55.self_attn.q_b_proj", - "model.layers.56.self_attn.b_proj", - "model.layers.56.self_attn.f_a_proj", - "model.layers.56.self_attn.f_b_proj", - "model.layers.56.self_attn.fused_qkvg_proj", - "model.layers.56.self_attn.g_proj", - "model.layers.56.self_attn.in_proj_qkvgfab", - "model.layers.56.self_attn.k_proj", - "model.layers.56.self_attn.o_proj", - "model.layers.56.self_attn.q_proj", - "model.layers.56.self_attn.v_proj", - "model.layers.57.self_attn.b_proj", - "model.layers.57.self_attn.f_a_proj", - "model.layers.57.self_attn.f_b_proj", - "model.layers.57.self_attn.fused_qkvg_proj", - "model.layers.57.self_attn.g_proj", - "model.layers.57.self_attn.in_proj_qkvgfab", - "model.layers.57.self_attn.k_proj", - "model.layers.57.self_attn.o_proj", - "model.layers.57.self_attn.q_proj", - "model.layers.57.self_attn.v_proj", - "model.layers.58.self_attn.b_proj", - "model.layers.58.self_attn.f_a_proj", - "model.layers.58.self_attn.f_b_proj", - "model.layers.58.self_attn.fused_qkvg_proj", - "model.layers.58.self_attn.g_proj", - "model.layers.58.self_attn.in_proj_qkvgfab", - "model.layers.58.self_attn.k_proj", - "model.layers.58.self_attn.o_proj", - "model.layers.58.self_attn.q_proj", - "model.layers.58.self_attn.v_proj", - "model.layers.59.self_attn.fused_qkv_a_proj", - "model.layers.59.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.59.self_attn.fused_qkvg_proj", - "model.layers.59.self_attn.g_proj", - "model.layers.59.self_attn.in_proj_qkvgfab", - "model.layers.59.self_attn.kv_a_proj_with_mqa", - "model.layers.59.self_attn.kv_b_proj", - "model.layers.59.self_attn.o_proj", - "model.layers.59.self_attn.q_a_proj", - "model.layers.59.self_attn.q_b_proj", - "model.layers.6.self_attn.b_proj", - "model.layers.6.self_attn.f_a_proj", - "model.layers.6.self_attn.f_b_proj", - "model.layers.6.self_attn.fused_qkvg_proj", - "model.layers.6.self_attn.g_proj", - "model.layers.6.self_attn.in_proj_qkvgfab", - "model.layers.6.self_attn.k_proj", - "model.layers.6.self_attn.o_proj", - "model.layers.6.self_attn.q_proj", - "model.layers.6.self_attn.v_proj", - "model.layers.60.self_attn.b_proj", - "model.layers.60.self_attn.f_a_proj", - "model.layers.60.self_attn.f_b_proj", - "model.layers.60.self_attn.fused_qkvg_proj", - "model.layers.60.self_attn.g_proj", - "model.layers.60.self_attn.in_proj_qkvgfab", - "model.layers.60.self_attn.k_proj", - "model.layers.60.self_attn.o_proj", - "model.layers.60.self_attn.q_proj", - "model.layers.60.self_attn.v_proj", - "model.layers.61.self_attn.b_proj", - "model.layers.61.self_attn.f_a_proj", - "model.layers.61.self_attn.f_b_proj", - "model.layers.61.self_attn.fused_qkvg_proj", - "model.layers.61.self_attn.g_proj", - "model.layers.61.self_attn.in_proj_qkvgfab", - "model.layers.61.self_attn.k_proj", - "model.layers.61.self_attn.o_proj", - "model.layers.61.self_attn.q_proj", - "model.layers.61.self_attn.v_proj", - "model.layers.62.self_attn.b_proj", - "model.layers.62.self_attn.f_a_proj", - "model.layers.62.self_attn.f_b_proj", - "model.layers.62.self_attn.fused_qkvg_proj", - "model.layers.62.self_attn.g_proj", - "model.layers.62.self_attn.in_proj_qkvgfab", - "model.layers.62.self_attn.k_proj", - "model.layers.62.self_attn.o_proj", - "model.layers.62.self_attn.q_proj", - "model.layers.62.self_attn.v_proj", - "model.layers.63.self_attn.fused_qkv_a_proj", - "model.layers.63.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.63.self_attn.fused_qkvg_proj", - "model.layers.63.self_attn.g_proj", - "model.layers.63.self_attn.in_proj_qkvgfab", - "model.layers.63.self_attn.kv_a_proj_with_mqa", - "model.layers.63.self_attn.kv_b_proj", - "model.layers.63.self_attn.o_proj", - "model.layers.63.self_attn.q_a_proj", - "model.layers.63.self_attn.q_b_proj", - "model.layers.64.self_attn.b_proj", - "model.layers.64.self_attn.f_a_proj", - "model.layers.64.self_attn.f_b_proj", - "model.layers.64.self_attn.fused_qkvg_proj", - "model.layers.64.self_attn.g_proj", - "model.layers.64.self_attn.in_proj_qkvgfab", - "model.layers.64.self_attn.k_proj", - "model.layers.64.self_attn.o_proj", - "model.layers.64.self_attn.q_proj", - "model.layers.64.self_attn.v_proj", - "model.layers.65.self_attn.b_proj", - "model.layers.65.self_attn.f_a_proj", - "model.layers.65.self_attn.f_b_proj", - "model.layers.65.self_attn.fused_qkvg_proj", - "model.layers.65.self_attn.g_proj", - "model.layers.65.self_attn.in_proj_qkvgfab", - "model.layers.65.self_attn.k_proj", - "model.layers.65.self_attn.o_proj", - "model.layers.65.self_attn.q_proj", - "model.layers.65.self_attn.v_proj", - "model.layers.66.self_attn.b_proj", - "model.layers.66.self_attn.f_a_proj", - "model.layers.66.self_attn.f_b_proj", - "model.layers.66.self_attn.fused_qkvg_proj", - "model.layers.66.self_attn.g_proj", - "model.layers.66.self_attn.in_proj_qkvgfab", - "model.layers.66.self_attn.k_proj", - "model.layers.66.self_attn.o_proj", - "model.layers.66.self_attn.q_proj", - "model.layers.66.self_attn.v_proj", - "model.layers.67.self_attn.fused_qkv_a_proj", - "model.layers.67.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.67.self_attn.fused_qkvg_proj", - "model.layers.67.self_attn.g_proj", - "model.layers.67.self_attn.in_proj_qkvgfab", - "model.layers.67.self_attn.kv_a_proj_with_mqa", - "model.layers.67.self_attn.kv_b_proj", - "model.layers.67.self_attn.o_proj", - "model.layers.67.self_attn.q_a_proj", - "model.layers.67.self_attn.q_b_proj", - "model.layers.68.self_attn.b_proj", - "model.layers.68.self_attn.f_a_proj", - "model.layers.68.self_attn.f_b_proj", - "model.layers.68.self_attn.fused_qkvg_proj", - "model.layers.68.self_attn.g_proj", - "model.layers.68.self_attn.in_proj_qkvgfab", - "model.layers.68.self_attn.k_proj", - "model.layers.68.self_attn.o_proj", - "model.layers.68.self_attn.q_proj", - "model.layers.68.self_attn.v_proj", - "model.layers.69.self_attn.b_proj", - "model.layers.69.self_attn.f_a_proj", - "model.layers.69.self_attn.f_b_proj", - "model.layers.69.self_attn.fused_qkvg_proj", - "model.layers.69.self_attn.g_proj", - "model.layers.69.self_attn.in_proj_qkvgfab", - "model.layers.69.self_attn.k_proj", - "model.layers.69.self_attn.o_proj", - "model.layers.69.self_attn.q_proj", - "model.layers.69.self_attn.v_proj", - "model.layers.7.self_attn.fused_qkv_a_proj", - "model.layers.7.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.7.self_attn.fused_qkvg_proj", - "model.layers.7.self_attn.g_proj", - "model.layers.7.self_attn.in_proj_qkvgfab", - "model.layers.7.self_attn.kv_a_proj_with_mqa", - "model.layers.7.self_attn.kv_b_proj", - "model.layers.7.self_attn.o_proj", - "model.layers.7.self_attn.q_a_proj", - "model.layers.7.self_attn.q_b_proj", - "model.layers.70.self_attn.b_proj", - "model.layers.70.self_attn.f_a_proj", - "model.layers.70.self_attn.f_b_proj", - "model.layers.70.self_attn.fused_qkvg_proj", - "model.layers.70.self_attn.g_proj", - "model.layers.70.self_attn.in_proj_qkvgfab", - "model.layers.70.self_attn.k_proj", - "model.layers.70.self_attn.o_proj", - "model.layers.70.self_attn.q_proj", - "model.layers.70.self_attn.v_proj", - "model.layers.71.self_attn.fused_qkv_a_proj", - "model.layers.71.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.71.self_attn.fused_qkvg_proj", - "model.layers.71.self_attn.g_proj", - "model.layers.71.self_attn.in_proj_qkvgfab", - "model.layers.71.self_attn.kv_a_proj_with_mqa", - "model.layers.71.self_attn.kv_b_proj", - "model.layers.71.self_attn.o_proj", - "model.layers.71.self_attn.q_a_proj", - "model.layers.71.self_attn.q_b_proj", - "model.layers.72.self_attn.b_proj", - "model.layers.72.self_attn.f_a_proj", - "model.layers.72.self_attn.f_b_proj", - "model.layers.72.self_attn.fused_qkvg_proj", - "model.layers.72.self_attn.g_proj", - "model.layers.72.self_attn.in_proj_qkvgfab", - "model.layers.72.self_attn.k_proj", - "model.layers.72.self_attn.o_proj", - "model.layers.72.self_attn.q_proj", - "model.layers.72.self_attn.v_proj", - "model.layers.73.self_attn.b_proj", - "model.layers.73.self_attn.f_a_proj", - "model.layers.73.self_attn.f_b_proj", - "model.layers.73.self_attn.fused_qkvg_proj", - "model.layers.73.self_attn.g_proj", - "model.layers.73.self_attn.in_proj_qkvgfab", - "model.layers.73.self_attn.k_proj", - "model.layers.73.self_attn.o_proj", - "model.layers.73.self_attn.q_proj", - "model.layers.73.self_attn.v_proj", - "model.layers.74.self_attn.b_proj", - "model.layers.74.self_attn.f_a_proj", - "model.layers.74.self_attn.f_b_proj", - "model.layers.74.self_attn.fused_qkvg_proj", - "model.layers.74.self_attn.g_proj", - "model.layers.74.self_attn.in_proj_qkvgfab", - "model.layers.74.self_attn.k_proj", - "model.layers.74.self_attn.o_proj", - "model.layers.74.self_attn.q_proj", - "model.layers.74.self_attn.v_proj", - "model.layers.75.self_attn.fused_qkv_a_proj", - "model.layers.75.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.75.self_attn.fused_qkvg_proj", - "model.layers.75.self_attn.g_proj", - "model.layers.75.self_attn.in_proj_qkvgfab", - "model.layers.75.self_attn.kv_a_proj_with_mqa", - "model.layers.75.self_attn.kv_b_proj", - "model.layers.75.self_attn.o_proj", - "model.layers.75.self_attn.q_a_proj", - "model.layers.75.self_attn.q_b_proj", - "model.layers.76.self_attn.b_proj", - "model.layers.76.self_attn.f_a_proj", - "model.layers.76.self_attn.f_b_proj", - "model.layers.76.self_attn.fused_qkvg_proj", - "model.layers.76.self_attn.g_proj", - "model.layers.76.self_attn.in_proj_qkvgfab", - "model.layers.76.self_attn.k_proj", - "model.layers.76.self_attn.o_proj", - "model.layers.76.self_attn.q_proj", - "model.layers.76.self_attn.v_proj", - "model.layers.77.self_attn.b_proj", - "model.layers.77.self_attn.f_a_proj", - "model.layers.77.self_attn.f_b_proj", - "model.layers.77.self_attn.fused_qkvg_proj", - "model.layers.77.self_attn.g_proj", - "model.layers.77.self_attn.in_proj_qkvgfab", - "model.layers.77.self_attn.k_proj", - "model.layers.77.self_attn.o_proj", - "model.layers.77.self_attn.q_proj", - "model.layers.77.self_attn.v_proj", - "model.layers.78.self_attn.b_proj", - "model.layers.78.self_attn.f_a_proj", - "model.layers.78.self_attn.f_b_proj", - "model.layers.78.self_attn.fused_qkvg_proj", - "model.layers.78.self_attn.g_proj", - "model.layers.78.self_attn.in_proj_qkvgfab", - "model.layers.78.self_attn.k_proj", - "model.layers.78.self_attn.o_proj", - "model.layers.78.self_attn.q_proj", - "model.layers.78.self_attn.v_proj", - "model.layers.79.self_attn.fused_qkv_a_proj", - "model.layers.79.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.79.self_attn.fused_qkvg_proj", - "model.layers.79.self_attn.g_proj", - "model.layers.79.self_attn.in_proj_qkvgfab", - "model.layers.79.self_attn.kv_a_proj_with_mqa", - "model.layers.79.self_attn.kv_b_proj", - "model.layers.79.self_attn.o_proj", - "model.layers.79.self_attn.q_a_proj", - "model.layers.79.self_attn.q_b_proj", - "model.layers.8.self_attn.b_proj", - "model.layers.8.self_attn.f_a_proj", - "model.layers.8.self_attn.f_b_proj", - "model.layers.8.self_attn.fused_qkvg_proj", - "model.layers.8.self_attn.g_proj", - "model.layers.8.self_attn.in_proj_qkvgfab", - "model.layers.8.self_attn.k_proj", - "model.layers.8.self_attn.o_proj", - "model.layers.8.self_attn.q_proj", - "model.layers.8.self_attn.v_proj", - "model.layers.80.self_attn.b_proj", - "model.layers.80.self_attn.f_a_proj", - "model.layers.80.self_attn.f_b_proj", - "model.layers.80.self_attn.fused_qkvg_proj", - "model.layers.80.self_attn.g_proj", - "model.layers.80.self_attn.in_proj_qkvgfab", - "model.layers.80.self_attn.k_proj", - "model.layers.80.self_attn.o_proj", - "model.layers.80.self_attn.q_proj", - "model.layers.80.self_attn.v_proj", - "model.layers.81.self_attn.b_proj", - "model.layers.81.self_attn.f_a_proj", - "model.layers.81.self_attn.f_b_proj", - "model.layers.81.self_attn.fused_qkvg_proj", - "model.layers.81.self_attn.g_proj", - "model.layers.81.self_attn.in_proj_qkvgfab", - "model.layers.81.self_attn.k_proj", - "model.layers.81.self_attn.o_proj", - "model.layers.81.self_attn.q_proj", - "model.layers.81.self_attn.v_proj", - "model.layers.82.self_attn.b_proj", - "model.layers.82.self_attn.f_a_proj", - "model.layers.82.self_attn.f_b_proj", - "model.layers.82.self_attn.fused_qkvg_proj", - "model.layers.82.self_attn.g_proj", - "model.layers.82.self_attn.in_proj_qkvgfab", - "model.layers.82.self_attn.k_proj", - "model.layers.82.self_attn.o_proj", - "model.layers.82.self_attn.q_proj", - "model.layers.82.self_attn.v_proj", - "model.layers.83.self_attn.fused_qkv_a_proj", - "model.layers.83.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.83.self_attn.fused_qkvg_proj", - "model.layers.83.self_attn.g_proj", - "model.layers.83.self_attn.in_proj_qkvgfab", - "model.layers.83.self_attn.kv_a_proj_with_mqa", - "model.layers.83.self_attn.kv_b_proj", - "model.layers.83.self_attn.o_proj", - "model.layers.83.self_attn.q_a_proj", - "model.layers.83.self_attn.q_b_proj", - "model.layers.84.self_attn.b_proj", - "model.layers.84.self_attn.f_a_proj", - "model.layers.84.self_attn.f_b_proj", - "model.layers.84.self_attn.fused_qkvg_proj", - "model.layers.84.self_attn.g_proj", - "model.layers.84.self_attn.in_proj_qkvgfab", - "model.layers.84.self_attn.k_proj", - "model.layers.84.self_attn.o_proj", - "model.layers.84.self_attn.q_proj", - "model.layers.84.self_attn.v_proj", - "model.layers.85.self_attn.b_proj", - "model.layers.85.self_attn.f_a_proj", - "model.layers.85.self_attn.f_b_proj", - "model.layers.85.self_attn.fused_qkvg_proj", - "model.layers.85.self_attn.g_proj", - "model.layers.85.self_attn.in_proj_qkvgfab", - "model.layers.85.self_attn.k_proj", - "model.layers.85.self_attn.o_proj", - "model.layers.85.self_attn.q_proj", - "model.layers.85.self_attn.v_proj", - "model.layers.86.self_attn.b_proj", - "model.layers.86.self_attn.f_a_proj", - "model.layers.86.self_attn.f_b_proj", - "model.layers.86.self_attn.fused_qkvg_proj", - "model.layers.86.self_attn.g_proj", - "model.layers.86.self_attn.in_proj_qkvgfab", - "model.layers.86.self_attn.k_proj", - "model.layers.86.self_attn.o_proj", - "model.layers.86.self_attn.q_proj", - "model.layers.86.self_attn.v_proj", - "model.layers.87.self_attn.fused_qkv_a_proj", - "model.layers.87.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.87.self_attn.fused_qkvg_proj", - "model.layers.87.self_attn.g_proj", - "model.layers.87.self_attn.in_proj_qkvgfab", - "model.layers.87.self_attn.kv_a_proj_with_mqa", - "model.layers.87.self_attn.kv_b_proj", - "model.layers.87.self_attn.o_proj", - "model.layers.87.self_attn.q_a_proj", - "model.layers.87.self_attn.q_b_proj", - "model.layers.88.self_attn.b_proj", - "model.layers.88.self_attn.f_a_proj", - "model.layers.88.self_attn.f_b_proj", - "model.layers.88.self_attn.fused_qkvg_proj", - "model.layers.88.self_attn.g_proj", - "model.layers.88.self_attn.in_proj_qkvgfab", - "model.layers.88.self_attn.k_proj", - "model.layers.88.self_attn.o_proj", - "model.layers.88.self_attn.q_proj", - "model.layers.88.self_attn.v_proj", - "model.layers.89.self_attn.b_proj", - "model.layers.89.self_attn.f_a_proj", - "model.layers.89.self_attn.f_b_proj", - "model.layers.89.self_attn.fused_qkvg_proj", - "model.layers.89.self_attn.g_proj", - "model.layers.89.self_attn.in_proj_qkvgfab", - "model.layers.89.self_attn.k_proj", - "model.layers.89.self_attn.o_proj", - "model.layers.89.self_attn.q_proj", - "model.layers.89.self_attn.v_proj", - "model.layers.9.self_attn.b_proj", - "model.layers.9.self_attn.f_a_proj", - "model.layers.9.self_attn.f_b_proj", - "model.layers.9.self_attn.fused_qkvg_proj", - "model.layers.9.self_attn.g_proj", - "model.layers.9.self_attn.in_proj_qkvgfab", - "model.layers.9.self_attn.k_proj", - "model.layers.9.self_attn.o_proj", - "model.layers.9.self_attn.q_proj", - "model.layers.9.self_attn.v_proj", - "model.layers.90.self_attn.b_proj", - "model.layers.90.self_attn.f_a_proj", - "model.layers.90.self_attn.f_b_proj", - "model.layers.90.self_attn.fused_qkvg_proj", - "model.layers.90.self_attn.g_proj", - "model.layers.90.self_attn.in_proj_qkvgfab", - "model.layers.90.self_attn.k_proj", - "model.layers.90.self_attn.o_proj", - "model.layers.90.self_attn.q_proj", - "model.layers.90.self_attn.v_proj", - "model.layers.91.self_attn.fused_qkv_a_proj", - "model.layers.91.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.91.self_attn.fused_qkvg_proj", - "model.layers.91.self_attn.g_proj", - "model.layers.91.self_attn.in_proj_qkvgfab", - "model.layers.91.self_attn.kv_a_proj_with_mqa", - "model.layers.91.self_attn.kv_b_proj", - "model.layers.91.self_attn.o_proj", - "model.layers.91.self_attn.q_a_proj", - "model.layers.91.self_attn.q_b_proj", - "model.layers.92.self_attn.fused_qkv_a_proj", - "model.layers.92.self_attn.fused_qkv_a_proj_with_mqa", - "model.layers.92.self_attn.fused_qkvg_proj", - "model.layers.92.self_attn.g_proj", - "model.layers.92.self_attn.in_proj_qkvgfab", - "model.layers.92.self_attn.kv_a_proj_with_mqa", - "model.layers.92.self_attn.kv_b_proj", - "model.layers.92.self_attn.o_proj", - "model.layers.92.self_attn.q_a_proj", - "model.layers.92.self_attn.q_b_proj" - ] - } - }, - "ignore": [ - "*block_sparse_moe.gate*", - "*mlp.gate*", - "*block_sparse_moe.routed_expert_up_proj*", - "*mlp.routed_expert_up_proj*", - "*block_sparse_moe.routed_expert_down_proj*", - "*mlp.routed_expert_down_proj*", - "*block_sparse_moe.routed_expert_norm*", - "*mlp.routed_expert_norm*", - "*block_sparse_moe.shared_experts*", - "*mlp.shared_experts*", - "*self_attn.q_conv1d*", - "*self_attn.k_conv1d*", - "*self_attn.v_conv1d*", - "*self_attn.o_norm*", - "*self_attn.q_a_layernorm*", - "*self_attn.kv_a_layernorm*", - "*_res_proj*", - "*_res_norm*", - "*layernorm*", - "language_model.model.layers.0.mlp.*", - "layers.0.mlp.*", - "model.layers.0.mlp.*", - "language_model.lm_head", - "vision_tower*", - "mm_projector*" - ], - "producer": { - "name": "modelopt", - "version": "0.45.0" - }, - "quant_algo": "MIXED_PRECISION", - "quant_method": "modelopt_mixed", - "quantized_layers": { - "language_model.model.layers.0.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.0.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.1.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.1.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.1.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.10.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.10.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.10.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.11.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.11.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.11.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.12.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.12.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.12.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.13.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.13.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.13.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.14.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.14.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.14.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.15.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.15.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.15.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.16.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.16.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.16.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.17.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.17.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.17.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.18.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.18.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.18.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.19.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.19.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.19.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.2.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.2.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.2.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.20.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.20.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.20.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.21.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.21.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.21.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.22.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.22.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.22.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.23.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.23.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.23.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.24.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.24.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.24.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.25.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.25.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.25.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.26.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.26.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.26.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.27.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.27.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.27.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.28.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.28.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.28.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.29.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.29.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.29.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.3.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.3.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.3.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.30.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.30.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.30.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.31.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.31.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.31.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.32.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.32.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.32.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.33.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.33.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.33.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.34.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.34.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.34.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.35.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.35.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.35.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.36.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.36.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.36.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.37.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.37.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.37.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.38.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.38.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.38.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.39.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.39.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.39.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.4.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.4.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.4.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.40.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.40.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.40.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.41.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.41.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.41.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.42.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.42.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.42.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.43.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.43.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.43.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.44.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.44.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.44.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.45.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.45.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.45.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.46.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.46.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.46.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.47.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.47.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.47.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.48.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.48.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.48.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.49.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.49.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.49.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.5.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.5.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.5.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.50.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.50.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.50.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.51.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.51.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.51.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.52.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.52.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.52.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.53.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.53.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.53.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.54.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.54.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.54.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.55.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.55.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.55.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.56.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.56.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.56.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.57.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.57.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.57.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.58.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.58.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.58.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.59.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.59.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.59.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.6.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.6.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.6.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.60.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.60.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.60.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.61.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.61.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.61.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.62.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.62.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.62.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.63.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.63.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.63.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.64.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.64.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.64.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.65.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.65.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.65.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.66.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.66.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.66.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.67.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.67.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.67.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.68.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.68.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.68.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.69.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.69.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.69.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.7.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.7.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.7.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.70.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.70.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.70.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.71.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.71.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.71.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.72.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.72.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.72.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.73.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.73.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.73.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.74.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.74.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.74.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.75.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.75.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.75.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.76.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.76.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.76.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.77.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.77.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.77.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.78.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.78.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.78.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.79.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.79.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.79.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.8.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.8.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.8.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.80.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.80.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.80.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.81.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.81.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.81.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.82.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.82.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.82.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.83.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.83.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.83.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.84.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.84.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.84.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.85.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.85.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.85.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.86.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.86.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.86.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.87.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.87.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.87.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.88.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.88.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.88.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.89.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.89.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.89.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.9.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.9.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.9.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.90.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.90.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.90.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.91.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.91.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.91.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.92.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "language_model.model.layers.92.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "language_model.model.layers.92.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.0.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.1.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.1.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.1.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.10.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.10.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.10.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.11.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.11.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.11.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.12.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.12.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.12.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.13.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.13.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.13.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.14.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.14.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.14.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.15.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.15.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.15.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.16.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.16.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.16.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.17.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.17.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.17.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.18.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.18.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.18.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.19.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.19.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.19.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.2.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.2.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.2.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.20.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.20.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.20.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.21.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.21.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.21.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.22.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.22.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.22.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.23.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.23.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.23.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.24.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.24.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.24.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.25.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.25.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.25.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.26.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.26.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.26.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.27.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.27.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.27.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.28.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.28.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.28.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.29.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.29.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.29.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.3.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.3.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.3.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.30.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.30.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.30.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.31.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.31.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.31.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.32.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.32.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.32.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.33.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.33.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.33.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.34.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.34.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.34.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.35.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.35.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.35.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.36.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.36.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.36.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.37.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.37.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.37.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.38.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.38.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.38.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.39.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.39.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.39.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.4.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.4.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.4.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.40.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.40.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.40.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.41.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.41.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.41.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.42.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.42.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.42.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.43.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.43.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.43.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.44.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.44.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.44.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.45.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.45.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.45.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.46.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.46.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.46.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.47.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.47.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.47.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.48.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.48.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.48.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.49.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.49.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.49.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.5.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.5.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.5.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.50.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.50.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.50.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.51.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.51.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.51.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.52.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.52.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.52.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.53.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.53.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.53.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.54.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.54.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.54.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.55.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.55.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.55.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.56.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.56.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.56.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.57.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.57.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.57.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.58.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.58.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.58.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.59.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.59.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.59.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.6.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.6.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.6.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.60.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.60.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.60.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.61.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.61.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.61.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.62.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.62.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.62.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.63.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.63.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.63.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.64.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.64.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.64.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.65.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.65.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.65.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.66.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.66.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.66.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.67.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.67.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.67.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.68.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.68.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.68.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.69.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.69.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.69.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.7.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.7.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.7.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.70.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.70.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.70.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.71.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.71.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.71.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.72.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.72.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.72.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.73.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.73.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.73.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.74.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.74.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.74.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.75.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.75.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.75.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.76.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.76.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.76.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.77.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.77.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.77.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.78.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.78.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.78.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.79.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.79.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.79.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.8.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.8.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.8.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.80.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.80.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.80.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.81.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.81.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.81.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.82.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.82.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.82.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.83.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.83.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.83.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.84.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.84.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.84.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.85.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.85.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.85.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.86.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.86.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.86.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.87.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.87.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.87.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.88.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.88.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.88.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.89.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.89.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.89.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.9.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.9.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.9.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.90.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.90.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.90.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.91.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.91.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.91.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.92.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "layers.92.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "layers.92.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.0.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.1.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.1.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.1.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.10.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.10.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.10.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.11.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.11.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.11.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.12.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.12.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.12.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.13.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.13.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.13.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.14.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.14.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.14.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.15.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.15.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.15.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.16.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.16.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.16.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.17.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.17.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.17.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.18.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.18.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.18.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.19.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.19.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.19.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.2.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.2.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.2.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.20.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.20.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.20.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.21.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.21.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.21.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.22.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.22.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.22.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.23.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.23.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.23.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.24.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.24.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.24.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.25.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.25.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.25.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.26.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.26.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.26.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.27.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.27.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.27.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.28.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.28.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.28.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.29.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.29.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.29.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.3.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.3.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.3.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.30.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.30.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.30.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.31.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.31.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.31.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.32.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.32.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.32.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.33.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.33.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.33.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.34.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.34.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.34.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.35.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.35.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.35.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.36.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.36.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.36.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.37.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.37.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.37.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.38.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.38.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.38.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.39.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.39.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.39.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.4.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.4.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.4.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.40.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.40.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.40.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.41.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.41.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.41.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.42.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.42.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.42.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.43.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.43.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.43.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.44.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.44.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.44.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.45.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.45.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.45.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.46.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.46.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.46.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.47.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.47.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.47.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.48.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.48.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.48.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.49.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.49.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.49.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.5.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.5.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.5.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.50.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.50.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.50.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.51.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.51.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.51.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.52.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.52.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.52.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.53.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.53.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.53.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.54.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.54.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.54.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.55.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.55.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.55.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.56.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.56.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.56.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.57.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.57.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.57.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.58.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.58.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.58.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.59.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.59.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.59.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.6.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.6.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.6.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.60.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.60.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.60.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.61.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.61.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.61.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.62.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.62.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.62.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.63.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.63.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.63.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.64.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.64.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.64.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.65.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.65.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.65.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.66.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.66.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.66.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.67.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.67.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.67.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.68.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.68.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.68.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.69.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.69.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.69.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.7.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.7.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.7.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.70.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.70.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.70.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.71.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.71.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.71.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.72.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.72.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.72.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.73.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.73.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.73.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.74.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.74.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.74.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.75.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.75.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.75.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.76.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.76.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.76.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.77.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.77.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.77.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.78.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.78.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.78.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.79.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.79.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.79.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.8.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.8.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.8.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.80.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.80.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.80.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.81.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.81.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.81.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.82.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.82.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.82.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.83.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.83.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.83.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.84.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.84.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.84.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.85.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.85.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.85.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.86.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.86.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.86.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.87.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.87.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.87.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.88.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.88.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.88.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.89.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.89.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.89.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.9.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.9.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.9.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.90.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.90.self_attn.b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.f_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.f_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.k_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.q_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.90.self_attn.v_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.91.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.91.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.91.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.block_sparse_moe.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.92.mlp.experts": { - "group_size": 16, - "quant_algo": "NVFP4" - }, - "model.layers.92.self_attn.fused_qkv_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.fused_qkv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.fused_qkvg_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.g_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.in_proj_qkvgfab": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.kv_a_proj_with_mqa": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.kv_b_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.o_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.q_a_proj": { - "quant_algo": "FP8_PB_WO" - }, - "model.layers.92.self_attn.q_b_proj": { - "quant_algo": "FP8_PB_WO" - } - } - }, - "text_config": { - "_name_or_path": "", - "activation_situ_beta": 4.0, - "activation_situ_linear_beta": 25.0, - "add_cross_attention": false, - "architectures": [ - "KimiLinearForCausalLM" - ], - "attn_res_block_size": 12, - "auto_map": { - "AutoConfig": "configuration_kimi_k3.KimiLinearConfig", - "AutoModel": "modeling_kimi_linear.KimiLinearModel", - "AutoModelForCausalLM": "modeling_kimi_linear.KimiLinearForCausalLM" - }, - "bad_words_ids": null, - "begin_suppress_tokens": null, - "bos_token_id": 163584, - "chunk_size_feed_forward": 0, - "cross_attention_hidden_size": null, - "decoder_start_token_id": null, - "diversity_penalty": 0.0, - "do_sample": false, - "dtype": "bfloat16", - "early_stopping": false, - "encoder_no_repeat_ngram_size": 0, - "eos_token_id": 163586, - "exponential_decay_length_penalty": null, - "finetuning_task": null, - "first_k_dense_replace": 1, - "forced_bos_token_id": null, - "forced_eos_token_id": null, - "hidden_act": "situ", - "hidden_size": 7168, - "id2label": { - "0": "LABEL_0", - "1": "LABEL_1" - }, - "initializer_range": 0.02, - "intermediate_size": 33792, - "is_decoder": false, - "is_encoder_decoder": false, - "kv_lora_rank": 512, - "label2id": { - "LABEL_0": 0, - "LABEL_1": 1 - }, - "latent_moe_use_norm": true, - "length_penalty": 1.0, - "linear_attn_config": { - "full_attn_layers": [ - 4, - 8, - 12, - 16, - 20, - 24, - 28, - 32, - 36, - 40, - 44, - 48, - 52, - 56, - 60, - 64, - 68, - 72, - 76, - 80, - 84, - 88, - 92, - 93 - ], - "gate_lower_bound": -5.0, - "head_dim": 128, - "kda_layers": [ - 1, - 2, - 3, - 5, - 6, - 7, - 9, - 10, - 11, - 13, - 14, - 15, - 17, - 18, - 19, - 21, - 22, - 23, - 25, - 26, - 27, - 29, - 30, - 31, - 33, - 34, - 35, - 37, - 38, - 39, - 41, - 42, - 43, - 45, - 46, - 47, - 49, - 50, - 51, - 53, - 54, - 55, - 57, - 58, - 59, - 61, - 62, - 63, - 65, - 66, - 67, - 69, - 70, - 71, - 73, - 74, - 75, - 77, - 78, - 79, - 81, - 82, - 83, - 85, - 86, - 87, - 89, - 90, - 91 - ], - "num_heads": 96, - "short_conv_kernel_size": 4, - "use_full_rank_gate": true - }, - "max_length": 20, - "max_position_embeddings": 1048576, - "min_length": 0, - "mla_use_nope": true, - "mla_use_output_gate": true, - "model_type": "kimi_linear", - "moe_intermediate_size": 3072, - "moe_layer_freq": 1, - "moe_renormalize": true, - "moe_router_activation_func": "sigmoid", - "no_repeat_ngram_size": 0, - "num_attention_heads": 96, - "num_beam_groups": 1, - "num_beams": 1, - "num_expert_group": 1, - "num_experts": 896, - "num_experts_per_token": 16, - "num_hidden_layers": 93, - "num_key_value_heads": 96, - "num_nextn_predict_layers": 0, - "num_return_sequences": 1, - "num_shared_experts": 2, - "output_attentions": false, - "output_hidden_states": false, - "output_scores": false, - "pad_token_id": 163839, - "prefix": null, - "problem_type": null, - "pruned_heads": {}, - "q_lora_rank": 1536, - "qk_nope_head_dim": 128, - "qk_rope_head_dim": 64, - "remove_invalid_values": false, - "repetition_penalty": 1.0, - "return_dict": true, - "return_dict_in_generate": false, - "rms_norm_eps": 1e-05, - "routed_expert_hidden_size": 3584, - "routed_scaling_factor": 1.0, - "sep_token_id": null, - "suppress_tokens": null, - "task_specific_params": null, - "temperature": 1.0, - "tf_legacy_loss": false, - "tie_encoder_decoder": false, - "tie_word_embeddings": false, - "tokenizer_class": null, - "top_k": 50, - "top_p": 1.0, - "topk_group": 1, - "topk_method": "noaux_tc", - "torchscript": false, - "transformers_version": "4.56.2", - "typical_p": 1.0, - "use_bfloat16": false, - "use_cache": true, - "use_grouped_topk": true, - "v_head_dim": 128, - "vocab_size": 163840 - }, - "tie_word_embeddings": false, - "vision_config": { - "_attn_implementation": "flash_attention_2", - "activation_func": "gelu_pytorch_tanh", - "attn_bias": false, - "init_pos_emb_height": 64, - "init_pos_emb_time": 4, - "init_pos_emb_width": 64, - "linear_bias": false, - "merge_kernel_size": [ - 2, - 2 - ], - "merge_type": "sd2_tpool", - "mlp_type": "mlp2", - "mm_hidden_size": 1024, - "mm_projector_type": "patchmergerv2", - "norm_type": "rmsnorm", - "patch_embed_proj_bias": false, - "patch_size": 14, - "pos_emb_interpolation_mode": "bilinear", - "pos_emb_type": "divided_fixed", - "projector_hidden_act": "gelu", - "projector_ln_eps": 1e-05, - "qkv_hidden_size": 1536, - "text_hidden_size": 7168, - "vt_hidden_size": 1024, - "vt_intermediate_size": 4096, - "vt_num_attention_heads": 12, - "vt_num_hidden_layers": 27 - } -} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json deleted file mode 100644 index 993b612989..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json +++ /dev/null @@ -1,121 +0,0 @@ -{ - "architectures": [ - "NemotronHForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "bos_token_id": 1, - "chunk_size": 128, - "conv_kernel": 4, - "dtype": "bfloat16", - "eos_token_id": 2, - "expand": 2, - "head_dim": 128, - "hidden_dropout": 0.0, - "hidden_size": 2688, - "initializer_range": 0.02, - "intermediate_size": 1856, - "layer_norm_epsilon": 1e-05, - "mamba_head_dim": 64, - "mamba_hidden_act": "silu", - "mamba_num_heads": 64, - "mamba_proj_bias": false, - "mamba_ssm_cache_dtype": "float32", - "max_position_embeddings": 262144, - "mlp_bias": false, - "mlp_hidden_act": "relu2", - "model_type": "nemotron_h", - "moe_intermediate_size": 1856, - "moe_latent_size": null, - "moe_shared_expert_intermediate_size": 3712, - "moe_shared_expert_overlap": true, - "n_group": 1, - "n_groups": 8, - "n_routed_experts": 128, - "n_shared_experts": 1, - "norm_eps": 1e-05, - "norm_topk_prob": true, - "num_attention_heads": 32, - "num_experts_per_tok": 6, - "num_hidden_layers": 52, - "num_key_value_heads": 2, - "num_logits_to_keep": 1, - "num_nextn_predict_layers": 1, - "pad_token_id": 0, - "partial_rotary_factor": 1.0, - "rescale_prenorm_residual": true, - "residual_in_fp32": false, - "rope_theta": 10000, - "routed_scaling_factor": 2.5, - "sliding_window": null, - "ssm_state_size": 128, - "tie_word_embeddings": false, - "time_step_floor": 0.0001, - "time_step_max": 0.1, - "time_step_min": 0.001, - "topk_group": 1, - "transformers_version": "4.57.6", - "use_bias": false, - "use_cache": true, - "use_conv_bias": true, - "use_mamba_kernels": true, - "vocab_size": 131072, - "layers_block_type": [ - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "attention", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "attention", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "attention", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "attention", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "attention", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "attention", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "moe", - "mamba", - "moe" - ], - "mtp_layers_block_type": [ - "attention", - "moe" - ] -} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json deleted file mode 100644 index 2b5e5dac5b..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_config.json +++ /dev/null @@ -1,2690 +0,0 @@ -{ - "architectures": [ - "Qwen3_5MoeForCausalLM" - ], - "attention_bias": false, - "attention_dropout": 0.0, - "attn_output_gate": true, - "base_model_ep_plan": { - "layers.*.mlp.experts": "torch_a2a_experts", - "layers.*.mlp.experts.down_proj": "grouped_gemm", - "layers.*.mlp.experts.gate_up_proj": "grouped_gemm", - "layers.*.mlp.gate": "torch_a2a_router" - }, - "bos_token_id": 248044, - "distributed_config": null, - "dtype": "bfloat16", - "eos_token_id": 248044, - "full_attention_interval": 4, - "head_dim": 256, - "hidden_act": "silu", - "hidden_size": 8192, - "initializer_range": 0.02, - "layer_types": [ - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention" - ], - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 128, - "linear_value_head_dim": 128, - "mamba_ssm_dtype": "float32", - "max_position_embeddings": 262144, - "model_type": "qwen3_5_moe_text", - "moe_intermediate_size": 2048, - "mtp_num_hidden_layers": 1, - "mtp_use_dedicated_embeddings": false, - "num_attention_heads": 64, - "num_experts": 512, - "num_experts_per_tok": 10, - "num_hidden_layers": 92, - "num_key_value_heads": 4, - "output_gate_type": "swish", - "output_router_logits": false, - "pad_token_id": null, - "partial_rotary_factor": 0.25, - "rms_norm_eps": 1e-06, - "rope_parameters": { - "mrope_interleaved": true, - "mrope_section": [ - 11, - 11, - 10 - ], - "partial_rotary_factor": 0.25, - "rope_theta": 10000000, - "rope_type": "default" - }, - "router_aux_loss_coef": 0.001, - "shared_expert_intermediate_size": 2048, - "tie_word_embeddings": false, - "transformers_version": "5.13.0", - "use_cache": true, - "vocab_size": 248320, - "quantization_config": { - "config_groups": { - "group_0": { - "input_activations": { - "dynamic": false, - "num_bits": 8, - "type": "float" - }, - "weights": { - "dynamic": false, - "num_bits": 8, - "type": "float" - }, - "targets": [ - "model.layers.0.linear_attn.conv1d", - "model.layers.0.linear_attn.in_proj_a", - "model.layers.0.linear_attn.in_proj_b", - "model.layers.0.linear_attn.in_proj_qkv", - "model.layers.0.linear_attn.in_proj_z", - "model.layers.0.linear_attn.out_proj", - "model.layers.1.linear_attn.conv1d", - "model.layers.1.linear_attn.in_proj_a", - "model.layers.1.linear_attn.in_proj_b", - "model.layers.1.linear_attn.in_proj_qkv", - "model.layers.1.linear_attn.in_proj_z", - "model.layers.1.linear_attn.out_proj", - "model.layers.10.linear_attn.conv1d", - "model.layers.10.linear_attn.in_proj_a", - "model.layers.10.linear_attn.in_proj_b", - "model.layers.10.linear_attn.in_proj_qkv", - "model.layers.10.linear_attn.in_proj_z", - "model.layers.10.linear_attn.out_proj", - "model.layers.11.self_attn.k_proj", - "model.layers.11.self_attn.o_proj", - "model.layers.11.self_attn.q_proj", - "model.layers.11.self_attn.v_proj", - "model.layers.12.linear_attn.conv1d", - "model.layers.12.linear_attn.in_proj_a", - "model.layers.12.linear_attn.in_proj_b", - "model.layers.12.linear_attn.in_proj_qkv", - "model.layers.12.linear_attn.in_proj_z", - "model.layers.12.linear_attn.out_proj", - "model.layers.13.linear_attn.conv1d", - "model.layers.13.linear_attn.in_proj_a", - "model.layers.13.linear_attn.in_proj_b", - "model.layers.13.linear_attn.in_proj_qkv", - "model.layers.13.linear_attn.in_proj_z", - "model.layers.13.linear_attn.out_proj", - "model.layers.14.linear_attn.conv1d", - "model.layers.14.linear_attn.in_proj_a", - "model.layers.14.linear_attn.in_proj_b", - "model.layers.14.linear_attn.in_proj_qkv", - "model.layers.14.linear_attn.in_proj_z", - "model.layers.14.linear_attn.out_proj", - "model.layers.15.self_attn.k_proj", - "model.layers.15.self_attn.o_proj", - "model.layers.15.self_attn.q_proj", - "model.layers.15.self_attn.v_proj", - "model.layers.16.linear_attn.conv1d", - "model.layers.16.linear_attn.in_proj_a", - "model.layers.16.linear_attn.in_proj_b", - "model.layers.16.linear_attn.in_proj_qkv", - "model.layers.16.linear_attn.in_proj_z", - "model.layers.16.linear_attn.out_proj", - "model.layers.17.linear_attn.conv1d", - "model.layers.17.linear_attn.in_proj_a", - "model.layers.17.linear_attn.in_proj_b", - "model.layers.17.linear_attn.in_proj_qkv", - "model.layers.17.linear_attn.in_proj_z", - "model.layers.17.linear_attn.out_proj", - "model.layers.18.linear_attn.conv1d", - "model.layers.18.linear_attn.in_proj_a", - "model.layers.18.linear_attn.in_proj_b", - "model.layers.18.linear_attn.in_proj_qkv", - "model.layers.18.linear_attn.in_proj_z", - "model.layers.18.linear_attn.out_proj", - "model.layers.19.self_attn.k_proj", - "model.layers.19.self_attn.o_proj", - "model.layers.19.self_attn.q_proj", - "model.layers.19.self_attn.v_proj", - "model.layers.2.linear_attn.conv1d", - "model.layers.2.linear_attn.in_proj_a", - "model.layers.2.linear_attn.in_proj_b", - "model.layers.2.linear_attn.in_proj_qkv", - "model.layers.2.linear_attn.in_proj_z", - "model.layers.2.linear_attn.out_proj", - "model.layers.20.linear_attn.conv1d", - "model.layers.20.linear_attn.in_proj_a", - "model.layers.20.linear_attn.in_proj_b", - "model.layers.20.linear_attn.in_proj_qkv", - "model.layers.20.linear_attn.in_proj_z", - "model.layers.20.linear_attn.out_proj", - "model.layers.21.linear_attn.conv1d", - "model.layers.21.linear_attn.in_proj_a", - "model.layers.21.linear_attn.in_proj_b", - "model.layers.21.linear_attn.in_proj_qkv", - "model.layers.21.linear_attn.in_proj_z", - "model.layers.21.linear_attn.out_proj", - "model.layers.22.linear_attn.conv1d", - "model.layers.22.linear_attn.in_proj_a", - "model.layers.22.linear_attn.in_proj_b", - "model.layers.22.linear_attn.in_proj_qkv", - "model.layers.22.linear_attn.in_proj_z", - "model.layers.22.linear_attn.out_proj", - "model.layers.23.self_attn.k_proj", - "model.layers.23.self_attn.o_proj", - "model.layers.23.self_attn.q_proj", - "model.layers.23.self_attn.v_proj", - "model.layers.24.linear_attn.conv1d", - "model.layers.24.linear_attn.in_proj_a", - "model.layers.24.linear_attn.in_proj_b", - "model.layers.24.linear_attn.in_proj_qkv", - "model.layers.24.linear_attn.in_proj_z", - "model.layers.24.linear_attn.out_proj", - "model.layers.25.linear_attn.conv1d", - "model.layers.25.linear_attn.in_proj_a", - "model.layers.25.linear_attn.in_proj_b", - "model.layers.25.linear_attn.in_proj_qkv", - "model.layers.25.linear_attn.in_proj_z", - "model.layers.25.linear_attn.out_proj", - "model.layers.26.linear_attn.conv1d", - "model.layers.26.linear_attn.in_proj_a", - "model.layers.26.linear_attn.in_proj_b", - "model.layers.26.linear_attn.in_proj_qkv", - "model.layers.26.linear_attn.in_proj_z", - "model.layers.26.linear_attn.out_proj", - "model.layers.27.self_attn.k_proj", - "model.layers.27.self_attn.o_proj", - "model.layers.27.self_attn.q_proj", - "model.layers.27.self_attn.v_proj", - "model.layers.28.linear_attn.conv1d", - "model.layers.28.linear_attn.in_proj_a", - "model.layers.28.linear_attn.in_proj_b", - "model.layers.28.linear_attn.in_proj_qkv", - "model.layers.28.linear_attn.in_proj_z", - "model.layers.28.linear_attn.out_proj", - "model.layers.29.linear_attn.conv1d", - "model.layers.29.linear_attn.in_proj_a", - "model.layers.29.linear_attn.in_proj_b", - "model.layers.29.linear_attn.in_proj_qkv", - "model.layers.29.linear_attn.in_proj_z", - "model.layers.29.linear_attn.out_proj", - "model.layers.3.self_attn.k_proj", - "model.layers.3.self_attn.o_proj", - "model.layers.3.self_attn.q_proj", - "model.layers.3.self_attn.v_proj", - "model.layers.30.linear_attn.conv1d", - "model.layers.30.linear_attn.in_proj_a", - "model.layers.30.linear_attn.in_proj_b", - "model.layers.30.linear_attn.in_proj_qkv", - "model.layers.30.linear_attn.in_proj_z", - "model.layers.30.linear_attn.out_proj", - "model.layers.31.self_attn.k_proj", - "model.layers.31.self_attn.o_proj", - "model.layers.31.self_attn.q_proj", - "model.layers.31.self_attn.v_proj", - "model.layers.32.linear_attn.conv1d", - "model.layers.32.linear_attn.in_proj_a", - "model.layers.32.linear_attn.in_proj_b", - "model.layers.32.linear_attn.in_proj_qkv", - "model.layers.32.linear_attn.in_proj_z", - "model.layers.32.linear_attn.out_proj", - "model.layers.33.linear_attn.conv1d", - "model.layers.33.linear_attn.in_proj_a", - "model.layers.33.linear_attn.in_proj_b", - "model.layers.33.linear_attn.in_proj_qkv", - "model.layers.33.linear_attn.in_proj_z", - "model.layers.33.linear_attn.out_proj", - "model.layers.34.linear_attn.conv1d", - "model.layers.34.linear_attn.in_proj_a", - "model.layers.34.linear_attn.in_proj_b", - "model.layers.34.linear_attn.in_proj_qkv", - "model.layers.34.linear_attn.in_proj_z", - "model.layers.34.linear_attn.out_proj", - "model.layers.35.self_attn.k_proj", - "model.layers.35.self_attn.o_proj", - "model.layers.35.self_attn.q_proj", - "model.layers.35.self_attn.v_proj", - "model.layers.36.linear_attn.conv1d", - "model.layers.36.linear_attn.in_proj_a", - "model.layers.36.linear_attn.in_proj_b", - "model.layers.36.linear_attn.in_proj_qkv", - "model.layers.36.linear_attn.in_proj_z", - "model.layers.36.linear_attn.out_proj", - "model.layers.37.linear_attn.conv1d", - "model.layers.37.linear_attn.in_proj_a", - "model.layers.37.linear_attn.in_proj_b", - "model.layers.37.linear_attn.in_proj_qkv", - "model.layers.37.linear_attn.in_proj_z", - "model.layers.37.linear_attn.out_proj", - "model.layers.38.linear_attn.conv1d", - "model.layers.38.linear_attn.in_proj_a", - "model.layers.38.linear_attn.in_proj_b", - "model.layers.38.linear_attn.in_proj_qkv", - "model.layers.38.linear_attn.in_proj_z", - "model.layers.38.linear_attn.out_proj", - "model.layers.39.self_attn.k_proj", - "model.layers.39.self_attn.o_proj", - "model.layers.39.self_attn.q_proj", - "model.layers.39.self_attn.v_proj", - "model.layers.4.linear_attn.conv1d", - "model.layers.4.linear_attn.in_proj_a", - "model.layers.4.linear_attn.in_proj_b", - "model.layers.4.linear_attn.in_proj_qkv", - "model.layers.4.linear_attn.in_proj_z", - "model.layers.4.linear_attn.out_proj", - "model.layers.40.linear_attn.conv1d", - "model.layers.40.linear_attn.in_proj_a", - "model.layers.40.linear_attn.in_proj_b", - "model.layers.40.linear_attn.in_proj_qkv", - "model.layers.40.linear_attn.in_proj_z", - "model.layers.40.linear_attn.out_proj", - "model.layers.41.linear_attn.conv1d", - "model.layers.41.linear_attn.in_proj_a", - "model.layers.41.linear_attn.in_proj_b", - "model.layers.41.linear_attn.in_proj_qkv", - "model.layers.41.linear_attn.in_proj_z", - "model.layers.41.linear_attn.out_proj", - "model.layers.42.linear_attn.conv1d", - "model.layers.42.linear_attn.in_proj_a", - "model.layers.42.linear_attn.in_proj_b", - "model.layers.42.linear_attn.in_proj_qkv", - "model.layers.42.linear_attn.in_proj_z", - "model.layers.42.linear_attn.out_proj", - "model.layers.43.self_attn.k_proj", - "model.layers.43.self_attn.o_proj", - "model.layers.43.self_attn.q_proj", - "model.layers.43.self_attn.v_proj", - "model.layers.44.linear_attn.conv1d", - "model.layers.44.linear_attn.in_proj_a", - "model.layers.44.linear_attn.in_proj_b", - "model.layers.44.linear_attn.in_proj_qkv", - "model.layers.44.linear_attn.in_proj_z", - "model.layers.44.linear_attn.out_proj", - "model.layers.45.linear_attn.conv1d", - "model.layers.45.linear_attn.in_proj_a", - "model.layers.45.linear_attn.in_proj_b", - "model.layers.45.linear_attn.in_proj_qkv", - "model.layers.45.linear_attn.in_proj_z", - "model.layers.45.linear_attn.out_proj", - "model.layers.46.linear_attn.conv1d", - "model.layers.46.linear_attn.in_proj_a", - "model.layers.46.linear_attn.in_proj_b", - "model.layers.46.linear_attn.in_proj_qkv", - "model.layers.46.linear_attn.in_proj_z", - "model.layers.46.linear_attn.out_proj", - "model.layers.47.self_attn.k_proj", - "model.layers.47.self_attn.o_proj", - "model.layers.47.self_attn.q_proj", - "model.layers.47.self_attn.v_proj", - "model.layers.48.linear_attn.conv1d", - "model.layers.48.linear_attn.in_proj_a", - "model.layers.48.linear_attn.in_proj_b", - "model.layers.48.linear_attn.in_proj_qkv", - "model.layers.48.linear_attn.in_proj_z", - "model.layers.48.linear_attn.out_proj", - "model.layers.49.linear_attn.conv1d", - "model.layers.49.linear_attn.in_proj_a", - "model.layers.49.linear_attn.in_proj_b", - "model.layers.49.linear_attn.in_proj_qkv", - "model.layers.49.linear_attn.in_proj_z", - "model.layers.49.linear_attn.out_proj", - "model.layers.5.linear_attn.conv1d", - "model.layers.5.linear_attn.in_proj_a", - "model.layers.5.linear_attn.in_proj_b", - "model.layers.5.linear_attn.in_proj_qkv", - "model.layers.5.linear_attn.in_proj_z", - "model.layers.5.linear_attn.out_proj", - "model.layers.50.linear_attn.conv1d", - "model.layers.50.linear_attn.in_proj_a", - "model.layers.50.linear_attn.in_proj_b", - "model.layers.50.linear_attn.in_proj_qkv", - "model.layers.50.linear_attn.in_proj_z", - "model.layers.50.linear_attn.out_proj", - "model.layers.51.self_attn.k_proj", - "model.layers.51.self_attn.o_proj", - "model.layers.51.self_attn.q_proj", - "model.layers.51.self_attn.v_proj", - "model.layers.52.linear_attn.conv1d", - "model.layers.52.linear_attn.in_proj_a", - "model.layers.52.linear_attn.in_proj_b", - "model.layers.52.linear_attn.in_proj_qkv", - "model.layers.52.linear_attn.in_proj_z", - "model.layers.52.linear_attn.out_proj", - "model.layers.53.linear_attn.conv1d", - "model.layers.53.linear_attn.in_proj_a", - "model.layers.53.linear_attn.in_proj_b", - "model.layers.53.linear_attn.in_proj_qkv", - "model.layers.53.linear_attn.in_proj_z", - "model.layers.53.linear_attn.out_proj", - "model.layers.54.linear_attn.conv1d", - "model.layers.54.linear_attn.in_proj_a", - "model.layers.54.linear_attn.in_proj_b", - "model.layers.54.linear_attn.in_proj_qkv", - "model.layers.54.linear_attn.in_proj_z", - "model.layers.54.linear_attn.out_proj", - "model.layers.55.self_attn.k_proj", - "model.layers.55.self_attn.o_proj", - "model.layers.55.self_attn.q_proj", - "model.layers.55.self_attn.v_proj", - "model.layers.56.linear_attn.conv1d", - "model.layers.56.linear_attn.in_proj_a", - "model.layers.56.linear_attn.in_proj_b", - "model.layers.56.linear_attn.in_proj_qkv", - "model.layers.56.linear_attn.in_proj_z", - "model.layers.56.linear_attn.out_proj", - "model.layers.57.linear_attn.conv1d", - "model.layers.57.linear_attn.in_proj_a", - "model.layers.57.linear_attn.in_proj_b", - "model.layers.57.linear_attn.in_proj_qkv", - "model.layers.57.linear_attn.in_proj_z", - "model.layers.57.linear_attn.out_proj", - "model.layers.58.linear_attn.conv1d", - "model.layers.58.linear_attn.in_proj_a", - "model.layers.58.linear_attn.in_proj_b", - "model.layers.58.linear_attn.in_proj_qkv", - "model.layers.58.linear_attn.in_proj_z", - "model.layers.58.linear_attn.out_proj", - "model.layers.59.self_attn.k_proj", - "model.layers.59.self_attn.o_proj", - "model.layers.59.self_attn.q_proj", - "model.layers.59.self_attn.v_proj", - "model.layers.6.linear_attn.conv1d", - "model.layers.6.linear_attn.in_proj_a", - "model.layers.6.linear_attn.in_proj_b", - "model.layers.6.linear_attn.in_proj_qkv", - "model.layers.6.linear_attn.in_proj_z", - "model.layers.6.linear_attn.out_proj", - "model.layers.60.linear_attn.conv1d", - "model.layers.60.linear_attn.in_proj_a", - "model.layers.60.linear_attn.in_proj_b", - "model.layers.60.linear_attn.in_proj_qkv", - "model.layers.60.linear_attn.in_proj_z", - "model.layers.60.linear_attn.out_proj", - "model.layers.61.linear_attn.conv1d", - "model.layers.61.linear_attn.in_proj_a", - "model.layers.61.linear_attn.in_proj_b", - "model.layers.61.linear_attn.in_proj_qkv", - "model.layers.61.linear_attn.in_proj_z", - "model.layers.61.linear_attn.out_proj", - "model.layers.62.linear_attn.conv1d", - "model.layers.62.linear_attn.in_proj_a", - "model.layers.62.linear_attn.in_proj_b", - "model.layers.62.linear_attn.in_proj_qkv", - "model.layers.62.linear_attn.in_proj_z", - "model.layers.62.linear_attn.out_proj", - "model.layers.63.self_attn.k_proj", - "model.layers.63.self_attn.o_proj", - "model.layers.63.self_attn.q_proj", - "model.layers.63.self_attn.v_proj", - "model.layers.64.linear_attn.conv1d", - "model.layers.64.linear_attn.in_proj_a", - "model.layers.64.linear_attn.in_proj_b", - "model.layers.64.linear_attn.in_proj_qkv", - "model.layers.64.linear_attn.in_proj_z", - "model.layers.64.linear_attn.out_proj", - "model.layers.65.linear_attn.conv1d", - "model.layers.65.linear_attn.in_proj_a", - "model.layers.65.linear_attn.in_proj_b", - "model.layers.65.linear_attn.in_proj_qkv", - "model.layers.65.linear_attn.in_proj_z", - "model.layers.65.linear_attn.out_proj", - "model.layers.66.linear_attn.conv1d", - "model.layers.66.linear_attn.in_proj_a", - "model.layers.66.linear_attn.in_proj_b", - "model.layers.66.linear_attn.in_proj_qkv", - "model.layers.66.linear_attn.in_proj_z", - "model.layers.66.linear_attn.out_proj", - "model.layers.67.self_attn.k_proj", - "model.layers.67.self_attn.o_proj", - "model.layers.67.self_attn.q_proj", - "model.layers.67.self_attn.v_proj", - "model.layers.68.linear_attn.conv1d", - "model.layers.68.linear_attn.in_proj_a", - "model.layers.68.linear_attn.in_proj_b", - "model.layers.68.linear_attn.in_proj_qkv", - "model.layers.68.linear_attn.in_proj_z", - "model.layers.68.linear_attn.out_proj", - "model.layers.69.linear_attn.conv1d", - "model.layers.69.linear_attn.in_proj_a", - "model.layers.69.linear_attn.in_proj_b", - "model.layers.69.linear_attn.in_proj_qkv", - "model.layers.69.linear_attn.in_proj_z", - "model.layers.69.linear_attn.out_proj", - "model.layers.7.self_attn.k_proj", - "model.layers.7.self_attn.o_proj", - "model.layers.7.self_attn.q_proj", - "model.layers.7.self_attn.v_proj", - "model.layers.70.linear_attn.conv1d", - "model.layers.70.linear_attn.in_proj_a", - "model.layers.70.linear_attn.in_proj_b", - "model.layers.70.linear_attn.in_proj_qkv", - "model.layers.70.linear_attn.in_proj_z", - "model.layers.70.linear_attn.out_proj", - "model.layers.71.self_attn.k_proj", - "model.layers.71.self_attn.o_proj", - "model.layers.71.self_attn.q_proj", - "model.layers.71.self_attn.v_proj", - "model.layers.72.linear_attn.conv1d", - "model.layers.72.linear_attn.in_proj_a", - "model.layers.72.linear_attn.in_proj_b", - "model.layers.72.linear_attn.in_proj_qkv", - "model.layers.72.linear_attn.in_proj_z", - "model.layers.72.linear_attn.out_proj", - "model.layers.73.linear_attn.conv1d", - "model.layers.73.linear_attn.in_proj_a", - "model.layers.73.linear_attn.in_proj_b", - "model.layers.73.linear_attn.in_proj_qkv", - "model.layers.73.linear_attn.in_proj_z", - "model.layers.73.linear_attn.out_proj", - "model.layers.74.linear_attn.conv1d", - "model.layers.74.linear_attn.in_proj_a", - "model.layers.74.linear_attn.in_proj_b", - "model.layers.74.linear_attn.in_proj_qkv", - "model.layers.74.linear_attn.in_proj_z", - "model.layers.74.linear_attn.out_proj", - "model.layers.75.self_attn.k_proj", - "model.layers.75.self_attn.o_proj", - "model.layers.75.self_attn.q_proj", - "model.layers.75.self_attn.v_proj", - "model.layers.76.linear_attn.conv1d", - "model.layers.76.linear_attn.in_proj_a", - "model.layers.76.linear_attn.in_proj_b", - "model.layers.76.linear_attn.in_proj_qkv", - "model.layers.76.linear_attn.in_proj_z", - "model.layers.76.linear_attn.out_proj", - "model.layers.77.linear_attn.conv1d", - "model.layers.77.linear_attn.in_proj_a", - "model.layers.77.linear_attn.in_proj_b", - "model.layers.77.linear_attn.in_proj_qkv", - "model.layers.77.linear_attn.in_proj_z", - "model.layers.77.linear_attn.out_proj", - "model.layers.78.linear_attn.conv1d", - "model.layers.78.linear_attn.in_proj_a", - "model.layers.78.linear_attn.in_proj_b", - "model.layers.78.linear_attn.in_proj_qkv", - "model.layers.78.linear_attn.in_proj_z", - "model.layers.78.linear_attn.out_proj", - "model.layers.79.self_attn.k_proj", - "model.layers.79.self_attn.o_proj", - "model.layers.79.self_attn.q_proj", - "model.layers.79.self_attn.v_proj", - "model.layers.8.linear_attn.conv1d", - "model.layers.8.linear_attn.in_proj_a", - "model.layers.8.linear_attn.in_proj_b", - "model.layers.8.linear_attn.in_proj_qkv", - "model.layers.8.linear_attn.in_proj_z", - "model.layers.8.linear_attn.out_proj", - "model.layers.80.linear_attn.conv1d", - "model.layers.80.linear_attn.in_proj_a", - "model.layers.80.linear_attn.in_proj_b", - "model.layers.80.linear_attn.in_proj_qkv", - "model.layers.80.linear_attn.in_proj_z", - "model.layers.80.linear_attn.out_proj", - "model.layers.81.linear_attn.conv1d", - "model.layers.81.linear_attn.in_proj_a", - "model.layers.81.linear_attn.in_proj_b", - "model.layers.81.linear_attn.in_proj_qkv", - "model.layers.81.linear_attn.in_proj_z", - "model.layers.81.linear_attn.out_proj", - "model.layers.82.linear_attn.conv1d", - "model.layers.82.linear_attn.in_proj_a", - "model.layers.82.linear_attn.in_proj_b", - "model.layers.82.linear_attn.in_proj_qkv", - "model.layers.82.linear_attn.in_proj_z", - "model.layers.82.linear_attn.out_proj", - "model.layers.83.self_attn.k_proj", - "model.layers.83.self_attn.o_proj", - "model.layers.83.self_attn.q_proj", - "model.layers.83.self_attn.v_proj", - "model.layers.84.linear_attn.conv1d", - "model.layers.84.linear_attn.in_proj_a", - "model.layers.84.linear_attn.in_proj_b", - "model.layers.84.linear_attn.in_proj_qkv", - "model.layers.84.linear_attn.in_proj_z", - "model.layers.84.linear_attn.out_proj", - "model.layers.85.linear_attn.conv1d", - "model.layers.85.linear_attn.in_proj_a", - "model.layers.85.linear_attn.in_proj_b", - "model.layers.85.linear_attn.in_proj_qkv", - "model.layers.85.linear_attn.in_proj_z", - "model.layers.85.linear_attn.out_proj", - "model.layers.86.linear_attn.conv1d", - "model.layers.86.linear_attn.in_proj_a", - "model.layers.86.linear_attn.in_proj_b", - "model.layers.86.linear_attn.in_proj_qkv", - "model.layers.86.linear_attn.in_proj_z", - "model.layers.86.linear_attn.out_proj", - "model.layers.87.self_attn.k_proj", - "model.layers.87.self_attn.o_proj", - "model.layers.87.self_attn.q_proj", - "model.layers.87.self_attn.v_proj", - "model.layers.88.linear_attn.conv1d", - "model.layers.88.linear_attn.in_proj_a", - "model.layers.88.linear_attn.in_proj_b", - "model.layers.88.linear_attn.in_proj_qkv", - "model.layers.88.linear_attn.in_proj_z", - "model.layers.88.linear_attn.out_proj", - "model.layers.89.linear_attn.conv1d", - "model.layers.89.linear_attn.in_proj_a", - "model.layers.89.linear_attn.in_proj_b", - "model.layers.89.linear_attn.in_proj_qkv", - "model.layers.89.linear_attn.in_proj_z", - "model.layers.89.linear_attn.out_proj", - "model.layers.9.linear_attn.conv1d", - "model.layers.9.linear_attn.in_proj_a", - "model.layers.9.linear_attn.in_proj_b", - "model.layers.9.linear_attn.in_proj_qkv", - "model.layers.9.linear_attn.in_proj_z", - "model.layers.9.linear_attn.out_proj", - "model.layers.90.linear_attn.conv1d", - "model.layers.90.linear_attn.in_proj_a", - "model.layers.90.linear_attn.in_proj_b", - "model.layers.90.linear_attn.in_proj_qkv", - "model.layers.90.linear_attn.in_proj_z", - "model.layers.90.linear_attn.out_proj", - "model.layers.91.self_attn.k_proj", - "model.layers.91.self_attn.o_proj", - "model.layers.91.self_attn.q_proj", - "model.layers.91.self_attn.v_proj" - ] - }, - "group_1": { - "input_activations": { - "dynamic": false, - "num_bits": 4, - "type": "float", - "group_size": 16 - }, - "weights": { - "dynamic": false, - "num_bits": 4, - "type": "float", - "group_size": 16 - }, - "targets": [ - "model.layers.0.mlp.experts", - "model.layers.1.mlp.experts", - "model.layers.10.mlp.experts", - "model.layers.11.mlp.experts", - "model.layers.12.mlp.experts", - "model.layers.13.mlp.experts", - "model.layers.14.mlp.experts", - "model.layers.15.mlp.experts", - "model.layers.16.mlp.experts", - "model.layers.17.mlp.experts", - "model.layers.18.mlp.experts", - "model.layers.19.mlp.experts", - "model.layers.2.mlp.experts", - "model.layers.20.mlp.experts", - "model.layers.21.mlp.experts", - "model.layers.22.mlp.experts", - "model.layers.23.mlp.experts", - "model.layers.24.mlp.experts", - "model.layers.25.mlp.experts", - "model.layers.26.mlp.experts", - "model.layers.27.mlp.experts", - "model.layers.28.mlp.experts", - "model.layers.29.mlp.experts", - "model.layers.3.mlp.experts", - "model.layers.30.mlp.experts", - "model.layers.31.mlp.experts", - "model.layers.32.mlp.experts", - "model.layers.33.mlp.experts", - "model.layers.34.mlp.experts", - "model.layers.35.mlp.experts", - "model.layers.36.mlp.experts", - "model.layers.37.mlp.experts", - "model.layers.38.mlp.experts", - "model.layers.39.mlp.experts", - "model.layers.4.mlp.experts", - "model.layers.40.mlp.experts", - "model.layers.41.mlp.experts", - "model.layers.42.mlp.experts", - "model.layers.43.mlp.experts", - "model.layers.44.mlp.experts", - "model.layers.45.mlp.experts", - "model.layers.46.mlp.experts", - "model.layers.47.mlp.experts", - "model.layers.48.mlp.experts", - "model.layers.49.mlp.experts", - "model.layers.5.mlp.experts", - "model.layers.50.mlp.experts", - "model.layers.51.mlp.experts", - "model.layers.52.mlp.experts", - "model.layers.53.mlp.experts", - "model.layers.54.mlp.experts", - "model.layers.55.mlp.experts", - "model.layers.56.mlp.experts", - "model.layers.57.mlp.experts", - "model.layers.58.mlp.experts", - "model.layers.59.mlp.experts", - "model.layers.6.mlp.experts", - "model.layers.60.mlp.experts", - "model.layers.61.mlp.experts", - "model.layers.62.mlp.experts", - "model.layers.63.mlp.experts", - "model.layers.64.mlp.experts", - "model.layers.65.mlp.experts", - "model.layers.66.mlp.experts", - "model.layers.67.mlp.experts", - "model.layers.68.mlp.experts", - "model.layers.69.mlp.experts", - "model.layers.7.mlp.experts", - "model.layers.70.mlp.experts", - "model.layers.71.mlp.experts", - "model.layers.72.mlp.experts", - "model.layers.73.mlp.experts", - "model.layers.74.mlp.experts", - "model.layers.75.mlp.experts", - "model.layers.76.mlp.experts", - "model.layers.77.mlp.experts", - "model.layers.78.mlp.experts", - "model.layers.79.mlp.experts", - "model.layers.8.mlp.experts", - "model.layers.80.mlp.experts", - "model.layers.81.mlp.experts", - "model.layers.82.mlp.experts", - "model.layers.83.mlp.experts", - "model.layers.84.mlp.experts", - "model.layers.85.mlp.experts", - "model.layers.86.mlp.experts", - "model.layers.87.mlp.experts", - "model.layers.88.mlp.experts", - "model.layers.89.mlp.experts", - "model.layers.9.mlp.experts", - "model.layers.90.mlp.experts", - "model.layers.91.mlp.experts" - ] - } - }, - "quantized_layers": { - "model.layers.0.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.0.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.1.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.1.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.2.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.2.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.3.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.4.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.4.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.5.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.5.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.6.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.6.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.7.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.8.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.8.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.9.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.9.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.10.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.10.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.11.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.12.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.12.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.13.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.13.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.14.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.14.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.15.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.16.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.16.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.17.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.17.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.18.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.18.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.19.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.20.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.20.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.21.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.21.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.22.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.22.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.23.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.24.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.24.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.25.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.25.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.26.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.26.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.27.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.28.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.28.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.29.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.29.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.30.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.30.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.31.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.32.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.32.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.33.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.33.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.34.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.34.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.35.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.36.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.36.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.37.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.37.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.38.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.38.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.39.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.40.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.40.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.41.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.41.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.42.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.42.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.43.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.44.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.44.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.45.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.45.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.46.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.46.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.47.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.48.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.48.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.49.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.49.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.50.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.50.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.51.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.52.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.52.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.53.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.53.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.54.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.54.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.55.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.56.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.56.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.57.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.57.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.58.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.58.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.59.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.60.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.60.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.61.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.61.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.62.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.62.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.63.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.64.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.64.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.65.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.65.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.66.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.66.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.67.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.68.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.68.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.69.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.69.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.70.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.70.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.71.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.72.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.72.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.73.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.73.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.74.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.74.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.75.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.76.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.76.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.77.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.77.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.78.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.78.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.79.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.80.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.80.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.81.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.81.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.82.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.82.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.83.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.84.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.84.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.85.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.85.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.86.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.86.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.87.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.88.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.88.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.89.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.89.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.90.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.90.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.91.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - } - }, - "ignore": [ - "mtp*", - "mtp.layers.0*" - ], - "quant_algo": "MIXED_PRECISION", - "kv_cache_scheme": { - "dynamic": false, - "num_bits": 8, - "type": "float" - }, - "producer": { - "name": "modelopt", - "version": "0.42.0" - }, - "quant_method": "modelopt" - } -} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json b/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json deleted file mode 100644 index 9a7da44038..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/nvidia--Qwen3.8-2.4T-A95B-NVFP4_hf_quant_config.json +++ /dev/null @@ -1,1902 +0,0 @@ -{ - "producer": { - "name": "modelopt", - "version": "0.42.0" - }, - "quantization": { - "quant_algo": "MIXED_PRECISION", - "kv_cache_quant_algo": "FP8", - "quantized_layers": { - "model.layers.0.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.0.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.0.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.1.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.1.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.1.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.2.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.2.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.2.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.3.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.3.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.4.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.4.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.4.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.5.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.5.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.5.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.6.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.6.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.6.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.7.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.7.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.8.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.8.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.8.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.9.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.9.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.9.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.10.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.10.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.10.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.11.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.11.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.12.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.12.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.12.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.13.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.13.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.13.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.14.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.14.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.14.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.15.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.15.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.16.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.16.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.16.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.17.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.17.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.17.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.18.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.18.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.18.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.19.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.19.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.20.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.20.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.20.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.21.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.21.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.21.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.22.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.22.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.22.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.23.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.23.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.24.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.24.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.24.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.25.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.25.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.25.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.26.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.26.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.26.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.27.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.27.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.28.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.28.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.28.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.29.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.29.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.29.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.30.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.30.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.30.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.31.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.31.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.32.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.32.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.32.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.33.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.33.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.33.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.34.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.34.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.34.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.35.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.35.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.36.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.36.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.36.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.37.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.37.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.37.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.38.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.38.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.38.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.39.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.39.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.40.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.40.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.40.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.41.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.41.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.41.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.42.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.42.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.42.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.43.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.43.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.44.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.44.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.44.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.45.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.45.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.45.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.46.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.46.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.46.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.47.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.47.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.48.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.48.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.48.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.49.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.49.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.49.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.50.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.50.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.50.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.51.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.51.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.52.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.52.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.52.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.53.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.53.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.53.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.54.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.54.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.54.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.55.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.55.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.56.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.56.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.56.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.57.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.57.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.57.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.58.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.58.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.58.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.59.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.59.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.60.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.60.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.60.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.61.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.61.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.61.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.62.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.62.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.62.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.63.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.63.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.64.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.64.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.64.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.65.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.65.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.65.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.66.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.66.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.66.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.67.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.67.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.68.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.68.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.68.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.69.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.69.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.69.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.70.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.70.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.70.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.71.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.71.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.72.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.72.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.72.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.73.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.73.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.73.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.74.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.74.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.74.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.75.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.75.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.76.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.76.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.76.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.77.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.77.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.77.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.78.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.78.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.78.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.79.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.79.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.80.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.80.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.80.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.81.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.81.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.81.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.82.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.82.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.82.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.83.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.83.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.84.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.84.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.84.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.85.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.85.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.85.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.86.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.86.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.86.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.87.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.87.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.88.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.88.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.88.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.89.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.89.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.89.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.90.linear_attn.conv1d": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.out_proj": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_qkv": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_z": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_b": { - "quant_algo": "FP8" - }, - "model.layers.90.linear_attn.in_proj_a": { - "quant_algo": "FP8" - }, - "model.layers.90.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - }, - "model.layers.91.self_attn.q_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.self_attn.k_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.self_attn.v_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.self_attn.o_proj": { - "quant_algo": "FP8" - }, - "model.layers.91.mlp.experts": { - "quant_algo": "NVFP4", - "group_size": 16 - } - }, - "exclude_modules": [ - "mtp*", - "mtp.layers.0*" - ] - } -} \ No newline at end of file diff --git a/python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json deleted file mode 100644 index 9860d7132a..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/openbmb--MiniCPM5-2B_config.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "_name_or_path": "openbmb/MiniCPM5-2B", - "architectures": [ - "LlamaForCausalLM" - ], - "bos_token_id": 0, - "eos_token_id": [1, 130073], - "pad_token_id": 1, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "intermediate_size": 6144, - "max_position_embeddings": 131072, - "model_type": "llama", - "num_attention_heads": 16, - "num_hidden_layers": 42, - "num_key_value_heads": 2, - "head_dim": 128, - "rms_norm_eps": 1e-06, - "rope_theta": 5000000, - "rope_scaling": null, - "tie_word_embeddings": false, - "torch_dtype": "bfloat16", - "transformers_version": "5.6.2", - "use_cache": true, - "vocab_size": 130560 -} diff --git a/python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json b/python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json deleted file mode 100644 index 865278c0e0..0000000000 --- a/python/aisimulate/src/aisimulate_core/model_configs/ornith-ai--Ornith-1.5-35B-A3B_config.json +++ /dev/null @@ -1,125 +0,0 @@ -{ - "architectures": [ - "Qwen3_5MoeForConditionalGeneration" - ], - "bos_token_id": null, - "dtype": "bfloat16", - "eos_token_id": 248046, - "hidden_size": 2048, - "image_token_id": 248056, - "model_type": "qwen3_5_moe", - "pad_token_id": 248044, - "text_config": { - "attention_bias": false, - "attention_dropout": 0.0, - "attn_output_gate": true, - "bos_token_id": 248044, - "dtype": "bfloat16", - "eos_token_id": 248044, - "full_attention_interval": 4, - "head_dim": 256, - "hidden_act": "silu", - "hidden_size": 2048, - "initializer_range": 0.02, - "layer_types": [ - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention", - "linear_attention", - "linear_attention", - "linear_attention", - "full_attention" - ], - "linear_conv_kernel_dim": 4, - "linear_key_head_dim": 128, - "linear_num_key_heads": 16, - "linear_num_value_heads": 32, - "linear_value_head_dim": 128, - "mamba_ssm_dtype": "float32", - "max_position_embeddings": 262144, - "model_type": "qwen3_5_moe_text", - "moe_intermediate_size": 512, - "mtp_num_hidden_layers": 1, - "mtp_use_dedicated_embeddings": false, - "num_attention_heads": 16, - "num_experts": 256, - "num_experts_per_tok": 8, - "num_hidden_layers": 40, - "num_key_value_heads": 2, - "output_router_logits": false, - "pad_token_id": 248044, - "partial_rotary_factor": 0.25, - "rms_norm_eps": 1e-06, - "rope_parameters": { - "mrope_interleaved": true, - "mrope_section": [ - 11, - 11, - 10 - ], - "partial_rotary_factor": 0.25, - "rope_theta": 10000000, - "rope_type": "default" - }, - "router_aux_loss_coef": 0.0, - "shared_expert_intermediate_size": 512, - "tie_word_embeddings": false, - "use_cache": false, - "vocab_size": 248320 - }, - "tie_word_embeddings": false, - "transformers_version": "5.8.1", - "video_token_id": 248057, - "vision_config": { - "deepstack_visual_indexes": [], - "depth": 27, - "dtype": "bfloat16", - "hidden_act": "gelu_pytorch_tanh", - "hidden_size": 1152, - "in_channels": 3, - "initializer_range": 0.02, - "intermediate_size": 4304, - "model_type": "qwen3_5_moe_vision", - "num_heads": 16, - "num_position_embeddings": 2304, - "out_hidden_size": 2048, - "patch_size": 16, - "spatial_merge_size": 2, - "temporal_patch_size": 2 - }, - "vision_end_token_id": 248054, - "vision_start_token_id": 248053 -} diff --git a/python/aisimulate/src/aisimulate_core/sdk/common.py b/python/aisimulate/src/aisimulate_core/sdk/common.py index c804f7f233..0ddaaf7bc5 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/common.py +++ b/python/aisimulate/src/aisimulate_core/sdk/common.py @@ -678,7 +678,6 @@ def get_default_models() -> set[str]: """ DefaultHFModels = { "deepseek-ai/DeepSeek-V4.1-Flash", - "nvidia/DeepSeek-V4.1-Flash-NVFP4", # Llama 3.1 Models "meta-llama/Meta-Llama-3.1-8B", "meta-llama/Meta-Llama-3.1-70B", @@ -693,7 +692,6 @@ def get_default_models() -> set[str]: "moonshotai/Kimi-K2.5", # Kimi K3 "moonshotai/Kimi-K3", - "nvidia/Kimi-K3-NVFP4", "nvidia/Kimi-K2.5-NVFP4", "nvidia/Kimi-K2.6-NVFP4", "nvidia/Kimi-K2.7-Code-NVFP4", @@ -773,14 +771,12 @@ def get_default_models() -> set[str]: # Qwen3.8-Max Models "Qwen/Qwen3.8-2.4T-A95B", "Qwen/Qwen3.8-2.4T-A95B-FP8", - "nvidia/Qwen3.8-2.4T-A95B-NVFP4", # MiMo Models "XiaomiMiMo/MiMo-V2-Flash", "XiaomiMiMo/MiMo-7B-Base", # NVIDIA Nemotron "nvidia/Llama-3_3-Nemotron-Super-49B-v1", "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4", - "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4", "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8", @@ -793,15 +789,6 @@ def get_default_models() -> set[str]: "google/gemma-4-26B-A4B", # Meta Muse Glimmer "meta-models/Muse-Glimmer-30B", - # Qwen3.8-27B (QWEN35 hybrid GDN dense; onboarding 2026-09-25) - "Qwen/Qwen3.8-27B", - "Qwen/Qwen3.8-27B-FP8", - # Ornith 1.5 (Qwen3.5-MoE architecture) - "ornith-ai/Ornith-1.5-35B-A3B", - # MiniCPM5 (Llama architecture) - "openbmb/MiniCPM5-2B", - # NeoHorse 1 (text-only dense Qwen3.5, Qwen3_5ForCausalLM) - "TokenRhythm/NeoHorse-1-9B", # StepFun Step-3.7 Models "stepfun-ai/Step-3.7-Flash", "stepfun-ai/Step-3.7-Flash-FP8", @@ -919,7 +906,6 @@ def get_default_models() -> set[str]: # Qwen3.8-Max: FLAT config (no text_config nesting) -- do NOT add to # MULTIMODAL_TEXT_CONFIG_KEY below, unlike the two VLM classes above. "Qwen3_5MoeForCausalLM": "QWEN35", - "Qwen3_5ForCausalLM": "QWEN35", # text-only dense Qwen3.5 checkpoint (config at root, no text_config) "Gemma4ForConditionalGeneration": "GEMMA4MIX", "MuseGlimmerForConditionalGeneration": "MUSEGLIMMER", "Mistral3ForConditionalGeneration": "MISTRAL3", diff --git a/python/aisimulate/src/aisimulate_core/sdk/utils.py b/python/aisimulate/src/aisimulate_core/sdk/utils.py index 7ce95dd878..5e3fde4354 100644 --- a/python/aisimulate/src/aisimulate_core/sdk/utils.py +++ b/python/aisimulate/src/aisimulate_core/sdk/utils.py @@ -1487,9 +1487,6 @@ def positive_int(value, name): # CausalLM architecture string instead of the VLM ConditionalGeneration # classes above. "Qwen3_5MoeForCausalLM", - # text-only dense Qwen3.5 checkpoints (e.g. Altworld/Hemmingway-1, - # TokenRhythm/NeoHorse-1-9B): flat config, no vision_config. - "Qwen3_5ForCausalLM", }: # Qwen3.5 hybrid GDN + full-attention model. layer_types_raw = config.get("layer_types", []) diff --git a/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py b/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py index 293c643586..d508e3d4f4 100644 --- a/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py +++ b/python/aisimulate/tests/unit/sdk/models/test_modelopt_mixed_precision_targets.py @@ -558,8 +558,6 @@ def test_packaged_targets_only_checkpoints_are_reviewed(): _PACKAGED: "DEEPSEEKV32", "nvidia/MiniMax-M3-NVFP4": "MINIMAXM3", "nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4": "NEMOTRONH", - # QWEN35 family: outside the changed DEEPSEEK/KIMIK25/DEEPSEEKV32 resolvers. - "nvidia/Qwen3.8-2.4T-A95B-NVFP4": "QWEN35", "nvidia/Qwen3.6-27B-NVFP4": "QWEN35", "nvidia/Qwen3.6-35B-A3B-NVFP4": "QWEN35", } diff --git a/python/aisimulate/tests/unit/sdk/test_utils.py b/python/aisimulate/tests/unit/sdk/test_utils.py index e53edd1591..f2a721ac27 100644 --- a/python/aisimulate/tests/unit/sdk/test_utils.py +++ b/python/aisimulate/tests/unit/sdk/test_utils.py @@ -497,51 +497,6 @@ def test_parse_qwen35_layer_types_length_mismatch_raises(self): with pytest.raises(ValueError, match="layer_types length"): _parse_hf_config_json(config) - def test_parse_qwen35_dense_causal_lm_flat_config(self): - """Text-only dense Qwen3.5 checkpoints (Altworld/Hemmingway-1, - TokenRhythm/NeoHorse-1-9B) ship Qwen3_5ForCausalLM with a FLAT config - and no vision_config; they must build the same Qwen35Config the VLM - class does (shape values from Altworld/Hemmingway-1).""" - layer_types = (["linear_attention"] * 3 + ["full_attention"]) * 16 # 64 layers - config = { - "architectures": ["Qwen3_5ForCausalLM"], - "hidden_size": 5120, - "num_hidden_layers": 64, - "layer_types": layer_types, - "num_attention_heads": 24, - "num_key_value_heads": 4, - "head_dim": 256, - "intermediate_size": 17408, - "linear_num_key_heads": 16, - "linear_key_head_dim": 128, - "linear_num_value_heads": 48, - "linear_value_head_dim": 128, - "linear_conv_kernel_dim": 4, - "full_attention_interval": 4, - "vocab_size": 248320, - "max_position_embeddings": 262144, - "rms_norm_eps": 1e-6, - "hidden_act": "silu", - "dtype": "bfloat16", - "tie_word_embeddings": False, - } - - result = _parse_hf_config_json(config) - - assert result["architecture"] == "Qwen3_5ForCausalLM" - assert result["layers"] == 64 - assert result["hidden_size"] == 5120 - assert result["n"] == 24 - assert result["n_kv"] == 4 - assert result["d"] == 256 - extra_params = result["extra_params"] - assert isinstance(extra_params, common.Qwen35Config) - assert extra_params.layer_types.count("linear_attention") == 48 - assert extra_params.layer_types.count("full_attention") == 16 - assert extra_params.linear_num_value_heads == 48 - assert extra_params.num_experts == 0 - assert extra_params.vision_config is None - def test_parse_qwen38_max_flat_config(self): """Test parsing Qwen3.8-Max (Qwen/Qwen3.8-2.4T-A95B) → Qwen35Config from a FLAT config. diff --git a/python/aisimulate/tests/unit/test_support_config_profile.py b/python/aisimulate/tests/unit/test_support_config_profile.py index ea368c5461..97cec392bb 100644 --- a/python/aisimulate/tests/unit/test_support_config_profile.py +++ b/python/aisimulate/tests/unit/test_support_config_profile.py @@ -135,7 +135,6 @@ def test_glm53_flash_config_is_runtime_resolved_not_rejected(): "nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16_config.json", "nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8_config.json", "nvidia--NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4_config.json", - "nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16_config.json", "nvidia--NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4_config.json", "nvidia--Nemotron-H-56B-Base-8K_config.json", } @@ -217,7 +216,6 @@ def test_packaged_unrecognized_architectures_are_pending_and_mla_keeps_explicit_ assert unloadable == { "nvidia--DeepSeek-V4-Flash-NVFP4", "nvidia--DeepSeek-V4-Pro-NVFP4", - "nvidia--DeepSeek-V4.1-Flash-NVFP4", "nvidia--Gemma-4-31B-IT-NVFP4", } assert unrecognized == {name: [f"unrecognized architecture {arch}"] for name, arch in _UNRECOGNIZED.items()} From 77bf6096d6c35e0c8f64d11339c02448ec43c35a Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 18:11:15 +0800 Subject: [PATCH 03/13] feat: one framework release per collector file; vLLM gemm/moe/kda/mhc move to 0.31.0 The owner policy of 2026-09-20 (version history lives in git, not in side-by-side files or version ranges) was recorded only in a commit message and four vLLM file headers, so later ports widened __compat__ ranges again. It is now a binding rule ("Version contract" in .claude/rules/collector/layer_permissions.md) and a test: every stock vLLM, SGLang and TRT-LLM registry module declares exactly `==` the runtime the manifest resolves for its op. aic-auto-collect, aic-collector-op-development and docs/perf-model/collector/upgrade.md follow the rule. - vLLM: gemm, moe, kda and mhc move to 0.31.0 through manifest family pins (owner decision): GLM-5.3-Flash already collects them there, so the families upgrade instead of carrying 0.30.0 and 0.31.0 side by side. Kimi-K3 KDA moves with its family. Every other vLLM CUDA collector pins 0.30.0. DeepSeek-V4 mHC is not admitted on 0.31.0: the architecture-free mhc_module_perf view its consumer reads would merge GLM-5.3-Flash rows at (hc_mult 4, hidden 4096); its 0.24.0-0.30.0 tables stay. - SGLang pins 0.5.21 and TRT-LLM 1.3.0rc29 in every collector. - Qwen3.8-2.4T model pins (vLLM 0.27.1, SGLang 0.5.17) fold into the defaults per the pin-consolidation rule; the new tables already carry the Qwen3.8 MoE and GDN shapes. - The SGLang 0.5.20 GLM-5.3-Flash registry lane retires (owner decision): model pin, KDA/mHC architecture gates and the collect_gemm_0520 fork. Its tables stay under 0.5.20; re-collect from main c540ff90a. The standalone GLM attention runner keeps its own exact pins. - SGLang KDA raises KdaRuntimeNotAuditedError (not KeyError) for an architecture with no audited release. Older collectors and data are untouched: data stays under its version key, code is `git log -- ` away. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- docs/perf-model/collector/upgrade.md | 11 +- .../rules/collector/layer_permissions.md | 23 ++ .../.claude/skills/aic-auto-collect/SKILL.md | 7 +- .../aic-collector-op-development/SKILL.md | 5 +- .../collector/framework_manifest.yaml | 107 +++---- .../aisimulate/collector/hash_closures.yaml | 2 - .../collector/sglang/collect_attn.py | 7 +- .../collector/sglang/collect_attn_encoder.py | 7 +- .../collector/sglang/collect_computescale.py | 7 +- .../collector/sglang/collect_dsv4_attn.py | 9 +- .../collector/sglang/collect_gdn.py | 21 +- .../collector/sglang/collect_gemm.py | 71 ++--- .../collector/sglang/collect_gemm_0520.py | 278 ------------------ .../collector/sglang/collect_kda.py | 23 +- .../collector/sglang/collect_mhc_module.py | 16 +- .../collector/sglang/collect_mla.py | 7 +- .../collector/sglang/collect_mla_bmm.py | 7 +- .../collector/sglang/collect_mla_module.py | 7 +- .../collector/sglang/collect_moe.py | 21 +- .../collector/sglang/collect_msa_module.py | 8 +- .../sglang/deepseekv4_sparse_modules.py | 9 +- .../sglang/glm5_dsa_sparse_modules.py | 7 +- .../collector/trtllm/collect_attn.py | 2 +- .../collector/trtllm/collect_attn_encoder.py | 2 +- .../collector/trtllm/collect_computescale.py | 2 +- .../collector/trtllm/collect_dsv4_attn.py | 2 +- .../collector/trtllm/collect_gdn.py | 2 +- .../collector/trtllm/collect_gemm.py | 2 +- .../collector/trtllm/collect_mamba2.py | 2 +- .../collector/trtllm/collect_mhc_module.py | 2 +- .../collector/trtllm/collect_mla.py | 2 +- .../collector/trtllm/collect_mla_bmm.py | 2 +- .../collector/trtllm/collect_mla_module.py | 2 +- .../collector/trtllm/collect_moe.py | 2 +- .../collector/trtllm/collect_msa_module.py | 2 +- .../aisimulate/collector/vllm/collect_attn.py | 2 +- .../collector/vllm/collect_attn_encoder.py | 2 +- .../collector/vllm/collect_computescale.py | 2 +- .../collector/vllm/collect_dsv4_attn.py | 2 +- .../aisimulate/collector/vllm/collect_gdn.py | 2 +- .../aisimulate/collector/vllm/collect_gemm.py | 10 +- .../aisimulate/collector/vllm/collect_kda.py | 17 +- .../collector/vllm/collect_mhc_module.py | 19 +- .../collector/vllm/collect_mla_bmm.py | 2 +- .../collector/vllm/collect_mla_module.py | 2 +- .../aisimulate/collector/vllm/collect_moe.py | 7 +- .../collector/vllm/collect_msa_module.py | 2 +- .../sglang/test_collect_kda_contract.py | 9 +- .../sglang/test_collect_mhc_module.py | 2 +- .../unit/collector/test_framework_manifest.py | 129 +++----- .../test_glm5next_collection_plan.py | 88 +----- .../collector/test_glm5next_runtime_pins.py | 28 +- .../collector/test_mhc_module_glm5next.py | 19 +- .../collector/test_sglang_glm5_sparse_0514.py | 5 +- .../unit/collector/test_version_resolver.py | 6 +- .../vllm/test_collect_kda_contract.py | 23 +- 56 files changed, 317 insertions(+), 747 deletions(-) delete mode 100644 python/aisimulate/collector/sglang/collect_gemm_0520.py diff --git a/docs/perf-model/collector/upgrade.md b/docs/perf-model/collector/upgrade.md index eaa3523180..d5bd025ffa 100644 --- a/docs/perf-model/collector/upgrade.md +++ b/docs/perf-model/collector/upgrade.md @@ -32,17 +32,18 @@ Record these decisions before changing code: - what may be committed, pushed, or opened as a PR. `collector/framework_manifest.yaml` is the source of truth for the stock -framework pin and image. Module-level `__compat__` metadata may narrow that -contract but must not silently widen it. Treat stock and WideEP/special images +framework pin and image. Each collector module pins exactly the release the +manifest resolves for its op (the "Version contract" in +`python/aisimulate/.claude/rules/collector/layer_permissions.md`). Treat stock and WideEP/special images as separate runtimes with separate evidence. Do not silently broaden the work to another framework, XPU, WideEP, host upgrades, or performance-data publication. If the work is stacked on another PR, keep that dependency explicit when creating the branch and eventual PR. -For a fixed-release collector, prefer an exact version contract when the -framework API and kernel selection changed materially. Do not add compatibility -branches for versions that the project no longer intends to collect. +Upgrade collectors in place: move the exact `__compat__` pin to the new +release and do not keep compatibility branches for earlier releases; their +code stays in git history and their data stays under its version key. ## 2. Pin and prove the runtime diff --git a/python/aisimulate/.claude/rules/collector/layer_permissions.md b/python/aisimulate/.claude/rules/collector/layer_permissions.md index 72c2722aed..483eb4b217 100644 --- a/python/aisimulate/.claude/rules/collector/layer_permissions.md +++ b/python/aisimulate/.claude/rules/collector/layer_permissions.md @@ -150,6 +150,29 @@ prediction) or a guard that raises a classified error citing the framework source line, or deletes the note. Do not implement guards from unverified claims, and never express these limits in YAML. +## Version contract: one release per collector file + +Owner policy (2026-09-20, made binding 2026-10-30): version history lives in +git, not in side-by-side files or version ranges. + +1. **Each collector module targets exactly one framework release.** Its + `__compat__` is a single exact pin, `"=="`, equal to + the runtime `framework_manifest.yaml` resolves for its op (the framework + default or a family pin). No ranges, no `!=` exclusions, no per-version + branches for releases the manifest does not collect. Per-architecture + gates inside a module may only narrow that one release. +2. **Upgrade in place.** A version bump rewrites the module for the new + release and moves its pin. Prior-version code is `git log -- ` + away; prior-version data stays permanent under its version key. Do not + add `_` fork modules or keep old releases admitted "so they + still work". +3. **Pin consolidation.** A model or family pin below the target version + upgrades with it; a pin above the default is the next version for that + family and needs its own GPU validation. +4. **Enforced by tests.** `tests/unit/collector/test_framework_manifest.py` + fails when a stock vLLM, SGLang or TRT-LLM registry module declares + anything but `==` its resolved runtime. + ## Module boundary: a collector task may only touch collector/ The collector's outward surface is its **data contract**: the perf row schema diff --git a/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md b/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md index 7c8dff7415..c630866cf2 100644 --- a/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md +++ b/python/aisimulate/.claude/skills/aic-auto-collect/SKILL.md @@ -336,17 +336,16 @@ Fix policy: 2. Read the failing `collector//collect_*.py`. 3. Search the framework source checkout for the missing class, function, import path, or attribute. 4. Verify the real API in source. Do not trust Python's "Did you mean" suggestions blindly. - 5. Patch the collector with a minimal compatibility-preserving change. -- For API changes, use version routing and `__compat__` where needed. + 5. Patch the collector in place for the target release (one release per file: `.claude/rules/collector/layer_permissions.md`, "Version contract"). +- For API changes, upgrade the collector in place and pin `__compat__` to exactly the target release. - For GPU capability gaps, filter test cases before execution using `get_sm_version()`. - For Blackwell/SM100+ features, gate FP4/NVFP4/FP8 paths carefully. - For dimension constraints, filter on per-rank dimensions such as `inter_size // tp`. - For CUDA-fatal errors, prefer worker restart and skip only the exact invalid region. - Preserve data quality. Do not reduce benchmark repetitions or broad test dimensions just to pass. -- Keep older backend versions working. - Keep shared test-case generators deterministic when unit tests depend on them. Put runtime availability probes in backend collector wrappers when the skip is caused by a partially installed framework package. - Guard `importlib.util.find_spec("a.b.c")` with `try/except ModuleNotFoundError`; partial packages can make `find_spec` raise instead of return `None`. -- If a collector route only exists for future framework versions, use `VersionRoute` in `collector//registry.py` and add a wrapper module with an explicit `__compat__`. +- Do not add side-by-side version fork modules; prior-version code is `git log -- ` away. - If a collector helper silently returns `None` after a dry-run or subprocess failure, make it raise or record the skip explicitly so failed coverage is visible. - If a valid production-like case fails in the framework kernel, file or prepare an upstream bug for SGLang, vLLM, or TRT-LLM with exact image/version, SM, minimal repro, and traceback. Link it in the PR or failure ledger. - If the case is not production-like or the runtime explicitly does not support that SM/path, skip it in `collect_xx.py` before execution with a clear comment and a narrow predicate. diff --git a/python/aisimulate/.claude/skills/aic-collector-op-development/SKILL.md b/python/aisimulate/.claude/skills/aic-collector-op-development/SKILL.md index 5cf9579748..3df1b79ae2 100644 --- a/python/aisimulate/.claude/skills/aic-collector-op-development/SKILL.md +++ b/python/aisimulate/.claude/skills/aic-collector-op-development/SKILL.md @@ -54,8 +54,9 @@ If the task proceeds to GPU collection, also use `$aic-auto-collect`. plan selection required by the resolved case plan. Generic resume, retry, checkpoint, logging, and output-finalization behavior require separate explicit scope. -- Anchor collector behavior to the requested framework version. Prefer one - verified `__compat__`/version route over speculative multi-version branches. +- Anchor collector behavior to the requested framework version: pin exactly + one release (`__compat__ = "=="`, see the "Version contract" + section of `.claude/rules/collector/layer_permissions.md`). ## Step 1: Establish the consumer contract diff --git a/python/aisimulate/collector/framework_manifest.yaml b/python/aisimulate/collector/framework_manifest.yaml index 2e031f59fd..b822d1b4c2 100644 --- a/python/aisimulate/collector/framework_manifest.yaml +++ b/python/aisimulate/collector/framework_manifest.yaml @@ -39,16 +39,13 @@ frameworks: # one model's entire run and leaves every other model's resolution (and # every other op's family/default resolution) unchanged. # - # SGLang v0.5.17 registers Qwen3_5MoeForCausalLM / - # Qwen3_5ForCausalLM; v0.5.14 predated that model integration and could - # not collect these cases. The 0.5.21 default now covers the registration, - # but these model runs stay pinned to the runtime they were validated on - # until a Qwen3.8 collection is re-validated on 0.5.21 (the 0.5.21 serving - # matrix fails Qwen3.8-2.4T-A95B / -FP8 at boot, results/sm90/ - # sglang-0.5.21.yaml). Both tags below resolve to the same multi-arch - # index digest. These are collector-runtime declarations, not claims of - # whole-checkpoint serving validation (in particular for the RadixArk - # NVFP4 collection fixture). + # Qwen3.8-2.4T model pins (0.5.17) folded into the 0.5.21 default + # 2026-10-30 per the pin-consolidation rule (pins below the target version + # upgrade with it): the 0.5.21 tables already carry the Qwen3.8 MoE and GDN + # shapes from the unfiltered campaign, and the op-probe serving matrix boots + # Qwen3.8-2.4T-A95B on SM90/SM100 and -FP8 on SM90 under 0.5.21 + # (benchmarks/evidence/collector/op-probe-2026-10). The 0.5.17 tables stay + # under their version key. # # Scope: the override is selected only when collect.py resolves a single # exact model_path (normally through --model-path, or through an @@ -56,41 +53,13 @@ frameworks: # has no single model identity and therefore uses the framework/family # runtime; run these model-pinned campaigns separately at their declared # runtime rather than mixing them into an unfiltered invocation. - models: - "Qwen/Qwen3.8-2.4T-A95B": - version: "0.5.17" - images: - default: "lmsysorg/sglang:v0.5.17@sha256:16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155" - cu130: "lmsysorg/sglang:v0.5.17-cu130@sha256:16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155" - "Qwen/Qwen3.8-2.4T-A95B-FP8": - version: "0.5.17" - images: - default: "lmsysorg/sglang:v0.5.17@sha256:16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155" - cu130: "lmsysorg/sglang:v0.5.17-cu130@sha256:16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155" - "RadixArk/Qwen3.8-2.4T-A95B-NVFP4": - version: "0.5.17" - images: - default: "lmsysorg/sglang:v0.5.17@sha256:16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155" - cu130: "lmsysorg/sglang:v0.5.17-cu130@sha256:16aba8925507e631e1dc1e23d95d026533602591775f6a8db68b74ee99746155" - # GLM-5.3-Flash (Glm5NextForConditionalGeneration) standalone per-op - # collection. SGLang first ships python/sglang/srt/models/glm5_next.py - # (with dsa_indexer_kpool.py and communicator_mhc.py) in the official - # v0.5.20 release, tag commit 94602c9c2b7cbdb8efd5c52802dac6a1c180089e; - # the 0.5.14 default has no glm5_next model. Docker Hub reported the - # multi-arch index digest below for both v0.5.20 and v0.5.20-cu130 on - # 2026-09-30; its linux/arm64 child (the GB300 platform image, imported - # for collection as b0d8718a...-auth3.sqsh) is - # sha256:b0d8718a4424bb22e448e04407ab3ce5f7399a4c5fc702d6fbe36c3772ec8862. - # Scope/precedence as documented for the Qwen3.8 pins above: the pin - # applies only when collect.py resolves exactly one of these model - # paths; every other model resolves exactly as before. - "zai-org/GLM-5.3-Flash": &glm53flash_sglang_runtime - version: "0.5.20" - source_commit: "94602c9c2b7cbdb8efd5c52802dac6a1c180089e" - images: - default: "lmsysorg/sglang:v0.5.20@sha256:06e4f2ed21afde4ff513cda65070124e727ba23ccaeff7712b8c40e1097d611f" - cu130: "lmsysorg/sglang:v0.5.20-cu130@sha256:06e4f2ed21afde4ff513cda65070124e727ba23ccaeff7712b8c40e1097d611f" - "nvidia/GLM-5.3-Flash-NVFP4": *glm53flash_sglang_runtime + # GLM-5.3-Flash SGLang 0.5.20 model pin retired 2026-10-30 (owner decision: + # one release per collector file; 0.5.20 is below the 0.5.21 target). The + # 0.5.20 tables stay under their version key; re-collect them from + # main at c540ff90a (before #649). Moving the lane to 0.5.21 needs the GLM + # attention runner, KDA, mHC and GEMM ports listed in the 2026-10-30 + # source audit, then a re-collection. + models: {} trtllm: source_repo: "https://github.com/NVIDIA/TensorRT-LLM.git" # 2026-10-02 (H20/sm90 upgrade audit, benchmarks/evidence/collector/op-probe-2026-10/findings.yaml @@ -122,6 +91,22 @@ frameworks: version: "0.30.0" images: default: "vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90" + # gemm / moe / kda / mhc move to vLLM 0.31.0 (owner decision 2026-10-30): + # GLM-5.3-Flash collects these families on 0.31.0, and each collector file + # targets exactly one release, so the families upgrade instead of carrying + # 0.30.0 and 0.31.0 side by side. A family pin moves the family for every + # model. The 0.30.0 tables stay under their version key; the 0.30.0 + # collectors are `git log` away. Same stock image as the GLM-5.3-Flash + # model pin below (multi-arch index; amd64 a4a4c043..., arm64 3f7dd5b7...). + families: + gemm: &vllm_0310_runtime + version: "0.31.0" + source_commit: "db9527a46873454610df6dbedf79a36d6bf1a7f6" + images: + default: "vllm/vllm-openai:v0.31.0@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3" + moe: *vllm_0310_runtime + kda: *vllm_0310_runtime + mhc: *vllm_0310_runtime # kda family pin RETIRED (owner decision 2026-09-28): the family ran on the # kimi-k3 preview image (0.1.dev19262) while the consumer's fused-decode # pricing did not match 0.27's packed-decode serving path (review #1559). @@ -134,7 +119,8 @@ frameworks: # default 2026-09-20 per the pin-consolidation rule: pins BELOW the # target version upgrade with it; the kda preview pin (an odd dev-build # pin) was revisited at the 0.30.0 move and retired, see above. - # moe: NO family pin (owner decision 2026-08-02, reverting the + # moe: the 0.31.0 family pin above is a stock release (2026-10-30). + # History: NO family pin (owner decision 2026-08-02, reverting the # 2026-08-01 family-wide preview pin — a temporary preview build must # not own a family default; a family pin here hijacks image selection # for every model). ALL vllm moe collection — including K3 — runs on @@ -159,34 +145,19 @@ frameworks: # frameworks.sglang.models above: an exact run model_path overrides both # family pins and the framework default for every op in that run. # - # Qwen3.8-Max needs vLLM v0.27.1. The vLLM model registry first maps the - # bare Qwen3_5MoeForCausalLM architecture string at v0.27.1 - # (model_executor/models/registry.py:202 at tag v0.27.1, commit - # 6e448d0ea9bf3d88d898b65449ca6dc2aec170ac); v0.24.0 has the model class - # but no standard registry route for that architecture. Docker Hub's - # registry API reported the immutable index digests below on 2026-08-21. - # v0.27.1 has default and cu129 variants, but no cu128/cu130 tags. + # Qwen3.8-2.4T model pins (0.27.1) folded into the 0.30.0 default + # 2026-10-30 per the pin-consolidation rule: v0.30.0 registers + # Qwen3_5MoeForCausalLM, the 0.30.0 tables already carry the Qwen3.8 MoE + # and GDN shapes, and the op-probe serving matrix boots -FP8 (SM90) and + # nvidia's NVFP4 checkpoint (SM90/SM100) under 0.30.0 + # (benchmarks/evidence/collector/op-probe-2026-10). + # The 0.27.1 tables stay under their version key. # # Scope: the override is selected only when collect.py resolves one exact # model_path. A full/unfiltered plan has no single model identity and uses # the normal framework/family runtime; run these campaigns separately at # their declared runtime. models: - "Qwen/Qwen3.8-2.4T-A95B": - version: "0.27.1" - images: - default: "vllm/vllm-openai:v0.27.1@sha256:0a51ea5b4ae2dc5d81890e5173f54203d2a3ae0cfffe51b8fd2afd4391bfd967" - cu129: "vllm/vllm-openai:v0.27.1-cu129@sha256:07913e94a58a4e61322c88f8d4647411b4dd394838b53abacaa32448d9d8de0a" - "Qwen/Qwen3.8-2.4T-A95B-FP8": - version: "0.27.1" - images: - default: "vllm/vllm-openai:v0.27.1@sha256:0a51ea5b4ae2dc5d81890e5173f54203d2a3ae0cfffe51b8fd2afd4391bfd967" - cu129: "vllm/vllm-openai:v0.27.1-cu129@sha256:07913e94a58a4e61322c88f8d4647411b4dd394838b53abacaa32448d9d8de0a" - "RadixArk/Qwen3.8-2.4T-A95B-NVFP4": - version: "0.27.1" - images: - default: "vllm/vllm-openai:v0.27.1@sha256:0a51ea5b4ae2dc5d81890e5173f54203d2a3ae0cfffe51b8fd2afd4391bfd967" - cu129: "vllm/vllm-openai:v0.27.1-cu129@sha256:07913e94a58a4e61322c88f8d4647411b4dd394838b53abacaa32448d9d8de0a" # GLM-5.3-Flash (Glm5NextForConditionalGeneration) standalone per-op # collection runs on the stock official v0.31.0 release (no overlay), # tag commit db9527a46873454610df6dbedf79a36d6bf1a7f6. v0.31.0 moves diff --git a/python/aisimulate/collector/hash_closures.yaml b/python/aisimulate/collector/hash_closures.yaml index 67062ed89e..35a8ea7cf3 100644 --- a/python/aisimulate/collector/hash_closures.yaml +++ b/python/aisimulate/collector/hash_closures.yaml @@ -131,8 +131,6 @@ collector.sglang.collect_kda: - collector/cases/base_ops/kda.yaml - __model_cases__ collector.sglang.collect_gemm: - # SGLang 0.5.20 runs are delegated to collect_gemm_0520.py. - - collector/sglang/collect_gemm_0520.py - collector/cases/base_ops/gemm.yaml - __model_cases__ collector.sglang.collect_mhc_module: diff --git a/python/aisimulate/collector/sglang/collect_attn.py b/python/aisimulate/collector/sglang/collect_attn.py index 7273ed1abb..75c5c04fb2 100644 --- a/python/aisimulate/collector/sglang/collect_attn.py +++ b/python/aisimulate/collector/sglang/collect_attn.py @@ -9,9 +9,10 @@ KV-cache setup, backend dispatch, and perf logging for the SGLang runtime. """ -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import math import os diff --git a/python/aisimulate/collector/sglang/collect_attn_encoder.py b/python/aisimulate/collector/sglang/collect_attn_encoder.py index b695753698..d9a7bd8e0c 100644 --- a/python/aisimulate/collector/sglang/collect_attn_encoder.py +++ b/python/aisimulate/collector/sglang/collect_attn_encoder.py @@ -17,9 +17,10 @@ Quant: bf16 only. SGLang upstream does not support fp8 ViT FMHA. """ -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" from importlib.metadata import ( version as _dist_version, # setuptools/pkg_resources is absent from recent framework images diff --git a/python/aisimulate/collector/sglang/collect_computescale.py b/python/aisimulate/collector/sglang/collect_computescale.py index a9caa1599f..5325e21e99 100644 --- a/python/aisimulate/collector/sglang/collect_computescale.py +++ b/python/aisimulate/collector/sglang/collect_computescale.py @@ -10,9 +10,10 @@ extra_perf_filenames[0] -> scale_matrix_perf.txt (the static per-tensor quant alone) """ -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" from importlib.metadata import version as _dist_version # setuptools/pkg_resources is absent from sglang>=0.5.21 images diff --git a/python/aisimulate/collector/sglang/collect_dsv4_attn.py b/python/aisimulate/collector/sglang/collect_dsv4_attn.py index 9c7da934fd..9a20301d79 100644 --- a/python/aisimulate/collector/sglang/collect_dsv4_attn.py +++ b/python/aisimulate/collector/sglang/collect_dsv4_attn.py @@ -23,12 +23,13 @@ --batch-sizes 1,4 --seq-lens 128,1024 """ -# Requires stock SGLang 0.5.14 with its matching ``sgl-kernel`` package. +# Requires stock SGLang 0.5.21 with its matching ``sgl-kernel`` package. from __future__ import annotations -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import argparse import contextlib diff --git a/python/aisimulate/collector/sglang/collect_gdn.py b/python/aisimulate/collector/sglang/collect_gdn.py index 9ded6052cf..c2a9cc2e6d 100644 --- a/python/aisimulate/collector/sglang/collect_gdn.py +++ b/python/aisimulate/collector/sglang/collect_gdn.py @@ -73,22 +73,11 @@ # unaffected) -- cosmetic relocations (signatures independently re-diffed # byte-identical at both locations), fixed with version-conditional imports # mirroring collect_gemm.py's established try/except-import pattern (this -# file had none of its own before this bump). 0.5.15/0.5.16 stay excluded: -# never verified, and version_resolver's __compat__ grammar (AND-of- -# comparators only, no OR) has no way to express a true two-point set either -# -- this bounded range with the two untested base releases carved out via -# != is the closest expressible approximation, NOT an exact {0.5.14, 0.5.17} -# gate (CORRECTED 2026-08-18, code review): `_check_compat`'s `!=` only -# excludes the literal point version, so 0.5.15.post1/0.5.15rc1/ -# 0.5.16.post2-style variants of the excluded releases still satisfy this -# specifier (see tests/unit/collector/test_version_resolver.py's -# TestExactVersionSet for the honest, probed semantics). This is an -# accepted, narrow gap: the framework_manifest digest-pinned gate is the -# true version enforcement upstream and only ever supplies exactly 0.5.14 -# or 0.5.17 in a sanctioned run, so the leak is unreachable there. -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19,!=0.5.20" +# file had none of its own before this bump). +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import gc import os diff --git a/python/aisimulate/collector/sglang/collect_gemm.py b/python/aisimulate/collector/sglang/collect_gemm.py index 59fb19bdee..9b99936b5a 100644 --- a/python/aisimulate/collector/sglang/collect_gemm.py +++ b/python/aisimulate/collector/sglang/collect_gemm.py @@ -37,33 +37,23 @@ # per layer_permissions.md); the caveat is for data consumers: fp8_block # gemm rows collected at 0.5.14 and at 0.5.17 are NOT cross-version # comparable, because they time different kernels for the identical -# (m, n, k) inputs. 0.5.15/0.5.16 stay excluded: never verified, and -# version_resolver's __compat__ grammar (AND-of-comparators only, no OR) has -# no way to express a true two-point set either -- this bounded range with -# the two untested base releases carved out via != is the closest -# expressible approximation, NOT an exact {0.5.14, 0.5.17} gate (CORRECTED -# 2026-08-18, code review): `_check_compat`'s `!=` only excludes the literal -# point version, so 0.5.15.post1/0.5.15rc1/0.5.16.post2-style variants of -# the excluded releases still satisfy this specifier (see -# tests/unit/collector/test_version_resolver.py's TestExactVersionSet for -# the honest, probed semantics). This is an accepted, narrow gap: the -# framework_manifest digest-pinned gate is the true version enforcement -# upstream and only ever supplies exactly 0.5.14 or 0.5.17 in a sanctioned -# run, so the leak is unreachable there. -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -# SGLang 0.5.20 (GLM-5.3-Flash model pin; audited 2026-09-30 against tag -# commit 94602c9c2b7c): serving changed WHICH kernels this module's raw -# calls map to (bf16 CuTe DSL/split-K dispatch, JIT per-token fp8 quant, -# Fp8ScaledMMOp, fp8_block Triton fallback for N%64/K%128, NVFP4 padding and -# cute-dsl activation quant), so 0.5.20 runs delegate get_gemm_test_cases/ -# run_gemm to collect_gemm_0520.py (serving-built linear layers; full audit -# there). That is a whole-implementation fork selected by the installed -# version at the bottom of this module, not a per-call shim; it lives here -# instead of a registry VersionRoute because tools/perf_database/ -# changed_ops.py fails closed on versioned registry entries. 0.5.18/0.5.19 -# are not audited and stay excluded (same != caveat as above). -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19" +# (m, n, k) inputs. +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +# The SGLang 0.5.20 fork (collect_gemm_0520.py, serving-built linear layers) +# retired with the 0.5.20 lane 2026-10-30; it is git history (main at c540ff90a (before #649)). +# FIXME(re-verify, 2026-10-30): two independent SM100/103 serving gaps, accepted +# by the owner for the 0.5.21 tables (no re-collection). Re-check at the next bump. +# 1. bf16 GEMM kernel: serving resolves the auto bf16 backend to CuTe DSL +# (quantization/unquant.py:216@v0.5.21) and runs the TGV kernel only where +# use_cutedsl_bf16_gemm(m, n, k) picks it (kernels/ops/gemm/ +# cutedsl_bf16_gemm.py:1357-1387: small m, 2048 <= k <= 6144, n >= 1024); +# everywhere else serving runs F.linear like this module. +# 2. nvfp4 activation quant: serving quantizes with FlashInfer's cute-dsl backend +# (quantization/fp4_utils.py:24@v0.5.21); this module's flashinfer_fp4_quantize +# uses the default cuda backend. The mm_fp4 GEMM itself matches. +__compat__ = "sglang==0.5.21" import os import random @@ -142,23 +132,6 @@ from sglang.srt.layers.quantization.fp8_utils import requant_weight_ue8m0 -def _sglang_release() -> tuple[int, ...]: - """Installed SGLang release as an int tuple (local/pre-release tags dropped). - - Collection always runs with SGLang installed (collect.py resolves the - pinned runtime from the installed version first); without the - distribution metadata (offline import tests) the legacy path is kept. - """ - from importlib.metadata import PackageNotFoundError - - from packaging.version import Version - - try: - return Version(_dist_version("sglang")).release - except PackageNotFoundError: - return (0,) - - def get_gemm_test_cases(): test_cases = [] @@ -494,13 +467,3 @@ def kernel_func(): finally: op_list.clear() torch.cuda.empty_cache() - - -# Whole-implementation version fork (see the __compat__ comment): SGLang -# 0.5.20 collects through serving-built linear layers. Exactly 0.5.20: the -# 0.5.21 default keeps this module's own (0.5.21-validated) implementation. -if _sglang_release()[:3] == (0, 5, 20): - from collector.sglang import collect_gemm_0520 as _fork_0520 - - get_gemm_test_cases = _fork_0520.get_gemm_test_cases - run_gemm = _fork_0520.run_gemm diff --git a/python/aisimulate/collector/sglang/collect_gemm_0520.py b/python/aisimulate/collector/sglang/collect_gemm_0520.py deleted file mode 100644 index a8001fcad7..0000000000 --- a/python/aisimulate/collector/sglang/collect_gemm_0520.py +++ /dev/null @@ -1,278 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 - -"""SGLang 0.5.20 GEMM collector (serving-built linear layers). - -At SGLang 0.5.20 the same raw kernel calls that collect_gemm.py times no -longer match serving for several dtypes, so this 0.5.20 fork (selected by collect_gemm.py) builds each -GEMM the way serving does -- a ``ReplicatedLinear`` whose quant method is -constructed from the checkpoint-style quant config -- and times -``layer.quant_method.apply`` after ``process_weights_after_loading``. The -framework therefore owns weight repacking, padding, fallbacks and kernel -selection; kernel_source records the branch the framework took. -""" - -# Audited 2026-09-30 against sglang v0.5.20 (tag commit -# 94602c9c2b7cbdb8efd5c52802dac6a1c180089e, source clone) vs v0.5.17 -# (29481685). -# Dependency pins moved sglang-kernel 0.4.5->0.4.7, flashinfer -# 0.6.15.post1->0.6.18, sgl-deep-gemm 0.1.5.post1->0.2.0, torch 2.11->2.13, -# so rows are not cross-version comparable with collect_gemm.py's. -# Serving initialization: the scheduler calls initialize_fp8_gemm_config, -# initialize_fp4_gemm_config and initialize_bf16_gemm_config -# (managers/scheduler.py:999-1001); _ensure_serving_gemm_config() does the -# same after publishing ServerArgs. -# bf16: UnquantizedLinearMethod.apply (quantization/unquant.py:460-499) -# routes bf16 through _bf16_gemm_dispatch_impl when the bf16 backend is -# cutedsl, which "auto" resolves to on SM100/103 (unquant.py:215-220). -# The dispatch order is FlashInfer split-K/direct for the tuned (m,n,k) -# table (:106-181,199-201,283-313, SGLANG_ENABLE_BF16_SPLITK_GEMM), -# CuTe DSL TGV when use_cutedsl_bf16_gemm(m,n,k) (cutedsl_bf16_gemm.py: -# 1357-1387), else F.linear (unquant.py:315-350). collect_gemm.py times -# F.linear unconditionally, which is not serving truth on SM10x. -# fp8_block: Fp8LinearMethod (quantization/fp8.py) with the auto runner -# deepgemm_w8a8_block_fp8_linear_with_fallback when JIT DeepGEMM is on -# (fp8_utils.py:799-810); shapes with N%64 or K%128 fall back to Triton -# with fp32 scales (fp8_utils.py:1094-1107) and their scales are not -# requantized to UE8M0 (fp8.py:758-770, model_loader/utils.py:265-291). -# nvfp4: ModelOptFp4LinearMethod (quantization/modelopt_quant.py:1706-2075) -# pads N/K to 32 (pad_nvfp4_weight, :195-250), quantizes activations with -# fp4_utils.fp4_quantize (FlashInfer "cute-dsl" backend on SM10x, -# fp4_utils.py:26-49) and runs mm_fp4 with the auto fp4 runner -# (fp4_utils.py:145-158: cute-dsl on SM100/103). -# The collect_gemm.py FIXME(kernel-limit) "n<128 or k<128" is false at this -# version (fp8_block falls back to Triton; nvfp4 pads to 32), so this fork -# queues those shapes and lets the framework handle them. -# Per-tensor fp8 is not collected at this version: no 0.5.20 consumer reads -# it (GLM-5.3-Flash serves fp8_block, nvfp4 and bf16 GEMMs only). -__compat__ = "sglang==0.5.20" - -import os -import random - -# SGLang reads this flag while importing ``deep_gemm_wrapper.compile_utils``. -# Set it before any SGLang imports so a task compiles only its requested M. -os.environ.setdefault("SGLANG_JIT_DEEPGEMM_PRECOMPILE", "0") - -from importlib.metadata import version as _dist_version - -import torch -from collector.case_generator import get_gemm_case_specs -from collector.helper import benchmark_with_power, get_sm_version, log_perf - -_SERVING_GEMM_CONFIG_READY = False - - -def _ensure_serving_gemm_config() -> None: - """Publish dummy ServerArgs, then run serving's GEMM backend initializers.""" - global _SERVING_GEMM_CONFIG_READY - if _SERVING_GEMM_CONFIG_READY: - return - from sglang.srt.runtime_context import get_server_args - from sglang.srt.server_args import ServerArgs, set_global_server_args_for_scheduler - - try: - get_server_args() # raises ValueError when nothing is published (runtime_context.py:1031-1037) - except ValueError: - set_global_server_args_for_scheduler(ServerArgs(model_path="dummy")) - from sglang.srt.layers.quantization.fp4_utils import initialize_fp4_gemm_config - from sglang.srt.layers.quantization.fp8_utils import initialize_fp8_gemm_config - from sglang.srt.layers.quantization.unquant import initialize_bf16_gemm_config - - initialize_fp8_gemm_config() - initialize_fp4_gemm_config() - initialize_bf16_gemm_config() - _SERVING_GEMM_CONFIG_READY = True - - -def get_gemm_test_cases(): - sm_version = get_sm_version() - if sm_version < 90: - gemm_list = ["bfloat16"] - elif sm_version < 100: - gemm_list = ["fp8_block", "bfloat16"] - elif sm_version < 110: - gemm_list = ["fp8_block", "bfloat16", "nvfp4"] - else: - # SM120 dense fp8_block serving is CUTLASS (fp8_utils.py:832-833); - # not collected here, unchanged from collect_gemm.py. - gemm_list = ["bfloat16", "nvfp4"] - - requested_gemm_types = os.environ.get("AIC_COLLECT_GEMM_TYPES") - if requested_gemm_types: - requested = {item.strip() for item in requested_gemm_types.split(",") if item.strip()} - gemm_list = [gemm_type for gemm_type in gemm_list if gemm_type in requested] - - test_cases = [[gemm_type, case.x, case.n, case.k] for case in get_gemm_case_specs() for gemm_type in gemm_list] - # Group DeepGEMM JIT cache hits the same way collect_gemm.py does. - random.seed(42) - random.shuffle(test_cases) - return test_cases - - -def _quant_config(gemm_type: str): - if gemm_type == "bfloat16": - return None - if gemm_type == "fp8_block": - from sglang.srt.layers.quantization.fp8 import Fp8Config - - return Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic", weight_block_size=[128, 128]) - if gemm_type == "nvfp4": - from sglang.srt.layers.quantization.modelopt_quant import ModelOptFp4Config - - # packed_modules_mapping must be a mapping: get_quant_method's - # is_layer_skipped iterates it (quantization/utils.py:84 @v0.5.20). - return ModelOptFp4Config( - is_checkpoint_nvfp4_serialized=True, group_size=16, exclude_modules=[], packed_modules_mapping={} - ) - raise ValueError(f"no serving quant config for {gemm_type!r}") - - -def _fill_checkpoint_tensors(layer: "torch.nn.Module", gemm_type: str) -> None: - """Populate the checkpoint-native parameters create_weights allocated.""" - with torch.no_grad(): - for name, param in layer.named_parameters(recurse=False): - data = param.data - if data.dtype == torch.uint8: - data.random_(0, 256) # packed e2m1 nibbles: no NaN/Inf encodings - elif data.dtype == torch.float8_e4m3fn: - if "scale" in name: - data.copy_(torch.ones(data.shape, device=data.device).to(data.dtype)) - else: - data.copy_(((torch.rand(data.shape, device=data.device) - 0.5) * 2).to(data.dtype)) - elif data.is_floating_point(): - if "scale" in name: - data.copy_(torch.rand(data.shape, device=data.device) * 0.5 + 0.5) - else: - data.copy_(torch.randn(data.shape, device=data.device) * 0.02) - - -def _build_linear(gemm_type: str, n: int, k: int, device: str): - from sglang.srt.layers.linear import ReplicatedLinear - - layer = ReplicatedLinear( - k, n, bias=False, params_dtype=torch.bfloat16, quant_config=_quant_config(gemm_type), prefix="collector_gemm" - ).to(device) - _fill_checkpoint_tensors(layer, gemm_type) - layer.quant_method.process_weights_after_loading(layer) - layer.requires_grad_(False) - return layer - - -def _bf16_kernel_source(m: int, n: int, k: int) -> str: - """Mirror the branch unquant.UnquantizedLinearMethod.apply takes (unquant.py:460-499).""" - from sglang.srt.layers.quantization import unquant - - if not unquant.get_bf16_gemm_backend().is_cutedsl(): - return "sglang_torch_linear" - if unquant._enable_bf16_splitk_gemm and unquant.use_bf16_splitk_gemm(m, n, k): - return "sglang_flashinfer_bf16_direct" if unquant._prefer_direct(m, n, k) else "sglang_flashinfer_bf16_splitk" - if unquant._use_cutedsl_bf16_gemm is not None and unquant._use_cutedsl_bf16_gemm(m, n, k): - return "sglang_cutedsl_tgv_bf16" - return "sglang_torch_linear" - - -def _fp8_block_kernel_source(layer, n: int, k: int) -> str: - from sglang.srt.layers.quantization.fp8_utils import deepgemm_w8a8_block_fp8_linear_with_fallback - - runner = layer.quant_method.w8a8_block_fp8_linear - if runner is deepgemm_w8a8_block_fp8_linear_with_fallback: - # fp8_utils.py:1094-1107: DeepGEMM only for N%64==0 and K%128==0. - if n % 64 == 0 and k % 128 == 0: - return "sglang_deepgemm_gemm_nt_f8f8bf16" - return "sglang_triton_w8a8_block_fp8" - return f"sglang_{getattr(runner, '__name__', type(runner).__name__)}" - - -def _nvfp4_kernel_source() -> str: - from sglang.srt.layers.quantization.fp4_utils import get_fp4_gemm_runner_backend - - backend = get_fp4_gemm_runner_backend().get_flashinfer_backend() - return f"sglang_flashinfer_{str(backend).replace('-', '')}_nvfp4" - - -def run_gemm(gemm_type, batch_size, N, K, *, perf_filename, device="cuda:0"): # noqa: N803 - if gemm_type not in {"fp8_block", "bfloat16", "nvfp4"}: - raise ValueError(f"unsupported SGLang GEMM type {gemm_type!r}") - torch.cuda.set_device(device) - torch.set_default_device(device) - _ensure_serving_gemm_config() - M = batch_size # noqa: N806 - dtype = torch.bfloat16 - - def create_gemm(): - x = torch.randn((M, K), dtype=dtype, device=device) - layer = _build_linear(gemm_type, N, K, device) - - def gemm_op(): - return layer.quant_method.apply(layer, x) - - gemm_op.layer = layer - return gemm_op - - op_list = [] - try: - gpu_mem = torch.cuda.get_device_properties(device).total_memory - persistent_bytes = M * K * 2 + M * N * 2 - transient_bytes = 0 - if gemm_type == "bfloat16": - persistent_bytes += N * K * 2 - elif gemm_type == "fp8_block": - persistent_bytes += N * K + M * K + N * 4 + M * 4 - transient_bytes = N * K * 4 - elif gemm_type == "nvfp4": - persistent_bytes += int(N * K * 0.75) - transient_bytes = N * K * 2 - budget = int(gpu_mem * 0.45) - if persistent_bytes + transient_bytes >= budget: - outside_loop_count = 1 - else: - outside_loop_count = max(1, min(6, (budget - transient_bytes) // max(persistent_bytes, 1))) - - for _ in range(outside_loop_count): - op_list.append(create_gemm()) - - # Eager dry run: JIT-compiles outside graph capture. - op_list[0]() - torch.cuda.synchronize() - if gemm_type == "bfloat16": - kernel_source = _bf16_kernel_source(M, N, K) - elif gemm_type == "fp8_block": - kernel_source = _fp8_block_kernel_source(op_list[0].layer, N, K) - else: - kernel_source = _nvfp4_kernel_source() - - def kernel_func(): - for op in op_list: - op() - - nvtx_tag = f"{gemm_type}_m{M}_n{N}_k{K}" - torch.cuda.nvtx.range_push(nvtx_tag) - try: - with benchmark_with_power( - device=device, - kernel_func=kernel_func, - num_warmups=3, - num_runs=6, - repeat_n=1, - ) as results: - pass - finally: - torch.cuda.nvtx.range_pop() - - if not log_perf( - item_list=[ - {"gemm_dtype": gemm_type, "m": M, "n": N, "k": K, "latency": results["latency_ms"] / len(op_list)} - ], - framework="SGLang", - version=_dist_version("sglang"), - device_name=torch.cuda.get_device_name(device), - op_name="gemm", - kernel_source=kernel_source, - perf_filename=perf_filename, - power_stats=results["power_stats"], - ): - raise RuntimeError(f"Failed to persist SGLang GEMM performance row to {perf_filename}") - finally: - op_list.clear() - torch.cuda.empty_cache() diff --git a/python/aisimulate/collector/sglang/collect_kda.py b/python/aisimulate/collector/sglang/collect_kda.py index d4ba1019c0..5c5156a02d 100644 --- a/python/aisimulate/collector/sglang/collect_kda.py +++ b/python/aisimulate/collector/sglang/collect_kda.py @@ -59,12 +59,12 @@ # packed_decode / fused_sigmoid_gating_delta_rule_update, kda_fused_decode, # causal_conv1d_fn/update — kda_backend.py:6-34, kernels/kda_triton.py:109-237; # H20 smoke 2026-10-01, 91 cells ok, the remaining 21 are the >=2**31 conv -# guard cells). 0.5.20 adds GLM-5.3-Flash (glm5_next) KDA, audited against -# the GB300 0.5.20 image (see the GLM section below). The file-level range -# spans all three; each KDA architecture is additionally gated at runtime to -# its own audited releases by _KDA_ARCHITECTURE_COMPAT (Kimi-K3 0.5.16 and -# 0.5.21, GLM-5.3-Flash 0.5.20), raising KdaRuntimeNotAuditedError otherwise. -__compat__ = "sglang>=0.5.16,<=0.5.21,!=0.5.17,!=0.5.18,!=0.5.19" +# guard cells). 0.5.20 added GLM-5.3-Flash (glm5_next) KDA, audited against +# the GB300 0.5.20 image (see the GLM section below). Each KDA architecture +# is additionally gated at runtime by _KDA_ARCHITECTURE_COMPAT, raising +# KdaRuntimeNotAuditedError otherwise. +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import gc import os @@ -1023,8 +1023,11 @@ def _is_glm5_next_kda(model_name: str) -> bool: # replicates; any other installed release raises KdaRuntimeNotAuditedError (a # classified failure) instead of timing a possibly different kernel path. _KDA_ARCHITECTURE_COMPAT = { - KIMI_K3_KDA_ARCHITECTURE: "sglang>=0.5.16,<=0.5.21,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20", - GLM5_NEXT_KDA_ARCHITECTURE: "sglang==0.5.20", + KIMI_K3_KDA_ARCHITECTURE: "sglang==0.5.21", + # GLM-5.3-Flash (0.5.20 only) retired 2026-10-30: its route is git history + # (main at c540ff90a (before #649)). On 0.5.21 serving fuses the unquantized + # qkvbfg/bfg projections (strided views), so the GLM path needs a port + # before it is re-admitted; GLM rows raise KdaRuntimeNotAuditedError. } @@ -1040,7 +1043,9 @@ def _kda_architecture(model_name: str) -> str: def _require_audited_runtime(model_name: str, runtime_version: str) -> None: architecture = _kda_architecture(model_name) - compat = _KDA_ARCHITECTURE_COMPAT[architecture] + compat = _KDA_ARCHITECTURE_COMPAT.get(architecture) + if compat is None: + raise KdaRuntimeNotAuditedError(f"sglang {runtime_version} is not an audited KDA runtime for {architecture}") if not _check_compat(compat, runtime_version): raise KdaRuntimeNotAuditedError( f"sglang {runtime_version} is not an audited KDA runtime for {architecture} (audited: {compat})" diff --git a/python/aisimulate/collector/sglang/collect_mhc_module.py b/python/aisimulate/collector/sglang/collect_mhc_module.py index 80f7a4e5cc..9685d3fbf8 100644 --- a/python/aisimulate/collector/sglang/collect_mhc_module.py +++ b/python/aisimulate/collector/sglang/collect_mhc_module.py @@ -10,11 +10,10 @@ # 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; # findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). -# The file-level range spans the audited routes; each architecture is -# additionally gated to its own audited releases by _ARCHITECTURE_COMPAT: -# DeepSeek-V4 on 0.5.14 and 0.5.21 (the 0.5.21 port uses the native -# DeepseekV4Config), GLM only on 0.5.20. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19" +# Each architecture is additionally gated by _ARCHITECTURE_COMPAT +# (DeepSeek-V4 on 0.5.21 via the native DeepseekV4Config). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import argparse import copy @@ -59,8 +58,11 @@ # Per-architecture audited SGLang releases; anything else raises # MhcRuntimeNotAuditedError (a classified failure) instead of running. _ARCHITECTURE_COMPAT = { - DSV4_ARCHITECTURE: "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20", - GLM5NEXT_ARCHITECTURE: "sglang==0.5.20", + DSV4_ARCHITECTURE: "sglang==0.5.21", + # GLM-5.3-Flash (0.5.20 only) retired 2026-10-30: its route is git history + # (main at c540ff90a (before #649)). 0.5.21 moved MHCState to layer_boundary, removed + # ParallelState/init_torch_distributed and fuses post+pre for T<=16, so the + # GLM path needs a port before it is re-admitted. } # GLM-5.3-Flash mHC call sites in SGLang 0.5.20 serving diff --git a/python/aisimulate/collector/sglang/collect_mla.py b/python/aisimulate/collector/sglang/collect_mla.py index f26a6a7e26..9415391dfd 100644 --- a/python/aisimulate/collector/sglang/collect_mla.py +++ b/python/aisimulate/collector/sglang/collect_mla.py @@ -9,9 +9,10 @@ mocking, runtime dispatch, and perf logging. """ -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import contextlib import inspect diff --git a/python/aisimulate/collector/sglang/collect_mla_bmm.py b/python/aisimulate/collector/sglang/collect_mla_bmm.py index d487a2f7ac..b34bd5f0c0 100644 --- a/python/aisimulate/collector/sglang/collect_mla_bmm.py +++ b/python/aisimulate/collector/sglang/collect_mla_bmm.py @@ -8,9 +8,10 @@ shapes, selects SGLang kernel helpers, and logs the resulting MLA BMM perf rows. """ -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" from importlib.metadata import version as _dist_version # setuptools/pkg_resources is absent from sglang>=0.5.21 images diff --git a/python/aisimulate/collector/sglang/collect_mla_module.py b/python/aisimulate/collector/sglang/collect_mla_module.py index 77ba8f8b59..307a56c817 100644 --- a/python/aisimulate/collector/sglang/collect_mla_module.py +++ b/python/aisimulate/collector/sglang/collect_mla_module.py @@ -22,9 +22,10 @@ python collect_mla_module.py --mode generation --attn-type mla """ -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import argparse import gc diff --git a/python/aisimulate/collector/sglang/collect_moe.py b/python/aisimulate/collector/sglang/collect_moe.py index 55568066c2..d16a3a80dc 100644 --- a/python/aisimulate/collector/sglang/collect_moe.py +++ b/python/aisimulate/collector/sglang/collect_moe.py @@ -42,21 +42,10 @@ # _pin_moe_runner_backend below for the full trace and the version-branched # fix: 0.5.14 keeps the exact original global read/write/restore, 0.5.17 # goes through the new RuntimeContext/Flags singleton via its own sanctioned -# override() primitive). 0.5.15/0.5.16 stay excluded: never verified, and -# version_resolver's __compat__ grammar (AND-of-comparators only, no OR) has -# no way to express a true two-point set either -- this bounded range with -# the two untested base releases carved out via != is the closest -# expressible approximation, NOT an exact {0.5.14, 0.5.17} gate (CORRECTED -# 2026-08-18, code review): `_check_compat`'s `!=` only excludes the literal -# point version, so 0.5.15.post1/0.5.15rc1/0.5.16.post2-style variants of -# the excluded releases still satisfy this specifier (see -# tests/unit/collector/test_version_resolver.py's TestExactVersionSet for -# the honest, probed semantics). This is an accepted, narrow gap: the -# framework_manifest digest-pinned gate is the true version enforcement -# upstream and only ever supplies exactly 0.5.14 or 0.5.17 in a sanctioned -# run, so the leak is unreachable there. -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. +# override() primitive). +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. # 0.5.20 audit (2026-09-30; source tag v0.5.20 == 94602c9c2b7c vs v0.5.17, # source-only). 0.5.18/0.5.19 # were never verified and are carved out with != (same caveat as above). @@ -90,7 +79,7 @@ # (moe_runner/flashinfer_cutlass.py:17,208-209), which the single-rank # parallel patch below does not replace, so flashinfer_cutlass lanes (the # SM120 nvfp4 map) are expected to fail until that module is added. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.18,!=0.5.19" +__compat__ = "sglang==0.5.21" import gc import importlib diff --git a/python/aisimulate/collector/sglang/collect_msa_module.py b/python/aisimulate/collector/sglang/collect_msa_module.py index 5a1d89cf2d..86feacd8b6 100644 --- a/python/aisimulate/collector/sglang/collect_msa_module.py +++ b/python/aisimulate/collector/sglang/collect_msa_module.py @@ -2,10 +2,10 @@ # SPDX-License-Identifier: Apache-2.0 # MiniMax-M3 support landed in SGLang v0.5.16 (models/minimax_m3.py + -# layers/attention/minimax_sparse_backend.py + minimax_sparse_ops/); the -# collector-default 0.5.14 pin has no M3. The framework manifest pins the -# msa family to the official v0.5.16 image. -__compat__ = "sglang>=0.5.16" +# layers/attention/minimax_sparse_backend.py + minimax_sparse_ops/); the msa +# family now runs on the 0.5.21 default (the v0.5.16 family pin is retired). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" """ MSA Module Collector for SGLang — MiniMax-M3 sparse-attention benchmarking. diff --git a/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py b/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py index b6fec83881..589f3d397f 100644 --- a/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py +++ b/python/aisimulate/collector/sglang/deepseekv4_sparse_modules.py @@ -31,12 +31,13 @@ c4 path (=4). """ -# Requires stock SGLang 0.5.14 with its matching ``sgl-kernel`` package. +# Requires stock SGLang 0.5.21 with its matching ``sgl-kernel`` package. from __future__ import annotations -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import functools import json diff --git a/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py b/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py index 9170f6bfaf..d266fbd60c 100644 --- a/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py +++ b/python/aisimulate/collector/sglang/glm5_dsa_sparse_modules.py @@ -27,9 +27,10 @@ from __future__ import annotations -# 0.5.21 added 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; -# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). Releases in between are unvalidated and excluded. -__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20" +# 0.5.21 since 2026-10-01 (H20/sm90 collector port: op_smoke + path gates in the v0.5.21 image; +# findings hopper_sglang_collector_port_0514_to_0521_2026_10_01). +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "sglang==0.5.21" import os import sys diff --git a/python/aisimulate/collector/trtllm/collect_attn.py b/python/aisimulate/collector/trtllm/collect_attn.py index 4d36121235..239ed6ac63 100644 --- a/python/aisimulate/collector/trtllm/collect_attn.py +++ b/python/aisimulate/collector/trtllm/collect_attn.py @@ -59,7 +59,7 @@ from tensorrt_llm.mapping import Mapping from tensorrt_llm.models.modeling_utils import QuantAlgo, QuantConfig -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" def _skip_trtllm_sm120_fp8_context_fmha( diff --git a/python/aisimulate/collector/trtllm/collect_attn_encoder.py b/python/aisimulate/collector/trtllm/collect_attn_encoder.py index 06170f11ce..ea029639ba 100644 --- a/python/aisimulate/collector/trtllm/collect_attn_encoder.py +++ b/python/aisimulate/collector/trtllm/collect_attn_encoder.py @@ -12,7 +12,7 @@ Quant: bf16 only. TRT-LLM upstream does not support fp8 on the encoder path. """ -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import os diff --git a/python/aisimulate/collector/trtllm/collect_computescale.py b/python/aisimulate/collector/trtllm/collect_computescale.py index eab200a839..667de7d235 100644 --- a/python/aisimulate/collector/trtllm/collect_computescale.py +++ b/python/aisimulate/collector/trtllm/collect_computescale.py @@ -15,7 +15,7 @@ extra_perf_filenames[0] -> scale_matrix_perf.txt (the static per-tensor quant alone) """ -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import tensorrt_llm import torch diff --git a/python/aisimulate/collector/trtllm/collect_dsv4_attn.py b/python/aisimulate/collector/trtllm/collect_dsv4_attn.py index f6590f24f3..a1c9c500b5 100644 --- a/python/aisimulate/collector/trtllm/collect_dsv4_attn.py +++ b/python/aisimulate/collector/trtllm/collect_dsv4_attn.py @@ -41,7 +41,7 @@ from __future__ import annotations -__compat__ = "trtllm>=1.3.0rc21" +__compat__ = "trtllm==1.3.0rc29" import gc import json diff --git a/python/aisimulate/collector/trtllm/collect_gdn.py b/python/aisimulate/collector/trtllm/collect_gdn.py index f34bdc8078..240401ca45 100644 --- a/python/aisimulate/collector/trtllm/collect_gdn.py +++ b/python/aisimulate/collector/trtllm/collect_gdn.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" """ GDN (Gated DeltaNet) Collector for AIConfigurator — TensorRT-LLM backend. diff --git a/python/aisimulate/collector/trtllm/collect_gemm.py b/python/aisimulate/collector/trtllm/collect_gemm.py index 2465b9e971..deec6122a3 100644 --- a/python/aisimulate/collector/trtllm/collect_gemm.py +++ b/python/aisimulate/collector/trtllm/collect_gemm.py @@ -9,7 +9,7 @@ quantized-weight setup, and perf logging. """ -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import ctypes import math diff --git a/python/aisimulate/collector/trtllm/collect_mamba2.py b/python/aisimulate/collector/trtllm/collect_mamba2.py index a77c37c01e..4ef76599bf 100644 --- a/python/aisimulate/collector/trtllm/collect_mamba2.py +++ b/python/aisimulate/collector/trtllm/collect_mamba2.py @@ -1,7 +1,7 @@ # SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" """ Mamba2 SSM Collector for AIConfigurator. diff --git a/python/aisimulate/collector/trtllm/collect_mhc_module.py b/python/aisimulate/collector/trtllm/collect_mhc_module.py index 26dc00dcec..8f5cdf92dc 100644 --- a/python/aisimulate/collector/trtllm/collect_mhc_module.py +++ b/python/aisimulate/collector/trtllm/collect_mhc_module.py @@ -52,7 +52,7 @@ from __future__ import annotations -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import argparse import os diff --git a/python/aisimulate/collector/trtllm/collect_mla.py b/python/aisimulate/collector/trtllm/collect_mla.py index 3d08d5955e..6518bddb7b 100644 --- a/python/aisimulate/collector/trtllm/collect_mla.py +++ b/python/aisimulate/collector/trtllm/collect_mla.py @@ -8,7 +8,7 @@ metadata objects, request/cache helpers, and backend-specific timing paths. """ -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import math from dataclasses import dataclass diff --git a/python/aisimulate/collector/trtllm/collect_mla_bmm.py b/python/aisimulate/collector/trtllm/collect_mla_bmm.py index 0581fe3bfd..8d970c5dde 100644 --- a/python/aisimulate/collector/trtllm/collect_mla_bmm.py +++ b/python/aisimulate/collector/trtllm/collect_mla_bmm.py @@ -31,7 +31,7 @@ and logs MLA BMM perf rows for pre/post generation operations. """ -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import tensorrt_llm import tensorrt_llm.quantization.utils.fp8_utils as fp8_utils diff --git a/python/aisimulate/collector/trtllm/collect_mla_module.py b/python/aisimulate/collector/trtllm/collect_mla_module.py index a1c8d8103a..697eebf4ac 100644 --- a/python/aisimulate/collector/trtllm/collect_mla_module.py +++ b/python/aisimulate/collector/trtllm/collect_mla_module.py @@ -45,7 +45,7 @@ # 1.3.0rc20 renamed the cache-manager sparse kwargs and moved attention # metadata to lowered SparseMetadataParams; this module follows those APIs. -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" """ MLA Module Collector for TRT-LLM — unified MLA and DSA benchmarking. diff --git a/python/aisimulate/collector/trtllm/collect_moe.py b/python/aisimulate/collector/trtllm/collect_moe.py index f49ba56158..7316439049 100755 --- a/python/aisimulate/collector/trtllm/collect_moe.py +++ b/python/aisimulate/collector/trtllm/collect_moe.py @@ -9,7 +9,7 @@ kernel-specific filters. """ -__compat__ = "trtllm>=1.3.0rc20" +__compat__ = "trtllm==1.3.0rc29" import gc import glob diff --git a/python/aisimulate/collector/trtllm/collect_msa_module.py b/python/aisimulate/collector/trtllm/collect_msa_module.py index 7645993986..a332f4e282 100644 --- a/python/aisimulate/collector/trtllm/collect_msa_module.py +++ b/python/aisimulate/collector/trtllm/collect_msa_module.py @@ -3,7 +3,7 @@ # MiniMax-M3 support landed in TRT-LLM 1.3.0rc19 (modeling_minimaxm3.py + # attention_backend/sparse/minimax_m3/); this collector follows the rc20 APIs. -__compat__ = "trtllm>=1.3.0rc19" +__compat__ = "trtllm==1.3.0rc29" """ MSA Module Collector for TRT-LLM — MiniMax-M3 sparse-attention benchmarking. diff --git a/python/aisimulate/collector/vllm/collect_attn.py b/python/aisimulate/collector/vllm/collect_attn.py index 2658d32d4c..9e6e53eca4 100644 --- a/python/aisimulate/collector/vllm/collect_attn.py +++ b/python/aisimulate/collector/vllm/collect_attn.py @@ -5,7 +5,7 @@ # targets the manifest pin; prior-version code is `git log -- ` # away and prior-version DATA stays permanent under its version key. The # 0.24->0.29 adaptations each carry a serving citation @0.29.0. -__compat__ = "vllm>=0.30.0,<=0.30.0" +__compat__ = "vllm==0.30.0" import os diff --git a/python/aisimulate/collector/vllm/collect_attn_encoder.py b/python/aisimulate/collector/vllm/collect_attn_encoder.py index 8b25ff8b00..cb96760a62 100644 --- a/python/aisimulate/collector/vllm/collect_attn_encoder.py +++ b/python/aisimulate/collector/vllm/collect_attn_encoder.py @@ -14,7 +14,7 @@ # encoder_attention: 8/8 representative cases. The native framework # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. -__compat__ = "vllm>=0.24.0,<=0.30.0" +__compat__ = "vllm==0.30.0" import torch from collector.case_generator import get_attention_encoder_head_configs, get_attention_encoder_shape_sweeps diff --git a/python/aisimulate/collector/vllm/collect_computescale.py b/python/aisimulate/collector/vllm/collect_computescale.py index c136ecdb85..48c07d5a23 100644 --- a/python/aisimulate/collector/vllm/collect_computescale.py +++ b/python/aisimulate/collector/vllm/collect_computescale.py @@ -14,7 +14,7 @@ # compute_scale: 8/8 representative cases. The native framework # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. -__compat__ = "vllm>=0.24.0,<=0.30.0" +__compat__ = "vllm==0.30.0" import torch from collector.case_generator import get_compute_scale_case_specs diff --git a/python/aisimulate/collector/vllm/collect_dsv4_attn.py b/python/aisimulate/collector/vllm/collect_dsv4_attn.py index 1208ab6612..0f007b22d5 100644 --- a/python/aisimulate/collector/vllm/collect_dsv4_attn.py +++ b/python/aisimulate/collector/vllm/collect_dsv4_attn.py @@ -76,7 +76,7 @@ # targets the manifest pin; prior-version code is `git log -- ` # away and prior-version DATA stays permanent under its version key. The # 0.24->0.29 adaptations each carry a serving citation @0.29.0. -__compat__ = "vllm>=0.30.0,<=0.30.0" +__compat__ = "vllm==0.30.0" DEFAULT_MODEL = _DSV4_DEFAULT_MODELS[0] diff --git a/python/aisimulate/collector/vllm/collect_gdn.py b/python/aisimulate/collector/vllm/collect_gdn.py index badbbd0420..4bb9bfb845 100644 --- a/python/aisimulate/collector/vllm/collect_gdn.py +++ b/python/aisimulate/collector/vllm/collect_gdn.py @@ -60,7 +60,7 @@ # 1968047). Existing framework construction/dispatch is unchanged. Invalid # model/quant shapes continue to raise and remain recorded failures; this is # API compatibility, not a claim that every Cartesian shape is supported. -__compat__ = "vllm>=0.24.0,<=0.30.0,!=0.25.1,!=0.26.0,!=0.27.0" +__compat__ = "vllm==0.30.0" import gc import os diff --git a/python/aisimulate/collector/vllm/collect_gemm.py b/python/aisimulate/collector/vllm/collect_gemm.py index 304d1de7b0..e0596c7b80 100644 --- a/python/aisimulate/collector/vllm/collect_gemm.py +++ b/python/aisimulate/collector/vllm/collect_gemm.py @@ -59,10 +59,10 @@ # (schemes/compressed_tensors_w4a4_nvfp4.py:29-31,95-141). This adds the # exact 0.25.0 release to the existing lane, not 0.25.1/0.26.0/0.27.0. # 0.30.0 source audit (2026-09-30: tag v0.30.0 == ced6857afa0e vs v0.27.1): -# the collector default (framework_manifest.yaml; the 2026-10 campaign -# tables) and the baseline of the 0.31.0 audit below, which admits the -# GLM-5.3-Flash model-pinned run. 0.28.0/0.29.0 stay admitted from the -# 0.27.1 -> 0.30.0 upgrade path. +# the 2026-10 campaign tables and the baseline of the 0.31.0 audit below. +# The gemm family runs on 0.31.0 (framework_manifest.yaml family pin, +# 2026-10-30), so this file targets exactly 0.31.0. +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. # bf16: UnquantizedLinearMethod now binds its GEMM at __init__ from # kernel_config.linear_backend (layers/linear.py:169-174, apply :234-238); # dispatch_unquantized_gemm (layers/utils.py:584-616) returns @@ -131,7 +131,7 @@ # Helpers with identical bodies: scaled_fp4_quant (_custom_ops.py:1513-1588), # per_block_cast_to_fp8 (utils/deep_gemm.py:983-1002). All imports below # resolve at 0.31.0. -__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0" +__compat__ = "vllm==0.31.0" from types import SimpleNamespace diff --git a/python/aisimulate/collector/vllm/collect_kda.py b/python/aisimulate/collector/vllm/collect_kda.py index 36ebc67da2..d8cf56c35b 100644 --- a/python/aisimulate/collector/vllm/collect_kda.py +++ b/python/aisimulate/collector/vllm/collect_kda.py @@ -48,16 +48,17 @@ kda_perf.txt — same column layout as the sglang kda collector. """ -# Kimi-K3 runs on the manifest default 0.30.0 since the kda family pin on -# the kimi-k3 preview image was retired 2026-09-28: 0.30's DS-layout probe +# The kda family runs on 0.31.0 (framework_manifest.yaml family pin, +# 2026-10-30) for both architectures. History: Kimi-K3 ran on 0.30.0 after +# the kimi-k3 preview pin was retired 2026-09-28: 0.30's DS-layout probe # (is_fused_kda_decode_supported, now with recurrent_state_dtype) selects # fused_kda_decode for the non-spec decode, and the packed conv-update + # recurrence pair is measured alongside as the spec-decode serving path. -# GLM-5.3-Flash runs on stock 0.31.0 (no overlay; model-pinned run). The -# file-level range spans the two audited routes; each KDA architecture is -# additionally gated at runtime to its own audited release by -# _KDA_ARCHITECTURE_COMPAT, raising KdaRuntimeNotAuditedError otherwise. -__compat__ = "vllm>=0.30.0,<=0.31.0" +# GLM-5.3-Flash runs on stock 0.31.0 (no overlay). Each KDA architecture is +# additionally gated at runtime by _KDA_ARCHITECTURE_COMPAT, raising +# KdaRuntimeNotAuditedError otherwise. +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "vllm==0.31.0" import gc import os @@ -882,7 +883,7 @@ def _is_glm5_next_kda(model_name: str) -> bool: # replicates; any other installed release raises KdaRuntimeNotAuditedError (a # classified failure) instead of timing a possibly different kernel path. _KDA_ARCHITECTURE_COMPAT = { - KIMI_K3_KDA_ARCHITECTURE: "vllm==0.30.0", + KIMI_K3_KDA_ARCHITECTURE: "vllm==0.31.0", # 0.30.0 route -> git history; GPU smoke 2026-10-30 GLM5_NEXT_KDA_ARCHITECTURE: "vllm==0.31.0", } diff --git a/python/aisimulate/collector/vllm/collect_mhc_module.py b/python/aisimulate/collector/vllm/collect_mhc_module.py index a5c187b9fd..c62a59a59b 100644 --- a/python/aisimulate/collector/vllm/collect_mhc_module.py +++ b/python/aisimulate/collector/vllm/collect_mhc_module.py @@ -28,12 +28,12 @@ # builders/selectors remain authoritative; no kernel fallback is introduced. # The campaign manifest still selects one exact release per run. # -# The file-level range spans the audited routes; each architecture is -# additionally gated to its own audited release(s) by ``_ARCHITECTURE_COMPAT`` -# below: DeepSeek-V4 on 0.24.0-0.30.0 (0.30.0 is the collector default), -# GLM-5.3-Flash on stock vLLM 0.31.0 only (no overlay); an architecture on -# any other release raises instead of running. -__compat__ = "vllm>=0.24.0,<=0.31.0" +# The mhc family runs on 0.31.0 (framework_manifest.yaml family pin, +# 2026-10-30). Each architecture is additionally gated by +# ``_ARCHITECTURE_COMPAT`` below; an architecture that is not audited on the +# installed release raises instead of running. +# One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +__compat__ = "vllm==0.31.0" # vLLM imports stay lazy in this module so that a mismatched install fails # inside collect.py's per-op error handling (after the __compat__ gate can @@ -62,8 +62,13 @@ def _mhc_tilelang_kernels(): # Per-architecture audited vLLM releases. A case whose architecture is not # audited on the installed release raises MhcRuntimeNotAuditedError (a # classified failure) instead of running an unverified dispatch. +# DeepSeek-V4 is NOT audited on 0.31.0. mhc_module_perf's architecture-free +# view (MhcKey = op_name, hc_mult, hidden_size; the DeepSeek-V4 consumer) +# would merge GLM-5.3-Flash's fused-site rows at (4, 4096) into one +# 0.31.0 table, so DeepSeek-V4 rows stay on the 0.24.0-0.30.0 tables and +# their collector route is `git log -- ` away. Re-admitting it +# needs an architecture-keyed DeepSeek-V4 lookup first (consumer change). _ARCHITECTURE_COMPAT = { - ARCHITECTURE: "vllm>=0.24.0,<=0.30.0", # Stock vllm 0.31.0 only (no overlay) — see the GLM audit below. GLM5NEXT_ARCHITECTURE: "vllm==0.31.0", } diff --git a/python/aisimulate/collector/vllm/collect_mla_bmm.py b/python/aisimulate/collector/vllm/collect_mla_bmm.py index b8a73fe40c..7313ddc22e 100644 --- a/python/aisimulate/collector/vllm/collect_mla_bmm.py +++ b/python/aisimulate/collector/vllm/collect_mla_bmm.py @@ -23,7 +23,7 @@ # MLA bmm call site unchanged in-image on GB300). # B200 0.25.0 qualification: both pre/post torch.bmm callbacks passed in # job 1968047. The NVIDIA bf16 BMM contract and input geometry are unchanged. -__compat__ = "vllm>=0.25.0,<=0.30.0,!=0.25.1,!=0.26.0" +__compat__ = "vllm==0.30.0" from importlib.metadata import ( version as _dist_version, # setuptools/pkg_resources is absent from recent framework images diff --git a/python/aisimulate/collector/vllm/collect_mla_module.py b/python/aisimulate/collector/vllm/collect_mla_module.py index bfd1e33c0d..2991901bb7 100644 --- a/python/aisimulate/collector/vllm/collect_mla_module.py +++ b/python/aisimulate/collector/vllm/collect_mla_module.py @@ -10,7 +10,7 @@ # # Portions adapted from vLLM dd10e03f95f94edbea1975c67ace3a35ec9a8a40. # Copyright contributors to the vLLM project. Modified for collector execution. -__compat__ = "vllm>=0.30.0,<=0.30.0" +__compat__ = "vllm==0.30.0" """ MLA Module Collector for vLLM — unified MLA and DSA benchmarking. diff --git a/python/aisimulate/collector/vllm/collect_moe.py b/python/aisimulate/collector/vllm/collect_moe.py index d058411701..041037d244 100644 --- a/python/aisimulate/collector/vllm/collect_moe.py +++ b/python/aisimulate/collector/vllm/collect_moe.py @@ -108,7 +108,10 @@ # (FlashInfer lanes are timed on the autotuned tactic like serving; see # helper.flashinfer_autotuned.) # 0.31.0 audit (2026-10-07, source-only: tag v0.31.0 == db9527a4 vs v0.30.0 -# ced6857a; no releases in between). GLM-5.3-Flash now runs STOCK 0.31.0 +# ced6857a; no releases in between). The moe family runs on 0.31.0 +# (framework_manifest.yaml family pin, 2026-10-30), so this file targets +# exactly 0.31.0. One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. +# GLM-5.3-Flash now runs STOCK 0.31.0 # (no overlay). Construction surface unchanged: FusedMoEFactory # (layer.py:88-141) keeps every kwarg passed here (swiglu_limit :108, # activation_situ_beta :111, router_logits_dtype :125, @@ -163,7 +166,7 @@ # K3 SiTU: FusedMoEFactory exposes activation_situ_beta on stock 0.30.0 and # 0.31.0, so the probe below passes the betas natively there; the # situ-as-silu approximation applies only to older stock builds. -__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0" +__compat__ = "vllm==0.31.0" import contextlib import json diff --git a/python/aisimulate/collector/vllm/collect_msa_module.py b/python/aisimulate/collector/vllm/collect_msa_module.py index f132d77ad4..8ccfdf34fc 100644 --- a/python/aisimulate/collector/vllm/collect_msa_module.py +++ b/python/aisimulate/collector/vllm/collect_msa_module.py @@ -7,7 +7,7 @@ # targets the manifest pin; prior-version code is `git log -- ` # away and prior-version DATA stays permanent under its version key. The # 0.24->0.29 adaptations each carry a serving citation @0.29.0. -__compat__ = "vllm>=0.30.0,<=0.30.0" +__compat__ = "vllm==0.30.0" """ MSA Module Collector for vLLM — MiniMax-M3 sparse-attention benchmarking. diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py index 4d4bba49ed..70e0b8bc7b 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py @@ -84,7 +84,7 @@ def test_glm5_next_decode_uses_bounded_gate_recurrence_not_packed_decode(): @pytest.mark.parametrize( ("version", "accepted"), - [("0.5.16", True), ("0.5.17", False), ("0.5.19", False), ("0.5.20", True), ("0.5.21", True), ("0.5.22", False)], + [("0.5.16", False), ("0.5.17", False), ("0.5.19", False), ("0.5.20", False), ("0.5.21", True), ("0.5.22", False)], ) def test_kda_compat_admits_kimi_branch_and_glm_runtime(version, accepted): from collector.version_resolver import _check_compat @@ -135,8 +135,7 @@ def _gate_namespace(): AUDITED_KDA_RELEASES = { - "KimiK3ForConditionalGeneration": ("0.5.16", "0.5.21"), - "Glm5NextForConditionalGeneration": ("0.5.20",), + "KimiK3ForConditionalGeneration": ("0.5.21",), } @@ -154,11 +153,11 @@ def test_kda_audited_release_sets_do_not_overlap(): @pytest.mark.parametrize( ("model_name", "version", "audited"), [ - ("moonshotai/Kimi-K3", "0.5.16", True), + ("moonshotai/Kimi-K3", "0.5.16", False), ("moonshotai/Kimi-K3", "0.5.21", True), ("moonshotai/Kimi-K3", "0.5.20", False), ("zai-org/GLM-5.3-Flash", "0.5.21", False), - ("zai-org/GLM-5.3-Flash", "0.5.20", True), + ("zai-org/GLM-5.3-Flash", "0.5.20", False), # 0.5.20 lane retired 2026-10-30 ("zai-org/GLM-5.3-Flash", "0.5.16", False), ], ) diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mhc_module.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mhc_module.py index 9ae8dc5b71..65a463f094 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_mhc_module.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mhc_module.py @@ -50,7 +50,7 @@ def test_mhc_sweep_fails_closed_after_an_inner_error(monkeypatch): runner = SimpleNamespace(model=SimpleNamespace(model=SimpleNamespace(layers=[layer]))) monkeypatch.setattr(module, "_load_one_layer_runner", lambda *_args, **_kwargs: runner) monkeypatch.setattr(module, "_hidden_size", lambda _layer: 4096) - monkeypatch.setattr(module, "get_version", lambda _package: "0.5.14") + monkeypatch.setattr(module, "get_version", lambda _package: "0.5.21") monkeypatch.setattr(module, "_make_residual", lambda _layer, num_tokens, _device: num_tokens) monkeypatch.setattr(module, "_make_kernel", lambda _layer, _op, residual: residual) monkeypatch.setattr(module, "_mhc_call_args", lambda _layer: (None, None)) diff --git a/python/aisimulate/tests/unit/collector/test_framework_manifest.py b/python/aisimulate/tests/unit/collector/test_framework_manifest.py index b8dd9a740c..8e824d569a 100644 --- a/python/aisimulate/tests/unit/collector/test_framework_manifest.py +++ b/python/aisimulate/tests/unit/collector/test_framework_manifest.py @@ -59,11 +59,9 @@ def test_manifest_exposes_pinned_vllm_xpu_runtime_identity(): def test_active_cuda_vllm_collectors_are_exactly_pinned_to_manifest_version(): assert all(not entry.versions for entry in VLLM_REGISTRY) - # Each module pins the runtime that actually collects it: the manifest - # default, or its family override (e.g. kda runs only on the vllm kimi-k3 - # preview image, frameworks.vllm.families.kda). The Qwen3.8 target-lane - # collectors accept both the default 0.24.0 runtime and model-pinned - # 0.27.1, so compare the declaration semantically just as collect.py does. + # Each module pins exactly the runtime that collects it: the manifest + # default, or its family override (frameworks.vllm.families, e.g. gemm on + # 0.31.0). One release per file (owner policy 2026-09-20): no ranges. module_versions: dict[str, set[str]] = {} for entry in VLLM_REGISTRY: module_versions.setdefault(entry.module, set()).add(resolve_op_runtime("vllm", entry.op).version) @@ -74,22 +72,28 @@ def test_active_cuda_vllm_collectors_are_exactly_pinned_to_manifest_version(): source = (REPO_ROOT / f"{module.replace('.', '/')}.py").read_text(encoding="utf-8") declarations = [line.strip() for line in source.splitlines() if line.startswith("__compat__")] assert len(declarations) == 1, module - declared = declarations[0].split("=", 1)[1].strip().strip('"') - assert _check_compat(declared, resolved_version), (module, declared, resolved_version) + assert declarations[0] == f'__compat__ = "vllm=={resolved_version}"', (module, declarations[0]) -@pytest.mark.parametrize( - "module", - ["collector.vllm.collect_moe", "collector.vllm.collect_gdn", "collector.vllm.collect_gemm"], -) -def test_vllm_target_lane_collectors_declare_the_exact_bumped_compat_range(module): - expected = '__compat__ = "vllm>=0.24.0,<=0.30.0,!=0.25.1,!=0.26.0,!=0.27.0"' - if module in {"collector.vllm.collect_gemm", "collector.vllm.collect_moe"}: - # GLM-5.3-Flash model pin: 0.31.0 is admitted on top of the 0.30.0 default lanes. - expected = '__compat__ = "vllm>=0.24.0,<=0.31.0,!=0.25.1,!=0.26.0,!=0.27.0"' - source = (REPO_ROOT / f"{module.replace('.', '/')}.py").read_text(encoding="utf-8") - declarations = [line.strip() for line in source.splitlines() if line.startswith("__compat__")] - assert declarations == [expected], module +@pytest.mark.parametrize("backend", ["sglang", "trtllm"]) +def test_stock_collectors_are_exactly_pinned_to_their_resolved_runtime(backend): + # One release per file (owner policy 2026-09-20): every registry module + # declares exactly the runtime the manifest resolves for its op (default or + # family pin). Earlier releases' code is `git log` away, never a range. + registry = {"sglang": SGLANG_REGISTRY, "trtllm": TRTLLM_REGISTRY}[backend] + module_versions: dict[str, set[str]] = {} + for entry in registry: + module_versions.setdefault(entry.module, set()).add(resolve_op_runtime(backend, entry.op).version) + for module, versions in sorted(module_versions.items()): + assert len(versions) == 1, (module, versions) + source = (REPO_ROOT / f"{module.replace('.', '/')}.py").read_text(encoding="utf-8") + declarations = [line.strip() for line in source.splitlines() if line.startswith("__compat__")] + version = next(iter(versions)) + allowed = {f'__compat__ = "{backend}=={version}"'} + # a module pinned to an exact source commit is an exact pin too + if declarations and "@" in declarations[0]: + allowed = {declarations[0]} + assert len(declarations) == 1 and declarations[0] in allowed, (module, declarations) def test_active_vllm_xpu_collectors_are_exactly_pinned_to_manifest_version(): @@ -384,81 +388,26 @@ def test_unknown_model_id_falls_back_to_default_resolution(): assert unmatched == baseline -@pytest.mark.parametrize( - "model_path", - [ - "Qwen/Qwen3.8-2.4T-A95B", - "Qwen/Qwen3.8-2.4T-A95B-FP8", - "RadixArk/Qwen3.8-2.4T-A95B-NVFP4", - ], -) -def test_model_pin_match_resolves_qwen38_max_to_sglang_0_5_17(model_path): - # Real manifest (no path= override): the checked-in frameworks.sglang.models - # entries added for Qwen3.8-Max day-0 support. - runtime = require_collector_runtime( - "sglang", "0.5.17", requested_ops={"gemm"}, wideep_ops=WIDEEP_OPS, model_path=model_path - ) - assert runtime.version == "0.5.17" - assert runtime.image().startswith("lmsysorg/sglang:v0.5.17@sha256:") - assert runtime.image("cu130").startswith("lmsysorg/sglang:v0.5.17-cu130@sha256:") - # Not a family classification — see _model_pinned_runtime docstring. - assert runtime.family is None - - -def test_model_pin_mismatch_error_names_the_model_scoped_image(): - with pytest.raises(RuntimeError) as excinfo: - require_collector_runtime( - "sglang", - "0.5.21", - requested_ops={"gemm"}, - wideep_ops=WIDEEP_OPS, - model_path="Qwen/Qwen3.8-2.4T-A95B", - ) - message = str(excinfo.value) - # Same template as the pre-4b guard ("~:249"), but naming the - # model-scoped runtime/image instead of the framework default. - assert "sglang stock collector requires exactly 0.5.17, found 0.5.21" in message - assert "use lmsysorg/sglang:v0.5.17@sha256:" in message - - -@pytest.mark.parametrize( - "model_path", - [ - "Qwen/Qwen3.8-2.4T-A95B", - "Qwen/Qwen3.8-2.4T-A95B-FP8", - "RadixArk/Qwen3.8-2.4T-A95B-NVFP4", - ], -) -def test_model_pin_match_resolves_qwen38_max_to_vllm_0_27_1(model_path): - runtime = require_collector_runtime( - "vllm", "0.27.1", requested_ops={"gemm"}, wideep_ops=set(), model_path=model_path - ) - assert runtime.version == "0.27.1" - assert runtime.image().startswith("vllm/vllm-openai:v0.27.1@sha256:") - assert runtime.image("cu129").startswith("vllm/vllm-openai:v0.27.1-cu129@sha256:") - assert runtime.family is None - - def test_vllm_model_pin_mismatch_error_names_the_model_scoped_image(): with pytest.raises(RuntimeError) as excinfo: require_collector_runtime( "vllm", - "0.24.0", - requested_ops={"gemm"}, + "0.30.0", + requested_ops={"attention_context"}, wideep_ops=set(), - model_path="Qwen/Qwen3.8-2.4T-A95B", + model_path="zai-org/GLM-5.3-Flash", ) message = str(excinfo.value) - assert "vllm stock collector requires exactly 0.27.1, found 0.24.0" in message - assert "use vllm/vllm-openai:v0.27.1@sha256:" in message + assert "vllm stock collector requires exactly 0.31.0, found 0.30.0" in message + assert "use vllm/vllm-openai:v0.31.0@sha256:" in message def test_vllm_unknown_model_id_falls_back_to_default_resolution(): - baseline = require_collector_runtime("vllm", "0.30.0", requested_ops={"gemm"}, wideep_ops=set()) + baseline = require_collector_runtime("vllm", "0.30.0", requested_ops={"attention_context"}, wideep_ops=set()) unmatched = require_collector_runtime( "vllm", "0.30.0", - requested_ops={"gemm"}, + requested_ops={"attention_context"}, wideep_ops=set(), model_path="some-org/not-a-pinned-model", ) @@ -789,20 +738,16 @@ def test_model_pin_images_must_be_digest_pinned(tmp_path): @pytest.mark.parametrize( "version,accepted", [ - ("0.24.0", True), - ("0.25.0", True), - ("0.25.1", False), - ("0.26.0", False), - ("0.27.0", False), - ("0.27.1", True), - ("0.28.0", True), - ("0.29.0", True), - ("0.30.0", True), - ("0.31.0", True), + ("0.24.0", False), + ("0.25.0", False), + ("0.27.1", False), + ("0.28.0", False), + ("0.30.0", False), # 0.30.0 gemm collector -> git history + ("0.31.0", True), # gemm family pin (2026-10-30) ("0.31.1", False), ], ) -def test_gemm_025_qualification_preserves_other_release_gaps(version, accepted): +def test_gemm_admits_only_its_family_release(version, accepted): import ast source = ast.parse((COLLECTOR_ROOT / "vllm" / "collect_gemm.py").read_text()) diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py index dcb41bad5c..0e9c86a013 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py @@ -5,8 +5,6 @@ import ast import json -import sys -import types from pathlib import Path import pytest @@ -18,10 +16,9 @@ moe_model_allows_quantization, ) from collector.model_cases import build_collection_case_plan -from collector.sglang.registry import REGISTRY as SGLANG_REGISTRY from collector.version_resolver import _check_compat -from .test_getter_deduplication import _install_vllm_stubs, _load_collector, _stub_module +from .test_getter_deduplication import _install_vllm_stubs, _load_collector pytestmark = pytest.mark.unit @@ -169,83 +166,28 @@ def test_vllm_nvfp4_uses_modelopt_only_for_the_glm53flash_checkpoint(monkeypatch @pytest.mark.parametrize( "version,accepted", [ - ("0.31.0", True), - ("0.30.0", True), # the collector default lanes - ("0.28.0", True), - ("0.29.0", True), + ("0.31.0", True), # gemm / moe family pins (2026-10-30) + ("0.30.0", False), # 0.30.0 collectors -> git history + ("0.28.0", False), + ("0.29.0", False), ("0.31.1", False), ("0.32.0", False), - ("0.24.0", True), + ("0.24.0", False), ], ) def test_vllm_collectors_admit_exactly_the_audited_releases(path, version, accepted): assert _check_compat(_compat(path), version) is accepted -@pytest.mark.parametrize( - "path,fork,names", - [("collector/sglang/collect_gemm.py", "collect_gemm_0520", ("get_gemm_test_cases", "run_gemm"))], -) -def test_sglang_0_5_20_runs_delegate_to_the_serving_built_fork(path, fork, names): - """The registry module stays unversioned (changed_ops rejects VersionRoute - entries) and swaps in the 0.5.20 implementation by installed release.""" - tree = ast.parse((REPO_ROOT / path).read_text()) - gate = next( - node - for node in tree.body - if isinstance(node, ast.If) - and "_sglang_release()" in ast.unparse(node.test) - and "(0, 5, 20)" in ast.unparse(node.test) - ) - body = ast.unparse(gate) - # exactly 0.5.20: the 0.5.21 default keeps the module's own implementation - assert "==" in ast.unparse(gate.test) - assert f"from collector.sglang import {fork} as _fork_0520" in body - for name in names: - assert f"{name} = _fork_0520.{name}" in body - entry = next(e for e in SGLANG_REGISTRY if e.module == f"collector.sglang.{Path(path).stem}") - assert not entry.versions - compat = _compat(path) - assert _check_compat(compat, "0.5.20") and _check_compat(compat, "0.5.14") - for version in ("0.5.18", "0.5.19"): - assert not _check_compat(compat, version) - assert _check_compat(_compat(f"collector/sglang/{fork}.py"), "0.5.20") - assert not _check_compat(_compat(f"collector/sglang/{fork}.py"), "0.5.17") - - -def test_sglang_moe_admits_0_5_20_but_not_unaudited_releases(): - compat = _compat("collector/sglang/collect_moe.py") - for version, accepted in [ - ("0.5.14", True), - ("0.5.17", True), - ("0.5.18", False), - ("0.5.19", False), - ("0.5.20", True), - ("0.5.21", True), - ]: - assert _check_compat(compat, version) is accepted, version - - -def _load_sglang_gemm_0520(monkeypatch): - _stub_module(monkeypatch, "torch", bfloat16="bfloat16", float32="float32") - helper = types.ModuleType("collector.helper") - helper.benchmark_with_power = None - helper.log_perf = None - helper.get_sm_version = lambda: 103 - monkeypatch.setitem(sys.modules, "collector.helper", helper) - return _load_collector(monkeypatch, "collector.sglang.collect_gemm_0520", "collector/sglang/collect_gemm_0520.py") - - -def test_sglang_0520_queues_small_shapes_the_v1_fixme_skipped(monkeypatch): - monkeypatch.setenv("COLLECTOR_MODEL_PATH", FP8) - module = _load_sglang_gemm_0520(monkeypatch) - cases = module.get_gemm_test_cases() - types_by_shape = {} - for gemm_type, x, n, k in cases: - types_by_shape.setdefault((n, k), set()).add(gemm_type) - # Per-tensor fp8 has no 0.5.20 consumer and is not planned. - assert types_by_shape[(288, 4096)] == {"bfloat16", "fp8_block", "nvfp4"} - assert types_by_shape[(32, 4096)] == {"bfloat16", "fp8_block", "nvfp4"} +def test_sglang_collectors_admit_only_0_5_21(): + # The 0.5.20 GLM-5.3-Flash lane (incl. the collect_gemm_0520 fork) retired + # 2026-10-30; one release per file. + for path in ("collector/sglang/collect_gemm.py", "collector/sglang/collect_moe.py"): + compat = _compat(path) + assert compat == "sglang==0.5.21", path + for version in ("0.5.14", "0.5.17", "0.5.20"): + assert not _check_compat(compat, version), (path, version) + assert not (REPO_ROOT / "collector/sglang/collect_gemm_0520.py").exists() def test_sglang_nvfp4_allowlist_admits_glm53flash_nvfp4_only(): diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py b/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py index 45db41e170..a310c4dca3 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py @@ -1,7 +1,8 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -"""GLM-5.3-Flash model-scoped collector runtime pins (stock vLLM 0.31.0, SGLang 0.5.20).""" +"""GLM-5.3-Flash model-scoped collector runtime pins (stock vLLM 0.31.0; the SGLang 0.5.20 pin +retired 2026-10-30, one release per collector file).""" import pytest from collector.framework_manifest import require_collector_runtime, validate_resolution @@ -10,7 +11,6 @@ GLM_MODEL_PATHS = ("zai-org/GLM-5.3-Flash", "nvidia/GLM-5.3-Flash-NVFP4") VLLM_INDEX = "vllm/vllm-openai:v0.31.0@sha256:c1c9f6fd5c109ba7f0546a59f5b2f15fb87f64c77782e90a27b648b42a8e67c3" -SGLANG_INDEX = "lmsysorg/sglang:v0.5.20@sha256:06e4f2ed21afde4ff513cda65070124e727ba23ccaeff7712b8c40e1097d611f" @pytest.mark.parametrize("model_path", GLM_MODEL_PATHS) @@ -27,19 +27,17 @@ def test_glm_vllm_pin_is_the_stock_0_31_0_release(model_path): @pytest.mark.parametrize("model_path", GLM_MODEL_PATHS) -def test_glm_sglang_pin_resolves_every_op_to_0_5_20(model_path): - runtime = require_collector_runtime( - "sglang", "0.5.20", requested_ops={"gemm", "moe", "kda", "mhc_module"}, model_path=model_path - ) - assert runtime.version == "0.5.20" - assert runtime.image() == SGLANG_INDEX - assert runtime.image("cu130").endswith(SGLANG_INDEX.split("@", 1)[1]) - assert runtime.source_commit == "94602c9c2b7cbdb8efd5c52802dac6a1c180089e" +def test_glm_has_no_sglang_model_pin(model_path): + # SGLang GLM-5.3-Flash resolves to the 0.5.21 default like every other model. + runtime = require_collector_runtime("sglang", "0.5.21", requested_ops={"gemm"}, model_path=model_path) + assert runtime.version == "0.5.21" + with pytest.raises(RuntimeError, match="requires exactly 0.5.21"): + require_collector_runtime("sglang", "0.5.20", requested_ops={"gemm"}, model_path=model_path) @pytest.mark.parametrize( "backend,wrong_version,expected", - [("vllm", "0.27.1", "0.31.0"), ("vllm", "0.30.0", "0.31.0"), ("sglang", "0.5.17", "0.5.20")], + [("vllm", "0.27.1", "0.31.0"), ("vllm", "0.30.0", "0.31.0")], ) def test_glm_pin_rejects_other_runtimes(backend, wrong_version, expected): with pytest.raises(RuntimeError, match=f"requires exactly {expected}"): @@ -49,12 +47,12 @@ def test_glm_pin_rejects_other_runtimes(backend, wrong_version, expected): @pytest.mark.parametrize( "backend,version,model_path", [ - ("vllm", "0.30.0", None), - ("vllm", "0.30.0", "zai-org/GLM-5.2-FP8"), - ("vllm", "0.27.1", "Qwen/Qwen3.8-2.4T-A95B"), + ("vllm", "0.31.0", None), # gemm family pin + ("vllm", "0.31.0", "zai-org/GLM-5.2-FP8"), + ("vllm", "0.31.0", "Qwen/Qwen3.8-2.4T-A95B"), # Qwen3.8 pins folded 2026-10-30 ("sglang", "0.5.21", None), ("sglang", "0.5.21", "zai-org/GLM-5.2-FP8"), - ("sglang", "0.5.17", "Qwen/Qwen3.8-2.4T-A95B"), + ("sglang", "0.5.21", "Qwen/Qwen3.8-2.4T-A95B"), ], ) def test_other_models_keep_their_runtime(backend, version, model_path): diff --git a/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py b/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py index eb0d9fe77e..0ea11358d2 100644 --- a/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py +++ b/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py @@ -103,16 +103,18 @@ def test_glm_row_conventions_match_the_shared_table(monkeypatch): @pytest.mark.parametrize( ("backend", "architecture", "version", "audited"), [ - ("vllm", DSV4, "0.24.0", True), - ("vllm", DSV4, "0.25.0", True), + # DeepSeek-V4 is not audited on 0.31.0 (shared architecture-free key, see the module) + ("vllm", DSV4, "0.30.0", False), ("vllm", DSV4, "0.31.0", False), ("vllm", GLM, "0.31.0", True), ("vllm", GLM, "0.30.0", False), ("vllm", GLM, "0.25.0", False), ("vllm", GLM, "0.27.1", False), - ("sglang", DSV4, "0.5.14", True), + ("sglang", DSV4, "0.5.21", True), + ("sglang", DSV4, "0.5.14", False), ("sglang", DSV4, "0.5.20", False), - ("sglang", GLM, "0.5.20", True), + ("sglang", GLM, "0.5.20", False), # 0.5.20 lane retired 2026-10-30 + ("sglang", GLM, "0.5.21", False), # not ported to 0.5.21 yet ("sglang", GLM, "0.5.14", False), ("sglang", "UnknownForCausalLM", "0.5.20", False), ], @@ -145,13 +147,10 @@ def test_audited_releases_never_overlap_across_architectures(monkeypatch, backen assert len(admitted) <= 1, (version, admitted) -def test_vllm_file_compat_admits_glm_runtime_and_keeps_dsv4_releases(monkeypatch): +def test_vllm_file_is_pinned_to_the_mhc_family_release(monkeypatch): module = _load(monkeypatch, "vllm") - from collector.version_resolver import _check_compat - - for version in ("0.24.0", "0.25.0", "0.31.0"): - assert _check_compat(module.__compat__, version), version - assert not _check_compat(module.__compat__, "0.32.0") + assert module.__compat__ == "vllm==0.31.0" + assert set(module._ARCHITECTURE_COMPAT) == {GLM} def test_vllm_glm_sites_import_the_0_31_0_module_location(): diff --git a/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py b/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py index 5160213b10..9e21624304 100644 --- a/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py +++ b/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py @@ -61,8 +61,9 @@ def test_glm5_flash_mla_keeps_the_production_topk_width(): def test_glm5_sparse_collector_is_pinned_to_sglang_0514_dsa_api(): source = SOURCE_PATH.read_text(encoding="utf-8") - # ported to the 0.5.21 pin on 2026-10-01 (same DSA module API, verified in collector/sglang/runtime_compat.py) - assert '__compat__ = "sglang>=0.5.14,<=0.5.21,!=0.5.15,!=0.5.16,!=0.5.17,!=0.5.18,!=0.5.19,!=0.5.20"' in source + # ported to the 0.5.21 pin on 2026-10-01 (same DSA module API, verified in collector/sglang/runtime_compat.py); + # one release per file since 2026-10-30 + assert '__compat__ = "sglang==0.5.21"' in source assert "SGLANG_DSA_FUSE_TOPK" in source assert "SGLANG_NSA_FUSE_TOPK" not in source diff --git a/python/aisimulate/tests/unit/collector/test_version_resolver.py b/python/aisimulate/tests/unit/collector/test_version_resolver.py index ee7cc3b073..7aecd5a68a 100644 --- a/python/aisimulate/tests/unit/collector/test_version_resolver.py +++ b/python/aisimulate/tests/unit/collector/test_version_resolver.py @@ -143,7 +143,11 @@ def test_invalid_compat_raises(self, compat): # --------------------------------------------------------------------------- @pytest.mark.unit class TestExactVersionSet: - """``_check_compat``'s grammar is AND-of-comparators only (see + """Grammar semantics only: collector modules no longer use this form (one + exact ``==`` pin per file, layer_permissions.md "Version contract"); the + tests below keep ``_check_compat``'s range/``!=`` behaviour documented. + + ``_check_compat``'s grammar is AND-of-comparators only (see ``_parse_compat_specifier``): no OR, so a __compat__ string can never directly express a discrete set like "0.5.14 or 0.5.17". When a module's verified-compatible versions are non-contiguous -- e.g. diff --git a/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py b/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py index 147e5a5f64..dc222dd46f 100644 --- a/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py +++ b/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py @@ -272,7 +272,7 @@ def test_glm5_next_decode_mirrors_serving_decode(): [ ("0.1.dev19262+gb6bbf29dd", False), # Kimi-K3 preview pin retired 2026-09-28 ("0.29.0", False), - ("0.30.0", True), + ("0.30.0", False), # kda family moved to 0.31.0 (2026-10-30) ("0.31.0", True), ("0.31.1", False), ], @@ -315,24 +315,25 @@ def _gate_namespace(): return namespace -def test_kda_audited_release_sets_do_not_overlap(): - from collector.version_resolver import _check_compat - +def test_kda_architecture_gates_follow_the_file_pin(): + # One release per file: every audited architecture runs on the file's release. compat = _gate_namespace()["_KDA_ARCHITECTURE_COMPAT"] - audited = {arch: spec.split("==", 1)[1] for arch, spec in compat.items()} - assert set(audited) == {"KimiK3ForConditionalGeneration", "Glm5NextForConditionalGeneration"} - for arch, version in audited.items(): - for other, spec in compat.items(): - assert _check_compat(spec, version) is (other == arch), (arch, other) + assert set(compat) == {"KimiK3ForConditionalGeneration", "Glm5NextForConditionalGeneration"} + file_pin = next( + node.value.value + for node in ast.parse(SOURCE_PATH.read_text(encoding="utf-8")).body + if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "__compat__" for t in node.targets) + ) + assert set(compat.values()) == {file_pin} == {"vllm==0.31.0"} @pytest.mark.parametrize( ("model_name", "version", "audited"), [ - ("moonshotai/Kimi-K3", "0.30.0", True), + ("moonshotai/Kimi-K3", "0.31.0", True), ("moonshotai/Kimi-K3", "0.1.dev19262+gb6bbf29dd", False), ("moonshotai/Kimi-K3", "0.27.0", False), - ("moonshotai/Kimi-K3", "0.31.0", False), + ("moonshotai/Kimi-K3", "0.30.0", False), ("zai-org/GLM-5.3-Flash", "0.31.0", True), ("nvidia/GLM-5.3-Flash-NVFP4", "0.31.0", True), ("zai-org/GLM-5.3-Flash", "0.30.0", False), From 3da11e09fa0cbbc90eed55abef207139f9177bb7 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 18:11:16 +0800 Subject: [PATCH 04/13] feat: drop the vLLM per-tensor fp8 MLA/DSA module cases Every bundled MLA/DSA FP8 checkpoint is block-scaled (fp8_block): the DeepSeek-V3/R1/V3.2/V4, Kimi-K2 and GLM-5.x FP8 configs all carry weight_block_size, and the NVIDIA ones are ModelOpt NVFP4. The only consumer of the per-tensor `fp8` rows in the vLLM MLA/DSA module tables was the prediction gate's forced per-tensor override (removed separately), so the two `gemm_type: fp8` combos leave mla_module.yaml, as the fictitious MoE fp8 cases did in AIC-1782 Task V2. collect_mla_module.py drops its per-tensor Fp8Config branch and the `--gemm-type fp8` choice. Shipped tables keep their rows. Collection harness shard plans that filter mla/dsa module shards by gemm type need their owner to drop the fp8 filter; this PR does not touch the harness. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .../collector/cases/base_ops/mla_module.yaml | 18 ++++-------------- .../collector/vllm/collect_mla_module.py | 16 +++------------- .../tests/unit/collector/test_model_cases.py | 8 -------- .../tests/unit/collector/test_vllm_mla_cli.py | 6 ++++-- .../test_vllm_mla_input_population.py | 12 +++++++----- 5 files changed, 18 insertions(+), 42 deletions(-) diff --git a/python/aisimulate/collector/cases/base_ops/mla_module.yaml b/python/aisimulate/collector/cases/base_ops/mla_module.yaml index 4777ac4821..a0540d2240 100644 --- a/python/aisimulate/collector/cases/base_ops/mla_module.yaml +++ b/python/aisimulate/collector/cases/base_ops/mla_module.yaml @@ -395,20 +395,10 @@ common_case_values: min_sm: 89 phases: [context, generation] attention_types: [mla, dsa] - # Native tensor-scaled FP8 weights with dynamic activations; distinct - # from block-scaled FP8 and from the fp8_static GEMM contract. - - compute_dtype: bfloat16 - kv_cache_dtype: bfloat16 - gemm_type: fp8 - min_sm: 89 - phases: [context, generation] - attention_types: [mla, dsa] - - compute_dtype: bfloat16 - kv_cache_dtype: fp8 - gemm_type: fp8 - min_sm: 90 - phases: [context, generation] - attention_types: [mla, dsa] + # No per-tensor fp8 GEMM combo (removed 2026-10-30): every bundled MLA/DSA + # FP8 checkpoint is block-scaled (fp8_block) and the only consumer was the + # prediction gate's forced per-tensor override. Existing tables keep their + # rows. Same reasoning as the AIC-1782 Task V2 MoE fp8 removal. - compute_dtype: bfloat16 kv_cache_dtype: fp8 gemm_type: fp8_block diff --git a/python/aisimulate/collector/vllm/collect_mla_module.py b/python/aisimulate/collector/vllm/collect_mla_module.py index 2991901bb7..07602dae10 100644 --- a/python/aisimulate/collector/vllm/collect_mla_module.py +++ b/python/aisimulate/collector/vllm/collect_mla_module.py @@ -596,19 +596,9 @@ def _create_gemm_quant_config(gemm_type: str): """ if gemm_type == "bfloat16": return None - if gemm_type in ("fp8", "fp8_block"): + if gemm_type == "fp8_block": from vllm.model_executor.layers.quantization.fp8 import Fp8Config - # Match the existing GEMM collector's fp8 contract: tensor-scaled - # weights with dynamic activations, not fp8_static. Native Fp8LinearMethod - # selects the activation granularity/kernel (fp8.py:298-322 at - # vLLM v0.25.1, commit 752a3a504485790a). - if gemm_type == "fp8": - return Fp8Config( - is_checkpoint_fp8_serialized=True, - activation_scheme="dynamic", - weight_block_size=None, - ) # vLLM requires is_checkpoint_fp8_serialized=True for block-scaled # FP8 (fp8.py raises ValueError otherwise). This routes through # Fp8LinearMethod (block_quant=True) → W8A8BlockFp8LinearOp → @@ -830,7 +820,7 @@ def _create_attention_module( _create_kv_cache_and_metadata fails closed if the decision does not match the case label. gemm_type: Precision for linear-layer GEMMs — "bfloat16", - "fp8", "fp8_block", or "nvfp4". + "fp8_block", or "nvfp4". """ from vllm.model_executor.models.deepseek_v2 import DeepseekV2MLAAttention @@ -1983,7 +1973,7 @@ def main(): parser.add_argument( "--gemm-type", type=str, - choices=["bfloat16", "fp8", "fp8_block", "nvfp4"], + choices=["bfloat16", "fp8_block", "nvfp4"], default=None, help="GEMM quantisation type for linear layers (default: run all supported by GPU)", ) diff --git a/python/aisimulate/tests/unit/collector/test_model_cases.py b/python/aisimulate/tests/unit/collector/test_model_cases.py index 7fbf01c4fa..f67cbeed43 100644 --- a/python/aisimulate/tests/unit/collector/test_model_cases.py +++ b/python/aisimulate/tests/unit/collector/test_model_cases.py @@ -1329,8 +1329,6 @@ def test_mla_module_metadata_and_micro_sweeps_are_yaml_backed(): ("bfloat16", "fp8", "bfloat16"), ("fp8", "fp8", "bfloat16"), ("bfloat16", "bfloat16", "fp8_block"), - ("bfloat16", "bfloat16", "fp8"), - ("bfloat16", "fp8", "fp8"), ("bfloat16", "fp8", "fp8_block"), ("fp8", "fp8", "fp8_block"), ("bfloat16", "bfloat16", "nvfp4"), @@ -1360,8 +1358,6 @@ def test_mla_module_metadata_and_micro_sweeps_are_yaml_backed(): ("bfloat16", "bfloat16", "bfloat16"), ("bfloat16", "fp8", "bfloat16"), ("bfloat16", "bfloat16", "fp8_block"), - ("bfloat16", "bfloat16", "fp8"), - ("bfloat16", "fp8", "fp8"), ("bfloat16", "fp8", "fp8_block"), # MSA-scoped combos (attention_types [msa]) at SM90: both KV dtypes # for the bf16 and fp8_block gemm tiers (fp8-KV has no SM floor for @@ -1383,8 +1379,6 @@ def test_mla_module_metadata_and_micro_sweeps_are_yaml_backed(): ("bfloat16", "bfloat16", "bfloat16"), ("bfloat16", "fp8", "bfloat16"), ("bfloat16", "bfloat16", "fp8_block"), - ("bfloat16", "bfloat16", "fp8"), - ("bfloat16", "fp8", "fp8"), ("bfloat16", "fp8", "fp8_block"), ("bfloat16", "bfloat16", "nvfp4"), ("bfloat16", "fp8", "nvfp4"), @@ -1400,8 +1394,6 @@ def test_mla_module_metadata_and_micro_sweeps_are_yaml_backed(): ("bfloat16", "fp8", "bfloat16"), ("fp8", "fp8", "bfloat16"), ("bfloat16", "bfloat16", "fp8_block"), - ("bfloat16", "bfloat16", "fp8"), - ("bfloat16", "fp8", "fp8"), ("bfloat16", "fp8", "fp8_block"), ("fp8", "fp8", "fp8_block"), ("bfloat16", "bfloat16", "nvfp4"), diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py index b36a45384d..925f24f041 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_cli.py @@ -112,10 +112,12 @@ def test_explicit_checkpoint_alias_is_preserved(cli): def test_dsa_context_cli_passes_declared_cached_prefix_to_native_worker(cli): - calls = cli("--mode", "context", "--model", "zai-org/GLM-5.2", "--gemm-type", "fp8", "--kv-cache-dtype", "fp8") + calls = cli( + "--mode", "context", "--model", "zai-org/GLM-5.2", "--gemm-type", "fp8_block", "--kv-cache-dtype", "fp8" + ) assert [(row["seq_len"], row["prefix_len"]) for row in calls] == [(128, 0), (128, 131072)] assert all(row["batch_size"] == 1 and row["num_heads"] == 8 for row in calls) - assert all(row["gemm_type"] == "fp8" and row["perf_filename"] == PerfFile.DSA_CONTEXT_MODULE for row in calls) + assert all(row["gemm_type"] == "fp8_block" and row["perf_filename"] == PerfFile.DSA_CONTEXT_MODULE for row in calls) @pytest.mark.parametrize( diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py index 378379dbb3..636774d5dd 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_input_population.py @@ -232,19 +232,21 @@ def test_indexer_history_rejects_an_incompatible_packed_cache(monkeypatch, torch ) -@pytest.mark.parametrize("mode,block", [("fp8", None), ("fp8_block", [128, 128])]) -def test_fp8_modes_construct_distinct_native_configs(monkeypatch, mode, block): +def test_fp8_block_constructs_the_native_block_config_and_per_tensor_fp8_is_gone(monkeypatch): monkeypatch.setitem( sys.modules, "vllm.model_executor.layers.quantization.fp8", SimpleNamespace(Fp8Config=lambda **kwargs: kwargs), ) - config = _load_function("_create_gemm_quant_config")(mode) - assert config == { + create = _load_function("_create_gemm_quant_config") + assert create("fp8_block") == { "is_checkpoint_fp8_serialized": True, "activation_scheme": "dynamic", - "weight_block_size": block, + "weight_block_size": [128, 128], } + # per-tensor fp8 MLA/DSA module combos were removed 2026-10-30 + with pytest.raises(ValueError, match="Unknown gemm_type"): + create("fp8") def test_nvfp4_config_preserves_native_modelopt_without_projection_exclusions(monkeypatch): From 331a2661cf3d3e3459fd09cc353b1abd322bb2b9 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 18:21:07 +0800 Subject: [PATCH 05/13] chore: keep only the cited op-probe evidence; leave the stale reuse report to a data PR The evidence bundle keeps the 20 findings collector and generator code cite and, in each results file, _meta plus the models the citing comments name (5,275 -> ~1,080 lines); the complete archive lands with the op-probe harness. benchmarks/reports/perf-data/perf-data-reuse-analysis.md goes back to main's copy: main never regenerated it after the #517 data, so a rebuild here carried ~2,200 lines of #517 churn for a 2-line change of this PR. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .../collector/op-probe-2026-10/README.md | 5 + .../collector/op-probe-2026-10/findings.yaml | 2195 +--------------- .../retests/sm100/sglang-0.5.21.moe_auto.yaml | 528 +--- .../results/sm100/sglang-0.5.21.yaml | 773 +----- .../results/sm100/trtllm-1.3.0rc29.yaml | 721 +----- .../perf-data/perf-data-reuse-analysis.md | 2237 ++++++++--------- 6 files changed, 1005 insertions(+), 5454 deletions(-) diff --git a/benchmarks/evidence/collector/op-probe-2026-10/README.md b/benchmarks/evidence/collector/op-probe-2026-10/README.md index 1dd99f0616..bcde8e2252 100644 --- a/benchmarks/evidence/collector/op-probe-2026-10/README.md +++ b/benchmarks/evidence/collector/op-probe-2026-10/README.md @@ -15,6 +15,11 @@ Citations in the code name either a finding key or a results file below. Grep a finding key, such as `sm100_probe_failures_recheck_2026_10_04`, to reach its statement. +Every file here is an excerpt: `findings.yaml` keeps the 20 findings the code +cites, and each results file keeps `_meta` plus the models the citing comments +name. `_meta.summary` still counts the full matrix. The complete archive lands +with the op-probe harness. + ## Files - `findings.yaml`: curated conclusions, one entry per key. Each statement gives diff --git a/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml b/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml index e695caea22..cacc8df43f 100644 --- a/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml +++ b/benchmarks/evidence/collector/op-probe-2026-10/findings.yaml @@ -1,369 +1,10 @@ +# Excerpt (2026-10-30): only the findings that collector and generator code in +# this repository cite. The complete archive lands with the op-probe harness. # OUTPUT: curated conclusions from probe campaigns (the facts archive's # human layer). Each finding states its own evidence paths, versions and # dates inline; applies_to lists where it was observed. Rots with # framework releases — re-verify on version bumps. Inputs live in targets.yaml. findings: - dummy_notes: - statement: layer-kind period is FRAMEWORK-HARDCODED (sglang llama4.py:217 use_rope=(layer_id+1)%4!=0) with no - config field -> _ARCH_IMPLICIT_PERIODS; aic-core config lacks vision_config -> merged from unsloth mirror - (vllm builds the vision tower and profiles it even for text-only probes). The manual mirror-merge step BIT - on B300 (2026-09-20) — the gated-config fallback used the core copy WITHOUT vision_config, vllm assumed - default vision sizes (1025 pos) vs the mirror preprocessor's 336px (577) and both Llama-4 probes 'failed'; - codified since as fetch_assets' mirror-config fallback (B300 patch, pending merge) — after rebuilding the - dummies both pass. - applies_to: - - meta-llama/Llama-4-Scout-17B-16E-Instruct - k3_deepep: - statement: 'NO working deepep mode on sglang 0.5.17 + bundled DeepEP @ SM90 tp8/ep8 (A/B 2026-08-24, both deterministic): - mode=auto -> AssertionError forward_deepgemm_contiguous is deprecated at the first MoE forward (normal dispatch - routed into the removed contiguous-deepgemm path; all 8 ranks); mode=low_latency -> DeepEP device assert internode_ll.cu:358 - num_topk <= kNumMaxTopK (K3 num_experts_per_token=16 exceeds the LL kernel cap). Additionally K3''s own SP-collective/MegaMoE - path is SM103-gated (startup log: requires SM103, TP4/TP8, MegaMoE/DeepEP, CustomAllReduceV2 multicast; falls - back to NCCL on SM90) — the independent kernel route is unreachable on H20 regardless. Non-deepep tp8/ep8 - (a2a=none) works and matches tp1 kernel families. Evidence facts/tepdep/k3_deepep_{auto_v2,ll}.rank*.json - + run logs.' - applies_to: - - moonshotai/Kimi-K3 - sglang_0516: - statement: 'CORRECTED 2026-08-23 (retracts the earlier framework_bug verdict): the ''Hidden size mismatch'' - chain was a DUMMY false positive for converted-FP8 checkpoints. sglang picks the DSV4 expert layout by probing - the safetensors HEADER dtype of one routed-expert tensor (configs/deepseek_v4.py try_detect_fp4_experts); - weightless dummies probed None -> env default fp4=True -> fp4-packed weights fed to the triton fp8 kernel. - Real header dtypes (fetched from HF): native deepseek-ai = I8 (fp4-packed), sgl-project FP8 = F8_E4M3, nvidia - NVFP4 = U8. With the dtype_probe.safetensors signal: converted-FP8 checkpoints RUN OUT OF THE BOX (ok); native/NVFP4 - (true fp4 experts) fail under runner=auto on SM90 and RUN with moe_runner_backend=marlin -> needs_flags (facts/bisect_dsv4_{fp8experts,native_marlin}.json). - The earlier five-runner-all-fail experiments ran under the wrong fp4 layout and are retracted. The ue8m0/deep_gemm - SM100-only branch remains a separate real limit for the deep_gemm runner choice.' - applies_to: - - family:dsv4 - sglang_0516_flaky: - statement: trace NONDETERMINISTICALLY fails (~1/3 of runs) in flashinfer cutlass-DSL fused_add_rmsnorm via tvm_ffi - 'Mismatched Tensor' — the same command text both passes and fails on the same GPU (12 controlled reruns 2026-08-21); - every generator flag excluded by bisection; suspected timing-dependent tensor layout under enable-mixed-chunk. - Also hits MiniMax-M2.5. Probe retries once and records warmup_retry; persistent failure is a record, not a - block. - applies_to: - - MiniMaxAI/MiniMax-M2 - sglang_marlin_ab: - statement: 'runner=marlin A/B''d 2026-08-24: SAME capability ValueError — the gate (loader.py:282) checks the - quant config''s declared min_capability=100 before model construction; runner choice never enters. Declaration-locked, - no flag escape. Evidence facts/bisect_m3mxfp8_marlin.json.' - applies_to: - - MiniMaxAI/MiniMax-M3-MXFP8 - sglang_silent_downgrade: - statement: 'SGLANG BUG (not ours), mechanism pinned: the artifact ships quant_method=''modelopt'' + quant_algo=''MIXED_PRECISION''; - sglang 0.5.16''s mixed detector matches only the literal ''modelopt_mixed'' (modelopt_quant.py:622) and the - generic modelopt fallback greps quant_algo for FP8/NVFP4/FP4 substrings (base_config.py:189-196) — MIXED_PRECISION - matches neither -> silently Unquantized, zero warnings. Our dummy carries the full real metadata and the SAME - metadata binds ModelOptNvFp4FusedMoE correctly on vllm. Worth reporting upstream; recheck on 0.5.17.' - applies_to: - - nvidia/MiniMax-M3-NVFP4 - sglang_sm90: - statement: 'NO working path: SM90 gates NVFP4 MoE to marlin only (flashinfer_mxfp4/flashinfer_cutlass/triton - all rejected ''platform does not support NVFP4 with selected backend''), and marlin fused_moe asserts ''Only - gated SiLU is supported'' — Gemma uses gelu_pytorch_tanh. framework gap: NVFP4xgelu-MoE unservable on sglang/SM90. - Dense 31B-IT-NVFP4 passes (no MoE). Evidence facts/bisect_gemma4nvfp4_*.json, 2026-08-24.' - applies_to: - - nvidia/Gemma-4-26B-A4B-NVFP4 - sglang_workaround: - statement: 'flake root-caused to flashinfer cutlass-DSL fused_add_rmsnorm tvm_ffi nondeterminism: SGLANG_IS_FLASHINFER_AVAILABLE=false - passes 4/4 (facts/bisect_m2_noflashinfer_*.json, 2026-08-22). Deterministic workaround exists; perf cost unmeasured.' - applies_to: - - MiniMaxAI/MiniMax-M2 - - MiniMaxAI/MiniMax-M2.5 - sm90_triple: - statement: 'Same checkpoint, three fates on SM90. vllm 0.24 RUNS but NOT at mxfp8 precision: binds ModelOptMxFp8FusedMoE/LinearMethod - yet executes marlin_moe_wna16::Marlin + marlin::Marlin — weight-only dequant (W8A16 semantics); perf collected - here is Marlin-dequant data, not mxfp8 tensor-core data (same class as NVFP4-on-SM90). sglang 0.5.16 DEFINITIVELY - unservable: explicit quant-method capability floor ''mxfp8 minimum capability 100, current 90'' — not runner-selectable, - no flag bypasses it. trtllm rc23 DEFINITIVELY unservable but for a NON-quant reason: all three M3 artifacts - die at the sparse-attention wiring wall before quant matters. Evidence: records + raws, 2026-08-24.' - applies_to: - - MiniMaxAI/MiniMax-M3-MXFP8 - support: - statement: 'Generator/SDK supports K3 since upstream rebase 2026-08-23 (KimiK3Config, KIMIK3 family); the golden - command renders. Failure now correctly sits at the FRAMEWORK layer: unregistered at the pins (sglang 0.5.16 - / vllm 0.24 / trtllm rc23); registered in latest releases (0.5.17 / 0.27.1 / rc24, probed).' - applies_to: - - moonshotai/Kimi-K3 - support_2: - statement: 'same as moonshotai/Kimi-K3: architecture unregistered at all three pinned framework versions (probe-verified - 2026-08-20).' - applies_to: - - nvidia/Kimi-K3-NVFP4 - topology_invariance: - statement: 'ep/tp sharding on the a2a=none (single-node default) path does NOT change kernel families — measured - across GLM-5.2-FP8 (tp2/tp8 x ep1/ep2/ep8: triton fused_moe throughout), DSV4-NVFP4 (tp2 x ep1/ep2: marlin), - Kimi-K2.5-FP8 (tp8 x ep1/ep8: fused_marlin_moe). Evidence facts/tepdep/*.rank0.json, 2026-08-21. Therefore - tp1 probes represent kernel identity for the default path; multi-rank probing is scoped to routes that change - kernels.' - applies_to: - - a2a=none sharding - topology_sm103: - statement: 'the owner-reported TEP8 kernel fork is SM-gated, from sglang''s own dispatch log: ''K3 SP collective - requires SM103, TP4/TP8, MegaMoE/DeepEP, and CustomAllReduceV2 with multicast; using NCCL.'' — the special - path never activates on SM90; verify on SM103 (Blackwell Ultra) during the Blackwell replay.' - applies_to: - - moonshotai/Kimi-K3 - topology_sm90: - statement: 'sglang 0.5.17 on SM90 (probed): tp1 and TEP8 and DEP8(a2a=none) all run Mxfp4MoEMethod -> bf16-dequant - triton fused_moe_kernel, IDENTICAL kernel sets (facts/tepdep/k3_{tep8,dep8}.rank0.json). DEP8+deepep (the - real DEP deployment route) CRASHES at warmup: ''AssertionError: forward_deepgemm_contiguous is deprecated'' - — the DeepEPMoE route for this quant recipe is broken at 0.5.17/SM90.' - applies_to: - - moonshotai/Kimi-K3 - topology_verdict: - statement: 'Kimi-K2.5 (compressed-tensors W4-packed -> marlin): TEP8 == DEP8 kernel family (sglang::fused_marlin_moe - + device::marlin_moe::Marlin, same align/reduce/activation set; only shape-tuning template args differ). facts/tepdep/k25_fp8_{tep8,dep8}.rank0.json, - 2026-08-21.' - applies_to: - - tp8/ep1 - trtllm_fp8kv: - statement: 'trtllm rc23/SM90 UNIFIED root cause: fp8 KV cache in the MLA/DSA path crashes natively in flash_mla - (frame#62 abort). Door 1 = engine kv_cache_config.dtype:fp8 (generator renders for fp8 profiles) -> config_gap, - kv-auto A/B passes (GLM-5/5.1/5.2-FP8, DSV3.2, MiniMax-M2/M2.5 whose ''Linear assert'' was the same trigger). - Door 2 = checkpoint hf_quant kv_cache_quant_algo:FP8 (all NVFP4 artifacts) -> framework_bug, kv-auto STILL - crashes (GLM-5/5.1-NVFP4, DSV3.2-NVFP4). Evidence facts/trt_ab_kvauto_*.json, 2026-08-22.' - applies_to: - - MiniMaxAI/MiniMax-M2 - - zai-org/GLM-5-FP8 - - zai-org/GLM-5.1-FP8 - - deepseek-ai/DeepSeek-V3.2 - - MiniMaxAI/MiniMax-M2.5 - - nvidia/GLM-5-NVFP4 - - nvidia/GLM-5.1-NVFP4 - - nvidia/DeepSeek-V3.2-NVFP4 - - family:glm - trtllm_layers: - statement: 'rc23 fails in QUANT PARSING before the arch check (quant_config_utils.py:43 NoneType on the 4-bit-float - compressed-tensors config) — two walls stacked: parser first, arch-unregistered behind it. rc24 passes parsing - AND registers the arch, then stops at the missing ''fla'' package in the image.' - applies_to: - - moonshotai/Kimi-K3 - trtllm_linear_assert: - statement: 'trtllm rc23: fp8 Linear.apply assertion (linear.py:1142) under the generator-rendered engine args; - probe-default args loaded fine — surfaced by the fidelity upgrade 2026-08-21.' - applies_to: - - MiniMaxAI/MiniMax-M2 - - MiniMaxAI/MiniMax-M2.5 - trtllm_quant_algo: - statement: hf_quant mixes NVFP4 experts (552 layers) with FP8_PB_WO attention projections (2790 layers, fp8 - per-block WEIGHT-ONLY). FP8_PB_WO is absent from the QuantAlgo enum in rc23 AND rc24; nearest member FP8_BLOCK_SCALES - is W8A8 (fp8 activations), NOT weight-only — no semantic equivalent exists, so this needs a new enum + fp8 - weight-only dequant path in trtllm, not a rename. W4A16 kernels exist but only apply to the expert half. Enum - grep-verified 2026-08-24. - applies_to: - - nvidia/Kimi-K3-NVFP4 - trtllm_quant_parser: - statement: rc23 quant_config_utils.py:43 crashes (NoneType) parsing the native compressed-tensors W4 quantization_config - — framework_bug; K2.5-NVFP4 sibling fails separately on NVFP4 MoE tactics (platform_floor SM100). - applies_to: - - moonshotai/Kimi-K2.5 - trtllm_rc23: - statement: 'trtllm rc23: NOT capacity (2026-08-21 explanation RETRACTED — the 64B/120GiB arithmetic used gated-3-mat - at hidden width, ignoring moe_latent_size=2048+relu2; true experts/layer ~10.7B). Measured 2026-08-23: peak - 33GB/96GB, weights fit; crash is a native ILLEGAL MEMORY ACCESS right after model init (cudaStreamDestroy - IMA -> abort in KVCacheManager teardown, exit 139, no raw written). Isolation: Ultra-BF16/FP8 pass trtllm, - Nano/Super-NVFP4 pass trtllm — Ultra-shape x NVFP4-path specific framework bug. Evidence facts/ultra_nvfp4_trt_ima.log - + mem monitor.' - applies_to: - - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - trtllm_rc23_2: - statement: 'DSV4 rejected at load: ''Missing COMPRESS pool metadata for compress ratio 4'' — CSA compression - unsupported on this path.' - applies_to: - - family:dsv4 - trtllm_rc24: - statement: rc24 registers KimiK3ForConditionalGeneration but the nvcr image lacks the 'fla' (flash-linear-attention) - package its model file imports — K3 is unserveable on the shipped image (probe-verified 2026-08-21). - applies_to: - - moonshotai/Kimi-K3 - trtllm_sparse_config: - statement: 'MiniMax-M3 on trtllm REQUIRES llmapi sparse_attention_config (algorithm: minimax_m3) to get MiniMaxM3KVCacheManagerV2 - — the generator does not render it (needs_flags). With it set, rc23/SM90 still fails deeper (MiniMaxM3SparseRuntimeBackend.forward - missing cache args — framework gap, recheck on rc24/Blackwell). Evidence facts/trt_ab_m3_sparse.json.' - applies_to: - - family:m3 - trtllm_tokens_per_block: - statement: DeepseekV4CacheManager requires tokens_per_block in [128,256]; the generator renders 64 (FPM example - value) — ILLEGAL FOR THIS ARCH ON ANY PLATFORM (needs_flags for the generator). With 128 the load proceeds - to the explicit pre-blackwell rejection -> final cause on SM90 is platform_floor. Evidence facts/trt_ab_dsv4_tpb128.json, - 2026-08-22. - applies_to: - - family:dsv4 - vllm_024: - statement: 'NotImplementedError at load: checkpoint hf_quant excludes lm_head but NOT model.embed_tokens while - weights are tied; vllm''s ModelOpt NVFP4 config has no embedding quant method. Real-artifact metadata x framework - combination — real deployment hits the same. Affects both 26B and 31B-IT.' - applies_to: - - nvidia/Gemma-4-26B-A4B-NVFP4 - vllm_024_2: - statement: same tied-embedding NotImplementedError as 26B (see its entry). - applies_to: - - nvidia/Gemma-4-31B-IT-NVFP4 - vllm_024_image_tilelang_stub: - statement: 'PRISTINE vllm/vllm-openai:v0.24.0 CANNOT load DSV4 on vllm: tilelang (hard dep of DSV4 mhc) ships - lib/libcudart_stub.so with undefined symbol cudaDeviceReset; vllm''s load path resolves the stub via ctypes - and crashes (A/B on pristine image: facts/ab_dsv4_vllm_pristine_image.json, 2026-08-24 — minimal ''import - tilelang; import flashinfer.comm'' does NOT trigger it, the engine load path does). Real 0.24.0-image deployments - of DSV4 hit the same. Upstream-fixed: v0.27.1 ships tilelang without stub files. The -fix image only symlinks - the stub to the real libcudart (no dispatch change); drop it when the vllm pin moves past 0.24.' - applies_to: - - vllm/vllm-openai:v0.24.0 - vllm_0271: - statement: 'vllm v0.27.1 probed OK: KimiK3DeltaAttention (+MTP model registered), Mxfp4MoEMethod — same quant - identity as sglang 0.5.17.' - applies_to: - - moonshotai/Kimi-K3 - vllm_0290_dsa_attention_swap: - statement: 'vllm 0.29.0/SM90: the sparse-MLA/DSA attention path changed implementation — probes now record FlashAttnMLASparseImpl - and the 0.24-era dsa_nsa/triton kernel-family mix no longer appears for DeepSeek-V3.2 and the GLM-5.x family - (7 models, all still pass). Consequences: (1) attention perf data for these models must be recollected on - 0.29; (2) kernel_taxonomy.yaml needs rules for the new kernel names before attention identity is machine-adjudicable - again (backlog). Also observed: Kimi-K2.5 W4A16 MoE binding class renamed (CompressedTensorsWNA16MarlinMoEMethod - -> CompressedTensorsWNA16MoEMethod) while the executed kernel is unchanged (marlin_moe_wna16::Marlin) — class - names alone are not identity. Evidence: records.jsonl 0.24-vs-0.29 diff, 2026-09-19.' - applies_to: - - deepseek-ai/DeepSeek-V3.2 - - zai-org/GLM-5 - - zai-org/GLM-5-FP8 - - zai-org/GLM-5.1 - - zai-org/GLM-5.1-FP8 - - zai-org/GLM-5.2 - - zai-org/GLM-5.2-FP8 - - moonshotai/Kimi-K2.5 - vllm_0290_dsa_blackwell_paths_unvalidated: - statement: 'HARDWARE-VALIDATED on B300/SM103 2026-09-20 (upgrades the source-derived pre-read; evidence - results/pathdiff/sm103/vllm-0.29.0/*.json in the B300 workspace, pending its patch landing): backend selection - matched the pre-read — DSA ctx bf16 -> FLASHMLA_SPARSE (2.42ms b1 s512), fp8 -> FLASHINFER_MLA_SPARSE (1.58ms); - MSA -> MiniMaxM3SparseMSAImpl; dense attn -> flashinfer trtllm prefill/decode. The 029 lane construction - carried over with ZERO arch edits — the one bug found was arch-NEUTRAL: the lane handed the indexer builder - block_table_width=get_block_table_width(max_model_len)=64 while its own block_table_tensor had ceil(seq/block)=32 - columns; serving allocates the table AT the width (v1/worker/block_table.py) so builders assume equality, and - only the family-100 decode path reads the expanded buffer (indexer.py:609,745) — latent on SM90. Fixed by - zero-padding the table to the width; SM90 re-smoke green (gen bf16 0.190ms FLASH_ATTN_MLA_SPARSE, fp8 0.187ms - FLASHMLA_SPARSE, ctx 1.97ms) and all 3 sm90 dsa verdicts re-emitted aligned. CORRECTION to the pre-read: - "B200 = B300 on the vllm chain" holds only at DISPATCH level (family-100, no 103 special-case) — at kernel - identity level B300 prefill runs fmhaSm103a* cubins while decode runs fmhaSm100f* (B300 probe evidence), so - SM100 and SM103 need separate perf rows. Prior pre-read kept for the record: FLASH_ATTN_MLA_SPARSE is - major==9-only (flashattn_mla_sparse.py:71-72); indexer family-100 uses varlen_paged_mqa_logits - (indexer.py:475-503); SM120: TRITON_MLA + FLASHINFER_MLA_SPARSE_SM120 (cuda.py:130-134). DSA attention perf - rows remain strictly per-SM.' - applies_to: - - family:sparse_attention - - platform:sm100 - - platform:sm103 - - platform:sm120 - vllm_0290_dsa_lane_indexer_drift: - statement: 'RESOLVED 2026-09-19 (same day): the 029 lane now constructs the serving-same DeepseekV32Attention - (vllm.models.deepseek_v32.attention:118 — 0.29 routes DeepseekV32ForCausalLM to the DSV32-specific fused implementation, - registry.py:94) instead of the generic DeepseekV2MLAAttention the 0.24 collector used. Adaptations, each serving-cited: - forward arity (positions, hidden_states — no residual arg), forward_context.slot_mapping dict populated per - layer name (attention.py:303-306), perf-log layer accessor (DSV32Attention IS the MLAAttention layer, no .mla_attn - nesting). path_diff verdicts ALL ALIGNED (bf16 s512, bf16 s4096 sparse branch, fp8-kv) with zero kernel drift - — the fused prelude (fused_norm_rope) now matches serving, and the short-seq dense/sparse branch puzzle dissolved - with the module swap (same class, same branch logic). Perf delta of construction parity (H20, same GPU, back-to-back): - context b1 s512 1.76→1.97 ms (+12% — the fused serving prelude is SLOWER at tiny seq here), context b1 s4096 - 26.25→26.88 ms (+2.4%), generation b4 s2048 0.209→0.189 ms (-9.6%). Non-trivial deltas = the old construction - was mispricing the module; serving-parity rows are the only trustworthy ones. Legacy construction remains - behind AIC_DSA_LEGACY_MODULE=1 for A/B only (loud banner, never for data).' - applies_to: - - family:sparse_attention - - collector.vllm.collect_mla_module_029 - vllm_0290_dsv4_cutedsl: - statement: 'CORRECTED 2026-09-19 (retracts the same-day ''framework regression, family dead'' verdict): DSV4 - RUNS on vllm 0.29.0/SM90 — the crash is a CONFIG-TRIGGERED alignment bug, single-variable isolated. Trigger: - the generator renders --max-num-batched-tokens 6012; the runner slices per-group slot mappings as rows of a - 2D buffer with row width = max_num_batched_tokens, so a non-multiple-of-8 budget puts every row after the - first at a 64B-misaligned int64 offset (6012*8=48096 bytes, not /64), and DSV4 is multi-cache-group (MLA main - + compressor state + indexer) so its state-cache slot_mapping lands on a misaligned row — cutedsl''s tvm_ffi - wrapper asserts 64B alignment on argument #3 (slot_mapping, NOT state_cache as first written). A/B on the - pristine 0.29 image, same dummy: minimal args PASS (full request, whole DSV4 path executes: cutedsl compress, - fused indexer_q, dsv4_topk, FlashMLA decode metadata, deep_gemm fp8 — facts/ab_dsv4_minargs_trace_0290.json, - probe_isl 4096); minimal + 6012 reproduces the Misaligned error verbatim (ab_dsv4_tok6012_0290.json); minimal - + 6016 (=64*94) PASSES (ab_dsv4_tok6016_0290.json). Routing is SM-agnostic (compressor.py:402 is_cuda & - head=512 -> cutedsl on every SM; triton only for AMD/XPU/head-128), so B200 would crash identically under - 6012 — not a Hopper-specific kernel bug. Dual attribution: vllm robustness bug (pad slot-mapping rows or - relax the assert; worth reporting upstream) + generator trigger (odd token budget). REPRODUCED ON B300/SM103 2026-09-20 - (same Misaligned crash on all 8 DSV4 checkpoints, same single-variable flip: 6012->6016 makes - DSV4-Pro-0813 serve with native fp4 kernels). Birthplace located: the vllm rule plugin''s agg formula - max_batch_size + isl + 1500 (generator/rule_plugin/vllm.rule) — and the rule runs AFTER --generator-set, - silently overriding a user-supplied params.agg.max_num_tokens (separate defect, still open). FIX SHIPPED on - branch fix/generator-round-vllm-batched-tokens: rule budgets rounded up to 64 at the source + the render-layer - mapping rounds as last-line guard; 6 unit tests. Consequence: the - sparse_attention family split is NO LONGER FORCED by dsv4 (see vllm_0290_sparse_family_split_needed).' - applies_to: - - deepseek-ai/DeepSeek-V4-Flash - - deepseek-ai/DeepSeek-V4-Pro - - deepseek-ai/DeepSeek-V4-Flash-0731 - - deepseek-ai/DeepSeek-V4-Pro-0813 - - nvidia/DeepSeek-V4-Flash-NVFP4 - - nvidia/DeepSeek-V4-Pro-NVFP4 - - sgl-project/DeepSeek-V4-Flash-FP8 - - sgl-project/DeepSeek-V4-Pro-FP8 - vllm_0290_nemoh_profile_oom: - statement: 'vllm 0.29.0/SM90: Nemotron-H mamba profile run OOMs under the generator-rendered default (_state_passing_fwd - allocates 8GiB with ~4GiB free on 143GB). Root knob is max_num_seqs (64 alone passes; batched-tokens reduction - alone does not) but TWO generator walls block the direct fix: the rule engine floors max_batch_size at 512 - (pipeline.py ''512 if x<512 else x*2'') and unknown params.agg keys are SILENTLY IGNORED instead of rejected. - Expressible rescue: gpu-memory-utilization 0.6 via extra_cli_args (now the checkpoint''s cli_extra_args). - Evidence facts/ab_nemoh_{seq,tok,small,mem}_0290.json, 2026-09-19.' - applies_to: - - nvidia/Nemotron-H-56B-Base-8K - - generator:rule-engine - vllm_0290_op_sweep_triage: - statement: 'vllm 0.29.0 op-collector sweep, first pass (H20, quick smokes in the official container, 2026-09-19). - WORK AS-IS on 0.29 (perf rows produced, no code change): gemm (incl. fp8 CutlassFP8ScaledMM kernel_source), - moe (108 cases), encoder_attention (long sweep, no errors), gdn (4k rows), mhc_module. WORK VIA THE 029 LANE''s - generic registry construction: dsa (aligned, done) AND plain mla — DeepSeek-V3 runs unmodified through collect_mla_module_029 - (registry resolves DeepseekV3ForCausalLM -> generic deepseek_v2 model -> serving-same DeepseekV2MLAAttention; - FLASH_ATTN_MLA, 1.81ms ctx b1 s512), i.e. the config-in construction generalizes across architectures with - zero per-model code. FORKED AND CLOSED (second pass, same day): collect_attn_029.py (0.29 replaced the - set/get kv-cache-layout API with get_supported_kv_cache_layouts + resolve_kv_cache_layout and merged the - per-layer cache view to [B,H,N,2*D], flash_attn.py:1013-1014; SM90 smokes green on vllm_flash_attn_fa3; - path_diff aligned ctx+gen vs Meta-Llama-3.1-8B serving) and collect_msa_029.py (0.29 removed - AttentionBackend.get_kv_cache_shape — both caches now allocated via compute_layer_kv_cache_shape_bytes - and bound via each layer''s bind_kv_cache, v1/kv_cache_interface.py:253-269 + v1/worker/utils.py:622; - SM90 smokes green bf16 ctx/gen + fp8 ctx on MiniMaxM3SparseTritonImpl; path_diff aligned ctx+gen vs - MiniMax-M3 serving). SMOKED VIA components/op_smoke.py (generic registry-driven single-case runner — - one standard, no per-op drivers): compute_scale (2/2 cases, dynamic-minus-static fp8 quant rows) and - mla_bmm_gen_pre/post (2/2 each, vllm_torch_bmm) run AS-IS on 0.29. EXPECTED BLOCKED: dsv4_* (0.29 cutedsl - framework regression). OUT OF SCOPE: kda (preview-image runtime lane), xpu variants. Evidence: workspace - facts/opsweep_0290/*.txt.' - applies_to: - - framework:vllm-0.29.0 - - ops:gemm,moe,encoder_attention,gdn,mhc,mla,dsa,attention,msa,compute_scale,mla_bmm - - ops-blocked:dsv4 - vllm_0290_sparse_family_split_needed: - statement: 'UPDATED 2026-09-19 (the forcing condition is GONE): the split was motivated by ''DSA works on 0.29, - DSV4 dead on 0.29'' — but the DSV4 death was retracted (config-triggered alignment crash, not a framework - regression; see vllm_0290_dsv4_cutedsl CORRECTED). With the token-budget fix (round to 64) DSV4 serves on - 0.29, so dsa_* and dsv4_* can move pins TOGETHER and no family split is required for the 0.29 upgrade. - Remaining owner decisions on this thread: (1) approve the generic generator rounding rule (or accept - per-checkpoint cli_extra_args as the interim); (2) dsv4 op-collector 0.29 lane SHIPPED 2026-09-20 - (collect_dsv4_029.py: storage_block_size/get_kv_cache_shape removal -> generic byte-shape pipeline; builder - block_table_width + zero-padded table, the B300 family-100 contract applied preventively; binding via - bind_kv_cache). SM90 smokes green csa/hca x ctx/gen, backend FLASHMLA_SPARSE_DSV4, fp8_ds_mla layout + - FP8 Lightning-Indexer cache engaged. path_diff verdict still pending: it needs an ok DSV4 serving record at - 0.29, which needs the 64-aligned budget in the workspace render chain (the aic rule is unfixed; fix lives in - aisimulate branch fix/generator-round-vllm-batched-tokens). Historical context kept: DSA lane collect_mla_module_029.py validated (FLASH_ATTN_MLA_SPARSE bf16-kv / - FLASHMLA_SPARSE fp8-kv, both keys need collection), first live copy-on-upgrade instance.' - applies_to: - - family:sparse_attention - - deepseek-ai/DeepSeek-V3.2 - - zai-org/GLM-5.x - - deepseek-ai/DeepSeek-V4-* - vllm_0290_upgrades: - statement: 'vllm 0.24.0 -> 0.29.0 improvements confirmed on SM90: Kimi-K3 and Kimi-K3-NVFP4 now pass (arch registered, - as predicted by the 0.27.1 probe); Gemma-4-26B-A4B-NVFP4 passes (0.24''s tied-embedding NotImplementedError - and the NVFP4 MoE path both fixed); Gemma-4-31B-IT-NVFP4 passes after a DUMMY fidelity fix — 0.29''s transformers - loads the multimodal processor, so dummies must carry processor_config.json (fetch it with configs/tokenizers - during onboarding; the 26B dummy already had it, the 31B one did not).' - applies_to: - - moonshotai/Kimi-K3 - - nvidia/Kimi-K3-NVFP4 - - nvidia/Gemma-4-26B-A4B-NVFP4 - - nvidia/Gemma-4-31B-IT-NVFP4 vllm_fp8kv: statement: 'vllm 0.24/SM90: head-576 MLA has NO attention backend with fp8 KV; kv-auto(bf16) resolves DeepseekV2MLAAttention+SparseAttnIndexer and runs. Door 1 engine kv dtype (fp8 ckpts) = config_gap, kv-auto A/B passes; door 2 ckpt hf_quant kv_cache_quant_algo:FP8 @@ -388,596 +29,6 @@ findings: - nvidia/DeepSeek-V3.2-NVFP4 - zai-org/GLM-5.2-FP8 - nvidia/GLM-5.2-NVFP4 - vllm_kv_required: - statement: 'INVERTED requirement vs GLM: vllm''s DeepseekV4 fp8_ds_mla layout REQUIRES fp8 kv-cache (attention.py:79 - assert, golden auto crashed 2026-08-23) while GLM head-576 has NO backend with fp8 KV. kv dtype must be rendered - per (model, backend) — neither blanket auto nor blanket profile->fp8 works.' - applies_to: - - family:dsv4 - probe_cudagraph_kernel_blindness_fixed: - statement: 'INSTRUMENT FIX (probe_vllm, 2026-09-19): the whole-run kernel tables were built by walking - torch.profiler CPU function_events — kernels replayed inside CUDA graphs have no CPU launch event and - were INVISIBLE. Consequence: vllm 0.29 MiniMax-M3 serving evidence showed ZERO attend/indexer kernels - (M3''s graph-safe attend runs under full cudagraph), and Llama FA evidence missed the in-graph - FlashAttnFwdCombine + prepare_varlen_num_blocks — path_diff produced FALSE "diverged" verdicts against - collector captures (collectors run eager, so their kernels were all visible). Fix in code, generic - (no per-model case): tables now come from the kineto DEVICE event stream (every executed kernel appears - there, graph-replayed or not), CPU-tree walk kept as fallback. Second generalized knob from the same - episode: probe prompt length is now --isl / AIC_PROBE_ISL (default 4096, owner-set 2026-09-19; the - historical implicit value was 32; recorded per-record as probe_isl) because - shape-conditional serving paths (M3 sparse threshold at topk_blocks*block_size tokens, FA split-KV) - are invisible below their thresholds — coverage is a parameter, never a per-model special case. - MiniMax-M3 and Meta-Llama-3.1-8B 0.29 raws re-probed at isl 4096 and superseded (old raws kept in - workspace facts/raw_superseded/); after the fix all four attn/msa 029-lane path_diff verdicts flipped - to aligned with zero collector changes.' - applies_to: - - instrument:probe_vllm - - framework:vllm-0.29.0 - - MiniMaxAI/MiniMax-M3 - - meta-llama/Meta-Llama-3.1-8B - pathdiff_fp8kv_coverage_complete: - statement: | - HARNESS COMPLETENESS (2026-09-23, driven by "fp8-KV must be probed and - compared, not skipped"; REVISED the same day after a second review found - four defects in the first pass — see probe_prefill_prefix_cache_contamination - for the largest). fp8-KV is a first-class path_diff dimension and the fp8 - attention paths are verified aligned with matched evidence: - (a) probe fp8-KV as a config axis: probe_vllm --kv-cache-dtype + - probe_driver kv_variants=[None,fp8] (vllm), path_diff --kv-dtype - selects the same-kv serving record (commit a2076196). - (b) build_ops orphan keep-rule is STRUCTURAL: every orphan the taxonomy - labels with an identity-bearing backend is kept; only unlabeled/ - noise orphans are capped (24, by device time). The first fix - (top-40 by time) was still a count; the original alphabetical - [:15] had truncated half the records (137/274 raws exceed 15). - (c) taxonomy: vllm kernel_unified_attention / reduce_segments - (TRITON_ATTN, the only fp8 head_dim>256 Gemma-4 path) labeled - `triton` — the shared vocabulary's name for the Triton attention - backend (kernel_source_backends.yaml: sglang `triton`, vllm - `vllm_triton_attn` -> triton). The first pass invented - `triton_attn`, breaking the one-vocabulary contract. Routing - kernels moved triton -> framework_native, so `triton` now means - attention only and left path_diff's infra set. - (d) verdict artifacts are reproducible: reports carry capture_file, - capture_cmd, op_hint, kv_dtype and the serving record (id, tp, kv, - isl, prefix_caching, candidate count); records carry runtime.isl; - path_diff --isl selects by prompt length and --serving-raw diffs - against explicit A/B evidence. The first pass saved verdicts to - results/pathdiff/sm90/ where the gate never reads them (it reads - sm90/-/). - VERDICTS (sm90, vllm 0.29.0, kv=fp8, results/pathdiff/sm90/vllm-0.29.0/): - - Gemma-4 head512 fp8 -> ALIGNED (triton both sides). - - DSV3.2 DSA fp8 GENERATION -> ALIGNED (flashmla sparse_fp8 decode + - dsa_nsa paged_mqa_logits + deepgemm). The first pass called this - capture "ctx": run_mla_module picks the phase from - `"context" in perf_filename` (collect_mla_module.py:920) and the - capture used /tmp/cap_perf.txt. - - DSV3.2 DSA fp8 CONTEXT s=4096 (128 heads, separate prefill path: - cp_gather_and_upconvert -> bf16 workspace -> sparse_attn_fwd_kernel) - -> ALIGNED against cache-cold serving at isl 4096. - - DSV3.2 DSA fp8 CONTEXT s=512 (dense FA3 MLA prefill, seq <= index_topk) - -> ALIGNED against cache-cold serving at isl 512 (--serving-raw - facts/reprobe_nopc/f208a101a9c8_isl512.json). Against the isl-4096 - record it is legitimately "diverged" (serving never ran FA3 at 4096): - prefill dispatch is length-conditional, so the serving evidence must - be isl-matched to the collector cell. - - heads=16 fp8 context (TP8-equivalent, num_heads < 32) follows the - mixed-batch path in the collector too (sparse_decode_fwd), matching - flashmla_sparse.py:307 — capture facts/pathdiff/cap_dsa_ctx_fp8_s4096_h16.json. - Capture precision must match serving: gemm_type=bfloat16 shows a false - fused_a-projection drift (cutlass vs fp8_blockscale); use fp8_block. - FOLLOW-UP RESOLVED 2026-09-24 (measured, not read): the Gemma-4 case - file declares sglang_backends {90: triton, ...} on every attention - profile (head_dim 256 and 512), which overrides collect_attn's default - SM90->fa3 table; op_smoke case 168 (Gemma4ForConditionalGeneration, - head_dim 512, triton) captured _fwd_kernel/_fwd_kernel_stage2 = the - serving record's TritonAttnBackend kernels -> ALIGNED - (attn_ctx_hd512_gemma-4-26B-A4B). Original: (unconfirmed, needs an - sglang capture): sglang 0.5.16 serves Gemma-4 (head512) on Triton - attention (_fwd_grouped_kernel_stage1 in the raws) while - collector/sglang/collect_attn.py pins FlashAttentionBackend on SM90 - unless head_dim==192 — same class as the vllm Gemma-4 override. - applies_to: - - instrument:path_diff - - instrument:kernel_taxonomy - - instrument:probe_driver - - framework:vllm-0.29.0 - - platform:sm90 - - google/gemma-4-26B-A4B - - deepseek-ai/DeepSeek-V3.2 - probe_prefill_prefix_cache_contamination: - statement: | - INSTRUMENT FIX (probe_vllm, 2026-09-23): the profiled request reused the - warmup prompt (list(range(isl))) and vllm enables prefix caching by - default, so every "prefill" step the probe profiled was a near-total - cache hit recomputing only the last block — a query of <= block_size - tokens with a full-length KV. ALL 173 vllm serving records with prefill - tables carried this residual, not an isl-token prefill. Identity verdicts - survived wherever the residual still routes to the same kernel family - (FA3, TRITON_ATTN), which is why it stayed invisible; DSA fp8 exposed it: - FlashMLASparse's builder classifies query_len <= 256 (128 heads) as - decode (flashmla_sparse.py:244 @0.29.0), so serving "prefill" evidence - showed the fp8 DECODE kernel and path_diff flagged the collector's - correct separate-path prefill (sparse_attn_fwd_kernel) as kernel drift. - Cache-cold re-probes confirm: DSV3.2 fp8 prefill 688us decode kernel -> - 26.9ms sparse_attn_fwd (+cp_gather_and_upconvert); Gemma-4 fp8 - kernel_unified_attention 863us -> 10.5ms; Llama-3.1-8B FA3 41us -> - 2.08ms. Fix: ea.enable_prefix_caching=False (same policy as the sglang - probe's --disable-radix-cache); records carry runtime.prefix_caching. - TARGETED RE-PROBE (owner decision 2026-09-23: no full re-sweep; GEMM/MoE - identity is not length-conditional, so only attention classes whose - DISPATCH changes with query length were redone). Triage over the 173 - records by mechanism, then 37 + 3 records re-probed cache-cold (all ok): - - DSA family (DSV3.2 variants, GLM-5/5.1/5.2 + FP8/NVFP4; 18): fp8-KV - prefill evidence flipped from the fp8 DECODE kernel to - sparse_attn_fwd_kernel (the separate-path bf16-workspace prefill), - matching the collector; bf16-KV DSV3.2 stayed FA3. - - MLA + fp8-KV (DeepSeek-R1/V3 fp8, Kimi-K2/K2.5, DSV3.1/K2.5/K2.6/ - K2.7 NVFP4; 12): flipped from flash_fwd_splitkv_mla (FlashMLA - decode) to FA3 prefill — the residual query had been routed through - the MLA decode path, so every MLA fp8 prefill row was wrong. - - MiniMax-M3 (7): prefill still showed NO labeled attention kernel — - that was a taxonomy gap, not the probe: 0.29 renamed the M3 Triton - kernels (_gqa_sparse_fwd/_decode, _index_block_score, - _decode_index_score, _topk_index_{partial,merge}); labeled `triton` - (kernel_source_backends.yaml: MiniMaxM3SparseTritonImpl -> triton), - closing the dsa_nsa-covers-M3 naming debt. msa029_m3_{ctx,gen} were - previously aligned on an EMPTY attention set; now kernel-matched. - - Dense FA3 / TRITON_ATTN / linear-attention (133 records): same - family before and after in the three cache-cold controls (Llama, - Gemma-4, DSV3.2 MoE/quant/routing identical; GEMM differs only in - cuBLAS nvjet tile shape) — identity verdicts are trustworthy, prefill - MAGNITUDES are not. build_records prints the count of such records - on every rebuild ("NOTE: N vllm records still carry cached-residual - prefill evidence"). Superseded raws: facts/raw_superseded/prefixcache_2026-09-23/. - DSV4 on vllm has NO serving evidence at all (36 records, mostly `load` - errors from the --max-num-batched-tokens 6012 trigger; the generator fix - lives on its own branch) — unrelated to this contamination. - The trtllm probe has the same shape (warmup and profiled prompt both - [[1]*16], block reuse on) plus no --isl and no phase tables — a weaker - lane that needs the same treatment. Evidence facts/reprobe_nopc/ - (candidates.json, postprocess.py), facts/pathdiff/cap_dsa_ctx_fp8_s4096_fp8gemm.json. - applies_to: - - instrument:probe_vllm - - instrument:probe_trtllm - - framework:vllm-0.29.0 - - platform:sm90 - op_coverage_vllm_0290: - statement: | - OP-LEVEL ALIGNMENT SWEEP (2026-09-23, after the probes were fixed): every - vllm registry op with a serving reference now has a path_diff verdict in - results/pathdiff/sm90/vllm-0.29.0/ (23 files, gate aligned). Captures are - one representative cell per op at a serving model's real shape, run via - the collector's own run_* entry (op_smoke semantics), diffed against - cache-cold serving records (re-probed for DSV3 bf16 and Qwen3.5-0.8B). - aligned: gemm {fp8_block/deepgemm vs DSV3.2, bf16/cublas vs Llama, - fp8 per-tensor vs Qwen3-32B-FP8}; moe {fp8_block DSV3.2, bf16 Gemma-4, - mxfp4 gpt-oss-120b}; mla_context {bf16 DSV3, fp8 R1}; mla_generation - {fp8 R1}; mla_bmm_gen_pre/post; gdn {ctx: flashinfer GDN + conv1d, - gen: fla_triton fused_recurrent} vs Qwen3.5-0.8B; compute_scale; - plus the earlier attention/dsa/msa set. - REAL DIVERGENCE FOUND AND FIXED — moe w4a16_mxfp4 (gpt-oss): the - collector built a bare Mxfp4Config -> Mxfp4MoEMethod -> MARLIN, while - vllm 0.29 overrides OCP "mxfp4" checkpoints of model_type gpt_oss to - GptOssMxfp4Config/GptOssMxfp4MoEMethod (quantization/mxfp4.py:119-139), - whose gpt-oss priority list (fused_moe/oracle/mxfp4.py:584-602) picks - TRITON matmul_ogs on SM90 — exactly what the gpt-oss-120b serving probe - shows. Same class as the Gemma-4 attention override: the collector - constructed the generic config and lost the model-specific method. - collect_moe.py now mirrors the override (commit in this series). - HARNESS FIXES surfaced by the sweep: (1) path_diff picked the LAST - matching serving record; a decode-only Qwen3.5 record beat the - cache-cold one and produced a false collector-only `flashinfer` — records - are now ranked by evidence quality (prefix_caching False > phase tables > - known isl); (2) Triton autotune tile suffixes (_matmul_ogs_..._16x256x128x1 - vs _128x256x128x1, BLOCK_M from the token count) are stripped by - normalize_kernel so kernel-drift compares kernels, not tiles; - (3) FlashInfer GDN kernels relabeled trtllm_internal -> flashinfer (the - vocabulary name; vllm runs them on SM90 @0.29). - NOT ALIGNED-ABLE TODAY (no serving reference, not a collector defect): - - mla_generation bf16-gemm cells: every DeepSeek/Kimi checkpoint is fp8 - or NVFP4, so the bf16-weight grid has no serving instance; the - collector's fused_a_gemm there is vllm's own small-M dispatch - (deepseek_v2.py:~915-925, num_tokens <= 16). Verdict file removed; - the fp8 cell is the reference comparison. - - dsv4_* (6 ops) and mhc_module: vllm DSV4 serving has no evidence - (load errors from the 6012 token-budget trigger; generator fix on - its own branch). - - kda: Kimi-K3 runs only on the vllm kimi-k3 preview image; its - records carry no kernels. - - encoder_attention: the probe prompt is text-only, so vision-encoder - kernels never execute in the profiled window (Qwen3-VL records have - none) — needs a multimodal probe request. - FOLLOW-UP: sglang (91 records at isl 32) and trtllm (74 at 16 tokens, - block reuse on) serving evidence predates their probe fixes; their - length-conditional classes (DSA/NSA, M3) need the same targeted - re-probe before sglang/trtllm op verdicts are drawn. - applies_to: - - instrument:path_diff - - instrument:probe_driver - - framework:vllm-0.29.0 - - platform:sm90 - - openai/gpt-oss-120b - - Qwen/Qwen3.5-0.8B - - deepseek-ai/DeepSeek-V3 - - deepseek-ai/DeepSeek-R1 - op_coverage_sglang_trtllm_2026_09_24: - statement: | - OP-LEVEL ALIGNMENT SWEEP, sglang 0.5.16 and trtllm 1.3.0rc23 on SM90, after - both probes were brought to the vllm standard (isl 4096, cache-cold, phase - tables) and their length-conditional records re-probed. Captures run the - collector through op_smoke (--case-filter/--case-prefix) or the module - CLI's --quick / in-process entry (the sglang DSA/MSA module collectors run - each case in a SUBPROCESS, which the parent profiler cannot see — a - capture of the worker wrapper is silently empty). - HARNESS DEFECT found first: path_diff graded a crashed or empty capture - "aligned" (empty set is a subset of everything) — six broken sglang - captures passed. Verdicts are now invalid-capture / no-collector-signal. - COLLECTOR DRIFT ON THE PINNED VERSIONS (all fixed, each surfaced as a - zero-kernel capture; 286 sglang + 114 vllm collector unit tests pass): - sglang 0.5.16 — ModelRunner takes ps: ParallelState (mla_module built - it with tp_rank/tp_size...), attention backends read model_runner.ps / - server_args.enable_prefill_cp / enable_dp_attention / - model_config.linear_attn_registry_result / hf_config.get_text_config() - / is_draft_worker (attention + MLA mocks lacked them), - Req.set_extend_input_len -> set_extend_range (mla_module, dsv4), - ForwardBatch.init_new requires return_hidden_states_before_norm - (mla_module context+generation, dsv4). The sglang collector had not - run on its own pin since the bump. - trtllm — gemm/computescale/mla_bmm imported case_generator and helper - as bare modules (only works from collector/ cwd); GDN prefill/decode - imported the vendored FLA Triton kernels directly while serving - dispatches to FlashInfer's cute-DSL GDN chunk + bf16-state decode on - SM90/SM100 (gdn_mixer.py:57-70,670; fused_sigmoid_gating_recurrent.py: - 205-238). The collector now calls the mixer's resolver for prefill - (packed [1,T,H,D] + cu_seqlens, fp32 g — FlashInfer's contract) and - serving's fused_sigmoid_gating_delta_rule_update for decode, labeling - rows by the dispatch that actually ran (new kernel_source values in - kernel_source_backends.yaml). - path_diff/normalize — cute-DSL symbols carry tensor specs - (…_kernel_tensorptrbf16gmemalign32o1291…): identity cut at - _tensorptr/_object; `trtllm::` launchers excluded like vllm/sglang; - collector EXIT_CODE_RESTART (10) is a protocol signal, not a failure. - VERDICTS (results/pathdiff/sm90/{sglang-0.5.16,trtllm-1.3.0rc23}/): - sglang ALIGNED: attn ctx+gen (Llama-3.1-8B shape, fa3), dsa ctx+gen - (DSV3.2 bf16: sparse_attn_fwd/FA3 + indexer), mla ctx+gen (DSV3, fa3), - msa ctx (M3), gemm fp8_block, moe fp8_block, gdn ctx+gen (Qwen3.5). - trtllm ALIGNED: attn ctx (Llama, fmha_v2 causal), dsa ctx+gen (DSV3.2), - mla ctx+gen (DSV3), gemm fp8_block, moe fp8_block, gdn ctx (FlashInfer - chunk + trtllm conv1d). - RESOLVED (2026-09-24, owner asked for these one at a time): - - trtllm attention_generation (Llama-3.1-8B): NOT a collector bug — - a cell mismatch. The first capture took the kv_len=1 cell, where - trtllm's own heuristic declines XQA (TLLM debug: "JIT XQA is not - used: maybe no performance gain" -> MMHA); the serving probe's - warmup does the same at short KV (8 of 424 decisions) and selects - XQA (kernel_mha) for the 4096-token decode. At the grid point that - matches the probe (batch 1, kv 4095) the collector selects XQA too - -> ALIGNED. TRTLLM_ENABLE_XQA_JIT no longer exists in rc23 binaries - (only TRTLLM_FORCE_XQA / TRTLLM_XQA_BLOCKS_PER_SEQUENCE); the - collector's toggle is documented as inert. Rule reinforced: capture - the collector cell at the probe's kv length, never the first cell. - - sglang msa_generation (MiniMax-M3): the PROBE is the unfaithful side. - The gate (flash_with_topk_idx.py:822-826) uses score.shape[2], which - under CUDA-graph capture is ceil(max_context_len / 128) (static - shapes, minimax_sparse_backend.py:175-178) and the live KV length in - eager. Production decodes under graphs; M3's context_len makes the - static block count exceed 4096 -> topk_split, which is what the - collector models (collect_msa_module.py:892-897, use_graph). The - identity probe disables CUDA graphs, so it saw the eager radix path. - probe_sglang gained --cuda-graph (recorded as probe_cuda_graph); - M3 re-probed with graphs on: decode now runs _topk_index_partial / - _topk_index_merge (topk_split), matching the collector -> ALIGNED - (verdict computed with --serving-raw against that record). All - three gates green (vllm 33, sglang 12, trtllm 9). - HARNESS FOLLOW-UP this opens (RESOLVED 2026-09-24, owner decision: - probes now run the framework's own mode for prefill AND decode — - see probe_execution_mode_is_the_framework_default_2026_09_24): - the identity probes ran eager on every backend. Any op whose - kernel choice reads capture-time static shapes (not just M3 top-k: - FlashMLA/FA split-KV planning, XQA multi-block, trtllm-gen decode - scheduling are candidates) is mis-identified by an eager probe. - The device-stream tables already see graph-replayed kernels, so the - probes can run decode under graphs; the question for the owner is - whether to make graph-mode the default decode evidence (production - truth) and keep eager only for prefill. - ROSTER GATE — RESOLVED 2026-09-24 (owner: "fp8 就用 zai-org/GLM-5.3"): - rename landed in ac274b36 — case YAML model_paths + both alias rows, - packaged model_configs (GLM-5.3 = Hub fp8 config, GLM-5.3-BF16 = the - former bf16 file, -FP8 removed), sdk/common.py list, support-matrix - b60.csv rows, two unit tests (178 pass); workspace roster - (configs/repos.txt, targets.yaml overlay) + GLM-5.3-BF16 dummies - generated; probe_driver --plan now enumerates GLM-5.3 (fp8), - GLM-5.3-BF16 (bfloat16) and GLM-5.3-NVFP4 on all three backends, and a - roster repo without dummy variants is skipped loudly instead of - crashing enumeration. Original: (probe_driver --plan refused: "no - fetched config for zai-org/GLM-5.3-FP8"): that repository does not - exist on Hugging Face - (404 with a token; zai-org lists GLM-5.3, GLM-5.3-BF16, GLM-5.3-Flash, - GLM-5.3-Flash-BF16). zai-org/GLM-5.3 itself is the FP8 artifact - (config quantization_config fp8 e4m3 block 128) and GLM-5.3-BF16 the - bf16 one — 5.3 inverted the 5.x naming. The collector case file - (cases/models/GlmMoeDsaForCausalLM_cases.yaml) aliases GLM-5.3 to the - bf16 row and GLM-5.3-FP8 to the fp8 row, and the SDK's supported-model - list (src/aisimulate_core/sdk/common.py:690-692) names GLM-5.3-FP8 - too — a producer+consumer naming fix (collector YAML + tests + SDK - list), so it is an OWNER DECISION: rename (GLM-5.3 -> fp8 row, - GLM-5.3-BF16 -> bf16 row) or sign an exclusion. configs/ already holds - zai-org_GLM-5.3.json and nvidia_GLM-5.3-NVFP4.json; GLM-5.3-BF16's - config fetches fine (200) once the rename lands. - EXPLAINED (kept out of the gate, same class as vllm kda_gen): trtllm - gdn_gen — FlashInfer decode row matches serving kernel-for-kernel; the - FLA fused_recurrent row is the deliberate fallback lane. - NOT RUN: 5 trtllm records probed on trtllm-probe:1.3.0rc20-onnxfix - (image gone); sglang gdn_ctx via op_smoke hits the known int32-offset - FIXME at the largest sweep points (captured via a direct 4096-token - call instead). - applies_to: - - instrument:path_diff - - instrument:op_smoke - - framework:sglang-0.5.16 - - framework:trtllm-1.3.0rc23 - - platform:sm90 - - meta-llama/Meta-Llama-3.1-8B - - deepseek-ai/DeepSeek-V3.2 - - deepseek-ai/DeepSeek-V3 - - MiniMaxAI/MiniMax-M3 - - Qwen/Qwen3.5-0.8B - unalignable_ops_followup_2026_09_24: - statement: | - Follow-up on the "no serving reference" ops (owner 2026-09-24: "if running - it produces evidence, run it; if load crashes, try to fix it"). - - encoder_attention: FIXED without a multimodal prompt. Engine construction - runs vLLM's own profile_run (full-model dummy forward; for VL models it - pushes max-size dummy images through the vision tower). probe_vllm now - records that run as profile_run_kernels and build_ops counts it as - identity evidence. Qwen3-VL-2B ViT: FlashAttnFwdSm90 x24 layers in - serving; collector encoder_attention (FLASH_ATTN for vit attention, - head_dim 64) -> ALIGNED. - - DeepSeek-V4 on vllm: the load crash is the rendered - --max-num-batched-tokens 6012 (cutedsl compress asserts 64-byte - slot-mapping alignment). The harness golden renderer runs the aic/ - checkout of aiconfigurator, not aisimulate, so the fix branch - (fix/generator-round-vllm-batched-tokens) never reached it. Mirrored - the rounding into aic/ (a71f27955: rule_plugin/vllm.rule, - benchmark/vllm.rule, backend_config_mapping.yaml); the 24 planned DSV4 - goldens re-render at 6016 and are being re-probed. Planning via - probe_driver is currently blocked by the roster gate ("no fetched - config for zai-org/GLM-5.3-FP8 — OWNER DECISION NEEDED"); the goldens - were rendered through render_golden() on the existing plan entries. - The product fix still needs the aisimulate branch merged. - - Kimi-K3: vllm 0.29.0 ships K3 natively (KimiK3ForConditionalGeneration, - vllm/models/kimi_k3/nvidia/kda importable) — the collect_kda docstring - ("kimi-k3 preview image") is stale. K3 0.29 records exist with kernel - tables; their kernels were unlabeled (FlashKDA _flash_kda_fwd_*, - kda_decode_fusion_many_heads, attn_res, layer_norm_gated, situ_and_mul) - — taxonomy rules added under the vocabulary names (flashkda, - vllm_kernel). K3 records are being re-probed cache-cold; the kda op - capture follows. - - bf16-gemm MLA cells: CONSUMED. The MLA module tables carry a gemm_quant - axis (crates/core/src/perfmodel/perf_database/mla.rs: query_context_module / - query_generation_module take GemmQuantMode); any aisimulate run that - models an MLA architecture with gemm_quant_mode=bfloat16 reads them - (readiness.rs even tests DeepseekV4 + bfloat16). They measure vLLM's - own bf16-weight dispatch (fused_a_gemm at num_tokens<=16) — faithful - to what vLLM would run, just with no deployed checkpoint to cross-check. - RESULTS (same day): 24 DSV4 vllm records + 6 K3 records re-probed clean - (6016 budget; first DSV4 serving evidence on vllm: sparse_attn_fwd, - cute-DSL compressor/indexer, mHC TileLang). Verdicts: dsv4_csa/hca - context+generation, dsv4_hca_attn, dsv4_paged_mqa_logits -> ALIGNED vs - sgl-project/DeepSeek-V4-Flash-FP8 fp8; kda_ctx -> ALIGNED after fixing - the collector's FlashKDA prefill call (0.29 takes caller-owned - out/final_state/workspace buffers, kda.py:193-215/536-547; the - preview-era call raised TypeError on every context cell). Two - divergences are EXPLAINED and deliberately kept out of the gate: - - mhc_module: serving runs mhc_pre_big_fuse_WITH_NORM (norm_weight - passed, tilelang.py:90) and mhc_fused_post_pre for middle layers; - the collector measures the norm_weight=None pre + separate post - because the SDK bills attn_norm separately (collect_mhc_module.py: - 118-134, measured 1-2% off the fused path). Row-semantics contract; - switching needs a producer+consumer decision. - - kda_gen: the collector measures BOTH the fused decode - (ops.fused_kda_decode = kda_decode_fusion_many_heads, serving's - pure-decode path, kda.py:686-695) and the Triton fallback lane - (causal_conv1d_update + fused_recurrent_kda_packed_decode, taken by - serving only for spec-decode / mixed batches). The fused row matches - serving kernel-for-kernel; the fallback rows are collector-only by - construction. A verdict per kernel_source lane would express this; - today's subset semantics cannot. - sglang/trtllm evidence: 45 length-conditional records re-probed with the - fixed probes (isl 4096, cache-cold, phase tables). sglang DSA prefill at - isl 32 was dense FA3 (SGLANG_DSA_PREFILL_DENSE_ATTN_KV_LEN_THRESHOLD: - use_mha when max_kv_len is short, dsa_backend.py:2999-3012 @0.5.16); - at 4096 it is sparse_attn_fwd — the old sglang prefill evidence was - the dense fallback. The old sglang fp8-KV decode evidence (flashmla_kv) - came from a retired probe injection (--kv-dtype fp8_e4m3): the golden - sglang CLI for fp8 profiles renders NO kv dtype, so current serving - truth is bf16 KV (dsa prefill flashmla_sparse, decode fa3). Added the - kv variant to sglang planning (fp8_e4m3) so the fp8-KV DSA/MLA paths are - probed as a separate, labeled config. 5 trtllm records were probed on - trtllm-probe:1.3.0rc20-onnxfix, an image no longer present — not - re-probed. 8 DSV4 sglang runs exhausted the probe's fixed 16384-token - KV pool at isl 4096 (multi-cache-group) — pool now scales with isl; - rerun in progress. - applies_to: - - instrument:probe_vllm - - instrument:probe_sglang - - instrument:render_golden - - framework:vllm-0.29.0 - - framework:sglang-0.5.16 - - platform:sm90 - - family:dsv4 - - moonshotai/Kimi-K3 - - Qwen/Qwen3-VL-2B-Instruct - probe_execution_mode_is_the_framework_default_2026_09_24: - statement: | - OWNER DECISION 2026-09-24: the identity probes run the framework's OWN - execution mode — torch.compile + piecewise/full CUDA graphs exactly as - the rendered config asks — for prefill AND decode; `--eager` is an A/B - switch only (records carry runtime.probe_eager / probe_cuda_graph; - None = a pre-2026-09-24 eager record). Why they ran eager before: a - kernel-visibility shortcut from before the device-stream tables existed - (graph-replayed kernels have no CPU launch event) plus capture time / - capture failures on dummies. It had become a fidelity liability: any - dispatch that reads capture-time static shapes is mis-identified by an - eager probe (sglang MiniMax-M3 top-k was the proven case), and eager - prefill is not serving's prefill. - A/B (9 records re-probed in framework mode, eager raws kept under - facts/raw_superseded/eager_2026-09-24/): vllm Llama-3.1-8B, - DeepSeek-V3.2 bf16-KV + fp8-KV, MiniMax-M3 bf16-KV + fp8-KV; sglang - DeepSeek-V3.2 bf16, MiniMax-M3 fp8-KV; trtllm Llama-3.1-8B, - DeepSeek-V3.2. Attention / GEMM / MoE / indexer kernel sets are - IDENTICAL to the eager records on every one of them (trtllm: same - 14+16 and 36+32 kernels; vllm DSV3.2 prefill 46/46, M3 38/38; sglang - DSV3.2 prefill 61/61). What changes is what the profiler SEES and the - glue: (a) the Python API spans (AIC::attn::*, AIC::quant_apply::*) - that attributed GEMM/quant/attention kernels to ops vanish under graph - replay and under torch.compile (vllm Llama: both phases) — the kernels - arrive as orphans; (b) vLLM compile replaces its custom elementwise - ops (act_and_mul, rms_norm, rotary_embedding, _typeConvert) with - Inductor triton_{poi,red}_fused_* kernels on Llama (not on DSV3.2/M3, - whose rendered configs keep them); (c) sglang M3 fp8-KV decode under - graphs runs _topk_index_partial/_merge (topk_split = the collector's - path) — the eager fp8-KV record had hidden it and diff_sgl picked that - record over the graph-mode bf16 one, so msa_gen read DIVERGED until - the re-probe. sglang graph mode adds _fused_dsa_decode_metadata_kernel - / _fused_metadata_kernel_general (metadata glue, framework_native). - HARNESS CONSEQUENCES (all landed): probe_driver keeps every - taxonomy-labeled orphan (cublas/vllm_kernel/sgl_kernel/torch were - previously capped as noise — under graphs they are the ONLY place - GEMM and quant kernels appear); identity.attn_backend falls back to - the attention kernel family when no span exists; taxonomy gained the - Inductor and sglang-metadata rules. path_diff: the first recompute - after the no-collector-signal rule flipped FOUR gemm-class verdicts - (gemm_bf16 Llama, gemm_fp8 Qwen3-32B, mla_bmm_gen DSV3, compute_scale) - that had been graded "aligned" BEFORE the rule existed — their only - families (cublas, vllm_kernel) are infra by design, so they can never - have a signal family. Rule refined: no signal family -> grade on - kernel-NAME overlap inside the infra families (infra_family_name_matches - in the report). Three re-align on names (the collector's nvjet / - scaled_fp8_quant instantiations are ones serving launched); - compute_scale did not — it had been paired with fp8-BLOCK DeepSeek-V3.2 - (per_token_group_quant), never a valid instance of scaled_fp8_quant; - re-paired with per-tensor Qwen3-32B-FP8 -> aligned. Gates after all of - it: vllm 33 / sglang 15 / trtllm 9 aligned, zero non-aligned files. - sglang DSA prefill is LENGTH-CONDITIONAL and the collector mirrors it: - at isl 512 (framework mode, fp8-KV and bf16-KV alike — both probed) - serving runs dense FA3 with NO indexer logits or top-k (only - _quantize_k_cache); at isl 4096 the - indexer (sm90_fp8_mqa_logits) + fp8 sparse decode kernel. The - whole-sweep fp8 ctx capture unions both regimes and read DIVERGED - (fa3 collector-only vs the 4096 record; mqa_logits collector-only vs - the 512 record — the prefix>0 cells push kv_len over the threshold). - Single-cell captures (frozen sweep spec replaced; AIC_DSA_CONTEXT_* - env filters do the same): s=4096 vs the framework-mode fp8-KV isl-4096 - record -> ALIGNED (dsa_ctx_fp8_DeepSeek-V3.2); s=512 prefix-0 vs the - isl-512 record -> ALIGNED (dsa_ctx_fp8_s512_DeepSeek-V3.2: FA3 only, - no indexer on either side). sglang gate: 15 aligned. - NOT DONE: the other ~600 records are still eager-probed (identity-safe - per the A/B, but their op attribution differs from framework-mode - records); re-probe them when a verdict needs one, not wholesale. - applies_to: - - instrument:probe_vllm - - instrument:probe_sglang - - instrument:probe_trtllm - - instrument:probe_driver - - instrument:path_diff - - instrument:kernel_taxonomy - - framework:vllm-0.29.0 - - framework:sglang-0.5.16 - - framework:trtllm-1.3.0rc23 - - platform:sm90 - harness_hygiene_2026_09_24: - statement: | - Owner asked for the open-issue list after the framework-mode change and - told me to fix the harness items ("别搞复杂问题"). Landed: - - probe_driver --matrix: the moe column read only API spans; framework- - mode records carry MoE kernels as orphans -> orphans are scanned too. - - First unit tests for the harness: tests/unit/collector/opharness/ - (path_diff.grade extracted as a pure function: subset semantics, - within-family drift, invalid-capture, no-collector-signal, gemm-class - infra-name grading; probe_driver: normalize_kernel, taxonomy contract - incl. "only attention rules emit `triton`", orphan keep rule, span/ - launcher exclusion). 34 tests, pytest.mark.unit. - - Taxonomy backlog: 61 -> 2 unmatched kernel names across all 637 - records (sources verified in the pinned images): identity-bearing - additions dsa_nsa (_fused_q_kernel, FilteredTopK, index_buf_accessor, - DeepGEMM smxx paged-logits helpers), fla_triton (Kimi-K3 KDA chunk / - recurrent / gate kernels, fused post-conv), triton_fused_moe - (_reduce_grouped); the rest is allocator / position / DSV4 SWA and - compress metadata / norm-rope / routing / sampler / load-time repack - glue. Left unmatched on purpose: sglang Kimi-K3 _score_kernel and - _combine_kernel (no source hit; 1 record each). - - path_diff --capture --env K=V sets collector cell filters - (AIC_DSA_CONTEXT_{SEQ_LENS,PREFIX_LENS,BATCH_SIZES}) and records - every AIC_*/AIS_* cell variable in the capture; the verdict carries - capture_env. Whole-sweep captures union length/prefix-conditional - paths and cannot match a single serving record. - - components/captures/: the capture scripts and the four verdict - helpers (verdicts_{all,dsv4,sgl,trt}.sh) that produce every file - under results/pathdiff/, parametrized on AIS_PROBE_WORKSPACE, so a - verdict can be re-derived from the repository alone. - Checked and already fixed (stale in my own list): vllm DSA context - mirrors serving's chunked prefill (context_dsa_chunk_prefill_size - new-token budget + context_dsa_prefix_lengths prefix axis, seq cap - 16384) — no monolithic large-isl construction remains. - Gates after all of it: vllm 33 / sglang 16 / trtllm 9 aligned. - applies_to: - - instrument:probe_driver - - instrument:path_diff - - instrument:kernel_taxonomy - - platform:sm90 - trtllm_kv_dtype_variants_2026_09_24: - statement: | - trtllm was the only backend whose fp8-KV path was never compared. Now: - probe_trtllm --kv-dtype overrides the rendered kv_cache_config.dtype - (recorded as probe_kv_cache_dtype + probe_overrides; the KV manager's - resolved dtype is the ground truth), probe_driver plans a `fp8` variant - per trtllm run (rid tag kvfp8), and the verdict helper pins the bf16 - entries to kv auto so a variant record can never be picked for them. - Four fp8-KV records probed (framework mode): Llama-3.1-8B, DeepSeek-V3, - Qwen3.5-0.8B (KV manager DataType.FP8 on all three), DeepSeek-V3.2 — - which FAILS at load: `RuntimeError: Expected kv.dtype() == - torch::kBFloat16` inside flash_mla_interface (mla.py forward_sparse -> - flash_mla). trtllm 1.3.0rc23 sparse MLA (DSA) requires bf16 KV; there is - no fp8-KV DSV3.2 serving instance on this pin, so the collector's DSA - fp8-KV rows (_get_precision_combos adds ('bfloat16','fp8') for dsa on - SM>=89) measure a path serving cannot run. Kept OUT of the gate (no - serving record); the collector rows are not wrong data, they are - unreachable data — flag to the trtllm collector owner, retest on the - next pin. - Single-cell captures at the bf16 gate cells with fp8 KV - (components/captures/trt_kv/): attention context and generation - (Llama, use_fp8_kv_cache=True) ALIGNED — serving keeps the bf16 - context FMHA and writes fp8 KV; the use_fp8_context_fmha=True cell - (e4m3 FMHA) is NOT Llama's serving path but IS DeepSeek-V3's - (fp8 weights + fp8 KV -> fmha_v2_flash_attention_e4m3 in the record); - MLA context + generation (DeepSeek-V3, kv 'fp8' + gemm fp8_block) - ALIGNED. Lesson repeated: a substring --case-filter picked a head_dim - 64 sliding-window cell first; the captures use --case-prefix with the - full leading tuple. - Also fixed on the way: the archived sglang MiniMax-M3 bf16 record was - the eager run (the graph-mode raw had only been used via - --serving-raw); superseded by the graph-mode raw so msa_gen holds - under kv auto. Gates: vllm 33 / sglang 16 / trtllm 13 aligned. - A/B (GLM-5.2-NVFP4, trtllm rc23, KV forced to bfloat16 to escape the - sparse-FlashMLA bf16-KV assert): llmapi REFUSES it — "Overriding KV - cache quantization with an invalid type ... Accepted types are - ('fp8', 'nvfp4', 'auto')" (facts/trt_kv/glm52nvfp4_kvbf16.json). So an - NVFP4 DSA artifact (hf_quant kv_cache_quant_algo FP8) has NO servable - KV dtype on trtllm rc23 / SM90: auto and fp8 hit the kernel assert, - bf16 is rejected at config validation. Every NVFP4 DSA cell in the - trtllm matrix (GLM-5 / 5.1 / 5.2 / 5.3-NVFP4, DeepSeek-V3.2-NVFP4) fails - for this one reason; upstream item 4 in facts/upstream_reports_2026-09-24.md. - applies_to: - - instrument:probe_trtllm - - instrument:probe_driver - - instrument:path_diff - - framework:trtllm-1.3.0rc23 - - platform:sm90 - - deepseek-ai/DeepSeek-V3.2 - - deepseek-ai/DeepSeek-V3 - - meta-llama/Meta-Llama-3.1-8B glm53_onboarding_2026_09_24: statement: | GLM-5.3 trio probed on all three backends after the rename (18 runs: @@ -1166,301 +217,6 @@ findings: - nvidia/Kimi-K3-NVFP4 - framework:vllm-0.29.0 - platform:sm90 - onboarding_recent_releases_2026_09_25: - statement: | - Owner request (2026-09-25): "再试试 qwen3.8 和 glm5.3-flash … 把这些模型也 - 弄进我们的列表 … inkling 啥的，最近这 1-2 月的模型 … probe 一把，增加对应的 - config". 33 Hub repos were surveyed and fetched with the new - components/fetch_inputs.py (config, hf_quant, every non-weight file into - configs/aux_files/); 31 have a transformers config, 2 do not - (mistralai/Leanstral-1.5-119B-A6B, MiniMaxAI/MiniMax-H3 — no config.json - on the Hub; recorded in targets.yaml roster.excluded, decided_by - pending-owner). The 30 not yet in the roster were added to - configs/repos.txt + roster.extra_repos; dummies for all 30 generated - without hand edits (tokenizer/custom code provisioned from aux_files). - Three gates decide whether a checkpoint yields an identity record, and - each of them rejected part of the list: - (1) THE GENERATOR (this repo's SDK) parses the architecture — rejected - 9 architectures / 19 repos, now recorded as "generator rejects" matrix - cells: Glm5NextForConditionalGeneration (GLM-5.3-Flash, -BF16, nvidia - -NVFP4), Qwen4ExpForConditionalGeneration (Qwen3.8-Flash-Next, -FP8, - -NVFP4), InklingForConditionalGeneration (Inkling, Inkling-Small and the - two baerquants NVFP4 artifacts), MiMoV2ForCausalLM (MiMo-V2.6-Pro-RL, - -Flash-RL; note the SDK knows MiMoV2FlashForCausalLM only), - Xing4_0ForCausalLM (x2), AliceAIForCausalLM, K2HorizonForCausalLM, - Spark2_5ForCausalLM. A tenth, Qwen3_5ForCausalLM (text-only dense - Qwen3.5: Altworld/Hemmingway-1, TokenRhythm/NeoHorse-1-9B), was the - flat-config twin of an already-supported class and is now mapped - (commit 8d8e154d) — both rendered and were probed in pass 2; - Hemmingway-1 was then DROPPED by owner decision (CC-BY-NC-4.0, see - below), its inputs/raws live in facts/dropped/. - (2) THE PINNED FRAMEWORKS know the architecture — checked in the three - images: vllm 0.29.0 lacks Glm5Next and DeepseekV41; sglang 0.5.16 lacks - Glm5Next, Qwen4Exp, Qwen3_5MoeForCausalLM, DeepseekV41, MuseGlimmer; - trtllm 1.3.0rc23 lacks Glm5Next, Qwen4Exp, DeepseekV41, MuseGlimmer, - Inkling, MiMoV2. Consequence for the owner's two named targets: - GLM-5.3-Flash is unservable on EVERY pinned framework AND unparsed by - the SDK; Qwen3.8-Flash-Next (Qwen4Exp) loads only under vllm 0.29. - Both need a framework pin bump before any identity can be recorded. - DeepSeek-V4.1-Flash (SDK-supported) fails on all three pins at the - transformers config stage ("model type deepseek_v41 … Transformers does - not recognize"); Muse-Glimmer-30B (+NVFP4) the same on sglang/trtllm - (`muse_glimmer`) but passes on vllm. - (3) THE PROBE GPU holds the dummy cut — Qwen3.8-2.4T-A95B: the driver - only ever queued dummy variant index 0 (depth8 = 8 x 26B params), a - permanent OOM. New driver rule (commit 4447cf41): a raw whose LOAD - stage failed with CUDA OOM / trtllm "insufficient GPU memory" advances - the representative to the next smaller faithful cut, recorded as - capacity_fallback_from on the run — observed, never predicted; the - smallest cut stays queued when everything OOMs. Outcome: bf16 and FP8 - 2.4T OOM even at depth4 (one full period, 4 x 26B params ~ 194 / 104 - GiB of weights on a 139.8 GiB GPU) on vllm and trtllm — no faithful - single-GPU cut exists, the cells stay honest OOM fails; sglang rejects - Qwen3_5MoeForCausalLM outright. NVFP4 2.4T: depth8 OOMs, depth4 result - in pass 2 (below); trtllm rc23 additionally fails NVFP4 MoE on SM90 - ("No supported MoE GEMM tactic remains after replacing unsupported - NO_SMEM epilogues") — the same NVFP4-on-Hopper wall class as the other - trtllm NVFP4 cells. - Identity records landed (framework mode, tp1, rendered + fp8-KV - variants) for: Qwen3.8-27B / -FP8 / nvidia -NVFP4 (all three backends; - vllm ChunkGatedDeltaRule + fla_triton/causal_conv1d + fa3, sglang - HybridLinearAttnBackend, trtllm fmha), NVIDIA-Nemotron-3.5-Lightning- - 30B-A3B-BF16 (all three), Ornith-1.5-35B-A3B (Qwen3_5MoeForConditional - Generation; all three), openbmb/MiniCPM5-2B (Llama; all three), - Muse-Glimmer-30B / -NVFP4 (vllm), plus the pass-2 additions. Taxonomy: - every vllm kernel of the new models is already labeled; the only - unlabeled sglang orphans were the per-variant metadata builder - (_fused_metadata_kernel_ps1_no_swa) and raw memcpy128/memset32 runtime - copies — both glue, rules added (commit 995c7430). No new op family - appeared: GDN/KDA/mamba/MoE/attention kernel families of the onboarded - models are all in the existing taxonomy. - Configs bundled into the SDK (byte-for-byte at pinned revisions, - THIRD_PARTY_NOTICES updated; commits 97bce88f, 8d8e154d, 895586de): - Qwen3.8-27B / -FP8, Qwen3.8-2.4T-A95B-NVFP4, DeepSeek-V4.1-Flash-NVFP4, - Nemotron-3.5-Lightning-BF16, Ornith-1.5, MiniCPM5-2B, NeoHorse-1-9B. - NOT bundled: every generator-rejected architecture (dead weight until - modeled), and the two modelopt mixed_precision NVFP4 artifacts - (Qwen3.8-27B-NVFP4, Muse-Glimmer-30B-NVFP4) — the SDK quant inference - maps mixed_precision to fp8_static for dense models and the support - matrix then reports FAIL instead of HW_INCOMPATIBLE on non-FP4 - systems (TODO.md); they stay in the probe roster and render from the - Hub. The other eight classify like their bundled siblings in a - tools/support_matrix dry run on h200_sxm. - Side answers: Kimi-K3 max-num-seqs bisection completed — 64, 256, 384, - 448, 480, 496 all pass framework-mode; only the generator's 512 fails, - i.e. the fault is the 512-batch CUDA-graph decode warm-up itself, not a - threshold "above 256" (facts/k3_ab/seqs*.json). The compute_scale - standalone-kernel deviation is now a governed standards exception - (collector/evidence_exceptions.yaml, expires 2026-12-31). - Pass 2 (after the Qwen3_5ForCausalLM alias and the capacity rule): - NeoHorse-1-9B (and Hemmingway-1 before its removal) pass on vllm and - trtllm, sglang 0.5.16 has no Qwen3_5ForCausalLM implementation - (framework gap); - Qwen3.8-2.4T-A95B-NVFP4 passes on vllm at depth4 (capacity_fallback_from - depth8; ModelOptNvFp4FusedMoE -> marlin W4A16 on SM90, as for the other - NVFP4 MoE checkpoints), sglang framework gap, trtllm NVFP4 MoE wall. - Matrix after onboarding (Hemmingway-1 removed): 129 rows per backend - — vllm 97 pass / 9 pass+custom / 23 fail, sglang 93 / 6 / 30, trtllm - 69 / 0 / 60; of the 29 new rows: vllm 10 pass, sglang 6 pass, trtllm - 7 pass, 19 rows - "generator rejects" on every backend, the rest framework gaps / walls - named above. Note (display, pre-existing): NemotronH rows on vllm show - attention "ReLUSquaredActivation" and the text-only Qwen3.5 rows show - the fa3 kernel family instead of ChunkGatedDeltaRule — the identity - fallback reads the first wrapped/attention span; kernel families in the - records are correct. - Evidence: facts/onboard_2026-09-25_repos.txt (33 surveyed repos), - facts/onboard_render/arch_support*.txt (framework registry checks), - facts/onboard_render/{plan.log,plan_onboard_pass1.json,queues*/}. - OWNER DECISIONS (tianhaox, 2026-09-25): (a) Leanstral-1.5 / MiniMax-H3 - dropped (params.json-only; roster scope = language models and models - with a vision module, no diffusion/DiT — none of the 30 was DiT); - (b) models with license problems are dropped: Hemmingway-1 - (CC-BY-NC-4.0) removed from roster, configs, dummies, plan and matrix - (raws in facts/dropped/); every other new repo is Apache-2.0 / MIT / - NVIDIA Open Model / OpenMDW / Qwen-community (Qwen3.8-Flash-Next only, - not bundled anyway); (c) framework gaps and walls stay as recorded — - no pin bumps now; (d) SDK modeling of the 9 rejected architectures - stays open. Structural notes for (d), from the Hub configs: Glm5Next - = GLM-5.3 DSA (kv_lora 512, indexer topk 2048, mHC fields) with 34/45 - layers swapped to KDA linear attention (linear_attn_config as in - Kimi-K3, 0-based lists) + 288-expert MoE + MTP — closest existing - families DEEPSEEKV32/GLM-5.3 + KIMIK3; Qwen4Exp = Qwen3.5 GDN hybrid - (interval 4, 512 experts x 640, MTP) plus new n-gram embedding - (ngram_*, ple_*) and indexer_* fields — extends QWEN35; AliceAI = - Qwen3.5-style GDN/KDA hybrid MoE (512 x 512, interval-4 layer_types) - with block attention residual — QWEN35-like; Inkling = 66-layer - GQA MoE (256 experts top-6, 2 shared) with SWA layers (local_layer_ids, - swa_* heads), short conv, vision hmlp encoder + audio — new; MiMoV2 = - MiMo-V2-Flash lineage (hybrid SWA 128 + full attention, 384-expert - MoE, head_dim 192/v 128, vision + audio configs) — extends HYBRIDMOE; - Xing4_0 = DeepSeek-V3-style MLA (q_lora 768, kv_lora 512, nope 128 / - rope 64) + 64-expert MoE + mHC + MTP — DEEPSEEK-like; K2Horizon = - GQA MoE (100 experts top-8 + MoVA expert set) — MOE-like; Spark2_5 = - dense 36-layer with 27 SWA(512)/9 full layers, per-kind rope — - Gemma-like SWA hybrid. Framework side: vllm 0.29 loads Qwen4Exp, - Inkling, MiMoV2 natively; none of the pins knows Glm5Next, Xing4_0, - AliceAI, K2Horizon or Spark2_5 (custom auto_map code only). - FOLLOW-UP DECISION (tianhaox, 2026-09-25, later the same day): AliceAI, - Xing4.0 (both artifacts), Spark-X2.5 and K2-Horizon are dropped from - the roster altogether (custom-code-only architectures no pinned - framework or the SDK knows); inputs, dummies and plan rows moved to - facts/dropped/. Remaining unmodeled architectures in the roster: 4 - (Glm5Next, Qwen4Exp, Inkling, MiMoV2 — 12 repos). Matrix after the - drops: 124 rows per backend — vllm 97 / 9 / 18, sglang 93 / 6 / 25, - trtllm 69 / 0 / 55. - DECISIONS CLOSED 2026-09-25 (tianhaox): the NVFP4 DSA artifacts on - trtllm rc23 / SM90 stay modeled as bf16-KV (rc23's sparse FlashMLA - asserts bf16 KV; the perf task keeps that dtype) — continue as is. - compute_scale rows are an approximation by design (governed exception - stands, no further follow-up). SDK modeling of the four remaining - unmodeled architectures is out of this work — see TODO.md. - DECOMPOSE (step 5, component landed 2026-09-25): every passing record - on the three pins decomposes with EMPTY residue after six glue rules - (torch xmma f32/tf32 GEMM, fa3 split-kv combine, vision-encoder - attention/conv glue, trtllm MoE tactic-profiler buffers, trtllm GDN - layout transposes). One identity fact surfaced: sglang 0.5.16 serves - MiniMax-M2 / M2.5 (block-fp8 checkpoints) through the Triton - _w8a8_block_fp8_matmul GEMM, while the collector's fp8_block gemm lane - is DeepGEMM (kernel_source_backends gemm_dtype fp8_block) — labeled - triton_block_fp8 so a gemm path_diff on those repos reads diverged - instead of hiding under the DeepGEMM label. No capture exists yet for - that cell; recorded, not graded. - KDA DECODE-BATCH CAP (owner decision 2026-09-25, "带kda的模型覆盖max num - seq到256"): vllm.rule caps max_batch_size at 256 for checkpoints with - linear_attn_config.kda_layers (generator commit a0820789; Kimi-K3 now - renders --max-num-seqs 256 --max-num-batched-tokens 5760). Re-probed - under the capped render in framework mode: moonshotai/Kimi-K3 and - nvidia/Kimi-K3-NVFP4 PASS on vllm 0.29.0 (prefill 63/70, decode 51/55 - kernels; identity KimiK3DeltaAttention + kda_decode_fusion_many_heads - + flash_kda prefill, MoE Mxfp4MoE->marlin / ModelOptNvFp4FusedMoE-> - marlin); the fp8-KV variants still fail at load with the known - "fp8 KV cache requires an fp8 prefill query" requirement (not - rendered). kda_ctx path_diff now grades against the framework-mode - record (default selection, --serving-raw dropped): ALIGNED. The old - 512 raws are kept in facts/k3_ab/pre_cap/. vllm matrix: 99 / 9 / 16. - NORMALIZER FACT (2026-09-25): "(anonymous namespace)::" was never - stripped, so every kernel defined in an anonymous namespace — ALL of - sgl_kernel's C++ kernels (store_kvcache, concat_mla_*, fused_*norm_rope*, - flash_c4/c128 DSV4 attention, topk_* indexer helpers, per-tensor fp8 - quant, act/silu) plus torch's at::native helpers and K3's KDA decode - kernel — collapsed into one bogus name "namespace" and was invisible - to the taxonomy, the identity column and the gates. Stripping it - exposed 26 sglang kernels, now labeled by role (flash_c4_*/flash_c128_* - = dsa_nsa attention; the DSA indexer top-k helpers dispatch by batch - — topk_small_batch_kernel in the probe's decode vs topk_main_kernel in - the collector's case — so they carry the sgl_kernel infra label and - are not name-graded). Every saved gate verdict stays aligned (61), - residue is empty on all pins, no pass row lost its attention identity. - applies_to: [zai-org/GLM-5.3-Flash, Qwen/Qwen3.8-27B, Qwen/Qwen3.8-Flash-Next, nvidia/Qwen3.8-2.4T-A95B-NVFP4, - deepseek-ai/DeepSeek-V4.1-Flash, meta-models/Muse-Glimmer-30B, baerquant/Inkling, Altworld/Hemmingway-1, - TokenRhythm/NeoHorse-1-9B, ornith-ai/Ornith-1.5-35B-A3B, openbmb/MiniCPM5-2B, - 'instrument:fetch_inputs', 'instrument:probe_driver', 'instrument:dummies', 'framework:vllm-0.29.0', - 'framework:sglang-0.5.16', 'framework:trtllm-1.3.0rc23', 'platform:sm90'] - codex_review_evidence_contracts_2026_09_25: - statement: | - Codex review of 17ee75c2 (workspace file "Review Comment Codex.md") - reproduced six ways the harness turned stale, missing or insufficient - evidence into "pass"; all six reproduced on HEAD 9d860284 and are - closed by contract changes, each with a regression test: - (1) TARGET-OP GRADING. path_diff graded a gate with no signal family - on ANY infra-kernel overlap, so a capture sharing only a quant helper - with serving "proved" a GEMM path (the committed fp8 GEMM verdict had - exactly that: only scaled_fp8_quant in infra_family_name_matches; - the real reason was the launcher filter dropping - vllm::cutlass_3x_gemm_sm90_fp8 — a kernel, not a launcher). grade() - now takes the gate's TARGET ROLES (taxonomy role axis; inferred from - the gate name, or --target-roles) and requires every collector kernel - of each target role to name-match a serving kernel of the same role; - a role the collector ran but serving did not, or a collector-only - backend family inside a target role, is diverged; auxiliary roles are - reported, never graded; explicit role-scoped equivalences only (GEMM - tile blobs with layout kept; sgl_kernel top-k small_batch/main batch - dispatch). - (2) PHASE. prefill/decode/profile-run orphans were one set, and span - attribution by NAME across phases hid a kernel from the phase whose - span did not cover it. Records carry kernel_phases (every device-table - kernel by phase) and orphan_phases; select_serving picks serving - evidence by the gate's phase; vllm api_trace ops carry phase None (was - a borrowed "decode"); trtllm's flat table stays unscoped and the - verdict says so (phase_scoped). Re-derivation under (1)+(2): 60 gates - aligned, 1 diverged — trtllm rc23 DeepSeek-V3 MoE routing: the - collector routes with customMoeRoutingKernel, serving with - deepseek_v3_topk_kernel + dsv3MinLatencyKernels::router_gemm_kernel - (a collector serving-parity finding, left red); the sglang DSV3.2 DSA - context gate needed a single-cell capture (the whole-sweep capture - unions short cells that run dense FA3 — opcov_sgl_dsa_ctx_s4096); - compute_scale moved to the explained set (governed approximation); - the two trtllm fp8-KV DSA gate lines are retired (rc23 cannot serve - fp8 KV for DSA — a recorded wall, not an alignment question). - (3) EXECUTION IDENTITY. The run id names the case only; a raw probed - under an older render counted as done. Every plan run now carries an - execution fingerprint (engine tokens from the render — first flag on, - deployment-wrapper values dropped —, dummy config, probe code, image, - kv); the queue writes it as a .fp sidecar next to the raw and skips a - run only when the sidecar matches; records carry evidence_status - current | stale | unverified (sidecar decisive; without one, vllm / - sglang raws are checked on the argv they recorded, trtllm on its - engine yaml semantically); the matrix is derived from the NEWEST plan - per cell, reads the record of ITS run id (never a sibling version's), - and shows stale evidence as a fail cell. First run over the archive: - sglang 216 current / 12 unverified (retired pins), trtllm 222 / 37, - vllm 121 current / 103 stale (mostly fp8-KV variant raws from before - the 64-alignment) / 98 unverified; exactly ONE current-pin matrix - cell rested on stale evidence — MiniMax-M3 on vllm, probed at - max-num-batched-tokens 6012 before the alignment rule, re-probed at - 6016 (the review's own counterexample, found by the mechanism). - (4) PLAN-RELATIVE PROGRESS. matrix_complete now requires every repo - the plan declares for (fw, version) to have a verdict (an empty - matrix is not complete; stale cells are not complete); - path_verdicts_aligned requires every gate the verdict scripts declare - to have a verdict naming this framework/version and reading aligned - (declared_gates parses components/captures/verdicts_*.sh); - component_pending never completes a step (implemented -> todo "no - completion predicate", unimplemented -> blocked); new_op_collector - steps got artifact predicates (family_observed / family_unit_defined / - family_collector_exists / family_gates_aligned) and onboard_model's - path_aligned became model_gates_aligned. A nonexistent op family now - shows four todo steps, not four done ones. - (5) ONE CODE SOURCE. Queues invoke the probes from this checkout - (components/probes, visible under the workspace mount or mounted at - /harness); build_images.sh builds venv_ais from the checkout; the - workspace holds data only (configs, dummies, archive with raw + .fp, - facts, jitcache). The workspace's twin probe scripts are retired. - (6) THIRD_PARTY_NOTICES root copy synced (check_packaged_legal_files - passes). - FOLLOW-THROUGH (same day): the 102 vllm fp8-KV variant raws that read - stale (probed before the 64-alignment) were re-probed through the new - queue lines — every record on the three pinned versions is now - 'current' (remaining 'unverified' = retired pins only); the six sglang - 0.5.16 customizations (DSV4 moe_backend=marlin) were A/B'd bare in a - separate workspace and all still_needed (results/retests/sm90/ - sglang-0.5.16.yaml); the clean-checkout acceptance ran end to end in a - fresh workspace (build_images.sh -> venv_ais -> fetch_inputs -> - dummies -> emit with the /harness read-only mount -> probe -> record - 'current' -> matrix pass for Qwen3-0.6B). Harness unit tests: 73. - EVIDENCE LAYERING (owner decision 2026-09-26, "facts 太多进 repo 不是好 - practice"): the PR had 33k lines of kernel-level decompositions and - 5k lines of verdict kernel lists committed as results. Consumers were - checked: the workflow predicates read role/backend SETS, residue and - verdict/framework/version only; kernel names are debugging evidence. - decompose.py now commits a summary (role -> backend -> count, residue, - ops, record id) and writes kernel names to /archive/evidence/; - path_diff commits a verdict summary (both inputs' identity: record id, - exec fingerprint, capture sha256; roles, phase, per-role counts; the - deciding names only when red) and writes the full report to the - evidence dir; evidence_bundle.py packs a campaign's raw + .fp, - records, plans, rendered engine configs, captures and full reports - into one content-addressed tar.gz (first bundle 2026-09-26_sm90: - 856 raws, 6.2 MB) indexed in results/evidence_index.yaml with sha256 - and location. Committed results shrink from ~44k to ~9k lines; LFS - was rejected for the evidence (it hides content without making it - reviewable); the bundle's storage location is the owner's call and - is recorded per campaign. - applies_to: ['instrument:path_diff', 'instrument:probe_driver', 'instrument:workflow_check', - 'instrument:build_images', 'framework:vllm-0.29.0', 'framework:sglang-0.5.16', - 'framework:trtllm-1.3.0rc23', 'platform:sm90', MiniMaxAI/MiniMax-M3, deepseek-ai/DeepSeek-V3, - deepseek-ai/DeepSeek-V3.2] vllm_0300_upgrade_2026_09_27: statement: | Owner decision 2026-09-27: move the vllm pin from 0.29.0 straight to @@ -1564,287 +320,6 @@ findings: applies_to: ['framework:vllm-0.30.0', 'instrument:probe_driver', 'instrument:path_diff', 'instrument:workflow_check', 'collector:vllm', 'platform:sm90', deepseek-ai/DeepSeek-V4.1-Flash, nvidia/Qwen3.8-2.4T-A95B-NVFP4, moonshotai/Kimi-K3, deepseek-ai/DeepSeek-V4-Flash, nvidia/Nemotron-H-56B-Base-8K] - vllm_0300_full_op_sweep_2026_09_28: - statement: | - Owner request 2026-09-28: sweep every vllm op except gemm on 0.30.0 - (H20, SM90) and look for new large-scale failures. Code frozen at - 863bfc75 (workspace staging_0300), one container per GPU, ops split - by --case-filter as GPUs freed; ~22 h wall, ~48 GPU-hours, 23 tables - finalized (evidence: workspace facts/opsweep_full_0300/{TRIAGE.md, - SUMMARY.txt,PARTITION.txt,logs/}, parquet under staging_0300/runs/). - VERDICT: no deterministic 0.29.0 -> 0.30.0 regression. Row parity with - the 0.29 sweep on every table except where 0.30 is BETTER: - context/generation attention +697/+928 rows (the fp8-KV x head512 - Gemma-4 cells now run vllm_triton_attn, the head_dim>256 + fp8 fix - verified at full grid); dsa_generation +406 rows and 0/17664 failures - (the 0.29 sparse-MLA decode IMA race is gone); moe +27 rows (the - Llama-4 fp8 128E power_law_1.01 OOM seen in the sweep reproduced - clean 3/3 on an idle GPU -> worker-history transient, not a - regression). Every other failure cluster matched 0.29 cell by cell: - msa_generation 1472 (heads 48/96 kernel-limit), gdn 5 (grid-y guard + - one OOM cell empty on 0.29 too), moe 40 (fp8_block block_n / K3 mxfp4 - tp32, no 0.29 rows either), dsv4 csa_context 16 + hca_context 4 (the - 0.29 residual IMA family and the same aten::new_empty cells at 262144 - tokens per forward on fp8 DSV4-Pro). NEW on 0.30 with no baseline: - kda full grid 12 tasks, 4 failed on Kimi-K3 TP1 shapes — decode - bs1024 hits the CUDA grid-y limit (fused_recurrent.py:645 grid = - (cdiv(V,BV), B*H); the kda collector lacks gdn's classified guard), - three context OOMs on the batch x heads = 1536 boundary at isl 32768 - (no generation-time memory filter in the kda context grid). - COLLECTOR DEFECT FOUND (mine): the msa index-head kernel-limit guard - of 2026-09-22 keyed on the head count alone and rejected 1808 - msa_context prefill rows the 0.29 sweep had collected — the limit is - the DECODE scorer's (0.29 evidence: context failed only at isl=1, - 904/976 rows per head otherwise). Fixed in c7269391 - (msa_decode_scorer_kernel_limit, query_len-scoped) and re-run: 904 + - 904 rows recovered, exactly 144 isl=1 cells classified. FOLLOW-UP - (owner 2026-09-29): MSA is one fixed geometry (M3 64q/4kv/4idx, - GQA-style sharding, index heads = kv heads per rank), so the shared - 12-value head grid was replaced by the model row's declared shards — - tensor_parallel_sizes [1,2,4,8,16] -> heads 64/32/16/8/4 - (a22e4553; case_generator.get_msa_head_counts). msa_context 11712 -> - 4880 cases, msa_generation 8826 -> 3680; the 128/96/48/24/12-head - rows were unreachable points and, per the owner ("数据别放这种临时 - 的了"), the 0.30 msa tables were re-collected on the new axis: - 4880 + 3680 rows, zero failures; the sweep's shared-grid msa tables - are superseded (workspace staging_0300/superseded/). dsa_context - grew from 22640 to 169512 rows (the 2026-09-22 chunked-prefill prefix - axis: 2 models x 4 combos x 12 heads x 1816 tuples = 174336 cases); - 4827 tasks failed, 2.7% deterministic ModelConfig rejects (prefix - 262144+ over DeepSeek-V3.2's 163840 context) plus ~20 largest-shape - OOMs. OWNER DECISION 2026-09-28: the dsa grid stays as declared - (over-collection + classified failures). Lessons: split large-grid - ops by (model x precision combo) from the start; the case order is - model-major; a producer's checkpoint can be finalized after an early - stop by a zero-work --resume with the slice's own filters. - applies_to: ['framework:vllm-0.30.0', 'collector:vllm', 'platform:sm90', 'instrument:opsweep', - meta-llama/Llama-4-Maverick-17B-128E-Instruct, moonshotai/Kimi-K3, MiniMaxAI/MiniMax-M3, - deepseek-ai/DeepSeek-V3.2, zai-org/GLM-5, sgl-project/DeepSeek-V4-Pro-FP8, google/gemma-4-26B-A4B] - vllm_0300_gate_set_restored_2026_09_29: - statement: | - Owner question 2026-09-29 ("msa 的 kernel 和 probe 的 align 吗") exposed - that the vllm 0.30.0 gate set I rebuilt kept only 19 of the 32 gates - the 0.29.0 set had — attention ctx/gen (Llama-3.1-8B), encoder_attn - (Qwen3-VL-2B), dsa ctx x4 + gen (DeepSeek-V3.2), gemm m=1 x2, msa - ctx/gen (MiniMax-M3) and kda decode were silently missing, and - path_verdicts_aligned reported 19/19. Restored: every family - re-captured in the 0.30 image with the current collectors and graded; - RESULT 32 declared gates, 32 ALIGNED (msa ctx 3/3 and gen 5/5 attention - kernels matched; dsa fp8 isl-512 graded against a dedicated isl-512 - serving raw of the same record as on 0.29, because isl 512 is below the - sparse threshold and runs dense FA3). Two deviations are EXPLAINED, not - gates: mhc (collector measures the no-norm mhc_pre variant by - producer/consumer contract — the SDK bills attn_norm separately, - collect_mhc_module._mhc_pre — while 0.30 serving selects the - with_norm big_fuse variant via MhcPreBigFuseTileLangKernel.CompileKey - .use_norm_weight) and compute_scale (governed approximation). kda - decode: the collector measures both the fused non-spec kernel and the - packed conv-update + recurrence pair (serving's spec-decode path); the - gate captures a non-spec probe, so the capture sets - AIS_KDA_DECODE_PATHS=fused (collect_kda._decode_paths, recorded in - capture_env) and the fused path is ALIGNED with - kda_decode_fusion_many_heads_kernel. MECHANISM: workflow_check gained - gates_cover_registry_ops (upgrade_op step gates_declared before - path_aligned): every backend registry op must map to a declared gate - family (_OP_GATE_FAMILIES) or be listed gate-free with a reason; an - unmapped op is a finding. It flagged mhc_module immediately (now - explained-by-contract). Lesson: a gate set is checked against the - registry, never against what the script happens to declare. - applies_to: ['framework:vllm-0.30.0', 'instrument:workflow_check', 'instrument:path_diff', 'platform:sm90', - MiniMaxAI/MiniMax-M3, deepseek-ai/DeepSeek-V3.2, moonshotai/Kimi-K3, sgl-project/DeepSeek-V4-Flash-FP8, - meta-llama/Meta-Llama-3.1-8B, Qwen/Qwen3-VL-2B-Instruct] - blackwell_handoffs_absorbed_2026_09_30: - statement: | - Three off-box campaigns handed back to the H20 harness owner on - 2026-09-30 — B300/sm103 (0.29.0, 2026-09-20), B200/sm100 (0.30.0, - 2026-09-29, 109/1/14, 32 gates 27 aligned) and two sm120 boxes (RTX PRO - 6000 96GB: 101/7/16, 30 gates 29 aligned; RTX PRO 5000 72GB: 100/1/23, - 31 gates 29 aligned + 2 platform floors). ABSORBED into pr-228 here: - the RTX 5000 bundle as delivered (patch + captures + sm120 results, - taxonomy_sm120, 8 sm120 findings, gemm_fp8 fp8_static governed - exception per the owner's 2026-09-30 decision, verdict script - parametrized by AIS_SM, --matrix single-backend + writes to the harness - results dir, sm120 fail causes, kv 'auto' declared on bf16 gates, - gemvx/cutlass hints); plus, re-implemented from the B200 / RTX 6000 - notes whose patches are not on this box: render_golden runs the - generator with cwd = workspace (aic-core safe_mkdir; a workspace on - another mount rendered 124/124 as generator rejects on B300 and B200), - path_diff prefers the RENDERED (kv None/auto) record when no - --kv-dtype is asked, and per-SM platform-floor gate markers - (FLOOR_SM= / FLOOR_NOTE= on a verdict line -> platform-floor verdict - file, excluded from path_aligned on that SM, still declared coverage): - sm120's two fp8-KV dense-MLA gates are floors (TRITON_MLA smem), so the - sm120 upgrade_op view is all done honestly and the sm90 view stays - 32/32. The 5000 bundle's platform pin (rtx5000_sm120) was NOT taken — - targets.platform stays the H20. NOT YET ABSORBED (need the patches or - real work): B200's collector fix for the block_size = 64 literal in - collect_attn.py / collect_mla_module.py (ask the framework per case; - re-grade attn/mla gates on sm90 AND sm100), kernel_taxonomy_sm100.yaml - seed and the b200-opharness-fixes.patch (on umb-b200-238), RTX 6000's - platform-scoped cli_extra_args and no-baseline fails_root_caused - semantics (sm120-opharness-fixes.patch on the 6000 box), - fetch_inputs gated-repo mirror fallback (both), identity.attention - column via taxonomy backend labels, platform-blind run ids, the sm120 - generator rule (bf16 KV for NVFP4 MLA on CC 12.0) and the silently - ignored --generator-set kv_cache_dtype (generator module, separate PR). - applies_to: ['platform:sm120', 'platform:sm100', 'platform:sm103', 'framework:vllm-0.30.0', - 'instrument:probe_driver', 'instrument:path_diff', 'instrument:workflow_check'] - taxonomy_dsa_nsa_family_covers_m3: - statement: 'RESOLVED 2026-09-23: M3 sparse-attention kernels now label `triton` (the vocabulary name of - MiniMaxM3SparseTritonImpl), dsa_nsa is DeepSeek-DSA only; see probe_prefill_prefix_cache_contamination. - Original: NAMING DEBT: the kernel_taxonomy family "dsa_nsa" matches MiniMax-M3 sparse-attention kernels - too (sparse_attn_fwd/_gqa_share_sparse/_merge_topk_attn_out/_decode_score are models/minimax_m3 Triton - kernels, ops/sparse_attn.py). Verdicts are computed within-family by kernel-name overlap so the lumping - is functionally harmless today, but the family name misleads readers of verdict files. Rename (e.g. - block_sparse_attn) or split when the owner next touches the taxonomy.' - applies_to: - - instrument:kernel_taxonomy - - MiniMaxAI/MiniMax-M3 - - deepseek-ai/DeepSeek-V3.2 - cross_sm_evidence_contamination: - statement: 'HARNESS STRUCTURAL LESSON (B300 first run outside H20, 2026-09-20): gate evidence keyed by - (fw, version) alone let a fresh arch INHERIT another arch''s green checks — with zero B300 jobs run, - workflow_check --param sm=sm103 reported path_aligned (7 sm90 verdicts) and customizations_retested (9 sm90 - records), and saving B300 verdicts would have OVERWRITTEN the sm90 files. Same class: kernel_taxonomy was a - de-facto SM90 vocabulary (Blackwell fmhaSm103a*/fmhaSm100f*/sm100 cutlass/deepgemm-sm100/fp4 kernels all - unmatched -> attn path_diff went "empty vs empty" ALIGNED, a false green; missing serving kernels made dsa - falsely diverged); per-checkpoint customizations keyed (repo, fw) carried a 143GB-H20 memory workaround to a - 275GB B300. Fixes landed on H20: results/pathdiff//-/ + results/retests//-.yaml - (sm90 evidence git-mv''d; sm103 view now shows honest todos), and path_diff emits "inconclusive" when the - collector signal set is empty (B300 patch, pending). Rule going forward: SM is a first-class dimension of - gate evidence, taxonomy rules (arch-anchored + audited against other arches'' records before merge), - customization facts, and case plans; editing a shared file re-opens verdicts for every SM with shipped data - (unverified_sms when no box is available). Collectors do NOT fork per arch — dispatch stays the framework''s.' - applies_to: - - instrument:workflow_check - - instrument:kernel_taxonomy - - instrument:path_diff - - platform:sm90 - - platform:sm103 - vllm_0290_full_op_sweep: - statement: 'FULL-GRID VALIDATION of the in-place 0.29 upgrade (H20, 4 GPUs, 2026-09-20/21, ~290k perf rows, - 15 op tables finalized to parquet; evidence workspace facts/opsweep_full_0290/ incl. TRIAGE.md + staging_patch.diff - — the staging rehearsal that became commit 993f9cdf). Row parity vs 0.24/0.25 baselines: gemm 111444 (exact), - attention ctx/gen 53777/66708 (exact), computescale/scale_matrix/encoder/mla_bmm exact, msa+gdn above baseline, - moe 58k. ALL four failure clusters root-caused, ZERO 0.29 regressions and ZERO upgrade-code bugs: (1) msa - Triton CompilationError 1616x = kernel domain boundary — sparse_num_index_heads shards to non-power-of-2 at - collector heads 48/96, unreachable in serving (real tp gives 4/2/1); (2+3) attention fp8-KV 1625x = HISTORICAL - blank: 0.24/0.25 baselines contain zero fp8 rows (100% bfloat16), the grid always swept an unsupported region, - only the error text changed; (4a) dsa ctx fp8-KV 776x = flashmla int64_stride_to_int input constraint on the - NEW fp8 collection key — FIXME(kernel-limit) class; (4b) dsa gen IMA ~4% = RACE, audit complete - 2026-09-22: bisect exonerated the block-table fix (22 vs 19 fails on identical 300 cases) AND cudagraph - (eager 21/300) AND the topk buffer (allocation parity with serving — both torch.empty; poison prefill 2e9 - and -1 both pass); CUDA_LAUNCH_BLOCKING pins the faulting kernel to vllm-flash-attn Hopper flash_fwd - (flash_fwd_launch_template.h:205); compute-sanitizer memcheck serializes and the fault VANISHES (8/8, 0 - errors) — a timing race in the 0.29 FA3 sparse-MLA decode chain, worth reporting upstream. Recovery is - operational: failures are stochastic with ~75%/round retry recovery, geometric to near-zero - (GLM gen 300->70->17->5->2 of 8832; DeepSeek 346->106->28->8) — --resume-retry-failed sweeps recover the - rows without touching measurement semantics. ADDENDA: (2+3) VERDICT FLIPPED AGAIN 2026-09-22 via live-engine - probe — collector-specific, NOT a framework selector gap. head_dim=512 dense attention comes only from - Gemma-4 (global layers: global_head_dim=512, num_global_key_value_heads=2). Real vllm serving of Gemma-4 with - --kv-cache-dtype fp8 on SM90 GENERATES fine: Gemma4Config.verify_and_update_config - (model_executor/models/config.py:209-256) is a model-level hook that, for heterogeneous head dims (256/512), - forces FA4 for all layers (or TRITON_ATTN fallback) because default FA3 on Hopper cannot handle head_dim>256 - — so serving NEVER routes Gemma-4 head512 to FlashInfer (resolved backend TRITON_ATTN, candidate list - [TRITON_ATTN]). collect_attn constructs the attention in isolation and calls the bare platform selector - (get_attn_backend_cls) WITHOUT running that model hook, so it picks FlashInfer whose SM90 JIT cannot compile - fp8+head512 -> the 1625 dsa... i.e. attention fp8 failures. Same class as DSA 4a / Gemma monolithic: the - collector reproduces a backend serving never selects. Fix: the collector must select the backend serving - actually uses for head_dim>256 (apply the Gemma4 heterogeneous-head override: FA4/Triton, never FlashInfer), - with the config.py:209-256 citation. Evidence facts/gemma4_fp8_probe (GENERATE OK, TRITON_ATTN). (4a) VERDICT FLIPPED 2026-09-22 via live-engine probe: - collector-specific, NOT a framework limit. fp8-KV + num_heads<32 (the collector head-sweep emulating TP>=5) - selects fp8 mixed mode (MIN_HEADS_FOR_BF16_PREFILL=32, flashmla_sparse.py:63/307), which runs the whole batch - as one MQA batch through the fp8 decode kernel (head padding 16->64); the collector builds the prefill as a - SINGLE monolithic forward so num_actual_tokens=isl(32768) and stride = tokens*padded_heads*576 breaks int32. - Real-engine tests (8 GPUs idle): TP=1 native-128-heads isl=32k fp8 runs (separate bf16 splitkv path); TP=8 - (16 heads/rank -> the SAME mixed fp8 decode kernel, padding 16->64) fed a 32768-token prompt -> GENERATE OK. - Serving chunks the prefill (max_num_batched_tokens), keeping per-forward num_actual_tokens well under 29k; - the collector does not chunk. Fix: the collector context construction should mirror serving''s chunked prefill - (split at max_num_batched_tokens) -- this both removes the 776 failures AND makes large-isl context rows - faithful to how serving actually invokes the kernel (the monolithic measurement is itself non-faithful for - large isl; even the passing large-isl context rows warrant review). Evidence - facts/ab_dsa_stride_{oneshot,tp8}.json. - (1) msa kernel limit converted to an invocation-site guard raising a classified error with the framework - citation (commit c0f7f035). dsv4 full-grid rerun on the merged lane, FINAL: - 22589/22604 collected (99.93%) across 6 ops — generation 3088/3088, paged_mqa_logits + hca_attn 3918/3918 - each, hca ctx 5840/5840 after retries; the sole residual is a DETERMINISTIC 15-case subcluster in csa ctx, - all one shape family (seq=16 chunk + batch=64 + fp8-KV + prefix=4096, both FP8 checkpoints, heads 1-8) — - the fp8 csa large-prefix small-tail-chunk path; classified, re-tested free on the next version bump. - Harness lessons banked: PerfFile.SCALE_MATRIX enum gap (fixed), measurement containers must never share host - network (setup_distributed port collisions), collection images need deps pre-baked, executor ignores --ops - CLI order, sharding is manual via --case-filter.' - applies_to: - - framework:vllm-0.29.0 - - platform:sm90 - - instrument:collect.py - sm120_vllm_0300_campaign: - statement: 'FIRST SM120 CAMPAIGN (RTX PRO 5000 Blackwell 72GB x8, compute capability 12.0, 2026-09-30; vllm 0.30.0, - golden renders with --system rtx_pro_6000_server as the 5000 proxy — no SDK entry for the 5000). 124 roster cells: - 100 pass / 1 pass+custom / 23 fail (12 generator rejects = the same SDK-gap repos as sm90; 4 platform limit; - 2 platform gap; 5 capacity). Deployed identity on sm120 differs from sm90 in every family: dense attention - = FA2 (flash::flash_fwd_splitkv_kernel; FA3 is sm90-only) with FlashInfer paged prefill + XQA decode under - fp8 KV; MLA = TRITON_MLA (prefill FA2 hd192, decode _fwd_grouped_kernel_stage1); sparse MLA (DSV3.2/GLM-5.2/DSV4) - = FLASHINFER_MLA_SPARSE_SM120 / DeepseekV4FlashInferSM120Attention (flashinfer::sparse_mla_sm120::*, - sparse_mla_prefill_mg*); fp8 linears AND fp8 MoE = DeepGEMM sm120 lane (deep_gemm::sm120_fp8_fp4_gemm_1d1d_impl - + ep_scatter/gather glue) instead of cutlass/triton_fused_moe; bf16 MoE (GLM-5.2, MiniMax-M3, Qwen1.5-MoE) = - FlashInfer CUTLASS fused MoE (fused_moe::run_global / MoeFCGemm) instead of triton fused_moe_kernel; M=1 - GEMMs = cuBLAS gemvx instead of nvjet; per-tensor fp8 (ModelOpt) decode GEMMs = cuBLAS sm89_xmma_gemm_e4m3. - kernel_taxonomy_sm120.yaml was seeded from sm90 and grew 25 evidence-based rules (every rule names a kernel - recorded on this box); decompose residue = 1 kernel (the DSV4 cute-DSL kernel_cutlass_device_kernel, same - owner-decided residue as sm90). Gates: 29 aligned, 1 explained, 2 ungradable; the 2 DeepSeek-R1 fp8-KV MLA gates have no serving - record (see sm120_triton_mla_fp8kv_smem); gemm_fp8 (dynamic-activation row, no faithful serving reference) is an explained deviation under an owner exception (sm120_collector_gemm_fp8_static_row); - compute_scale and mhc stay explained deviations as on sm90. Evidence bundle 2026-09-30_sm120_vllm-0.30.0 - (results/evidence_index.yaml).' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - sm120_triton_mla_fp8kv_smem: - statement: 'PLATFORM LIMIT (vllm 0.30.0 on sm120): every MLA checkpoint that resolves an fp8 KV cache dies at - CUDA-graph capture with triton OutOfResources "shared memory, Required: 102400, Hardware limit: 101376" in - TRITON_MLA decode (_decode_grouped_att_m_fwd, v1/attention/ops/triton_decode_attention.py:534; BLOCK_DMODEL 512 - + BLOCK_DPE 64, num_stages 2 — the guard at :524-532 only drops num_stages for BLOCK_DMODEL >= 1024). The sm120 - opt-in smem/block is 101376 B (torch.cuda.get_device_properties inside the image), 102400 B/SM. Hit by the fp8-KV - probe sweep of DeepSeek-V3, DeepSeek-R1, Kimi-K2-Instruct, Kimi-K2.5 (bf16-KV cells of the same repos PASS, - identity TRITON_MLA) AND by the RENDERED config of the NVFP4 artifacts whose hf_quant pins fp8 KV - (nvidia/DeepSeek-V3.1-NVFP4, Kimi-K2.5/K2.6/K2.7-Code-NVFP4: matrix fail cells). The collector''s mla_gen_fp8 - capture crashes identically (observe-don''t-predict: same dispatch, same limit), so the mla_ctx_fp8 / mla_gen_fp8 - DeepSeek-R1 gates have no serving record on this SM and cannot be graded. Not a generator defect: `cli generate` - renders no kv dtype; the crash is the framework''s only MLA decode backend on major==12 (platforms/cuda.py:132-135 - lists TRITON_MLA + FLASHINFER_MLA_SPARSE_SM120). A deployment of those NVFP4 checkpoints on RTX PRO 5000/6000 - with vllm 0.30.0 does not boot; upstream-report candidate (reduce num_stages/BLOCK for fp8 K on sm12x). - Evidence: archive/raw/{175d26fcbb87,c78550b69fb3,c73c64978344,f6e846bb5051,f157c1c39890,691d6cce3327, - 9c912acc9024,c91c53ea605c}.json, facts/pathdiff/opcov_mla_gen_fp8@0.30.0.json (RUN ERROR).' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - deepseek-ai/DeepSeek-V3 - - deepseek-ai/DeepSeek-R1 - - moonshotai/Kimi-K2-Instruct - - moonshotai/Kimi-K2.5 - - nvidia/DeepSeek-V3.1-NVFP4 - - nvidia/Kimi-K2.5-NVFP4 - - nvidia/Kimi-K2.6-NVFP4 - - nvidia/Kimi-K2.7-Code-NVFP4 - sm120_dsv41_sparse_mla_no_decode_shape: - statement: 'PLATFORM GAP (vllm 0.30.0 on sm120), re-verified 2026-10-01 on the faithful dsv41 dummy (7-layer source+consumer - cut, upstream a04c849c) after the first verdict ran on the unfaithful 2-layer cut: DeepSeek-V4.1-Flash and - nvidia/DeepSeek-V4.1-Flash-NVFP4 cannot boot on sm120 under ANY kv block size. The KV block size is pinned from two - sides that disagree off Hopper: DeepseekV41IndexerBackend.get_supported_kernel_block_sizes() returns [64] only on - capability family 90 and [128] everywhere else (vllm/v1/attention/backends/mla/indexer.py:262-263), while the sm120 - sparse-MLA decode path (flashinfer 0.6.18 _sparse_mla_sm120.py: _DECODE_DSV4_PAGE_BLOCK_SIZE = 64; 0.7.0.post1 - _sparse_mla_sm120_plan.py: _PAGE_BLOCK_SIZE = 64) accepts page 64 only. Observed: default block -> DeepGEMM - "block_kv == 32 or block_kv == 64" (attention.hpp:409) from the indexer metadata build; --block-size 64 (the V4.1 facts - value that rescues H20/sm90) -> "No common block size for 64" (worker/utils.py:386, select_common_block_size); the - original 2-layer run showed the third face of the same conflict ("no decode kernel for this shape ... page_block_size=32"). - Not config-rescuable; needs a vllm change (indexer block 64 when the sm120 FlashInfer sparse backend is selected). - The latest vllm RELEASE is still 0.30.0 (PyPI 2026-10-01), so no released vllm runs V4.1 on RTX PRO / sm120; vllm main - not probed. DeepSeek-V4-Flash/Pro (indexer block 256, no V4.1 indexer class) PASS on sm120. A/B evidence: - facts/ab_sm120/{32100c75d1df,c78c20c47673}_bs64.json; stale goldens/dummies in archive/superseded_*/2026-10-01_*.' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - model:deepseek-ai/DeepSeek-V4.1-Flash - - model:nvidia/DeepSeek-V4.1-Flash-NVFP4 rtx_systems_have_no_generator_facts_hardware_profile: statement: 'GENERATOR GAP (found while re-probing V4.1 on sm120, 2026-10-01): model-default cli args from generator/facts/models.yaml (e.g. V4.1 block-size 64, trust-remote-code, no-enable-flashinfer-autotune; V4 block-size 256) @@ -1862,109 +337,6 @@ findings: - platform:sm120 - instrument:generator - instrument:probe_driver - sm120_capacity_72gb: - statement: 'CAPACITY (72GB probe GPU vs 143GB H20): Qwen/Qwen3.8-2.4T-A95B{,-FP8} and nvidia/Qwen3.8-2.4T-A95B-NVFP4 - OOM at load even on the depth4 cut (one MoE layer of the 2.4T model exceeds the card: 32 GiB single allocation - bf16), meta-llama/Llama-4-Maverick-17B-128E-Instruct OOMs on its smallest cut (depth4, 20 GiB expert tensor) — - "no faithful cut fits one probe GPU"; Qwen3.5-397B-A17B needed the depth8 -> depth4 fallback and passes. The - capacity fallback (probe_driver enumerate_runs) worked as designed but needs a second --emit-queues after the - first sweep (the OOM must be observed first) — documented, not automatic. Nemotron-3-Ultra-550B: all_kinds (3 - layers) OOMs; the fallback ranks single-kind cuts ALPHABETICALLY and picked layers_block_type_attention, which - passes for BF16/FP8 (identity: attention + no MoE column) but for the NVFP4 artifact strips every quantized layer - -> vllm rejects MIXED_PRECISION with an empty quantized_layers map (dummy artifact). Owner-side variant_override - to the MoE cut then fails the probe''s attention hook (no attention module in a MoE-only cut). Both are recorded - as capacity cells, not framework facts. Lesson: single-kind cuts are not identity probes; a 72GB box needs a - 2-layer mixed cut (attention+moe) variant for Nemotron-H-class models.' - applies_to: - - platform:sm120 - - Qwen/Qwen3.8-2.4T-A95B - - Qwen/Qwen3.8-2.4T-A95B-FP8 - - nvidia/Qwen3.8-2.4T-A95B-NVFP4 - - meta-llama/Llama-4-Maverick-17B-128E-Instruct - - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - sm120_gemm_fp8_kernel_level_drift: - statement: 'gemm_fp8 gate DIVERGED on sm120 (GEMM-class gates are graded on kernel-name overlap; the collector''s - one GEMM kernel is not among serving''s 14): the collector''s per-tensor fp8 case (Fp8LinearMethod -> CutlassFP8ScaledMMLinearKernel, op_smoke confirms the - selection) runs cutlass MainloopSm120TmaWarpSpecialized, while serving Llama-3.1-70B-Instruct-FP8 - (ModelOptLinearMethod, fp8 per-tensor weights) runs MainloopSm120TmaWarpSpecializedBlockwiseScaling for the - prefill GEMMs and cuBLAS sm89_xmma_gemm_e4m3 (torch._scaled_mm) for the M=1 decode GEMMs. Same sm90 pairing was - kernel-identical. The vllm gemm fp8 perf rows on sm120 therefore measure a different mainloop than the ModelOpt - checkpoints deploy; whether ModelOpt''s sm120 blockwise mainloop is a vllm dispatch quirk or the intended path - for per-tensor scales needs a collector-side serving-parity audit before the sm120 gemm data is trusted. - Evidence facts/pathdiff/opcov_gemm_fp8@0.30.0.json vs archive/raw/da1900656836.json.' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - nvidia/Llama-3.1-70B-Instruct-FP8 - harness_generalization_sm120: - statement: 'HARNESS LESSONS from the first campaign on a fresh SM + fresh workspace (2026-09-30), each fixed in - this checkout unless noted: (1) build_matrix aborted when ANY targets backend lacked a plan (single-backend - campaign impossible) -> now skips backends without a plan; (2) build_matrix wrote results// under the - WORKSPACE, so workflow_check (which reads the checkout) never saw the matrix -> now writes the harness results/; - (3) the taxonomy SM comes from the AIS_SM env var, not targets.platform (forgot once = sm90 vocabulary graded - sm120 records) — NOT fixed, documented; (4) verdicts_vllm_0300.sh hardcoded AIS_SM=sm90, the sm90 output dir - and a checkout-inside-workspace path -> SM-parametric + script-relative; (5) 9 of the 32 vllm gates (attn_ctx/gen, - dsa_ctx_*, dsa_gen_fp8, msa_ctx/gen) had NO committed capture recipe (produced ad hoc via op_smoke on sm90) -> - capture scripts added; kda_gen needed AIS_KDA_DECODE_PATHS=fused that only lived in a shell history -> now set - in the script; compute_scale capture had drifted from the registry signature (extra_perf_filenames) -> fixed; - (6) gates for bf16-KV models declared kv "" and path_diff then admitted the fp8-KV record of the same repo (newest - wins) -> attn/gemm_bf16 gates declare kv auto; op hints lacked gemvx (the M=1 cuBLAS gemv on sm120), which hid - serving GEMM evidence; (7) the matrix attention-identity heuristic matched the kv-cache glue - vllm::concat_and_cache_mla_kernel on TRITON_MLA records -> excluded, _fwd_grouped_kernel_stage1 added; (8) - fetch_inputs hard-stops on gated repos with no mirror fallback: meta-llama x5 and the gated Nemotron-3-Ultra-FP8 - were provisioned by hand from unsloth mirrors / SDK model_configs (configs/GATED_SOURCES.md in the workspace); - (9) the generator vllm.rule has no SM branch — sm90 and sm120 render byte-identical engine args, so every sm120 - fail above is a framework/platform fact, none a rendering fact. Data plane grew: kernel_taxonomy_sm120.yaml, - results/sm120/{vllm-0.30.0.yaml,decompose/}, results/pathdiff/sm120/vllm-0.30.0/ (30 verdicts), - results/retests/sm120/vllm-0.30.0.yaml, _fail_cause vocabulary for the sm120 causes.' - applies_to: - - platform:sm120 - - instrument:probe_driver - - instrument:path_diff - - instrument:workflow_check - sm120_config_ab_2026_09_30: - statement: 'CONFIG-OR-FRAMEWORK A/B for every non-SDK-gap fail cell on sm120 (owner question: is the rendered config - wrong, and does the framework hint at a backend/flag that makes it boot). Method: the rendered run.sh with ONE - extra engine arg appended, same dummy, same image, same probe (facts/ab_sm120/_.{sh,json}). - RESULTS. (a) NVFP4 MLA checkpoints (nvidia/Kimi-K2.5-NVFP4 691d6cce3327, nvidia/DeepSeek-V3.1-NVFP4 f157c1c39890, nvidia/Kimi-K2.6-NVFP4 9c912acc9024, nvidia/Kimi-K2.7-Code-NVFP4 c91c53ea605c): - + --kv-cache-dtype bfloat16 -> PASS on all four (identity TRITON_MLA bf16 KV, DeepGEMM sm120 linears). So the cell IS - config-rescuable: the checkpoint''s hf_quant pins fp8 KV, and on cc 12.0 the only non-sparse MLA backend that - declares support is TRITON_MLA (enumerated: FLASHINFER_MLA / CUTLASS_MLA / FLASHMLA / FLASH_ATTN_MLA / - TOKENSPEED_MLA all report cc12=False), whose fp8-KV decode kernel overflows smem — bf16 KV is the only route. - GENERATOR GAP: params.agg.kv_cache_dtype=bfloat16 renders `--kv-cache-dtype auto` (backend_config_mapping.yaml - kv_cache_dtype guard "vLLM doesn''t accept bfloat16") and auto re-resolves to the checkpoint''s fp8 -> the - rescue cannot be expressed through the generator today; vllm 0.30.0 accepts the literal bfloat16 (this A/B). - Bucket-1 candidate: drop/condition the guard, and an SM120 rule "MLA + quantized KV -> kv_cache_dtype bfloat16". - (b) DeepSeek-V3 / R1 / Kimi-K2 fp8-KV probe variants: + --max-num-seqs 64 --max-num-batched-tokens 4608 -> same - smem failure (kernel tile config, not batch); no alternative backend exists (see a). Not a config issue; the - rendered default (bf16 KV) passes. - (c) DeepSeek-V4.1-Flash (both repos): + --attention-backend FLASHINFER_MLA_SPARSE_DSV41 / TRITON_MLA, + --max-num-seqs - 32, + --block-size 64 / 128 / 256 -> all FAIL with the same "no decode kernel ... page_block_size=32". Root: the DSV41 sm120 - backend pins kernel block size [128] (models/deepseek_v41/nvidia/flashinfer_sparse.py:112) and computes the - compressed-layer page as block_size // compress_ratio = 128 // 4 = 32 (:759, :839), while FlashInfer''s sm120 DSV4 - decode requires page 64 (_DECODE_DSV4_PAGE_BLOCK_SIZE = 64; (64,128) IS in _DECODE_DSV4_DISPATCH). No engine flag - reaches that arithmetic -> framework gap, upstream-report candidate (vllm/flashinfer sm120 DSV4.1 decode). - (d) Kimi-K3 fp8-KV variant: + --attention-config {"use_prefill_query_quantization": true} (the flag vLLM''s own - assertion names) -> same AssertionError: on sm120 the prefill backend is FA2, which cannot consume fp8 queries - (mla_attention.py determine_prefill_query_data_type -> backend_supports_prefill_query_quantization False), so the - flag is silently ignored and the K3 assert fires. fp8 KV for Kimi-K3 is not reachable on sm120 by any flag; bf16 KV - (the rendered default) passes. Arch-neutral half of this is already in findings (0.29 K3 followups). - (e) nvidia/Nemotron-H-56B-Base-8K: bare render OOMs on 72GB too; + --max-num-seqs 64 (no gpu-memory-utilization) - -> PASS, confirming the root knob is max_num_seqs on sm120 as on sm90; the generator still floors max_batch_size - at 512 (vllm.rule) so the expressible rescue stays extra_cli_args gpu-memory-utilization 0.6 (retest: still_needed). - (f) capacity cells (Qwen3.8-2.4T x3, Llama-4-Maverick, Nemotron-Ultra-NVFP4 single-kind cut): dummy weights do not - fit 72GB — not a config question; no A/B.' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - nvidia/Kimi-K2.5-NVFP4 - - nvidia/Kimi-K2.6-NVFP4 - - nvidia/Kimi-K2.7-Code-NVFP4 - - nvidia/DeepSeek-V3.1-NVFP4 - - deepseek-ai/DeepSeek-V4.1-Flash - - nvidia/DeepSeek-V4.1-Flash-NVFP4 - - moonshotai/Kimi-K3 - - nvidia/Nemotron-H-56B-Base-8K sm120_collector_gemm_fp8_static_row: statement: 'COLLECTOR-SIDE DISPOSITION of the gemm_fp8 divergence (serving-parity audit + owner decision, 2026-09-30). Audit: vLLM 0.30.0 selects the fp8 scaled-mm kernel by activation scheme — FlashInferFP8ScaledMMLinearKernel is @@ -1993,77 +365,6 @@ findings: - platform:sm103 - framework:vllm-0.30.0 - instrument:collect_gemm - sm120_dsa_head_axis_failures_accepted: - statement: 'OWNER DECISION (tianhaox, 2026-09-30) on the sm120 vllm 0.30.0 collection: the dsa_context/generation_module - failures for per-rank head counts 96/48/24/12 (and 4/2/1) are ACCEPTED as classified failures — no sm120-specific - axis pruning, no skip. Facts: mla_module_vllm.head_counts is a shared grid [128,96,64,48,32,24,16,12,8,4,2,1]; the - 96/48/24/12 values are Kimi-K3 (96 heads) TP1/2/4/8 shards and are REAL for the dense mla module, but no DSA model - (DeepSeek-V3.2 128 heads, GLM-5 64 heads) produces them at any TP. sm90 FlashMLA-sparse and sm100 FlashInfer-sparse - pad heads, so those rows collect as fabricated configurations there; sm120 FlashInfer sparse-MLA supports only - h_q in {8,16,32,64,128} and refuses ("Unsupported sparse-MLA prefill configuration" / "no decode kernel for this - shape"), so on sm120 the rows fail at metadata build in ~0.3 s each (~37% of dsa_context cases together with the - prefix >= 262144 max_model_len group; ~40 min per 21.8k-case shard). Shipped vllm 0.24.0 dsa tables on every system - carry heads {8,16,32,64,128} (+1/2/4 on Hopper) only, so nothing regresses. Rationale: layer_permissions offers no - head-axis negative gate (max_sm_exclusive is quant-mode only) and failure_handling wants framework gaps to fail and - re-test on the next pin. FOLLOW-UP (low priority, arch-neutral, for the H20 owner at the next bump): derive the - mla/dsa head axis per model row from declared tensor_parallel_sizes like get_msa_head_counts does (mla keeps - {128,96,64,48,32,24,16,12,8} via Kimi-K3, dsa drops to {128,64,32,16,8}), and cap dsa_prefix_lengths by the model - row''s max_position_embeddings (262144/524288/1048575 exceed DeepSeek-V3.2 163840 / GLM-5 202752 on every SM).' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - instrument:collect_mla_module - sm120_vllm_0300_collection_2026_10_01: - statement: 'FULL PERF-DATA COLLECTION on the RTX PRO 5000 (sm120) box, vllm 0.30.0, --model-cases-full --sm 120: - 20 ops / 634,878 planned cases / 41 case-filter shards on 8 GPUs, ~12 h wall time incl. clean-GPU retries; - launcher = tools/perf_database/collect_campaign.py (written during this run, see aic-auto-collect skill). - DATA (rows, after retries): gemm 144,142 (bf16 37,472 / fp8 37,518 / fp8_block 31,598 / nvfp4 37,518); - attention_context bf16 27,218; attention_generation bf16 33,785; dsa_context 106,020 (12 x 8,835); - dsa_generation 10,862; mla_context 8,487; mla_generation 7,059; msa 7,320 + 5,511; dsv4 csa/hca ctx 5,718 / - 5,789, gen 1,544 / 1,544; encoder_attention 7,679; moe 76,947; compute_scale 3,256 (2 tables); mla_bmm 300 + - 336; gdn 10,284; kda 1,247. attention fp8-KV ctx/gen re-collected from scratch after the collector fix below: context 27,144 / generation 32,370 rows (labels fiprefill / trtllmapidecode_xqa / triton_attn), 90 + 1,448 XQA power-of-2 failures left; campaign total 524,972 rows. - FAILURE GROUPS, all classified, none hidden: (1) dsa heads 96/48/24/12/4/2/1 + prefix >= 262144 — owner - decision sm120_dsa_head_axis_failures_accepted (~60% of dsa_context cases, fail at metadata build); - (2) TRITON_MLA fp8-KV smem 102400 > 101376 (mla_generation 6,036; the platform limit from the probe campaign); - (3) fp8 prefill query on FA2 (mla_context compute=fp8 4,392: FA2 cannot take fp8 Q, collector refuses to - mislabel — declared min_sm 100 but SM120 is >= 100 by number while lacking the SM100 path: the combo axis - needs a family gate, not a floor); (4) sparse-MLA sm120 decode shape table (dsa_generation 9,996); - (5) gemm fp8_block: cutlass c3x "Invalid status" 4,491 (documented SM120 gap) + triton_scaled_mm scale-shape - assert 546 at N in {4, 8}; (6) XQA JIT static_assert isPowerOf2 for MQA group size 48/24 (attention fp8-KV, - n_kv=1, ~1,450 cases; the same shapes IMA under bf16 FA2 decode: 3 cases); (7) large-shape CUDA faults — - dsv4_csa_context b*s = 262144 fp8-KV (IMA / capture "unjoined work" / aten::new_empty ~50), mla_generation - b*s = 16.7M tokens (3 IMA per gemm type; int32-index class); (8) moe: NVFP4 w1/w3 intermediate padding - unsupported 69, gate/up not divisible by block 128: 36, inter_size % alignment 3; (9) gdn packed recurrent - grid-y limit 4, kda strict-completeness 1; (10) residual OOMs: 30 -> 19 (attention_generation bf16), 16 -> 5 - (attention_context bf16), 46 -> 10 (gemm bf16 32768 x 65536 x 51200-class, 6 weight copies ~40 GB) after - a clean-GPU --resume-retry-failed pass; the ones that reproduce are genuine 72 GB capacity limits. - COLLECTOR FIXES that came out of it (branch tianhaox/aisimulate:sm120-collect-campaign): collect_attn lays - the paged KV cache out in the framework-resolved kv_cache_layout (NHD on FlashInfer outside SM100; the fixed - HND memory failed every fp8-KV GQA decode on sm120, 20,458 + 1,354 cases, while serving passed) and the - FlashInfer decode kernel_source names the kernel (…trtllmapidecode_xqa / _trtllm_gen; 0.30 merged both into - one metadata class); collector/helper _merge_perf_rows treats string/large_string as one type (the third - finalize of a table used to OVERWRITE it — 27,303 rows silently replaced by 5,067; mechanism change flagged - for owner review). DELIVERY BLOCKER: no SDK system definition for the RTX PRO 5000 (playbook §8 forbids the - rtx_pro_6000_server relabel); data stays in the workspace until systems/.yaml exists.' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - instrument:collect.py - - instrument:collect_campaign - sm120_dsa_kv_dtype_axis_collapses: - statement: 'DSA module rows on sm120 carry kv_cache_dtype=fp8 for BOTH declared KV variants: vLLM 0.30.0''s - FLASHINFER_MLA_SPARSE_SM120 path resolves the cache to fp8_ds_mla regardless of the requested dtype (opharness - serving probes of DeepSeek-V3.2 and GLM-5.2: kv None and kv fp8 both -> cache_config_dtype fp8_ds_mla), and the - collector records the RESOLVED dtype (serving truth). Consequence: the bf16-KV and fp8-KV shards of dsa_context / - dsa_generation produce identical identity keys (8,835 of 8,835 per shard pair, latency ratio 1.01), so half of - the sm120 DSA collection (~4 h x 8 GPUs) re-measured the same physical configuration; the merged tables hold - 53,010 + 5,432 unique rows (106,020 + 10,862 collected). Not a data defect — the rows are truthful — but the - case plan''s kv axis is not a real axis for DSA on this SM. Candidate: dedup DSA cases on the resolved kv dtype at - plan time (the collector already collapses physical keys for mla specs), or declare the DSA kv axis per SM.' - applies_to: - - platform:sm120 - - framework:vllm-0.30.0 - - instrument:collect_mla_module dsv41_dummy_cut_and_vllm_block_size_2026_10_01: statement: 'CORRECTION of the DeepSeek-V4.1 rows in vllm_0300_upgrade_2026_09_27 and of the 09-30 support probe: the V4.1 dummy those verdicts ran on was not V4.1. The generic depth cut kept original layers [0,1] — two sliding-window @@ -2120,122 +421,6 @@ findings: - framework:sglang-main-2026-09-30 - instrument:dummies - platform:sm90 - dsv41_generator_profile_block_size_2026_10_01: - statement: 'FOLLOW-UP to dsv41_dummy_cut_and_vllm_block_size_2026_10_01 (owner request: fix the generator default - instead of carrying a harness override). facts/models.yaml now has a `deepseek-v4.1` profile (match - deepseek-ai/DeepSeek-V4.1*, nvidia/DeepSeek-V4.1*; arch DeepseekV41ForCausalLM; model_type deepseek_v41) whose - vllm defaults are trust-remote-code + block-size 64 + no-enable-flashinfer-autotune and whose sglang defaults - equal V4''s; the `deepseek-v4` globs narrowed from `DeepSeek-V4*` to `DeepSeek-V4-*` so V4.1 no longer inherits - V4''s block-size 256. V4''s gb200-specific defaults (deep_gemm_mega_moe, compilation-config) were NOT copied — - they are validated for V4 only. Re-probed on H20 with the native render (block-size 64 for BOTH V4.1 artifacts, - no harness cli_extra_args): vllm 0.30.0 rendered + fp8-KV PASS x4; sglang main (nightly 0930) PASS x4 (the - nvidia artifact now also renders trust-remote-code / disable-flashinfer-autotune). Matrix vllm-0.30.0 back to - 110/1/13 with deepseek-ai/DeepSeek-V4.1-Flash a plain pass. Unit test - tests/unit/generator/test_model_profile_deepseek_v41.py pins both the glob split and the 64/256 defaults. - Still open on the generator side: nvidia/DeepSeek-V4-*-NVFP4 artifacts of V4 itself are matched by no profile - (deliberately left alone — changing it would alter V4-NVFP4 golden renders that currently pass).' - applies_to: - - deepseek-ai/DeepSeek-V4.1-Flash - - nvidia/DeepSeek-V4.1-Flash-NVFP4 - - module:generator - - framework:vllm-0.30.0 - - platform:sm90 - hopper_trtllm_upgrade_rc23_to_rc29_2026_10_01: - statement: 'HOPPER UPGRADE trtllm 1.3.0rc23 -> 1.3.0rc29 (H20/sm90, 2026-10-01; latest nvcr tag, no 1.3.0 GA / 1.4 rc yet). - IDENTITY: 224 runs (112 repos x rendered + fp8-KV), matrix results/sm90/trtllm-1.3.0rc29.yaml 70 pass / 6 pass+custom / 48 - fail vs rc23 69/0/55. Verdict flips: nvidia/Qwen3.6-27B-NVFP4 and nvidia/Qwen3.6-35B-A3B-NVFP4 fail->pass (rc29 serves them - through the W4A16 NVFP4 lane on SM90: W4A16NVFP4CutlassFusedMoE); nvidia/Qwen3.8-27B-NVFP4 pass->fail ("CutlassFp4GemmRunner - Arch unsupported for CUTLASS FP4 GEMM": rc29 routes its dense FP4 GEMM to the W4A4 runner, SM100+ only — the two Qwen artifacts - swapped lanes between rcs; platform floor on Hopper). DeepSeek-V4 family UNLOCKED on Hopper: rc29 drops the "not supported on - pre-blackwell" assert and instead requires kv_cache_config.dtype=fp8_ds_mla (attention/backends/sparse/deepseek_v4/ - cache_manager.py:346); the generator renders auto, so the dsv4 family trtllm customization gained params.agg.kv_cache_dtype= - fp8_ds_mla next to tokens_per_block=128 (A/B facts/upgrade_rc29/ab_dsv4_kv_*.json) -> 6 pass+custom (V4-Pro/Flash/-0813/-0731, - sgl-project FP8 x2), attention DeepseekV4 sparse (dsa_nsa). The harness fp8-KV variant (--kv-dtype fp8) cannot spell - fp8_ds_mla and stays a recorded fail for DSV4 on trtllm. NVFP4 DSV4 (and every other NVFP4 MoE artifact) now fails uniformly - at MoE resolution ("CutlassFusedMoE NVFP4 only supports SM100/103/107/120/121") — the same Hopper floor rc23 reported as - per-model GEMM/tactic errors. Other cause changes: moonshotai/Kimi-K3 + Kimi-K3-NVFP4 "No module named fla" (the rc29 image - ships kimi_kda importing the external fla package; rc23 failed earlier on quant parsing); google/gemma-4-26B-A4B hard-aborts - before the probe can dump ("Assertion failed: Head size 512 is not supported by MMHA", attentionOp.cpp:2954, - facts/upgrade_rc29/gemma4_bf16_repro.log) — Hopper floor for head_dim 512 decode; MiniMax-M3 x3 now fail with "requires - sparse_attention_config.algorithm=minimax_m3": the generator never rendered that key (rc23 template emits only - sparse_attention_config.implementation from msa_sparse_implementation; the targets customization params.agg. - sparse_attention_config={...} is an unknown params.agg key and is silently dropped — rc23 masked this with an earlier - "M3-sparse not wired" error). A/B with the key hand-added to the engine yaml (facts/upgrade_rc29/ab_m3/) gets past the - KV-cache manager and fails in MiniMaxM3SparseRuntimeBackend.forward ("requires k, v, k_cache, v_cache, idx_k_cache and - out_cache_loc") -> framework gap on Hopper either way; the generator-side key is still a gap to close when M3 works. - Identity on pass->pass rows unchanged except Qwen3-Coder-480B-A35B moe label gaining its full backend set (richer trace). - COLLECTOR ON rc29 (the audit the harness exists for): (1) tensorrt_llm._torch.attention_backend.* moved to - _torch.attention.backends.* and _torch.modules.fused_moe to _torch.moe.fused_moe (old root is a deprecation shim without - submodules) -> seven collectors import-shimmed (path only, same classes/kernels; commit f49e76c3); (2) collect_gdn''s extra FLA - row fused_recurrent_gated_delta_rule no longer exists (fla/fused_recurrent.py keeps only *_update; serving''s decode is - fused_sigmoid_gating_delta_rule_update at rc23:670 / rc29:706, never the removed kernel) -> the row is omitted with a printed - note, serving-dispatch rows unchanged; (3) collect_moe fp8_block passes dtype=torch.float8_e4m3fn to create_moe and rc29''s - MoE resolution now refuses it ("CutlassFusedMoE FP8_BLOCK_SCALES requires torch.bfloat16") — the moe gate was already DIVERGED - at rc23 (routing kernel drift) and is now INVALID-CAPTURE; serving passes the bf16 model dtype with the quant config, so the - collector''s fp8_block construction needs a serving-parity fix (collect_moe.py:313-319) — OPEN. GATES (captures re-taken in the - rc29 image as facts/pathdiff/opcov_trt_*@1.3.0rc29.json, components/captures/verdicts_trt_rc29.sh): 12/13 ALIGNED - (attn ctx/gen bf16+fp8-KV, dsa ctx/gen, mla ctx/gen bf16+fp8-KV, gemm fp8_block, gdn ctx), moe_fp8block invalid-capture. Lesson - recorded in the launcher: re-take captures from the opcov json `cmd` (exact --case-prefix cells); the first pass reused an older - --case-filter launcher and the rc29 plan order handed it head_dim-64 / sliding-window cells that looked like divergences. - 11 gate families remain undeclared for trtllm (8 dsv4_* now gradable since DSV4 loads on Hopper, msa x2, encoder/mla_bmm/mhc/ - compute_scale per workflow_check) — backfill scope is the owner''s call. RETESTS: all 11 trtllm customizations still_needed - (results/retests/sm90/trtllm-1.3.0rc29.yaml; bare DSV4 renders fail on tokens_per_block 32, NVFP4/M3 fail identically with or - without). Collector pins (framework_manifest trtllm default 1.3.0rc20) untouched — owner decision.' - applies_to: - - framework:trtllm-1.3.0rc29 - - platform:sm90 - - deepseek-ai/DeepSeek-V4-Flash - - deepseek-ai/DeepSeek-V4-Pro - - nvidia/Qwen3.8-27B-NVFP4 - - MiniMaxAI/MiniMax-M3 - - google/gemma-4-26B-A4B - - instrument:collect_moe - - instrument:collect_gdn - hopper_sglang_upgrade_0516_to_0521_2026_10_01: - statement: 'HOPPER UPGRADE sglang 0.5.16 -> 0.5.21 (H20/sm90, 2026-10-01; v0.5.21 is the newest release tag). GENERATOR - FIRST: 0.5.20 turned ServerArgs into a msgspec Struct whose argparse flags derive from field names; the 0.5.16-0.5.19 - aliases `--cuda-graph-bs` / `--cuda-graph-max-bs` are gone and 0.5.21 rejects the rendered `--cuda-graph-bs` as ambiguous, - so every golden boot failed in seconds. New versioned template cli_args.0.5.20.j2 renders the *-decode spellings (commit - 4da917b7, boundary verified from the v0.5.20 server_args.py; prior templates untouched). IDENTITY: 224 runs, matrix - results/sm90/sglang-0.5.21.yaml 99 pass / 3 pass+custom / 22 fail vs 0.5.16 93/6/25. fail->pass: moonshotai/Kimi-K3, - meta-models/Muse-Glimmer-30B, nvidia/Muse-Glimmer-30B-NVFP4, TokenRhythm/NeoHorse-1-9B. pass->fail: MiniMaxAI/MiniMax-M3 - ("Mixed chunk prefill requires the breakable prefill CUDA graph backend; got tc_piecewise" — the golden renders - --enable-mixed-chunk; A/B adding `--cuda-graph-backend-prefill breakable` PASSES on 0.5.21 (facts/upgrade_rc29/ab_m3_sgl/ - m3_A.json, MiniMaxHybridAttnBackend), dropping --enable-mixed-chunk does not; the generator has no cuda_graph_backend_prefill - parameter and no sglang extra_cli_args passthrough, so this rescue is not expressible from targets — generator gap); - nvidia/MiniMax-M3-NVFP4 (deep_gemm fp8_fp4_gemm_nt "Unknown SF transformation", layout.hpp:57); nvidia/Qwen3.8-27B-NVFP4 - (sglang::apply_fp4_marlin_linear receives a tuple for `input` — modelopt_quant.py:2055, framework bug on the SM90 NVFP4 - dense path). Identity changes on passing rows: Qwen3-VL family attention label FlashAttentionBackend -> HybridAttnBackend - (fa3 prefill underneath, same role); DSA/GLM/DSV4 kv_allocated column now reads the server arg (auto) instead of the pool - dtype — label change only. CUSTOMIZATIONS: 0.5.21 runner=auto now loads fp4-expert DeepSeek-V4/V4.1 on SM90 and selects - FLASHINFER_MXFP4 (trtllm cutlass MoE kernels, Mxfp4FlashinferCutlassMoE) — a different identity from the marlin_moe the - 0.5.16 rescue forced. Per the retest rule the five deepseek-ai V4/V4.1 marlin customizations are DROPPED and the cells - re-probed bare (results/retests/sm90/sglang-0.5.21.yaml; bare raws facts/upgrade_rc29/retest_bare_sgl_*); the three nvidia - NVFP4 artifacts still refuse auto ("Current platform does not support NVFP4 quantization with the selected MoE backend") - and keep marlin (pass+custom). Consequence for the collector: the sglang DSV4 moe row on SM90 must follow - flashinfer_mxfp4, not marlin, once the collector pin moves. GATES: captures re-taken in the v0.5.21 image - (facts/pathdiff/opcov_sgl_*@0.5.21.json, components/captures/verdicts_sgl_0521.sh) are the compat audit the collector - modules (`__compat__ sglang==0.5.14` / 0.5.16) never had: 13/15 INVALID-CAPTURE — pkg_resources absent from the 0.5.21 - image (collect_attn/mla/mla_bmm/computescale import it for the version string; 8 captures), get_device_memory_capacity - moved out of sglang.srt.server_args (dsv4/dsa context modules), initialize_moe_config() now takes no args after - publish(server_args) (dsa generation module; probe_sglang already carries the publish-then-initialize pattern), - sglang.srt.distributed.parallel_state_wrapper gone (msa, mla, attn); gdn ctx/gen capture but DIVERGE (0.5.21 serving runs - chunk_gated_delta_rule_fwd_kernel_h_blockdim64 / ChunkGatedDeltaRuleFunction for prefill and _causal_conv1d_update_kernel - for decode where the collector lane differs). The short-isl DSA gate (s512) is parked: its serving side is a 0.5.16 raw. - 15 gate families undeclared (dsv4 x5, msa, encoder, mla_bmm, mhc, compute_scale, kda, ...). OWNER DECISIONS: (1) move - the sglang collector pin to 0.5.21 (a full module audit: imports, moe init contract, gdn lanes, dsv4 moe backend) or keep - sglang path gates at 0.5.16; (2) M3 generator parameter for the prefill cuda-graph backend; (3) gate backfill scope. - framework_manifest sglang pins (0.5.14 default) untouched.' - applies_to: - - framework:sglang-0.5.21 - - platform:sm90 - - module:generator - - MiniMaxAI/MiniMax-M3 - - nvidia/Qwen3.8-27B-NVFP4 - - deepseek-ai/DeepSeek-V4-Flash - - deepseek-ai/DeepSeek-V4.1-Flash - - instrument:collect_attn - - instrument:collect_mla_module hopper_sglang_collector_port_0514_to_0521_2026_10_01: statement: 'SGLANG COLLECTOR PORT 0.5.14 -> 0.5.21 + GATE BACKFILL (H20/sm90, 2026-10-01; owner decision 2026-10-01 ''跟到0.5.21''). PIN: framework_manifest sglang default 0.5.14 -> 0.5.21 (lmsysorg/sglang:v0.5.21, index digest b1259f3e..., cu130 tag @@ -2325,248 +510,6 @@ findings: - deepseek-ai/DeepSeek-V3 - zai-org/GLM-5 - sgl-project/DeepSeek-V4-Flash-FP8 - hopper_vision_evidence_and_tp4_probe_routes_2026_10_02: - statement: 'VISION-TOWER EVIDENCE + TP>1 PROBE ROUTES (H20/sm90, 2026-10-02; owner: ''3.8这个probe走tp4试试, vision 那个加一下, - trtllm也看一下''). VISION: the plan probes are text-only, so VL checkpoints never ran their encoder on sglang/trtllm and - the encoder_attn gate had no serving evidence there (vLLM gets it from its own multimodal profile_run). Added probes/vision_{sglang,trtllm}.py: - load the dummy like the text probe, push one synthetic 448x448 image through the checkpoint''s own image processor (preprocessor_config.json - travels with the dummy dir) into model.visual (sglang qwen3_vl.py:1459) / model.mm_encoder.visual (trtllm modeling_qwen3vl.py:1206, - encoder owned by the normal worker :1358) under torch.profiler, and write archive/raw/.vision.json; build_records - merges it as the profile_run phase. It is a SEPARATE script on purpose: the probe code hash is part of every run''s - execution fingerprint, so touching probe_.py would have made all 448 text records stale. Qwen3-VL-2B serving - vision attention: sglang = sgl_kernel::fwd (FA3 FlashAttnFwdSm90), trtllm = fmha_v2_flash_attention_bf16 (trtllm_mha). - encoder_attn gates now ALIGNED on both -> sglang 0.5.21 29/29 gates, upgrade_op all_done=True; trtllm rc29 18/18 (6 - families still undeclared: gdn_gen, mamba2, mla_bmm x2, msa ctx/gen). TP>1: targets.topologies had tp8 ''implemented'' - but no record in the archive ever ran at tp>1 and the driver put every run on one GPU. New: topology_policy.tp_required - (repo, backend, tp) replaces tp1 for the listed checkpoints; the driver gives such a run the first tp GPUs of --gpu-list - on one queue (run those plans ALONE, via --only) and hashes the route script in the fingerprint. Routes = the real servers - inside one N-GPU container: probe_sglang_server.py (launch_server with the golden CLI; identity from /get_server_info; - kernels from /start_profile profile_by_stage num_steps=1 activities CPU+GPU -> -TP-0-{EXTEND,DECODE}.trace.json.gz; - never /stop_profile in by-stage mode — manual_stop raises in the scheduler and the gloo barrier kills every rank; with_stack=False - — rank 0 segfaulted in kineto finalizeTrace() with python stacks on the bf16 depth8 run) and probe_trtllm_server.py - (trtllm-serve --tp_size N --extra_llm_api_options + load_format dummy; kernels from /start_profile - num_steps windows 1 and 2 diffed per kernel -> trtllm-trace-*-rank-0.json). What the server routes cannot see is flagged - identity_source: quant_methods / param_dtypes / api span maps (path_diff grades from the device-stream tables regardless). - RESULT Qwen3.8-2.4T-A95B (+FP8) at tp4: sglang 0.5.21 PASS both (server up in ~70 s, attention fa3, GDN prefill = flashinfer - cute-DSL delta rule, fused_moe triton; kv auto and fp8_e4m3 variants), trtllm rc29 PASS both (165-245 s startup, trtllm_mha - fmha_v2 + cutlass GDN decode kernel, fused_moe_oss / deep_gemm fp8 for the FP8 checkpoint). Why tp1 failed: the depth-4 - (trtllm) / depth-8 (sglang) dummies keep all 512 experts (bf16 ~51 GB per MoE layer); 0.5.16 never reached loading (no - model class) and the 0.5.17 manifest pin was a collector-runtime declaration, never a serving validation. Matrices: - sglang 101/4/19, trtllm 72/6/46 (vllm unchanged; vllm tp>1 route not implemented — its probe is the golden run.sh in-process - and would need the same server treatment). Evidence: archive/raw/{b9f90ee85db2,9bce9bf007e4,5f3e9962ae80,6ff23fb0fc31,699fceaf86e2,937771098635,f9b89de0ef48,5d8a431b0ade}.json - (+ .profile/ server logs and traces), archive/raw/{1a0e76dd419d,a1255b13536c}.vision.json, facts/upgrade_rc29/qwen38_tp4*_queue.log. - PITFALL recorded: --emit-queues rewrites archive/queues/gpuN.sh — never emit while a queue is executing from that file - (bash reads it lazily); copy the queue aside first.' - applies_to: - - framework:sglang-0.5.21 - - framework:trtllm-1.3.0rc29 - - platform:sm90 - - module:opharness - - instrument:probe_sglang_server - - instrument:probe_trtllm_server - - instrument:vision_sglang - - instrument:vision_trtllm - - Qwen/Qwen3.8-2.4T-A95B - - Qwen/Qwen3.8-2.4T-A95B-FP8 - - Qwen/Qwen3-VL-2B-Instruct - hopper_trtllm_gate_backfill_complete_and_full_campaign_2026_10_02: - statement: 'TRTLLM GATE BACKFILL COMPLETE + FULL H20 CAMPAIGN STARTED (2026-10-02; owner: ''先把 trtllm 那 3 个 gate 补了，然后开一轮完整采集''). - GATES rc29 now 22/22 ALIGNED: gdn_gen (the earlier verdict script had it parked as ''explained'' because the collector''s - FLA fused_recurrent row was a deliberate fallback lane; on rc29 collect_gdn''s decode lane matches serving''s FlashInfer - decode, so it is a gate again), mla_bmm (cublas bmm pre/post), mamba2 ctx+gen (Nemotron-H-56B). mamba2_ctx first came - back DIVERGED: rc29 Mamba2Mixer defaults TRTLLM_MAMBA_TOKEN_MAJOR_CONV=1 (mamba2_mixer.py:319) and hands causal_conv1d_fn - a channel-last VIEW of the token-major projection plus a separate token-major out (:541-569), which selects causal_conv1d_channellast_fwd_kernel - (causal_conv1d.py:26-60), while the collector ran the channel-major kernel on (batch, dim, seq) inputs. collect_mamba2 - now builds the serving varlen token-major buffer with cu_seqlens / cache_indices / has_initial_state=False whenever - the runtime''s causal_conv1d_fn has the `out` parameter and the env default is on, records kernel_source=causal_conv1d_fn_token_major - (label added to kernel_source_backends.yaml), and keeps the old call for older runtimes; gate ALIGNED after the change. - The 6 msa gate families on trtllm remain undeclared: MiniMax-M3 does not boot on rc29 (MiniMaxM3SparseRuntimeBackend.forward), - so there is no serving record to grade against — framework gap, recorded only. CAMPAIGN: the first full collection on - the new pins runs from a frozen snapshot staging_1002 (ea76d9ab + the ba56d55d collect_mamba2/kernel_source_backends - overlay) via staging_1002/run_op.sh ( [filters]: --model-cases-full, own run + checkpoint dir, --resume - --resume-retry-failed, images = manifest defaults, HF offline + shared jitcache). Partition facts/campaign_1002/PARTITION.txt: - sglang on GPUs 2-4, trtllm on GPUs 5-7, three op chains per backend (light ops first; dsa/dsv4 modules on their own - GPU). Excluded: trtllm moe_ep (WideEP runtime, separate container), DSV4.1 (no collector ops), sglang dsv4 *_attn/paged_mqa - (not in the sm90 plan). First plan sizes: gemm 104636 (sglang) / 104294 (trtllm) cases, mla_context 3080, sglang dsa_context_module - 264 (plan-time memory filter) vs trtllm 7808. Attempt 1 of the launcher died on an empty --case-filter argument (printf - ''%q'' on an empty array yields ''''); fixed before any case ran. Logs facts/campaign_1002/logs/, run dirs staging_1002/runs/_/ - (perf evidence stays in the workspace, never committed).' - applies_to: - - framework:trtllm-1.3.0rc29 - - framework:sglang-0.5.21 - - platform:sm90 - - module:collector - - instrument:trtllm.collect_mamba2 - - nvidia/Nemotron-H-56B-Base-8K - - Qwen/Qwen3.5-0.8B - - deepseek-ai/DeepSeek-V3 - hopper_full_campaign_1002_complete_2026_10_04: - statement: 'FULL H20 CAMPAIGN COMPLETE (2026-10-02 14:3x -> 2026-10-04 04:54, ~37.5 h wall on 6 GPUs; sglang 0.5.21 + trtllm 1.3.0rc29; - everything except DSV4.1 and trtllm msa, which the owner skipped this round). 967,892 perf rows / 925,704 unique keys across 48 - (backend, op) tables; 65,405 failed tasks, all recorded. Duplicate keys (sglang moe 22,627, trtllm moe 8,991) come from two mid-run - splits done without seeding the new run dir''s checkpoint — the re-measurements agree (median 0.3%, p90 3.6%) and dedupe at merge; - lesson: seed ckpt//..json with every prior dir''s done+failed before splitting (case strings carry quant and model but no - single substring spans both, so model/dist filters only work on seeded checkpoints). Failure families: (a) classified grid/gate - raises — sglang attention_context 17,239 fa3 FP8-KV x bf16 guard; sglang moe 17,577 Triton fp8_block divisibility, 3,726 w4a16_mxfp4 - version gate; sglang/trtllm gemm 666/1,110 fp8 16-byte alignment on tiny N; trtllm attention 4,182+4,148 Gemma no-SM90-kernel; - trtllm moe 658 TP-shard alignment; gdn/kda own >=2**31 conv guard cells. (b) collector gaps found by this run — sglang moe - "bias is required for non-softmax routing" 7,533 = Llama-4 Maverick/Scout every quant mode; sglang moe "parallel name tp_group - has not been written" 3,807 = Nemotron-3 Ultra-550B/Nano-30B BF16 on 0.5.21. (c) real crashes awaiting the serving-parity audit — - trtllm attention_context 500 IMA (96 heads x kv 1/2/4/8 x head_dim 256 and 96/96/128, seq 16384, TRTLLM backend), trtllm - attention_generation 512 launch failure + 162 IMA (96 heads, kv 4, head_dim 64, fp8 KV), trtllm mla_context_module 41 IMA - (DeepSeek-V3). (d) trtllm dsv4 csa/hca ctx 47+47 OOM on DeepSeek-V4-Pro largest-prefix cells with the snapshot''s free*0.5 pool — - re-run with the max_tokens cap (d8d9aafb). Evidence: workspace facts/campaign_1002/{REPORT.md,SUMMARY.txt,PARTITION.txt,logs/}, - perf rows in staging_1002/runs/ (never committed).' - applies_to: - - framework:trtllm-1.3.0rc29 - - framework:sglang-0.5.21 - - platform:sm90 - - module:collector - - meta-llama/Llama-4-Maverick-17B-128E-Instruct - - meta-llama/Llama-4-Scout-17B-16E-Instruct - - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 - - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - - deepseek-ai/DeepSeek-V3 - - sgl-project/DeepSeek-V4-Pro-FP8 - hopper_campaign_1002_followup_fixes_2026_10_04: - statement: 'CAMPAIGN FOLLOW-UP CLEARED (2026-10-04, owner: ''一步步清掉剩余的问题''; WideEP explicitly not upgraded). (1) sglang moe: - Llama-4 Maverick/Scout now build the router like serving (Llama4MoE.custom_routing_function) and the 0.5.21 flashinfer_cutlass - runner gets the single-process get_parallel stand-in; re-collected 10,530 Llama-4 rows (0 failures) and 8,531 Nemotron Ultra/Nano - BF16 rows (243 remaining = the collector''s own high-TP alignment guard). (2) weight-only lanes re-verified on 0.5.21 for - SM89/90 only (int4_wo -> Marlin, compressed_tensors.py:838-877; w4a16_mxfp4 -> Triton, mxfp4.py:1423-1430, the 0.5.17 gpt-oss - override is gone from arg_groups/overrides.py); 4,455 Kimi int4_wo + 5,751 gpt-oss mxfp4 rows re-collected, SM100/103 - declarations stay closed. (3) trtllm attention_context 500 IMA / attention_generation 512+162 / mla_context_module 41: - the context FMHA faults exactly when (batch-1)*seq*heads*head_dim*2 >= 2**31 (fitted cell-by-cell on three head grids, - reproduced b16 s3072 h96 d256 vs b8 s6144 passing), decode fails for fp8 KV at GQA ratio 24/32 — FIXME(kernel-limit) at - both invocation sites, cases remain classified. (4) trtllm dsv4 Pro OOM cells all pass with the max_tokens cap (floor - lowered 4M -> 2M after Pro reproduced the free*0.5 pool). (5) WideEP same-pin assumption dropped: moe_ep leaves the default - trtllm plans and resolves on wideep_trtllm 1.3.0rc20 alone; mixed stock+moe_ep requests fail-close like sglang. New - classified family: Nemotron-3-Super-120B bf16 tp32 (81 cells) asserts in sglang''s JIT activation.cuh:186 on the shard width. - Evidence: workspace facts/campaign_1002/REPORT.md, merged tables facts/campaign_1002/merged/ (sglang moe 109,781 unique rows).' - applies_to: - - framework:sglang-0.5.21 - - framework:trtllm-1.3.0rc29 - - platform:sm90 - - module:collector - - meta-llama/Llama-4-Maverick-17B-128E-Instruct - - meta-llama/Llama-4-Scout-17B-16E-Instruct - - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16 - - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16 - - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8 - - openai/gpt-oss-120b - - moonshotai/Kimi-K2.5 - - sgl-project/DeepSeek-V4-Pro-FP8 - - sm100_vllm_0300_collector_verdict_2026_10_03: - statement: 'COLLECTOR VERDICT FOR THE vllm 0.30.0 UPGRADE ON sm100 (B200 umbriel-b200-049, 8 GPUs, PR head d8d9aafb, 2026-10-03; - the Hopper port of the same pin was already graded on H20). Question: do the 0.30 collectors need sm100-specific fixes? - Answer: mostly NO, two YES. All 35 vllm gate captures were re-taken in the 0.30.0 image on B200 (32 from components/captures - + mla_ctx_fp8/mla_gen_fp8, which are sm120 FLOOR lines but real gates on sm100) and graded against B200 serving records - (224 runs = 112 rendered + 112 fp8-KV): 26 ALIGNED — gemm bf16/fp8_block (+m1), moe fp8_block/bf16/mxfp4 (all on trtllm_gen_moe), - mla_ctx bf16/fp8, mla_bmm, dsa_ctx_fp8 s512 (vs a B200 isl-512 raw) and s4096, dsa_gen_fp8, msa ctx/gen, dsv4 x6, gdn - x2, kda x2, attn_ctx. NEEDS FIX (1): utils.kv_block_size branches by SM (sm100: attention 16 / mla 32, owner decision - 2026-09-30) — it fixed attn_ctx (P64->P16, aligned) but REGRESSED dsa_ctx_bf16 and dsa_ctx_bf16_s4096 (ALIGNED on 2026-09-29 - with page 64): with page 32 the collector''s bf16 DSA cell selects the TRT-LLM FMHA sparse path (fmhaSm100fKernel_QkvBfloat16...HQk576HV512HVPerCta256PagedKvDenseStaticTok...) - while serving runs vLLM cute-DSL FA sm100 + the indexer. Serving picks the page per (backend, kv dtype, head shape), not - per SM — ask the framework per case (get_supported_kernel_block_sizes / cache_config.block_size) as the B200 handoff proposed; - keep AIS_KV_BLOCK_SIZE as the A/B hook. NEEDS DECISION (2): the remaining attention drifts are tile variants of the same - cubin family — attn_gen MultiCtasKvCga vs MultiCtasKv, mla_gen_fp8/bf16 HVPerCta128 vs 256 — chosen by the FMHA''s kv-length/batch - heuristic; whether perf rows must match at tile level is a granularity call. gemm_fp8 is the sm120 governed exception - (cutlass sm100 GEMM template instance); encoder_attn needs an encoder-scoped op-hint (the decoder''s paged-causal FMHA - spans are swept in; the ViT path is fa_cute_sm100 on both sides); compute_scale/mhc are the known explained deviations. - No 0.30 API drift showed up on sm100 beyond what the Hopper port already fixed.' - applies_to: - - platform:sm100 - - framework:vllm-0.30.0 - - module:collector - - instrument:vllm.collect_attn - - instrument:vllm.collect_mla_module - - instrument:path_diff - - sm100_vllm_0300_identity_2026_10_03: - statement: 'IDENTITY MATRIX sm100 / vllm 0.30.0 on PR head d8d9aafb (results/sm100/vllm-0.30.0.yaml): 106 pass / 1 pass+custom - / 17 fail over 124 rows. 12 fails share sm90''s causes (11 generator rejects: Qwen3.8-Flash-Next x3, MiMo-V2.6 x2, Inkling - x4, GLM-5.3-Flash x2; Qwen3.8-2.4T-A95B capacity). sm100-specific: (a) deepseek-ai/DeepSeek-V4.1-Flash and nvidia/DeepSeek-V4.1-Flash-NVFP4 - ''ValueError: No common block size for 64'' — the generator fact facts/models.yaml V4.1 block-size 64 (added 2026-10-01 - for sm90/sm120) has no accepting backend on sm100; A/B: the same rendered run.sh without --block-size 64 boots and runs - (DeepseekV41ForCausalLM) — a platform-blind generator fact; (b) nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 ''no attention-ish - module classes'' — targets.yaml declares variant_overrides vllm: layers_block_type_moe for it, so the probe ran a moe-only - cut; (c) Qwen/Qwen3.8-2.4T-A95B-FP8 OOM on 178 GiB (passes on the 143 GB H20 — dummy residency differs); (d) fp8-KV variant: - Kimi-K3 and Kimi-K3-NVFP4 require --attention-config ''{"use_prefill_query_quantization": true}'' (framework assertion). - DSV4 family all pass (64-aligned budgets). Identity flips vs sm90 at the same pin: dense attention FA3 -> flashinfer TRT-LLM - FMHA cubins (prefill fmhaSm100a*, decode fmhaSm100f*; page 16 Llama/Qwen, 32 Nemotron and MLA fp8, 64 Gemma-4), DSV3.2 - FlashAttnMLASparse -> FlashMLASparse, NVFP4 DSA FlashMLASparse -> FlashInferMLASparse, M3 SparseTritonImpl -> SparseMSAImpl, - every NVFP4/MXFP4/FP8 MoE marlin/triton_fused_moe -> trtllm_gen_moe. Customization retest (results/retests/sm100/vllm-0.30.0.yaml): - Nemotron-H gpu-memory-utilization 0.6 still_needed (bare default OOMs in the profile run on a clean 178 GiB B200, 2026-09-29 - evidence). Perf-data collection was started on the same box (26 shards, gemm bf16 37,518 / gemm fp8 37,518 / attention_context - 54,468 rows finished with 0 failed at the 2.6 h mark; dsa_context_module plans 261,504 cases and is the long tail) — data - stays in the workspace per the owner''s 2026-10-04 rule, never in this PR.' - applies_to: - - platform:sm100 - - framework:vllm-0.30.0 - - deepseek-ai/DeepSeek-V4.1-Flash - - nvidia/DeepSeek-V4.1-Flash-NVFP4 - - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4 - - moonshotai/Kimi-K3 - - nvidia/Nemotron-H-56B-Base-8K - - harness_issues_sm100_third_run_2026_10_03: - statement: 'HARNESS ISSUES FROM THE THIRD BLACKWELL RUN (sm100, 2026-10-03), each reproduced: (1) tools/perf_database/collect_campaign.py - image_digest() uses ''{{index .RepoDigests 0}}'' — a locally built collect image has no RepoDigests, so `run` refuses - the image `build-image` just produced (''image not present''); fixed here with ''{{if .RepoDigests}}...{{end}}|{{.Id}}''. - (2) kernel_taxonomy_sm100.yaml did not exist — a new SM starts with an empty vocabulary and every gate reads no-collector-signal; - seeded here from the sm90 file + the B300-validated Blackwell block + fa_cute_sm100 + the cutlass sm100 GEMM symbol (enable_sm100_to_sm120 - guard). (3) dummy_overrides.tokenizer_from only resolves to a ROSTER repo''s dummy dir, not to configs/aux_files/; - gated meta-llama repos on a token-less box (bundled config fallback ships no tokenizer) need a public sibling IN the roster, - and Llama-4''s unsloth mirrors had to be copied into the meta-llama aux dirs by hand (the bundled Llama-4 configs also - lack vision_config). (4) verdicts_vllm_0300.sh is AIS_SM-parametric but the dsa_ctx_fp8 SERVING_RAW is the H20 path facts/reprobe_nopc/04cea4da0eff_isl512.json - (pre-platform run id); on another SM it needs a symlink or a derived path. (5) FLOOR_SM=sm120 lines are real gates on - sm100 — a capture list derived from the `run` lines alone misses mla_ctx_fp8/mla_gen_fp8. (6) path_diff --capture only - fails at the final open() when the output dir is missing — the whole capture runs for nothing; mkdir -p first. (7) the - collect image has no git, so collection_meta.collector_ref records ''unknown''. (8) a generator fact (V4.1 block-size - 64) and a targets variant_override (Nemotron-Ultra-NVFP4 moe-only cut) are platform-blind inputs that turn pass cells - into fails on a new SM — same class as the cli_extra_args lesson from B300. (9) throughput facts for planning: identity - 224 runs on 7 B200 ~2 h (first wave JIT-heavy); collection shard plan must split dsa_context_module (261,504 cases) and - gemm fp8_block (DeepGEMM JIT) across GPUs — one shard per op leaves 7 GPUs idle behind a 20 h tail.' - applies_to: - - platform:sm100 - - instrument:collect_campaign - - instrument:path_diff - - instrument:dummies - - instrument:fetch_inputs - - instrument:probe_driver - - - sm100_kv_block_size_framework_query_2026_10_04: - statement: 'COLLECTOR FIX (owner go-ahead 2026-10-04 ''如果有好的机制，可以实现一下''): the KV page is now ASKED from the framework per - case instead of tabled per SM. Root cause found while implementing: collector/vllm/utils.create_vllm_config built CacheConfig(block_size=16|64), - and pydantic marks any explicit value user_specified_block_size=True — the selector (v1/attention/selector.py:131-135) - then filters backends by that page exactly as --block-size would, which serving never does unless the user asked. On sm100 - that constraint is what moved the collector onto the wrong backends: at page 16 FLASHMLA_SPARSE/FLASHINFER_MLA_SPARSE - are rejected (''No valid attention backend''), at page 32 the bf16 DSA cell landed on the TRT-LLM FMHA sparse cubin. Mechanism, - mirroring serving: (1) CacheConfig(block_size=None) -> default 16, NOT user-specified, selection unconstrained (AttentionSelectorConfig.block_size=None - in collect_attn); (2) after the backend is known (selector result in collect_attn; the built module''s layer in collect_mla_module._create_kv_cache_and_metadata), - utils.framework_kv_block_size(backend_cls) = select_common_block_size(backend.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE), - [backend]) — the two calls platforms/interface.py update_block_size_for_backend and v1/worker/utils.prepare_kernel_block_sizes - make; cache_config.block_size is set to it before the KV cache/metadata are built. AIS_KV_BLOCK_SIZE stays the A/B hook; - utils.kv_block_size (the per-SM table) is no longer called. VALIDATION on umbriel-b200-040 (B200, vllm 0.30.0 image, 10 - captures, families compared with the new attention tile equivalence against the 2026-10-03 sm100 serving kernels): attn_ctx/attn_gen - P16 (FLASHINFER) = serving; dsa_ctx_bf16_s4096 back on FLASHMLA_SPARSE with the FA-cute prefill kernel = serving (DIVERGED - under the table); mla_gen_fp8 FLASHINFER_MLA P32 = serving; mla_gen_bf16 P32; dsa fp8 ctx/gen FLASHINFER_MLA_SPARSE = - serving identity. Open: dsa_ctx_bf16 at s=512 runs the sparse kernel on sm100 while the isl-4096 serving record shows - the FA-cute prefill — a length-conditional path, so on sm100 that gate needs an isl-512 serving raw like dsa_ctx_fp8 already - has. Also in this change: path_diff _EQUIV_RULES gains an ''attention'' rule that keys TRT-LLM FMHA cubins by (sm, qkv - dtypes, head dims, paged-kv kind, PAGE) and drops the tile/CTA tail (HVPerCta, MultiCtasKv[Cga], VarSeqQKv, Persistent/Static/Grouped) - — owner decision 2026-10-04 to relax tile variants; a page mismatch or a backend mismatch still reads diverged.' - applies_to: - - platform:sm100 - - platform:sm90 - - framework:vllm-0.30.0 - - instrument:vllm.collect_attn - - instrument:vllm.collect_mla_module - - instrument:vllm.utils - - instrument:path_diff - sm100_moe_backend_fact_rejected_2026_10_04: statement: 'GENERATOR FACT BUG (B200 / sm100, 2026-10-04, first sglang 0.5.21 + trtllm 1.3.0rc29 identity probes on Blackwell; Hopper never saw it): generator/facts/hardware.yaml b200 and gb200 profiles carry `moe_backend: {trtllm: WIDEEP, sglang: @@ -2614,26 +557,6 @@ findings: collector_impact: - {op: moe, action: guard, ref: '95085922 collect_moe._raise_if_unverified_moe_lane: int4_wo opened on SM100/103 for 0.5.21 — the moe_auto retest here (Kimi-K2.5 FLASHINFER_TRTLLM / trtllm_gen_moe) is the evidence; lane_evidence.yaml now grades it'} - {op: none, action: pipeline, ref: 'generator hardware fact (moe_backend deepep_moe) is the generator plane; collector cases unaffected'} - sm100_trtllm_moe_default_cutlass_2026_10_04: - statement: 'GENERATOR DEFAULT, PLATFORM-BLIND (B200, trtllm 1.3.0rc29, 2026-10-04): with the Blackwell moe_backend fact gone, - the rc29 engine template (config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2:16) still renders - `moe_config.backend: {{ moe_config.backend | default(''CUTLASS'') }}` unconditionally, so every sm100 trtllm MoE cell is - served with an explicit CUTLASS — the same literal the Hopper profiles pin. On SM100 rc29''s CutlassFusedMoE refuses - FP8_BLOCK_SCALES ("only supports SM90/SM120, got SM100"), so all fp8-block MoE checkpoints fail under the generator''s - render while AUTO serves them (A/B above): DeepSeek-R1, DeepSeek-V3/V3.1/V3.2, DeepSeek-V4 x6 (also page-blocked), - GLM-5/5.1/5.2/5.3 FP8, MiniMax-M2/M2.5/M2.7, Qwen3-30B-A3B-FP8, Qwen3-235B-A22B-FP8, Kimi-K2.x FP8 — 19 cells in - results/sm100/trtllm-1.3.0rc29.yaml carry the new cause "rendered MoE backend cannot serve this layer on this SM" (the - rule also covers the sm90 "no MoE implementation can serve this layer" NVFP4-on-Hopper cells, previously "framework gap"). - The matrix is the as-rendered truth: 78 pass / 0 custom / 46 fail; the AUTO table (results/retests/sm100/ - trtllm-1.3.0rc29.moe_auto.yaml) is what the framework picks when allowed. OWNER DECISION: the trtllm MoE default must be - platform-aware — AUTO on sm100 (rc29 resolves fp8-block to TrtllmGen/DeepGEMM, NVFP4 to NVFP4Cutlass/TrtllmGen, bf16 to - Cutlass fused_moe_oss), CUTLASS stays correct on sm90. Collector note: label_kernels files rc29''s cutlass-based - `fused_moe_oss::run_global` under triton_fused_moe (taxonomy rule shared with sm90) — a vocabulary nit, not a verdict.' - applies_to: - - platform:sm100 - - framework:trtllm-1.3.0rc29 - - instrument:generator.backend_templates.trtllm - - instrument:probe_driver sm100_trtllm_dsv4_facts_unservable_2026_10_04: statement: 'GENERATOR DSV4 FACTS ARE A HOPPER RECIPE (B200, trtllm 1.3.0rc29, 2026-10-04): the 8 DeepSeek-V4 trtllm cells (Flash/Pro x {bf16, FP8, NVFP4, 0731/0813}) render `kv_cache_config: {dtype: fp8_ds_mla, tokens_per_block: 128}`. On SM100 @@ -2689,61 +612,6 @@ findings: - instrument:path_diff collector_impact: - {op: none, action: none, ref: 'verdict plane only (gates/captures); collector grids unchanged'} - sm100_precision_smoke_2026_10_04: - statement: 'PRECISION-COVERING OP SMOKE on B200 (umbriel-b200-021, 2026-10-04; driver facts/smoke/op_smoke_sweep.py in the - workspace: every registered sglang (30) and trtllm (23) collector op, cases grouped by the precision tokens in the case text, - ONE case per group, op_smoke''s executor contract, one container process per op — a shared process poisons later ops through - torch default-device state and produced false "can''t convert cuda:0 tensor to numpy" errors; collect_moe / collect_mamba2 - exit with EXIT_CODE_RESTART by design after writing their rows, which a sweep must not read as failure). FOUND AND FIXED - (sm100 lanes Hopper never exercised): (1) trtllm collect_gemm and collect_mla_module declared NVFP4 with group_size 128; - 1.3.0rc29 NVFP4LinearMethod.resolve_scaling_vector_size requires 16 — gemm nvfp4 and every nvfp4 group of the mla/dsa module - collectors failed to construct (nvfp4 is a platform floor on sm90, so the lane never ran there); now 16, smoke OK. - (2) sglang collect_mla and collect_attn MockModelRunner lacked max_running_requests / is_draft_worker / - prefill_attention_backend_str / decode_attention_backend_str / model — attributes the 0.5.21 TRTLLMMLABackend and - TRTLLMHAAttnBackend read, and those are the backends sglang picks on SM100 (fa3/flashinfer on Hopper never asked); added, - mla_generation and attention_* smoke OK. (3) trtllm collect_dsv4_attn pinned the fp8 KV pool + kv_cache_quant_algo FP8 on - every SM; on SM100 rc29 cannot serve fp8_ds_mla (finding sm100_trtllm_dsv4_facts_unservable_2026_10_04) and kv auto resolves - to the bf16 DeepseekV4CacheManager at the generator''s page 128 (facts/ab_dsv4_p128: boots, fmhaSm100f QkvBfloat16 - DynamicTokenSparse) — the collector lane is now bf16-KV on SM100 (_serving_kv_cache_dtype; AIS_DSV4_KV_CACHE_DTYPE hook) - and all four trtllm dsv4 gates read ALIGNED against that serving raw. (4) GLM-5 DSA attention on sglang: the fp8-KV lane is - NOT a new code path — collect_mla_module ''dsa'' with kv fp8 already serves GLM-5 (GlmMoeDsaForCausalLM); recipes - sgl_glm5_dsa_attn_fp8_{ctx,gen} capture it: attention family == sm100 serving (fmhaSm100f QkvE4m3 HQk576HV512 TokenSparse), - residual drift is one tiny cutlass_80 fp16 wmma 32x32 GEMM the module lane issues and serving does not (gemm family) — - owner call whether the glm5 gate should move to the module lane on Blackwell. FIXED LATER (umbriel-b200-028, 2026-10-05): (5) sglang mla_context on - SM100: the collector times the prefill kernel under CUDA-graph capture; flashinfer.prefill.trtllm_ragged_attention_deepseek - (prefill.py:5088) refuses capture without q_seq_lens_cpu/kv_seq_lens_cpu and sglang''s trtllm_mla _run_prefill_kernel does - not pass them (serving never graph-captures prefill) — collect_mla.benchmark_layer now times the SM100 trtllm_mla PREFILL lane eagerly (use_cuda_graph=False; decode and - the Hopper backends keep CUDA-graph timing); mla_context bf16/fp8 smoke OK on B200. (6) sglang moe mxfp4 lanes: the guard is - extended to 0.5.21 on SM100/103 for w4a16_mxfp4 AND w4a8_mxfp4_mxfp8, verified from the identity records (gpt-oss-120b/20b - and the five deepseek-ai/DeepSeek-V4* fp4-experts checkpoints serve with Mxfp4MoEMethod -> FLASHINFER_MXFP4, kernel family - trtllm_gen_moe, under the default render and under AUTO; the marlin prescription is the SM90 workaround); the leaf check - that only knew Hopper''s cutlass_sm90 now maps the SM100 leaf trtllm_sm100 -> kernel_source sglang_mxfp4_flashinfer_trtllm_moe - (and SM120 flashinfer_cutlass_sm120). Smoke on B200: gpt-oss-120b w4a16_mxfp4 and DeepSeek-V4-Flash w4a8_mxfp4_mxfp8 both run - and record sglang_mxfp4_flashinfer_trtllm_moe = the serving lane. Still open: (7) the kda/gdn int32 guard at very large - shapes (owner: leave it; only batch>=16 x seq>=8k is affected). (7) sglang kda: the collector''s own int32-overflow guard - (causal_conv1d_triton token-offset) rejects every Kimi-K3 case at the sweep''s shapes — not sm100-specific; the same guard - trips 6/112 sglang gdn context sub-cases (batch 16-64 x seq 8192-32768, total_tokens*6144 >= 2**31) and the op''s - strict-completeness check then fails the whole sweep (gate cell gdn_ctx Qwen3.5-0.8B itself ALIGNED). FINAL TABLE: - trtllm 58/59 cells pass (9 = designed restart exits; 1 stale shared-process entry for a group that no longer exists on - sm100), sglang 47/52 (open: mla_context x2, moe mxfp4 guard, kda guard, gdn guard); vLLM 0.30.0 (umbriel-b200-037, 23 ops, 55 cells across bf16/fp8/fp8_block/nvfp4 groups) 54/55 — the one failure is kda: the Kimi-K3 KDA sweep OOMs at batch 64 x seq 16384/32768 (24-144 GiB allocations on a 178 GiB B200; no shape budget filter in that collector) and the strict-completeness check fails the whole op; the gate cell is unaffected. BOOTSTRAP ROBUSTNESS (037): the sequential roster fetch hit Hub HTTPErrors on 30 repos and fell back to the bundled SDK copies — for MiniMaxAI/MiniMax-M3 the bundled copy is the FLAT text-only config (model_type minimax_m3) while the Hub repo is the VL wrapper (minimax_m3_vl, layers under text_config), so dummies.py''s M3 adapter died with KeyError text_config and took the whole 256-variant run with it; a sibling hf_quant fallback without exclude_modules did the same (KeyError). Both adapters now tolerate the other layout / log the gap instead of aborting; the fetch should retry on 429/HTTPError before falling back, and dummies.py should isolate per-repo failures. Everything else - (attention, gemm bf16/fp8/fp8_block, mla ctx/gen, mla_bmm, dsa/dsv4 modules across bf16/fp8/fp8_block/nvfp4, gdn, mamba2, - mhc, compute_scale, encoder, moe fp8/fp8_block/nvfp4/bf16, msa) runs on B200. Per-cell table: facts/smoke/sweep_*.json + - summarize_smoke.py in the 021 workspace.' - applies_to: - - platform:sm100 - - framework:sglang-0.5.21 - - framework:trtllm-1.3.0rc29 - - instrument:trtllm.collect_gemm - - instrument:trtllm.collect_mla_module - - instrument:trtllm.collect_dsv4_attn - - instrument:sglang.collect_mla - - instrument:sglang.collect_attn - - instrument:sglang.collect_mla_module - - instrument:op_smoke - collector_impact: - - {op: mla_context, action: collector_fix, ref: '8f4f36c (sm100 branch): trtllm_mla prefill timed eagerly on SM100'} - - {op: moe, action: guard, ref: '8f4f36c: w4a16_mxfp4 + w4a8_mxfp4_mxfp8 opened on SM100/103 from these identity records'} - - {op: gemm, action: collector_fix, ref: '8f4f36c: trtllm nvfp4 group_size 16; mock runner attributes for 0.5.21 TRTLLM backends'} sm100_probe_failures_recheck_2026_10_04: statement: 'RE-CHECK OF EVERY sm100 PROBE FAILURE / NON-ALIGNED GATE FOR A CONFIG CAUSE (owner ask; umbriel-b200-037, fresh workspace, Hub configs re-fetched). OURS (generator / harness / dummy): (a) 12 generator rejects per framework — roster repos @@ -2784,40 +652,6 @@ findings: collector_impact: - {op: mla_generation, action: guard, ref: '95085922 collect_mla.run_mla: FIXME(kernel-limit) classified raise for trtllm_mla decode with 64 < local heads < 128 on SM100/103 (item (i)); job 469988017 failed exactly the 362 num_heads=96 cases before this landed'} - {op: none, action: none, ref: 'item (j) g1_scale_c: collector passes the runner explicitly, not reachable from the grid; dummy/fetch fixes are the probe plane'} - sm100_smoke_vs_perfdb_regressions_2026_10_04: - statement: 'REGRESSION CHECK, new pins vs the shipped B200 perf database (umbriel-b200-037, 2026-10-04; every smoke perf row - joined on the full case key — and on the kernel lane where an op writes one row per kernel — against the newest b200_sxm - parquet of the same framework: vllm 0.25.0/0.27.1, sglang 0.5.14/0.5.16/0.5.17, trtllm 1.3.0rc20/rc23; compare_perfdb.py + - compare_perfdb_report.txt in the handoff dir). Calibration: this box runs compute-bound GEMMs 5-10% slower than the database - box (40 bf16 gemm cases, ratio 1.05-1.10), and sub-10 us kernels carry a ~2 us launch offset here (mla_bmm 1-token, dsv4 - paged_mqa_logits, topk calib) — ratios <= 1.15 and tiny-kernel deltas are environment, not framework. FUNCTIONAL: every lane - the database has rows for runs on sm100 at the new pins (final per-cell smoke on 037: vllm 54/55, trtllm 58/58, sglang 51/56 — - the sglang misses are the collector gaps of sm100_precision_smoke_2026_10_04 plus kda/gdn guards; vllm kda OOM). IDENTITY - CHANGES: vllm attention decode kernel_source -> flashinfertrtllmapidecode_trtllm_gen (trtllm-gen decode, x1.22 single cell); - trtllm mamba2 conv -> causal_conv1d_fn_token_major (rc29; collector mirrors it, mamba2 gates ALIGNED; per-case ratios 0.03-2.5 - = a different kernel, not comparable). LATENCY REGRESSIONS (same kernel_source): (1) vllm 0.30 moe nvfp4, DeepSeek shape - (hidden 7168, inter 2048, top-8, 256 experts, tp1 ep1, flashinfer trtllm nvfp4 experts): 8192 tokens 1.86 -> 4.05/4.15/4.13 ms - (x2.2, three repeats) while 6144 (1.57 -> 1.46) and 12288 (2.75 -> 2.74) are unchanged and 16384 is x1.14 — an isolated, - reproducible spike at exactly 8192 tokens (8192 slower than 12288): a bad tactic/tile pick at that token count. (2) vllm 0.30 gdn - context, chunk_gated_delta_rule_flashinfer lane, every small shape (seq <= 256, d_model 1024): ~20 us -> ~40 us (x2.0 across 21 - cases), large shapes x0.9 — a fixed per-call overhead added in the 0.30 lane. (3) sglang 0.5.21 vs 0.5.14 DSA (DeepSeek-V3.2, - bf16 KV): context dense_mha_trtllm_ragged lane median x1.23, up to x2.07 (bs 1, isl 16-64, past_kv 1-1.5k); generation - dsa_indexer_trtllm at bs 1024: 1.3 -> 1.8-2.3 ms (x1.4-1.8); sparse/indexer context lanes x1.04-1.13; GLM-5.2 lanes x1.0-1.12 - — REPEATED (two more full sweeps each): generation bs 1024 is x1.37 / x1.39 / x1.39 (systematic ~x1.4; the x1.77 was a single-shot - outlier); context dense_mha_trtllm_ragged lane median x1.21 / x1.23 / x1.23 with max x1.35-1.47 (systematic ~x1.2, i.e. ~x1.1 - after the 5-10% box offset — borderline); sparse/indexer context lanes x1.02-1.05 (environment). NOT regressions: trtllm bf16 512-expert moe_torch_flow_nongated x1.3-1.7 - at 4-16 tokens (a torch reference flow, noisy), sglang dsv4 paged_mqa_logits / topk_calib x1.1-1.2 (3-60 us kernels). - FASTER: vllm dsa modules x0.57 (0.30 vs 0.24), trtllm msa context x0.30 (rc29 vs rc23), sglang msa context x0.71, sglang - mla_bmm / mla gen x0.7, vllm gemm fp8_block 128 -> 111 ms. Everything else within +-25%.' - applies_to: - - platform:sm100 - - framework:vllm-0.30.0 - - framework:sglang-0.5.21 - - framework:trtllm-1.3.0rc29 - - instrument:op_smoke - - instrument:perf_database - collector_impact: - - {op: none, action: none, ref: 'latency regressions are data for owner review, no collector grid change'} sm89_vllm_0300_probe_2026_10_04: statement: 'FIRST sm89 (L40, 8 x 44.4 GiB usable, CC 8.9, driver 595.58) identity campaign, vllm 0.30.0 (image digest 8a69ffad015f), generator = this branch, system l40s (needs the new l40s hardware profile: without it resolve_facts raised @@ -3077,28 +911,3 @@ findings: - instrument:dummies - component:generator_facts - family:m3 - sm103_prepared_from_sm100_best_effort_2026_10_05: - statement: 'SM103 (B300) PREPARED FROM THE SM100 ROUND, BEST EFFORT WITHOUT A B300 BOX (2026-10-05; owner: the sglang encoder - gate passed on sm100 and failed on sm103). Root cause found by reading: collector/sglang/collect_attn_encoder.py picked - FA4 with `sm == 100` and sent SM103 down the Triton lane, while serving selects by capability MAJOR (vision.py:1298-1303 - @0.5.21: major 9 -> fa3, major 10 -> fa4, else triton_attn) — a wrong-kernel measurement, not a crash; the collector now - calls serving_vision_backend(sm) (unit-tested for 89/90/100/103/120/121) and the H20 smoke still records flash_attention_v3. - Audit of every exact-SM test in the collectors: the only other `== 100` is trtllm collect_moe''s nvfp4 min-latency option - (opened to SM103 under the owner''s 2026-10-05 decision that sm103 is treated as sm100; re-verify on a B300 box); sglang gemm / - gdn / mla_module already use {100, 103}. Harness: kernel_taxonomy_sm103.yaml seeded verbatim from sm100 (its Blackwell - block was validated on B300 on 2026-09-20; probe_driver refuses to run without a per-SM file), generator facts already map - b300_sxm / gb300 onto the b200 / gb200 profiles, the Blackwell fail-cause rule matches SM1xx, the trtllm msa floors list - sm103. NOT done without the box: labelling the sm103 vocabulary from B300 raws, results/sm103 matrices and gate verdicts, - re-running the sm103 encoder gate. On the B300 box: set targets.platform (b300_sm103 / sm 103 / b300_sxm; facts_systems - b300_sxm, gb300) and run the workflow as on sm100.' - applies_to: - - platform:sm103 - - platform:sm100 - - framework:sglang-0.5.21 - - module:collector - - instrument:sglang.collect_attn_encoder - - instrument:probe_driver - - Qwen/Qwen3-VL-2B-Instruct - collector_impact: - - {op: encoder_attention, action: collector_fix, ref: '14111be3: encoder collector picks the vision lane by capability family on sm103'} - - {op: moe, action: guard, ref: '26d09519: sm103 treated as sm100 — nvfp4 min-latency lane opens on SM103'} diff --git a/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml b/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml index 3da6d96b99..0ecf6dd91b 100644 --- a/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml +++ b/benchmarks/evidence/collector/op-probe-2026-10/results/retests/sm100/sglang-0.5.21.moe_auto.yaml @@ -1,3 +1,5 @@ +# Excerpt (2026-10-30): _meta plus the models collector comments cite; _meta.summary +# counts the full matrix, which lands with the op-probe harness. _meta: platform: b200_sm100 framework: sglang @@ -10,173 +12,6 @@ _meta: fail: 3 evidence: archive/retests/moe_auto/{raw,*.engine.yaml} in the B200 workspace (umbriel-b200-040, 2026-10-04) results: - MiniMaxAI/MiniMax-M2: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - MiniMaxAI/MiniMax-M2.5: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - MiniMaxAI/MiniMax-M2.7: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - MiniMaxAI/MiniMax-M3: - verdict: pass+override - attention: MiniMaxHybridAttnBackend - moe_runner: TRITON - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - MiniMaxAI/MiniMax-M3-MXFP8: - verdict: pass+override - attention: MiniMaxHybridAttnBackend - moe_runner: DEEP_GEMM - moe_kernel_families: null - moe_kernels_seen: 1 - topology: tp1 - Qwen/Qwen1.5-MoE-A2.7B: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - Qwen/Qwen3-235B-A22B: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3-235B-A22B-FP8: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - Qwen/Qwen3-235B-A22B-Instruct-2507: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3-30B-A3B: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3-30B-A3B-FP8: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - Qwen/Qwen3-Coder-480B-A35B-Instruct: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3-VL-235B-A22B-Instruct: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3-VL-30B-A3B-Instruct: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3.5-122B-A10B: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3.5-35B-A3B: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - Qwen/Qwen3.5-397B-A17B: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - XiaomiMiMo/MiMo-V2-Flash: - verdict: pass+override - attention: FlashAttentionBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - deepseek-ai/DeepSeek-R1: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - deepseek-ai/DeepSeek-V3: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - deepseek-ai/DeepSeek-V3.2: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 deepseek-ai/DeepSeek-V4-Flash: verdict: pass+override attention: DeepseekV4AttnBackend @@ -217,54 +52,6 @@ results: - trtllm_gen_moe moe_kernels_seen: 4 topology: tp1 - google/gemma-4-26B-A4B: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - meta-llama/Llama-4-Maverick-17B-128E-Instruct: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 4 - topology: tp1 - meta-llama/Llama-4-Scout-17B-16E-Instruct: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 4 - topology: tp1 - mistralai/Mixtral-8x22B-v0.1: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 4 - topology: tp1 - mistralai/Mixtral-8x7B-v0.1: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 4 - topology: tp1 - moonshotai/Kimi-K2-Instruct: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 moonshotai/Kimi-K2.5: verdict: pass+override attention: TRTLLMMLABackend @@ -273,229 +60,6 @@ results: - trtllm_gen_moe moe_kernels_seen: 5 topology: tp1 - moonshotai/Kimi-K3: - verdict: fail - cause: framework gap - error: 'ValueError: multi_ctas_kv_counter_buffer is only supported when a trtllm-gen runner is selected' - topology: tp1 - nvidia/DeepSeek-V3.1-NVFP4: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/DeepSeek-V3.2-NVFP4: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/GLM-5-NVFP4: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/GLM-5.1-NVFP4: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/GLM-5.2-NVFP4: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/GLM-5.3-NVFP4: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Gemma-4-26B-A4B-NVFP4: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Kimi-K2.5-NVFP4: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Kimi-K2.6-NVFP4: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Kimi-K2.7-Code-NVFP4: - verdict: pass+override - attention: TRTLLMMLABackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Kimi-K3-NVFP4: - verdict: fail - cause: framework gap - error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' - topology: tp1 - nvidia/MiniMax-M2.5-NVFP4: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM_ROUTED - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 4 - topology: tp1 - nvidia/MiniMax-M2.7-NVFP4: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM_ROUTED - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 4 - topology: tp1 - nvidia/MiniMax-M3-NVFP4: - verdict: pass+override - attention: MiniMaxHybridAttnBackend - moe_runner: FLASHINFER_TRTLLM_ROUTED - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 4 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: MARLIN - moe_kernel_families: - - marlin - moe_kernels_seen: 6 - topology: tp1 - nvidia/Qwen3-235B-A22B-NVFP4: - verdict: pass+override - attention: TRTLLMHAAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Qwen3.5-122B-A10B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Qwen3.5-397B-A17B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - nvidia/Qwen3.6-35B-A3B-NVFP4: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: MARLIN - moe_kernel_families: - - marlin - moe_kernels_seen: 6 - topology: tp1 - nvidia/Qwen3.8-2.4T-A95B-NVFP4: - verdict: fail - cause: framework gap - error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' - topology: tp1 openai/gpt-oss-120b: verdict: pass+override attention: TRTLLMHAAttnBackend @@ -512,91 +76,3 @@ results: - trtllm_gen_moe moe_kernels_seen: 5 topology: tp1 - ornith-ai/Ornith-1.5-35B-A3B: - verdict: pass+override - attention: HybridLinearAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - sgl-project/DeepSeek-V4-Flash-FP8: - verdict: pass+override - attention: DeepseekV4AttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - sgl-project/DeepSeek-V4-Pro-FP8: - verdict: pass+override - attention: DeepseekV4AttnBackend - moe_runner: AUTO - moe_kernel_families: - - triton_fused_moe - moe_kernels_seen: 6 - topology: tp1 - zai-org/GLM-5: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - zai-org/GLM-5-FP8: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - zai-org/GLM-5.1: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - zai-org/GLM-5.1-FP8: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - zai-org/GLM-5.2: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 - zai-org/GLM-5.2-FP8: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - zai-org/GLM-5.3: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 6 - topology: tp1 - zai-org/GLM-5.3-BF16: - verdict: pass+override - attention: DeepseekSparseAttnBackend - moe_runner: FLASHINFER_TRTLLM - moe_kernel_families: - - trtllm_gen_moe - moe_kernels_seen: 5 - topology: tp1 diff --git a/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml index 84a58280ba..9f4188b3cb 100644 --- a/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml +++ b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/sglang-0.5.21.yaml @@ -1,3 +1,5 @@ +# Excerpt (2026-10-30): _meta plus the models collector comments cite; _meta.summary +# counts the full matrix, which lands with the op-probe harness. _meta: platform: b200_sm100 framework: sglang @@ -9,417 +11,6 @@ _meta: fail: 16 golden_facts_not_applied: 0 results: - MiniMaxAI/MiniMax-M2: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: Fp8MoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - MiniMaxAI/MiniMax-M2.5: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: Fp8MoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - MiniMaxAI/MiniMax-M2.7: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: Fp8MoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - MiniMaxAI/MiniMax-M3: - verdict: pass+custom - extra_args: --generator-set params.agg.cuda_graph_backend_prefill=breakable - evidence: current - attention: MiniMaxHybridAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - MiniMaxAI/MiniMax-M3-MXFP8: - verdict: pass+custom - extra_args: --generator-set params.agg.cuda_graph_backend_prefill=breakable - evidence: current - attention: MiniMaxHybridAttnBackend - moe: Fp8MoE - kv_allocated: auto - topology: tp1 - Qwen/Qwen1.5-MoE-A2.7B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-0.6B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-1.7B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-235B-A22B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-235B-A22B-FP8: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-235B-A22B-Instruct-2507: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-30B-A3B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-30B-A3B-FP8: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-32B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-32B-FP8: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-32B-FP8-Static-PerTensor: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-8B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-Coder-480B-A35B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-235B-A22B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-2B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-30B-A3B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-32B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-32B-Thinking: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-4B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3-VL-8B-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-0.8B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-122B-A10B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-27B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-2B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-35B-A3B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-397B-A17B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-4B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.5-9B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.8-2.4T-A95B: - verdict: pass - evidence: current - attention: trtllm_mha - kv_allocated: auto - topology: tp4 - Qwen/Qwen3.8-2.4T-A95B-FP8: - verdict: fail - cause: no raw (crashed before dump) - Qwen/Qwen3.8-27B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.8-27B-FP8: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - Qwen/Qwen3.8-Flash-Next: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next'' is not a valid HuggingFace model path or local pa' - Qwen/Qwen3.8-Flash-Next-FP8: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next-FP8'' is not a valid HuggingFace model path or loca' - TokenRhythm/NeoHorse-1-9B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - XiaomiMiMo/MiMo-7B-Base: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - XiaomiMiMo/MiMo-V2-Flash: - verdict: pass - evidence: current - attention: FlashAttentionBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - XiaomiMiMo/MiMo-V2.6-Flash-RL: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Flash-RL'' is not a valid HuggingFace model path or lo' - XiaomiMiMo/MiMo-V2.6-Pro-RL: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Pro-RL'' is not a valid HuggingFace model path or loca' - baerquant/Inkling: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling'' is not a valid HuggingFace model path or local path wit' - baerquant/Inkling-Small: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling-Small'' is not a valid HuggingFace model path or local pa' - baerquants/Inkling-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-NVFP4'' is not a valid HuggingFace model path or local p' - baerquants/Inkling-Small-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-Small-NVFP4'' is not a valid HuggingFace model path or l' - deepseek-ai/DeepSeek-R1: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V3: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V3.2: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V4-Flash: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V4-Flash-0731: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V4-Pro: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V4-Pro-0813: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - deepseek-ai/DeepSeek-V4.1-Flash: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Mxfp4FlashinferTrtllmMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - google/gemma-4-26B-A4B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - meta-llama/Llama-4-Maverick-17B-128E-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - meta-llama/Llama-4-Scout-17B-16E-Instruct: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - meta-llama/Meta-Llama-3.1-405B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - meta-llama/Meta-Llama-3.1-70B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - meta-llama/Meta-Llama-3.1-8B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - meta-models/Muse-Glimmer-30B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - mistralai/Mixtral-8x22B-v0.1: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - mistralai/Mixtral-8x7B-v0.1: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: UnquantizedFusedMoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - moonshotai/Kimi-K2-Instruct: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 moonshotai/Kimi-K2.5: verdict: pass evidence: current @@ -427,275 +18,6 @@ results: moe: CompressedTensorsFusedMoE->trtllm_gen_moe kv_allocated: auto topology: tp1 - moonshotai/Kimi-K3: - verdict: fail - cause: framework gap - error: 'ValueError: multi_ctas_kv_counter_buffer is only supported when a trtllm-gen runner is selected' - nvidia/DeepSeek-V3.1-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/DeepSeek-V3.2-NVFP4: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/DeepSeek-V4-Flash-NVFP4: - verdict: pass+custom - extra_args: --generator-set params.agg.moe_backend=marlin - evidence: current - attention: DeepseekV4AttnBackend - moe: ModelOptNvFp4FusedMoE->marlin - kv_allocated: auto - topology: tp1 - nvidia/DeepSeek-V4-Pro-NVFP4: - verdict: pass+custom - extra_args: --generator-set params.agg.moe_backend=marlin - evidence: current - attention: DeepseekV4AttnBackend - moe: ModelOptNvFp4FusedMoE->marlin - kv_allocated: auto - topology: tp1 - nvidia/DeepSeek-V4.1-Flash-NVFP4: - verdict: pass+custom - extra_args: --generator-set params.agg.moe_backend=marlin - evidence: current - attention: DeepseekV4AttnBackend - moe: ModelOptNvFp4FusedMoE->marlin - kv_allocated: auto - topology: tp1 - nvidia/GLM-5-NVFP4: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/GLM-5.1-NVFP4: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/GLM-5.2-NVFP4: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/GLM-5.3-Flash-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/GLM-5.3-Flash-NVFP4'' is not a valid HuggingFace model path or local' - nvidia/GLM-5.3-NVFP4: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Gemma-4-26B-A4B-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Gemma-4-31B-IT-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/Kimi-K2.5-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Kimi-K2.6-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Kimi-K2.7-Code-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMMLABackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Kimi-K3-NVFP4: - verdict: fail - cause: framework gap - error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' - nvidia/Llama-3.1-70B-Instruct-FP8: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/Llama-3_3-Nemotron-Super-49B-v1: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/MiniMax-M2.5-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/MiniMax-M2.7-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/MiniMax-M3-NVFP4: - verdict: pass+custom - extra_args: --generator-set params.agg.cuda_graph_backend_prefill=breakable - evidence: current - attention: MiniMaxHybridAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Muse-Glimmer-30B-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptFp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptFp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->marlin - kv_allocated: auto - topology: tp1 - nvidia/Nemotron-H-56B-Base-8K: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/Qwen3-235B-A22B-NVFP4: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Qwen3.5-122B-A10B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Qwen3.5-397B-A17B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - nvidia/Qwen3.6-27B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/Qwen3.6-35B-A3B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: ModelOptNvFp4FusedMoE->marlin - kv_allocated: auto - topology: tp1 - nvidia/Qwen3.8-2.4T-A95B-NVFP4: - verdict: fail - cause: framework gap - error: 'AttributeError: ''FusedMoE'' object has no attribute ''g1_scale_c''' - nvidia/Qwen3.8-27B-NVFP4: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - kv_allocated: auto - topology: tp1 - nvidia/Qwen3.8-Flash-Next-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/Qwen3.8-Flash-Next-NVFP4'' is not a valid HuggingFace model path or ' openai/gpt-oss-120b: verdict: pass evidence: current @@ -710,94 +32,3 @@ results: moe: Mxfp4MoE->trtllm_gen_moe kv_allocated: auto topology: tp1 - openbmb/MiniCPM5-2B: - verdict: pass - evidence: current - attention: TRTLLMHAAttnBackend - kv_allocated: auto - topology: tp1 - ornith-ai/Ornith-1.5-35B-A3B: - verdict: pass - evidence: current - attention: HybridLinearAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - sgl-project/DeepSeek-V4-Flash-FP8: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Fp8MoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - sgl-project/DeepSeek-V4-Pro-FP8: - verdict: pass - evidence: current - attention: DeepseekV4AttnBackend - moe: Fp8MoE->triton_fused_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5-FP8: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.1: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.1-FP8: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.2: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.2-FP8: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.3: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: Fp8MoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.3-BF16: - verdict: pass - evidence: current - attention: DeepseekSparseAttnBackend - moe: UnquantizedFusedMoE->trtllm_gen_moe - kv_allocated: auto - topology: tp1 - zai-org/GLM-5.3-Flash: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash'' is not a valid HuggingFace model path or local path' - zai-org/GLM-5.3-Flash-BF16: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash-BF16'' is not a valid HuggingFace model path or local' diff --git a/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml index b390b4482f..fb3e2ed530 100644 --- a/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml +++ b/benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml @@ -1,3 +1,5 @@ +# Excerpt (2026-10-30): _meta plus the models collector comments cite; _meta.summary +# counts the full matrix, which lands with the op-probe harness. _meta: platform: b200_sm100 framework: trtllm @@ -9,726 +11,7 @@ _meta: fail: 46 golden_facts_not_applied: 0 results: - MiniMaxAI/MiniMax-M2: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - MiniMaxAI/MiniMax-M2.5: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - MiniMaxAI/MiniMax-M2.7: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - MiniMaxAI/MiniMax-M3: - verdict: fail - cause: quant parser gap - error: 'AttributeError: ''NoneType'' object has no attribute ''shape''' - MiniMaxAI/MiniMax-M3-MXFP8: - verdict: fail - cause: quant parser gap - error: 'AttributeError: ''NoneType'' object has no attribute ''shape''' - Qwen/Qwen1.5-MoE-A2.7B: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3-0.6B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-1.7B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-235B-A22B: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3-235B-A22B-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - Qwen/Qwen3-235B-A22B-Instruct-2507: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3-30B-A3B: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3-30B-A3B-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - Qwen/Qwen3-32B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-32B-FP8: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-32B-FP8-Static-PerTensor: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-8B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-Coder-480B-A35B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-235B-A22B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-2B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-30B-A3B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-32B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-32B-Thinking: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-4B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3-VL-8B-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-0.8B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-122B-A10B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-27B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-2B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-35B-A3B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-397B-A17B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-4B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.5-9B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.8-2.4T-A95B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp4 - Qwen/Qwen3.8-2.4T-A95B-FP8: - verdict: fail - cause: framework gap - error: '[10/04/2026-11:07:37] [TRT-LLM] [W] [_torch][RANK 0] PyTorchModelEngine cleanup failed during destruction: ''PyTorchModelEngine'' object has no attribute ''_torch_compile_backend''' - Qwen/Qwen3.8-27B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.8-27B-FP8: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - Qwen/Qwen3.8-Flash-Next: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next'' is not a valid HuggingFace model path or local pa' - Qwen/Qwen3.8-Flash-Next-FP8: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''Qwen/Qwen3.8-Flash-Next-FP8'' is not a valid HuggingFace model path or loca' - TokenRhythm/NeoHorse-1-9B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - XiaomiMiMo/MiMo-7B-Base: - verdict: fail - cause: arch not registered - error: 'ValueError: Unknown architecture for AutoModelForCausalLM: MiMoForCausalLM' - XiaomiMiMo/MiMo-V2-Flash: - verdict: fail - cause: arch not registered - error: 'ValueError: Unknown architecture for AutoModelForCausalLM: MiMoV2FlashForCausalLM' - XiaomiMiMo/MiMo-V2.6-Flash-RL: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Flash-RL'' is not a valid HuggingFace model path or lo' - XiaomiMiMo/MiMo-V2.6-Pro-RL: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''XiaomiMiMo/MiMo-V2.6-Pro-RL'' is not a valid HuggingFace model path or loca' - baerquant/Inkling: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling'' is not a valid HuggingFace model path or local path wit' - baerquant/Inkling-Small: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquant/Inkling-Small'' is not a valid HuggingFace model path or local pa' - baerquants/Inkling-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-NVFP4'' is not a valid HuggingFace model path or local p' - baerquants/Inkling-Small-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''baerquants/Inkling-Small-NVFP4'' is not a valid HuggingFace model path or l' - deepseek-ai/DeepSeek-R1: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V3: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V3.2: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V4-Flash: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V4-Flash-0731: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V4-Pro: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V4-Pro-0813: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - deepseek-ai/DeepSeek-V4.1-Flash: - verdict: fail - cause: framework gap - error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports - this mod - google/gemma-4-26B-A4B: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - meta-llama/Llama-4-Maverick-17B-128E-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - meta-llama/Llama-4-Scout-17B-16E-Instruct: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - meta-llama/Meta-Llama-3.1-405B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - meta-llama/Meta-Llama-3.1-70B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - meta-llama/Meta-Llama-3.1-8B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - meta-models/Muse-Glimmer-30B: - verdict: fail - cause: framework gap - error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports - this mod - mistralai/Mixtral-8x22B-v0.1: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - mistralai/Mixtral-8x7B-v0.1: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - moonshotai/Kimi-K2-Instruct: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' moonshotai/Kimi-K2.5: verdict: fail cause: quant parser gap error: 'TypeError: ''NoneType'' object is not subscriptable' - moonshotai/Kimi-K3: - verdict: fail - cause: framework gap - error: 'ModuleNotFoundError: No module named ''fla''' - nvidia/DeepSeek-V3.1-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/DeepSeek-V3.2-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/DeepSeek-V4-Flash-NVFP4: - verdict: fail - cause: generator DSV4 kv facts unservable on this SM - error: 'ValueError: DeepSeek-V4 fp8_ds_mla KV cache requires tokens_per_block=256 on SM100, got 128.' - nvidia/DeepSeek-V4-Pro-NVFP4: - verdict: fail - cause: generator DSV4 kv facts unservable on this SM - error: 'ValueError: DeepSeek-V4 fp8_ds_mla KV cache requires tokens_per_block=256 on SM100, got 128.' - nvidia/DeepSeek-V4.1-Flash-NVFP4: - verdict: fail - cause: framework gap - error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports - this mod - nvidia/GLM-5-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/GLM-5.1-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/GLM-5.2-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/GLM-5.3-Flash-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/GLM-5.3-Flash-NVFP4'' is not a valid HuggingFace model path or local' - nvidia/GLM-5.3-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Gemma-4-26B-A4B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Gemma-4-31B-IT-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - nvidia/Kimi-K2.5-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Kimi-K2.6-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Kimi-K2.7-Code-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Kimi-K3-NVFP4: - verdict: fail - cause: framework gap - error: 'ModuleNotFoundError: No module named ''fla''' - nvidia/Llama-3.1-70B-Instruct-FP8: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - nvidia/Llama-3_3-Nemotron-Super-49B-v1: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - nvidia/MiniMax-M2.5-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/MiniMax-M2.7-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/MiniMax-M3-NVFP4: - verdict: fail - cause: quant parser gap - error: 'AttributeError: ''NoneType'' object has no attribute ''shape''' - nvidia/Muse-Glimmer-30B-NVFP4: - verdict: fail - cause: framework gap - error: You can update Transformers with the command `pip install --upgrade transformers`. If this does not work, and the checkpoint is very new, then there may not be a release version that supports - this mod - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-FP8: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: FP8QDQFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-FP8: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: FP8QDQFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-BF16: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: UnquantizedFusedMoE - kv_allocated: null - topology: tp1 - nvidia/NVIDIA-Nemotron-3.5-Lightning-30B-A3B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - moe: W4A16NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Nemotron-H-56B-Base-8K: - verdict: pass - evidence: current - attention: trtllm_fmha + _chunk_scan_fwd_kernel - kv_allocated: null - topology: tp1 - nvidia/Qwen3-235B-A22B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Qwen3.5-122B-A10B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Qwen3.5-397B-A17B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Qwen3.6-27B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - nvidia/Qwen3.6-35B-A3B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Qwen3.8-2.4T-A95B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: NVFP4CutlassFusedMoE - kv_allocated: null - topology: tp1 - nvidia/Qwen3.8-27B-NVFP4: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - kv_allocated: null - topology: tp1 - nvidia/Qwen3.8-Flash-Next-NVFP4: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''nvidia/Qwen3.8-Flash-Next-NVFP4'' is not a valid HuggingFace model path or ' - openai/gpt-oss-120b: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: W4A8MXFP4MXFP8CutlassFusedMoE - kv_allocated: null - topology: tp1 - openai/gpt-oss-20b: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: W4A8MXFP4MXFP8CutlassFusedMoE - kv_allocated: null - topology: tp1 - openbmb/MiniCPM5-2B: - verdict: pass - evidence: current - attention: trtllm_fmha - kv_allocated: null - topology: tp1 - ornith-ai/Ornith-1.5-35B-A3B: - verdict: pass - evidence: current - attention: trtllm_fmha + kernel_cutlass_gdn_decode_bf16state_mtp_ilp4_kernel - moe: UnquantizedFusedMoE->causal_conv1d/cutlass/fla_triton/flashinfer/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - sgl-project/DeepSeek-V4-Flash-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - sgl-project/DeepSeek-V4-Pro-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - zai-org/GLM-5: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - zai-org/GLM-5-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - zai-org/GLM-5.1: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - zai-org/GLM-5.1-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - zai-org/GLM-5.2: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - zai-org/GLM-5.2-FP8: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - zai-org/GLM-5.3: - verdict: fail - cause: rendered MoE backend cannot serve this layer on this SM - error: 'ValueError: no MoE implementation can serve this layer. MoE resolution: none (via failed, requested CUTLASS, env 492ea16eac8bc124); turned down: CutlassFusedMoE=sm_unsupported Each candidate''s - reason:' - zai-org/GLM-5.3-BF16: - verdict: pass - evidence: current - attention: trtllm_fmha - moe: UnquantizedFusedMoE->cutlass/dsa_nsa/triton_fused_moe/trtllm_fmha/trtllm_internal - kv_allocated: null - topology: tp1 - zai-org/GLM-5.3-Flash: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash'' is not a valid HuggingFace model path or local path' - zai-org/GLM-5.3-Flash-BF16: - verdict: fail - cause: generator rejects - error: 'golden render failed: aiconfigurator generate: error: argument --model-path/--model: ''zai-org/GLM-5.3-Flash-BF16'' is not a valid HuggingFace model path or local' diff --git a/benchmarks/reports/perf-data/perf-data-reuse-analysis.md b/benchmarks/reports/perf-data/perf-data-reuse-analysis.md index ae08b65e75..f907b2d169 100644 --- a/benchmarks/reports/perf-data/perf-data-reuse-analysis.md +++ b/benchmarks/reports/perf-data/perf-data-reuse-analysis.md @@ -9,41 +9,41 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur ## Headline numbers -- Total rows scanned: **31,755,054** -- Within-framework cross-version dedup-able rows: **47,131** (~0.1%) +- Total rows scanned: **15,891,586** +- Within-framework cross-version dedup-able rows: **8,584** (~0.1%) - Tier distribution (groups / rows): - - `shared`: 1299 groups · 31,156,118 rows - - `shared_fallback`: 54 groups · 598,936 rows + - `shared`: 1084 groups · 15,585,305 rows + - `shared_fallback`: 52 groups · 306,281 rows ## `computescale_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3242 | 0 / 1623 | 1 | — | — | — | -| b200_sxm | `sglang` | shared | sglang | sglang:2 | 0 / 1 | 0 | — | — | — | -| b200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 3256 | 0 | — | — | — | -| b300_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3255 | 0 / 1628 | 10 | — | — | — | -| b300_sxm | `sglang` | shared | sglang | sglang:2 | 0 / 1 | 0 | — | — | — | -| b300_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | -| gb200 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3253 | 0 / 1628 | 6 | — | — | — | -| gb200 | `sglang` | shared | sglang | sglang:5 | 0 / 4 | 0 | — | — | — | -| gb200 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | -| gb300 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:4880 | 0 / 1628 | 22 | — | — | — | -| gb300 | `sglang` | shared | sglang | sglang:16 | 0 / 10 | 0 | — | — | — | -| gb300 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 1 | — | — | — | -| h100_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 15 | — | — | — | -| h100_sxm | `sglang` | shared | sglang | sglang:11 | 0 / 8 | 0 | — | — | — | -| h100_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 10 | — | — | — | -| h200_sxm | `sglang` | shared | sglang | sglang:17 | 0 / 11 | 0 | — | — | — | -| h200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | -| l40s | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3155 | 0 / 1584 | 182 | — | — | — | -| l40s | `sglang` | shared | sglang | sglang:35 | 0 / 18 | 1 | — | — | — | -| l40s | `torch_ops` | shared | trtllm | trtllm:3234 | 0 / 1627 | 1 | — | — | — | -| rtx_pro_6000_server | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:3216 | 0 / 1615 | 7 | — | — | — | -| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:20 | 0 / 10 | 0 | — | — | — | -| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:3237 | 0 / 1625 | 0 | — | — | — | +| b200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1620 | 0 / 1620 | 0 | — | — | — | +| b200_sxm | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | +| b200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| b300_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| b300_sxm | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | +| b300_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| gb200 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1625 | 0 / 1625 | 0 | — | — | — | +| gb200 | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | +| gb200 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| gb300 | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1626 | 0 / 1626 | 0 | — | — | — | +| gb300 | `sglang` | shared | sglang | sglang:1 | 0 / 1 | 0 | — | — | — | +| gb300 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `sglang` | shared | sglang | sglang:3 | 0 / 3 | 0 | — | — | — | +| h100_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `sglang` | shared | sglang | sglang:6 | 0 / 6 | 0 | — | — | — | +| h200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| l40s | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1575 | 0 / 1575 | 0 | — | — | — | +| l40s | `sglang` | shared | sglang | sglang:18 | 0 / 18 | 0 | — | — | — | +| l40s | `torch_ops` | shared | trtllm | trtllm:1607 | 0 / 1607 | 0 | — | — | — | +| rtx_pro_6000_server | `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | shared | vllm | vllm:1611 | 0 / 1611 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:10 | 0 / 10 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:1613 | 0 / 1613 | 0 | — | — | — | ## `context_attention_perf.parquet` @@ -52,101 +52,90 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `flash_attention` | shared | sglang | sglang:5627 | 0 / 5627 | 0 | — | — | — | | a100_sxm | `torch_flow` | shared | trtllm | trtllm:4864 | 0 / 4864 | 0 | — | — | — | | a100_sxm | `vllm_flash_attn` | shared | vllm | vllm:5457 | 0 / 5049 | 0 | — | — | — | -| b200_sxm | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 4 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:139563 | 0 / 139563 | 0 | — | — | — | -| b200_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 5100 | 0 | — | — | — | -| b200_sxm | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 15 | — | — | — | -| b200_sxm | `trtllm_mha` | shared | sglang | sglang:65618 | 0 / 33526 | 32 | — | — | — | -| b200_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| b200_sxm | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:68535 | 0 / 68535 | 0 | — | — | — | +| b200_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | +| b200_sxm | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | +| b200_sxm | `trtllm_mha` | shared | sglang | sglang:32093 | 0 / 32093 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer` | shared | vllm | vllm:35360 | 0 / 35360 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | -| b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3756 | 0 / 3756 | 0 | — | — | — | -| b200_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:139508 | 0 / 47304 | 243 | — | — | — | -| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:7222 | 0 / 3961 | 6 | — | — | — | -| b300_sxm | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 5 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:138136 | 0 / 71030 | 53 | — | — | — | -| b300_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 2788 | 6 | — | — | — | -| b300_sxm | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 14 | — | — | — | -| b300_sxm | `trtllm_mha` | shared | sglang | sglang:65619 | 0 / 33526 | 30 | — | — | — | -| b300_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:1626 | 0 / 1626 | 0 | — | — | — | +| b200_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:92848 | 0 / 47304 | 183 | — | — | — | +| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:4896 | 0 / 3264 | 3 | — | — | — | +| b300_sxm | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:67106 | 0 / 67106 | 0 | — | — | — | +| b300_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | +| b300_sxm | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | +| b300_sxm | `trtllm_mha` | shared | sglang | sglang:32097 | 0 / 32097 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | -| b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3668 | 0 / 3668 | 0 | — | — | — | -| b300_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:138440 | 0 / 47304 | 303 | — | — | — | -| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:5588 | 0 / 2329 | 15 | — | — | — | -| b60 | `vllm_flash_attn` | shared | vllm | vllm:31688 | 0 / 17930 | 1 | — | — | — | -| gb200 | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 5 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:139562 | 0 / 71027 | 53 | — | — | — | -| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 2788 | 2 | — | — | — | -| gb200 | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 20 | — | — | — | -| gb200 | `trtllm_mha` | shared | sglang | sglang:64661 | 0 / 33525 | 28 | — | — | — | -| gb200 | `vllm_flash_attn_fa2` | shared | vllm | vllm:1630 | 0 / 1630 | 0 | — | — | — | +| b300_sxm | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:91780 | 0 / 47304 | 178 | — | — | — | +| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:3264 | 0 / 1632 | 7 | — | — | — | +| b60 | `vllm_flash_attn` | shared | vllm | vllm:34118 | 0 / 17930 | 1 | — | — | — | +| gb200 | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:68535 | 0 / 68535 | 0 | — | — | — | +| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | +| gb200 | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | +| gb200 | `trtllm_mha` | shared | sglang | sglang:31138 | 0 / 31138 | 0 | — | — | — | | gb200 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | -| gb200 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | gb200 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3740 | 0 / 3740 | 0 | — | — | — | -| gb200 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:139320 | 0 / 47304 | 232 | — | — | — | -| gb200 | `vllm_triton_attn` | shared | vllm | vllm:5591 | 0 / 2329 | 6 | — | — | — | -| gb300 | `flashinfer` | shared | sglang | sglang:5168 | 0 / 2584 | 3 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:139563 | 0 / 71030 | 49 | — | — | — | -| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:5100 | 0 / 2788 | 5 | — | — | — | -| gb300 | `triton` | shared | sglang | sglang:18972 | 0 / 9486 | 4 | — | — | — | -| gb300 | `trtllm_mha` | shared | sglang | sglang:64667 | 0 / 33525 | 18 | — | — | — | -| gb300 | `vllm_flash_attn_fa2` | shared | vllm | vllm:1629 | 0 / 1629 | 0 | — | — | — | +| gb200 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:92660 | 0 / 47304 | 124 | — | — | — | +| gb200 | `vllm_triton_attn` | shared | vllm | vllm:3264 | 0 / 1632 | 3 | — | — | — | +| gb300 | `flashinfer` | shared | sglang | sglang:2584 | 0 / 2584 | 0 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:68533 | 0 / 68533 | 0 | — | — | — | +| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:2312 | 0 / 2312 | 0 | — | — | — | +| gb300 | `triton` | shared | sglang | sglang:9486 | 0 / 9486 | 0 | — | — | — | +| gb300 | `trtllm_mha` | shared | sglang | sglang:31144 | 0 / 31144 | 0 | — | — | — | | gb300 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:3900 | 0 / 3900 | 0 | — | — | — | -| gb300 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:3847 | 0 / 3847 | 0 | — | — | — | | gb300 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:3740 | 0 / 3740 | 0 | — | — | — | -| gb300 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:139320 | 0 / 47304 | 301 | — | — | — | -| gb300 | `vllm_triton_attn` | shared | vllm | vllm:5587 | 0 / 2329 | 5 | — | — | — | -| h100_sxm | `fa3` | shared | sglang | sglang:66100 | 0 / 34478 | 28 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:154782 | 0 / 79050 | 57 | — | — | — | -| h100_sxm | `triton` | shared | sglang | sglang:7650 | 0 / 4182 | 0 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:156400 | 0 / 53074 | 93 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1972 | 0 / 697 | 3 | — | — | — | -| h100_sxm | `vllm_triton_attn` | shared | vllm | vllm:697 | 0 / 697 | 0 | — | — | — | -| h200_sxm | `fa3` | shared | sglang | sglang:67052 | 0 / 34478 | 31 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:156199 | 0 / 79038 | 69 | — | — | — | -| h200_sxm | `triton` | shared | sglang | sglang:7650 | 0 / 4182 | 1 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:157352 | 0 / 53074 | 320 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1972 | 0 / 697 | 3 | — | — | — | -| h200_sxm | `vllm_triton_attn` | shared | vllm | vllm:697 | 0 / 697 | 0 | — | — | — | -| l40s | `flashinfer` | shared | sglang | sglang:52506 | 0 / 29092 | 477 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:154712 | 0 / 78973 | 1671 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:26622 | 0 / 16830 | 39 | — | — | — | -| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:51624 | 0 / 26537 | 1065 | — | — | — | -| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:3690 | 0 / 1946 | 90 | — | — | — | -| l40s | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:45334 | 0 / 23644 | 294 | — | — | — | -| l40s | `vllm_triton_attn` | shared | vllm | vllm:3961 | 0 / 2329 | 35 | — | — | — | -| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:49692 | 0 / 26282 | 50 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:147105 | 0 / 75176 | 179 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:32946 | 0 / 16830 | 15 | — | — | — | -| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:51662 | 0 / 26537 | 71 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:1781 | 0 / 1781 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:45312 | 0 / 23635 | 29 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_flashinfertrtllmapidecode_xqa` | shared | vllm | vllm:1807 | 0 / 1807 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:3961 | 0 / 2329 | 1 | — | — | — | +| gb300 | `vllm_flashinfer_trtllmprefill` | shared | vllm | vllm:92660 | 0 / 47304 | 207 | — | — | — | +| gb300 | `vllm_triton_attn` | shared | vllm | vllm:3264 | 0 / 1632 | 4 | — | — | — | +| h100_sxm | `fa3` | shared | sglang | sglang:31622 | 0 / 31622 | 0 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:75840 | 0 / 75840 | 0 | — | — | — | +| h100_sxm | `triton` | shared | sglang | sglang:3468 | 0 / 3468 | 0 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:103326 | 0 / 53074 | 12 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1275 | 0 / 697 | 1 | — | — | — | +| h200_sxm | `fa3` | shared | sglang | sglang:32574 | 0 / 32574 | 0 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:77265 | 0 / 77265 | 0 | — | — | — | +| h200_sxm | `triton` | shared | sglang | sglang:3468 | 0 / 3468 | 0 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:104278 | 0 / 53074 | 155 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1275 | 0 / 697 | 1 | — | — | — | +| l40s | `flashinfer` | shared | sglang | sglang:26236 | 0 / 26236 | 0 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:75768 | 0 / 75768 | 0 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:9792 | 0 / 9792 | 0 | — | — | — | +| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:25092 | 0 / 25092 | 0 | — | — | — | +| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:1744 | 0 / 1744 | 0 | — | — | — | +| l40s | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:21690 | 0 / 21690 | 0 | — | — | — | +| l40s | `vllm_triton_attn` | shared | vllm | vllm:1632 | 0 / 1632 | 0 | — | — | — | +| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:23410 | 0 / 23410 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:71929 | 0 / 71929 | 0 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:16116 | 0 / 16116 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:25126 | 0 / 25126 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:1728 | 0 / 1728 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_fiprefill` | shared | vllm | vllm:21677 | 0 / 21677 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:1632 | 0 / 1632 | 0 | — | — | — | ## `context_mla_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | a100_sxm | `default` | shared_fallback | trtllm | trtllm:2436 | 0 / 2436 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:6160 | 0 / 4620 | 1 | — | — | — | -| b200_sxm | `trtllm_mla` | shared | sglang | sglang:6160 | 0 / 3080 | 4 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 3 | — | — | — | -| b300_sxm | `trtllm_mla` | shared | sglang | sglang:6160 | 0 / 3080 | 7 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 4 | — | — | — | -| gb200 | `trtllm_mla` | shared | sglang | sglang:6600 | 0 / 3960 | 0 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 4 | — | — | — | -| gb300 | `trtllm_mla` | shared | sglang | sglang:6600 | 0 / 3960 | 3 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 3 | — | — | — | -| h100_sxm | `flash_attention` | shared | sglang | sglang:6600 | 0 / 3080 | 1 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:6160 | 0 / 4620 | 4 | — | — | — | -| h200_sxm | `flash_attention` | shared | sglang | sglang:6160 | 0 / 3080 | 2 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:3080 | 0 / 3080 | 0 | — | — | — | +| b200_sxm | `trtllm_mla` | shared | sglang | sglang:3080 | 0 / 3080 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | +| b300_sxm | `trtllm_mla` | shared | sglang | sglang:3080 | 0 / 3080 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | +| gb200 | `trtllm_mla` | shared | sglang | sglang:3520 | 0 / 1760 | 0 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | +| gb300 | `trtllm_mla` | shared | sglang | sglang:3520 | 0 / 1760 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | +| h100_sxm | `flash_attention` | shared | sglang | sglang:3520 | 0 / 1760 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:3080 | 0 / 3080 | 0 | — | — | — | +| h200_sxm | `flash_attention` | shared | sglang | sglang:3080 | 0 / 3080 | 0 | — | — | — | | l40s | `default` | shared_fallback | trtllm | trtllm:2436 | 0 / 2436 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:2420 | 0 / 1540 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:4840 | 0 / 3080 | 4 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:3300 | 0 / 1540 | 2 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:880 | 0 / 880 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:1760 | 0 / 1760 | 0 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:1760 | 0 / 880 | 0 | — | — | — | ## `custom_allreduce_perf.parquet` @@ -162,35 +151,34 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | b200_sxm | `TRTLLM` | shared | trtllm | trtllm:207 | 0 / 138 | 0 | — | — | — | | b200_sxm | `vLLM_custom_eager` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | | b200_sxm | `vLLM_custom_graph` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | -| b300_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | -| b300_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | +| b300_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | +| b300_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | | b300_sxm | `TRTLLM` | shared | trtllm | trtllm:138 | 0 / 69 | 1 | — | — | — | -| b300_sxm | `vLLM_custom_eager` | shared | vllm | vllm:207 | 0 / 69 | 0 | — | — | — | -| b300_sxm | `vLLM_custom_graph` | shared | vllm | vllm:207 | 0 / 69 | 0 | — | — | — | -| b60 | `vLLM_custom_eager` | shared | vllm | vllm:115 | 0 / 69 | 0 | — | — | — | -| b60 | `vLLM_custom_graph` | shared | vllm | vllm:46 | 0 / 46 | 0 | — | — | — | -| gb200 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | -| gb200 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | +| b300_sxm | `vLLM_custom_eager` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | +| b300_sxm | `vLLM_custom_graph` | shared | vllm | vllm:138 | 0 / 69 | 0 | — | — | — | +| b60 | `vLLM_custom_eager` | shared | vllm | vllm:207 | 0 / 69 | 23 | — | — | — | +| gb200 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | +| gb200 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | | gb200 | `TRTLLM` | shared | trtllm | trtllm:92 | 0 / 46 | 0 | — | — | — | -| gb200 | `vLLM_custom_eager` | shared | vllm | vllm:184 | 0 / 46 | 0 | — | — | — | -| gb200 | `vLLM_custom_graph` | shared | vllm | vllm:184 | 0 / 46 | 0 | — | — | — | -| gb300 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | -| gb300 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 92 | 0 | — | — | — | +| gb200 | `vLLM_custom_eager` | shared | vllm | vllm:138 | 0 / 46 | 0 | — | — | — | +| gb200 | `vLLM_custom_graph` | shared | vllm | vllm:138 | 0 / 46 | 0 | — | — | — | +| gb300 | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | +| gb300 | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:230 | 0 / 92 | 0 | — | — | — | | gb300 | `TRTLLM` | shared | trtllm | trtllm:92 | 0 / 46 | 0 | — | — | — | | gb300 | `TRTLLM_MNNVL_oneshot` | shared | trtllm | trtllm:19 | 0 / 19 | 0 | — | — | — | | gb300 | `TRTLLM_MNNVL_twoshot` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | -| gb300 | `vLLM_custom_eager` | shared | vllm | vllm:276 | 0 / 92 | 0 | — | — | — | -| gb300 | `vLLM_custom_graph` | shared | vllm | vllm:276 | 0 / 92 | 0 | — | — | — | -| h100_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | +| gb300 | `vLLM_custom_eager` | shared | vllm | vllm:230 | 0 / 92 | 0 | — | — | — | +| gb300 | `vLLM_custom_graph` | shared | vllm | vllm:230 | 0 / 92 | 0 | — | — | — | +| h100_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | | h100_sxm | `TRTLLM` | shared | trtllm | trtllm:276 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `vLLM_custom_eager` | shared | vllm | vllm:276 | 0 / 69 | 0 | — | — | — | -| h100_sxm | `vLLM_custom_graph` | shared | vllm | vllm:276 | 0 / 69 | 1 | — | — | — | -| h200_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | -| h200_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:276 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `vLLM_custom_eager` | shared | vllm | vllm:207 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `vLLM_custom_graph` | shared | vllm | vllm:207 | 0 / 69 | 1 | — | — | — | +| h200_sxm | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | +| h200_sxm | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | | h200_sxm | `TRTLLM` | shared | trtllm | trtllm:276 | 0 / 69 | 0 | — | — | — | -| h200_sxm | `vLLM_custom_eager` | shared | vllm | vllm:345 | 0 / 69 | 0 | — | — | — | -| h200_sxm | `vLLM_custom_graph` | shared | vllm | vllm:345 | 0 / 69 | 1 | — | — | — | +| h200_sxm | `vLLM_custom_eager` | shared | vllm | vllm:276 | 0 / 69 | 0 | — | — | — | +| h200_sxm | `vLLM_custom_graph` | shared | vllm | vllm:276 | 0 / 69 | 1 | — | — | — | | l40s | `SGLang_CustomAllReduce_eager` | shared | sglang | sglang:207 | 0 / 69 | 0 | — | — | — | | l40s | `SGLang_CustomAllReduce_graph` | shared | sglang | sglang:207 | 0 / 69 | 1 | — | — | — | | l40s | `TRTLLM` | shared | trtllm | trtllm:276 | 0 / 69 | 0 | — | — | — | @@ -206,95 +194,89 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254766 | 0 / 159801 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:138371 | 0 / 87864 | 4 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:29274 | 0 / 29274 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:69992 | 0 / 35080 | 14 | — | — | — | -| b200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:34946 | 0 / 29790 | 3 | — | — | — | -| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:34716 | 0 / 29702 | 1 | — | — | — | -| b200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:25657 | 0 / 12963 | 5 | — | — | — | -| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:25654 | 0 / 12960 | 3 | — | — | — | -| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254692 | 0 / 159780 | 5 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:138360 | 0 / 87864 | 4 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:29277 | 0 / 14640 | 19 | — | — | — | -| b300_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:62869 | 0 / 34972 | 1 | — | — | — | -| b300_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:34605 | 0 / 29449 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:34946 | 0 / 29790 | 2 | — | — | — | -| b300_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:19086 | 0 / 19086 | 0 | — | — | — | -| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:19174 | 0 / 19174 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254692 | 0 / 159780 | 2 | — | — | — | -| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:138360 | 0 / 87864 | 7 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:29276 | 0 / 14640 | 18 | — | — | — | -| gb200 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:81451 | 0 / 35080 | 0 | — | — | — | -| gb200 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:49046 | 0 / 29678 | 0 | — | — | — | -| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:43463 | 0 / 29678 | 2 | — | — | — | -| gb200 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:20966 | 0 / 19062 | 0 | — | — | — | -| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:20572 | 0 / 19062 | 0 | — | — | — | -| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:254692 | 0 / 159780 | 2 | — | — | — | -| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:138360 | 0 / 87864 | 3 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:29275 | 0 / 14640 | 19 | — | — | — | -| gb300 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:63144 | 0 / 35080 | 1 | — | — | — | -| gb300 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:34834 | 0 / 29678 | 0 | — | — | — | -| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:34834 | 0 / 29678 | 1 | — | — | — | -| gb300 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:19062 | 0 / 19062 | 0 | — | — | — | -| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:19062 | 0 / 19062 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:227400 | 0 / 143058 | 15 | — | — | — | -| h100_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:194364 | 0 / 128988 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:19444 | 0 / 7790 | 1 | — | — | — | -| h100_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:80601 | 0 / 31203 | 5 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:42717 | 0 / 24685 | 3 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:41017 | 0 / 23131 | 1 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:22083 | 0 / 11047 | 11 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:19024 | 0 / 9512 | 2 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:227700 | 0 / 143244 | 12 | — | — | — | -| h200_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:194364 | 0 / 128988 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:19520 | 0 / 7808 | 10 | — | — | — | -| h200_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:64510 | 0 / 32280 | 4 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:29171 | 0 / 24799 | 3 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:29157 | 0 / 24785 | 2 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:22146 | 0 / 11126 | 4 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:22216 | 0 / 11112 | 2 | — | — | — | -| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_SM120` | shared | vllm | vllm:107980 | 0 / 71660 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:4392 | 0 / 4392 | 0 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:35080 | 0 / 35080 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:17096 | 0 / 17096 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:17104 | 0 / 17104 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:12963 | 0 / 12963 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:12960 | 0 / 12960 | 0 | — | — | — | +| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:4392 | 0 / 4392 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:27896 | 0 / 20940 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:16918 | 0 / 16918 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:17096 | 0 / 17096 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:6392 | 0 / 6392 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:6480 | 0 / 6480 | 0 | — | — | — | +| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:4391 | 0 / 4391 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | +| gb200 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:46371 | 0 / 21048 | 0 | — | — | — | +| gb200 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:31196 | 0 / 16984 | 0 | — | — | — | +| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:25613 | 0 / 16984 | 0 | — | — | — | +| gb200 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:8272 | 0 / 6368 | 0 | — | — | — | +| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:7878 | 0 / 6368 | 0 | — | — | — | +| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:10248 | 0 / 10248 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:4392 | 0 / 4392 | 0 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:14640 | 0 / 14640 | 0 | — | — | — | +| gb300 | `sglang_dsa_dense_mha_trtllm_ragged` | shared | sglang | sglang:28064 | 0 / 21048 | 0 | — | — | — | +| gb300 | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:16984 | 0 / 16984 | 0 | — | — | — | +| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:16984 | 0 / 16984 | 0 | — | — | — | +| gb300 | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:6368 | 0 / 6368 | 0 | — | — | — | +| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:6368 | 0 / 6368 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:14668 | 0 / 14668 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:11654 | 0 / 7770 | 1 | — | — | — | +| h100_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:50621 | 0 / 31203 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:27320 | 0 / 13660 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:27312 | 0 / 13619 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:11037 | 0 / 11037 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:9512 | 0 / 9512 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:14816 | 0 / 14816 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 7808 | 10 | — | — | — | +| h200_sxm | `sglang_dsa_dense_mha_fa3` | shared | sglang | sglang:32280 | 0 / 32280 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:13660 | 0 / 13660 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_flashmla_sparse` | shared | sglang | sglang:13660 | 0 / 13660 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:11125 | 0 / 11125 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_flashmla_sparse` | shared | sglang | sglang:11111 | 0 / 11111 | 0 | — | — | — | ## `dsa_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40222 | 0 / 26706 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:19894 | 0 / 13248 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:22077 | 0 / 22077 | 0 | — | — | — | -| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 2 | — | — | — | -| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:4912 | 0 / 2464 | 6 | — | — | — | -| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40584 | 0 / 26916 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:22070 | 0 / 11039 | 120 | — | — | — | -| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 3 | — | — | — | -| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:3688 | 0 / 3688 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40584 | 0 / 26916 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:22070 | 0 / 11040 | 60 | — | — | — | -| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 2 | — | — | — | -| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:3688 | 0 / 3688 | 0 | — | — | — | -| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:40584 | 0 / 26916 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:22062 | 0 / 11040 | 78 | — | — | — | -| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:7256 | 0 / 6072 | 3 | — | — | — | -| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:3688 | 0 / 3688 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:46782 | 0 / 31083 | 0 | — | — | — | -| h100_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:19548 | 0 / 13032 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:14720 | 0 / 5888 | 10 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:5396 | 0 / 3020 | 0 | — | — | — | -| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:5396 | 0 / 3020 | 1 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:2456 | 0 / 1232 | 8 | — | — | — | -| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:2456 | 0 / 1232 | 11 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:46998 | 0 / 31227 | 0 | — | — | — | -| h200_sxm | `FLASH_ATTN_MLA_SPARSE` | shared | vllm | vllm:19872 | 0 / 13248 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:14720 | 0 / 5888 | 16 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:3612 | 0 / 3020 | 3 | — | — | — | -| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:3612 | 0 / 3020 | 0 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:2456 | 0 / 1232 | 7 | — | — | — | -| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:2456 | 0 / 1232 | 1 | — | — | — | -| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_SM120` | shared | vllm | vllm:11040 | 0 / 7360 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7705 | 0 / 7705 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:3294 | 0 / 3294 | 0 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:11040 | 0 / 11040 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | +| b200_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:2448 | 0 / 2448 | 0 | — | — | — | +| b300_sxm | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7576 | 0 / 7576 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:3277 | 0 / 3277 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:11037 | 0 / 11037 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | +| b300_sxm | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| gb200 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7728 | 0 / 7728 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE` | shared | vllm | vllm:3311 | 0 / 3311 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:11036 | 0 / 11036 | 0 | — | — | — | +| gb200 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | +| gb200 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| gb300 | `FLASHINFER_MLA_SPARSE` | shared | vllm | vllm:7727 | 0 / 7727 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE` | shared | vllm | vllm:3312 | 0 / 3312 | 0 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:11025 | 0 / 11025 | 0 | — | — | — | +| gb300 | `sglang_dsa_indexer_trtllm` | shared | sglang | sglang:3600 | 0 / 3600 | 0 | — | — | — | +| gb300 | `sglang_dsa_skip_indexer_trtllm` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:11679 | 0 / 11679 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 5888 | 2 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:3568 | 0 / 1784 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:3568 | 0 / 1784 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| h100_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE` | shared | vllm | vllm:11776 | 0 / 11776 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 5888 | 7 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_fa3` | shared | sglang | sglang:1784 | 0 / 1784 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_indexer_flashmla_kv` | shared | sglang | sglang:1784 | 0 / 1784 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_fa3` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | +| h200_sxm | `sglang_dsa_skip_indexer_flashmla_kv` | shared | sglang | sglang:1224 | 0 / 1224 | 0 | — | — | — | ## `dsv41_module_perf.parquet` @@ -325,117 +307,116 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11421 | 0 / 11421 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17510 | 0 / 8760 | 39 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:89024 | 0 / 67120 | 12 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17373 | 0 / 8624 | 35 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:137145 | 0 / 70160 | 6 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11419 | 0 / 11419 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17507 | 0 / 8760 | 27 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:100528 | 0 / 68354 | 11 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17371 | 0 / 8624 | 28 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:142106 | 0 / 70152 | 8 | — | — | — | -| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5739 | 0 / 5739 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17325 | 0 / 5790 | 47 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:92982 | 0 / 40384 | 9 | — | — | — | -| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5840 | 0 / 5840 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17510 | 0 / 5840 | 62 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:80976 | 0 / 40592 | 11 | — | — | — | -| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:5780 | 0 / 5780 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5587 | 0 / 5587 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 20 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:43804 | 0 / 43804 | 0 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11544 | 0 / 8624 | 22 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:89689 | 0 / 45449 | 0 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5584 | 0 / 5584 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 9 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:53376 | 0 / 42610 | 0 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11544 | 0 / 8624 | 10 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:94650 | 0 / 45424 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11565 | 0 / 5790 | 17 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:52902 | 0 / 40384 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 5840 | 45 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:40592 | 0 / 40592 | 0 | — | — | — | ## `dsv4_csa_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 17 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 4 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4136 | 0 / 1820 | 22 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 1 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 13 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 2 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4136 | 0 / 1820 | 17 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 3 | — | — | — | -| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 43 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:6396 | 0 / 3200 | 4 | — | — | — | -| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 91 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 3200 | 10 | — | — | — | -| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:1544 | 0 / 1544 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 9 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:2592 | 0 / 1820 | 12 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 6 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:2592 | 0 / 1820 | 6 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 10 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 29 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | | rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:909 | 0 / 868 | 0 | — | — | — | ## `dsv4_csa_topk_calib_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | -| b200_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | -| b300_sxm | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | -| b300_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | -| gb200 | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | -| gb200 | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | -| gb300 | `topk_transform_v1` | shared | sglang | sglang:7156 | 0 / 4598 | 0 | — | — | — | -| gb300 | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | -| h100_sxm | `topk_transform_v1` | shared | sglang | sglang:5750 | 0 / 3686 | 0 | — | — | — | -| h100_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | -| h200_sxm | `topk_transform_v1` | shared | sglang | sglang:5750 | 0 / 3686 | 0 | — | — | — | -| h200_sxm | `topk_transform_v2` | shared | sglang | sglang:384 | 0 / 244 | 0 | — | — | — | +| b200_sxm | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | +| b200_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | +| b300_sxm | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | +| b300_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | +| gb200 | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | +| gb200 | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | +| gb300 | `topk_transform_v1` | shared | sglang | sglang:4598 | 0 / 4598 | 0 | — | — | — | +| gb300 | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | +| h100_sxm | `topk_transform_v1` | shared | sglang | sglang:3686 | 0 / 3686 | 0 | — | — | — | +| h100_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | +| h200_sxm | `topk_transform_v1` | shared | sglang | sglang:3686 | 0 / 3686 | 0 | — | — | — | +| h200_sxm | `topk_transform_v2` | shared | sglang | sglang:244 | 0 / 244 | 0 | — | — | — | + +## `dsv4_hca_attn_module_perf.parquet` + +| system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | +|---|---|---|---|---|---|---|---|---|---| +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 155 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 112 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 116 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 90 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 42 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:5877 | 0 / 3918 | 128 | — | — | — | ## `dsv4_hca_context_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11429 | 0 / 11429 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 32 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:95152 | 0 / 71424 | 3 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 51 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:109223 | 0 / 71470 | 11 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11424 | 0 / 11424 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 19 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:95152 | 0 / 71424 | 9 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:11440 | 0 / 11440 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 8760 | 37 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:102620 | 0 / 71472 | 11 | — | — | — | -| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5793 | 0 / 5793 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17379 | 0 / 5793 | 65 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:79855 | 0 / 40624 | 14 | — | — | — | -| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5840 | 0 / 5840 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:17520 | 0 / 5840 | 118 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:81376 | 0 / 40784 | 8 | — | — | — | -| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:5810 | 0 / 5810 | 0 | — | — | — | -| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:79856 | 0 / 40048 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5589 | 0 / 5589 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 17 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:47456 | 0 / 47456 | 0 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 43 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:61527 | 0 / 47550 | 0 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5584 | 0 / 5584 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 15 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:47456 | 0 / 47456 | 0 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:5600 | 0 / 5600 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 8760 | 10 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:54924 | 0 / 47552 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11586 | 0 / 5793 | 42 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:40383 | 0 / 40383 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:11680 | 0 / 5840 | 65 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:40592 | 0 / 40592 | 0 | — | — | — | +| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:40384 | 0 / 20192 | 0 | — | — | — | ## `dsv4_hca_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 17 | — | — | — | -| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 10 | — | — | — | -| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 24 | — | — | — | -| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 4 | — | — | — | -| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 12 | — | — | — | -| gb200 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 11 | — | — | — | -| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:3088 | 0 / 3088 | 0 | — | — | — | -| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 2316 | 29 | — | — | — | -| gb300 | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 4800 | 14 | — | — | — | -| h100_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 56 | — | — | — | -| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 3200 | 5 | — | — | — | -| h200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | -| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:4632 | 0 / 1544 | 123 | — | — | — | -| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:6400 | 0 / 3200 | 2 | — | — | — | -| rtx_pro_6000_server | `FLASHINFER_MLA_SPARSE_DSV4` | shared | vllm | vllm:1544 | 0 / 1544 | 0 | — | — | — | -| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:4800 | 0 / 3200 | 0 | — | — | — | +| b200_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| b200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 10 | — | — | — | +| b200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| b300_sxm | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| b300_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 20 | — | — | — | +| b300_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| gb200 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| gb200 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 6 | — | — | — | +| gb200 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| gb300 | `DeepseekV4TrtllmAttention` | shared | trtllm | trtllm:1544 | 0 / 1544 | 0 | — | — | — | +| gb300 | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 2316 | 7 | — | — | — | +| gb300 | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| h100_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 10 | — | — | — | +| h100_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| h200_sxm | `FLASHMLA_SPARSE_DSV4` | shared | vllm | vllm:3088 | 0 / 1544 | 47 | — | — | — | +| h200_sxm | `compressed_flashmla` | shared | sglang | sglang:3200 | 0 / 3200 | 0 | — | — | — | +| rtx_pro_6000_server | `compressed_flashmla` | shared | sglang | sglang:1600 | 0 / 1600 | 0 | — | — | — | ## `dsv4_megamoe_module_perf.parquet` @@ -449,50 +430,47 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:6971 | 0 / 4790 | 0 | — | — | — | -| b200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 718 | — | — | — | -| b300_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:9152 | 0 / 4790 | 0 | — | — | — | -| b300_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 879 | — | — | — | -| gb200 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:9051 | 0 / 4790 | 0 | — | — | — | -| gb200 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 398 | — | — | — | -| gb300 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:9152 | 0 / 4790 | 0 | — | — | — | -| gb300 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 471 | — | — | — | -| h100_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:6165 | 0 / 4110 | 0 | — | — | — | -| h100_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 351 | — | — | — | -| h200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:6165 | 0 / 4110 | 0 | — | — | — | -| h200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:9795 | 0 / 3918 | 432 | — | — | — | -| rtx_pro_6000_server | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:3918 | 0 / 3918 | 0 | — | — | — | +| b200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:2181 | 0 / 2181 | 0 | — | — | — | +| b200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 226 | — | — | — | +| b300_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:4362 | 0 / 2181 | 0 | — | — | — | +| b300_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 263 | — | — | — | +| gb200 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:4261 | 0 / 2181 | 0 | — | — | — | +| gb200 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 86 | — | — | — | +| gb300 | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:4362 | 0 / 2181 | 0 | — | — | — | +| gb300 | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 149 | — | — | — | +| h100_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:2055 | 0 / 2055 | 0 | — | — | — | +| h100_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 77 | — | — | — | +| h200_sxm | `deep_gemm.fp8_paged_mqa_logits` | shared | sglang | sglang:2055 | 0 / 2055 | 0 | — | — | — | +| h200_sxm | `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | shared | vllm | vllm:5877 | 0 / 3918 | 192 | — | — | — | ## `encoder_attention_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `flash_attention_v4` | shared | sglang | sglang:15358 | 0 / 7679 | 12 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 15358 | 0 | — | — | — | -| b200_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 62 | — | — | — | -| b300_sxm | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 21 | — | — | — | +| b200_sxm | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| b200_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 32 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | | b300_sxm | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| b300_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 53 | — | — | — | -| gb200 | `flash_attention_v4` | shared | sglang | sglang:15358 | 0 / 7679 | 14 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 14 | — | — | — | -| gb200 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 49 | — | — | — | -| gb300 | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 17 | — | — | — | +| b300_sxm | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 28 | — | — | — | +| gb200 | `flash_attention_v4` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| gb200 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 26 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | | gb300 | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | -| gb300 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:23037 | 0 / 7679 | 45 | — | — | — | -| h100_sxm | `flash_attention_v3` | shared | sglang | sglang:15358 | 0 / 7679 | 3 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 16 | — | — | — | -| h100_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:23037 | 0 / 7679 | 21 | — | — | — | -| h200_sxm | `flash_attention_v3` | shared | sglang | sglang:16458 | 0 / 8779 | 6 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 16 | — | — | — | -| h200_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:24137 | 0 / 8779 | 60 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 96 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:15358 | 0 / 7679 | 121 | — | — | — | -| l40s | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:15358 | 0 / 7679 | 228 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:15358 | 0 / 7679 | 27 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:15358 | 0 / 7679 | 15 | — | — | — | -| rtx_pro_6000_server | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:15358 | 0 / 7679 | 16 | — | — | — | +| gb300 | `vllm_vit_flash_attn_fa4` | shared | vllm | vllm:15358 | 0 / 7679 | 19 | — | — | — | +| h100_sxm | `flash_attention_v3` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| h100_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:15358 | 0 / 7679 | 5 | — | — | — | +| h200_sxm | `flash_attention_v3` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| h200_sxm | `vllm_vit_flash_attn_fa3` | shared | vllm | vllm:15358 | 0 / 7679 | 17 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| l40s | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:7679 | 0 / 7679 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:7679 | 0 / 7679 | 0 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:7679 | 0 / 7679 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_vit_flash_attn_fa2` | shared | vllm | vllm:7679 | 0 / 7679 | 0 | — | — | — | ## `gdn_perf.parquet` @@ -501,77 +479,58 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `causal_conv1d_fn` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | | a100_sxm | `causal_conv1d_update` | shared | sglang | sglang:88 | 0 / 88 | 0 | — | — | — | | a100_sxm | `fused_recurrent_gated_delta_rule` | shared | sglang | sglang:86 | 0 / 86 | 0 | — | — | — | -| b200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5470, vllm:14112 | 4616 / 5558 | 38 | 89.8 | 208.1 | 277.9 | -| b200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 550 | 2 | 65.6 | 101.6 | 161.9 | -| b200_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:854 | 0 / 5383 | 1 | — | — | — | -| b200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4616, vllm:14112 | 4616 / 4704 | 0 | 9.9 | 23.1 | 30.6 | +| b200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:854, vllm:9408 | 4529 / 5558 | 8 | 6.1 | 12.0 | 28.2 | +| b200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 550 | 2 | 9.8 | 26.8 | 33.8 | +| b200_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:854 | 0 / 5383 | 0 | — | — | — | +| b200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 0 | — | — | — | | b200_sxm | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | -| b200_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | b200_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| b200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 7 | 0.4 | 3.4 | 9.3 | -| b200_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| b300_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5487, vllm:14112 | 4650 / 4704 | 18 | 92.8 | 208.2 | 278.3 | -| b300_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 462 | 0 | 62.9 | 98.4 | 160.3 | -| b300_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:836 | 799 / 4566 | 0 | 39.6 | 56.6 | 61.1 | -| b300_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4650, vllm:14112 | 4650 / 4704 | 1 | 8.4 | 22.2 | 30.1 | +| b200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 3 | 0.5 | 4.5 | 9.3 | +| b300_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:837, vllm:9408 | 4567 / 4704 | 3 | 8.1 | 138.5 | 277.1 | +| b300_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 462 | 0 | 10.2 | 70.9 | 129.4 | +| b300_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:836 | 799 / 4566 | 0 | 31.2 | 46.0 | 50.6 | +| b300_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 1 | — | — | — | | b300_sxm | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | -| b300_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | b300_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| b300_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 2 | 0.3 | 3.6 | 5.3 | -| b300_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| gb200 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5470, vllm:14112 | 4616 / 4704 | 11 | 97.8 | 210.7 | 278.2 | -| gb200 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 462 | 2 | 83.7 | 135.7 | 163.8 | -| gb200 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:854 | 824 / 4559 | 1 | 13.5 | 49.1 | 70.1 | -| gb200 | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4616, vllm:14112 | 4616 / 4704 | 1 | 11.4 | 27.0 | 33.3 | +| b300_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 0 | 1.2 | 6.3 | 9.3 | +| gb200 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:854, vllm:9408 | 4559 / 4704 | 5 | 3.5 | 137.4 | 276.6 | +| gb200 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 462 | 2 | 9.8 | 112.0 | 134.2 | +| gb200 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:854 | 824 / 4559 | 0 | 22.8 | 54.0 | 68.0 | +| gb200 | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 0 | — | — | — | | gb200 | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | -| gb200 | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | gb200 | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| gb200 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 0 | 2.7 | 4.7 | 5.9 | -| gb200 | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| gb300 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:9058, trtllm:5487, vllm:14112 | 4650 / 4704 | 12 | 101.6 | 208.0 | 277.4 | -| gb300 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:550, vllm:1383 | 462 / 462 | 1 | 82.1 | 153.6 | 198.4 | -| gb300 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:9058, trtllm:836 | 799 / 4566 | 0 | 17.6 | 41.9 | 55.4 | -| gb300 | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4650, vllm:14112 | 4650 / 4704 | 1 | 10.2 | 25.7 | 32.5 | +| gb200 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 0 | 1.0 | 4.4 | 6.1 | +| gb300 | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4529, trtllm:837, vllm:9408 | 4567 / 4704 | 7 | 5.6 | 140.3 | 276.0 | +| gb300 | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:462, trtllm:88, vllm:922 | 461 / 462 | 1 | 10.2 | 143.1 | 182.1 | +| gb300 | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4529, trtllm:836 | 799 / 4566 | 0 | 20.3 | 59.3 | 67.4 | +| gb300 | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:9408 | 0 / 4704 | 1 | — | — | — | | gb300 | `flashinfer_gated_delta_rule_decode` | shared | sglang | sglang:52 | 0 / 52 | 0 | — | — | — | -| gb300 | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | gb300 | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| gb300 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:914, vllm:1371 | 457 / 457 | 2 | 0.4 | 3.6 | 5.3 | -| gb300 | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| h100_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5325, vllm:13507 | 4529 / 4694 | 5 | 92.8 | 220.5 | 281.1 | -| h100_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:1329 | 461 / 462 | 1 | 79.4 | 115.8 | 160.1 | +| gb300 | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:457, vllm:914 | 457 / 457 | 1 | 1.4 | 7.2 | 18.1 | +| h100_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:816, vllm:8824 | 4004 / 4683 | 0 | 9.0 | 134.5 | 278.5 | +| h100_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:868 | 407 / 461 | 0 | 17.1 | 100.9 | 135.4 | | h100_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:806 | 806 / 4004 | 0 | 42.2 | 59.4 | 69.5 | -| h100_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4509, vllm:13507 | 4498 / 4694 | 0 | 18.5 | 65.1 | 83.8 | -| h100_sxm | `flashinfer_chunk_gated_delta_rule` | shared | sglang | sglang:4529 | 0 / 4529 | 0 | — | — | — | -| h100_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | +| h100_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:8824 | 0 / 4683 | 0 | — | — | — | | h100_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| h100_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:1319 | 457 / 457 | 0 | 1.0 | 3.0 | 5.7 | -| h100_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| h200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5400, vllm:13550 | 4602 / 4703 | 14 | 93.7 | 220.8 | 281.1 | -| h200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:1329 | 461 / 462 | 2 | 82.2 | 116.6 | 151.5 | +| h100_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:862 | 405 / 457 | 0 | 4.3 | 11.0 | 13.4 | +| h200_sxm | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:798, vllm:8847 | 4017 / 4703 | 8 | 5.2 | 194.5 | 279.0 | +| h200_sxm | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:868 | 407 / 461 | 2 | 14.7 | 100.8 | 133.3 | | h200_sxm | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:797 | 785 / 4016 | 0 | 44.5 | 130.0 | 145.6 | -| h200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4584, vllm:13550 | 4584 / 4703 | 1 | 18.1 | 60.7 | 67.5 | -| h200_sxm | `flashinfer_chunk_gated_delta_rule` | shared | sglang | sglang:4529 | 0 / 4529 | 0 | — | — | — | -| h200_sxm | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | +| h200_sxm | `chunk_gated_delta_rule_flashinfer` | shared | vllm | vllm:8847 | 0 / 4703 | 1 | — | — | — | | h200_sxm | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| h200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:1319 | 457 / 457 | 5 | 0.7 | 3.1 | 6.6 | -| h200_sxm | `fused_sigmoid_gating_delta_rule_update_flashinfer` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| l40s | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5186, vllm:8364 | 4511 / 4638 | 74 | 100.5 | 197.6 | 275.8 | -| l40s | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:868 | 461 / 462 | 3 | 109.5 | 151.3 | 181.4 | -| l40s | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:8533, trtllm:5186 | 4400 / 4529 | 2 | 7.5 | 33.9 | 65.3 | -| l40s | `chunk_gated_delta_rule_triton` | shared | vllm | vllm:8330 | 0 / 4395 | 0 | — | — | — | -| l40s | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | +| h200_sxm | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:862 | 405 / 457 | 2 | 1.9 | 71.1 | 104.6 | +| l40s | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:786, vllm:3950 | 3956 / 4100 | 0 | 2.4 | 147.1 | 276.1 | +| l40s | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:407 | 407 / 407 | 0 | 14.1 | 137.2 | 168.2 | +| l40s | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:786 | 786 / 4004 | 0 | 21.5 | 104.5 | 118.2 | +| l40s | `chunk_gated_delta_rule_triton` | shared | vllm | vllm:3935 | 0 / 3935 | 0 | — | — | — | | l40s | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| l40s | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:862 | 457 / 457 | 3 | 0.4 | 3.0 | 24.3 | -| l40s | `fused_sigmoid_gating_delta_rule_update_triton` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | -| rtx_pro_6000_server | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:8533, trtllm:5353, vllm:8653 | 4529 / 4701 | 29 | 86.9 | 199.3 | 274.0 | -| rtx_pro_6000_server | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:550, vllm:868 | 461 / 462 | 1 | 70.8 | 94.0 | 146.9 | -| rtx_pro_6000_server | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:8533, trtllm:824 | 824 / 4529 | 4 | 21.2 | 87.6 | 99.6 | -| rtx_pro_6000_server | `chunk_gated_delta_rule_flashinfer` | shared | trtllm, vllm | trtllm:4529, vllm:4701 | 4529 / 4701 | 0 | 21.8 | 53.8 | 103.1 | +| l40s | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:405 | 405 / 405 | 0 | 0.4 | 4.3 | 11.7 | +| rtx_pro_6000_server | `causal_conv1d_fn` | shared | sglang, trtllm, vllm | sglang:4004, trtllm:824, vllm:3952 | 3965 / 4102 | 0 | 3.1 | 137.7 | 272.9 | +| rtx_pro_6000_server | `causal_conv1d_update` | shared | sglang, trtllm, vllm | sglang:407, trtllm:88, vllm:407 | 407 / 407 | 0 | 9.2 | 105.0 | 139.6 | +| rtx_pro_6000_server | `chunk_gated_delta_rule` | shared | sglang, trtllm | sglang:4004, trtllm:824 | 824 / 4004 | 0 | 17.7 | 84.7 | 97.0 | | rtx_pro_6000_server | `chunk_gated_delta_rule_triton` | shared | vllm | vllm:3935 | 0 / 3935 | 0 | — | — | — | -| rtx_pro_6000_server | `fused_qkvzba_causal_conv1d_update_contiguous` | shared | sglang | sglang:462 | 0 / 462 | 0 | — | — | — | | rtx_pro_6000_server | `fused_recurrent_gated_delta_rule` | shared | trtllm | trtllm:86 | 0 / 86 | 0 | — | — | — | -| rtx_pro_6000_server | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:862, vllm:862 | 457 / 457 | 0 | 0.3 | 5.7 | 24.9 | -| rtx_pro_6000_server | `fused_sigmoid_gating_delta_rule_update_triton` | shared | trtllm | trtllm:462 | 0 / 462 | 0 | — | — | — | +| rtx_pro_6000_server | `fused_recurrent_gated_delta_rule_packed_decode` | shared | sglang, vllm | sglang:405, vllm:405 | 405 / 405 | 0 | 0.3 | 8.2 | 23.2 | ## `gemm_perf.parquet` @@ -582,108 +541,92 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0` | shared | trtllm | trtllm:6048 | 0 / 6048 | 0 | — | — | — | | a100_sxm | `trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0` | shared | trtllm | trtllm:12096 | 0 / 12096 | 0 | — | — | — | | a100_sxm | `vllm_default` | shared | vllm | vllm:9240 | 0 / 9240 | 0 | — | — | — | -| b200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:146668 | 0 / 37518 | 222 | — | — | — | -| b200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:15466 | 0 / 5328 | 14 | — | — | — | -| b200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:93166 | 0 / 31080 | 38 | — | — | — | -| b200_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:146668 | 0 / 37518 | 695 | — | — | — | -| b200_sxm | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 58610 | 0 | — | — | — | +| b200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 81 | — | — | — | +| b200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | +| b200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | +| b200_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 482 | — | — | — | +| b200_sxm | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | | b200_sxm | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | b200_sxm | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| b200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:88652 | 0 / 29600 | 36 | — | — | — | -| b200_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:88652 | 0 / 29600 | 70 | — | — | — | -| b200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:108410 | 0 / 36852 | 175 | — | — | — | -| b200_sxm | `sglang_torch_linear` | shared | sglang | sglang:109150 | 0 / 37518 | 190 | — | — | — | -| b200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:146668 | 0 / 37518 | 489 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 204292 | 0 | — | — | — | -| b300_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 298 | — | — | — | -| b300_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:20572 | 0 / 5624 | 28 | — | — | — | -| b300_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:124542 | 0 / 31450 | 213 | — | — | — | -| b300_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 929 | — | — | — | -| b300_sxm | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 29342 | 23 | — | — | — | -| b300_sxm | `sglang_cutedsl_tgv_bf16` | shared | sglang | sglang:1317 | 0 / 1317 | 0 | — | — | — | -| b300_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:120028 | 0 / 31450 | 64 | — | — | — | -| b300_sxm | `sglang_flashinfer_bf16_direct` | shared | sglang | sglang:8 | 0 / 8 | 0 | — | — | — | -| b300_sxm | `sglang_flashinfer_bf16_splitk` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:125282 | 0 / 36704 | 122 | — | — | — | -| b300_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:108558 | 0 / 36852 | 73 | — | — | — | -| b300_sxm | `sglang_torch_linear` | shared | sglang | sglang:144642 | 0 / 38215 | 330 | — | — | — | -| b300_sxm | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | -| b300_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:183298 | 0 / 38258 | 848 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 103404 | 72 | — | — | — | -| b60 | `vllm_default` | shared | vllm | vllm:61698 | 0 / 23058 | 1 | — | — | — | -| gb200 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 357 | — | — | — | -| gb200 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:20572 | 0 / 5624 | 46 | — | — | — | -| gb200 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:124542 | 0 / 31450 | 173 | — | — | — | -| gb200 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:183298 | 0 / 38258 | 812 | — | — | — | -| gb200 | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 29342 | 18 | — | — | — | +| b200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 15 | — | — | — | +| b200_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 26 | — | — | — | +| b200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71558 | 0 / 35816 | 85 | — | — | — | +| b200_sxm | `sglang_torch_linear` | shared | sglang | sglang:71632 | 0 / 35890 | 86 | — | — | — | +| b200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 214 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | +| b300_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 52 | — | — | — | +| b300_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | +| b300_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | +| b300_sxm | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 387 | — | — | — | +| b300_sxm | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | +| b300_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 10 | — | — | — | +| b300_sxm | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 30 | — | — | — | +| b300_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71706 | 0 / 35964 | 49 | — | — | — | +| b300_sxm | `sglang_torch_linear` | shared | sglang | sglang:71854 | 0 / 36112 | 104 | — | — | — | +| b300_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 306 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | +| b60 | `vllm_default` | shared | vllm | vllm:40530 | 0 / 22176 | 1 | — | — | — | +| gb200 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 67 | — | — | — | +| gb200 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | +| gb200 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | +| gb200 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 281 | — | — | — | +| gb200 | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | | gb200 | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | gb200 | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| gb200 | `sglang_cutedsl_tgv_bf16` | shared | sglang | sglang:1317 | 0 / 1317 | 0 | — | — | — | -| gb200 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:120028 | 0 / 31450 | 78 | — | — | — | -| gb200 | `sglang_flashinfer_bf16_direct` | shared | sglang | sglang:8 | 0 / 8 | 0 | — | — | — | -| gb200 | `sglang_flashinfer_bf16_splitk` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:125282 | 0 / 36704 | 134 | — | — | — | -| gb200 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:108410 | 0 / 36852 | 56 | — | — | — | -| gb200 | `sglang_torch_linear` | shared | sglang | sglang:144420 | 0 / 38215 | 347 | — | — | — | -| gb200 | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | -| gb200 | `torch.nn.functional.linear` | shared | vllm | vllm:183298 | 0 / 38258 | 850 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 103404 | 108 | — | — | — | -| gb300 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:183150 | 0 / 38184 | 437 | — | — | — | -| gb300 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:20572 | 0 / 5624 | 41 | — | — | — | -| gb300 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:124542 | 0 / 31450 | 118 | — | — | — | -| gb300 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:183150 | 0 / 38184 | 723 | — | — | — | -| gb300 | `deepgemm` | shared | trtllm | trtllm:58610 | 0 / 29342 | 16 | — | — | — | -| gb300 | `sglang_cutedsl_tgv_bf16` | shared | sglang | sglang:1295 | 0 / 1295 | 0 | — | — | — | -| gb300 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:120028 | 0 / 31450 | 67 | — | — | — | -| gb300 | `sglang_flashinfer_bf16_direct` | shared | sglang | sglang:8 | 0 / 8 | 0 | — | — | — | -| gb300 | `sglang_flashinfer_bf16_splitk` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:125134 | 0 / 36556 | 145 | — | — | — | -| gb300 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:144892 | 0 / 37518 | 126 | — | — | — | -| gb300 | `sglang_torch_linear` | shared | sglang | sglang:144294 | 0 / 38163 | 428 | — | — | — | -| gb300 | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1850 | 0 / 1850 | 0 | — | — | — | -| gb300 | `torch.nn.functional.linear` | shared | vllm | vllm:183150 | 0 / 38184 | 888 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:204292 | 0 / 103404 | 97 | — | — | — | -| h100_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:147407 | 0 / 38258 | 97 | — | — | — | -| h100_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:15762 | 0 / 5624 | 25 | — | — | — | -| h100_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:72018 | 0 / 24225 | 90 | — | — | — | -| h100_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:21484 | 0 / 7225 | 60 | — | — | — | +| gb200 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 14 | — | — | — | +| gb200 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 21 | — | — | — | +| gb200 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71558 | 0 / 35816 | 43 | — | — | — | +| gb200 | `sglang_torch_linear` | shared | sglang | sglang:71632 | 0 / 35890 | 61 | — | — | — | +| gb200 | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 266 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | +| gb300 | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 98 | — | — | — | +| gb300 | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:10138 | 0 / 5328 | 0 | — | — | — | +| gb300 | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:62086 | 0 / 31080 | 0 | — | — | — | +| gb300 | `FlashInferCuteDslNvFp4LinearKernel` | shared | vllm | vllm:109150 | 0 / 37518 | 247 | — | — | — | +| gb300 | `deepgemm` | shared | trtllm | trtllm:29268 | 0 / 29268 | 0 | — | — | — | +| gb300 | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:59052 | 0 / 29526 | 4 | — | — | — | +| gb300 | `sglang_flashinfer_cutedsl_nvfp4` | shared | sglang | sglang:59052 | 0 / 29526 | 46 | — | — | — | +| gb300 | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:71558 | 0 / 35816 | 69 | — | — | — | +| gb300 | `sglang_torch_linear` | shared | sglang | sglang:71632 | 0 / 35890 | 85 | — | — | — | +| gb300 | `torch.nn.functional.linear` | shared | vllm | vllm:109150 | 0 / 37518 | 228 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:100888 | 0 / 100888 | 0 | — | — | — | +| h100_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 1 | — | — | — | +| h100_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:5328 | 0 / 5328 | 0 | — | — | — | +| h100_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:23940 | 0 / 23940 | 0 | — | — | — | +| h100_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:7140 | 0 / 7140 | 0 | — | — | — | | h100_sxm | `sglang` | shared | sglang | sglang:35280 | 0 / 35280 | 0 | — | — | — | | h100_sxm | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | h100_sxm | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h100_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:90501 | 0 / 31450 | 43 | — | — | — | -| h100_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 8 | — | — | — | -| h100_sxm | `sglang_torch_linear` | shared | sglang | sglang:109878 | 0 / 38258 | 42 | — | — | — | -| h100_sxm | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | -| h100_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:147408 | 0 / 38258 | 147 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:203852 | 0 / 103184 | 30 | — | — | — | -| h200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:147400 | 0 / 38258 | 193 | — | — | — | -| h200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:15762 | 0 / 5624 | 44 | — | — | — | -| h200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:72048 | 0 / 24225 | 128 | — | — | — | -| h200_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:21488 | 0 / 7225 | 81 | — | — | — | +| h100_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:29526 | 0 / 29526 | 0 | — | — | — | +| h100_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| h100_sxm | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| h100_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:73260 | 0 / 37518 | 23 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:100668 | 0 / 100668 | 0 | — | — | — | +| h200_sxm | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 48 | — | — | — | +| h200_sxm | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:5328 | 0 / 5328 | 0 | — | — | — | +| h200_sxm | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:23940 | 0 / 23940 | 0 | — | — | — | +| h200_sxm | `FlashInferFp8BlockScaledMMKernel` | shared | vllm | vllm:7140 | 0 / 7140 | 0 | — | — | — | | h200_sxm | `sglang.srt.layers.vocab_parallel_embedding.ParallelLMHead.quant_method.apply/sglang.srt.layers.quantization.unquant.UnquantizedEmbeddingMethod` | shared | sglang | sglang:30 | 0 / 30 | 0 | — | — | — | | h200_sxm | `sglang.srt.models.deepseek_v2.MoEGate.forward` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:90502 | 0 / 31450 | 36 | — | — | — | -| h200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 4 | — | — | — | -| h200_sxm | `sglang_torch_linear` | shared | sglang | sglang:109890 | 0 / 38258 | 53 | — | — | — | -| h200_sxm | `sglang_triton_w8a8_block_fp8` | shared | sglang | sglang:1998 | 0 / 1998 | 0 | — | — | — | -| h200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:147408 | 0 / 38258 | 266 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:203852 | 0 / 103184 | 49 | — | — | — | -| l40s | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73217 | 0 / 37516 | 34 | — | — | — | -| l40s | `MarlinFP8ScaledMMLinearKernel` | shared | vllm | vllm:37518 | 0 / 37518 | 0 | — | — | — | +| h200_sxm | `sglang_deepgemm_gemm_nt_f8f8bf16` | shared | sglang | sglang:29526 | 0 / 29526 | 0 | — | — | — | +| h200_sxm | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| h200_sxm | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| h200_sxm | `torch.nn.functional.linear` | shared | vllm | vllm:73260 | 0 / 37518 | 67 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:100668 | 0 / 100668 | 0 | — | — | — | +| l40s | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:35718 | 0 / 35718 | 0 | — | — | — | | l40s | `TritonFp8BlockScaledMMKernel` | shared | vllm | vllm:35676 | 0 / 35676 | 0 | — | — | — | -| l40s | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 469 | — | — | — | -| l40s | `sglang_torch_linear` | shared | sglang | sglang:73260 | 0 / 37518 | 371 | — | — | — | -| l40s | `torch.nn.functional.linear` | shared | vllm | vllm:73168 | 0 / 37505 | 1491 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:203964 | 0 / 103304 | 917 | — | — | — | -| rtx_pro_6000_server | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 39 | — | — | — | -| rtx_pro_6000_server | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:1036 | 0 / 518 | 0 | — | — | — | -| rtx_pro_6000_server | `DeepGemmFp8BlockScaledMMKernel` | shared | vllm | vllm:31080 | 0 / 31080 | 0 | — | — | — | -| rtx_pro_6000_server | `FlashInferCutlassNvFp4LinearKernel` | shared | vllm | vllm:73260 | 0 / 37518 | 123 | — | — | — | -| rtx_pro_6000_server | `sglang_flashinfer_cutlass_nvfp4` | shared | sglang | sglang:59126 | 0 / 29600 | 77 | — | — | — | -| rtx_pro_6000_server | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:72594 | 0 / 36852 | 113 | — | — | — | -| rtx_pro_6000_server | `sglang_torch_linear` | shared | sglang | sglang:73260 | 0 / 37518 | 148 | — | — | — | -| rtx_pro_6000_server | `torch.nn.functional.linear` | shared | vllm | vllm:73260 | 0 / 37518 | 103 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:263014 | 0 / 132866 | 100 | — | — | — | +| l40s | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| l40s | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| l40s | `torch.nn.functional.linear` | shared | vllm | vllm:35672 | 0 / 35672 | 0 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:100668 | 0 / 100668 | 0 | — | — | — | +| rtx_pro_6000_server | `CutlassFP8ScaledMMLinearKernel` | shared | vllm | vllm:35742 | 0 / 35742 | 0 | — | — | — | +| rtx_pro_6000_server | `CutlassFp8BlockScaledMMKernel` | shared | vllm | vllm:518 | 0 / 518 | 0 | — | — | — | +| rtx_pro_6000_server | `FlashInferCutlassNvFp4LinearKernel` | shared | vllm | vllm:35742 | 0 / 35742 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_flashinfer_cutlass_nvfp4` | shared | sglang | sglang:29526 | 0 / 29526 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_sgl_kernel_fp8_scaled_mm` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_torch_linear` | shared | sglang | sglang:35742 | 0 / 35742 | 0 | — | — | — | +| rtx_pro_6000_server | `torch.nn.functional.linear` | shared | vllm | vllm:35742 | 0 / 35742 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:130156 | 0 / 130156 | 0 | — | — | — | ## `generation_attention_perf.parquet` @@ -692,146 +635,117 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `flash_attention` | shared | sglang | sglang:5093 | 0 / 5093 | 0 | — | — | — | | a100_sxm | `torch_flow` | shared | trtllm | trtllm:5026 | 0 / 5026 | 0 | — | — | — | | a100_sxm | `vllm_flash_attn` | shared | vllm | vllm:5431 | 0 / 5431 | 0 | — | — | — | -| b200_sxm | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 11 | — | — | — | -| b200_sxm | `torch_flow` | shared | trtllm | trtllm:98310 | 0 / 98310 | 0 | — | — | — | +| b200_sxm | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | +| b200_sxm | `torch_flow` | shared | trtllm | trtllm:48314 | 0 / 48314 | 0 | — | — | — | | b200_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | -| b200_sxm | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 19 | — | — | — | -| b200_sxm | `trtllm_mha` | shared | sglang | sglang:76995 | 0 / 39358 | 78 | — | — | — | -| b200_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:2224 | 0 / 2224 | 0 | — | — | — | +| b200_sxm | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | +| b200_sxm | `trtllm_mha` | shared | sglang | sglang:37643 | 0 / 37643 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer` | shared | vllm | vllm:36240 | 0 / 36240 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | -| b200_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:60916 | 0 / 60916 | 0 | — | — | — | -| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:9835 | 0 / 5384 | 30 | — | — | — | -| b300_sxm | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 16 | — | — | — | -| b300_sxm | `torch_flow` | shared | trtllm | trtllm:96976 | 0 / 49996 | 75 | — | — | — | +| b200_sxm | `vllm_triton_attn` | shared | vllm | vllm:6684 | 0 / 4456 | 17 | — | — | — | +| b300_sxm | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | +| b300_sxm | `torch_flow` | shared | trtllm | trtllm:46980 | 0 / 46980 | 0 | — | — | — | | b300_sxm | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | -| b300_sxm | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 18 | — | — | — | -| b300_sxm | `trtllm_mha` | shared | sglang | sglang:76840 | 0 / 39360 | 68 | — | — | — | -| b300_sxm | `vllm_flash_attn_fa2` | shared | vllm | vllm:2227 | 0 / 2227 | 0 | — | — | — | +| b300_sxm | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | +| b300_sxm | `trtllm_mha` | shared | sglang | sglang:37490 | 0 / 37490 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | -| b300_sxm | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | b300_sxm | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:58510 | 0 / 58510 | 0 | — | — | — | -| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:7600 | 0 / 3156 | 14 | — | — | — | -| b60 | `vllm_flash_attn` | shared | vllm | vllm:54675 | 0 / 31096 | 0 | — | — | — | -| gb200 | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 4 | — | — | — | -| gb200 | `torch_flow` | shared | trtllm | trtllm:98312 | 0 / 49996 | 123 | — | — | — | -| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:7688 | 0 / 4148 | 7 | — | — | — | -| gb200 | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 10 | — | — | — | -| gb200 | `trtllm_mha` | shared | sglang | sglang:75434 | 0 / 39360 | 54 | — | — | — | -| gb200 | `vllm_flash_attn_fa2` | shared | vllm | vllm:2225 | 0 / 2225 | 0 | — | — | — | +| b300_sxm | `vllm_triton_attn` | shared | vllm | vllm:4456 | 0 / 2228 | 7 | — | — | — | +| b60 | `vllm_flash_attn` | shared | vllm | vllm:57209 | 0 / 30948 | 3 | — | — | — | +| gb200 | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | +| gb200 | `torch_flow` | shared | trtllm | trtllm:48316 | 0 / 48316 | 0 | — | — | — | +| gb200 | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | +| gb200 | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | +| gb200 | `trtllm_mha` | shared | sglang | sglang:36082 | 0 / 36082 | 0 | — | — | — | | gb200 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | -| gb200 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | gb200 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:59844 | 0 / 59844 | 0 | — | — | — | -| gb200 | `vllm_triton_attn` | shared | vllm | vllm:7601 | 0 / 3156 | 15 | — | — | — | -| gb300 | `flashinfer` | shared | sglang | sglang:7140 | 0 / 3570 | 4 | — | — | — | -| gb300 | `torch_flow` | shared | trtllm | trtllm:98312 | 0 / 49996 | 88 | — | — | — | -| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:7688 | 0 / 4148 | 12 | — | — | — | -| gb300 | `triton` | shared | sglang | sglang:17240 | 0 / 8620 | 8 | — | — | — | -| gb300 | `trtllm_mha` | shared | sglang | sglang:75428 | 0 / 39354 | 58 | — | — | — | -| gb300 | `vllm_flash_attn_fa2` | shared | vllm | vllm:2226 | 0 / 2226 | 0 | — | — | — | +| gb200 | `vllm_triton_attn` | shared | vllm | vllm:4456 | 0 / 2228 | 7 | — | — | — | +| gb300 | `flashinfer` | shared | sglang | sglang:3570 | 0 / 3570 | 0 | — | — | — | +| gb300 | `torch_flow` | shared | trtllm | trtllm:48316 | 0 / 48316 | 0 | — | — | — | +| gb300 | `torch_flow_flashinfer` | shared | trtllm | trtllm:3540 | 0 / 3540 | 0 | — | — | — | +| gb300 | `triton` | shared | sglang | sglang:8620 | 0 / 8620 | 0 | — | — | — | +| gb300 | `trtllm_mha` | shared | sglang | sglang:36078 | 0 / 36078 | 0 | — | — | — | | gb300 | `vllm_flashinfer_flashinfertrtllmapidecode` | shared | vllm | vllm:63180 | 0 / 63180 | 0 | — | — | — | -| gb300 | `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | shared | vllm | vllm:62252 | 0 / 62252 | 0 | — | — | — | | gb300 | `vllm_flashinfer_trtllmdecode` | shared | vllm | vllm:59844 | 0 / 59844 | 0 | — | — | — | -| gb300 | `vllm_triton_attn` | shared | vllm | vllm:7603 | 0 / 3156 | 11 | — | — | — | -| h100_sxm | `fa3` | shared | sglang | sglang:78420 | 0 / 41244 | 20 | — | — | — | -| h100_sxm | `torch_flow` | shared | trtllm | trtllm:107080 | 0 / 55047 | 170 | — | — | — | -| h100_sxm | `triton` | shared | sglang | sglang:7120 | 0 / 3828 | 3 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:192926 | 0 / 65780 | 416 | — | — | — | -| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:2656 | 0 / 928 | 1 | — | — | — | -| h100_sxm | `vllm_triton_attn` | shared | vllm | vllm:928 | 0 / 928 | 0 | — | — | — | -| h200_sxm | `fa3` | shared | sglang | sglang:79754 | 0 / 41244 | 29 | — | — | — | -| h200_sxm | `torch_flow` | shared | trtllm | trtllm:108414 | 0 / 55047 | 219 | — | — | — | -| h200_sxm | `triton` | shared | sglang | sglang:7120 | 0 / 3828 | 7 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:194260 | 0 / 65780 | 1415 | — | — | — | -| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:2656 | 0 / 928 | 5 | — | — | — | -| h200_sxm | `vllm_triton_attn` | shared | vllm | vllm:928 | 0 / 928 | 0 | — | — | — | -| l40s | `flashinfer` | shared | sglang | sglang:57836 | 0 / 33226 | 624 | — | — | — | -| l40s | `torch_flow` | shared | trtllm | trtllm:108442 | 0 / 55728 | 1314 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:25108 | 0 / 15724 | 290 | — | — | — | -| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:63551 | 0 / 32887 | 1907 | — | — | — | -| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:58928 | 0 / 31312 | 1428 | — | — | — | -| l40s | `vllm_triton_attn` | shared | vllm | vllm:5384 | 0 / 3156 | 122 | — | — | — | -| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:53808 | 0 / 29114 | 77 | — | — | — | -| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:106394 | 0 / 54704 | 160 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:30912 | 0 / 15724 | 45 | — | — | — | -| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:63543 | 0 / 32888 | 81 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:28324 | 0 / 28324 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_flashinfer_flashinfertrtllmapidecode_xqa` | shared | vllm | vllm:29214 | 0 / 29214 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:5376 | 0 / 3153 | 8 | — | — | — | +| gb300 | `vllm_triton_attn` | shared | vllm | vllm:4456 | 0 / 2228 | 7 | — | — | — | +| h100_sxm | `fa3` | shared | sglang | sglang:37176 | 0 / 37176 | 0 | — | — | — | +| h100_sxm | `torch_flow` | shared | trtllm | trtllm:52033 | 0 / 52033 | 0 | — | — | — | +| h100_sxm | `triton` | shared | sglang | sglang:3292 | 0 / 3292 | 0 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:127146 | 0 / 65780 | 23 | — | — | — | +| h100_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1728 | 0 / 928 | 1 | — | — | — | +| h200_sxm | `fa3` | shared | sglang | sglang:38510 | 0 / 38510 | 0 | — | — | — | +| h200_sxm | `torch_flow` | shared | trtllm | trtllm:53367 | 0 / 53367 | 0 | — | — | — | +| h200_sxm | `triton` | shared | sglang | sglang:3292 | 0 / 3292 | 0 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa3` | shared | vllm | vllm:128480 | 0 / 65780 | 485 | — | — | — | +| h200_sxm | `vllm_flash_attn_fa4` | shared | vllm | vllm:1728 | 0 / 928 | 4 | — | — | — | +| l40s | `flashinfer` | shared | sglang | sglang:29158 | 0 / 29158 | 0 | — | — | — | +| l40s | `torch_flow` | shared | trtllm | trtllm:52714 | 0 / 52714 | 0 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:9384 | 0 / 9384 | 0 | — | — | — | +| l40s | `vllm_flash_attn_fa2` | shared | vllm | vllm:30679 | 0 / 30679 | 0 | — | — | — | +| l40s | `vllm_flashinfer_fidecode` | shared | vllm | vllm:27616 | 0 / 27616 | 0 | — | — | — | +| l40s | `vllm_triton_attn` | shared | vllm | vllm:2228 | 0 / 2228 | 0 | — | — | — | +| rtx_pro_6000_server | `flashinfer` | shared | sglang | sglang:24694 | 0 / 24694 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_flow` | shared | trtllm | trtllm:51690 | 0 / 51690 | 0 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:15188 | 0 / 15188 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flash_attn_fa2` | shared | vllm | vllm:30677 | 0 / 30677 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_flashinfer_fidecode` | shared | vllm | vllm:27396 | 0 / 27396 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_triton_attn` | shared | vllm | vllm:2228 | 0 / 2228 | 0 | — | — | — | ## `generation_mla_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | a100_sxm | `default` | shared_fallback | trtllm | trtllm:5068 | 0 / 5068 | 0 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 7602 | 0 | — | — | — | -| b200_sxm | `trtllm_mla` | shared | sglang | sglang:7602 | 0 / 6154 | 2 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 4706 | 12 | — | — | — | -| b300_sxm | `trtllm_mla` | shared | sglang | sglang:10498 | 0 / 6154 | 1 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 4706 | 8 | — | — | — | -| gb200 | `trtllm_mla` | shared | sglang | sglang:10498 | 0 / 6154 | 4 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:7602 | 0 / 4706 | 9 | — | — | — | -| gb300 | `trtllm_mla` | shared | sglang | sglang:10498 | 0 / 6154 | 2 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:7964 | 0 / 5068 | 16 | — | — | — | -| h100_sxm | `flash_attention` | shared | sglang | sglang:10860 | 0 / 5068 | 4 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:7964 | 0 / 5068 | 12 | — | — | — | -| h200_sxm | `flash_attention` | shared | sglang | sglang:7964 | 0 / 5068 | 1 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| b200_sxm | `trtllm_mla` | shared | sglang | sglang:2896 | 0 / 2896 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| b300_sxm | `trtllm_mla` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| gb200 | `trtllm_mla` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| gb300 | `trtllm_mla` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| h100_sxm | `flash_attention` | shared | sglang | sglang:5792 | 0 / 2896 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| h200_sxm | `flash_attention` | shared | sglang | sglang:2896 | 0 / 2896 | 0 | — | — | — | | l40s | `default` | shared_fallback | trtllm | trtllm:5068 | 0 / 5068 | 0 | — | — | — | -| l40s | `triton` | shared | sglang | sglang:3470 | 0 / 2308 | 10 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:7964 | 0 / 5068 | 5 | — | — | — | -| rtx_pro_6000_server | `triton` | shared | sglang | sglang:4928 | 0 / 2528 | 5 | — | — | — | - -## `glm53_attention_module_perf.parquet` - -| system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | -|---|---|---|---|---|---|---|---|---|---| -| b300_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:926 | 0 / 926 | 0 | — | — | — | -| b300_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:926 | 0 / 926 | 0 | — | — | — | -| b300_sxm | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | shared | vllm | vllm:2315 | 0 / 2315 | 0 | — | — | — | -| gb200 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:463 | 0 / 463 | 0 | — | — | — | -| gb200 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:925 | 0 / 925 | 0 | — | — | — | -| gb200 | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | shared | vllm | vllm:1389 | 0 / 1389 | 0 | — | — | — | -| gb300 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:926 | 0 / 926 | 0 | — | — | — | -| gb300 | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:1366 | 0 / 1366 | 0 | — | — | — | -| gb300 | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | shared | vllm | vllm:2315 | 0 / 2315 | 0 | — | — | — | -| h100_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:433 | 0 / 433 | 0 | — | — | — | -| h100_sxm | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseSM90Impl` | shared | vllm | vllm:449 | 0 / 449 | 0 | — | — | — | -| h200_sxm | `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | shared | sglang | sglang:911 | 0 / 911 | 0 | — | — | — | -| h200_sxm | `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseSM90Impl` | shared | vllm | vllm:925 | 0 / 925 | 0 | — | — | — | +| l40s | `triton` | shared | sglang | sglang:1162 | 0 / 1162 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:2896 | 0 / 2896 | 0 | — | — | — | +| rtx_pro_6000_server | `triton` | shared | sglang | sglang:2400 | 0 / 1200 | 0 | — | — | — | ## `glm5_dsa_attn_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | -| b300_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6910 | 0 / 3333 | 0 | — | — | — | -| gb200 | `flash_mla_sparse_fwd` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | -| gb300 | `flash_mla_sparse_fwd` | shared | sglang | sglang:6883 | 0 / 3333 | 0 | — | — | — | -| h100_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | -| h200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | +| b200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | +| b300_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3577 | 0 / 3332 | 0 | — | — | — | +| gb200 | `flash_mla_sparse_fwd` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | +| gb300 | `flash_mla_sparse_fwd` | shared | sglang | sglang:3550 | 0 / 3332 | 0 | — | — | — | +| h100_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | +| h200_sxm | `flash_mla_sparse_fwd` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | ## `glm5_mqa_logits_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | -| b300_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | -| gb200 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | -| gb300 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6665 | 0 / 3333 | 0 | — | — | — | -| h100_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | -| h200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:6051 | 0 / 3026 | 0 | — | — | — | +| b200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | +| b300_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | +| gb200 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | +| gb300 | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3332 | 0 / 3332 | 0 | — | — | — | +| h100_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | +| h200_sxm | `deep_gemm.fp8_mqa_logits` | shared | sglang | sglang:3025 | 0 / 3025 | 0 | — | — | — | ## `glm5_topk_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | -| b300_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | -| gb200 | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | -| gb300 | `fast_topk_transform_fused` | shared | sglang | sglang:8442 | 0 / 4222 | 0 | — | — | — | -| h100_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:7214 | 0 / 3608 | 0 | — | — | — | -| h200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:7214 | 0 / 3608 | 0 | — | — | — | +| b200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | +| b300_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | +| gb200 | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | +| gb300 | `fast_topk_transform_fused` | shared | sglang | sglang:4220 | 0 / 4220 | 0 | — | — | — | +| h100_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:3606 | 0 / 3606 | 0 | — | — | — | +| h200_sxm | `fast_topk_transform_fused` | shared | sglang | sglang:3606 | 0 / 3606 | 0 | — | — | — | ## `kda_perf.parquet` @@ -846,58 +760,50 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | b200_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | | b200_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 13.5 | 68.4 | 68.9 | | b200_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | -| b300_sxm | `causal_conv1d_fn` | shared | sglang, vllm | sglang:448, vllm:420 | 420 / 448 | 0 | 19.6 | 122.7 | 142.1 | | b300_sxm | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 184.0 | 193.6 | 199.4 | -| b300_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:77, vllm:224 | 77 / 224 | 0 | 6.0 | 52.5 | 57.5 | -| b300_sxm | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | -| b300_sxm | `flashkda_fwd` | shared | vllm | vllm:847 | 0 / 847 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:33, vllm:152 | 33 / 152 | 0 | 16.4 | 37.4 | 52.4 | +| b300_sxm | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | +| b300_sxm | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | | b300_sxm | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | | b300_sxm | `fused_kda_decode_mtp_dspark` | shared | sglang | sglang:107 | 0 / 107 | 0 | — | — | — | -| b300_sxm | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | +| b300_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | | b300_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 12.7 | 62.7 | 69.6 | -| b300_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:44 | 0 / 44 | 0 | — | — | — | | b300_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | -| gb200 | `causal_conv1d_fn` | shared | sglang, vllm | sglang:447, vllm:419 | 419 / 447 | 0 | 30.3 | 129.5 | 139.1 | | gb200 | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 193.3 | 197.1 | 197.6 | -| gb200 | `causal_conv1d_update` | shared | sglang, vllm | sglang:77, vllm:224 | 77 / 224 | 0 | 3.6 | 49.0 | 54.0 | -| gb200 | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | -| gb200 | `flashkda_fwd` | shared | vllm | vllm:844 | 0 / 844 | 0 | — | — | — | +| gb200 | `causal_conv1d_update` | shared | sglang, vllm | sglang:33, vllm:152 | 33 / 152 | 0 | 13.1 | 27.5 | 28.5 | +| gb200 | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | +| gb200 | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | | gb200 | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | | gb200 | `fused_kda_decode_mtp_dspark` | shared | sglang | sglang:107 | 0 / 107 | 0 | — | — | — | -| gb200 | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | +| gb200 | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | | gb200 | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 6.7 | 69.5 | 77.5 | -| gb200 | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:44 | 0 / 44 | 0 | — | — | — | | gb200 | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | -| gb300 | `causal_conv1d_fn` | shared | sglang, vllm | sglang:232, vllm:327 | 218 / 341 | 0 | 30.3 | 127.7 | 140.0 | | gb300 | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:848 | 396 / 448 | 0 | 24.8 | 196.3 | 197.7 | -| gb300 | `causal_conv1d_update` | shared | sglang, vllm | sglang:55, vllm:358 | 55 / 206 | 0 | 9.5 | 41.5 | 52.2 | -| gb300 | `chunk_kda` | shared | sglang | sglang:619 | 0 / 619 | 0 | — | — | — | -| gb300 | `flashkda_fwd` | shared | vllm | vllm:1173 | 0 / 773 | 0 | — | — | — | +| gb300 | `causal_conv1d_update` | shared | sglang, vllm | sglang:33, vllm:304 | 33 / 152 | 0 | 18.5 | 34.4 | 39.6 | +| gb300 | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | +| gb300 | `flashkda_fwd` | shared | vllm | vllm:847 | 0 / 447 | 0 | — | — | — | | gb300 | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | | gb300 | `fused_kda_decode_mtp_dspark` | shared | sglang | sglang:107 | 0 / 107 | 0 | — | — | — | -| gb300 | `fused_recurrent_kda` | shared | vllm | vllm:269 | 0 / 161 | 0 | — | — | — | +| gb300 | `fused_recurrent_kda` | shared | vllm | vllm:216 | 0 / 108 | 0 | — | — | — | | gb300 | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:86 | 32 / 44 | 1 | 11.8 | 61.4 | 65.1 | -| gb300 | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:22 | 0 / 22 | 0 | — | — | — | | gb300 | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | -| h100_sxm | `causal_conv1d_fn` | shared | sglang, vllm | sglang:444, vllm:416 | 416 / 444 | 0 | 18.2 | 122.7 | 176.4 | | h100_sxm | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 181.9 | 193.1 | 194.3 | -| h100_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:185, vllm:224 | 185 / 224 | 0 | 13.4 | 57.9 | 75.9 | -| h100_sxm | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | -| h100_sxm | `flashkda_fwd` | shared | vllm | vllm:838 | 0 / 838 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:141, vllm:152 | 141 / 152 | 0 | 19.2 | 60.3 | 75.9 | +| h100_sxm | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | +| h100_sxm | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | | h100_sxm | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | -| h100_sxm | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | +| h100_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | | h100_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 16.7 | 66.6 | 72.7 | -| h100_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:152 | 0 / 152 | 0 | — | — | — | +| h100_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:108 | 0 / 108 | 0 | — | — | — | | h100_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | -| h200_sxm | `causal_conv1d_fn` | shared | sglang, vllm | sglang:447, vllm:419 | 419 / 447 | 0 | 17.7 | 119.3 | 179.3 | | h200_sxm | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 189.5 | 196.1 | 196.6 | -| h200_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:185, vllm:224 | 185 / 224 | 0 | 15.7 | 67.0 | 79.3 | -| h200_sxm | `chunk_kda` | shared | sglang | sglang:824 | 0 / 824 | 0 | — | — | — | -| h200_sxm | `flashkda_fwd` | shared | vllm | vllm:844 | 0 / 844 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_update` | shared | sglang, vllm | sglang:141, vllm:152 | 141 / 152 | 0 | 21.0 | 70.4 | 79.3 | +| h200_sxm | `chunk_kda` | shared | sglang | sglang:396 | 0 / 396 | 0 | — | — | — | +| h200_sxm | `flashkda_fwd` | shared | vllm | vllm:428 | 0 / 428 | 0 | — | — | — | | h200_sxm | `fused_kda_decode` | shared | vllm | vllm:44 | 0 / 44 | 0 | — | — | — | -| h200_sxm | `fused_recurrent_kda` | shared | vllm | vllm:179 | 0 / 179 | 0 | — | — | — | +| h200_sxm | `fused_recurrent_kda` | shared | vllm | vllm:108 | 0 / 108 | 0 | — | — | — | | h200_sxm | `fused_recurrent_kda_packed_decode` | shared | sglang, vllm | sglang:33, vllm:43 | 32 / 44 | 0 | 19.4 | 49.9 | 52.6 | -| h200_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:152 | 0 / 152 | 0 | — | — | — | +| h200_sxm | `fused_sigmoid_gating_delta_rule_update` | shared | sglang | sglang:108 | 0 / 108 | 0 | — | — | — | | h200_sxm | `kda_fused_decode` | shared | sglang | sglang:11 | 0 / 11 | 0 | — | — | — | | l40s | `causal_conv1d_fn_qkv3` | shared | sglang, vllm | sglang:396, vllm:428 | 396 / 428 | 0 | 190.5 | 196.8 | 198.0 | | l40s | `causal_conv1d_update` | shared | sglang, vllm | sglang:141, vllm:152 | 141 / 152 | 0 | 17.0 | 53.6 | 102.9 | @@ -921,274 +827,185 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | b200_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| b200_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| b200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 132 | 0 | — | — | — | -| b200_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| b200_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| b200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | b300_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:586 | 0 / 586 | 0 | — | — | — | -| b300_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:624 | 0 / 624 | 0 | — | — | — | -| b300_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | -| b300_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:624 | 0 / 624 | 0 | — | — | — | -| b300_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| b300_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | gb200 | `causal_conv1d_fn` | shared | trtllm | trtllm:624 | 0 / 624 | 0 | — | — | — | -| gb200 | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| gb200 | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | -| gb200 | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| gb200 | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| gb200 | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | gb300 | `causal_conv1d_fn` | shared | trtllm | trtllm:586 | 0 / 586 | 0 | — | — | — | -| gb300 | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:577 | 0 / 577 | 0 | — | — | — | -| gb300 | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | -| gb300 | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:576 | 0 / 576 | 0 | — | — | — | -| gb300 | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| gb300 | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | h100_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:592 | 0 / 592 | 0 | — | — | — | -| h100_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:579 | 0 / 579 | 0 | — | — | — | -| h100_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 1 | — | — | — | -| h100_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:579 | 0 / 579 | 0 | — | — | — | -| h100_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| h100_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | h200_sxm | `causal_conv1d_fn` | shared | trtllm | trtllm:619 | 0 / 619 | 0 | — | — | — | -| h200_sxm | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:609 | 0 / 609 | 0 | — | — | — | -| h200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 1 | — | — | — | -| h200_sxm | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:609 | 0 / 609 | 0 | — | — | — | -| h200_sxm | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| h200_sxm | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | l40s | `causal_conv1d_fn` | shared | trtllm | trtllm:559 | 0 / 559 | 0 | — | — | — | -| l40s | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:559 | 0 / 559 | 0 | — | — | — | -| l40s | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 2 | — | — | — | -| l40s | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:559 | 0 / 559 | 0 | — | — | — | -| l40s | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| l40s | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | | rtx_pro_6000_server | `causal_conv1d_fn` | shared | trtllm | trtllm:598 | 0 / 598 | 0 | — | — | — | -| rtx_pro_6000_server | `causal_conv1d_fn_token_major` | shared | trtllm | trtllm:592 | 0 / 592 | 0 | — | — | — | -| rtx_pro_6000_server | `causal_conv1d_update` | shared | trtllm | trtllm:132 | 0 / 66 | 0 | — | — | — | -| rtx_pro_6000_server | `mamba_chunk_scan_combined` | shared | trtllm | trtllm:592 | 0 / 592 | 0 | — | — | — | -| rtx_pro_6000_server | `selective_state_update_flashinfer` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | +| rtx_pro_6000_server | `causal_conv1d_update` | shared | trtllm | trtllm:66 | 0 / 66 | 0 | — | — | — | ## `mhc_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 8 | — | — | — | -| b200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 6 | — | — | — | -| b200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 140 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:50 | 0 / 50 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:58 | 0 / 58 | 0 | — | — | — | -| b200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 30 | 0 | — | — | — | -| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 29 | — | — | — | -| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:204 | 0 / 70 | 15 | — | — | — | -| b300_sxm | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 12 | — | — | — | -| b300_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 4 | — | — | — | -| b300_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 3 | — | — | — | -| b300_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:52 | 0 / 28 | 0 | — | — | — | -| b300_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:58 | 0 / 32 | 0 | — | — | — | -| b300_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 16 | 0 | — | — | — | -| b300_sxm | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | -| b300_sxm | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | -| b300_sxm | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 28 | — | — | — | -| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:203 | 0 / 70 | 15 | — | — | — | -| b300_sxm | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| b300_sxm | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 3 | — | — | — | -| gb200 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 4 | — | — | — | -| gb200 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 4 | — | — | — | -| gb200 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:48 | 0 / 26 | 0 | — | — | — | -| gb200 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:60 | 0 / 33 | 0 | — | — | — | -| gb200 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 16 | 0 | — | — | — | -| gb200 | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | -| gb200 | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | -| gb200 | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 21 | — | — | — | -| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 17 | — | — | — | -| gb200 | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb200 | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 2 | — | — | — | -| gb300 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 4 | — | — | — | -| gb300 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 0 | — | — | — | -| gb300 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:51 | 0 / 28 | 0 | — | — | — | -| gb300 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:59 | 0 / 33 | 0 | — | — | — | -| gb300 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:30 | 0 / 16 | 3 | — | — | — | -| gb300 | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | -| gb300 | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | -| gb300 | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 20 | — | — | — | -| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 26 | — | — | — | -| gb300 | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| gb300 | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:136 | 0 / 68 | 0 | — | — | — | -| h100_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:139 | 0 / 70 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:138 | 0 / 69 | 4 | — | — | — | -| h100_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:45 | 0 / 24 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:61 | 0 / 34 | 0 | — | — | — | -| h100_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:28 | 0 / 16 | 0 | — | — | — | -| h100_sxm | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | -| h100_sxm | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:28 | 0 / 28 | 0 | — | — | — | -| h100_sxm | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:207 | 0 / 69 | 11 | — | — | — | -| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 2 | — | — | — | -| h100_sxm | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h100_sxm | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `sglang.hc_post[tilelang]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `sglang.kernels.ops.layernorm.mhc.hc_contract` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `sglang.kernels.ops.layernorm.mhc.hc_expand` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:140 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:140 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:140 | 0 / 70 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:48 | 0 / 26 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:59 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:31 | 0 / 19 | 0 | — | — | — | -| h200_sxm | `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | shared | vllm | vllm:6 | 0 / 6 | 0 | — | — | — | -| h200_sxm | `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:29 | 0 / 29 | 0 | — | — | — | -| h200_sxm | `vllm.mhc_post_tilelang` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 21 | — | — | — | -| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:210 | 0 / 70 | 13 | — | — | — | -| h200_sxm | `vllm.model_executor.layers.mhc.hc_contract` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| h200_sxm | `vllm.model_executor.layers.mhc.hc_expand` | shared | vllm | vllm:35 | 0 / 35 | 0 | — | — | — | -| l40s | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:134 | 0 / 67 | 6 | — | — | — | -| l40s | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:130 | 0 / 65 | 1 | — | — | — | -| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:134 | 0 / 67 | 6 | — | — | — | -| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:138 | 0 / 69 | 2 | — | — | — | +| b200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| b200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | +| b200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | +| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 8 | — | — | — | +| b200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:134 | 0 / 70 | 2 | — | — | — | +| b300_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| b300_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| b300_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | +| b300_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:28 | 0 / 28 | 0 | — | — | — | +| b300_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | +| b300_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | +| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 9 | — | — | — | +| b300_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:133 | 0 / 70 | 5 | — | — | — | +| gb200 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| gb200 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| gb200 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | +| gb200 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | +| gb200 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | +| gb200 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | +| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 6 | — | — | — | +| gb200 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 7 | — | — | — | +| gb300 | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| gb300 | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| gb300 | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | +| gb300 | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:28 | 0 / 28 | 0 | — | — | — | +| gb300 | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | +| gb300 | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | +| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 10 | — | — | — | +| gb300 | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 13 | — | — | — | +| h100_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:68 | 0 / 68 | 0 | — | — | — | +| h100_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:69 | 0 / 69 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:24 | 0 / 24 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:27 | 0 / 27 | 0 | — | — | — | +| h100_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:16 | 0 / 16 | 0 | — | — | — | +| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:138 | 0 / 69 | 5 | — | — | — | +| h100_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `sglang_tilelang_mhc_post` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:70 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:70 | 0 / 70 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:26 | 0 / 26 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:24 | 0 / 24 | 0 | — | — | — | +| h200_sxm | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:19 | 0 / 19 | 0 | — | — | — | +| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 5 | — | — | — | +| h200_sxm | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:140 | 0 / 70 | 8 | — | — | — | +| l40s | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:67 | 0 / 67 | 0 | — | — | — | +| l40s | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:65 | 0 / 65 | 0 | — | — | — | +| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:67 | 0 / 67 | 0 | — | — | — | +| l40s | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:69 | 0 / 69 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_tilelang_mhc_post` | shared | sglang | sglang:67 | 0 / 67 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_tilelang_mhc_pre` | shared | sglang | sglang:35 | 0 / 35 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_torch_mhc_pre` | shared | sglang | sglang:67 | 0 / 67 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:138 | 0 / 69 | 13 | — | — | — | -| rtx_pro_6000_server | `trtllm_mhc_pre_dg_nosplit` | shared | trtllm | trtllm:18 | 0 / 18 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_mhc_pre_dg_splitk` | shared | trtllm | trtllm:37 | 0 / 37 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:79 | 0 / 67 | 3 | — | — | — | -| rtx_pro_6000_server | `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | shared | vllm | vllm:69 | 0 / 69 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | shared | vllm | vllm:70 | 0 / 70 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_mhc_post_mapping` | shared | trtllm | trtllm:69 | 0 / 69 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_mhc_pre_fma` | shared | trtllm | trtllm:67 | 0 / 67 | 0 | — | — | — | ## `mla_bmm_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| | a100_sxm | `default` | shared_fallback | sglang, trtllm | sglang:424, trtllm:424 | 224 / 224 | 0 | 1.4 | 18.4 | 115.9 | -| b200_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | b200_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | -| b200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1272 | 0 / 336 | 1 | — | — | — | -| b200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 760 | 0 | — | — | — | -| b200_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 760 | 0 | — | — | — | -| b200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 5 | — | — | — | -| b300_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| b200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| b200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 424 | 0 | — | — | — | +| b200_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 424 | 0 | — | — | — | +| b200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 1 | — | — | — | | b300_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| b300_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 1 | — | — | — | -| b300_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 2 | — | — | — | -| b300_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| b300_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 9 | — | — | — | -| gb200 | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| b300_sxm | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | +| b300_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| b300_sxm | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| b300_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 3 | — | — | — | | gb200 | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| gb200 | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | -| gb200 | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| gb200 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 336 | 1 | — | — | — | -| gb200 | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 2 | — | — | — | -| gb300 | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| gb200 | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | +| gb200 | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| gb200 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| gb200 | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 1 | — | — | — | | gb300 | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| gb300 | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | -| gb300 | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| gb300 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:1060 | 0 / 336 | 2 | — | — | — | -| gb300 | `vllm_torch_bmm` | shared | vllm | vllm:2344 | 0 / 336 | 5 | — | — | — | -| h100_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| gb300 | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | +| gb300 | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| gb300 | `trtllm_bmm_out_dequant_bf16` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| gb300 | `vllm_torch_bmm` | shared | vllm | vllm:1708 | 0 / 336 | 4 | — | — | — | | h100_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| h100_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | -| h100_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 1 | — | — | — | -| h100_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| h100_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 2 | — | — | — | -| h200_sxm | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| h100_sxm | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | +| h100_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| h100_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| h100_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 1 | — | — | — | | h200_sxm | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | -| h200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:1060 | 0 / 336 | 0 | — | — | — | -| h200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| h200_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| h200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1908 | 0 / 336 | 5 | — | — | — | +| h200_sxm | `sglang_torch_bmm` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | +| h200_sxm | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| h200_sxm | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| h200_sxm | `vllm_torch_bmm` | shared | vllm | vllm:1272 | 0 / 336 | 2 | — | — | — | | l40s | `default` | shared_fallback | trtllm | trtllm:848 | 0 / 448 | 0 | — | — | — | -| l40s | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | | l40s | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | -| l40s | `sglang_torch_bmm` | shared | sglang | sglang:1060 | 0 / 336 | 2 | — | — | — | -| l40s | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| l40s | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| l40s | `vllm_torch_bmm` | shared | vllm | vllm:636 | 0 / 336 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_flashinfer_bmm_fp8` | shared | sglang | sglang:636 | 0 / 336 | 0 | — | — | — | +| l40s | `sglang_torch_bmm` | shared | sglang | sglang:424 | 0 / 224 | 0 | — | — | — | +| l40s | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| l40s | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_sgl_kernel_bmm_fp8` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_torch_bmm` | shared | sglang | sglang:1484 | 0 / 336 | 0 | — | — | — | -| rtx_pro_6000_server | `trtllm_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 1 | — | — | — | -| rtx_pro_6000_server | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:1060 | 0 / 336 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_torch_bmm` | shared | vllm | vllm:636 | 0 / 336 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_torch_bmm` | shared | sglang | sglang:848 | 0 / 224 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | +| rtx_pro_6000_server | `trtllm_fp8_block_scaling_bmm_out` | shared | trtllm | trtllm:424 | 0 / 224 | 0 | — | — | — | ## `mla_context_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:2475 | 0 / 972 | 35 | — | — | — | -| b200_sxm | `FLASH_ATTN` | shared | vllm | vllm:21696 | 0 / 8136 | 118 | — | — | — | -| b200_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:10848 | 0 / 4068 | 58 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 11712 | 0 | — | — | — | +| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:1503 | 0 / 855 | 5 | — | — | — | +| b200_sxm | `FLASH_ATTN` | shared | vllm | vllm:13560 | 0 / 8136 | 58 | — | — | — | +| b200_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:6780 | 0 / 4068 | 27 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | | b200_sxm | `mla_trtllm_mla` | shared | sglang | sglang:218 | 0 / 218 | 0 | — | — | — | -| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:2475 | 0 / 972 | 17 | — | — | — | -| b300_sxm | `FLASH_ATTN` | shared | vllm | vllm:21696 | 0 / 8136 | 27 | — | — | — | -| b300_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:10848 | 0 / 4068 | 8 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 5856 | 44 | — | — | — | +| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:1503 | 0 / 855 | 0 | — | — | — | +| b300_sxm | `FLASH_ATTN` | shared | vllm | vllm:13560 | 0 / 8136 | 0 | — | — | — | +| b300_sxm | `TRTLLM_RAGGED` | shared | vllm | vllm:6780 | 0 / 4068 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | | b300_sxm | `mla_trtllm_mla` | shared | sglang | sglang:218 | 0 / 218 | 0 | — | — | — | -| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:2475 | 0 / 972 | 16 | — | — | — | -| gb200 | `FLASH_ATTN` | shared | vllm | vllm:21696 | 0 / 8136 | 26 | — | — | — | -| gb200 | `TRTLLM_RAGGED` | shared | vllm | vllm:10848 | 0 / 4068 | 13 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 5856 | 46 | — | — | — | -| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:4091 | 0 / 1944 | 15 | — | — | — | -| gb300 | `FLASH_ATTN` | shared | vllm | vllm:35197 | 0 / 16272 | 98 | — | — | — | -| gb300 | `TRTLLM_RAGGED` | shared | vllm | vllm:17591 | 0 / 8136 | 30 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:11712 | 0 / 5856 | 26 | — | — | — | -| h100_sxm | `FLASHMLA` | shared | vllm | vllm:2826 | 0 / 1080 | 11 | — | — | — | -| h100_sxm | `FLASH_ATTN` | shared | vllm | vllm:8704 | 0 / 3264 | 9 | — | — | — | -| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:4032 | 0 / 1512 | 20 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:7740 | 0 / 3880 | 14 | — | — | — | -| h200_sxm | `FLASHMLA` | shared | vllm | vllm:2826 | 0 / 1080 | 4 | — | — | — | -| h200_sxm | `FLASH_ATTN` | shared | vllm | vllm:8704 | 0 / 3264 | 0 | — | — | — | -| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:4032 | 0 / 1512 | 5 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:7739 | 0 / 3879 | 7 | — | — | — | -| l40s | `FLASH_ATTN` | shared | vllm | vllm:4502 | 0 / 2712 | 0 | — | — | — | -| l40s | `TRITON_MLA` | shared | vllm | vllm:360 | 0 / 216 | 0 | — | — | — | -| rtx_pro_6000_server | `FLASH_ATTN` | shared | vllm | vllm:11752 | 0 / 8136 | 0 | — | — | — | -| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:513 | 0 / 351 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:9719 | 0 / 5837 | 24 | — | — | — | +| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:1503 | 0 / 855 | 0 | — | — | — | +| gb200 | `FLASH_ATTN` | shared | vllm | vllm:13560 | 0 / 8136 | 0 | — | — | — | +| gb200 | `TRTLLM_RAGGED` | shared | vllm | vllm:6780 | 0 / 4068 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | +| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:3119 | 0 / 1827 | 6 | — | — | — | +| gb300 | `FLASH_ATTN` | shared | vllm | vllm:27061 | 0 / 16272 | 49 | — | — | — | +| gb300 | `TRTLLM_RAGGED` | shared | vllm | vllm:13523 | 0 / 8136 | 15 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:5856 | 0 / 5856 | 0 | — | — | — | +| h100_sxm | `FLASHMLA` | shared | vllm | vllm:1746 | 0 / 1044 | 0 | — | — | — | +| h100_sxm | `FLASH_ATTN` | shared | vllm | vllm:5440 | 0 / 3264 | 0 | — | — | — | +| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:2520 | 0 / 1512 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:3866 | 0 / 3866 | 0 | — | — | — | +| h200_sxm | `FLASHMLA` | shared | vllm | vllm:1746 | 0 / 1044 | 0 | — | — | — | +| h200_sxm | `FLASH_ATTN` | shared | vllm | vllm:5440 | 0 / 3264 | 0 | — | — | — | +| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:2520 | 0 / 1512 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:3869 | 0 / 3869 | 0 | — | — | — | +| l40s | `FLASH_ATTN` | shared | vllm | vllm:1790 | 0 / 1790 | 0 | — | — | — | +| l40s | `TRITON_MLA` | shared | vllm | vllm:144 | 0 / 144 | 0 | — | — | — | +| rtx_pro_6000_server | `FLASH_ATTN` | shared | vllm | vllm:3616 | 0 / 3616 | 0 | — | — | — | +| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:162 | 0 / 162 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:5828 | 0 / 5828 | 0 | — | — | — | ## `mla_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:33480 | 0 / 13248 | 359 | — | — | — | -| b200_sxm | `default` | shared_fallback | trtllm | trtllm:17662 | 0 / 17662 | 0 | — | — | — | -| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:33480 | 0 / 13248 | 196 | — | — | — | -| b300_sxm | `default` | shared_fallback | trtllm | trtllm:17663 | 0 / 8832 | 256 | — | — | — | -| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:33480 | 0 / 13248 | 161 | — | — | — | -| gb200 | `default` | shared_fallback | trtllm | trtllm:17663 | 0 / 8832 | 204 | — | — | — | -| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:55460 | 0 / 26496 | 252 | — | — | — | -| gb300 | `default` | shared_fallback | trtllm | trtllm:17664 | 0 / 8832 | 182 | — | — | — | -| h100_sxm | `FLASHMLA` | shared | vllm | vllm:11776 | 0 / 4416 | 25 | — | — | — | -| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:11584 | 0 / 4344 | 99 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:11776 | 0 / 5888 | 199 | — | — | — | -| h200_sxm | `FLASHMLA` | shared | vllm | vllm:11776 | 0 / 4416 | 38 | — | — | — | -| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:11776 | 0 / 4416 | 39 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:11776 | 0 / 5888 | 244 | — | — | — | -| l40s | `TRITON_MLA` | shared | vllm | vllm:6977 | 0 / 4321 | 0 | — | — | — | -| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:10122 | 0 / 7132 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:14719 | 0 / 8832 | 31 | — | — | — | +| b200_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:20232 | 0 / 11400 | 70 | — | — | — | +| b200_sxm | `default` | shared_fallback | trtllm | trtllm:8831 | 0 / 8831 | 0 | — | — | — | +| b300_sxm | `FLASHINFER_MLA` | shared | vllm | vllm:20232 | 0 / 11400 | 0 | — | — | — | +| b300_sxm | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | +| gb200 | `FLASHINFER_MLA` | shared | vllm | vllm:20232 | 0 / 11400 | 0 | — | — | — | +| gb200 | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | +| gb300 | `FLASHINFER_MLA` | shared | vllm | vllm:42212 | 0 / 24648 | 30 | — | — | — | +| gb300 | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | +| h100_sxm | `FLASHMLA` | shared | vllm | vllm:7360 | 0 / 4416 | 0 | — | — | — | +| h100_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:7240 | 0 / 4344 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:5888 | 0 / 5888 | 0 | — | — | — | +| h200_sxm | `FLASHMLA` | shared | vllm | vllm:7360 | 0 / 4416 | 0 | — | — | — | +| h200_sxm | `FLASH_ATTN_MLA` | shared | vllm | vllm:7360 | 0 / 4416 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:5888 | 0 / 5888 | 0 | — | — | — | +| l40s | `TRITON_MLA` | shared | vllm | vllm:2656 | 0 / 2656 | 0 | — | — | — | +| rtx_pro_6000_server | `TRITON_MLA` | shared | vllm | vllm:2990 | 0 / 2990 | 0 | — | — | — | +| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:8832 | 0 / 8832 | 0 | — | — | — | ## `moe_a2a_perf.parquet` @@ -1209,252 +1026,218 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | a100_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:36531 | 0 / 33453 | 0 | — | — | — | | a100_sxm | `sglang_marlin_moe` | shared | sglang | sglang:19035 | 0 / 16686 | 0 | — | — | — | | a100_sxm | `vllm_fused_moe` | shared | vllm | vllm:6360 | 0 / 6360 | 0 | — | — | — | -| b200_sxm | `deepgemm` | shared | trtllm | trtllm:79080 | 0 / 79080 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:17334 | 0 / 13122 | 5 | — | — | — | -| b200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 239760 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:75510 | 0 / 75510 | 0 | — | — | — | -| b200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 50868 | 0 | — | — | — | -| b200_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:241292 | 0 / 137294 | 5 | — | — | — | -| b200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:80352 | 0 / 43416 | 4 | — | — | — | -| b200_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:28056 | 0 / 22872 | 0 | — | — | — | +| b200_sxm | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:10854 | 0 / 10854 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:36045 | 0 / 36045 | 0 | — | — | — | +| b200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | +| b200_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:124821 | 0 / 124821 | 0 | — | — | — | +| b200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:36936 | 0 / 36936 | 0 | — | — | — | +| b200_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:5214 | 0 / 5214 | 0 | — | — | — | | b200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:10287 | 0 / 4131 | 7 | — | — | — | -| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:45318 | 0 / 16154 | 34 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:6156 | 0 / 3402 | 4 | — | — | — | +| b200_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:29786 | 0 / 15101 | 13 | — | — | — | | b200_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| b200_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | | b200_sxm | `vllm_flashinfer_trtllm_moe_fp4` | shared | vllm | vllm:10287 | 0 / 9234 | 0 | — | — | — | -| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 3 | — | — | — | -| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:37422 | 0 / 12474 | 21 | — | — | — | -| b200_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2754 | 0 / 2754 | 0 | — | — | — | +| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 1 | — | — | — | +| b200_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 12 | — | — | — | | b200_sxm | `vllm_fused_moe` | shared | vllm | vllm:83407 | 0 / 80329 | 0 | — | — | — | -| b200_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 8 | — | — | — | +| b200_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 6 | — | — | — | | b200_sxm | `vllm_marlin_int4_moe` | shared | vllm | vllm:10209 | 0 / 9237 | 0 | — | — | — | -| b200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 5 | — | — | — | -| b200_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:36738 | 0 / 12503 | 39 | — | — | — | +| b200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 1 | — | — | — | +| b200_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24573 | 0 / 12503 | 17 | — | — | — | | b200_sxm | `vllm_mxfp4_moe` | shared | vllm | vllm:1138 | 0 / 1138 | 0 | — | — | — | -| b200_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 4 | — | — | — | -| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 5 | — | — | — | +| b200_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 3 | — | — | — | +| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 0 | — | — | — | | b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25720 | 0 / 13556 | 9 | — | — | — | -| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:16180 | 0 / 15127 | 4 | — | — | — | -| b200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| b300_sxm | `deepgemm` | shared | trtllm | trtllm:79212 | 0 / 40497 | 57 | — | — | — | -| b300_sxm | `moe_torch_flow` | shared | trtllm | trtllm:19843 | 0 / 7371 | 18 | — | — | — | -| b300_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 125712 | 123 | — | — | — | -| b300_sxm | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:39465 | 0 / 39465 | 0 | — | — | — | -| b300_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 27945 | 1 | — | — | — | -| b300_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:320425 | 0 / 136404 | 15 | — | — | — | -| b300_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:104998 | 0 / 43416 | 5 | — | — | — | -| b300_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:32562 | 0 / 22842 | 0 | — | — | — | +| b200_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| b200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| b300_sxm | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | +| b300_sxm | `moe_torch_flow` | shared | trtllm | trtllm:12472 | 0 / 7371 | 9 | — | — | — | +| b300_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | +| b300_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | +| b300_sxm | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:198607 | 0 / 118584 | 0 | — | — | — | +| b300_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:61582 | 0 / 36936 | 0 | — | — | — | +| b300_sxm | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:9720 | 0 / 5184 | 0 | — | — | — | | b300_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | | b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:8181 | 0 / 4131 | 1 | — | — | — | -| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:44266 | 0 / 16180 | 38 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:4050 | 0 / 3402 | 0 | — | — | — | +| b300_sxm | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:28785 | 0 / 15127 | 23 | — | — | — | | b300_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| b300_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 7 | — | — | — | -| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:38232 | 0 / 13284 | 33 | — | — | — | -| b300_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2835 | 0 / 2835 | 0 | — | — | — | -| b300_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 7 | — | — | — | -| b300_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 2 | — | — | — | -| b300_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:36116 | 0 / 12555 | 26 | — | — | — | -| b300_sxm | `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:891 | 0 / 891 | 0 | — | — | — | -| b300_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 3 | — | — | — | -| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 7 | — | — | — | +| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 5 | — | — | — | +| b300_sxm | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 21 | — | — | — | +| b300_sxm | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 5 | — | — | — | +| b300_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 1 | — | — | — | +| b300_sxm | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:23821 | 0 / 12555 | 13 | — | — | — | +| b300_sxm | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 3 | — | — | — | +| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 1 | — | — | — | | b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25728 | 0 / 13582 | 15 | — | — | — | -| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:16102 | 0 / 15049 | 0 | — | — | — | -| b300_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| b60 | `vllm_xpu_moe` | shared | vllm | vllm:3996 | 0 / 2808 | 0 | — | — | — | +| b300_sxm | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| b300_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| b60 | `vllm_xpu_moe` | shared | vllm | vllm:1836 | 0 / 1404 | 0 | — | — | — | | b60 | `vllm_xpu_moe_mxfp4` | shared | vllm | vllm:2592 | 0 / 1296 | 0 | — | — | — | -| gb200 | `deepgemm` | shared | trtllm | trtllm:79299 | 0 / 40500 | 60 | — | — | — | -| gb200 | `moe_torch_flow` | shared | trtllm | trtllm:19683 | 0 / 7371 | 10 | — | — | — | -| gb200 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 125712 | 124 | — | — | — | -| gb200 | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:75510 | 0 / 39465 | 48 | — | — | — | -| gb200 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 27945 | 1 | — | — | — | -| gb200 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:281187 | 0 / 142638 | 3 | — | — | — | -| gb200 | `sglang_fused_moe_triton` | shared | sglang | sglang:94041 | 0 / 43416 | 2 | — | — | — | -| gb200 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:30405 | 0 / 22872 | 0 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 1 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| gb200 | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | +| gb200 | `moe_torch_flow` | shared | trtllm | trtllm:12312 | 0 / 7371 | 4 | — | — | — | +| gb200 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | +| gb200 | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:36045 | 0 / 36045 | 0 | — | — | — | +| gb200 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | +| gb200 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:159372 | 0 / 124818 | 0 | — | — | — | +| gb200 | `sglang_fused_moe_triton` | shared | sglang | sglang:50625 | 0 / 36936 | 0 | — | — | — | +| gb200 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:7563 | 0 / 5214 | 0 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | | gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:10287 | 0 / 4131 | 5 | — | — | — | -| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:45343 | 0 / 16232 | 40 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:6156 | 0 / 3402 | 5 | — | — | — | +| gb200 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:29864 | 0 / 15179 | 23 | — | — | — | | gb200 | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| gb200 | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 3 | — | — | — | -| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:38232 | 0 / 13284 | 35 | — | — | — | -| gb200 | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2835 | 0 / 2835 | 0 | — | — | — | -| gb200 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 5 | — | — | — | -| gb200 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 2 | — | — | — | -| gb200 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:35988 | 0 / 12555 | 41 | — | — | — | -| gb200 | `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:891 | 0 / 891 | 0 | — | — | — | -| gb200 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 1 | — | — | — | -| gb200 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 5 | — | — | — | +| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 2 | — | — | — | +| gb200 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 13 | — | — | — | +| gb200 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 1 | — | — | — | +| gb200 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 1 | — | — | — | +| gb200 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:23693 | 0 / 12555 | 21 | — | — | — | +| gb200 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 1 | — | — | — | +| gb200 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 3 | — | — | — | | gb200 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25876 | 0 / 13556 | 12 | — | — | — | -| gb200 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:15945 | 0 / 14919 | 1 | — | — | — | -| gb200 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| gb300 | `deepgemm` | shared | trtllm | trtllm:79299 | 0 / 40500 | 66 | — | — | — | -| gb300 | `moe_torch_flow` | shared | trtllm | trtllm:19845 | 0 / 7371 | 17 | — | — | — | -| gb300 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:239760 | 0 / 125712 | 152 | — | — | — | -| gb300 | `moe_torch_flow_min_latency` | shared | trtllm | trtllm:39465 | 0 / 39465 | 0 | — | — | — | -| gb300 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50868 | 0 / 27945 | 1 | — | — | — | -| gb300 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:284303 | 0 / 142619 | 3 | — | — | — | -| gb300 | `sglang_fused_moe_triton` | shared | sglang | sglang:94043 | 0 / 43416 | 1 | — | — | — | -| gb300 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:30375 | 0 / 22842 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| gb200 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1026 | 0 / 1026 | 0 | — | — | — | +| gb200 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| gb300 | `deepgemm` | shared | trtllm | trtllm:38799 | 0 / 38799 | 0 | — | — | — | +| gb300 | `moe_torch_flow` | shared | trtllm | trtllm:12474 | 0 / 7371 | 6 | — | — | — | +| gb300 | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:114048 | 0 / 114048 | 0 | — | — | — | +| gb300 | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22923 | 0 / 22923 | 0 | — | — | — | +| gb300 | `sglang_flashinfer_trtllm_moe` | shared | sglang | sglang:162504 | 0 / 124818 | 0 | — | — | — | +| gb300 | `sglang_fused_moe_triton` | shared | sglang | sglang:50627 | 0 / 36936 | 0 | — | — | — | +| gb300 | `sglang_mxfp4_flashinfer_trtllm_moe` | shared | sglang | sglang:7533 | 0 / 5184 | 0 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | | gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:8181 | 0 / 4131 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:44269 | 0 / 16180 | 40 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | shared | vllm | vllm:4050 | 0 / 3402 | 0 | — | — | — | +| gb300 | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:28785 | 0 / 15127 | 23 | — | — | — | | gb300 | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| gb300 | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:12555 | 0 / 4374 | 5 | — | — | — | -| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:38232 | 0 / 13284 | 34 | — | — | — | -| gb300 | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2835 | 0 / 2835 | 0 | — | — | — | -| gb300 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 7 | — | — | — | -| gb300 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:4779 | 0 / 1944 | 5 | — | — | — | -| gb300 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:36327 | 0 / 12555 | 46 | — | — | — | -| gb300 | `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | shared | vllm | vllm:891 | 0 / 891 | 0 | — | — | — | -| gb300 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:6561 | 0 / 2187 | 4 | — | — | — | -| gb300 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:5508 | 0 / 2835 | 5 | — | — | — | +| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | shared | vllm | vllm:8181 | 0 / 4374 | 4 | — | — | — | +| gb300 | `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:24948 | 0 / 12474 | 9 | — | — | — | +| gb300 | `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 4 | — | — | — | +| gb300 | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:2835 | 0 / 1863 | 2 | — | — | — | +| gb300 | `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | shared | vllm | vllm:23850 | 0 / 12555 | 20 | — | — | — | +| gb300 | `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | shared | vllm | vllm:4374 | 0 / 2187 | 4 | — | — | — | +| gb300 | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:3726 | 0 / 2835 | 3 | — | — | — | | gb300 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | shared | vllm | vllm:25849 | 0 / 13608 | 15 | — | — | — | -| gb300 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:16049 | 0 / 15023 | 0 | — | — | — | -| gb300 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| h100_sxm | `moe_torch_flow` | shared | trtllm | trtllm:2592 | 0 / 1053 | 1 | — | — | — | -| h100_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:301482 | 0 / 134622 | 95 | — | — | — | -| h100_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:52164 | 0 / 25191 | 9 | — | — | — | -| h100_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:17187 | 0 / 8601 | 0 | — | — | — | -| h100_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:256932 | 0 / 124497 | 10 | — | — | — | -| h100_sxm | `sglang_marlin_moe` | shared | sglang | sglang:28269 | 0 / 20493 | 0 | — | — | — | +| gb300 | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | shared | vllm | vllm:1026 | 0 / 1026 | 0 | — | — | — | +| gb300 | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| h100_sxm | `moe_torch_flow` | shared | trtllm | trtllm:1539 | 0 / 1053 | 1 | — | — | — | +| h100_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:185328 | 0 / 129357 | 16 | — | — | — | +| h100_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:32643 | 0 / 24462 | 9 | — | — | — | +| h100_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:8601 | 0 / 8601 | 0 | — | — | — | +| h100_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:157464 | 0 / 112509 | 1 | — | — | — | +| h100_sxm | `sglang_marlin_moe` | shared | sglang | sglang:22113 | 0 / 17415 | 0 | — | — | — | | h100_sxm | `sglang_marlin_moe_situ_as_silu` | shared | sglang | sglang:541 | 0 / 541 | 0 | — | — | — | | h100_sxm | `sglang_mxfp4_humming_moe` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h100_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 0 | — | — | — | +| h100_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | | h100_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| h100_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| h100_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:30213 | 0 / 10692 | 4 | — | — | — | -| h100_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:22323 | 0 / 9801 | 5 | — | — | — | -| h100_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 5 | — | — | — | -| h100_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:45927 | 0 / 16200 | 9 | — | — | — | -| h100_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:52753 | 0 / 18306 | 11 | — | — | — | -| h200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:3402 | 0 / 1134 | 1 | — | — | — | -| h200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:292734 | 0 / 134622 | 125 | — | — | — | -| h200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:52320 | 0 / 25272 | 14 | — | — | — | -| h200_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:17349 | 0 / 8682 | 0 | — | — | — | -| h200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:256932 | 0 / 124497 | 7 | — | — | — | -| h200_sxm | `sglang_marlin_moe` | shared | sglang | sglang:28269 | 0 / 20493 | 1 | — | — | — | +| h100_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:19521 | 0 / 10287 | 3 | — | — | — | +| h100_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:12549 | 0 / 6561 | 3 | — | — | — | +| h100_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 4 | — | — | — | +| h100_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:29727 | 0 / 15309 | 4 | — | — | — | +| h100_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:34559 | 0 / 18306 | 11 | — | — | — | +| h200_sxm | `moe_torch_flow` | shared | trtllm | trtllm:2268 | 0 / 1134 | 1 | — | — | — | +| h200_sxm | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:176580 | 0 / 129357 | 12 | — | — | — | +| h200_sxm | `moe_torch_flow_nongated` | shared | trtllm | trtllm:32724 | 0 / 24543 | 14 | — | — | — | +| h200_sxm | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:8682 | 0 / 8682 | 0 | — | — | — | +| h200_sxm | `sglang_fused_moe_triton` | shared | sglang | sglang:157464 | 0 / 112509 | 1 | — | — | — | +| h200_sxm | `sglang_marlin_moe` | shared | sglang | sglang:22113 | 0 / 17415 | 0 | — | — | — | | h200_sxm | `sglang_marlin_moe_situ_as_silu` | shared | sglang | sglang:488 | 0 / 488 | 0 | — | — | — | | h200_sxm | `sglang_mxfp4_humming_moe` | shared | sglang | sglang:15 | 0 / 15 | 0 | — | — | — | -| h200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:2916 | 0 / 972 | 0 | — | — | — | +| h200_sxm | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:1944 | 0 / 972 | 0 | — | — | — | | h200_sxm | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:2106 | 0 / 1053 | 0 | — | — | — | -| h200_sxm | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| h200_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:30213 | 0 / 10692 | 17 | — | — | — | -| h200_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:22356 | 0 / 9801 | 4 | — | — | — | +| h200_sxm | `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:19521 | 0 / 10287 | 14 | — | — | — | +| h200_sxm | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:12555 | 0 / 6561 | 0 | — | — | — | | h200_sxm | `vllm_fused_moe` | shared | vllm | vllm:36288 | 0 / 33210 | 0 | — | — | — | -| h200_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:5832 | 0 / 1944 | 4 | — | — | — | +| h200_sxm | `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | shared | vllm | vllm:3888 | 0 / 1944 | 3 | — | — | — | | h200_sxm | `vllm_marlin_int4_moe` | shared | vllm | vllm:10206 | 0 / 9234 | 0 | — | — | — | -| h200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:45927 | 0 / 16200 | 11 | — | — | — | +| h200_sxm | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:29727 | 0 / 15309 | 5 | — | — | — | | h200_sxm | `vllm_mxfp4_moe` | shared | vllm | vllm:1701 | 0 / 1296 | 0 | — | — | — | -| h200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:52893 | 0 / 18306 | 4 | — | — | — | +| h200_sxm | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:34587 | 0 / 18306 | 1 | — | — | — | | l40s | `moe_torch_flow` | shared | trtllm | trtllm:1960 | 0 / 1960 | 0 | — | — | — | -| l40s | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:153460 | 0 / 76385 | 2095 | — | — | — | -| l40s | `moe_torch_flow_nongated` | shared | trtllm | trtllm:32919 | 0 / 18746 | 15 | — | — | — | -| l40s | `sglang_fused_moe_triton` | shared | sglang | sglang:120390 | 0 / 65988 | 77 | — | — | — | +| l40s | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:77077 | 0 / 72092 | 7 | — | — | — | +| l40s | `moe_torch_flow_nongated` | shared | trtllm | trtllm:14372 | 0 / 14372 | 0 | — | — | — | +| l40s | `sglang_fused_moe_triton` | shared | sglang | sglang:54402 | 0 / 54402 | 0 | — | — | — | | l40s | `sglang_fused_moe_triton_situ_as_silu` | shared | sglang | sglang:2997 | 0 / 2997 | 0 | — | — | — | -| l40s | `sglang_marlin_moe` | shared | sglang | sglang:6156 | 0 / 3078 | 2 | — | — | — | -| l40s | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | +| l40s | `sglang_marlin_moe` | shared | sglang | sglang:3078 | 0 / 3078 | 0 | — | — | — | | l40s | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:1026 | 0 / 1026 | 0 | — | — | — | -| l40s | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| l40s | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:34573 | 0 / 19515 | 2 | — | — | — | -| l40s | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:3888 | 0 / 1944 | 1 | — | — | — | -| l40s | `vllm_modeloptfp8moe_triton_tritonexperts` | shared | vllm | vllm:30345 | 0 / 16200 | 2 | — | — | — | -| l40s | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:34042 | 0 / 18069 | 3 | — | — | — | -| rtx_pro_6000_server | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:250452 | 0 / 131058 | 102 | — | — | — | -| rtx_pro_6000_server | `moe_torch_flow_nongated` | shared | trtllm | trtllm:50785 | 0 / 27943 | 0 | — | — | — | -| rtx_pro_6000_server | `moe_torch_flow_triton_fp8_block` | shared | trtllm | trtllm:65205 | 0 / 32967 | 25 | — | — | — | -| rtx_pro_6000_server | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:91530 | 0 / 56052 | 2 | — | — | — | -| rtx_pro_6000_server | `sglang_fused_moe_triton` | shared | sglang | sglang:167184 | 0 / 89829 | 6 | — | — | — | +| l40s | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:15118 | 0 / 15118 | 0 | — | — | — | +| l40s | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:1944 | 0 / 1944 | 0 | — | — | — | +| l40s | `vllm_modeloptfp8moe_triton_tritonexperts` | shared | vllm | vllm:14174 | 0 / 14174 | 0 | — | — | — | +| l40s | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:16055 | 0 / 16055 | 0 | — | — | — | +| rtx_pro_6000_server | `moe_torch_flow_cutlass` | shared | trtllm | trtllm:119394 | 0 / 119394 | 0 | — | — | — | +| rtx_pro_6000_server | `moe_torch_flow_nongated` | shared | trtllm | trtllm:22921 | 0 / 22921 | 0 | — | — | — | +| rtx_pro_6000_server | `moe_torch_flow_triton_fp8_block` | shared | trtllm | trtllm:32238 | 0 / 32238 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_flashinfer_cutlass_moe` | shared | sglang | sglang:35478 | 0 / 35478 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang_fused_moe_triton` | shared | sglang | sglang:77355 | 0 / 77355 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_marlin_moe` | shared | sglang | sglang:4456 | 0 / 4456 | 0 | — | — | — | | rtx_pro_6000_server | `sglang_marlin_moe_situ_as_silu` | shared | sglang | sglang:551 | 0 / 551 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | shared | vllm | vllm:972 | 0 / 972 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:35505 | 0 / 19683 | 5 | — | — | — | +| rtx_pro_6000_server | `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:15822 | 0 / 15822 | 0 | — | — | — | | rtx_pro_6000_server | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_compressedtensorswna16moe_marlin_marlinexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_fp8moe_deepgemm_deepgemmexperts` | shared | vllm | vllm:16821 | 0 / 16821 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_fp8moe_triton_tritonexperts` | shared | vllm | vllm:2754 | 0 / 2754 | 0 | — | — | — | -| rtx_pro_6000_server | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:3888 | 0 / 1944 | 1 | — | — | — | -| rtx_pro_6000_server | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:29484 | 0 / 16200 | 6 | — | — | — | -| rtx_pro_6000_server | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:32670 | 0 / 17901 | 9 | — | — | — | -| rtx_pro_6000_server | `vllm_unquantizedfusedmoe_triton_tritonexperts` | shared | vllm | vllm:1053 | 0 / 1053 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_gptossmxfp4moe_marlin_marlinexperts` | shared | vllm | vllm:1944 | 0 / 1944 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:13284 | 0 / 13284 | 0 | — | — | — | +| rtx_pro_6000_server | `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | shared | vllm | vllm:15903 | 0 / 15903 | 0 | — | — | — | ## `msa_context_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33079 | 0 / 14396 | 122 | — | — | — | -| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | -| b200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60326 | 0 / 33649 | 0 | — | — | — | -| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33095 | 0 / 14431 | 147 | — | — | — | -| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | -| b300_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60534 | 0 / 34014 | 0 | — | — | — | -| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33080 | 0 / 14388 | 99 | — | — | — | -| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | -| gb200 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60001 | 0 / 33660 | 0 | — | — | — | -| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:33160 | 0 / 14403 | 95 | — | — | — | -| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:8106 | 0 / 5070 | 0 | — | — | — | -| gb300 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:60947 | 0 / 34014 | 0 | — | — | — | -| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:23958 | 0 / 11567 | 103 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:5791 | 0 / 3382 | 12 | — | — | — | -| h100_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:58866 | 0 / 32870 | 0 | — | — | — | -| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:23956 | 0 / 11567 | 268 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:5808 | 0 / 3388 | 8 | — | — | — | -| h200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:59957 | 0 / 33505 | 0 | — | — | — | -| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:12688 | 0 / 7808 | 35 | — | — | — | -| l40s | `default` | shared_fallback | trtllm | trtllm:5671 | 0 / 3313 | 11 | — | — | — | -| l40s | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:15731 | 0 / 15731 | 0 | — | — | — | -| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:15128 | 0 / 10248 | 77 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:3630 | 0 / 3630 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:32732 | 0 / 32732 | 0 | — | — | — | +| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25822 | 0 / 14396 | 47 | — | — | — | +| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | +| b200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26933 | 0 / 26933 | 0 | — | — | — | +| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25849 | 0 / 14431 | 73 | — | — | — | +| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | +| b300_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26850 | 0 / 26850 | 0 | — | — | — | +| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25820 | 0 / 14388 | 57 | — | — | — | +| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | +| gb200 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26608 | 0 / 26608 | 0 | — | — | — | +| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:25886 | 0 / 14403 | 26 | — | — | — | +| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:5070 | 0 / 5070 | 0 | — | — | — | +| gb300 | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:27242 | 0 / 27242 | 0 | — | — | — | +| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:19078 | 0 / 11567 | 30 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:3382 | 0 / 3382 | 0 | — | — | — | +| h100_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26348 | 0 / 26348 | 0 | — | — | — | +| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:19076 | 0 / 11567 | 144 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:3388 | 0 / 3388 | 0 | — | — | — | +| h200_sxm | `sglang_minimax_prefill_triton_sparse` | shared | sglang | sglang:26845 | 0 / 26845 | 0 | — | — | — | +| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:7808 | 0 / 7808 | 0 | — | — | — | +| l40s | `default` | shared_fallback | trtllm | trtllm:3313 | 0 / 3313 | 0 | — | — | — | +| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:7808 | 0 / 7808 | 0 | — | — | — | ## `msa_generation_module_perf.parquet` | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | -| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | -| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:268 | 0 / 149 | 0 | — | — | — | -| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3056 | 0 / 1698 | 69 | — | — | — | -| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | -| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | -| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:270 | 0 / 150 | 0 | — | — | — | -| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3060 | 0 / 1700 | 87 | — | — | — | -| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | -| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | -| gb200 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:268 | 0 / 149 | 0 | — | — | — | -| gb200 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3056 | 0 / 1698 | 17 | — | — | — | -| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:14352 | 0 / 14352 | 0 | — | — | — | -| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:6624 | 0 / 6624 | 0 | — | — | — | -| gb300 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:270 | 0 / 150 | 0 | — | — | — | -| gb300 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:3060 | 0 / 1700 | 67 | — | — | — | -| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:9544 | 0 / 9544 | 0 | — | — | — | -| h100_sxm | `default` | shared_fallback | trtllm | trtllm:4360 | 0 / 4360 | 0 | — | — | — | -| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:528 | 0 / 298 | 0 | — | — | — | -| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:2755 | 0 / 1533 | 16 | — | — | — | -| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:9562 | 0 / 9562 | 0 | — | — | — | -| h200_sxm | `default` | shared_fallback | trtllm | trtllm:4376 | 0 / 4376 | 0 | — | — | — | -| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:550 | 0 / 308 | 1 | — | — | — | -| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:2768 | 0 / 1539 | 19 | — | — | — | -| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:9512 | 0 / 9512 | 0 | — | — | — | -| l40s | `default` | shared_fallback | trtllm | trtllm:4292 | 0 / 4292 | 0 | — | — | — | -| l40s | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:370 | 0 / 370 | 0 | — | — | — | -| l40s | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:515 | 0 / 515 | 0 | — | — | — | -| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:11402 | 0 / 11402 | 0 | — | — | — | -| rtx_pro_6000_server | `default` | shared_fallback | trtllm | trtllm:2730 | 0 / 2730 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:294 | 0 / 294 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1531 | 0 / 1531 | 0 | — | — | — | +| b200_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | +| b200_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | +| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:119 | 0 / 119 | 0 | — | — | — | +| b200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1358 | 0 / 1358 | 0 | — | — | — | +| b300_sxm | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | +| b300_sxm | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | +| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:120 | 0 / 120 | 0 | — | — | — | +| b300_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1360 | 0 / 1360 | 0 | — | — | — | +| gb200 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | +| gb200 | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | +| gb200 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:119 | 0 / 119 | 0 | — | — | — | +| gb200 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1358 | 0 / 1358 | 0 | — | — | — | +| gb300 | `MiniMaxM3SparseMSAImpl` | shared | vllm | vllm:8832 | 0 / 8832 | 0 | — | — | — | +| gb300 | `msa_fmha_sm100` | shared | trtllm | trtllm:3864 | 0 / 3864 | 0 | — | — | — | +| gb300 | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:120 | 0 / 120 | 0 | — | — | — | +| gb300 | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1360 | 0 / 1360 | 0 | — | — | — | +| h100_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5870 | 0 / 5870 | 0 | — | — | — | +| h100_sxm | `default` | shared_fallback | trtllm | trtllm:2548 | 0 / 2548 | 0 | — | — | — | +| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:240 | 0 / 240 | 0 | — | — | — | +| h100_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1227 | 0 / 1227 | 0 | — | — | — | +| h200_sxm | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5882 | 0 / 5882 | 0 | — | — | — | +| h200_sxm | `default` | shared_fallback | trtllm | trtllm:2556 | 0 / 2556 | 0 | — | — | — | +| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_radix` | shared | sglang | sglang:246 | 0 / 246 | 0 | — | — | — | +| h200_sxm | `sglang_minimax_decode_triton_sparse_topk_split` | shared | sglang | sglang:1231 | 0 / 1231 | 0 | — | — | — | +| l40s | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5850 | 0 / 5850 | 0 | — | — | — | +| l40s | `default` | shared_fallback | trtllm | trtllm:2508 | 0 / 2508 | 0 | — | — | — | +| rtx_pro_6000_server | `MiniMaxM3SparseTritonImpl` | shared | vllm | vllm:5882 | 0 / 5882 | 0 | — | — | — | ## `nccl_perf.parquet` @@ -1466,30 +1249,30 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur | system | kernel_source | tier | frameworks | rows_per_fw | overlap_keys | dedup rows | median % | p95 % | max % | |---|---|---|---|---|---|---|---|---|---| -| b200_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 7 | — | — | — | -| b200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 11 | — | — | — | -| b200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 3256 | 0 | — | — | — | -| b300_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | -| b300_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 14 | — | — | — | -| b300_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 15 | — | — | — | -| gb200 | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | -| gb200 | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 9 | — | — | — | -| gb200 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 11 | — | — | — | -| gb300 | `sglang` | shared | sglang | sglang:4884 | 0 / 1628 | 1 | — | — | — | -| gb300 | `static_scaled_fp8_quant` | shared | vllm | vllm:4884 | 0 / 1628 | 25 | — | — | — | -| gb300 | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 7 | — | — | — | -| h100_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 0 | — | — | — | -| h100_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 17 | — | — | — | -| h200_sxm | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 0 | — | — | — | -| h200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 2 | — | — | — | -| h200_sxm | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 24 | — | — | — | -| l40s | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 66 | — | — | — | -| l40s | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 272 | — | — | — | -| l40s | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 0 | — | — | — | -| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:3256 | 0 / 1628 | 1 | — | — | — | -| rtx_pro_6000_server | `static_scaled_fp8_quant` | shared | vllm | vllm:3256 | 0 / 1628 | 7 | — | — | — | -| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:3256 | 0 / 1628 | 2 | — | — | — | +| b200_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| b200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| b200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| b300_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| b300_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| b300_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| gb200 | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| gb200 | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| gb200 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| gb300 | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| gb300 | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| gb300 | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| h100_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| h200_sxm | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| l40s | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| l40s | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| l40s | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | +| rtx_pro_6000_server | `sglang` | shared | sglang | sglang:1628 | 0 / 1628 | 0 | — | — | — | +| rtx_pro_6000_server | `static_scaled_fp8_quant` | shared | vllm | vllm:1628 | 0 / 1628 | 0 | — | — | — | +| rtx_pro_6000_server | `torch_ops` | shared | trtllm | trtllm:1628 | 0 / 1628 | 0 | — | — | — | ## `trtllm_alltoall_perf.parquet` @@ -1599,86 +1382,71 @@ Rows with a blank/`` kernel_source are skipped during the scan (the cur Each row is one distinct `kernel_source` value seen in the corpus, with the union of frameworks, op files, and systems it appears in. Tier is determined by the kernel_source name alone, so a single kernel_source has one tier across the whole corpus. -### `shared` (211 kernel sources) +### `shared` (175 kernel sources) | kernel_source | frameworks | op files | systems | rows | |---|---|---|---|---| -| `causal_conv1d_fn` | sglang, trtllm, vllm | gdn_perf.parquet, kda_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 223,690 | +| `causal_conv1d_fn` | sglang, trtllm, vllm | gdn_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 109,550 | | `causal_conv1d_fn_qkv3` | sglang, vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 7,012 | -| `causal_conv1d_fn_token_major` | trtllm | mamba2_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 4,778 | -| `causal_conv1d_update` | sglang, trtllm, vllm | gdn_perf.parquet, kda_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 21,550 | -| `chunk_gated_delta_rule` | sglang, trtllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 72,299 | -| `chunk_gated_delta_rule_flashinfer` | trtllm, vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 120,360 | -| `chunk_gated_delta_rule_triton` | vllm | gdn_perf.parquet | l40s, rtx_pro_6000_server | 12,265 | -| `chunk_kda` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,103 | +| `causal_conv1d_update` | sglang, trtllm, vllm | gdn_perf.parquet, kda_perf.parquet, mamba2_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 13,098 | +| `chunk_gated_delta_rule` | sglang, trtllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 40,725 | +| `chunk_gated_delta_rule_flashinfer` | vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 55,303 | +| `chunk_gated_delta_rule_triton` | vllm | gdn_perf.parquet | l40s, rtx_pro_6000_server | 7,870 | +| `chunk_kda` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,168 | | `chunk_kda_with_fused_gate` | vllm | kda_perf.parquet | l40s | 414 | -| `compressed_flashmla` | sglang | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 1,368,500 | -| `CutlassFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 109,742 | -| `CutlassFP8ScaledMMLinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,137,698 | -| `deep_gemm.fp8_mqa_logits` | sglang | glm5_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 38,762 | -| `deep_gemm.fp8_paged_mqa_logits` | sglang | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 46,656 | +| `compressed_flashmla` | sglang | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 748,644 | +| `CutlassFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 51,726 | +| `CutlassFP8ScaledMMLinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 654,580 | +| `deep_gemm.fp8_mqa_logits` | sglang | glm5_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 19,378 | +| `deep_gemm.fp8_paged_mqa_logits` | sglang | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 19,276 | | `deepep` | sglang, trtllm, vllm | moe_a2a_perf.parquet, wideep_deepep_ll_perf.parquet, wideep_deepep_normal_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 23,512 | | `deepepmoe` | sglang | wideep_context_moe_perf.parquet, wideep_generation_moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 6,333 | -| `deepgemm` | trtllm | gemm_perf.parquet, moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 551,330 | +| `deepgemm` | trtllm | gemm_perf.parquet, moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 272,268 | | `deepgemm_megamoe` | sglang | dsv4_megamoe_module_perf.parquet | b200_sxm, gb200, gb300 | 776 | -| `DeepGemmFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 641,938 | +| `DeepGemmFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 296,224 | | `deepseek_v3` | sglang | wideep_context_mlp_perf.parquet, wideep_generation_mlp_perf.parquet | h100_sxm, h200_sxm | 72 | -| `DeepseekV4TrtllmAttention` | trtllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 145,545 | -| `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | vllm | computescale_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 27,513 | -| `fa3` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | h100_sxm, h200_sxm | 296,574 | -| `fast_topk_transform_fused` | sglang | glm5_topk_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 48,196 | -| `flash_attention` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | a100_sxm, h100_sxm, h200_sxm | 42,304 | -| `flash_attention_v3` | sglang | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 31,816 | -| `flash_attention_v4` | sglang | encoder_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 46,074 | -| `FLASH_ATTN` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 133,947 | -| `FLASH_ATTN_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 31,424 | -| `FLASH_ATTN_MLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 428,148 | -| `flash_mla_sparse_fwd` | sglang | glm5_dsa_attn_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 39,225 | -| `flashinfer` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 268,399 | -| `flashinfer_chunk_gated_delta_rule` | sglang | gdn_perf.parquet | h100_sxm, h200_sxm | 9,058 | +| `DeepseekV4TrtllmAttention` | trtllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 57,096 | +| `dynamic_per_token_scaled_fp8_quant_minus_static_scaled_fp8_quant` | vllm | computescale_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 12,941 | +| `fa3` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | h100_sxm, h200_sxm | 145,130 | +| `fast_topk_transform_fused` | sglang | glm5_topk_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 24,092 | +| `flash_attention` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | a100_sxm, h100_sxm, h200_sxm | 26,008 | +| `flash_attention_v3` | sglang | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 15,358 | +| `flash_attention_v4` | sglang | encoder_attention_perf.parquet | b200_sxm, gb200 | 15,358 | +| `FLASH_ATTN` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 84,027 | +| `FLASH_ATTN_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 19,640 | +| `flash_mla_sparse_fwd` | sglang | glm5_dsa_attn_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 19,841 | +| `flashinfer` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 133,439 | | `flashinfer_gated_delta_rule_decode` | sglang | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 208 | -| `FLASHINFER_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 167,416 | -| `FLASHINFER_MLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 1,180,816 | -| `FLASHINFER_MLA_SPARSE_DSV4` | vllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | rtx_pro_6000_server | 14,678 | -| `FLASHINFER_MLA_SPARSE_SM120` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | rtx_pro_6000_server | 119,020 | -| `FlashInferCuteDslNvFp4LinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 696,414 | -| `FlashInferCutlassNvFp4LinearKernel` | vllm | gemm_perf.parquet | rtx_pro_6000_server | 73,260 | -| `FlashInferFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | h100_sxm, h200_sxm | 42,972 | -| `flashkda_fwd` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 5,402 | -| `FLASHMLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 29,204 | -| `FLASHMLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 1,181,841 | -| `FLASHMLA_SPARSE_DSV4` | vllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 264,167 | +| `FLASHINFER_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 110,536 | +| `FLASHINFER_MLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 71,728 | +| `FlashInferCuteDslNvFp4LinearKernel` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 436,600 | +| `FlashInferCutlassNvFp4LinearKernel` | vllm | gemm_perf.parquet | rtx_pro_6000_server | 35,742 | +| `FlashInferFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | h100_sxm, h200_sxm | 14,280 | +| `flashkda_fwd` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 3,415 | +| `FLASHMLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | h100_sxm, h200_sxm | 18,212 | +| `FLASHMLA_SPARSE` | vllm | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 83,700 | +| `FLASHMLA_SPARSE_DSV4` | vllm | dsv4_csa_context_module_perf.parquet, dsv4_csa_generation_module_perf.parquet, dsv4_hca_attn_module_perf.parquet, dsv4_hca_context_module_perf.parquet, dsv4_hca_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 211,005 | | `fused_kda_decode` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 308 | | `fused_kda_decode_mtp_dspark` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 428 | -| `fused_qkvzba_causal_conv1d_update_contiguous` | sglang | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,696 | | `fused_recurrent_gated_delta_rule` | sglang, trtllm | gdn_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 774 | -| `fused_recurrent_gated_delta_rule_packed_decode` | sglang, vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 16,950 | -| `fused_recurrent_kda` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,309 | +| `fused_recurrent_gated_delta_rule_packed_decode` | sglang, vllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 9,638 | +| `fused_recurrent_kda` | vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 972 | | `fused_recurrent_kda_packed_decode` | sglang, vllm | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 651 | -| `fused_sigmoid_gating_delta_rule_update` | sglang | kda_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 630 | -| `fused_sigmoid_gating_delta_rule_update_flashinfer` | trtllm | gdn_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 2,772 | -| `fused_sigmoid_gating_delta_rule_update_triton` | trtllm | gdn_perf.parquet | l40s, rtx_pro_6000_server | 924 | +| `fused_sigmoid_gating_delta_rule_update` | sglang | kda_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 432 | | `kda_fused_decode` | sglang | kda_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 77 | -| `mamba_chunk_scan_combined` | trtllm | mamba2_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 4,777 | -| `MarlinFP8ScaledMMLinearKernel` | vllm | gemm_perf.parquet | l40s | 37,518 | -| `MiniMaxM3SparseMSAImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 189,822 | -| `MiniMaxM3SparseTritonImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 115,750 | +| `MiniMaxM3SparseMSAImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 138,705 | +| `MiniMaxM3SparseTritonImpl` | vllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 77,254 | | `mla_trtllm_mla` | sglang | mla_context_module_perf.parquet | b200_sxm, b300_sxm | 436 | -| `moe_torch_flow` | trtllm | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 93,599 | -| `moe_torch_flow_cutlass` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,957,168 | -| `moe_torch_flow_min_latency` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 229,950 | -| `moe_torch_flow_nongated` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 391,660 | -| `moe_torch_flow_triton_fp8_block` | trtllm | moe_perf.parquet | rtx_pro_6000_server | 65,205 | -| `msa_fmha_sm100` | trtllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 58,920 | +| `moe_torch_flow` | trtllm | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 62,819 | +| `moe_torch_flow_cutlass` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,014,571 | +| `moe_torch_flow_min_latency` | trtllm | moe_perf.parquet | b200_sxm, gb200 | 72,090 | +| `moe_torch_flow_nongated` | trtllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 194,352 | +| `moe_torch_flow_triton_fp8_block` | trtllm | moe_perf.parquet | rtx_pro_6000_server | 32,238 | +| `msa_fmha_sm100` | trtllm | msa_context_module_perf.parquet, msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 35,736 | | `NCCL` | trtllm | nccl_perf.parquet | b300_sxm | 1,512 | | `NVLinkOneSided` | trtllm | trtllm_alltoall_perf.parquet | gb200 | 296 | | `NVLinkTwoSided` | trtllm | trtllm_alltoall_perf.parquet | gb200 | 1,800 | -| `selective_state_update_flashinfer` | trtllm | mamba2_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 528 | -| `sglang` | sglang | computescale_perf.parquet, gemm_perf.parquet, scale_matrix_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 98,806 | -| `sglang.hc_post[tilelang]` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `sglang.hc_pre[tilelang,prenorm_gemm=deepgemm,norm=fused]` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `sglang.kernels.ops.layernorm.mhc.hc_contract` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `sglang.kernels.ops.layernorm.mhc.hc_expand` | sglang | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | +| `sglang` | sglang | computescale_perf.parquet, gemm_perf.parquet, scale_matrix_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 84,087 | | `sglang.srt.layers.engram.Engram.apply_gate+sglang.srt.layers.engram.EngramEmbedding._owned_rows+sglang.srt.layers.linear.ReplicatedLinear.forward/sglang.srt.layers.quantization.fp8.Fp8LinearMethod` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 240 | | `sglang.srt.layers.linear.MergedColumnParallelLinear.forward/sglang.srt.layers.quantization.fp8.Fp8LinearMethod` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 120 | | `sglang.srt.layers.linear.RowParallelLinear.forward/sglang.srt.layers.quantization.fp8.Fp8LinearMethod` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 120 | @@ -1686,132 +1454,111 @@ Each row is one distinct `kernel_source` value seen in the corpus, with the unio | `sglang.srt.models.deepseek_v2.MoEGate.forward` | sglang | gemm_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 60 | | `sglang.srt.models.deepseek_v4.DeepseekV4DecoderLayer._hc_mix_and_combine+sglang.srt.models.deepseek_v4.DeepseekV4DecoderLayer.hc_post` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 60 | | `sglang.srt.models.deepseek_v4.MQALayer.forward` | sglang | dsv41_module_perf.parquet | b200_sxm, gb200, h100_sxm, h200_sxm | 9,040 | -| `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=Fp8LinearMethod|q_b=Fp8LinearMethod|kv_b=UnquantizedLinearMethod|o=Fp8LinearMethod|attn_backend=dsa|method=MLA` | sglang | glm53_attention_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 3,659 | -| `sglang0.5.20|DeepseekV2AttentionMLA|IndexerKPool|qkv_a=UnquantizedLinearMethod|q_b=UnquantizedLinearMethod|kv_b=UnquantizedLinearMethod|o=UnquantizedLinearMethod|attn_backend=dsa|method=MLA` | sglang | glm53_attention_module_perf.parquet | b300_sxm, gb200, gb300 | 3,217 | -| `SGLang_CustomAllReduce_eager` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 2,024 | -| `SGLang_CustomAllReduce_graph` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 2,024 | -| `sglang_cutedsl_tgv_bf16` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300 | 3,929 | -| `sglang_deepgemm_gemm_nt_f8f8bf16` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 629,739 | -| `sglang_dsa_dense_mha_fa3` | sglang | dsa_context_module_perf.parquet | h100_sxm, h200_sxm | 145,111 | -| `sglang_dsa_dense_mha_trtllm_ragged` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 277,456 | -| `sglang_dsa_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 9,008 | -| `sglang_dsa_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 80,896 | -| `sglang_dsa_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 223,605 | -| `sglang_dsa_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 176,983 | -| `sglang_dsa_skip_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 4,912 | -| `sglang_dsa_skip_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 49,141 | -| `sglang_dsa_skip_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 126,011 | -| `sglang_dsa_skip_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 100,438 | -| `sglang_flashinfer_bf16_direct` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300 | 24 | -| `sglang_flashinfer_bf16_splitk` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300 | 105 | -| `sglang_flashinfer_bmm_fp8` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,088 | -| `sglang_flashinfer_cutedsl_nvfp4` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 464,350 | -| `sglang_flashinfer_cutlass_moe` | sglang | moe_perf.parquet | h100_sxm, h200_sxm, rtx_pro_6000_server | 126,066 | -| `sglang_flashinfer_cutlass_nvfp4` | sglang | gemm_perf.parquet | rtx_pro_6000_server | 59,126 | -| `sglang_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 1,127,207 | -| `sglang_fused_moe_triton` | sglang | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,211,403 | +| `SGLang_CustomAllReduce_eager` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,725 | +| `SGLang_CustomAllReduce_graph` | sglang | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,725 | +| `sglang_deepgemm_gemm_nt_f8f8bf16` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 295,260 | +| `sglang_dsa_dense_mha_fa3` | sglang | dsa_context_module_perf.parquet | h100_sxm, h200_sxm | 82,901 | +| `sglang_dsa_dense_mha_trtllm_ragged` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 137,411 | +| `sglang_dsa_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 5,352 | +| `sglang_dsa_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 46,332 | +| `sglang_dsa_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 123,166 | +| `sglang_dsa_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 91,197 | +| `sglang_dsa_skip_indexer_fa3` | sglang | dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 2,448 | +| `sglang_dsa_skip_indexer_flashmla_kv` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | h100_sxm, h200_sxm | 24,610 | +| `sglang_dsa_skip_indexer_flashmla_sparse` | sglang | dsa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 54,618 | +| `sglang_dsa_skip_indexer_trtllm` | sglang | dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 39,806 | +| `sglang_flashinfer_cutedsl_nvfp4` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 236,208 | +| `sglang_flashinfer_cutlass_moe` | sglang | moe_perf.parquet | h100_sxm, h200_sxm, rtx_pro_6000_server | 52,761 | +| `sglang_flashinfer_cutlass_nvfp4` | sglang | gemm_perf.parquet | rtx_pro_6000_server | 29,526 | +| `sglang_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 645,304 | +| `sglang_fused_moe_triton` | sglang | moe_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 682,986 | | `sglang_fused_moe_triton_situ_as_silu` | sglang | moe_perf.parquet | l40s | 2,997 | -| `sglang_marlin_moe` | sglang | moe_perf.parquet | a100_sxm, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 86,185 | +| `sglang_marlin_moe` | sglang | moe_perf.parquet | a100_sxm, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 70,795 | | `sglang_marlin_moe_situ_as_silu` | sglang | moe_perf.parquet | h100_sxm, h200_sxm, rtx_pro_6000_server | 1,580 | -| `sglang_minimax_decode_triton_sparse_topk_radix` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 2,818 | -| `sglang_minimax_decode_triton_sparse_topk_split` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 19,801 | -| `sglang_minimax_prefill_triton_sparse` | sglang | msa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 409,094 | -| `sglang_mxfp4_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 121,398 | +| `sglang_minimax_decode_triton_sparse_topk_radix` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 964 | +| `sglang_minimax_decode_triton_sparse_topk_split` | sglang | msa_generation_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 7,894 | +| `sglang_minimax_prefill_triton_sparse` | sglang | msa_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 160,826 | +| `sglang_mxfp4_flashinfer_trtllm_moe` | sglang | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 30,030 | | `sglang_mxfp4_humming_moe` | sglang | moe_perf.parquet | h100_sxm, h200_sxm | 30 | | `sglang_sgl_kernel_bmm_fp8` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,724 | -| `sglang_sgl_kernel_fp8_scaled_mm` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 760,646 | -| `sglang_tilelang_mhc_post` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 903 | -| `sglang_tilelang_mhc_pre` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 874 | -| `sglang_torch_bmm` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 10,812 | -| `sglang_torch_linear` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 908,794 | +| `sglang_sgl_kernel_fp8_scaled_mm` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 429,348 | +| `sglang_tilelang_mhc_post` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 485 | +| `sglang_tilelang_mhc_pre` | sglang | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 420 | +| `sglang_torch_bmm` | sglang | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 5,724 | +| `sglang_torch_linear` | sglang | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 429,718 | | `sglang_torch_mhc_pre` | sglang | mhc_module_perf.parquet | rtx_pro_6000_server | 67 | -| `sglang_triton_w8a8_block_fp8` | sglang | gemm_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 9,842 | -| `static_scaled_fp8_quant` | vllm | scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 27,676 | -| `topk_transform_v1` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 40,124 | -| `topk_transform_v2` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 2,304 | -| `torch.nn.functional.linear` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,137,658 | -| `torch_flow` | trtllm | context_attention_perf.parquet, encoder_attention_perf.parquet, gemm_perf.parquet, generation_attention_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,825,706 | -| `torch_flow_flashinfer` | trtllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 42,856 | -| `torch_ops` | trtllm | computescale_perf.parquet, scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 52,055 | -| `triton` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, encoder_attention_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 350,168 | -| `TRITON_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | l40s, rtx_pro_6000_server | 17,972 | +| `static_scaled_fp8_quant` | vllm | scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 13,024 | +| `topk_transform_v1` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 25,764 | +| `topk_transform_v2` | sglang | dsv4_csa_topk_calib_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 1,464 | +| `torch.nn.functional.linear` | vllm | gemm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 654,534 | +| `torch_flow` | trtllm | context_attention_perf.parquet, encoder_attention_perf.parquet, gemm_perf.parquet, generation_attention_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,891,515 | +| `torch_flow_flashinfer` | trtllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 23,408 | +| `torch_ops` | trtllm | computescale_perf.parquet, scale_matrix_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 26,012 | +| `triton` | sglang | context_attention_perf.parquet, context_mla_perf.parquet, encoder_attention_perf.parquet, generation_attention_perf.parquet, generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 173,342 | +| `TRITON_MLA` | vllm | mla_context_module_perf.parquet, mla_generation_module_perf.parquet | l40s, rtx_pro_6000_server | 5,952 | | `TritonFp8BlockScaledMMKernel` | vllm | gemm_perf.parquet | l40s | 35,676 | | `trt_flow_/smooth_quant_gemm_L96/PLUGIN_V2_SmoothQuantGemm_0` | trtllm | gemm_perf.parquet | a100_sxm | 6,048 | | `trt_flow_/weight_only_quant_matmul_L257/PLUGIN_V2_WeightOnlyQuantMatmul_0` | trtllm | gemm_perf.parquet | a100_sxm | 12,096 | | `TRTLLM` | trtllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,518 | -| `trtllm_bmm_out` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 8,480 | -| `trtllm_bmm_out_dequant_bf16` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 4,240 | -| `trtllm_fp8_block_scaling_bmm_out` | trtllm | mla_bmm_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 4,240 | -| `trtllm_mha` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 565,262 | -| `trtllm_mhc_post_mapping` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,110 | -| `trtllm_mhc_pre_dg_nosplit` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 312 | -| `trtllm_mhc_pre_dg_splitk` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 392 | -| `trtllm_mhc_pre_fma` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 388 | -| `trtllm_mla` | sglang | context_mla_perf.parquet, generation_mla_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 70,527 | +| `trtllm_bmm_out` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 3,392 | +| `trtllm_bmm_out_dequant_bf16` | trtllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 1,696 | +| `trtllm_fp8_block_scaling_bmm_out` | trtllm | mla_bmm_perf.parquet | h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,696 | +| `trtllm_mha` | sglang | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 273,765 | +| `trtllm_mhc_post_mapping` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 555 | +| `trtllm_mhc_pre_dg_nosplit` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 159 | +| `trtllm_mhc_pre_dg_splitk` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 156 | +| `trtllm_mhc_pre_fma` | trtllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 231 | +| `trtllm_mla` | sglang | context_mla_perf.parquet, generation_mla_perf.parquet, wideep_context_mla_perf.parquet, wideep_generation_mla_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 39,383 | | `TRTLLM_MNNVL_oneshot` | trtllm | custom_allreduce_perf.parquet | gb300 | 19 | | `TRTLLM_MNNVL_twoshot` | trtllm | custom_allreduce_perf.parquet | gb300 | 27 | -| `TRTLLM_RAGGED` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 50,135 | -| `vllm.mhc_fused_post_pre_tilelang[fused_post_gemm,norm=fused]` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 30 | -| `vllm.mhc_fused_post_pre_tilelang[post+prenorm_gemm=deepgemm,norm=fused]` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 144 | -| `vllm.mhc_post_tilelang` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `vllm.mhc_pre_tilelang[prenorm_gemm=deepgemm,norm=fused]` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,460 | -| `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,455 | -| `vllm.model_executor.layers.mhc.hc_contract` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `vllm.model_executor.layers.mhc.hc_expand` | vllm | mhc_module_perf.parquet | b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 175 | -| `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | vllm | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 62,688 | -| `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseImpl` | vllm | glm53_attention_module_perf.parquet | b300_sxm, gb200, gb300 | 6,019 | -| `vllm0.31.0|Glm5NextMLAAttention|SparseAttnIndexerKpool|projections=UnquantizedLinearMethod|mla_impl=FlashInferMLASparseSM90Impl` | vllm | glm53_attention_module_perf.parquet | h100_sxm, h200_sxm | 1,374 | +| `TRTLLM_RAGGED` | vllm | mla_context_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 33,863 | +| `vllm.model_executor.kernels.mhc.tilelang.mhc_post_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 905 | +| `vllm.model_executor.kernels.mhc.tilelang.mhc_pre_tilelang` | vllm | mhc_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 896 | +| `vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits` | vllm | dsv4_paged_mqa_logits_module_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 35,262 | | `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm | 1,944 | -| `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 17,496 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutedsl_flashinfercutedslexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 3,888 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 43,281 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 36,936 | -| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 179,196 | +| `vllm_compressedtensorsw4a4mxfp4moe_marlin_marlinexperts_situ_as_silu` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 9,720 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 23,598 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 20,412 | +| `vllm_compressedtensorsw4a4nvfp4moe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 117,220 | | `vllm_compressedtensorswna16marlinmoe_marlin_marlinexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 14,715 | -| `vllm_compressedtensorswna16moe_marlin_marlinexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 8,424 | -| `vLLM_custom_eager` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, b60, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,840 | -| `vLLM_custom_graph` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, b60, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,771 | -| `vllm_default` | vllm | gemm_perf.parquet | a100_sxm, b60 | 70,938 | -| `vllm_flash_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | a100_sxm, b60 | 97,251 | -| `vllm_flash_attn_fa2` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, l40s, rtx_pro_6000_server | 245,795 | -| `vllm_flash_attn_fa3` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 700,938 | -| `vllm_flash_attn_fa4` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 9,256 | +| `vLLM_custom_eager` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, b60, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,633 | +| `vLLM_custom_graph` | vllm | custom_allreduce_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 1,426 | +| `vllm_default` | vllm | gemm_perf.parquet | a100_sxm, b60 | 49,770 | +| `vllm_flash_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | a100_sxm, b60 | 102,215 | +| `vllm_flash_attn_fa2` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | l40s, rtx_pro_6000_server | 111,574 | +| `vllm_flash_attn_fa3` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 463,230 | +| `vllm_flash_attn_fa4` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | h100_sxm, h200_sxm | 6,006 | | `vllm_flashinfer` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm | 71,600 | -| `vllm_flashinfer_fidecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | l40s, rtx_pro_6000_server | 92,723 | -| `vllm_flashinfer_fiprefill` | vllm | context_attention_perf.parquet | l40s, rtx_pro_6000_server | 90,646 | +| `vllm_flashinfer_fidecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | l40s, rtx_pro_6000_server | 58,484 | +| `vllm_flashinfer_fiprefill` | vllm | context_attention_perf.parquet | l40s, rtx_pro_6000_server | 43,367 | | `vllm_flashinfer_flashinfertrtllmapidecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 268,320 | -| `vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 264,396 | -| `vllm_flashinfer_flashinfertrtllmapidecode_xqa` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | rtx_pro_6000_server | 31,021 | | `vllm_flashinfer_trtllm_moe_fp4` | vllm | moe_perf.parquet | b200_sxm | 10,287 | | `vllm_flashinfer_trtllmdecode` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 254,018 | -| `vllm_flashinfer_trtllmprefill` | vllm | context_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 556,588 | -| `vllm_fp8moe_deepgemm_deepgemmexperts` | vllm | moe_perf.parquet | rtx_pro_6000_server | 16,821 | -| `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 60,426 | -| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 50,220 | -| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 152,118 | -| `vllm_fp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 93,265 | +| `vllm_flashinfer_trtllmprefill` | vllm | context_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 369,948 | +| `vllm_fp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 39,042 | +| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 32,724 | +| `vllm_fp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 99,792 | +| `vllm_fp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | h100_sxm, h200_sxm, l40s | 40,222 | | `vllm_fused_moe` | vllm | moe_perf.parquet | a100_sxm, b200_sxm, h200_sxm | 126,055 | -| `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 23,328 | -| `vllm_gptossmxfp4moe_marlin_marlinexperts` | vllm | moe_perf.parquet | l40s, rtx_pro_6000_server | 7,776 | -| `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 11,664 | +| `vllm_gptossmxfp4moe_flashinfer_trtllm_mxfp4_bf16_trtllmmxfp4expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 15,552 | +| `vllm_gptossmxfp4moe_marlin_marlinexperts` | vllm | moe_perf.parquet | l40s, rtx_pro_6000_server | 3,888 | +| `vllm_gptossmxfp4moe_triton_oaitritonmxfp4expertsmonolithic` | vllm | moe_perf.parquet | h100_sxm, h200_sxm | 7,776 | | `vllm_marlin_int4_moe` | vllm | moe_perf.parquet | b200_sxm, h200_sxm | 20,415 | -| `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 140,454 | -| `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 145,169 | -| `vllm_modeloptfp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | l40s | 30,345 | -| `vllm_modeloptnvfp4fusedmoe_flashinfer_trtllm_trtllmnvfp4expertsmonolithic` | vllm | moe_perf.parquet | b300_sxm, gb200, gb300 | 2,673 | +| `vllm_modeloptfp8moe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 84,078 | +| `vllm_modeloptfp8moe_flashinfer_trtllm_trtllmfp8expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 95,937 | +| `vllm_modeloptfp8moe_triton_tritonexperts` | vllm | moe_perf.parquet | l40s | 14,174 | | `vllm_mxfp4_moe` | vllm | moe_perf.parquet | b200_sxm, h200_sxm | 2,839 | -| `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 26,244 | -| `vllm_torch_bmm` | vllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 13,156 | -| `vllm_triton_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 78,559 | -| `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 54,702 | +| `vllm_mxfp4moe_flashinfer_trtllm_mxfp4_mxfp8_trtllmmxfp4expertsmodular` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 17,496 | +| `vllm_torch_bmm` | vllm | mla_bmm_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm | 8,068 | +| `vllm_triton_attn` | vllm | context_attention_perf.parquet, generation_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, l40s, rtx_pro_6000_server | 42,460 | +| `vllm_unquantizedfusedmoe_flashinfer_cutlass_flashinferexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, rtx_pro_6000_server | 30,807 | | `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16experts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 103,173 | -| `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 64,276 | -| `vllm_unquantizedfusedmoe_triton_tritonexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 149,165 | -| `vllm_vit_flash_attn_fa2` | vllm | encoder_attention_perf.parquet | l40s, rtx_pro_6000_server | 30,716 | -| `vllm_vit_flash_attn_fa3` | vllm | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 47,174 | -| `vllm_vit_flash_attn_fa4` | vllm | encoder_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 92,148 | -| `vllm_xpu_moe` | vllm | moe_perf.parquet | b60 | 3,996 | +| `vllm_unquantizedfusedmoe_flashinfer_trtllm_trtllmbf16expertsmonolithic` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 4,158 | +| `vllm_unquantizedfusedmoe_triton_tritonexperts` | vllm | moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s | 89,413 | +| `vllm_vit_flash_attn_fa2` | vllm | encoder_attention_perf.parquet | l40s, rtx_pro_6000_server | 15,358 | +| `vllm_vit_flash_attn_fa3` | vllm | encoder_attention_perf.parquet | h100_sxm, h200_sxm | 30,716 | +| `vllm_vit_flash_attn_fa4` | vllm | encoder_attention_perf.parquet | b200_sxm, b300_sxm, gb200, gb300 | 61,432 | +| `vllm_xpu_moe` | vllm | moe_perf.parquet | b60 | 1,836 | | `vllm_xpu_moe_mxfp4` | vllm | moe_perf.parquet | b60 | 2,592 | | `wideep_compute_cutlass` | trtllm | wideep_moe_perf.parquet | b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, rtx_pro_6000_server | 26,298 | @@ -1819,4 +1566,4 @@ Each row is one distinct `kernel_source` value seen in the corpus, with the unio | kernel_source | frameworks | op files | systems | rows | |---|---|---|---|---| -| `default` | sglang, trtllm | context_mla_perf.parquet, dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet, generation_mla_perf.parquet, mla_bmm_perf.parquet, mla_context_module_perf.parquet, mla_generation_module_perf.parquet, msa_context_module_perf.parquet, msa_generation_module_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 598,936 | +| `default` | sglang, trtllm | context_mla_perf.parquet, dsa_context_module_perf.parquet, dsa_generation_module_perf.parquet, generation_mla_perf.parquet, mla_bmm_perf.parquet, mla_context_module_perf.parquet, mla_generation_module_perf.parquet, msa_context_module_perf.parquet, msa_generation_module_perf.parquet | a100_sxm, b200_sxm, b300_sxm, gb200, gb300, h100_sxm, h200_sxm, l40s, rtx_pro_6000_server | 306,281 | From 5a99ffad9ccbda07f6750b3ed5f672b3b45ca8f4 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 19:22:56 +0800 Subject: [PATCH 06/13] fix: the failures only CI's unfiltered test run sees `pytest -m unit` (used for local runs so far) deselects the unmarked generator tests; CI runs tests/unit and tests/golden unfiltered. - extra_engine_args.1.3.0rc29.yaml.j2 was copied from rc23 before #626 and missed its rule: cache_transceiver_config renders when a backend is set, max_tokens_in_buffer only when given (test_sweeper_request, rc29 trace candidate). - The Dynamo 1.2.0 Slurm goldens follow the 64-aligned token budgets, as the release-1.2.0 contract already does: vLLM 0.20.1 2140 -> 2176, SGLang 0.5.11 1628 -> 1664, TRT-LLM 1.3.0rc14 decode max_num_tokens 32 -> 64. - test_collect_mla_module_native_quant's fake helper lacked PhaseClock and render_phase_totals, which sglang/collect_mla_module.py now imports; it passed only when another test had loaded the module first. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .../trtllm/extra_engine_args.1.3.0rc29.yaml.j2 | 4 +++- .../tests/golden/generator/slurm/sglang/0.5.11/agg.yaml | 2 +- .../tests/golden/generator/slurm/sglang/0.5.11/disagg.yaml | 4 ++-- .../tests/golden/generator/slurm/trtllm/1.3.0rc14/disagg.yaml | 4 ++-- .../tests/golden/generator/slurm/vllm/0.20.1/agg.yaml | 2 +- .../tests/golden/generator/slurm/vllm/0.20.1/disagg.yaml | 2 +- .../collector/sglang/test_collect_mla_module_native_quant.py | 4 ++++ 7 files changed, 14 insertions(+), 8 deletions(-) diff --git a/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 b/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 index 16ab331702..31e78186df 100644 --- a/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 +++ b/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 @@ -50,11 +50,13 @@ kv_cache_config: {% endif %} enable_block_reuse: {{ kv_cache_config.enable_block_reuse | default(false) }} -{% if cache_transceiver_config.max_tokens_in_buffer is defined %} +{% if cache_transceiver_config.max_tokens_in_buffer is defined or cache_transceiver_config.backend is defined %} cache_transceiver_config: backend: {{ cache_transceiver_config.backend | default('DEFAULT') }} # The communication backend type to use for the cache transceiver ("DEFAULT","UCX","NIXL","MPI") +{% if cache_transceiver_config.max_tokens_in_buffer is defined %} max_tokens_in_buffer: {{ cache_transceiver_config.max_tokens_in_buffer }} {% endif %} +{% endif %} cuda_graph_config: enable_padding: {{ cuda_graph_config.enable_padding | default(false) }} # Pad CUDA graph diff --git a/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/agg.yaml b/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/agg.yaml index 94126be193..de4bf7ad99 100644 --- a/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/agg.yaml +++ b/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/agg.yaml @@ -35,7 +35,7 @@ deployment.json: |- "--max-running-requests", "512", "--max-prefill-tokens", - "1628", + "1664", "--enable-mixed-chunk", "--cuda-graph-bs", "1", diff --git a/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/disagg.yaml b/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/disagg.yaml index d5e774b21b..17e5ee5747 100644 --- a/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/disagg.yaml +++ b/python/aisimulate/tests/golden/generator/slurm/sglang/0.5.11/disagg.yaml @@ -35,7 +35,7 @@ deployment.json: |- "--max-running-requests", "8", "--max-prefill-tokens", - "1628", + "1664", "--disaggregation-transfer-backend", "nixl", "--disaggregation-mode", @@ -72,7 +72,7 @@ deployment.json: |- "--max-running-requests", "512", "--max-prefill-tokens", - "1628", + "1664", "--disaggregation-transfer-backend", "nixl", "--cuda-graph-bs", diff --git a/python/aisimulate/tests/golden/generator/slurm/trtllm/1.3.0rc14/disagg.yaml b/python/aisimulate/tests/golden/generator/slurm/trtllm/1.3.0rc14/disagg.yaml index 03ac01d2c4..21973c615f 100644 --- a/python/aisimulate/tests/golden/generator/slurm/trtllm/1.3.0rc14/disagg.yaml +++ b/python/aisimulate/tests/golden/generator/slurm/trtllm/1.3.0rc14/disagg.yaml @@ -65,7 +65,7 @@ deployment.json: |- "--max-batch-size", "8", "--max-num-tokens", - "32", + "64", "--extra-engine-args", "decode_config.yaml", "--gpus-per-node", @@ -129,7 +129,7 @@ decode_config.yaml: | enable_attention_dp: false enable_chunked_prefill: false max_batch_size: 8 - max_num_tokens: 32 + max_num_tokens: 64 kv_cache_config: free_gpu_memory_fraction: 0.8 dtype: auto diff --git a/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/agg.yaml b/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/agg.yaml index 58e9faf3f0..689c37d6cc 100644 --- a/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/agg.yaml +++ b/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/agg.yaml @@ -37,7 +37,7 @@ deployment.json: |- "--max-num-seqs", "512", "--max-num-batched-tokens", - "2140" + "2176" ], "env": { "VLLM_DP_SIZE": "1", diff --git a/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/disagg.yaml b/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/disagg.yaml index 57aaf7a702..f4d1f9cb95 100644 --- a/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/disagg.yaml +++ b/python/aisimulate/tests/golden/generator/slurm/vllm/0.20.1/disagg.yaml @@ -35,7 +35,7 @@ deployment.json: |- "--max-num-seqs", "8", "--max-num-batched-tokens", - "1628", + "1664", "--disaggregation-mode", "prefill", "--kv-transfer-config", diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py index 62b3911543..e2c189b2fe 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py @@ -15,6 +15,7 @@ from unittest.mock import patch import pytest +from collector.helper import PhaseClock, render_phase_totals pytestmark = pytest.mark.unit @@ -43,6 +44,9 @@ def _mock_helper_imports(monkeypatch): fake_helper.get_sm_version = lambda: 90 # default Hopper fake_helper.log_perf = lambda **kw: None fake_helper.benchmark_with_power = lambda **kw: None + # the collector imports the phase-timing helpers at module load + fake_helper.PhaseClock = PhaseClock + fake_helper.render_phase_totals = render_phase_totals monkeypatch.setitem(sys.modules, "helper", fake_helper) fake_torch = types.ModuleType("torch") From 4527380ca1eb9287bd6088aafe4ea09736580762 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 19:23:26 +0800 Subject: [PATCH 07/13] test: drop the collector tests that only restate code; close two checks Removes about 50 test functions (~1,000 lines) that would not catch a wrong row or a wrong kernel: duplicates of the one-release-per-file check (per-file __compat__ / admitted-release tests), tombstone tests for retired ops, log/print-format pins no tool parses, positive controls, exact code-string and docstring needles, and the test of dead code. Same-branch parametrizations are trimmed and near-duplicates merged. - collector/vllm/utils.kv_block_size had no callers (superseded by framework_kv_block_size); it goes with its test. Gap left open: framework_kv_block_size itself has no unit test. - The strict pin test names its one exception, collect_dsv41_module's pre-existing source-build pin from #160 (never declared in the manifest), instead of accepting every `@` pin. - The no-eager-fallback test flags any allow_graph_fail other than literal False (a wrapper forwarding its own keyword-only False default is allowed), replacing five per-site call-count pins. - The mHC audited-release overlap guard also checks the live 0.5.21. - test_vllm_attn_kv_cache_layout needs real torch: registered as an optional collection skip in .github/application-test-inventory.json with the same reason text as the other three torch-only collector tests. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .github/application-test-inventory.json | 3 +- python/aisimulate/collector/vllm/utils.py | 26 +- .../test_collect_attn_encoder_sm_dispatch.py | 2 +- .../test_collect_attn_flashinfer_workspace.py | 26 +- .../sglang/test_collect_kda_contract.py | 31 --- .../sglang/test_collect_mla_graph_capture.py | 38 +-- .../sglang/test_collect_moe_population.py | 4 - .../test_collect_moe_unverified_lane_guard.py | 12 +- .../sglang/test_moe_config_observation.py | 3 - .../test_collect_ops_not_in_plan_exit_code.py | 6 - .../unit/collector/test_collect_replay.py | 18 +- .../collector/test_collect_trtllm_alltoall.py | 15 +- .../test_dsv4_retired_sidecar_ops_removed.py | 48 ---- .../collector/test_error_classification.py | 24 -- .../unit/collector/test_framework_manifest.py | 42 +-- .../test_glm5next_collection_plan.py | 42 --- .../collector/test_glm5next_runtime_pins.py | 12 - .../test_helper_flashinfer_autotune.py | 241 +----------------- .../unit/collector/test_helper_phase_clock.py | 47 +--- .../unit/collector/test_kv_block_size.py | 48 ---- .../collector/test_mhc_module_glm5next.py | 16 +- .../tests/unit/collector/test_model_cases.py | 8 +- .../collector/test_msa_getter_contract.py | 6 +- .../collector/test_msa_kernel_limit_scope.py | 2 +- .../test_no_eager_graph_fallback_sites.py | 58 ++--- .../test_perf_output_finalization.py | 30 +-- .../test_registry_table_producers.py | 28 +- .../collector/test_sglang_glm5_sparse_0514.py | 6 +- .../test_vllm_attn_kv_cache_layout.py | 34 +-- .../unit/collector/test_vllm_collect_attn.py | 7 +- .../collector/test_vllm_mla_graph_timing.py | 31 +-- .../trtllm/test_collect_dsv4_attn.py | 6 +- .../trtllm/test_collect_gdn_fail_closed.py | 32 +-- .../trtllm/test_collect_mamba2_fail_closed.py | 34 +-- .../trtllm/test_collect_module_layer_cache.py | 19 -- .../collector/trtllm/test_collect_moe_ep.py | 7 +- .../trtllm/test_collect_moe_worker_recycle.py | 65 +---- .../vllm/test_collect_kda_contract.py | 33 --- .../vllm/test_rope_missing_key_shim.py | 35 +-- .../generator/test_facts_hardware_l40s.py | 3 - .../generator/test_facts_hardware_sm120.py | 14 +- .../test_model_profile_deepseek_v41.py | 2 - .../test_vllm_kda_decode_batch_cap.py | 7 - .../test_vllm_nvfp4_dsa_kv_cache_dtype.py | 21 -- 44 files changed, 148 insertions(+), 1044 deletions(-) delete mode 100644 python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py delete mode 100644 python/aisimulate/tests/unit/collector/test_kv_block_size.py diff --git a/.github/application-test-inventory.json b/.github/application-test-inventory.json index 00807374c2..e6812f8846 100644 --- a/.github/application-test-inventory.json +++ b/.github/application-test-inventory.json @@ -9,6 +9,7 @@ "optional_collection_skips": { "tests/unit/collector/test_dsv4_megamoe_workload.py": "real torch required for tensor operations", "tests/unit/collector/test_helper_moe_distribution.py": "real torch required for tensor operations", - "tests/unit/collector/test_sglang_moe_ep_routing.py": "real torch required for tensor operations" + "tests/unit/collector/test_sglang_moe_ep_routing.py": "real torch required for tensor operations", + "tests/unit/collector/test_vllm_attn_kv_cache_layout.py": "real torch required for tensor operations" } } diff --git a/python/aisimulate/collector/vllm/utils.py b/python/aisimulate/collector/vllm/utils.py index 96e2c18ff7..4f62b6e478 100644 --- a/python/aisimulate/collector/vllm/utils.py +++ b/python/aisimulate/collector/vllm/utils.py @@ -664,7 +664,7 @@ def framework_kv_block_size(backend_cls, *, user_block_size: int | None = None) update_block_size_for_backend, phase 1), and the kernel page is ``select_common_block_size(cache_block_size, [backend])`` (v1/worker/utils.py: the manager size when every backend accepts it, else the largest explicit - size that divides it). The per-SM table (kv_block_size) was the owner's + size that divides it). The former per-SM table (removed) was the owner's 2026-09-30 interim; the B200 re-run on 2026-10-03 showed it is not equivalent: with mla=32 the bf16 DSA cell selected the TRT-LLM FMHA sparse cubin while serving ran FA-cute (page 64) — the page is a per-(backend, kv @@ -683,27 +683,3 @@ def framework_kv_block_size(backend_cls, *, user_block_size: int | None = None) user_block_size if user_block_size else backend_cls.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE) ) return int(select_common_block_size(int(cache_block), [backend_cls])) - - -def kv_block_size(sm_version: int, op: str = "attention") -> int: - """KV-cache page size the collector builds its fake engine with, per SM. - - Serving picks the page size per attention backend; SM90 backends (FA3, - FlashMLA, Triton) take 64 and the collectors used a 64 literal — which the - B200 campaign (2026-09-29) caught as a cross-arch defect: on SM100 vLLM - 0.30 serves dense attention through the flashinfer TRT-LLM FMHA at page - 16 (Llama/Qwen; Gemma-4 head-512 takes 64) and MLA decode at page 32, and - a 64-page collector cell ran a different cubin (attn_ctx P64 vs serving - P16; mla_gen_fp8 P64 vs P32). The A/B with page 16 aligned attn_ctx. - Owner decision 2026-09-30: branch by SM rather than query the framework - per case. SM120 keeps 64 (its 31 gates aligned with it). AIS_KV_BLOCK_SIZE - is the single-variable A/B hook and is never set in collection runs. - """ - import os - - override = os.environ.get("AIS_KV_BLOCK_SIZE") - if override: - return int(override) - if 100 <= sm_version < 120: - return 32 if op == "mla" else 16 - return 64 diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py index bc9620893c..df6d1eb3e5 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_encoder_sm_dispatch.py @@ -22,7 +22,7 @@ def _serving_vision_backend(): @pytest.mark.parametrize( "sm, expected", - [(89, "triton_attn"), (90, "fa3"), (100, "fa4"), (103, "fa4"), (120, "triton_attn"), (121, "triton_attn")], + [(90, "fa3"), (100, "fa4"), (103, "fa4"), (120, "triton_attn")], ) def test_vision_backend_follows_capability_major(sm, expected): assert _serving_vision_backend()(sm) == expected diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py index 8882c056ac..5acf02b7a3 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_attn_flashinfer_workspace.py @@ -42,30 +42,14 @@ def set(self, v): self.v = v -def test_0521_envs_knob_is_raised_to_the_floor(monkeypatch): +@pytest.mark.parametrize(("initial", "expected"), [(384 * 1024 * 1024, 4 << 30), (8 << 30, 8 << 30)]) +def test_0521_envs_knob_is_raised_to_the_floor_and_a_larger_setting_is_kept(monkeypatch, initial, expected): fn, floor = _load() - knob = _EnvInt(384 * 1024 * 1024) + assert floor == 4 << 30 + knob = _EnvInt(initial) monkeypatch.setitem(sys.modules, "sglang", types.ModuleType("sglang")) monkeypatch.setitem(sys.modules, "sglang.srt", types.ModuleType("sglang.srt")) environ = types.ModuleType("sglang.srt.environ") environ.envs = types.SimpleNamespace(SGLANG_FLASHINFER_WORKSPACE_SIZE=knob) monkeypatch.setitem(sys.modules, "sglang.srt.environ", environ) - assert fn() == floor and knob.get() == floor == 4 << 30 - - -def test_larger_setting_is_kept(monkeypatch): - fn, floor = _load() - knob = _EnvInt(8 << 30) - monkeypatch.setitem(sys.modules, "sglang", types.ModuleType("sglang")) - monkeypatch.setitem(sys.modules, "sglang.srt", types.ModuleType("sglang.srt")) - environ = types.ModuleType("sglang.srt.environ") - environ.envs = types.SimpleNamespace(SGLANG_FLASHINFER_WORKSPACE_SIZE=knob) - monkeypatch.setitem(sys.modules, "sglang.srt.environ", environ) - assert fn() == 8 << 30 and knob.get() == 8 << 30 - - -def test_no_knob_is_a_reported_noop(monkeypatch): - fn, _ = _load() - for name in ("sglang", "sglang.srt", "sglang.srt.environ", "sglang.global_config"): - monkeypatch.setitem(sys.modules, name, None) # import fails - assert fn() is None + assert fn() == expected and knob.get() == expected diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py index 70e0b8bc7b..26376a8405 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_kda_contract.py @@ -82,21 +82,6 @@ def test_glm5_next_decode_uses_bounded_gate_recurrence_not_packed_decode(): assert "fused_recurrent_kda_packed_decode" not in names -@pytest.mark.parametrize( - ("version", "accepted"), - [("0.5.16", False), ("0.5.17", False), ("0.5.19", False), ("0.5.20", False), ("0.5.21", True), ("0.5.22", False)], -) -def test_kda_compat_admits_kimi_branch_and_glm_runtime(version, accepted): - from collector.version_resolver import _check_compat - - declaration = next( - node.value.value - for node in ast.parse(SOURCE_PATH.read_text(encoding="utf-8")).body - if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "__compat__" for t in node.targets) - ) - assert _check_compat(declaration, version) is accepted - - def test_glm5_next_chunk_kda_int32_offset_guard_precedes_the_scan(): # chunk_kda's Triton kernels use int32 (bos * H + i_h) * K offsets # (fla/kda.py:265-268 @v0.5.20); an overflowing launch is an illegal @@ -134,22 +119,6 @@ def _gate_namespace(): return namespace -AUDITED_KDA_RELEASES = { - "KimiK3ForConditionalGeneration": ("0.5.21",), -} - - -def test_kda_audited_release_sets_do_not_overlap(): - from collector.version_resolver import _check_compat - - compat = _gate_namespace()["_KDA_ARCHITECTURE_COMPAT"] - assert set(compat) == set(AUDITED_KDA_RELEASES) - for arch, versions in AUDITED_KDA_RELEASES.items(): - for version in versions: - for other, spec in compat.items(): - assert _check_compat(spec, version) is (other == arch), (arch, other, version) - - @pytest.mark.parametrize( ("model_name", "version", "audited"), [ diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py index b1673b0c16..a68041f4cf 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_graph_capture.py @@ -47,22 +47,14 @@ def _fake_torch(): ) -def _install_fake_flashinfer(monkeypatch, *, accepts_cpu_lens): +def _install_fake_flashinfer(monkeypatch): calls = [] - if accepts_cpu_lens: - - def trtllm_ragged_attention_deepseek( - *, query, cum_seq_lens_q, cum_seq_lens_kv, q_seq_lens_cpu=None, kv_seq_lens_cpu=None - ): - calls.append({"q_seq_lens_cpu": q_seq_lens_cpu, "kv_seq_lens_cpu": kv_seq_lens_cpu}) - return "out" - - else: - - def trtllm_ragged_attention_deepseek(*, query, cum_seq_lens_q, cum_seq_lens_kv): - calls.append({}) - return "out" + def trtllm_ragged_attention_deepseek( + *, query, cum_seq_lens_q, cum_seq_lens_kv, q_seq_lens_cpu=None, kv_seq_lens_cpu=None + ): + calls.append({"q_seq_lens_cpu": q_seq_lens_cpu, "kv_seq_lens_cpu": kv_seq_lens_cpu}) + return "out" prefill = SimpleNamespace(trtllm_ragged_attention_deepseek=trtllm_ragged_attention_deepseek) monkeypatch.setitem(sys.modules, "flashinfer", SimpleNamespace(prefill=prefill)) @@ -108,7 +100,7 @@ def _extend_batch(lens): def test_no_prefix_prefill_call_carries_the_cpu_length_mirrors(monkeypatch): - prefill, calls, original = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=True) + prefill, calls, original = _install_fake_flashinfer(monkeypatch) backend = _backend() backend.init_forward_metadata(_extend_batch([512, 512])) cum = object() @@ -125,7 +117,7 @@ def test_no_prefix_prefill_call_carries_the_cpu_length_mirrors(monkeypatch): def test_chunked_prefix_call_goes_through_sglang_unchanged(monkeypatch): - _, calls, _ = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=True) + _, calls, _ = _install_fake_flashinfer(monkeypatch) backend = _backend() backend.init_forward_metadata(_extend_batch([512])) @@ -135,7 +127,7 @@ def test_chunked_prefix_call_goes_through_sglang_unchanged(monkeypatch): def test_decode_metadata_leaves_the_prefill_hook_inert(monkeypatch): - _, calls, _ = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=True) + _, calls, _ = _install_fake_flashinfer(monkeypatch) backend = _backend() backend.init_forward_metadata(SimpleNamespace(forward_mode=FORWARD_MODE.DECODE, seq_lens_cpu=[9])) cum = object() @@ -145,17 +137,6 @@ def test_decode_metadata_leaves_the_prefill_hook_inert(monkeypatch): assert calls == [{"q_seq_lens_cpu": None, "kv_seq_lens_cpu": None}] -def test_flashinfer_without_the_parameters_is_called_as_is(monkeypatch): - prefill, calls, original = _install_fake_flashinfer(monkeypatch, accepts_cpu_lens=False) - backend = _backend() - backend.init_forward_metadata(_extend_batch([512])) - cum = object() - - assert backend._run_prefill_kernel(q="q", cum_seq_lens_q=cum, cum_seq_lens_kv=cum) == "out" - assert calls == [{}] - assert prefill.trtllm_ragged_attention_deepseek is original - - def test_benchmark_layer_refuses_eager_timing(): benchmark_kwargs = {} @@ -184,7 +165,6 @@ def benchmark_with_power(**kwargs): def test_run_mla_has_no_eager_prefill_switch(): source = SOURCE.read_text(encoding="utf-8") - assert "_eager_prefill" not in source assert "use_cuda_graph=False" not in source run_mla = next(node for node in _tree().body if isinstance(node, ast.FunctionDef) and node.name == "run_mla") constructed = { diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py index fe5c2cc1b8..9ca556fc29 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_population.py @@ -786,10 +786,6 @@ def test_framework_moe_source_comes_from_the_constructed_quant_method(): "ModelOptNvFp4FusedMoEMethod", ): assert method_name in framework_bench - assert ( - 'on_result(num_tokens, results["latency_ms"] / len(logits), results["power_stats"], kernel_source)' - in framework_bench - ) assert 'model_name == "deepseek-ai/DeepSeek-V4-Pro"' not in run_moe diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py index 26950d79ec..f6da27effe 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_moe_unverified_lane_guard.py @@ -76,21 +76,19 @@ def test_guard_never_fires_for_qwen38max_collected_lanes(moe_type, installed_ver assert guard(moe_type) == installed_version -@pytest.mark.parametrize("installed_version", ["0.5.21", "0.5.21.post1", "0.5.21+cu130"]) -@pytest.mark.parametrize("sm_version", [89, 90]) +@pytest.mark.parametrize("installed_version", ["0.5.21", "0.5.21+cu130"]) @pytest.mark.parametrize("moe_type", ["int4_wo", "w4a16_mxfp4"]) -def test_guard_accepts_0521_hopper_lanes_reverified_2026_10_04(moe_type, installed_version, sm_version): +def test_guard_accepts_0521_hopper_lanes_reverified_2026_10_04(moe_type, installed_version): """0.5.21 Marlin (int4_wo) and Triton (w4a16_mxfp4) dispatch re-verified on SM89/90 only.""" - guard = _load_guard(installed_version, sm_version) + guard = _load_guard(installed_version, 90) assert guard(moe_type) == installed_version @pytest.mark.parametrize("moe_type", ["int4_wo", "w4a16_mxfp4", "w4a8_mxfp4_mxfp8"]) -@pytest.mark.parametrize("sm_version", [100, 103]) -def test_guard_accepts_0521_blackwell_mxfp4_lanes(moe_type, sm_version): +def test_guard_accepts_0521_blackwell_mxfp4_lanes(moe_type): """SM100/103 MXFP4 lanes re-verified on 0.5.21 from the B200 identity records (6400345f); int4_wo re-opened 2026-10-06 from the same records (Kimi-K2.5 -> FLASHINFER_TRTLLM / trtllm_gen_moe).""" - guard = _load_guard("0.5.21", sm_version) + guard = _load_guard("0.5.21", 100) assert guard(moe_type) == "0.5.21" diff --git a/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py b/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py index 19b03ff100..0cfbcad28b 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_moe_config_observation.py @@ -86,7 +86,4 @@ def test_sglang_collector_wraps_the_timed_call_and_writes_triton_rows_only(): source = (Path(__file__).resolve().parents[4] / "collector/sglang/collect_moe.py").read_text() assert "with observe_triton_config_selection(_triton_config_namespaces())" in source assert "return [sys.modules[get_config_dtype_str.__module__], globals()]" in source - # one task times a num_tokens series: each point's row gets its own counters assert re.search(r'if moe_backend == "triton":\n\s+append_triton_config_record\(', source) - assert "point_observed = dict(triton_config_observed)" in source - assert "triton_config_observed[key] = 0" in source diff --git a/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py b/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py index 5e00ce3506..d0c3ef88f1 100644 --- a/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py +++ b/python/aisimulate/tests/unit/collector/test_collect_ops_not_in_plan_exit_code.py @@ -44,9 +44,3 @@ def test_op_outside_the_backend_plan_exits_with_the_dedicated_code(): assert result.returncode == EXIT_CODE_OPS_NOT_IN_PLAN, result.stderr[-2000:] assert EXIT_CODE_OPS_NOT_IN_PLAN not in (0, 1, 2) assert "Requested ops are not present in the collector v2 case plan: mamba2" in result.stderr - - -def test_planned_op_still_passes_the_ops_check(): - result = _run_collect("--backend", "vllm", "--model-cases-full", "--plan-only", "--ops", "gemm") - - assert result.returncode == 0, result.stderr[-2000:] diff --git a/python/aisimulate/tests/unit/collector/test_collect_replay.py b/python/aisimulate/tests/unit/collector/test_collect_replay.py index bf36467df3..0aca3340da 100644 --- a/python/aisimulate/tests/unit/collector/test_collect_replay.py +++ b/python/aisimulate/tests/unit/collector/test_collect_replay.py @@ -59,16 +59,6 @@ def test_control_sample_is_seeded_and_excludes_requested(): assert len(big.select("op", CASES, random.Random(1))) == len(CASES) # capped at the rest -def test_report_lists_unmatched_requests_across_ops(): - spec = _spec({"[32, 255, 128, 1, 128, False, 'flashinfer']", "[9, 9, 9]"}) - spec.select("attention_generation", CASES, random.Random(42)) - spec.select("gemm", [], random.Random(42)) - rep = spec.report() - assert rep["requested"] == 2 and rep["matched"] == 1 - assert rep["unmatched"] == ["[9, 9, 9]"] - assert rep["matched_by_op"] == {"attention_generation": 1, "gemm": 0} - - def test_load_replay_cases_from_errors_json_summary_gz_and_plain_list(tmp_path: Path): errors = [ {"task_params": "[32, 255, 128, 1, 128, False, 'flashinfer']", "error_type": "RuntimeError"}, @@ -96,15 +86,15 @@ def test_load_replay_cases_from_errors_json_summary_gz_and_plain_list(tmp_path: collect.load_replay_cases(tmp_path / "empty.json") -def test_finish_replay_writes_marker_and_warns_on_unmatched(tmp_path: Path, caplog): +def test_finish_replay_writes_a_marker_listing_unmatched_requests_across_ops(tmp_path: Path): spec = _spec({"[32, 255, 128, 1, 128, False, 'flashinfer']", "[9, 9, 9]"}) spec.select("attention_generation", CASES, random.Random(42)) + spec.select("gemm", [], random.Random(42)) + assert spec.report()["matched_by_op"] == {"attention_generation": 1, "gemm": 0} import logging log = logging.getLogger("replay-test") - with caplog.at_level(logging.INFO, logger="replay-test"): - collect._finish_replay(spec, tmp_path, log) + collect._finish_replay(spec, tmp_path, log) marker = json.loads((tmp_path / collect.ReplaySpec.MARKER).read_text()) assert marker["requested"] == 2 and marker["matched"] == 1 and marker["unmatched"] == ["[9, 9, 9]"] - assert any("not in this run's case plan" in r.message for r in caplog.records) assert collect._finish_replay(None, tmp_path, log) is None diff --git a/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py b/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py index 5f6ecbf7c5..65b565065c 100644 --- a/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py +++ b/python/aisimulate/tests/unit/collector/test_collect_trtllm_alltoall.py @@ -541,20 +541,13 @@ def test_no_fabricated_node_num_derivation_remains(): # --------------------------------------------------------------------------- -def test_benchmark_op_agrees_capture_before_timing_and_never_falls_back(): +def test_benchmark_op_agrees_capture_before_timing(): import ast tree = ast.parse(SOURCE_TEXT) function = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_benchmark_op") callees = [ast.unparse(node.func) for node in ast.walk(function) if isinstance(node, ast.Call)] assert callees.index("_agree_graph_capturable") < callees.index("benchmark_with_power") - bench = next( - node - for node in ast.walk(function) - if isinstance(node, ast.Call) and ast.unparse(node.func) == "benchmark_with_power" - ) - assert all(keyword.arg != "allow_graph_fail" for keyword in bench.keywords) - assert "allow_graph_fail=True" not in SOURCE_TEXT class _Flag: @@ -608,12 +601,6 @@ def test_capture_probe_warms_eagerly_then_captures_and_agrees(monkeypatch): assert events == ["func", "sync", "capture", "func", "sync", "all_reduce"] -def test_capture_probe_skips_the_collective_in_a_single_rank_world(monkeypatch): - events = _fake_runtime(monkeypatch, local_capture_fails=False, peer_fails=False, world_size=1) - ata._agree_graph_capturable(lambda: None, "dispatch", "cuda:0") - assert "all_reduce" not in events - - def test_local_capture_failure_is_agreed_then_raises_classified(monkeypatch): events = _fake_runtime(monkeypatch, local_capture_fails=True, peer_fails=False, world_size=8) with pytest.raises(ata.TrtllmAlltoallBenchmarkError, match=r"CUDA graph capture of combine failed \(RuntimeError"): diff --git a/python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py b/python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py deleted file mode 100644 index f3f6617e7f..0000000000 --- a/python/aisimulate/tests/unit/collector/test_dsv4_retired_sidecar_ops_removed.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""The DeepSeek-V4 FMLA sidecar ops ``dsv4_hca_attn_module`` / ``dsv4_csa_attn_module`` -were removed end to end on 2026-10-15: collector registries and case plan, ``PerfFile``, -the SDK ``PerfDataFilename``, the op catalog, the capability floors and the shipped -tables. Their SDK readers had been retired placeholders with no query path (csa_attn -never shipped data), and as registry-only ops every sm100 shard plan generated from -the registry since 2026-10-05 carried them and failed one or two shards per cell. -This guard keeps them out of every layer a shard plan or the SDK reads.""" - -from __future__ import annotations - -import importlib -from pathlib import Path - -import pytest -from collector.capabilities import _load_capabilities -from collector.op_catalog import load_family_map -from collector.registry_types import PerfFile - -pytestmark = pytest.mark.unit - -RETIRED_OPS = frozenset({"dsv4_hca_attn_module", "dsv4_csa_attn_module"}) -PKG_ROOT = Path(__file__).resolve().parents[3] - - -@pytest.mark.parametrize("backend", ("vllm", "sglang", "trtllm")) -def test_retired_sidecar_ops_are_not_registered(backend): - registered = {entry.op for entry in importlib.import_module(f"collector.{backend}.registry").REGISTRY} - assert not registered & RETIRED_OPS - - -def test_retired_sidecar_tables_have_no_perf_file_catalog_entry_or_floor(): - assert not {member.name for member in PerfFile} & {"DSV4_HCA_ATTN_MODULE", "DSV4_CSA_ATTN_MODULE"} - family_map = load_family_map() - assert family_map is not None - assert not {f"{op}_perf" for op in RETIRED_OPS} & set(family_map) - _, op_min_sm = _load_capabilities() - assert not RETIRED_OPS & set(op_min_sm) - - -def test_retired_sidecar_tables_have_no_sdk_filename_or_shipped_data(): - from aisimulate_core.sdk.common import PerfDataFilename - - assert not {member.name for member in PerfDataFilename} & RETIRED_OPS - data_root = PKG_ROOT / "src" / "aisimulate_core" / "systems" / "data" - shipped = sorted(str(p.relative_to(data_root)) for op in RETIRED_OPS for p in data_root.rglob(f"{op}_perf.parquet")) - assert shipped == [] diff --git a/python/aisimulate/tests/unit/collector/test_error_classification.py b/python/aisimulate/tests/unit/collector/test_error_classification.py index ba4c4ae91a..b7a92080b8 100644 --- a/python/aisimulate/tests/unit/collector/test_error_classification.py +++ b/python/aisimulate/tests/unit/collector/test_error_classification.py @@ -29,27 +29,3 @@ def test_kernel_limit_prefix_is_classified(): assert collect._classify_exception(ValueError(" FIXME(kernel-limit): divisibility")) == "kernel_limit" assert collect._classify_exception(RuntimeError("CUDA error: an illegal memory access")) == "unexpected" assert collect._classify_exception(AssertionError()) == "unexpected" - - -@pytest.mark.parametrize( - "path, needle", - [ - ("collector/sglang/collect_moe.py", "FIXME(kernel-limit): SGLang Triton gated BF16 GELU"), - ("collector/trtllm/collect_moe.py", "FIXME(kernel-limit): TRT-LLM fused MoE requires"), - ("collector/trtllm/collect_attn.py", "FIXME(kernel-limit): TRT-LLM trtllm-gen FMHA has no head_dim=192"), - ("collector/vllm/collect_gemm.py", "FIXME(kernel-limit): vLLM fp8_block GEMM"), - # 2026-10-11 fix list: bare errors that were deterministic kernel limits - ("collector/trtllm/collect_gemm.py", "FIXME(kernel-limit): TRT-LLM fp8 GEMM (cublas/cutlass scaled_mm)"), - ("collector/sglang/collect_moe.py", "FIXME(kernel-limit): SGLang Triton gated BF16 GELU/SiLU"), - ("collector/sglang/collect_attn.py", "FIXME(kernel-limit): SGLang flashinfer has no FP8 prefill compute path"), - ( - "collector/sglang/collect_attn.py", - "FIXME(kernel-limit): SGLang {attn_backend_name} quantizes Q to FP8 internally", - ), - ], -) -def test_deterministic_guards_carry_the_prefix(path, needle): - from pathlib import Path - - root = Path(__file__).resolve().parents[3] - assert needle in (root / path).read_text(encoding="utf-8") diff --git a/python/aisimulate/tests/unit/collector/test_framework_manifest.py b/python/aisimulate/tests/unit/collector/test_framework_manifest.py index 8e824d569a..8b6d5a28a1 100644 --- a/python/aisimulate/tests/unit/collector/test_framework_manifest.py +++ b/python/aisimulate/tests/unit/collector/test_framework_manifest.py @@ -15,7 +15,6 @@ ) from collector.sglang.registry import REGISTRY as SGLANG_REGISTRY from collector.trtllm.registry import REGISTRY as TRTLLM_REGISTRY -from collector.version_resolver import _check_compat from collector.vllm.registry import REGISTRY as VLLM_REGISTRY from collector.vllm.registry import REGISTRY_XPU as VLLM_XPU_REGISTRY from collector.wideep.sglang import dataset_version_label @@ -40,9 +39,6 @@ def test_manifest_exposes_current_framework_versions_and_images(): assert trtllm.image().startswith("nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc29@sha256:") assert vllm.version == "0.30.0" assert vllm.image().startswith("vllm/vllm-openai:v0.30.0@sha256:") - # cu129 variant intentionally dropped at the 0.29.0 bump (no cu129 build); - # unknown variants fall back to the pinned default image below. - assert vllm.image("cu129").startswith("vllm/vllm-openai:v0.30.0@sha256:") # Unknown variants intentionally fall back to the pinned default image. assert vllm.image("cu130") == vllm.image() @@ -75,6 +71,15 @@ def test_active_cuda_vllm_collectors_are_exactly_pinned_to_manifest_version(): assert declarations[0] == f'__compat__ = "vllm=={resolved_version}"', (module, declarations[0]) +# Known exception, not a pattern: collect_dsv41_module carries the pre-existing +# source-build pin from #160 (sglang@1aa0e962...). framework_manifest.yaml never +# declares that build, so the module cannot run on any manifest-resolved runtime; +# fixing the pin is out of scope here. Any other `@` pin fails the test below. +_SOURCE_BUILD_PIN_EXCEPTIONS = { + "collector.sglang.collect_dsv41_module": '__compat__ = "sglang@1aa0e962b206102b7c439a4a0c4981cfec6e87bc"', +} + + @pytest.mark.parametrize("backend", ["sglang", "trtllm"]) def test_stock_collectors_are_exactly_pinned_to_their_resolved_runtime(backend): # One release per file (owner policy 2026-09-20): every registry module @@ -90,9 +95,8 @@ def test_stock_collectors_are_exactly_pinned_to_their_resolved_runtime(backend): declarations = [line.strip() for line in source.splitlines() if line.startswith("__compat__")] version = next(iter(versions)) allowed = {f'__compat__ = "{backend}=={version}"'} - # a module pinned to an exact source commit is an exact pin too - if declarations and "@" in declarations[0]: - allowed = {declarations[0]} + if module in _SOURCE_BUILD_PIN_EXCEPTIONS: + allowed = {_SOURCE_BUILD_PIN_EXCEPTIONS[module]} assert len(declarations) == 1 and declarations[0] in allowed, (module, declarations) @@ -735,30 +739,6 @@ def test_model_pin_images_must_be_digest_pinned(tmp_path): get_collector_runtime("sglang", path=manifest) -@pytest.mark.parametrize( - "version,accepted", - [ - ("0.24.0", False), - ("0.25.0", False), - ("0.27.1", False), - ("0.28.0", False), - ("0.30.0", False), # 0.30.0 gemm collector -> git history - ("0.31.0", True), # gemm family pin (2026-10-30) - ("0.31.1", False), - ], -) -def test_gemm_admits_only_its_family_release(version, accepted): - import ast - - source = ast.parse((COLLECTOR_ROOT / "vllm" / "collect_gemm.py").read_text()) - declaration = next( - node.value.value - for node in source.body - if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == "__compat__" for t in node.targets) - ) - assert _check_compat(declaration, version) is accepted - - @pytest.fixture def explicit_vllm_025_manifest(tmp_path): """Synthetic loader fixture, not a shipped image or collection declaration.""" diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py index 0e9c86a013..04d92ee23c 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py @@ -3,7 +3,6 @@ """GLM-5.3-Flash standalone per-op collection: case plan, routing and version gates.""" -import ast import json from pathlib import Path @@ -16,7 +15,6 @@ moe_model_allows_quantization, ) from collector.model_cases import build_collection_case_plan -from collector.version_resolver import _check_compat from .test_getter_deduplication import _install_vllm_stubs, _load_collector @@ -30,15 +28,6 @@ GLM_OFFGRID_WIDTHS = {8, 16, 32, 288, 3336, 6416, 12576, 19360, 24576, 24896, 38720, 77440, 154880} -def _compat(relative_path: str) -> str: - tree = ast.parse((REPO_ROOT / relative_path).read_text()) - return next( - node.value.value - for node in tree.body - if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "__compat__" for t in node.targets) - ) - - @pytest.mark.parametrize("backend", ["vllm", "sglang"]) @pytest.mark.parametrize("model_path", [FP8, NVFP4]) def test_plan_collects_gemm_moe_kda_and_mhc(backend, model_path): @@ -159,37 +148,6 @@ def test_vllm_nvfp4_uses_modelopt_only_for_the_glm53flash_checkpoint(monkeypatch assert (qc["quant_method"], qc["quant_algo"]) == ("modelopt", "NVFP4") -@pytest.mark.parametrize( - "path", - ["collector/vllm/collect_gemm.py", "collector/vllm/collect_moe.py"], -) -@pytest.mark.parametrize( - "version,accepted", - [ - ("0.31.0", True), # gemm / moe family pins (2026-10-30) - ("0.30.0", False), # 0.30.0 collectors -> git history - ("0.28.0", False), - ("0.29.0", False), - ("0.31.1", False), - ("0.32.0", False), - ("0.24.0", False), - ], -) -def test_vllm_collectors_admit_exactly_the_audited_releases(path, version, accepted): - assert _check_compat(_compat(path), version) is accepted - - -def test_sglang_collectors_admit_only_0_5_21(): - # The 0.5.20 GLM-5.3-Flash lane (incl. the collect_gemm_0520 fork) retired - # 2026-10-30; one release per file. - for path in ("collector/sglang/collect_gemm.py", "collector/sglang/collect_moe.py"): - compat = _compat(path) - assert compat == "sglang==0.5.21", path - for version in ("0.5.14", "0.5.17", "0.5.20"): - assert not _check_compat(compat, version), (path, version) - assert not (REPO_ROOT / "collector/sglang/collect_gemm_0520.py").exists() - - def test_sglang_nvfp4_allowlist_admits_glm53flash_nvfp4_only(): import yaml diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py b/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py index a310c4dca3..fabaa97cfa 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_runtime_pins.py @@ -26,15 +26,6 @@ def test_glm_vllm_pin_is_the_stock_0_31_0_release(model_path): assert runtime.family is None -@pytest.mark.parametrize("model_path", GLM_MODEL_PATHS) -def test_glm_has_no_sglang_model_pin(model_path): - # SGLang GLM-5.3-Flash resolves to the 0.5.21 default like every other model. - runtime = require_collector_runtime("sglang", "0.5.21", requested_ops={"gemm"}, model_path=model_path) - assert runtime.version == "0.5.21" - with pytest.raises(RuntimeError, match="requires exactly 0.5.21"): - require_collector_runtime("sglang", "0.5.20", requested_ops={"gemm"}, model_path=model_path) - - @pytest.mark.parametrize( "backend,wrong_version,expected", [("vllm", "0.27.1", "0.31.0"), ("vllm", "0.30.0", "0.31.0")], @@ -49,10 +40,7 @@ def test_glm_pin_rejects_other_runtimes(backend, wrong_version, expected): [ ("vllm", "0.31.0", None), # gemm family pin ("vllm", "0.31.0", "zai-org/GLM-5.2-FP8"), - ("vllm", "0.31.0", "Qwen/Qwen3.8-2.4T-A95B"), # Qwen3.8 pins folded 2026-10-30 - ("sglang", "0.5.21", None), ("sglang", "0.5.21", "zai-org/GLM-5.2-FP8"), - ("sglang", "0.5.21", "Qwen/Qwen3.8-2.4T-A95B"), ], ) def test_other_models_keep_their_runtime(backend, version, model_path): diff --git a/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py b/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py index fd404e34d3..f427d9deb6 100644 --- a/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py +++ b/python/aisimulate/tests/unit/collector/test_helper_flashinfer_autotune.py @@ -10,21 +10,17 @@ import sys import types from contextlib import contextmanager -from typing import ClassVar import pytest from collector import helper from collector.helper import ( FLASHINFER_AUTOTUNE_BUDGET_ENV, FLASHINFER_AUTOTUNE_ENV, - FlashInferTuneBudget, flashinfer_autotuned, ) pytestmark = pytest.mark.unit -_REAL_DEVICE_TAG = helper._flashinfer_device_tag - class _FakeTuner: instance = None @@ -80,7 +76,7 @@ def autotune(tune_mode=True, cache=None, tuning_buckets=None, round_up=None, ski return types.SimpleNamespace(events=events, active=active, tuner=_FakeTuner) -def test_tunes_once_at_the_exact_token_count_then_times_under_lookup_only(fake_flashinfer, capsys): +def test_tunes_once_at_the_exact_token_count_then_times_under_lookup_only(fake_flashinfer): seen: list[tuple] = [] # A key tuned by an earlier case on this worker is not reported again. @@ -108,14 +104,9 @@ def tune_fn(): ("enter", False, (320,)), ("exit", False, (320,)), ] - out = capsys.readouterr().out - assert "FlashInfer autotune: 1 new tactic entries at num_tokens=320 vllm_trtllm_nvfp4 [" in out - assert "s, warmup=1 repeat=3 delay_us=1000]" in out - assert "FlashInfer autotune chosen at num_tokens=320 vllm_trtllm_nvfp4: ('moe', 320) -> tactic 'tactic'" in out - assert "WARNING" not in out -def test_budget_env_overrides_or_keeps_flashinfer_defaults(fake_flashinfer, monkeypatch, capsys): +def test_budget_env_overrides_or_keeps_flashinfer_defaults(fake_flashinfer, monkeypatch): tuner = fake_flashinfer.tuner.get() seen = [] @@ -124,36 +115,20 @@ def test_budget_env_overrides_or_keeps_flashinfer_defaults(fake_flashinfer, monk pass assert seen == [(2, 5, 2000)] assert tuner.budget() == (3, 10, 5000) - assert "warmup=2 repeat=5 delay_us=2000]" in capsys.readouterr().out monkeypatch.setenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, "flashinfer") with flashinfer_autotuned(64, lambda: seen.append(tuner.budget())): pass assert seen[-1] == (3, 10, 5000) - assert "flashinfer defaults]" in capsys.readouterr().out -@pytest.mark.parametrize("raw", ["3,10", "a,b,c", "1,0,1000", "-1,3,1000", "1,3,-5"]) +@pytest.mark.parametrize("raw", ["3,10", "1,0,1000"]) def test_budget_env_rejects_malformed_values(monkeypatch, raw): monkeypatch.setenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, raw) with pytest.raises(ValueError, match=FLASHINFER_AUTOTUNE_BUDGET_ENV): helper.flashinfer_autotune_budget() -def test_budget_default_and_restore_when_tune_fn_raises(fake_flashinfer, monkeypatch): - monkeypatch.delenv(FLASHINFER_AUTOTUNE_BUDGET_ENV, raising=False) - assert helper.flashinfer_autotune_budget() == FlashInferTuneBudget(1, 3, 1000) - tuner = fake_flashinfer.tuner.get() - - def boom(): - raise RuntimeError("tuning forward failed") - - with pytest.raises(RuntimeError, match="tuning forward failed"), flashinfer_autotuned(8, boom): - pass - assert tuner.budget() == (3, 10, 5000) - assert fake_flashinfer.active == [] - - def test_env_switch_restores_the_untuned_measurement(fake_flashinfer, monkeypatch): monkeypatch.setenv(FLASHINFER_AUTOTUNE_ENV, "0") calls = [] @@ -166,23 +141,10 @@ def test_env_switch_restores_the_untuned_measurement(fake_flashinfer, monkeypatc assert fake_flashinfer.events == [] -def test_without_flashinfer_the_body_runs_untouched(monkeypatch): - monkeypatch.delenv(FLASHINFER_AUTOTUNE_ENV, raising=False) - monkeypatch.setattr(helper, "_flashinfer_autotune_api", lambda: (None, None)) - calls = [] - - with flashinfer_autotuned(64, lambda: calls.append("tuned")) as tuned: - assert tuned is False - - assert calls == [] - - -def test_default_is_on_and_accepts_the_shared_bool_grammar(monkeypatch): +def test_default_is_on(monkeypatch): + # Off by default would silently time FlashInfer's untuned fallback tactic. monkeypatch.delenv(FLASHINFER_AUTOTUNE_ENV, raising=False) assert helper.flashinfer_autotune_enabled() is True - for value, expected in (("false", False), ("no", False), ("1", True), ("TRUE", True)): - monkeypatch.setenv(FLASHINFER_AUTOTUNE_ENV, value) - assert helper.flashinfer_autotune_enabled() is expected def test_both_moe_collectors_wrap_their_timing_in_the_tuning_context(): @@ -221,12 +183,16 @@ def test_poisoned_context_after_the_tuning_pass_is_reported_as_the_autotune_fail assert fake_flashinfer.events == [("enter", True, (1,)), ("exit", True, (1,))] -def test_tune_fn_error_on_a_healthy_context_propagates_unchanged(fake_flashinfer): +def test_tune_fn_error_on_a_healthy_context_propagates_unchanged_and_restores_the_budget(fake_flashinfer): + tuner = fake_flashinfer.tuner.get() + def tune_fn(): raise ValueError("shape rejected") with pytest.raises(ValueError, match="shape rejected"), flashinfer_autotuned(8, tune_fn): pass + assert tuner.budget() == (3, 10, 5000) + assert fake_flashinfer.active == [] def test_cuda_context_error_probe_reports_a_dead_context(monkeypatch): @@ -258,190 +224,3 @@ def item(self): fake_torch = types.SimpleNamespace(cuda=_HealthyCuda, ones=lambda *a, **k: _Tensor()) monkeypatch.setitem(sys.modules, "torch", fake_torch) assert helper.cuda_context_error(0) is None - - -def test_autotuner_fallback_messages_are_surfaced_and_logging_is_restored(fake_flashinfer, capsys): - """b300 sglang 0.5.21 (2026-10-28): the Nemotron-3-Nano nvfp4 relu2 shape came back - identical to the untuned table while every other shape moved -- FlashInfer's - ``MoERunner.get_valid_tactics`` swallows the C++ exception, logs it at DEBUG and - returns ``[]``, so the op is timed on the fallback with no visible trace. The tuning - pass captures that log and prints the explanation next to the case.""" - import logging - - fi_logger = logging.getLogger("flashinfer") - # Other test modules leave this logger at an explicit level (seen at ERROR in - # the full collector run); the scenario needs the production default, NOTSET. - inherited_level = fi_logger.level - fi_logger.setLevel(logging.NOTSET) - # The threshold the capture pins pre-existing handlers at is the logger's EFFECTIVE level, - # inherited from the root logger — WARNING by default, INFO when another test module - # configured logging in this process; read it instead of assuming. - effective_before = fi_logger.getEffectiveLevel() - stream = logging.StreamHandler() - fi_logger.addHandler(stream) - try: - seen_levels = [] - - def tune_fn(): - seen_levels.append((fi_logger.level, stream.level)) - fi_logger.debug( - "[Autotuner]: Failed to get valid tactics for (E2m1, 6, 2688, 1920, 128, Relu2). " - "Error occurred: No kernel found for the given options: mActType 6 tile 192" - ) - fi_logger.debug("[Autotuner]: profiling runner MoERunner tactic (128, 3)") - fi_logger.debug("[AutoTuner]: Using fallback tactic for trtllm_fp4_block_scale_moe") - fi_logger.debug("[AutoTuner]: Using fallback tactic for trtllm_fp4_block_scale_moe") # duplicate - - with flashinfer_autotuned(768, tune_fn, label="sglang_flashinfer_trtllm_moe"): - pass - - # Captured at DEBUG, while the pre-existing stream handler was pinned at the old - # effective threshold so FlashInfer's stderr output did not change. - assert seen_levels == [(logging.DEBUG, effective_before)] - assert fi_logger.level == logging.NOTSET - assert stream.level == logging.NOTSET - assert all(h is not fi_logger for h in fi_logger.handlers) and stream in fi_logger.handlers - finally: - fi_logger.removeHandler(stream) - fi_logger.setLevel(inherited_level) - - out = capsys.readouterr().out - assert "FlashInfer autotune: 0 new tactic entries at num_tokens=768 sglang_flashinfer_trtllm_moe [" in out - warning = [line for line in out.splitlines() if line.startswith("FlashInfer autotune WARNING")] - assert len(warning) == 1 - assert "num_tokens=768 sglang_flashinfer_trtllm_moe: 2 autotuner messages" in warning[0] - assert "No kernel found for the given options: mActType 6 tile 192" in warning[0] - assert warning[0].count("Using fallback tactic") == 1 - assert "profiling runner" not in warning[0] - assert "tactics failed while profiling" not in warning[0] - - -def test_per_tactic_profiling_failures_are_counted_from_the_tuner_statistics(fake_flashinfer, capsys): - """Every candidate tactic runs the whole op on synthetic inputs; when e.g. the routing - kernel faults on them FlashInfer marks the tactic inf at DEBUG, records it in - ``stats.failed_tactics`` and -- if none survives -- serves the fallback. The pass - reports the new failures from the statistics even when no log line matched.""" - import logging - - tuner = fake_flashinfer.tuner.get() - op = "flashinfer::trtllm_fp4_block_scale_moe::MoERunner" - tuner.stats = types.SimpleNamespace(failed_tactics={op: {(8, 1)}}) - fi_logger = logging.getLogger("flashinfer") - - def tune_fn(): - tuner.stats.failed_tactics[op].update({(16, 2), (32, 5)}) - fi_logger.debug( - "[Autotuner]: Failed when profiling MoERunner (16, 2), shapes=[...]. Error occurred: " - "CUDA error in trtllm_fused_moe_routing_custom.cu:1045: an illegal memory access" - ) - fi_logger.info("[Autotuner]: Skipped 2 unsupported tactic(s) for trtllm_fp4_block_scale_moe") - - with flashinfer_autotuned(3072, tune_fn, label="sglang_flashinfer_trtllm_moe"): - pass - - out = capsys.readouterr().out - warning = [line for line in out.splitlines() if line.startswith("FlashInfer autotune WARNING")] - assert len(warning) == 1 - assert f"tactics failed while profiling: {op}=2" in warning[0] - assert "routing_custom.cu:1045" in warning[0] - assert "Skipped 2 unsupported tactic(s)" in warning[0] - - -def test_newly_enumerated_tactic_tables_are_printed_per_instance_key(fake_flashinfer, monkeypatch, capsys): - """Probe 77139257 (b200 sglang 0.5.21, 2026-10-28): the Nemotron relu2 nvfp4 cases wrote one - cache entry each after a 0.02-0.9 s pass and still measured the untuned latency -- the - autotuner may simply have had one candidate. Print the launcher-enumerated tactic list - per instance key (intermediate size, activation, ... included) for every key the pass - added, reading FlashInfer's already-built module only (no JIT build from here).""" - - class _Runner: - valid_tactics_dict: ClassVar[dict] = {("E2m1", 8, 7168, 2048, 256, "Swiglu", 768): [(64, 0), (64, 1), (128, 0)]} - - calls = [] - - class _Impl: - currsize = 0 - - @staticmethod - def cache_info(): - return types.SimpleNamespace(currsize=_Impl.currsize) - - def get_module(): - calls.append("get") - return types.SimpleNamespace(MoERunner=_Runner) - - core = types.ModuleType("flashinfer.fused_moe.core") - core._get_trtllm_moe_sm100_module_impl = _Impl - core.get_trtllm_moe_sm100_module = get_module - fused = types.ModuleType("flashinfer.fused_moe") - fused.core = core - sys.modules["flashinfer"].fused_moe = fused - monkeypatch.setitem(sys.modules, "flashinfer.fused_moe", fused) - monkeypatch.setitem(sys.modules, "flashinfer.fused_moe.core", core) - - # Module not built yet: nothing is read and the factory is not called. - with flashinfer_autotuned(8, lambda: None, label="cold"): - pass - assert calls == [] - assert "FlashInfer autotune tactics" not in capsys.readouterr().out - - def tune_fn(): - _Impl.currsize = 1 # the forward built the module ... - _Runner.valid_tactics_dict[("E2m1", 6, 2688, 1920, 128, "Relu2", 768)] = [(64, 3)] # ... and enumerated - - with flashinfer_autotuned(768, tune_fn, label="sglang_flashinfer_trtllm_moe"): - pass - out = capsys.readouterr().out - lines = [line for line in out.splitlines() if line.startswith("FlashInfer autotune tactics")] - # The module came up during this pass, so every key it holds is new to the snapshot: - # the control shape's three tactics and the single Relu2 candidate. - assert len(lines) == 2 - assert ( - "FlashInfer autotune tactics at num_tokens=768 sglang_flashinfer_trtllm_moe: trtllm_sm100 " - "('E2m1', 8, 7168, 2048, 256, 'Swiglu', 768) -> 3 tactics [(64, 0), (64, 1), (128, 0)]" - ) in lines - assert ( - "FlashInfer autotune tactics at num_tokens=768 sglang_flashinfer_trtllm_moe: trtllm_sm100 " - "('E2m1', 6, 2688, 1920, 128, 'Relu2', 768) -> 1 tactics [(64, 3)]" - ) in lines - - # Next pass: only keys added since the snapshot are reported. - def tune_fn_ep4(): - _Runner.valid_tactics_dict[("E2m1", 6, 2688, 1920, 32, "Relu2", 768)] = [(64, 3)] - - with flashinfer_autotuned(768, tune_fn_ep4, label="sglang_flashinfer_trtllm_moe"): - pass - lines = [line for line in capsys.readouterr().out.splitlines() if line.startswith("FlashInfer autotune tactics")] - assert lines == [ - "FlashInfer autotune tactics at num_tokens=768 sglang_flashinfer_trtllm_moe: trtllm_sm100 " - "('E2m1', 6, 2688, 1920, 32, 'Relu2', 768) -> 1 tactics [(64, 3)]" - ] - - -def test_autotune_lines_carry_the_cuda_device_tag(fake_flashinfer, monkeypatch, capsys): - """Parallel workers share one log and tune the same key independently; the device tag - is what lets a per-worker tactic choice be reconstructed afterwards (gb200 sglang part07 - could not be: no worker tag on the autotune lines).""" - - class _Cuda: - @staticmethod - def is_available(): - return True - - @staticmethod - def current_device(): - return 3 - - monkeypatch.setitem(sys.modules, "torch", types.SimpleNamespace(cuda=_Cuda)) - monkeypatch.setattr(helper, "_flashinfer_device_tag", _REAL_DEVICE_TAG) # fixture blanks it - assert helper._flashinfer_device_tag() == " dev3" - - def tune_fn(): - fake_flashinfer.tuner.get().profiling_cache[("moe", 96)] = ((128, 2), None) - - with flashinfer_autotuned(96, tune_fn, label="sglang_flashinfer_trtllm_moe"): - pass - out = capsys.readouterr().out - assert "FlashInfer autotune: 1 new tactic entries at num_tokens=96 sglang_flashinfer_trtllm_moe [" in out - assert "delay_us=1000] dev3\n" in out - assert "-> tactic (128, 2) dev3\n" in out diff --git a/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py b/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py index e7fdd2a2c8..b40cf0787e 100644 --- a/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py +++ b/python/aisimulate/tests/unit/collector/test_helper_phase_clock.py @@ -1,46 +1,26 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -import time - import pytest from collector.helper import PhaseClock, render_phase_totals pytestmark = pytest.mark.unit -def test_phase_clock_marks_absorbs_and_renders(): +def test_phase_clock_accumulates_phases_and_point_totals(): + # The rendered text only feeds human-read log lines; this is a smoke test of the accounting. clock = PhaseClock() - time.sleep(0.01) clock.mark("build") clock.mark("build") # a second slice accumulates into the same phase clock.absorb({"warmup": 0.5, "timed": 0.25}, prefix="bench_") clock.absorb(None) - assert clock.phases["build"] >= 0.01 - assert clock.phases["bench_warmup"] == 0.5 - assert clock.phases["bench_timed"] == 0.25 - rendered = clock.render() - assert rendered.startswith("build=") - assert "bench_warmup=0.50s" in rendered - assert rendered.split()[-1].startswith("total=") and rendered.endswith("s") + assert clock.phases["bench_warmup"] == 0.5 and clock.phases["bench_timed"] == 0.25 assert clock.total() >= clock.phases["build"] - - -def test_phase_totals_accumulate_points_and_render_without_the_counter(): - clock = PhaseClock() - clock.mark("build") - clock.absorb({"timed": 0.25}, prefix="bench_") totals: dict[str, float] = {} clock.add_to(totals) clock.add_to(totals) - assert totals["points"] == 2 - assert totals["bench_timed"] == 0.5 - assert totals["total"] >= 2 * totals["build"] - text = render_phase_totals(totals) - assert text.startswith("2 point(s): ") - assert "bench_timed=0.5s" in text - assert "points=" not in text - assert render_phase_totals({}) == "0 point(s): " + assert totals["points"] == 2 and totals["bench_timed"] == 0.5 + assert isinstance(clock.render(), str) and isinstance(render_phase_totals(totals), str) def _fake_torch(monkeypatch, allocated): @@ -88,20 +68,3 @@ def test_periodic_gc_collects_early_on_memory_growth(monkeypatch): assert len(calls) == 1 assert pgc.calls == 0 assert pgc.floor == 1000 # the floor never rises - - -def test_periodic_gc_without_cuda_counts_calls_only(monkeypatch): - import gc - import sys - import types - - from collector.helper import PeriodicGarbageCollector - - fake = types.ModuleType("torch") - fake.cuda = types.SimpleNamespace(is_available=lambda: False) - monkeypatch.setitem(sys.modules, "torch", fake) - calls = [] - monkeypatch.setattr(gc, "collect", lambda: calls.append(1) or 0) - pgc = PeriodicGarbageCollector(every=2) - assert [pgc.maybe_collect() for _ in range(4)] == [False, True, False, True] - assert len(calls) == 2 diff --git a/python/aisimulate/tests/unit/collector/test_kv_block_size.py b/python/aisimulate/tests/unit/collector/test_kv_block_size.py deleted file mode 100644 index 0996cf02d0..0000000000 --- a/python/aisimulate/tests/unit/collector/test_kv_block_size.py +++ /dev/null @@ -1,48 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""KV page size is a per-SM branch (owner decision 2026-09-30 after the B200 -campaign caught the 64 literal running a different FMHA cubin on SM100).""" - -import ast -from pathlib import Path - -import pytest - -SRC = Path(__file__).resolve().parents[3] / "collector" / "vllm" / "utils.py" - - -def _kv_block_size(): - """The module imports vllm at import time; lift just the pure function.""" - tree = ast.parse(SRC.read_text()) - fn = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "kv_block_size") - ns = {} - exec(compile(ast.Module(body=[fn], type_ignores=[]), str(SRC), "exec"), ns) - return ns["kv_block_size"] - - -kv_block_size = _kv_block_size() - -pytestmark = pytest.mark.unit - - -@pytest.mark.parametrize( - "sm,op,expected", - [ - (90, "attention", 64), - (90, "mla", 64), - (100, "attention", 16), - (103, "attention", 16), - (100, "mla", 32), - (103, "mla", 32), - (120, "attention", 64), - (120, "mla", 64), - ], -) -def test_page_size_per_sm(sm, op, expected, monkeypatch): - monkeypatch.delenv("AIS_KV_BLOCK_SIZE", raising=False) - assert kv_block_size(sm, op) == expected - - -def test_ab_override_hook(monkeypatch): - monkeypatch.setenv("AIS_KV_BLOCK_SIZE", "16") - assert kv_block_size(90, "attention") == 16 diff --git a/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py b/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py index 0ea11358d2..c2e6b7e098 100644 --- a/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py +++ b/python/aisimulate/tests/unit/collector/test_mhc_module_glm5next.py @@ -104,19 +104,13 @@ def test_glm_row_conventions_match_the_shared_table(monkeypatch): ("backend", "architecture", "version", "audited"), [ # DeepSeek-V4 is not audited on 0.31.0 (shared architecture-free key, see the module) - ("vllm", DSV4, "0.30.0", False), ("vllm", DSV4, "0.31.0", False), ("vllm", GLM, "0.31.0", True), ("vllm", GLM, "0.30.0", False), - ("vllm", GLM, "0.25.0", False), - ("vllm", GLM, "0.27.1", False), ("sglang", DSV4, "0.5.21", True), - ("sglang", DSV4, "0.5.14", False), ("sglang", DSV4, "0.5.20", False), - ("sglang", GLM, "0.5.20", False), # 0.5.20 lane retired 2026-10-30 ("sglang", GLM, "0.5.21", False), # not ported to 0.5.21 yet - ("sglang", GLM, "0.5.14", False), - ("sglang", "UnknownForCausalLM", "0.5.20", False), + ("sglang", "UnknownForCausalLM", "0.5.21", False), ], ) def test_each_architecture_runs_only_on_its_audited_release(monkeypatch, backend, architecture, version, audited): @@ -141,18 +135,12 @@ def test_audited_releases_never_overlap_across_architectures(monkeypatch, backen module = _load(monkeypatch, backend) from collector.version_resolver import _check_compat - candidates = ["0.24.0", "0.25.0", "0.27.1", "0.30.0", "0.31.0", "0.5.14", "0.5.16", "0.5.20"] + candidates = ["0.24.0", "0.25.0", "0.27.1", "0.30.0", "0.31.0", "0.5.14", "0.5.16", "0.5.20", "0.5.21"] for version in candidates: admitted = [arch for arch, spec in module._ARCHITECTURE_COMPAT.items() if _check_compat(spec, version)] assert len(admitted) <= 1, (version, admitted) -def test_vllm_file_is_pinned_to_the_mhc_family_release(monkeypatch): - module = _load(monkeypatch, "vllm") - assert module.__compat__ == "vllm==0.31.0" - assert set(module._ARCHITECTURE_COMPAT) == {GLM} - - def test_vllm_glm_sites_import_the_0_31_0_module_location(): # vllm.models.glm5next.nvidia.model moved to .common.model at 0.31.0. source = (COLLECTOR_DIR / "vllm" / "collect_mhc_module.py").read_text(encoding="utf-8") diff --git a/python/aisimulate/tests/unit/collector/test_model_cases.py b/python/aisimulate/tests/unit/collector/test_model_cases.py index f67cbeed43..6cbffe1876 100644 --- a/python/aisimulate/tests/unit/collector/test_model_cases.py +++ b/python/aisimulate/tests/unit/collector/test_model_cases.py @@ -2390,14 +2390,8 @@ def test_msa_head_axis_derives_from_declared_shards(): """MSA is one fixed geometry (MiniMax-M3 64q/4kv/4idx): the collectors sweep native // tp for the model row's tensor_parallel_sizes, never the shared GQA-style head grid (owner decision 2026-09-29; TP 16 is the ceiling).""" - from collector.case_generator import get_mla_module_model_specs, get_msa_head_counts + from collector.case_generator import get_msa_head_counts - (spec,) = [ - s - for s in get_mla_module_model_specs(attention_type="msa", apply_model_filter=False) - if s.model_path == "MiniMaxAI/MiniMax-M3" - ] - assert spec.tensor_parallel_sizes == (1, 2, 4, 8, 16) for backend in ("vllm", "trtllm", "sglang"): heads = get_msa_head_counts(backend) assert heads == [64, 32, 16, 8, 4], (backend, heads) diff --git a/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py b/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py index 916fcfd24e..6fdf2c3fcc 100644 --- a/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py +++ b/python/aisimulate/tests/unit/collector/test_msa_getter_contract.py @@ -245,10 +245,10 @@ def test_targeted_msa_getters_keep_prefixes_and_worker_tuple_compatibility(monke @pytest.mark.parametrize( "sm_version,expected_max_tokens", - [(89, 32768), (90, 32768), (120, 32768), (100, 131072), (103, 131072)], + [(90, 32768), (100, 131072)], ) def test_trtllm_context_getter_caps_only_the_triton_reference_implementation( - monkeypatch, capsys, sm_version, expected_max_tokens + monkeypatch, sm_version, expected_max_tokens ): """mla_module_trtllm.context.msa_triton_max_tokens binds where the collector resolves implementation="triton" (off SM100/103); fmha_sm100 systems keep @@ -262,8 +262,6 @@ def test_trtllm_context_getter_caps_only_the_triton_reference_implementation( assert max(s * b for s, b in shapes) == expected_max_tokens assert (32768, 1) in shapes - dropped = "[trtllm-msa-module-cases] context: dropped" in capsys.readouterr().out - assert dropped == (expected_max_tokens == 32768) def test_vllm_public_generation_getter_applies_live_memory_filter(monkeypatch, capsys): diff --git a/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py b/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py index 3a024cd2ca..dd84b03968 100644 --- a/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py +++ b/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py @@ -32,7 +32,7 @@ def test_non_power_of_two_index_heads_fail_only_where_the_decode_scorer_runs(idx assert limit(idx_heads, num_heads, is_context=True, query_len=32768) is None -@pytest.mark.parametrize("idx_heads", [1, 2, 4, 8]) +@pytest.mark.parametrize("idx_heads", [1, 4]) def test_power_of_two_index_heads_never_limited(idx_heads): limit = _predicate() for is_context, q in ((False, 1), (True, 1), (True, 4096)): diff --git a/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py b/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py index 42c2be3bc2..da68bbc3b2 100644 --- a/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py +++ b/python/aisimulate/tests/unit/collector/test_no_eager_graph_fallback_sites.py @@ -10,7 +10,11 @@ The sglang trtllm_mla prefill (B1) and the trtllm GDN lanes were closed on 2026-10-11, the last four sites — trtllm mamba2 and moe, wideep moe_compute, network trtllm alltoall — on 2026-10-12. This test keeps the count at zero for -every file under ``collector/``.""" +every file under ``collector/``: any ``allow_graph_fail=`` other than the literal +``False`` (a variable could carry ``True``) is an offender. The one sound exception +is a wrapper forwarding its own keyword-only ``allow_graph_fail=False`` parameter: +every call of that wrapper passes the keyword (or the False default) and is scanned +by the same walk.""" import ast from pathlib import Path @@ -21,45 +25,35 @@ COLLECTOR_ROOT = Path(__file__).resolve().parents[3] / "collector" -# Former fallback sites: number of benchmark_with_power calls each still makes. -FORMER_SITES = { - "network/slurm/collect_trtllm_alltoall.py": 1, - "trtllm/collect_mamba2.py": 8, - "trtllm/collect_moe.py": 1, - "vllm/collect_moe_xpu.py": 1, - "wideep/trtllm/collect_moe_compute.py": 1, -} +def _forwards_its_false_default(function, value): + """``allow_graph_fail=allow_graph_fail`` inside a function whose keyword-only parameter defaults to False.""" + if function is None or not (isinstance(value, ast.Name) and value.id == "allow_graph_fail"): + return False + for arg, default in zip(function.args.kwonlyargs, function.args.kw_defaults, strict=True): + if arg.arg == "allow_graph_fail": + return isinstance(default, ast.Constant) and default.value is False + return False -def _benchmark_calls(tree): - return [ - node - for node in ast.walk(tree) - if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "benchmark_with_power" - ] - -def test_no_collector_passes_allow_graph_fail_true(): +def test_no_collector_passes_allow_graph_fail_other_than_literal_false(): offenders = [] for path in sorted(COLLECTOR_ROOT.rglob("*.py")): tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + enclosing = {} # node -> innermost enclosing function (ast.walk visits outer functions first) + for function in ast.walk(tree): + if isinstance(function, ast.FunctionDef | ast.AsyncFunctionDef): + for child in ast.walk(function): + enclosing[child] = function for node in ast.walk(tree): if not isinstance(node, ast.Call): continue for keyword in node.keywords: - if ( - keyword.arg == "allow_graph_fail" - and isinstance(keyword.value, ast.Constant) - and keyword.value.value is True - ): - offenders.append(f"{path.relative_to(COLLECTOR_ROOT)}:{node.lineno}") + if keyword.arg != "allow_graph_fail": + continue + if isinstance(keyword.value, ast.Constant) and keyword.value.value is False: + continue + if _forwards_its_false_default(enclosing.get(node), keyword.value): + continue + offenders.append(f"{path.relative_to(COLLECTOR_ROOT)}:{node.lineno}") assert offenders == [], f"eager-timed rows would ship from: {offenders}" - - -@pytest.mark.parametrize(("relative_path", "expected_calls"), sorted(FORMER_SITES.items())) -def test_former_fallback_sites_time_under_cuda_graphs_only(relative_path, expected_calls): - tree = ast.parse((COLLECTOR_ROOT / relative_path).read_text(encoding="utf-8")) - calls = _benchmark_calls(tree) - assert len(calls) == expected_calls - for call in calls: - assert all(keyword.arg != "allow_graph_fail" for keyword in call.keywords), relative_path diff --git a/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py b/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py index e6a0b4708f..9311db5887 100644 --- a/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py +++ b/python/aisimulate/tests/unit/collector/test_perf_output_finalization.py @@ -1124,30 +1124,29 @@ def test_finalize_keeps_power_disabled_schema_column_free(tmp_path): assert pq.read_table(parquet).column_names == ["shape", "latency"] -def test_finalize_drops_rows_re_appended_by_a_retried_task_keeping_the_newest(tmp_path, caplog): +def test_finalize_drops_rows_re_appended_by_a_retried_task_keeping_the_newest(tmp_path): """The staging CSV is append-only across resumed windows and a task retried with --resume-retry-failed logs its successful rows again (rtxpro msa_context: 10,342 of 13,972 rows were such re-appends). Finalization keeps the newest row per identity key, in the original order, like the merge-with-existing path already did.""" perf = tmp_path / "gemm_perf.txt" _write_keyed_perf_csv(perf, [("s1", 1.0), ("s2", 2.0), ("s1", 1.5), ("s3", 3.0), ("s2", 2.5)]) + finalization_info = {} - with caplog.at_level("INFO", logger="collector.helper"): - [parquet] = finalize_perf_files([perf], delete_source=False) + [parquet] = finalize_perf_files([perf], delete_source=False, finalization_info=finalization_info) table = pq.read_table(parquet) assert table.column("shape").to_pylist() == ["s1", "s3", "s2"] assert table.column("latency").to_pylist() == [1.5, 3.0, 2.5] - assert "dropped 2 re-appended duplicate rows" in caplog.text - + [info] = finalization_info.values() + assert info.new_rows == 3 # the deduplicated count is what gets reported -def test_finalize_keeps_rows_that_differ_in_a_non_metric_column(tmp_path): - perf = tmp_path / "gemm_perf.txt" + # Rows that differ only in metric columns (power, latency) share one identity: the newer one wins. + metric_only = tmp_path / "metric_only" + metric_only.mkdir() + perf = metric_only / "gemm_perf.txt" perf.write_text("op,shape,power,latency\nmatmul,s1,100.0,1.0\nmatmul,s1,120.0,1.1\nmatmul,s2,100.0,2.0\n") - [parquet] = finalize_perf_files([perf], delete_source=False) - - # power is a metric, so the two s1 rows share one identity: the newer one wins assert pq.read_table(parquet).column("latency").to_pylist() == [1.1, 2.0] @@ -1166,14 +1165,3 @@ def test_finalize_dedupe_ignores_moe_eligibility_metadata_and_tolerates_empty_co assert table.column("shape").to_pylist() == ["s1", "s2"] assert table.column("default_eligible").to_pylist() == [False, True] assert table.column("latency").to_pylist() == [1.2, 2.0] - - -def test_finalize_reports_deduplicated_row_count(tmp_path): - perf = tmp_path / "gemm_perf.txt" - _write_keyed_perf_csv(perf, [("s1", 1.0), ("s1", 1.5)]) - finalization_info = {} - - finalize_perf_files([perf], delete_source=False, finalization_info=finalization_info) - - [info] = finalization_info.values() - assert info.new_rows == 1 diff --git a/python/aisimulate/tests/unit/collector/test_registry_table_producers.py b/python/aisimulate/tests/unit/collector/test_registry_table_producers.py index d4a51c3ca2..1c4dac5e0e 100644 --- a/python/aisimulate/tests/unit/collector/test_registry_table_producers.py +++ b/python/aisimulate/tests/unit/collector/test_registry_table_producers.py @@ -49,14 +49,12 @@ def test_op_entry_rejects_duplicate_tables(): ) -@pytest.mark.parametrize("backend", BACKENDS) -def test_provenance_collections_carry_extra_tables(backend): +def test_provenance_collections_carry_extra_tables(): from collector.version_resolver import build_collections - version = {"vllm": "0.30.0", "sglang": "0.5.14", "trtllm": "1.3.0rc23"}[backend] (cs,) = [ c - for c in build_collections(_registry(backend), backend, version, ops=["compute_scale"]) + for c in build_collections(_registry("vllm"), "vllm", "0.30.0", ops=["compute_scale"]) if c["type"] == "compute_scale" ] assert cs["perf_filename"] == PerfFile.COMPUTESCALE @@ -102,17 +100,6 @@ def test_executor_binds_every_declared_table_to_the_producer(): collect._registered_checkpoint_tables({**identity, "module": "vllm.nope"}, backend="vllm") -@pytest.mark.parametrize("backend", BACKENDS) -def test_compute_scale_collector_writes_the_handed_in_tables(backend): - src = importlib.util.find_spec(f"collector.{backend}.collect_computescale").origin - text = Path(src).read_text() - assert "def run_computescale(m, k, *, perf_filename, extra_perf_filenames" in text - assert "def run_scale_matrix" not in text - # no table name is hard-coded any more: the executor hands both in - assert 'perf_filename="scale_matrix_perf.txt"' not in text - assert "perf_filename=scale_matrix_filename" in text - - # ── shared-table producers: sglang dsa_*_module_skip_indexer ───────────────── # The skip-indexer rows live in the full-module table (op_name "_skip_indexer" # suffix; the SDK reads them from there), so the skip ops declare that table @@ -151,16 +138,7 @@ def test_provenance_collections_carry_run_kwargs(): assert by_type["dsa_context_module_skip_indexer"]["perf_filename"] == by_type["dsa_context_module"]["perf_filename"] -@pytest.mark.parametrize( - "bad", [(("perf_filename", "x.txt"),), (("device", "cuda:1"),), (("", True),), (("a", 1), ("a", 2))] -) +@pytest.mark.parametrize("bad", [(("perf_filename", "x.txt"),), (("a", 1), ("a", 2))]) def test_op_entry_rejects_executor_owned_or_malformed_run_kwargs(bad): with pytest.raises(ValueError, match="run_kwargs"): OpEntry(op="x", module="m", get_func="g", run_func="r", perf_filename=PerfFile.GEMM, run_kwargs=bad) - - -def test_skip_indexer_worker_takes_the_explicit_argument_and_does_not_sniff_the_table_name(): - src = importlib.util.find_spec("collector.sglang.collect_mla_module").origin - text = Path(src).read_text(encoding="utf-8") - assert 'skip_indexer: bool = False,\n):\n """Worker-compatible wrapper' in text - assert '"skip_indexer" in os.path.basename(perf_filename)' not in text diff --git a/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py b/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py index 9e21624304..c7c162dd99 100644 --- a/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py +++ b/python/aisimulate/tests/unit/collector/test_sglang_glm5_sparse_0514.py @@ -58,12 +58,10 @@ def test_glm5_flash_mla_keeps_the_production_topk_width(): assert "if k % 64" not in source -def test_glm5_sparse_collector_is_pinned_to_sglang_0514_dsa_api(): +def test_glm5_sparse_collector_uses_the_dsa_fuse_topk_switch_not_the_nsa_spelling(): + # the file pin itself is enforced by test_stock_collectors_are_exactly_pinned_to_their_resolved_runtime source = SOURCE_PATH.read_text(encoding="utf-8") - # ported to the 0.5.21 pin on 2026-10-01 (same DSA module API, verified in collector/sglang/runtime_compat.py); - # one release per file since 2026-10-30 - assert '__compat__ = "sglang==0.5.21"' in source assert "SGLANG_DSA_FUSE_TOPK" in source assert "SGLANG_NSA_FUSE_TOPK" not in source diff --git a/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py b/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py index b87e0318a2..b4a1b4db25 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_attn_kv_cache_layout.py @@ -13,9 +13,13 @@ import pytest -torch = pytest.importorskip("torch") -if not isinstance(getattr(torch, "__version__", None), str): # collector conftest stubs torch with a Mock - pytest.skip("real torch required (the helper is exercised on real tensors)", allow_module_level=True) +try: + import torch +except ImportError: + torch = None +if not isinstance(getattr(torch, "__version__", None), str): # absent, or the collector conftest's Mock stub + # reason string must match .github/application-test-inventory.json optional_collection_skips + pytest.skip("real torch required for tensor operations", allow_module_level=True) pytestmark = pytest.mark.unit _SRC = Path(__file__).resolve().parents[3] / "collector" / "vllm" / "collect_attn.py" @@ -41,27 +45,3 @@ def test_logical_view_is_unchanged_and_physical_order_is_contiguous(order): out = f(kv, order) assert out.shape == kv.shape and torch.equal(out, kv) # same logical [B, H, N, C] values assert out.permute(*order).is_contiguous() # what the backend's permute(*layer_view_order) needs - - -def test_identity_layout_returns_contiguous_input(): - f = _helper() - kv = torch.zeros(2, 3, 4, 6) - assert f(kv, LBHNC).data_ptr() == kv.data_ptr() - - -def test_collector_uses_the_resolved_layout(): - src = _SRC.read_text() - assert "kv_cache_in_resolved_layout(" in src - assert "get_resolved_kv_cache_layout().layer_view_order" in src - - -def test_flashinfer_decode_label_names_the_kernel(): - """FlashInferTrtllmAPIDecode carries XQA and trtllm-gen in one class; the label must keep the kernel.""" - src = _SRC.read_text() - assert 'kernel = getattr(phase_metadata, "kernel", None)' in src - vocab = (Path(__file__).resolve().parents[3] / "collector" / "kernel_source_backends.yaml").read_text() - for label in ( - "vllm_flashinfer_flashinfertrtllmapidecode_xqa", - "vllm_flashinfer_flashinfertrtllmapidecode_trtllm_gen", - ): - assert label in vocab diff --git a/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py b/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py index 952cbf97bd..56613cf309 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_collect_attn.py @@ -169,9 +169,6 @@ def benchmark(**kwargs): "benchmark_with_power": benchmark, "log_perf": lambda **kwargs: calls.setdefault("log", kwargs), "vllm_version": "0.24.0", - # per-SM page size (collect_attn:125 -> utils.kv_block_size); the test pins SM90's 16 - "get_sm_version": lambda: 90, - "kv_block_size": lambda _sm, _op: 16, # 1d0492eb: the page is asked from the framework per backend (collect_attn:201 # -> utils.framework_kv_block_size); the test pins the same 16 "framework_kv_block_size": lambda _backend_cls: 16, @@ -179,7 +176,8 @@ def benchmark(**kwargs): "get_supported_kv_cache_layouts": lambda _backends: [SimpleNamespace(name="LBHNC")], "resolve_kv_cache_layout": lambda _cfg, _layouts, _specs: None, "get_kv_cache_layout": lambda: SimpleNamespace(name="LBHNC", layer_view_order=(0, 1, 2, 3)), - "kv_cache_in_resolved_layout": lambda kv, _order: kv, + # spy: the paged KV must be laid out in the framework-RESOLVED layout order + "kv_cache_in_resolved_layout": lambda kv, order: (calls.setdefault("kv_layout_order", order), kv)[1], } _load_function(ATTN_SOURCE, "_dense_kernel_source", namespace) run = _load_function(ATTN_SOURCE, "run_attention_torch", namespace) @@ -202,6 +200,7 @@ def benchmark(**kwargs): assert calls["log"]["item_list"][0]["isl"] == 1 assert calls["log"]["item_list"][0]["step"] == 64 assert calls["log"]["kernel_source"] == "vllm_flash_attn_fa3" + assert calls["kv_layout_order"] == (0, 1, 2, 3) # cache_config.get_resolved_kv_cache_layout() assert [event[0] for event in events] == ["update", "update", "forward", "update", "forward"] assert events[0][5] is history_slot_mapping for update, forward in zip(events[1::2], events[2::2], strict=True): diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py index 7f21920022..485007b329 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py @@ -245,23 +245,7 @@ def test_ordinary_mla_does_not_publish_dsa_reuse_rows(): assert [row["op_name"] for row in published] == ["mla_context_module"] -@pytest.mark.parametrize("reuse_frequency", [1, 4]) -def test_each_case_prints_one_phase_timing_line(capsys, reuse_frequency): - execute, _, _, calls, _ = _module_runner(reuse_frequency=reuse_frequency) - execute() - lines = [line for line in capsys.readouterr().out.splitlines() if "module timing" in line] - assert len(lines) == 1 - (line,) = lines - assert line.startswith(" [DSA module timing] context b=2, s=16, prefix=128, heads=8, gemm=bfloat16, kv=fp8: ") - phases = ["module", "weights", "kv_metadata", "dry_run", "measure_full"] - if reuse_frequency == 4: - phases.append("measure_skip") - phases += ["indexer_fill", "total"] - assert [field.split("=")[0] for field in line.split(": ", 1)[1].split()] == phases - assert len(calls) == len([p for p in phases if p.startswith("measure")]) - - -def test_series_builds_the_module_once_and_measures_every_point_largest_first(capsys): +def test_series_builds_the_module_once_and_measures_every_point_largest_first(): execute, events, published, calls, _ = _module_runner(reuse_frequency=4) assert execute(seq_len=[8, 16, 8]) == [0.015, 0.015] assert events.count("workspace init") == 1 @@ -273,16 +257,9 @@ def test_series_builds_the_module_once_and_measures_every_point_largest_first(ca ("dsa_context_module_skip_indexer", 8), ] assert len(calls) == 4 and events[-1] == "cleanup" and events.count("cleanup") == 1 - out = capsys.readouterr().out - timing = [line for line in out.splitlines() if "module timing" in line] - assert len(timing) == 2 - assert "module=" in timing[0] and "weights=" in timing[0], "setup phases are reported once, on the first point" - assert "module=" not in timing[1] and "kv_metadata=" in timing[1] - series_lines = [line for line in out.splitlines() if "[DSA module series] context b=2, prefix=128" in line] - assert len(series_lines) == 1 and "2/2 points" in series_lines[0] -def test_series_sweeps_the_isl_x_prefix_grid_of_one_module_largest_total_context_first(capsys): +def test_series_sweeps_the_isl_x_prefix_grid_of_one_module_largest_total_context_first(): """A DSA context case carries its prefix list: one module, every (isl, prefix) point of the plan grid measured on it, largest total context first (module build was 0.9-1.2 s/pt under the campaign's 8 workers when it was amortised over one prefix only, 2026-10-29).""" @@ -297,10 +274,6 @@ def test_series_sweeps_the_isl_x_prefix_grid_of_one_module_largest_total_context if row["op_name"] == "dsa_context_module" ] assert rows == [(16, 128), (8, 128), (16, 0), (8, 0)] - out = capsys.readouterr().out - assert "[DSA module] context b=2, s=[16, 8], prefix=[128, 0]," in out - series_lines = [line for line in out.splitlines() if "[DSA module series] context b=2, prefix=[128, 0]" in line] - assert len(series_lines) == 1 and "4/4 points" in series_lines[0] assert len(calls) == 8 and events.count("cleanup") == 1 diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py index cf0d286df9..9b8fc6fc1f 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_dsv4_attn.py @@ -468,7 +468,7 @@ def test_constructor_args_match_the_serving_oracle(monkeypatch): @pytest.mark.parametrize( "bs, sl, is_ctx, prefix", - [(1, 1, True, 0), (16, 10240, True, 4096), (1024, 1, True, 0), (16, 1, False, 0), (128, 65536, False, 0)], + [(1, 1, True, 0), (16, 10240, True, 4096), (128, 65536, False, 0)], ) def test_kv_quota_is_capped_by_this_batch_s_tokens(monkeypatch, bs, sl, is_ctx, prefix): """Per-case KV pool: the byte quota stays the serving half-of-free bound, @@ -490,9 +490,7 @@ def test_kv_quota_is_capped_by_this_batch_s_tokens(monkeypatch, bs, sl, is_ctx, assert floor > 1_572_864 # H20 calibration: 1,048,576 tokens -> i32, 1,572,864 -> i64 -@pytest.mark.parametrize( - ("sm", "expected"), [(100, "bfloat16"), (103, "bfloat16"), (90, "fp8"), (120, "fp8"), (121, "fp8")] -) +@pytest.mark.parametrize(("sm", "expected"), [(100, "bfloat16"), (90, "fp8"), (120, "fp8")]) def test_serving_kv_cache_dtype_is_an_sdk_enum_member_name(monkeypatch, sm, expected): # engine_table_view rehydrates kv_cache_dtype through KVCacheQuantMode; "bf16" raised # KeyError on every SM100 DSV4 table in Full CI (2026-10-29). diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py index cb179bfb6e..406abb3d1b 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gdn_fail_closed.py @@ -11,7 +11,7 @@ collector's contract).""" import ast -from contextlib import contextmanager, nullcontext +from contextlib import contextmanager from pathlib import Path from types import SimpleNamespace @@ -37,14 +37,14 @@ def test_trtllm_gdn_benchmarks_never_fall_back_to_eager_timing(): for node in ast.walk(_tree()) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "benchmark_with_power" ] - assert len(calls) == 9 + assert calls + # allow_graph_fail is covered repo-wide by test_no_eager_graph_fallback_sites.py for call in calls: - keyword = next((kw for kw in call.keywords if kw.arg == "allow_graph_fail"), None) - assert keyword is None or not (isinstance(keyword.value, ast.Constant) and keyword.value.value is True) assert all(kw.arg != "use_cuda_graph" for kw in call.keywords) -@pytest.mark.parametrize("function_name", ["run_gdn_context_benchmark", "run_gdn_generation_benchmark"]) +# The context runner's final raise is exercised behaviourally below. +@pytest.mark.parametrize("function_name", ["run_gdn_generation_benchmark"]) def test_trtllm_gdn_phase_runners_end_in_a_strict_completeness_raise(function_name): function = _function(function_name) # A per-cell `continue` is only acceptable because the function re-raises at the end. @@ -145,25 +145,3 @@ def log_perf(**kwargs): assert "CUDA graph capture failed" in str(info.value) # the first cell shipped both of its rows; the failed cell shipped none assert logged == ["causal_conv1d_fn", "chunk_gated_delta_rule_flashinfer"] - - -def test_trtllm_gdn_context_all_cells_captured_returns_quietly(): - @contextmanager - def benchmark_with_power(**_kwargs): - yield {"latency_ms": 0.01, "power_stats": None, "used_cuda_graph": True} - - run = _load_context_runner(benchmark_with_power, lambda **_k: True) - with nullcontext(): - run( - d_model=2048, - d_conv=4, - num_k_heads=16, - head_k_dim=128, - num_v_heads=32, - head_v_dim=128, - batch_size_list=[1, 2], - seq_len_list=[32], - model_name="Qwen/Qwen3.5-test", - perf_filename="gdn_perf.txt", - trtllm_version="1.3.0rc29", - ) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py index 7f56e2797e..b433c1a503 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_mamba2_fail_closed.py @@ -45,22 +45,10 @@ def test_trtllm_mamba2_benchmarks_never_fall_back_to_eager_timing(): calls = [ node for node in ast.walk(_tree()) if isinstance(node, ast.Call) and _callee(node) == "benchmark_with_power" ] - assert len(calls) == 8 + assert calls + # allow_graph_fail is covered repo-wide by test_no_eager_graph_fallback_sites.py for call in calls: - assert all(keyword.arg not in ("allow_graph_fail", "use_cuda_graph") for keyword in call.keywords) - - -@pytest.mark.parametrize("function_name", ["run_mamba2_context_benchmark", "run_mamba2_generation_benchmark"]) -def test_trtllm_mamba2_phase_runners_end_in_a_strict_completeness_raise(function_name): - function = _function(function_name) - # A per-cell `continue` is only acceptable because the function re-raises at the end. - raises = [ - node - for node in function.body - if isinstance(node, ast.If) and any(isinstance(statement, ast.Raise) for statement in node.body) - ] - assert raises, f"{function_name} swallows cell failures without a final raise" - assert "strict completeness" in ast.unparse(raises[-1]) + assert all(keyword.arg != "use_cuda_graph" for keyword in call.keywords) def test_every_ssm_kernel_call_hands_over_the_caller_owned_out_buffer(): @@ -84,21 +72,6 @@ def test_every_ssm_kernel_call_hands_over_the_caller_owned_out_buffer(): assert any(keyword.arg == "state_batch_indices" for keyword in call.keywords), ast.unparse(call) -def test_generation_runner_never_calls_the_triton_state_update_directly(): - # The decode kernel comes from the serving gate, never from a hard-coded name. - direct = [ - node - for node in ast.walk(_function("run_mamba2_generation_benchmark")) - if isinstance(node, ast.Call) and _callee(node) == "selective_state_update" - ] - assert direct == [] - for literal in ("selective_state_update", "selective_state_update_flashinfer", "selective_state_update_triton"): - assert all( - not (isinstance(node, ast.Constant) and node.value == literal) - for node in ast.walk(_function("run_mamba2_generation_benchmark")) - ), f"{literal!r} is hard-coded in the generation runner" - - def _load_gate(): tree = _tree() body = [ @@ -117,7 +90,6 @@ def _load_gate(): [ (64, 8, 64, 128, True), # Nemotron-3 Nano / generic 4K: ratio 8 (256, 8, 64, 256, True), # Nemotron-H 56B: ratio 32, d_state 256 - (16, 4, 64, 64, True), # generic 1K: ratio 4 (rc29 widened the list; rc20 had [1, 8, 16]) (64, 8, 80, 128, False), # Nemotron-v2-Nano head_dim 80: no FlashInfer kernel (48, 16, 64, 128, False), # ratio 3 (64, 8, 64, 32, False), # d_state 32 diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py index f8769b6e4c..3a9a0e7e47 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_module_layer_cache.py @@ -65,22 +65,3 @@ def test_msa_layer_cache_keys_on_model_heads_gemm_and_device(): assert builds[0] == expected cached("MiniMaxAI/MiniMax-M3", 64, "fp8_block", "cuda:0") assert len(builds) == 2 and evictions == ["gc", "empty_cache"] - - -@pytest.mark.parametrize( - "filename,runner,helper", - [ - ("collect_mla_module.py", "run_mla_module", "_cached_attention_layer"), - ("collect_msa_module.py", "run_msa_module", "_cached_msa_attention_layer"), - ], -) -def test_runner_builds_its_layer_through_the_cache(filename, runner, helper): - tree = ast.parse((COLLECTOR / filename).read_text()) - body = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == runner) - calls = [ - n.func.id - for n in ast.walk(body) - if isinstance(n, ast.Call) and isinstance(n.func, ast.Name) and n.func.id.startswith(("create_", "_cached_")) - ] - assert helper in calls - assert not any(c in ("create_attention_layer", "create_msa_attention_layer") for c in calls) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py index 919f82bdb9..bbc60c27dc 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_ep.py @@ -434,10 +434,9 @@ def test_moe_ep_is_requested_independently_of_the_default_trtllm_plan(): # model plans on both backends so a default run resolves to one runtime. from collector.model_cases import build_collection_case_plan - for model in ("deepseek-ai/DeepSeek-V3", "moonshotai/Kimi-K2.5", "zai-org/GLM-5", "MiniMaxAI/MiniMax-M2.5"): - plan = build_collection_case_plan(backend="trtllm", model_path=model) - assert "moe_ep" not in plan.selected_ops, model - assert "trtllm_moe_wideep" not in plan.selected_ops + plan = build_collection_case_plan(backend="trtllm", model_path="deepseek-ai/DeepSeek-V3") + assert "moe_ep" not in plan.selected_ops + assert "trtllm_moe_wideep" not in plan.selected_ops sglang_plan = build_collection_case_plan(backend="sglang", model_path="deepseek-ai/DeepSeek-V3") assert "moe_ep" not in sglang_plan.selected_ops diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py index 492f9df7f1..6d546316dc 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_moe_worker_recycle.py @@ -75,7 +75,7 @@ def clean_env(monkeypatch): monkeypatch.delenv(name, raising=False) -def test_default_policy_recycles_after_sixteen_cases(clean_env, capsys): +def test_default_policy_recycles_after_sixteen_cases(clean_env): namespace, cases, gc_calls = _load() _run_cases(namespace, 15) assert len(cases) == 15 and len(gc_calls) == 15 @@ -83,49 +83,33 @@ def test_default_policy_recycles_after_sixteen_cases(clean_env, capsys): _run_cases(namespace, 1) assert caught.value.code == EXIT_CODE_RESTART assert len(cases) == 16, "the 16th case must still run (and persist) before the worker exits" - out = capsys.readouterr().out - assert "trtllm.moe: recycling worker after 16 case(s)" in out - assert "every 16" in out - assert out.count("MiB allocated, ") == 16, "one memory line per case, recycle or not" -def test_leak_guard_recycles_immediately_when_memory_stays_allocated(clean_env, capsys): +def test_leak_guard_recycles_immediately_when_memory_stays_allocated(clean_env): namespace, cases, _ = _load(allocated_mb=2048) with pytest.raises(SystemExit) as caught: _run_cases(namespace, 1) assert caught.value.code == EXIT_CODE_RESTART assert len(cases) == 1 - out = capsys.readouterr().out - assert "2048 MiB still allocated, leak guard" in out -def test_every_one_restores_the_legacy_per_case_recycle(clean_env, monkeypatch): - monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "1") - namespace, cases, _ = _load() - with pytest.raises(SystemExit) as caught: - _run_cases(namespace, 1) - assert caught.value.code == EXIT_CODE_RESTART and len(cases) == 1 - - -def test_legacy_switch_disables_recycling(clean_env, monkeypatch): - monkeypatch.setenv("TRTLLM_MOE_RESTART_WORKER", "0") - monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "1") - namespace, cases, _ = _load(allocated_mb=8192) - _run_cases(namespace, 40) - assert len(cases) == 40 - - -def test_zero_disables_the_count_and_the_guard(clean_env, monkeypatch): +def test_zero_disables_the_count_and_the_guard_and_the_legacy_switch_disables_recycling(clean_env, monkeypatch): monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "0") monkeypatch.setenv("TRTLLM_MOE_RESTART_LEAK_MB", "0") namespace, cases, gc_calls = _load(allocated_mb=8192) _run_cases(namespace, 50) assert len(cases) == 50 and len(gc_calls) == 50 + # the legacy master switch wins over any count and over the default leak guard + monkeypatch.delenv("TRTLLM_MOE_RESTART_LEAK_MB") + monkeypatch.setenv("TRTLLM_MOE_RESTART_WORKER", "0") + monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "1") + namespace, cases, _ = _load(allocated_mb=8192) + _run_cases(namespace, 40) + assert len(cases) == 40 -@pytest.mark.parametrize("name", ["TRTLLM_MOE_RESTART_EVERY", "TRTLLM_MOE_RESTART_LEAK_MB"]) -def test_negative_values_are_rejected(clean_env, monkeypatch, name): - monkeypatch.setenv(name, "-1") +def test_negative_values_are_rejected(clean_env, monkeypatch): + monkeypatch.setenv("TRTLLM_MOE_RESTART_EVERY", "-1") namespace, _, _ = _load() with pytest.raises(ValueError, match="must be >= 0"): _run_cases(namespace, 1) @@ -139,22 +123,6 @@ def test_wrapper_forwards_the_case_unchanged_and_recycles_after_it(clean_env, mo assert gc_calls == [True], "cleanup runs after the case body returned" -def test_case_body_no_longer_exits_on_its_own(): - tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) - body = next(node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "_run_moe_case") - exits = [ - node - for node in ast.walk(body) - if isinstance(node, ast.Call) - and isinstance(node.func, ast.Attribute) - and node.func.attr == "exit" - and isinstance(node.func.value, ast.Name) - and node.func.value.id == "sys" - ] - assert exits == [], "worker recycling belongs to run_moe_torch's policy, not the case body" - assert "TRTLLM_MOE_RESTART_WORKER" not in ast.get_source_segment(SOURCE_PATH.read_text(), body) - - def test_release_drops_every_trtllm_workspace_block_including_reserved(monkeypatch): import types @@ -189,12 +157,3 @@ def __init__(self, mib, reserved): monkeypatch.setitem(sys.modules, "tensorrt_llm._torch.memory_buffer_utils", utils) assert namespace[RELEASE]() == 7168 + 3584 + 28 assert pool.buffers == {} - - -def test_release_is_a_no_op_without_the_trtllm_module(monkeypatch): - tree = ast.parse(SOURCE_PATH.read_text(), filename=str(SOURCE_PATH)) - node = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == RELEASE) - namespace: dict = {} - exec(compile(ast.Module(body=[node], type_ignores=[]), str(SOURCE_PATH), "exec"), namespace) - monkeypatch.setitem(sys.modules, "tensorrt_llm", None) - assert namespace[RELEASE]() == 0 diff --git a/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py b/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py index dc222dd46f..7762971329 100644 --- a/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py +++ b/python/aisimulate/tests/unit/collector/vllm/test_collect_kda_contract.py @@ -267,27 +267,6 @@ def test_glm5_next_decode_mirrors_serving_decode(): assert ast.unparse(keywords["lower_bound"]) == "GLM5_NEXT_KDA_LOWER_BOUND" -@pytest.mark.parametrize( - ("version", "accepted"), - [ - ("0.1.dev19262+gb6bbf29dd", False), # Kimi-K3 preview pin retired 2026-09-28 - ("0.29.0", False), - ("0.30.0", False), # kda family moved to 0.31.0 (2026-10-30) - ("0.31.0", True), - ("0.31.1", False), - ], -) -def test_kda_compat_admits_kimi_and_glm_runtimes(version, accepted): - from collector.version_resolver import _check_compat - - declaration = next( - node.value.value - for node in ast.parse(SOURCE_PATH.read_text(encoding="utf-8")).body - if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "__compat__" for t in node.targets) - ) - assert _check_compat(declaration, version) is accepted - - _GATE_NAMES = { "GLM5_NEXT_KDA_MODEL_PATHS", "_is_glm5_next_kda", @@ -315,18 +294,6 @@ def _gate_namespace(): return namespace -def test_kda_architecture_gates_follow_the_file_pin(): - # One release per file: every audited architecture runs on the file's release. - compat = _gate_namespace()["_KDA_ARCHITECTURE_COMPAT"] - assert set(compat) == {"KimiK3ForConditionalGeneration", "Glm5NextForConditionalGeneration"} - file_pin = next( - node.value.value - for node in ast.parse(SOURCE_PATH.read_text(encoding="utf-8")).body - if isinstance(node, ast.Assign) and any(getattr(t, "id", None) == "__compat__" for t in node.targets) - ) - assert set(compat.values()) == {file_pin} == {"vllm==0.31.0"} - - @pytest.mark.parametrize( ("model_name", "version", "audited"), [ diff --git a/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py b/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py index b9c37c545e..e49a154cb6 100644 --- a/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py +++ b/python/aisimulate/tests/unit/collector/vllm/test_rope_missing_key_shim.py @@ -118,12 +118,6 @@ def forward(positions, query, key=None): assert key.tag == "q'" # the data was copied from the query -def test_throwaway_copy_of_an_empty_query_falls_back_to_clone(): - copy = _load_all()["_strided_throwaway_copy"] - key = copy(_Q("q", (0, 8, 64), (8 * 256, 256, 1))) - assert key.made_by == "clone" - - def test_present_key_passes_through_unchanged(): def forward(positions, query, key=None, offsets=None): return (query, key) @@ -156,6 +150,15 @@ def forward(positions, query, key=None): assert shim(once) is once and shim(shim(once)) is once assert getattr(once, "_aisim_rope_shim", False) is True + # ... and the installer, re-run on the same module for every case, keeps that one wrapper. + install = _load_all()["_install_rope_missing_key_shims"] + dsv32_rope = DeepseekScalingRotaryEmbedding() + attn = _Attn(rotary_emb=dsv32_rope) + install(attn) + wrapped = dsv32_rope.forward + install(attn) + assert dsv32_rope.forward is wrapped + # --- which rotary classes get the shim ------------------------------------------------- @@ -238,24 +241,6 @@ def test_installer_shims_deepseek_scaling_rope_and_leaves_glm_rope_on_servings_c assert (q.tag, k) == ("q", None) -def test_installer_is_idempotent_across_cases(): - ns = _load_all() - dsv32_rope = DeepseekScalingRotaryEmbedding() - attn = _Attn(rotary_emb=dsv32_rope) - ns["_install_rope_missing_key_shims"](attn) - once = dsv32_rope.forward - ns["_install_rope_missing_key_shims"](attn) - assert dsv32_rope.forward is once - - -def test_run_mla_module_installs_shims_through_the_class_filter_only(): +def test_run_mla_module_installs_shims_through_the_class_filter(): text = SOURCE.read_text(encoding="utf-8") assert "_install_rope_missing_key_shims(attn_module)" in text - # The old unconditional wrap of every *RotaryEmbedding submodule must not come back. - assert "_m.forward = _rope_tolerating_missing_key(_m.forward)" not in text - - -def test_fp8_block_guard_present_before_layer_construction(): - text = (Path(__file__).resolve().parents[4] / "collector" / "vllm" / "collect_gemm.py").read_text() - guard = text.index("FIXME(kernel-limit): vLLM fp8_block GEMM needs n and k multiples of 16") - assert guard < text.index("def create_gemm():") diff --git a/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py b/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py index c7b1167ef7..21bb3a6076 100644 --- a/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py +++ b/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py @@ -29,10 +29,8 @@ def _l40s(): def test_l40s_is_its_own_profile_and_no_phantom_l40_system(): assert hardware_key_for_system("l40s") == "l40s" - assert hardware_key_for_system("l40") == "l40" # not an SDK system: falls through, resolve_facts raises clearly hw = _l40s() assert hw["node_selector"]["nvidia.com/gpu.product"] == "NVIDIA-L40S" - assert hw["moe_backend"] == {"trtllm": {"nvfp4": "MARLIN"}} @pytest.mark.parametrize( @@ -43,7 +41,6 @@ def test_l40s_is_its_own_profile_and_no_phantom_l40_system(): ({"quant_method": "mxfp4"}, "mxfp4"), # gpt-oss ({"quant_method": "fp8", "weight_block_size": [128, 128]}, "fp8_block"), # DeepSeek-V3 class ({"quant_method": "fp8"}, "fp8"), - ({"quant_algo": "FP8"}, "fp8"), # modelopt MIXED_PRECISION: the experts' algo decides (nvidia/Qwen3.8-2.4T-A95B-NVFP4 layout) ( { diff --git a/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py b/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py index 78c2b2a1a7..766c232076 100644 --- a/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py +++ b/python/aisimulate/tests/unit/generator/test_facts_hardware_sm120.py @@ -25,7 +25,7 @@ def test_rtx_pro_6000_server_maps_to_the_sm120_profile(): assert hardware_key_for_system("not_a_system") == "not_a_system" -def test_sm120_profile_resolves_and_carries_no_wideep_moe_backend(): +def test_sm120_profile_resolves_and_model_defaults_reach_its_engine_args(): facts = resolve_facts( model_profile_id="deepseek-v4.1", hardware="sm120", @@ -33,20 +33,8 @@ def test_sm120_profile_resolves_and_carries_no_wideep_moe_backend(): dynamo_version=_latest_dynamo(), backend="vllm", ) - assert facts.hardware["arch"] == "x86_64" - assert facts.hardware["node_selector"]["nvidia.com/gpu.product"].startswith("NVIDIA-RTX-PRO-6000") # PCIe Blackwell: no wide-EP MoE backend is asserted for any backend assert "moe_backend" not in facts.hardware - - -def test_model_defaults_reach_sm120_engine_args(): - facts = resolve_facts( - model_profile_id="deepseek-v4.1", - hardware="sm120", - transport="nvlink", - dynamo_version=_latest_dynamo(), - backend="vllm", - ) tokens: list[str] = [] apply_model_default_args(tokens, facts.model, backend="vllm", system="sm120", role="agg", variant=None) assert tokens[tokens.index("--block-size") + 1] == "64" diff --git a/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py b/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py index f800497751..337054daa6 100644 --- a/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py +++ b/python/aisimulate/tests/unit/generator/test_model_profile_deepseek_v41.py @@ -17,8 +17,6 @@ "path,profile", [ ("deepseek-ai/DeepSeek-V4-Flash-0731", "deepseek-v4"), - ("deepseek-ai/DeepSeek-V4-Pro", "deepseek-v4"), - ("sgl-project/DeepSeek-V4-Flash-FP8", "deepseek-v4"), ("deepseek-ai/DeepSeek-V4.1-Flash", "deepseek-v4.1"), ("nvidia/DeepSeek-V4.1-Flash-NVFP4", "deepseek-v4.1"), ], diff --git a/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py b/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py index f277654b91..40b6992e86 100644 --- a/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py +++ b/python/aisimulate/tests/unit/generator/test_vllm_kda_decode_batch_cap.py @@ -50,13 +50,6 @@ def test_non_kda_model_keeps_the_512_floor(): assert pv["params"]["agg"]["max_batch_size"] == 512 -def test_small_requested_batch_is_not_raised_by_the_cap(): - pv = _pv(True, batch=64) - apply_rule_plugins(pv, backend="vllm") - # the 512 floor lifts 64 -> 512, the KDA cap brings it to 256 (never above the cap) - assert pv["params"]["agg"]["max_batch_size"] == 256 - - def test_preserve_engine_limits_disables_the_cap(): pv = _pv(True, batch=512, preserve=True) apply_rule_plugins(pv, backend="vllm") diff --git a/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py b/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py index a6fcc9723d..9940b328bf 100644 --- a/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py +++ b/python/aisimulate/tests/unit/generator/test_vllm_nvfp4_dsa_kv_cache_dtype.py @@ -24,7 +24,6 @@ "nvidia/DeepSeek-V3.2-NVFP4": "DeepseekV32ForCausalLM", "zai-org/GLM-5.3": "GlmMoeDsaForCausalLM", "nvidia/Llama-3.3-70B-Instruct-FP4": "LlamaForCausalLM", - "nvidia/DeepSeek-R1-0528-FP4": "DeepseekV3ForCausalLM", "local/unbundled": None, } @@ -71,16 +70,11 @@ def _stub_sdk_lookups(monkeypatch): # naive path (no quant mode): the artifact's fp8 KV fact, either spelling ("vllm", "nvidia/GLM-5.2-NVFP4", None, "fp8"), # config.json kv_cache_scheme ("vllm", "nvidia/GLM-5.3-NVFP4", None, "fp8"), # hf_quant kv_cache_quant_algo - ("vllm", "nvidia/DeepSeek-V3.2-NVFP4", None, "fp8"), ("vllm", "nvidia/GLM-5-NVFP4-unquant-bundle", None, None), # no KV fact anywhere: no prescription ("vllm", "zai-org/GLM-5.3", "fp8_block", None), # fp8 DSA artifact: auto is fine - ("vllm", "zai-org/GLM-5.3", None, None), ("vllm", "nvidia/Llama-3.3-70B-Instruct-FP4", "nvfp4", None), # NVFP4 but not DSA - ("vllm", "nvidia/Llama-3.3-70B-Instruct-FP4", None, None), - ("vllm", "nvidia/DeepSeek-R1-0528-FP4", "nvfp4", None), # plain MLA, not DSA ("vllm", "local/unbundled", "nvfp4", None), # unresolvable config: no prescription ("sglang", "nvidia/GLM-5.2-NVFP4", "nvfp4", None), # vllm-only spelling gap - ("trtllm", "nvidia/GLM-5.2-NVFP4", None, None), ], ) def test_prescription_keyed_on_dsa_architecture_and_an_artifact_fact(backend, model, gemm, expected): @@ -94,21 +88,6 @@ def test_artifact_kv_scheme_detection(): assert not utils._artifact_pins_fp8_kv(None) -class _FakeTask: - def __init__(self, backend, model): - self.primary_backend_name = backend - self.primary_model_path = model - - -def test_bridge_wrapper_passes_task_facts_through(): - # module_bridge imports the SDK (native runtime); skip where it is not built - bridge = pytest.importorskip("aisimulate.generator.module_bridge", exc_type=ImportError) - _vllm_dsa_kv_cache_dtype = bridge._vllm_dsa_kv_cache_dtype - - assert _vllm_dsa_kv_cache_dtype(_FakeTask("vllm", "nvidia/GLM-5.3-NVFP4"), "nvfp4") == "fp8" - assert _vllm_dsa_kv_cache_dtype(_FakeTask("vllm", "zai-org/GLM-5.3"), "fp8_block") is None - - def test_fp8_kv_dtype_renders_as_the_explicit_vllm_flag(): out = render_backend_parameters({"kv_cache_dtype": "fp8"}, "vllm") assert out["kv_cache_dtype"]["kv-cache-dtype"] == "fp8" From 57cebf4bd527d905eaadd39f5ab8f19e11d3c7af Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 20:55:55 +0800 Subject: [PATCH 08/13] fix: address the CodeRabbit review of the collector upgrade Collector runtime: - The post-failure context probe skips XPU devices; every XPU failure used to recycle its worker. - WorkerStallTimeout records carry the case's task_params and group, so --cases-from can replay a stalled case. - Resume finalization counts only cases that passed inside the open event (_done & _attempted), not the cumulative ledger. - The DSV4 subprocess wait beats the worker heartbeat every 60 s; the 1800 s stall watchdog would otherwise fail a healthy hour-long sweep. - collect.py refuses AIS_SM/AIC_SM: it is the case-inventory stand-in for a missing GPU, not a collection setting. SGLang: - The single-rank parallel override is back on run_attention_torch; it had slid onto default_attention_backend. No measured change: the offline runtime publishes TP/CP/DP 1. - The FlashInfer GDN prefill lane gathers the state rows inside the timed call, as FlashInferGDNKernel.extend does (gdn_flashinfer.py:523-541). - KDA decode drops the source probe and the packed lane: at the pinned 0.5.21 a lower-bound layer always takes the plain lane (kda_backend.py:737-742), and Kimi-K3 always has one. - mHC and MLA modules reserve a free NCCL port (free_local_port) instead of a random guess. vLLM: - AIS_KDA_DECODE_PATHS rejects unknown or unrunnable selections instead of finishing "ok" with zero rows. - The AIS_DSA_LEGACY_MODULE A/B switch (a 0.29 comparison) and the AIS_KV_BLOCK_SIZE override are removed; neither may change rows unseen. - gpt-oss MXFP4 imports GptOssMxfp4Config unconditionally (mxfp4.py:110 @0.31.0). - The fp8_block n/k % 16 guard applies from SM90, where the cutlass block tier exists; SM89 selects Marlin first. Re-cited at 0.31.0. Generator and tools: - trtllm renders load the bundled quantization only for MoE models whose hardware moe_backend is quantization-keyed. - collect_campaign counts a failed shard whose GPU slot is reused at once. - test_error_classification imports collect without a global torch mock. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- python/aisimulate/collector/collect.py | 18 ++- python/aisimulate/collector/helper.py | 2 + .../collector/sglang/collect_attn.py | 16 +-- .../collector/sglang/collect_dsv4_attn.py | 22 +++- .../collector/sglang/collect_gdn.py | 5 +- .../collector/sglang/collect_kda.py | 121 ++++++------------ .../collector/sglang/collect_mhc_module.py | 9 +- .../collector/sglang/collect_mla_module.py | 6 +- .../collector/sglang/runtime_compat.py | 10 ++ .../aisimulate/collector/vllm/collect_gemm.py | 8 +- .../aisimulate/collector/vllm/collect_kda.py | 7 +- .../collector/vllm/collect_mla_module.py | 9 +- .../aisimulate/collector/vllm/collect_moe.py | 14 +- python/aisimulate/collector/vllm/utils.py | 7 +- .../config/backend_config_mapping.yaml | 7 +- .../aisimulate/generator/rendering/engine.py | 13 +- .../collector/test_error_classification.py | 16 +-- .../collector/test_vllm_mla_graph_timing.py | 32 +++-- .../tools/perf_database/collect_campaign.py | 7 +- 19 files changed, 155 insertions(+), 174 deletions(-) diff --git a/python/aisimulate/collector/collect.py b/python/aisimulate/collector/collect.py index f5b02e18cd..4f4f846e32 100644 --- a/python/aisimulate/collector/collect.py +++ b/python/aisimulate/collector/collect.py @@ -2092,16 +2092,18 @@ def _dump_and_kill_stalled_workers(stalled_for: float, *, stall_round: int, clos for i, p in live: active_task_id = current_task_ids.get(i) if active_task_id is not None and active_task_id not in accounted: + # same task_params string a worker-side failure stores, so --cases-from can replay it + params = next((t["params"] for t in task_infos if t["id"] == active_task_id), None) errors.append( { "module": module_name, "device_id": i, "task_id": active_task_id, - "task_params": None, + "task_params": None if params is None else str(params), "error_type": "WorkerStallTimeout", "error_message": f"worker made no progress for {int(stalled_for)} s; stacks dumped via SIGUSR1", "classification": "unexpected", - "group": None, + "group": None if params is None else _failure_group(params), "traceback": "", "timestamp": datetime.now().isoformat(), } @@ -3279,7 +3281,11 @@ def _pending_resume_perf_outputs( context="resume finalization", ) has_pending_attempts = has_pending_attempts or bool(resume_tracker and resume_tracker._attempted) - has_completed_cases = has_completed_cases or bool(resume_tracker and resume_tracker._done) + # _done is cumulative across invocations; only a case that passed inside the open + # event (attempted there) can have staged rows + has_completed_cases = has_completed_cases or bool( + resume_tracker and resume_tracker._done & resume_tracker._attempted + ) if has_pending_attempts: if perf_path.is_symlink() or not perf_path.is_file(): if not has_completed_cases: @@ -6425,6 +6431,12 @@ def main(): else: os.environ.pop("COLLECTOR_MODEL_PATH", None) + # helper.get_sm_version lets AIS_SM stand in for a missing GPU when enumerating cases; + # under a collection it would dispatch and label every case for that SM instead. + forced_sm = os.environ.get("AIS_SM") or os.environ.get("AIC_SM") + if forced_sm: + parser.error(f"AIS_SM/AIC_SM={forced_sm} is a case-inventory override; unset it to collect") + # Setup logging - debug flag is handled inside setup_logging global _LOGGER_CONFIGURED if not _LOGGER_CONFIGURED: diff --git a/python/aisimulate/collector/helper.py b/python/aisimulate/collector/helper.py index 605cdbdbfa..8b29efd589 100644 --- a/python/aisimulate/collector/helper.py +++ b/python/aisimulate/collector/helper.py @@ -697,6 +697,8 @@ def cuda_context_error(device=None) -> str | None: synchronises the device and launches one trivial kernel; it returns the error text when either raises, ``None`` when the context is healthy or torch is absent. """ + if str(device).startswith("xpu"): + return None # XPU workers run the same failure path; their context is not CUDA's try: import torch except ImportError: diff --git a/python/aisimulate/collector/sglang/collect_attn.py b/python/aisimulate/collector/sglang/collect_attn.py index 75c5c04fb2..2e48fa90ba 100644 --- a/python/aisimulate/collector/sglang/collect_attn.py +++ b/python/aisimulate/collector/sglang/collect_attn.py @@ -444,14 +444,6 @@ def get_generation_attention_test_cases(): return test_cases -@get_parallel().override( - attn_tp_size=1, - attn_tp_rank=0, - attn_cp_size=1, - attn_cp_rank=0, - attn_dp_size=1, - attn_dp_rank=0, -) def default_attention_backend(sm_version: int, has_attention_sink: bool) -> str | None: """SGLang's default dense-attention backend per SM (the table run_attention_torch documents above its call site); None for SMs outside the supported set.""" @@ -518,6 +510,14 @@ def _ensure_flashinfer_workspace() -> int | None: return None +@get_parallel().override( + attn_tp_size=1, + attn_tp_rank=0, + attn_cp_size=1, + attn_cp_rank=0, + attn_dp_size=1, + attn_dp_rank=0, +) def run_attention_torch( batch_size, input_len, diff --git a/python/aisimulate/collector/sglang/collect_dsv4_attn.py b/python/aisimulate/collector/sglang/collect_dsv4_attn.py index 9a20301d79..107c924a8e 100644 --- a/python/aisimulate/collector/sglang/collect_dsv4_attn.py +++ b/python/aisimulate/collector/sglang/collect_dsv4_attn.py @@ -42,6 +42,7 @@ import subprocess import sys import tempfile +import time import traceback from collections.abc import Iterable from fractions import Fraction @@ -64,6 +65,7 @@ benchmark_with_power, log_perf, render_phase_totals, + worker_heartbeat, ) except ModuleNotFoundError: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -74,6 +76,7 @@ benchmark_with_power, log_perf, render_phase_totals, + worker_heartbeat, ) try: @@ -1820,11 +1823,20 @@ def _run_once(nccl_port: int) -> tuple[int, str]: stderr=subprocess.STDOUT, cwd=os.path.dirname(os.path.abspath(__file__)), ) - try: - proc.wait(timeout=3600) # up to 1 hour per (kind, tp, gemm, bs) - except subprocess.TimeoutExpired: - proc.kill() - proc.wait() + # Up to 1 hour per (kind, tp, gemm, bs). The child reports no liveness of its + # own, so beat while waiting: the executor's stall watchdog + # (STALL_TIMEOUT_SEC, 1800 s default) would otherwise fail a healthy sweep. + deadline = time.monotonic() + 3600 + while True: + try: + proc.wait(timeout=60) + break + except subprocess.TimeoutExpired: + if time.monotonic() >= deadline: + proc.kill() + proc.wait() + break + worker_heartbeat() try: with open(log_path, encoding="utf-8", errors="replace") as logf: diff --git a/python/aisimulate/collector/sglang/collect_gdn.py b/python/aisimulate/collector/sglang/collect_gdn.py index c2a9cc2e6d..b47c699e39 100644 --- a/python/aisimulate/collector/sglang/collect_gdn.py +++ b/python/aisimulate/collector/sglang/collect_gdn.py @@ -362,10 +362,11 @@ def run_gdn_scan(): prefill_fn, qkv_prepare = flashinfer_prefill cu_seqlens_i64 = cu_seqlens.to(torch.int64) state_rows = state_indices.to(torch.int64) - initial_state_fi = recurrent_state[state_rows].to(torch.float32) - output_state_fi = torch.empty_like(initial_state_fi) def run_gdn_scan(): + # gather + output buffer per call, as extend does (:523-541) + initial_state_fi = recurrent_state[state_rows].to(torch.float32) + output_state_fi = torch.empty_like(initial_state_fi) q_fi, k_fi, v_fi = qkv_prepare(q[0], k[0], v[0]) prefill_fn( q=q_fi, diff --git a/python/aisimulate/collector/sglang/collect_kda.py b/python/aisimulate/collector/sglang/collect_kda.py index 5c5156a02d..c4238cc7a0 100644 --- a/python/aisimulate/collector/sglang/collect_kda.py +++ b/python/aisimulate/collector/sglang/collect_kda.py @@ -17,7 +17,8 @@ Generation (decode) phase (kda_backend.forward_decode): - causal_conv1d_update: packed single-step conv over 3P channels - - fused_recurrent_kda_packed_decode: packed KDA recurrence (T=1) + - fused_sigmoid_gating_delta_rule_update: KDA recurrence with lower_bound (T=1; + the packed kernel is skipped for lower-bound layers, kda_backend.py:737-742) - "kda_fused_decode" where serving's attempt-and-verify covered() probe accepts the shard (see _fused_decode_module below) @@ -71,9 +72,6 @@ from typing import TYPE_CHECKING if TYPE_CHECKING: - from sglang.kernels.ops.attention.fla.fused_recurrent import ( - fused_recurrent_kda_packed_decode, - ) from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) @@ -381,30 +379,6 @@ def _fused_decode_module(): return kda_fused_decode -def _serving_decode_lane_for_lower_bound() -> str: - """Which recurrence kernel serving runs for a KDA layer that carries a - ``lower_bound`` (Kimi-K3: KDA_LOWER_BOUND). - - sglang>=0.5.21 kda_backend.KDAAttnBackend.forward_decode:737-741 takes the - packed Triton kernel only when ``supports_packed_decode`` AND the layer has - no lower_bound; a lower-bound layer goes through kernel_dispatcher.decode - (:771-786) = TritonKDAKernel.decode -> fused_sigmoid_gating_delta_rule_update - with lower_bound (kernels/kda_triton.py:125-157). The kimi-k3 branch build - (0.5.16) packed every T=1 decode. Probed from the backend source rather than - the version string so a backport is followed too. H20 serving golden - 2026-10-01 (archive raw 7ef3a64cae10 decode_kernels) carries - fused_sigmoid_gating_delta_rule_update_kernel, not the packed kernel. - """ - try: - import inspect - - from sglang.srt.layers.attention.linear.kda_backend import KDAAttnBackend - except ImportError: - return "packed" - src = inspect.getsource(KDAAttnBackend.forward_decode) - return "plain" if 'getattr(layer, "lower_bound", None) is None' in src else "packed" - - def run_kda_generation_benchmark( d_model: int, d_conv: int, @@ -441,7 +415,7 @@ def run_kda_generation_benchmark( print(f" Benchmarking batch_size={batch_size}") a = a_log = b = conv_pool = dt_bias = mixed_qkv = None - output = recurrent_state = state_indices = None + recurrent_state = state_indices = None try: num_warmups = 3 num_runs = 10 @@ -456,7 +430,6 @@ def run_kda_generation_benchmark( b = torch.randn(batch_size, num_v_heads, dtype=dtype, device=device) a_log = torch.zeros(num_v_heads, dtype=torch.float32, device=device) dt_bias = torch.ones(num_v_heads * head_k_dim, dtype=torch.float32, device=device) - output = torch.empty(batch_size, 1, num_v_heads, head_v_dim, dtype=dtype, device=device) common_log_data = { "phase": "generation", @@ -578,61 +551,43 @@ def run_conv1d_update(): ): raise RuntimeError(f"failed to persist SGLang KDA generation row to {perf_filename}") - if _serving_decode_lane_for_lower_bound() == "plain" and KDA_LOWER_BOUND is not None: - # kda_backend.py:771-786 @0.5.21: after the conv update, q/k/v are the - # [Q|K|V] split viewed as 1 x n x h x d, and the plain Triton decode runs - # fused_sigmoid_gating_delta_rule_update over cu_seqlens with lower_bound. - recurrence_kernel_source = "fused_sigmoid_gating_delta_rule_update" - plain_q, plain_k, plain_v = ( - t.unflatten(-1, (-1, dim)).unsqueeze(0) - for t, dim in zip( - mixed_qkv.split([proj_size] * 3, dim=-1), (head_k_dim, head_k_dim, head_v_dim), strict=True - ) + # kda_backend.py:737-742 @0.5.21 skips the packed T=1 kernel for a layer with a + # lower_bound (Kimi-K3: KDA_LOWER_BOUND); :771-786: after the conv update, q/k/v are the + # [Q|K|V] split viewed as 1 x n x h x d, and the plain Triton decode runs + # fused_sigmoid_gating_delta_rule_update over cu_seqlens with lower_bound. + recurrence_kernel_source = "fused_sigmoid_gating_delta_rule_update" + plain_q, plain_k, plain_v = ( + t.unflatten(-1, (-1, dim)).unsqueeze(0) + for t, dim in zip( + mixed_qkv.split([proj_size] * 3, dim=-1), (head_k_dim, head_k_dim, head_v_dim), strict=True ) - plain_a = a.view(1, batch_size, num_v_heads, head_k_dim) - plain_b = b.view(1, batch_size, num_v_heads) - plain_cu_seqlens = torch.arange(0, batch_size + 1, dtype=torch.int32, device=device) - - def run_kda_packed_decode(): - fused_sigmoid_gating_delta_rule_update( - A_log=a_log, - dt_bias=dt_bias, - q=plain_q, - k=plain_k, - v=plain_v, - a=plain_a, - b=plain_b, - initial_state_source=recurrent_state, - initial_state_indices=state_indices, - cu_seqlens=plain_cu_seqlens, - use_qk_l2norm_in_kernel=True, - softplus_beta=1.0, - softplus_threshold=20.0, - is_kda=True, - lower_bound=KDA_LOWER_BOUND, - ) - - else: - recurrence_kernel_source = "fused_recurrent_kda_packed_decode" + ) + plain_a = a.view(1, batch_size, num_v_heads, head_k_dim) + plain_b = b.view(1, batch_size, num_v_heads) + plain_cu_seqlens = torch.arange(0, batch_size + 1, dtype=torch.int32, device=device) - def run_kda_packed_decode(): - fused_recurrent_kda_packed_decode( - mixed_qkv=mixed_qkv, - a=a, - b=b, - A_log=a_log, - dt_bias=dt_bias, - scale=head_k_dim**-0.5, - initial_state=recurrent_state, - out=output, - ssm_state_indices=state_indices, - use_qk_l2norm_in_kernel=True, - lower_bound=KDA_LOWER_BOUND, - ) + def run_kda_decode(): + fused_sigmoid_gating_delta_rule_update( + A_log=a_log, + dt_bias=dt_bias, + q=plain_q, + k=plain_k, + v=plain_v, + a=plain_a, + b=plain_b, + initial_state_source=recurrent_state, + initial_state_indices=state_indices, + cu_seqlens=plain_cu_seqlens, + use_qk_l2norm_in_kernel=True, + softplus_beta=1.0, + softplus_threshold=20.0, + is_kda=True, + lower_bound=KDA_LOWER_BOUND, + ) with benchmark_with_power( device=device, - kernel_func=run_kda_packed_decode, + kernel_func=run_kda_decode, num_warmups=num_warmups, num_runs=num_runs, repeat_n=1, @@ -657,7 +612,7 @@ def run_kda_packed_decode(): continue finally: a = a_log = b = conv_pool = dt_bias = mixed_qkv = None - output = recurrent_state = state_indices = None + recurrent_state = state_indices = None _cleanup("generation") summary = f"ok={successful_points} error={failed_points} skip=0" @@ -1402,9 +1357,6 @@ def run_kda_torch( contextlib.redirect_stdout(_devnull_file), contextlib.redirect_stderr(_devnull_file), ): - from sglang.kernels.ops.attention.fla.fused_recurrent import ( - fused_recurrent_kda_packed_decode, - ) from sglang.kernels.ops.attention.fla.fused_sigmoid_gating_recurrent import ( fused_sigmoid_gating_delta_rule_update, ) @@ -1418,7 +1370,6 @@ def run_kda_torch( "causal_conv1d_fn": causal_conv1d_fn, "causal_conv1d_update": causal_conv1d_update, "chunk_kda": chunk_kda, - "fused_recurrent_kda_packed_decode": fused_recurrent_kda_packed_decode, "fused_sigmoid_gating_delta_rule_update": fused_sigmoid_gating_delta_rule_update, } ) diff --git a/python/aisimulate/collector/sglang/collect_mhc_module.py b/python/aisimulate/collector/sglang/collect_mhc_module.py index 9685d3fbf8..1224d7ebd9 100644 --- a/python/aisimulate/collector/sglang/collect_mhc_module.py +++ b/python/aisimulate/collector/sglang/collect_mhc_module.py @@ -20,7 +20,6 @@ import gc import json import os -import random import sys import tempfile from collections.abc import Sequence @@ -293,8 +292,9 @@ def _load_one_layer_runner( _set_envs_and_config(server_args) model_config = ModelConfig.from_server_args(server_args) - nccl_port = 29500 + random.randint(0, 10000) - from collector.sglang.runtime_compat import init_runtime_config + from collector.sglang.runtime_compat import free_local_port, init_runtime_config + + nccl_port = free_local_port() _runner_parallel_kwargs = init_runtime_config(server_args, gpu_id, nccl_port=nccl_port, model_config=model_config) return ModelRunner( @@ -596,6 +596,7 @@ def _init_glm5next_runtime(model_path: str, *, device: str, mem_fraction_static: (model_runner.py L456-458). No model weights are loaded: the mHC kernels only need this process state plus the layer's own parameters. """ + from collector.sglang.runtime_compat import free_local_port from sglang.srt.configs.model_config import ModelConfig from sglang.srt.distributed import bootstrap from sglang.srt.distributed.parallel_state_wrapper import ParallelState @@ -638,7 +639,7 @@ def _init_glm5next_runtime(model_path: str, *, device: str, mem_fraction_static: model_config=model_config, device="cuda", ps=ParallelState.trivial(gpu_id=gpu_id), - dist_port=29500 + random.randint(0, 10000), + dist_port=free_local_port(), is_draft_worker=False, local_omp_cpuid=None, ) diff --git a/python/aisimulate/collector/sglang/collect_mla_module.py b/python/aisimulate/collector/sglang/collect_mla_module.py index 307a56c817..a6809a4c9d 100644 --- a/python/aisimulate/collector/sglang/collect_mla_module.py +++ b/python/aisimulate/collector/sglang/collect_mla_module.py @@ -1158,8 +1158,6 @@ def load_model_runner( "Marlin is valid only for INT4-WO" ) - import random - from sglang.srt.configs.model_config import ModelConfig from sglang.srt.entrypoints.engine import _set_envs_and_config from sglang.srt.model_executor.model_runner import ModelRunner @@ -1237,7 +1235,9 @@ def load_model_runner( _set_envs_and_config(server_args) - nccl_port = 29500 + random.randint(0, 10000) + gpu_id * 100 + from collector.sglang.runtime_compat import free_local_port + + nccl_port = free_local_port() model_config = ModelConfig.from_server_args(server_args) expected_architecture = _module_model_architecture(model_path) diff --git a/python/aisimulate/collector/sglang/runtime_compat.py b/python/aisimulate/collector/sglang/runtime_compat.py index 3f51dc38a0..c4e9b9344b 100644 --- a/python/aisimulate/collector/sglang/runtime_compat.py +++ b/python/aisimulate/collector/sglang/runtime_compat.py @@ -86,6 +86,16 @@ def init_runtime_config( return {} +def free_local_port() -> int: + """A port the OS reports free right now: an arithmetic guess can collide between + concurrent per-GPU workers, and the collision surfaces as a framework failure.""" + import socket + + with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + return sock.getsockname()[1] + + _OFFLINE_PUBLISHED = False diff --git a/python/aisimulate/collector/vllm/collect_gemm.py b/python/aisimulate/collector/vllm/collect_gemm.py index e0596c7b80..35d47f6d26 100644 --- a/python/aisimulate/collector/vllm/collect_gemm.py +++ b/python/aisimulate/collector/vllm/collect_gemm.py @@ -206,15 +206,17 @@ def get_gemm_test_cases(): @with_exit_stack def run_gemm(exit_stack, gemm_type, m, n, k, *, perf_filename, device="cuda:0"): - if gemm_type == "fp8_block" and (n % 16 or k % 16): + if gemm_type == "fp8_block" and (n % 16 or k % 16) and get_sm_version() >= 90: # FIXME(kernel-limit): vLLM routes block-fp8 GEMMs whose weight dims are not multiples # of 16 away from cutlass (_custom_ops.cutlass_scaled_mm: `cutlass_compatible_b = - # b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0` @0.30.0:819) into triton_scaled_mm, + # b.shape[0] % 16 == 0 and b.shape[1] % 16 == 0` @0.31.0:807) into triton_scaled_mm, # whose scale-shape assert cannot take 128x128 block scales (triton_scaled_mm.py:201). # Deterministic per shape: gb200 1,110 cases, every one N in {1,4,8,12,24} (pipeline # 72343556; b200/b300 ~130 of the same shard). The ed9109ec attempt to classify by a # pre-flight forward never ran: create_gemm()'s own dry run asserts first. Refuse before - # the layer is built; serving linears with these N take the same path. + # the layer is built; serving linears with these N take the same path. SM89 never + # reaches it: the cutlass block tier needs SM90 and Marlin is selected first + # (_POSSIBLE_FP8_BLOCK_KERNELS, model_executor/kernels/linear/__init__.py:458-467 @0.31.0). raise RuntimeError( f"FIXME(kernel-limit): vLLM fp8_block GEMM needs n and k multiples of 16 for cutlass_scaled_mm " f"(n={n}, k={k}); the triton_scaled_mm fallback asserts on 128x128 block scales" diff --git a/python/aisimulate/collector/vllm/collect_kda.py b/python/aisimulate/collector/vllm/collect_kda.py index d8cf56c35b..d3c58dc506 100644 --- a/python/aisimulate/collector/vllm/collect_kda.py +++ b/python/aisimulate/collector/vllm/collect_kda.py @@ -102,7 +102,10 @@ def _decode_paths() -> set[str]: decoding). The opharness kda decode gate captures serving WITHOUT spec decoding, so it sets AIS_KDA_DECODE_PATHS=fused to compare like with like (recorded in the capture's env). Collection runs never set it.""" - return {p.strip() for p in os.environ.get("AIS_KDA_DECODE_PATHS", "fused,packed").split(",") if p.strip()} + paths = {p.strip() for p in os.environ.get("AIS_KDA_DECODE_PATHS", "fused,packed").split(",") if p.strip()} + if not paths or not paths <= {"fused", "packed"}: + raise ValueError(f"AIS_KDA_DECODE_PATHS must name fused and/or packed, got {sorted(paths)}") + return paths NUM_WARMUPS = 3 @@ -506,6 +509,8 @@ def run_kda_generation_benchmark( fused_ok = is_fused_kda_decode_supported( nh, hd, cw, num_spec=0, input_dtype=dtype, conv_state_dtype=dtype, recurrent_state_dtype=torch.float32 ) + if decode_paths == {"fused"} and not fused_ok: + raise RuntimeError("AIS_KDA_DECODE_PATHS=fused, but the fused KDA decode kernel does not support this shard") conv_weight = torch.randn(3 * dim, cw, dtype=torch.float32, device=device) fused_weight = conv_weight.reshape(3, dim, cw).transpose(1, 2).contiguous() norm_weight = torch.randn(hd, dtype=torch.float32, device=device) diff --git a/python/aisimulate/collector/vllm/collect_mla_module.py b/python/aisimulate/collector/vllm/collect_mla_module.py index 07602dae10..400a0e3637 100644 --- a/python/aisimulate/collector/vllm/collect_mla_module.py +++ b/python/aisimulate/collector/vllm/collect_mla_module.py @@ -949,9 +949,8 @@ def _create_attention_module( # (MLA backends only support bfloat16, not float32). from vllm.utils.torch_utils import set_default_torch_dtype - use_legacy = bool(os.environ.get("AIS_DSA_LEGACY_MODULE") or os.environ.get("AIC_DSA_LEGACY_MODULE")) with set_current_vllm_config(vllm_config), set_default_torch_dtype(vllm_config.model_config.dtype): - if attn_type == "dsa" and not use_legacy: + if attn_type == "dsa": # GENERIC serving-parity construction (owner rule 2026-09-19: # config in -> the module SERVING would build comes out; never a # per-model class import — that is how the prelude-fusion drift @@ -984,12 +983,6 @@ def _create_attention_module( if getattr(attn_module, "topk_indices_buffer", None) is not None: attn_module.topk_indices_buffer = topk_indices_buffer else: - if attn_type == "dsa" and use_legacy: - print( - " [AB-ONLY] AIS_DSA_LEGACY_MODULE=1: building the LEGACY generic " - "DeepseekV2MLAAttention — NOT serving-parity on 0.29; rows are for " - "comparison only, never for the database" - ) attn_module = DeepseekV2MLAAttention( vllm_config=vllm_config, config=hf_config, diff --git a/python/aisimulate/collector/vllm/collect_moe.py b/python/aisimulate/collector/vllm/collect_moe.py index 041037d244..9acdefca37 100644 --- a/python/aisimulate/collector/vllm/collect_moe.py +++ b/python/aisimulate/collector/vllm/collect_moe.py @@ -647,21 +647,17 @@ def gemma4_routing_function(hidden_states, gating_output, topk, renormalize): # matmul_ogs on SM90, as the gpt-oss-120b serving probe shows — # while the generic Mxfp4Config resolves to Mxfp4MoEMethod -> MARLIN # (path_diff moe_mxfp4 vs gpt-oss-120b was diverged on exactly this, - # 2026-09-23). Mirror the framework's own override here. A version - # without the class performs no override, so Mxfp4Config is that - # version's serving truth — this is dispatch, not a fallback. + # 2026-09-23). Mirror the framework's own override here + # (GptOssMxfp4Config, quantization/mxfp4.py:110 @0.31.0). model_cfg = _load_model_moe_config(model_name) quant_config = Mxfp4Config() if ( checkpoint_qc.get("quant_method") in ("mxfp4", "gpt_oss_mxfp4") and model_cfg.get("model_type") == "gpt_oss" ): - try: - from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4Config - except ImportError: # pre-override vLLM: no gpt_oss_mxfp4 method exists - GptOssMxfp4Config = None # noqa: N806 - if GptOssMxfp4Config is not None: - quant_config = GptOssMxfp4Config() + from vllm.model_executor.layers.quantization.mxfp4 import GptOssMxfp4Config + + quant_config = GptOssMxfp4Config() elif moe_type == "w4a8_mxfp4_mxfp8": # Native DeepSeek-V4 (expert_dtype=fp4) serving path: vLLM overrides # the checkpoint's fp8 quant_method to DeepseekV4FP8Config diff --git a/python/aisimulate/collector/vllm/utils.py b/python/aisimulate/collector/vllm/utils.py index 4f62b6e478..2e9880ab29 100644 --- a/python/aisimulate/collector/vllm/utils.py +++ b/python/aisimulate/collector/vllm/utils.py @@ -669,13 +669,8 @@ def framework_kv_block_size(backend_cls, *, user_block_size: int | None = None) equivalent: with mla=32 the bf16 DSA cell selected the TRT-LLM FMHA sparse cubin while serving ran FA-cute (page 64) — the page is a per-(backend, kv dtype, head shape) decision, so it must come from the backend the case - actually selected. AIS_KV_BLOCK_SIZE stays the single-variable A/B hook. + actually selected. """ - import os - - override = os.environ.get("AIS_KV_BLOCK_SIZE") - if override: - return int(override) from vllm.config.cache import CacheConfig from vllm.v1.worker.utils import select_common_block_size diff --git a/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml b/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml index 6e1f8f3eb4..d287fe2726 100644 --- a/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml +++ b/python/aisimulate/src/aisimulate/generator/config/backend_config_mapping.yaml @@ -284,9 +284,10 @@ parameters: sglang: null trtllm: sparse_attention_config.algorithm -# sglang >= 0.5.20 prefill CUDA-graph backend ({full,breakable,tc_piecewise,disabled}); -# mixed-chunk prefill requires `breakable` on models whose backend enables the prefill -# graph (MiniMax-M3 at 0.5.21: prefill_cuda_graph_runner.py:293). CLI only on sglang. +# sglang >= 0.5.20 prefill CUDA-graph backend ({full,breakable,tc_piecewise,disabled}). +# Rendered only when a deployment sets it: `breakable` is already the CUDA default +# (runner/prefill_cuda_graph_runner.py:17 @0.5.21), which mixed-chunk prefill requires +# (:291-296). CLI only on sglang. - param_key: cuda_graph_backend_prefill vllm: null sglang: cuda-graph-backend-prefill diff --git a/python/aisimulate/src/aisimulate/generator/rendering/engine.py b/python/aisimulate/src/aisimulate/generator/rendering/engine.py index 5f88847833..8873560e27 100644 --- a/python/aisimulate/src/aisimulate/generator/rendering/engine.py +++ b/python/aisimulate/src/aisimulate/generator/rendering/engine.py @@ -379,14 +379,15 @@ def render_backend_templates( from aisimulate.generator.facts.apply import apply_moe_backend from aisimulate.generator.utils import _bundled_quantization + hardware = getattr(resolved_facts, "hardware", None) # the artifact's quantization decides the MoE kernel family on SMs where one value cannot - # serve every quant (hardware.yaml l40s); None for unquantized / unresolvable checkpoints - apply_moe_backend( - context, - getattr(resolved_facts, "hardware", None), - backend=backend, - quantization=_bundled_quantization(str(context.get("model_path") or "")), + # serve every quant (hardware.yaml l40s); None for unquantized / unresolvable checkpoints. + # Loaded only for that case: it may fetch the config from the Hub. + keyed = isinstance(((hardware or {}).get("moe_backend") or {}).get(backend), dict) + quantization = ( + _bundled_quantization(str(context.get("model_path") or "")) if keyed and context.get("is_moe") else None ) + apply_moe_backend(context, hardware, backend=backend, quantization=quantization) # Find template files template_path = Path(templates_dir) diff --git a/python/aisimulate/tests/unit/collector/test_error_classification.py b/python/aisimulate/tests/unit/collector/test_error_classification.py index b7a92080b8..fd6d11263b 100644 --- a/python/aisimulate/tests/unit/collector/test_error_classification.py +++ b/python/aisimulate/tests/unit/collector/test_error_classification.py @@ -4,27 +4,13 @@ everything else as ``unexpected`` (pipeline summaries classified by hand until 2026-10-08). Runs with PYTHONPATH=collector like test_registry_table_producers.""" -import sys - +import collect import pytest pytestmark = pytest.mark.unit -def _collect(): - if "torch" not in sys.modules: - from unittest.mock import MagicMock - - _torch = MagicMock() - _torch.AcceleratorError = type("AcceleratorError", (Exception,), {}) - sys.modules["torch"] = _torch - import collect - - return collect - - def test_kernel_limit_prefix_is_classified(): - collect = _collect() assert collect._classify_exception(RuntimeError("FIXME(kernel-limit): no head_dim=192 kernel")) == "kernel_limit" assert collect._classify_exception(ValueError(" FIXME(kernel-limit): divisibility")) == "kernel_limit" assert collect._classify_exception(RuntimeError("CUDA error: an illegal memory access")) == "unexpected" diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py index 485007b329..8e76b6e0cf 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py @@ -360,9 +360,11 @@ def test_indexer_output_budget_excludes_cached_prefix(monkeypatch, batch, query, """Run actual construction and inspect both tensor shape and context capacity.""" allocations, configs = [], [] - class Module: - def __init__(self, **kwargs): - assert kwargs["topk_indices_buffer"] == (expected_tokens, 2048) + class Attention: + topk_indices_buffer = "meta" # what the meta-device model allocated + + def named_modules(self): + return [("", self)] def parameters(self): return [] @@ -378,9 +380,19 @@ def eval(self): def requires_grad_(self, enabled): assert enabled is False + attention = Attention() + + class Model: + def __init__(self, *, vllm_config, prefix): + assert prefix == "" + + def named_modules(self): + return [("model.layers.0.self_attn", attention)] + def create_config(**kwargs): configs.append(kwargs) hf = SimpleNamespace( + architectures=["DeepseekV32ForCausalLM"], index_topk=2048, hidden_size=7168, qk_nope_head_dim=192, @@ -400,7 +412,12 @@ def empty(*shape, **kwargs): return shape monkeypatch.setitem( - sys.modules, "vllm.model_executor.models.deepseek_v2", SimpleNamespace(DeepseekV2MLAAttention=Module) + sys.modules, "vllm.model_executor.models.deepseek_v2", SimpleNamespace(DeepseekV2MLAAttention=object) + ) + monkeypatch.setitem( + sys.modules, + "vllm.model_executor.models.registry", + SimpleNamespace(ModelRegistry=SimpleNamespace(resolve_model_cls=lambda archs, cfg: (Model, archs[0]))), ) monkeypatch.setitem( sys.modules, @@ -410,14 +427,10 @@ def empty(*shape, **kwargs): monkeypatch.setitem( sys.modules, "vllm.utils.torch_utils", SimpleNamespace(set_default_torch_dtype=lambda dtype: nullcontext()) ) - # The generic serving-parity construction resolves the model class through - # vLLM's ModelRegistry; the stubs below cover the explicit-class path, so - # select it (AB-only switch) for this allocation check. - monkeypatch.setenv("AIS_DSA_LEGACY_MODULE", "1") namespace = { "math": math, "os": os, - "torch": SimpleNamespace(empty=empty, int32="int32", no_grad=nullcontext), + "torch": SimpleNamespace(empty=empty, int32="int32", no_grad=nullcontext, device=lambda name: nullcontext()), "_resolve_model_path": lambda model: model, "create_vllm_config": create_config, "_create_gemm_quant_config": lambda gemm: None, @@ -437,6 +450,7 @@ def empty(*shape, **kwargs): is_context=is_context, ) assert allocations == [(expected_tokens, 2048)] + assert attention.topk_indices_buffer == (expected_tokens, 2048) # swapped in for the meta buffer assert configs[0]["max_model_len"] == max(prefix + query, 4096) # The module is built before the backend (and its page) is known: the KV # capacity is an upper bound at the smallest page (16) and is recomputed diff --git a/python/aisimulate/tools/perf_database/collect_campaign.py b/python/aisimulate/tools/perf_database/collect_campaign.py index fde9087df1..44a52a4c66 100644 --- a/python/aisimulate/tools/perf_database/collect_campaign.py +++ b/python/aisimulate/tools/perf_database/collect_campaign.py @@ -281,11 +281,10 @@ def run_campaign( g = _free_gpu(busy, gpus) if g is None: time.sleep(15) - for p in busy.values(): - if p is not None and p.poll() not in (None, 0): - failures += 1 - busy[[k for k, v in busy.items() if v is p][0]] = None continue + finished = busy.get(g) + if finished is not None and finished.returncode != 0: # _free_gpu polled it + failures += 1 s = todo.pop(0) print(f"{time.strftime('%T')} gpu{g} <- {s.key}") busy[g] = subprocess.Popen( From 2c5940314422a3392604801b38a049cc1a886a12 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 21:16:39 +0800 Subject: [PATCH 09/13] fix: kernel limits raise in the worker with cited evidence; exceptions cover every version Owner decisions (2026-10-30) on the CodeRabbit review of the guards: - layer_permissions.md: an evidence-backed FIXME(kernel-limit) guard is legal when it raises at run time, in the worker, before the expensive step; cites the framework source line or a reproducible failing run whose passing rows bound the envelope; and is re-verified on every version bump. Plan-time drops for kernel limits stay forbidden. - vLLM and TRT-LLM MLA modules: DSA per-rank heads 1/2/4 on SM100/103 are no longer dropped from the plan; run_mla_module_worker raises before the module is built (FlashInfer trtllm_fmha_kernel_launcher.cu:318 @0.6.18; TRT-LLM StoreSmemP.h:125 @1.3.0rc29). The cases show in every failure log again at no GPU cost. - TRT-LLM MoE int4_wo cites moe_resolution.py:665 @1.3.0rc29 and drops the version range (the file pins rc29); TRT-LLM SM90 FP8-KV decode names its evidence shard. - Governed exceptions: `versions: all` covers every version of the backend; an exception follows the deviation, and `expires` bounds it. The compute_scale, fp8_static and trtllm mHC entries use it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .../rules/collector/layer_permissions.md | 22 ++++++- .../collector/evidence_exceptions.yaml | 6 +- .../collector/trtllm/collect_attn.py | 5 +- .../collector/trtllm/collect_mla_module.py | 27 ++++---- .../collector/trtllm/collect_moe.py | 11 +--- .../collector/vllm/collect_mla_module.py | 43 ++++--------- .../tests/unit/collector/test_model_cases.py | 62 ++++++++++++------- 7 files changed, 93 insertions(+), 83 deletions(-) diff --git a/python/aisimulate/.claude/rules/collector/layer_permissions.md b/python/aisimulate/.claude/rules/collector/layer_permissions.md index 483eb4b217..45b98d9fd0 100644 --- a/python/aisimulate/.claude/rules/collector/layer_permissions.md +++ b/python/aisimulate/.claude/rules/collector/layer_permissions.md @@ -150,6 +150,22 @@ prediction) or a guard that raises a classified error citing the framework source line, or deletes the note. Do not implement guards from unverified claims, and never express these limits in YAML. +**Evidence-backed guards** (owner decision 2026-10-30). Some limits cannot be +shown in source (a precompiled cubin, a JIT kernel generator inside a shared +library), and some failures are expensive (a worker-killing abort, a full +module load per case). A guard for such a limit is legal only when it: + +1. **Raises at run time, in the worker, before the expensive step.** Never at + plan time: the case stays queued and lands in the failure log as + `kernel_limit`, so every run still shows it. +2. **Starts its message with `FIXME(kernel-limit):`.** +3. **Cites its evidence.** The framework source line when one exists; + otherwise a reproducible failing run (job or shard, collector revision, + failure counts) whose passing rows bound the guarded envelope exactly. A + single failing case or an unbounded envelope is not evidence. +4. **Is re-verified on every version bump** like any other + `FIXME(kernel-limit)`, by a source check or by one run without it. + ## Version contract: one release per collector file Owner policy (2026-09-20, made binding 2026-10-30): version history lives in @@ -209,8 +225,10 @@ A rule in this file may be temporarily excepted ONLY through an entry in the only when the entry carries ALL of: 1. **A self-contained machine-readable scope** — the rule excepted, plus the - exact tables/backend/versions/systems the exception covers. Prose-only - scoping is invalid. + exact tables/backend/systems the exception covers. Prose-only scoping is + invalid. `versions: all` (the default; owner decision 2026-10-30) covers + every version of the backend: an exception follows the deviation, not a + release, and `expires` is what bounds it. A version list narrows it. 2. **The standards owner's `approved_by` sign-off.** These rule files are human-owned policy; only their owner can except them. 3. **An `expires` date and a linked retirement work item.** An exception is diff --git a/python/aisimulate/collector/evidence_exceptions.yaml b/python/aisimulate/collector/evidence_exceptions.yaml index 7dc741bbdb..052ad5b19b 100644 --- a/python/aisimulate/collector/evidence_exceptions.yaml +++ b/python/aisimulate/collector/evidence_exceptions.yaml @@ -97,7 +97,7 @@ standards_exceptions: scope: tables: [scale_matrix_perf] backend: vllm - versions: ["0.30.0"] # 0.29.0 was never collected; the fusion is unchanged on 0.30.0 (opharness compute_scale gate, 2026-09-27) + versions: all # observed on 0.30.0 (opharness compute_scale gate, 2026-09-27) systems: [h100_sxm, h200_sxm] retirement: "model the fused quant (fold compute_scale into the gemm row when the deployment compiles) or drop the standalone row for compiled deployments" approved_by: tianhaox @@ -126,7 +126,7 @@ standards_exceptions: scope: tables: [gemm_perf] backend: vllm - versions: ["0.30.0"] + versions: all # observed on 0.30.0; 0.31.0 dispatch unchanged systems: [rtx_pro_6000_server, b200_sxm, b300_sxm, gb200, gb300] # every CC>=100 system; only rtx_pro_6000_server-class (SM120) hardware-observed so far retirement: "collect a static per-tensor fp8 gemm row (Fp8Config activation_scheme=static) on CC>=100 systems and let perf_database prefer the measured fp8_static row, or drop fp8_static support for ModelOpt FP8 checkpoints on those systems" approved_by: tianhaox @@ -148,7 +148,7 @@ standards_exceptions: scope: tables: [mhc_module_perf] backend: trtllm - versions: ["1.3.0rc23"] + versions: all # observed on 1.3.0rc23 systems: [b200_sxm, b300_sxm, gb200, gb300] retirement: "https://github.com/ai-dynamo/aiconfigurator/issues/1580 — collect fused_hc on SM100/103 + SDK consumer switch" approved_by: tianhaox diff --git a/python/aisimulate/collector/trtllm/collect_attn.py b/python/aisimulate/collector/trtllm/collect_attn.py index 239ed6ac63..cc21ac99cf 100644 --- a/python/aisimulate/collector/trtllm/collect_attn.py +++ b/python/aisimulate/collector/trtllm/collect_attn.py @@ -188,8 +188,9 @@ def run_attention_torch( f"({'context' if is_context_phase else 'decode'}; trtllm_fmha_kernel_launcher.cu:320)" ) # FIXME(kernel-limit): SM90 FP8-KV decode crashes ("CUDA error: unspecified launch failure" - # + SIGABRT) in two measured envelopes (h200 rc29, job 2026-10-08, 681 + 681 cases; the - # passing rows of the same shard bound them exactly): + # + SIGABRT) in two measured envelopes (h200 rc29 attention_generation shard, collector 3c6b058f, + # 2026-10-07: 512 launch failures + 169 illegal accesses, each followed by a worker SIGABRT; the + # shard's 55,047 passing rows bound them exactly): # (a) num_heads 96 / kv_heads 4 (GQA 24): EVERY head_dim (64/128/256), every batch, isl # and window — 512 cases, 0 passing rows; # (b) head_dim 256 at GQA 32 (32/1, 64/2, 128/4), full attention (window 0): isl >= 255 diff --git a/python/aisimulate/collector/trtllm/collect_mla_module.py b/python/aisimulate/collector/trtllm/collect_mla_module.py index 697eebf4ac..d07a6bd34a 100644 --- a/python/aisimulate/collector/trtllm/collect_mla_module.py +++ b/python/aisimulate/collector/trtllm/collect_mla_module.py @@ -40,7 +40,8 @@ # hardware-observed on SM103 (2026-07-19, B300): identical signature # (SmemTile.cpp:53 "Num. rows must be a multiple of 8 ") for h in # {1,2,4} while h=8 ctx+gen pass — the limit covers the trtllm-gen path on -# both SM100 and SM103. Cases stay as classified runtime failures. +# both SM100 and SM103 (StoreSmemP.h:125 @1.3.0rc29: NumRows >= 8). +# run_mla_module_worker raises for them before the module is built. # Re-verify on the next version bump. # 1.3.0rc20 renamed the cache-manager sparse kwargs and moved attention @@ -277,9 +278,7 @@ def _build_module_test_cases(attn_type: str, mode: str): base_cases = get_context_test_cases(attn_type) if mode == "context" else get_generation_test_cases(attn_type) model_specs = get_mla_module_model_specs(attention_type=attn_type, backend="trtllm") cases = [] - dropped_small_heads = 0 dropped_non_shard_heads = 0 - sm_version = get_sm_version() for model_spec in model_specs: # Model-correlated narrowing (case_authoring.md; get_msa_head_counts # precedent): a row declaring its real tensor_parallel_sizes measures @@ -295,23 +294,14 @@ def _build_module_test_cases(attn_type: str, mode: str): if shard_heads is not None and h not in shard_heads: dropped_non_shard_heads += 1 continue - if attn_type == "dsa" and sm_version in (100, 103) and h < 8: - # FIXME(kernel-limit): TRT-LLM's sparse-MLA path on SM100/103 falls back to the FMHA - # that requires "Num. rows must be a multiple of 8" — per-rank heads 1/2/4 (TP 32/64/128 - # of a 128-head model) failed 15,408 of the b200 rc29 DSA cases after a full module load - # each (job 469988228); vLLM's trtllm-gen sparse decode has no kernel for the same cells - # (tileSizeQ < 8). No serving deployment reaches them. Dropped at plan time, counted. - dropped_small_heads += 1 - continue case = [s, b, h, kv_dtype, compute_dtype, gemm_type, model_spec.model_path, attn_type] if rest: case.append(rest[0]) cases.append(case) - if dropped_small_heads or dropped_non_shard_heads: + if dropped_non_shard_heads: print( - f"[trtllm-mla-module-cases] {attn_type}/{mode}: dropped {dropped_small_heads} cases with per-rank " - f"heads < 8 (SM{sm_version} sparse-MLA FMHA row-multiple-of-8 domain) and {dropped_non_shard_heads} " - "cases whose per-rank heads no declared tensor_parallel_sizes shard produces" + f"[trtllm-mla-module-cases] {attn_type}/{mode}: dropped {dropped_non_shard_heads} cases whose " + "per-rank heads no declared tensor_parallel_sizes shard produces" ) return cases @@ -1122,6 +1112,13 @@ def run_mla_module_worker( device: str = "cuda:0", ): """Worker-compatible positional wrapper used by collector/collect.py.""" + # FIXME(kernel-limit): the SM100 DSA note at the top of this file — per-rank heads 1/2/4 failed + # 15,408 b200 rc29 cases after a full module load each (job 469988228). Raise before the build. + if attn_type == "dsa" and get_sm_version() in (100, 103) and num_heads < 8: + raise RuntimeError( + "FIXME(kernel-limit): TRT-LLM trtllm-gen sparse-MLA FMHA on SM100/103 needs per-rank heads >= 8 " + f"(num_heads={num_heads}; StoreSmemP.h:125 @1.3.0rc29)" + ) return run_mla_module( seq_len=seq_len, batch_size=batch_size, diff --git a/python/aisimulate/collector/trtllm/collect_moe.py b/python/aisimulate/collector/trtllm/collect_moe.py index 7316439049..163f9815bb 100755 --- a/python/aisimulate/collector/trtllm/collect_moe.py +++ b/python/aisimulate/collector/trtllm/collect_moe.py @@ -65,7 +65,6 @@ log_perf, power_law_logits_v3, ) -from collector.version_resolver import _check_compat aic_debug = int(os.getenv("aic_moe_debug", "0")) # noqa: SIM112 @@ -375,13 +374,9 @@ def _run_moe_case( # SM100/103 ("no MoE implementation can serve this layer ... CutlassFusedMoE: quant_unsupported # (quant_algo=W4A16)", 114/114 int4_wo cases, b200 job 469988228). Serving cannot load the # W4A16 checkpoint either (benchmarks/evidence/collector/op-probe-2026-10/results/sm100/trtllm-1.3.0rc29.yaml: - # Kimi-K2.5 fail, finding sm100_probe_failures_recheck_2026_10_04 (g)). The lane is unreachable on this pin, not a - # per-shape failure; refuse before building weights. - if ( - moe_type == "int4_wo" - and get_sm_version() in (100, 103) - and _check_compat("tensorrt_llm>=1.3.0rc29", _TRTLLM_VERSION) - ): + # Kimi-K2.5 fail, finding sm100_probe_failures_recheck_2026_10_04 (g)). The lane is unreachable on this pin + # (moe_resolution.py:665 @1.3.0rc29 raises it), not a per-shape failure; refuse before building weights. + if moe_type == "int4_wo" and get_sm_version() in (100, 103): raise RuntimeError( "FIXME(kernel-limit): TRT-LLM int4_wo (W4A16) MoE has no implementation on SM100/103 at " f"{_TRTLLM_VERSION}; serving fails identically (Kimi-K2.5 identity probe)" diff --git a/python/aisimulate/collector/vllm/collect_mla_module.py b/python/aisimulate/collector/vllm/collect_mla_module.py index 400a0e3637..c68c0fc6d1 100644 --- a/python/aisimulate/collector/vllm/collect_mla_module.py +++ b/python/aisimulate/collector/vllm/collect_mla_module.py @@ -416,12 +416,6 @@ def _install_rope_missing_key_shims(attn_module) -> tuple[list[str], list[str]]: return patched, left -# SM100/103: trtllm-gen sparse-MLA decode has no kernel below tileSizeQ 8 (see the -# FIXME in _build_module_test_cases). Hopper/Ada/SM120 route sparse decode elsewhere. -_DSA_SPARSE_DECODE_MIN_HEADS = 8 -_DSA_SPARSE_DECODE_MIN_HEADS_SMS = (100, 103) - - def _model_max_position_embeddings(model_id: str) -> int | None: """The model's max context length from its cached config (same file the sglang module collector reads); None when unavailable (no cap applied).""" @@ -445,10 +439,8 @@ def _build_module_test_cases(attn_type: str, mode: str): """ base_cases = get_context_test_cases(attn_type) if mode == "context" else get_generation_test_cases(attn_type) cases = [] - dropped_small_heads = 0 dropped_over_ceiling = 0 dropped_non_shard_heads = 0 - sm_version = get_sm_version() for model_spec in get_mla_module_model_specs(attention_type=attn_type, backend="vllm"): max_position = _model_max_position_embeddings(model_spec.model_path) if attn_type == "dsa" else None # Model-correlated narrowing (case_authoring.md; the MSA collectors' @@ -468,17 +460,6 @@ def _build_module_test_cases(attn_type: str, mode: str): if shard_heads is not None and h not in shard_heads: dropped_non_shard_heads += 1 continue - small_head_hole = sm_version in _DSA_SPARSE_DECODE_MIN_HEADS_SMS and h < _DSA_SPARSE_DECODE_MIN_HEADS - if attn_type == "dsa" and small_head_hole: - # FIXME(kernel-limit): FlashInfer's trtllm-gen sparse-MLA decode ships no kernel for - # tileSizeQ < 8 (= per-rank heads at q_len 1): "Missing TRTLLM-GEN kernel (decode) ... - # tileSizeQ=4, sparseMlaType=1" (trtllm_fmha_kernel_launcher.cu:320, flashinfer 0.6.x). - # Heads 1/2/4 (TP 32/64/128 of a 128-head model) failed 68,435 of the b200 vllm 0.30.0 - # DSA cases AFTER loading a full module each (job 471043976); trtllm's fallback FMHA - # rejects the same cells ("Num. rows must be a multiple of 8"). Serving has no such - # deployment either. Dropped at plan time, counted here, not silently. - dropped_small_heads += 1 - continue prefix_len = rest[0] if rest else 0 if attn_type == "dsa" and max_position is not None and prefix_len + s > max_position: # The shared dsa_prefix_lengths axis reaches 1,048,575 (GLM-5.2's ceiling); a model @@ -492,10 +473,9 @@ def _build_module_test_cases(attn_type: str, mode: str): if rest: case.append(rest[0]) cases.append(case) - if dropped_small_heads or dropped_over_ceiling or dropped_non_shard_heads: + if dropped_over_ceiling or dropped_non_shard_heads: print( - f"[vllm-mla-module-cases] {attn_type}/{mode}: dropped {dropped_small_heads} cases with per-rank heads " - f"< {_DSA_SPARSE_DECODE_MIN_HEADS} (SM{sm_version} sparse-MLA decode kernel domain), " + f"[vllm-mla-module-cases] {attn_type}/{mode}: dropped " f"{dropped_over_ceiling} cases whose prefix+isl exceeds the model's max_position_embeddings and " f"{dropped_non_shard_heads} cases whose per-rank heads no declared tensor_parallel_sizes shard produces" ) @@ -1637,7 +1617,7 @@ def _run_point(point_stack, seq_len: int, prefix_len: int, point_phase: dict[str # flashinfer 0.6.12) whenever vLLM selects FLASHINFER_MLA_SPARSE — always # for FP8 KV, and for BF16 KV at heads <= 16 (platforms/cuda.py:98-116 # @0.24.0). Heads 8..128 pass; boundary measured on B200. Serving fails - # identically, so the affected cases stay observed runtime failures. + # identically; run_mla_module_worker raises for them before the build. # Re-verify on the next vLLM/FlashInfer bump. # FIXME(kernel-limit): on SM120 and SM89, vLLM's dense-MLA decode backend # is TRITON_MLA (SM120: platforms/cuda.py:130-134; SM89: the else-branch @@ -1859,19 +1839,22 @@ def run_mla_module_worker( ``seq_len`` is one point (the CLI, single-point fills) or the case's list of points measured on one module (the registry plan). """ + # FIXME(kernel-limit): on SM100/103 FlashInfer's trtllm-gen sparse-MLA kernels start at tileSizeQ 8 + # (= per-rank heads at q_len 1): DSA heads 1/2/4 (TP 32/64/128 of a 128-head model) raise "Missing + # TRTLLM-GEN kernel" (csrc/trtllm_fmha_kernel_launcher.cu:318 @flashinfer 0.6.18, vLLM 0.30.0's pin), + # 68,435 b200 cases after a full module load each (job 471043976). Serving fails identically. + if attn_type == "dsa" and get_sm_version() in (100, 103) and num_heads < 8: + raise RuntimeError( + "FIXME(kernel-limit): FlashInfer trtllm-gen sparse-MLA on SM100/103 has no kernel for per-rank " + f"heads < 8 (num_heads={num_heads}; trtllm_fmha_kernel_launcher.cu:318 @flashinfer 0.6.18)" + ) # FIXME(kernel-limit): on SM100/103 with FP8 KV, vLLM selects FLASHINFER_MLA_SPARSE for DSA and # FlashInfer's auto backend has nothing for 64 < per-rank heads < 128: trtllm-gen refuses the head # range, cute-dsl refuses sparse_mla_top_k ("auto: no backend supports this configuration", 11,442 # cases = every heads-96 fp8 cell, both phases, job 471043976). Serving fails identically (the same # gap closes Kimi-K3 tp1 on sglang: finding sm100_probe_failures_recheck_2026_10_04 (i)). Raise # before the module is built so the cell is a classified failure, not a 20 s load + crash. - if ( - attn_type == "dsa" - and "fp8" in str(kv_cache_dtype) - and get_sm_version() in (100, 103) # SM100 family (_DSA_SPARSE_DECODE_MIN_HEADS_SMS); inline so the - # worker stays self-contained for the CLI tests that load it alone - and 64 < num_heads < 128 - ): + if attn_type == "dsa" and "fp8" in str(kv_cache_dtype) and get_sm_version() in (100, 103) and 64 < num_heads < 128: raise RuntimeError( "FIXME(kernel-limit): vLLM FLASHINFER_MLA_SPARSE on SM100/103 has no decode backend for " f"64 < per-rank heads < 128 with FP8 KV (num_heads={num_heads}); trtllm-gen rejects the head " diff --git a/python/aisimulate/tests/unit/collector/test_model_cases.py b/python/aisimulate/tests/unit/collector/test_model_cases.py index 6cbffe1876..82abd4897e 100644 --- a/python/aisimulate/tests/unit/collector/test_model_cases.py +++ b/python/aisimulate/tests/unit/collector/test_model_cases.py @@ -1875,28 +1875,52 @@ def _load_module_case_builder(backend: str, namespace: dict): @pytest.mark.parametrize("backend", ["vllm", "trtllm"]) -def test_dsa_module_plan_drops_sub_8_head_cells_on_sm100_only(backend): +def test_dsa_sub_8_head_cells_stay_queued_and_raise_before_the_module_build(backend): """b200 2026-10-06 (jobs 471043976 / 469988228): per-rank heads 1/2/4 have no sparse-MLA - decode kernel on SM100/103 (trtllm-gen tileSizeQ < 8; TRT-LLM FMHA rows % 8) and failed - 68,435 + 15,408 cases after a full module load each. The planners drop them for DSA on - SM100/103 and keep them for MLA and on other SMs.""" + kernel on SM100/103 and failed 68,435 + 15,408 cases after a full module load each. The + plan keeps them (a kernel limit is never a plan-time drop); the worker raises a + kernel_limit failure before building anything, on SM100/103 and for DSA only.""" + from contextlib import nullcontext from types import SimpleNamespace heads = [128, 64, 8, 4, 2, 1] base = [[1, 1, h, "bfloat16", "bfloat16", "bfloat16", 0] for h in heads] - for sm, expected in ((100, [128, 64, 8]), (103, [128, 64, 8]), (90, heads)): - ns = { - "get_context_test_cases": lambda _t, base=base: base, - "get_generation_test_cases": lambda _t: [], - "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], + ns = { + "get_context_test_cases": lambda _t: base, + "get_generation_test_cases": lambda _t: [], + "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], + "get_sm_version": lambda: 100, + "_model_max_position_embeddings": lambda _m: None, + } + build = _load_module_case_builder(backend, ns) + assert sorted({c[2] for c in build("dsa", "context")}) == sorted(heads) + + source_path = REPO_ROOT / "collector" / backend / "collect_mla_module.py" + tree = ast.parse(source_path.read_text(encoding="utf-8"), filename=str(source_path)) + worker = [node for node in tree.body if isinstance(node, ast.FunctionDef) and node.name == "run_mla_module_worker"] + built = [] + for sm, attn_type, num_heads, raises in ( + (100, "dsa", 4, True), + (103, "dsa", 1, True), + (100, "dsa", 8, False), + (100, "mla", 4, False), + (90, "dsa", 4, False), + ): + namespace = { "get_sm_version": lambda sm=sm: sm, - "_DSA_SPARSE_DECODE_MIN_HEADS": 8, - "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), - "_model_max_position_embeddings": lambda _m: None, + "run_mla_module": lambda **kwargs: built.append(kwargs), + "torch": SimpleNamespace(inference_mode=nullcontext), } - build = _load_module_case_builder(backend, ns) - assert [c[2] for c in build("dsa", "context")] == expected, (backend, sm) - assert [c[2] for c in build("mla", "context")] == heads, (backend, sm) # dense MLA untouched + exec(compile(ast.Module(body=worker, type_ignores=[]), str(source_path), "exec"), namespace) + args = (16, 1, num_heads, "bfloat16", "bfloat16", "bfloat16", "deepseek-ai/DeepSeek-V3.2", attn_type) + before = len(built) + if raises: + with pytest.raises(RuntimeError, match=r"^FIXME\(kernel-limit\): "): + namespace["run_mla_module_worker"](*args, perf_filename="x") + assert len(built) == before, (backend, sm, attn_type, num_heads) + else: + namespace["run_mla_module_worker"](*args, perf_filename="x") + assert len(built) == before + 1, (backend, sm, attn_type, num_heads) @pytest.mark.parametrize("backend", ["vllm", "trtllm"]) @@ -1917,8 +1941,6 @@ def test_module_plan_keeps_only_the_heads_of_declared_tensor_parallel_shards(bac "get_generation_test_cases": lambda _t: [], "get_mla_module_model_specs": lambda **_k: [glm, kimi], "get_sm_version": lambda: 100, - "_DSA_SPARSE_DECODE_MIN_HEADS": 8, - "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), "_model_max_position_embeddings": lambda _m: None, } build = _load_module_case_builder(backend, ns) @@ -1950,8 +1972,6 @@ def test_vllm_module_plan_groups_the_seq_len_points_of_one_module_into_one_case( "get_generation_test_cases": lambda _t: [], "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], "get_sm_version": lambda: 100, - "_DSA_SPARSE_DECODE_MIN_HEADS": 8, - "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), "_model_max_position_embeddings": lambda _m: None, } build = _load_module_case_builder("vllm", ns) @@ -1982,8 +2002,6 @@ def test_vllm_dsa_module_plan_refuses_a_grid_the_runner_would_not_reproduce(): "get_generation_test_cases": lambda _t: [], "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], "get_sm_version": lambda: 100, - "_DSA_SPARSE_DECODE_MIN_HEADS": 8, - "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), "_model_max_position_embeddings": lambda _m: None, } build = _load_module_case_builder("vllm", ns) @@ -2003,8 +2021,6 @@ def test_vllm_dsa_module_plan_caps_prefix_at_the_model_max_position(): "get_generation_test_cases": lambda _t: [], "get_mla_module_model_specs": lambda **_k: [SimpleNamespace(model_path="deepseek-ai/DeepSeek-V3.2")], "get_sm_version": lambda: 100, - "_DSA_SPARSE_DECODE_MIN_HEADS": 8, - "_DSA_SPARSE_DECODE_MIN_HEADS_SMS": (100, 103), "_model_max_position_embeddings": lambda _m: 163840, } build = _load_module_case_builder("vllm", ns) From 8ef3e21765415c54bdc233ec76cc42436eccef97 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Fri, 30 Oct 2026 22:55:51 +0800 Subject: [PATCH 10/13] fix: standards exceptions are re-verified on every pin bump instead of expiring An exception now follows the deviation (versions: all), so a calendar date bounded nothing that a reviewer or CI ever checked. Each standards exception carries the release it was last verified on; every bump of the covered tables' collector pin re-checks the deviation by source or run, as for a FIXME(kernel-limit), and either moves verified_on or deletes the entry. An entry behind the pin is stale. Re-verified on today's pins: - vLLM compute_scale: 0.30.0, the op's own pin (opharness gate 2026-09-27). - vLLM fp8_static: 0.31.0. FlashInfer still precedes Cutlass (kernels/linear/__init__.py:422-423) and still takes CC>=100 per-tensor scales (scaled_mm/flashinfer.py:50-62). - TRT-LLM mHC: 1.3.0rc29. DeepSeek-V4 serving still defaults to fused_hc (models/modeling_deepseekv4.py:292-297) while the collector times pre/post_mapping. Evidence waivers (design section 9) keep their expiry. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .../rules/collector/layer_permissions.md | 17 +++++++++------ .../collector/evidence_exceptions.yaml | 21 ++++++++++--------- .../collector/trtllm/collect_mhc_module.py | 4 ++-- 3 files changed, 24 insertions(+), 18 deletions(-) diff --git a/python/aisimulate/.claude/rules/collector/layer_permissions.md b/python/aisimulate/.claude/rules/collector/layer_permissions.md index 45b98d9fd0..d20caff886 100644 --- a/python/aisimulate/.claude/rules/collector/layer_permissions.md +++ b/python/aisimulate/.claude/rules/collector/layer_permissions.md @@ -228,19 +228,24 @@ only when the entry carries ALL of: exact tables/backend/systems the exception covers. Prose-only scoping is invalid. `versions: all` (the default; owner decision 2026-10-30) covers every version of the backend: an exception follows the deviation, not a - release, and `expires` is what bounds it. A version list narrows it. + release, and re-verification (item 3) is what bounds it. A version list + narrows it. 2. **The standards owner's `approved_by` sign-off.** These rule files are human-owned policy; only their owner can except them. -3. **An `expires` date and a linked retirement work item.** An exception is - a debt with a due date, not a policy change; on expiry the exception is - renewed by the owner or the deviation is fixed. +3. **A `verified_on` release and a linked retirement work item** (owner + decision 2026-10-30; replaces the former `expires` date). An exception + is a debt, not a policy change. Every bump of the collector pin for the + covered tables re-checks the deviation against the new release, by a + source check or a run, as for a `FIXME(kernel-limit)`: if it still + holds, `verified_on` moves to that release; if not, the entry is + deleted. An entry whose `verified_on` is behind the pin is stale. 4. **Consumer-visible provenance** — the deviation must be discoverable from the shipped data's own declaration layer (e.g. the `reuse.yaml` reason or sidecar for the affected tables), never only from this registry. An exception documents and bounds a deviation; it does not weaken the rule. -Reviews treat an in-scope, unexpired entry as compliant and everything -outside its scope as an ordinary violation. +Reviews treat an in-scope entry verified on the current pin as compliant; +a stale entry, and everything outside its scope, is an ordinary violation. ## Meta rules diff --git a/python/aisimulate/collector/evidence_exceptions.yaml b/python/aisimulate/collector/evidence_exceptions.yaml index 052ad5b19b..f5a55ed715 100644 --- a/python/aisimulate/collector/evidence_exceptions.yaml +++ b/python/aisimulate/collector/evidence_exceptions.yaml @@ -4,13 +4,14 @@ # Two registries in one file: # evidence_exceptions — waivers against resolver-emitted evidence # requirements (design §9 "Exceptions"). -# standards_exceptions — owner-approved, expiring exceptions to collector -# RULES (mechanism defined in -# .claude/rules/collector/layer_permissions.md +# standards_exceptions — owner-approved exceptions to collector RULES, +# re-verified on every pin bump (mechanism defined +# in .claude/rules/collector/layer_permissions.md # §"Governed exceptions"). # Both are human-reviewed against `evidence_check.py` output until the # enforcing CI gate lands; every entry carries a self-contained -# machine-readable scope, an approver, and an expiry. +# machine-readable scope and an approver. Evidence waivers carry an expiry; +# standards exceptions carry the release they were last verified on. schema_version: 2 @@ -97,11 +98,11 @@ standards_exceptions: scope: tables: [scale_matrix_perf] backend: vllm - versions: all # observed on 0.30.0 (opharness compute_scale gate, 2026-09-27) + versions: all systems: [h100_sxm, h200_sxm] retirement: "model the fused quant (fold compute_scale into the gemm row when the deployment compiles) or drop the standalone row for compiled deployments" approved_by: tianhaox - expires: 2026-12-31 + verified_on: "0.30.0" # opharness compute_scale gate, 2026-09-27; compute_scale is pinned to 0.30.0 # --- fp8_static GEMM is DERIVED, and on CC>=100 serving runs a different kernel --- # Owner decision (tianhaox, 2026-09-30): keep fp8_static as an APPROXIMATION, @@ -126,11 +127,11 @@ standards_exceptions: scope: tables: [gemm_perf] backend: vllm - versions: all # observed on 0.30.0; 0.31.0 dispatch unchanged + versions: all systems: [rtx_pro_6000_server, b200_sxm, b300_sxm, gb200, gb300] # every CC>=100 system; only rtx_pro_6000_server-class (SM120) hardware-observed so far retirement: "collect a static per-tensor fp8 gemm row (Fp8Config activation_scheme=static) on CC>=100 systems and let perf_database prefer the measured fp8_static row, or drop fp8_static support for ModelOpt FP8 checkpoints on those systems" approved_by: tianhaox - expires: 2026-12-31 + verified_on: "0.31.0" # observed on 0.30.0; 0.31.0 source unchanged: kernels/linear/__init__.py:422-423 priority, scaled_mm/flashinfer.py:50-62 # --- PR #1486: Blackwell mHC fused_hc serving-dispatch parity ------------- # Owner decision (tianhaox, 2026-08-19/20): the fused_hc collection on @@ -148,8 +149,8 @@ standards_exceptions: scope: tables: [mhc_module_perf] backend: trtllm - versions: all # observed on 1.3.0rc23 + versions: all systems: [b200_sxm, b300_sxm, gb200, gb300] retirement: "https://github.com/ai-dynamo/aiconfigurator/issues/1580 — collect fused_hc on SM100/103 + SDK consumer switch" approved_by: tianhaox - expires: 2026-10-31 + verified_on: "1.3.0rc29" # observed on rc23; rc29 serving still defaults to fused_hc (models/modeling_deepseekv4.py:292-297) while the collector times pre/post_mapping diff --git a/python/aisimulate/collector/trtllm/collect_mhc_module.py b/python/aisimulate/collector/trtllm/collect_mhc_module.py index 8f5cdf92dc..0f0a7e9765 100644 --- a/python/aisimulate/collector/trtllm/collect_mhc_module.py +++ b/python/aisimulate/collector/trtllm/collect_mhc_module.py @@ -45,8 +45,8 @@ of the approximation if fused_hc were free, NOT a measured Blackwell fused/unfused delta (which is unmeasured; owner-accepted). Direction is conservative (cost-overestimating). Governed standards exception: -collector/evidence_exceptions.yaml standards_exceptions (expires -2026-10-31); retirement = the coordinated fused_hc table + SDK consumer +collector/evidence_exceptions.yaml standards_exceptions (verified on +1.3.0rc29); retirement = the coordinated fused_hc table + SDK consumer switch for SM100/103. """ From 0935e3b0a38cf07dd421bf69ad85310b2f445124 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Sat, 31 Oct 2026 00:00:07 +0800 Subject: [PATCH 11/13] fix: kernel limits stay queued and classified; re-verify the cited raises at rc29 The rule this branch wrote (plan-time kernel-limit drops forbidden; a guard raises in the worker with FIXME(kernel-limit) and a citation) did not yet hold in its own code. - SGLang attention: the flashinfer fp8_context_fmha case is no longer dropped at plan time. Both FP8-prefill refusals (flashinfer: no FP8 prefill compute; fa3/trtllm_mha: no BF16 prefill on an FP8 cache) run before the model runner and tensors are built, cited at 0.5.21 (flashinfer_backend.py:1833-1834, flashattention_backend.py:1322-1332, trtllm_mha_backend.py:1600-1609). - vLLM MSA: the decode-scorer limit message starts with FIXME(kernel-limit), so it classifies as kernel_limit instead of unexpected. - failure_handling.md describes both classifications. - Pre-existing limit raises carry the prefix where the cited source still holds at the pinned release, re-cited there: - TRT-LLM FlashInferAttention FP8 FMHA: attention/backends/flashinfer.py:2740-2746 @1.3.0rc29. - TRT-LLM MLA below SM90: attentionOp.cpp:3234/:3240 @1.3.0rc29 (stock and module collectors). - WideEP fp8_block on non-SM90: deep_gemm/compiler.cuh unchanged rc20 -> rc29. - Not prefixed: the TRT-LLM DSA module's SM120 refusal. rc29 pins DeepGEMM 8b1392b, whose attention.hpp dispatches SM120 for both indexer phases (sm120_mqa_logits :205-212, sm120_paged_mqa_logits :414-418). A FIXME(re-verify) note records it; an SM120 rc29 run decides it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- .../rules/collector/failure_handling.md | 6 +- .../collector/sglang/collect_attn.py | 88 +++++++++---------- .../collector/trtllm/collect_attn.py | 9 +- .../collector/trtllm/collect_mla.py | 4 +- .../collector/trtllm/collect_mla_module.py | 9 +- .../collector/vllm/collect_msa_module.py | 2 +- .../wideep/trtllm/collect_moe_compute.py | 4 +- .../collector/test_msa_kernel_limit_scope.py | 2 +- .../collector/test_sglang_attention_0514.py | 14 +-- 9 files changed, 75 insertions(+), 63 deletions(-) diff --git a/python/aisimulate/.claude/rules/collector/failure_handling.md b/python/aisimulate/.claude/rules/collector/failure_handling.md index 45b84099d9..364d9a3a73 100644 --- a/python/aisimulate/.claude/rules/collector/failure_handling.md +++ b/python/aisimulate/.claude/rules/collector/failure_handling.md @@ -18,8 +18,10 @@ group is failing, it should be FIXED, not silently tolerated. ## What happens automatically (no action needed) - Every worker failure is recorded in `errors_.json` and - `collection_summary_.json` with `classification: "unexpected"`, - the case parameters, exception type/message, and a `(model, dtype)` group + `collection_summary_.json` with a classification: `kernel_limit` + when the message starts with `FIXME(kernel-limit)` (a cited guard, see + layer_permissions.md), otherwise `unexpected`. Each record carries the + case parameters, exception type/message, and a `(model, dtype)` group label. The summary aggregates `errors_by_group`; a group with many failures is logged as a fix-me warning. - CUDA-fatal errors reset the worker process; the task is already recorded diff --git a/python/aisimulate/collector/sglang/collect_attn.py b/python/aisimulate/collector/sglang/collect_attn.py index 2e48fa90ba..f8787758ed 100644 --- a/python/aisimulate/collector/sglang/collect_attn.py +++ b/python/aisimulate/collector/sglang/collect_attn.py @@ -315,22 +315,11 @@ def get_context_attention_test_cases(): continue if b * s * num_kv_heads * head_dim * 2 >= max_kv_elements: continue - # the backend this head config runs on (per-model map, else the SM default table) — - # the same resolution run_attention_torch makes before refusing a combo - lane = head_config.kernel_source or default_attention_backend( - sm_version, head_config.has_attention_sink - ) for precision_case in shape_sweep["precision_cases"]: use_fp8_kv_cache = bool(precision_case["fp8_kv_cache"]) use_fp8_context_fmha = bool(precision_case["fp8_context_fmha"]) if skip_fp8 and use_fp8_kv_cache: continue - if use_fp8_context_fmha and lane == "flashinfer": - # FlashInfer has no FP8 prefill compute path (BF16 Q reads the FP8 cache through - # descales), so run_attention_torch refuses this case by design; scheduling it on - # the flashinfer SMs (sm89 / sm120) only made a third of the op's tasks fail - # (sm89 sample 2026-10-04: 13 of 40). Not an SM gate: the fp8_kv_cache case stays. - continue test_cases.append( [ b, @@ -581,6 +570,42 @@ def run_attention_torch( if page_size is None: page_size = 64 if attn_backend_name == "trtllm_mha" else 1 + # Label prefill rows by the compute dtype the backend actually uses in + # SGLang 0.5.21 — the input dtype is an API contract, not the compute + # dtype: + # - fa3 (SM90): whenever the KV cache is FP8 and head_dim <= 256 it + # casts Q to the KV dtype itself (flashattention_backend.py:1322-1332), + # so FP8-KV prefill IS FP8 FMHA compute. + # - trtllm_mha (TRTLLM-GEN on SM100/103): takes BF16 inputs and casts Q + # to FP8 itself when the KV cache is FP8 (trtllm_mha_backend.py:1600-1609). + # Feed BF16 and let the backend quantize; an external FP8 cast would + # break its input contract without changing the (already FP8) compute. + # - flashinfer: the prefill wrapper plans BF16 Q against the FP8 KV cache + # (flashinfer_backend.py:1833-1834) and reads it with descales + # (:1388-1389); there is no FP8 prefill compute path, so an fp8-labeled + # row would be a lie. + # Consequence: on fa3/trtllm_mha a "BF16-compute prefill on FP8 KV" + # combo does not exist — the backend force-quantizes Q — so collecting + # it would record FP8 compute under a bfloat16 label. Both refusals run + # here, before the model runner and tensors are built; the cases stay + # queued and land in the failure log as kernel_limit. + if is_context_phase: + if use_fp8_context_fmha and attn_backend_name == "flashinfer": + raise ValueError( + "FIXME(kernel-limit): SGLang flashinfer has no FP8 prefill compute path " + "(flashinfer_backend.py:1833-1834 @0.5.21 plans BF16 Q against the FP8 KV cache)" + ) + if ( + not use_fp8_context_fmha + and use_fp8_kv_cache + and (attn_backend_name == "trtllm_mha" or (attn_backend_name == "fa3" and head_dim <= 256)) + ): + raise ValueError( + f"FIXME(kernel-limit): SGLang {attn_backend_name} quantizes Q to FP8 internally when the KV cache " + "is FP8, so a BF16-compute prefill on an FP8 KV cache does not exist; this " + "combination is the fp8_context_fmha case" + ) + model_runner = MockModelRunner( torch_device, kv_cache_dtype="fp8_e4m3" if use_fp8_kv_cache else "auto", @@ -786,41 +811,12 @@ def run_attention_torch( with forward_context(ForwardContext(attn_backend=attn_backend)): attn_backend.init_forward_metadata(forward_batch) - # Label prefill rows by the compute dtype the backend actually uses in - # SGLang 0.5.14 — the input dtype is an API contract, not the compute - # dtype: - # - fa3 (SM90): whenever the KV cache is FP8 and head_dim <= 256 it - # casts Q to the KV dtype itself (flashattention_backend.py:857-872), - # so FP8-KV prefill IS FP8 FMHA compute. - # - trtllm_mha (TRTLLM-GEN on SM100/103): requires BF16 inputs and - # quantizes Q internally via scaled_fp8_quant when the KV cache is - # FP8 (trtllm_mha_backend.py:154-158, 291-301). Feed BF16 and let - # the backend quantize; an external FP8 cast would break its input - # contract without changing the (already FP8) compute. - # - flashinfer: BF16 Q reads the FP8 KV cache with descales; there is - # no FP8 prefill compute path, so an fp8-labeled row would be a lie. - # Consequence: on fa3/trtllm_mha a "BF16-compute prefill on FP8 KV" - # combo does not exist — the backend force-quantizes Q — so collecting - # it would record FP8 compute under a bfloat16 label. Fail closed. - if is_context_phase: - if use_fp8_context_fmha: - if attn_backend_name == "flashinfer": - raise ValueError( - "FIXME(kernel-limit): SGLang flashinfer has no FP8 prefill compute path " - "(plan-time skip missed this case)" - ) - if attn_backend_name != "trtllm_mha": - q = q.to(kvtype) - k = k.to(kvtype) - v = v.to(kvtype) - elif use_fp8_kv_cache and ( - attn_backend_name == "trtllm_mha" or (attn_backend_name == "fa3" and head_dim <= 256) - ): - raise ValueError( - f"FIXME(kernel-limit): SGLang {attn_backend_name} quantizes Q to FP8 internally when the KV cache " - "is FP8, so a BF16-compute prefill on an FP8 KV cache does not exist; this " - "combination is the fp8_context_fmha case" - ) + # fp8_context_fmha (labels: see the refusals before the model runner): + # trtllm_mha casts Q itself; every other lane takes FP8 inputs. + if is_context_phase and use_fp8_context_fmha and attn_backend_name != "trtllm_mha": + q = q.to(kvtype) + k = k.to(kvtype) + v = v.to(kvtype) # Mirror the serving call contract: only sink-carrying models pass the # ``sinks`` kwarg (e.g. gpt_oss.py), and SGLang 0.5.14 diff --git a/python/aisimulate/collector/trtllm/collect_attn.py b/python/aisimulate/collector/trtllm/collect_attn.py index cc21ac99cf..9968d56ae7 100644 --- a/python/aisimulate/collector/trtllm/collect_attn.py +++ b/python/aisimulate/collector/trtllm/collect_attn.py @@ -169,15 +169,18 @@ def run_attention_torch( ) if is_flashinfer and use_fp8_context_fmha: # Mirror of the sglang collector's compute-dtype labeling rule: - # TRT-LLM 1.3.0rc20 FlashInferAttention has no FP8 FMHA compute path, + # TRT-LLM FlashInferAttention has no FP8 FMHA compute path, # including under the serving-pinned trtllm-gen sub-backend set below. # trtllm-gen's only fp8 handling is the fp8 KV cache: it casts Q to the # KV dtype to reuse the QkvE4m3OBfloat16 cubins but the FMHA output is - # still BF16 (attention_backend/flashinfer.py:1789-1791@1.3.0rc20). The + # still BF16 (attention/backends/flashinfer.py:2740-2746@1.3.0rc29; :1789-1791@1.3.0rc20). The # collector's fp8-context-FMHA case (quant_algo=FP8 + out_scale) is a # TRTLLM-backend contract flashinfer does not consume, so an fp8-labeled # flashinfer row would record BF16 compute under an fp8 label. Fail closed. - raise ValueError("TRT-LLM 1.3.0rc20 FlashInferAttention has no FP8 FMHA compute path") + raise ValueError( + "FIXME(kernel-limit): TRT-LLM FlashInferAttention has no FP8 FMHA compute path " + "(attention/backends/flashinfer.py:2740-2746 @1.3.0rc29)" + ) # FIXME(kernel-limit): trtllm-gen has no FMHA for head_dim 192 on SM100/103 — context # ("Missing TRTLLM-GEN kernel (context): headDimQk=192", 2,904 b200 rc29 cases) and decode # ("Missing TRTLLM-GEN kernel (decode)", 1,624 b200 / 1,641 b300). Serving routes the same diff --git a/python/aisimulate/collector/trtllm/collect_mla.py b/python/aisimulate/collector/trtllm/collect_mla.py index 6518bddb7b..28b1d8b93e 100644 --- a/python/aisimulate/collector/trtllm/collect_mla.py +++ b/python/aisimulate/collector/trtllm/collect_mla.py @@ -229,9 +229,9 @@ def run_mla( if get_sm_version() < 90: phase = "context" if is_context_phase else "generation" raise ValueError( - f"TRT-LLM MLA has no pre-Hopper FMHA kernel; DeepSeek MLA {phase} " + f"FIXME(kernel-limit): TRT-LLM MLA has no pre-Hopper FMHA kernel; DeepSeek MLA {phase} " f"is unsupported on SM{get_sm_version()} (attentionOp.cpp:" - f"{'3097' if is_context_phase else '3091'} assert @1.3.0rc20)" + f"{'3240' if is_context_phase else '3234'} assert @1.3.0rc29)" ) scenario = Scenario() q_lora_rank = scenario.q_lora_rank diff --git a/python/aisimulate/collector/trtllm/collect_mla_module.py b/python/aisimulate/collector/trtllm/collect_mla_module.py index d07a6bd34a..4c13838aaa 100644 --- a/python/aisimulate/collector/trtllm/collect_mla_module.py +++ b/python/aisimulate/collector/trtllm/collect_mla_module.py @@ -888,6 +888,13 @@ def run_mla_module( # (Gemma4 dense-attention precedent in collect_attn.py) instead of # paying construction + a CUDA-level abort per case. Re-verify against # deepgemm's supported-arch set on the next framework version bump. + # FIXME(re-verify): NOT re-verified at 1.3.0rc29, and the source no longer + # supports it. rc29 pins DeepGEMM 8b1392b (3rdparty/fetch_content.json), + # whose attention.hpp dispatches SM120 for both indexer phases + # (sm120_mqa_logits :205-212, sm120_paged_mqa_logits :414-418; heads + # 16/32/64). Until an SM120 rc29 run decides it, the raise stays and + # keeps its unprefixed message, so the failure log reports it as + # unexpected instead of an accepted kernel limit. if attn_type == "dsa" and get_sm_version() >= 120: raise ValueError( f"DeepGEMM sparse-attention indexer has no RTX Blackwell kernel; DSA " @@ -905,7 +912,7 @@ def run_mla_module( # in the worker (facts/mla_bypass_*.log). The wall is unchanged across rc20 -> rc29: keep failing closed. if attn_type == "mla" and get_sm_version() < 90: raise ValueError( - f"TRT-LLM MLA has no pre-Hopper FMHA kernel; MLA modules are " + f"FIXME(kernel-limit): TRT-LLM MLA has no pre-Hopper FMHA kernel; MLA modules are " f"unsupported on SM{get_sm_version()} " f"(attentionOp.cpp:3234 assert @1.3.0rc29, :3091/:3097 @1.3.0rc20)" ) diff --git a/python/aisimulate/collector/vllm/collect_msa_module.py b/python/aisimulate/collector/vllm/collect_msa_module.py index 8ccfdf34fc..713d08bb4e 100644 --- a/python/aisimulate/collector/vllm/collect_msa_module.py +++ b/python/aisimulate/collector/vllm/collect_msa_module.py @@ -317,7 +317,7 @@ def msa_decode_scorer_kernel_limit( decode_scorer_runs = (not is_context) or (query_len is not None and query_len <= 1) if decode_scorer_runs and num_idx_heads & (num_idx_heads - 1): return ( - f"kernel-limit: num_idx_heads={num_idx_heads} (from num_heads={num_heads}) " + f"FIXME(kernel-limit): num_idx_heads={num_idx_heads} (from num_heads={num_heads}) " "is not a power of 2; MiniMax-M3 sparse decode indexer Triton kernel requires " "power-of-2 tl.arange tiles (models/minimax_m3/common/ops/" "sparse_attn.py@0.29.0) and serving tp shards can never produce " diff --git a/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py b/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py index e00d1ecc85..673981452d 100644 --- a/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py +++ b/python/aisimulate/collector/wideep/trtllm/collect_moe_compute.py @@ -897,8 +897,8 @@ def run_moe_ep( # Re-verify on the next framework version bump. if moe_type == "fp8_block" and get_sm_version() != 90: raise ValueError( - f"WideEP fp8_block grouped GEMM is DeepGEMM-backed and SM90-only in " - f"trtllm 1.3.0rc20 (deep_gemm/compiler.cuh:330); got SM{get_sm_version()}" + f"FIXME(kernel-limit): WideEP fp8_block grouped GEMM is DeepGEMM-backed and SM90-only " + f"(deep_gemm/compiler.cuh:330 @1.3.0rc20, unchanged @1.3.0rc29); got SM{get_sm_version()}" ) _install_slot_lora_pad_patch() diff --git a/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py b/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py index dd84b03968..f0472d92d6 100644 --- a/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py +++ b/python/aisimulate/tests/unit/collector/test_msa_kernel_limit_scope.py @@ -26,7 +26,7 @@ def _predicate(): @pytest.mark.parametrize("idx_heads,num_heads", [(3, 48), (6, 96)]) def test_non_power_of_two_index_heads_fail_only_where_the_decode_scorer_runs(idx_heads, num_heads): limit = _predicate() - assert limit(idx_heads, num_heads, is_context=False, query_len=1) # generation + assert limit(idx_heads, num_heads, is_context=False, query_len=1).startswith("FIXME(kernel-limit):") # generation assert limit(idx_heads, num_heads, is_context=True, query_len=1) # one-token context query assert limit(idx_heads, num_heads, is_context=True, query_len=16) is None # prefill path assert limit(idx_heads, num_heads, is_context=True, query_len=32768) is None diff --git a/python/aisimulate/tests/unit/collector/test_sglang_attention_0514.py b/python/aisimulate/tests/unit/collector/test_sglang_attention_0514.py index 7c51b4b811..29e06ccb46 100644 --- a/python/aisimulate/tests/unit/collector/test_sglang_attention_0514.py +++ b/python/aisimulate/tests/unit/collector/test_sglang_attention_0514.py @@ -323,6 +323,7 @@ def test_sglang_attention_runtime_fields_are_not_persisted_dimensions(): @pytest.mark.unit +@pytest.mark.parametrize("kernel_source", ["triton", "flashinfer"]) # flashinfer refuses it in the worker @pytest.mark.parametrize( ("sm_version", "num_heads", "num_kv_heads", "head_dim", "max_kv_elements"), [ @@ -333,6 +334,7 @@ def test_sglang_attention_runtime_fields_are_not_persisted_dimensions(): ], ) def test_blackwell_context_cases_are_not_silently_removed( + kernel_source, sm_version, num_heads, num_kv_heads, @@ -358,7 +360,7 @@ def test_blackwell_context_cases_are_not_silently_removed( attention_chunk_size=None, has_attention_sink=False, scaling=None, - kernel_source="triton", + kernel_source=kernel_source, architecture="TestForCausalLM", ) getter = _load_collector_function( @@ -388,12 +390,12 @@ def _run_attention_torch_source() -> str: @pytest.mark.unit def test_fp8_prefill_labels_follow_backend_compute_dtype(): - """FP8 prefill labeling mirrors SGLang 0.5.14 per-backend compute truth. + """FP8 prefill labeling mirrors SGLang 0.5.21 per-backend compute truth. flashinfer has no FP8 prefill compute path at all (BF16 Q reads the FP8 KV cache with descales) and must reject the fp8_context_fmha case. trtllm_mha - (TRTLLM-GEN) requires BF16 inputs but quantizes Q internally via - scaled_fp8_quant (trtllm_mha_backend.py:154-158, 291-301), so its + (TRTLLM-GEN) takes BF16 inputs but casts Q to FP8 itself + (trtllm_mha_backend.py:1600-1609), so its fp8_context_fmha case runs WITHOUT an external cast — the FP8 compute happens inside the backend. """ @@ -404,6 +406,8 @@ def test_fp8_prefill_labels_follow_backend_compute_dtype(): # trtllm_mha must be exempt from the external cast, not rejected. assert 'attn_backend_name != "trtllm_mha"' in source assert 'attn_backend_name in {"flashinfer", "trtllm_mha"}' not in source + # the refusal runs before the model runner is built: the queued case fails cheaply + assert source.index("has no FP8 prefill compute path") < source.index("MockModelRunner(") @pytest.mark.unit @@ -411,7 +415,7 @@ def test_bf16_prefill_on_fp8_kv_fails_closed_on_internally_quantizing_backends() """A BF16-compute prefill on an FP8 KV cache does not exist on fa3/trtllm_mha. fa3 casts Q to the KV dtype whenever the cache is FP8 and head_dim <= 256 - (flashattention_backend.py:857-872); TRTLLM-GEN always quantizes Q when the + (flashattention_backend.py:1322-1332); TRTLLM-GEN always quantizes Q when the cache is FP8. Collecting that combination would record FP8 compute under a bfloat16 attn_dtype label, so it must raise instead. """ From 3e6f2ccba52ed0d8d5c23d073f4946b7543ba4f3 Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Sat, 31 Oct 2026 00:28:47 +0800 Subject: [PATCH 12/13] fix: the vLLM compute_scale exception covers SM89, where the probe observed it The op-probe sm89 vllm-0.30.0 entry (benchmarks/evidence/collector/ op-probe-2026-10/findings.yaml) records compute_scale as the same governed approximation as on sm90: torch.compile folds the activation quant, so the standalone row has no serving instance. The scope listed only h100_sxm and h200_sxm, leaving l40s outside it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- python/aisimulate/collector/evidence_exceptions.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/aisimulate/collector/evidence_exceptions.yaml b/python/aisimulate/collector/evidence_exceptions.yaml index f5a55ed715..6efd7f3090 100644 --- a/python/aisimulate/collector/evidence_exceptions.yaml +++ b/python/aisimulate/collector/evidence_exceptions.yaml @@ -99,7 +99,7 @@ standards_exceptions: tables: [scale_matrix_perf] backend: vllm versions: all - systems: [h100_sxm, h200_sxm] + systems: [l40s, h100_sxm, h200_sxm] # SM89 too: op-probe-2026-10 findings.yaml, sm89 vllm-0.30.0 entry retirement: "model the fused quant (fold compute_scale into the gemm row when the deployment compiles) or drop the standalone row for compiled deployments" approved_by: tianhaox verified_on: "0.30.0" # opharness compute_scale gate, 2026-09-27; compute_scale is pinned to 0.30.0 From 78e42f49c82221bcef17c75d40befa505721338b Mon Sep 17 00:00:00 2001 From: Tianhao Xu Date: Sat, 31 Oct 2026 14:47:25 +0800 Subject: [PATCH 13/13] fix: address the round-3 review of the collector runtime upgrade Blocking: - SGLang full-node ops took no `replay` keyword, so collect_sglang raised a TypeError after the pool ops and lost the summary and provenance. collect_sglang_fullnode_op now accepts replay and selects cases in the pool ops' order. A test checks every keyword collect.py passes. - TRT-LLM MoE gates allocate e_score_correction_bias with torch.empty, and only load_weights fills it, so leftover memory picked the experts. _serving_moe_gate zeroes it, also on the min-latency gate, and passes moe_backend upper-cased (TRTLLM keeps a bf16 bias). DeepSeek-V4 gets DeepseekV4Gate (scored layer) and MiniMax-M3 gets MiniMaxM3Gate, as rc29 serving builds them. - vLLM DSA page: serving sizes the cache block from the first non-SSM attention layer, the indexer cache, which accepts only 64 on CUDA. framework_kv_block_size takes that cache backend (current_platform._find_non_ssm_backend), so FLASHINFER_MLA_SPARSE runs at 64, not 32. vLLM DSA series: - The module is sized at the model's max_position_embeddings. The indexer buffers scale with max_model_len, so a series-max size bent batch scaling. - A failed point's traceback frames are cleared, so its memory does not stay pinned for the smaller points. - FIXME(gap) note: masked-MHA prefill needs model-level heads x TP, which a TP=1 per-rank module never reaches. Collector harness: - helper.run_measurement_subprocess beats the stall watchdog while a child's output grows, and on Linux kills the child with its worker (PR_SET_PDEATHSIG). The SGLang MLA/DSA and MSA collectors use it. - --cases-from refuses an output root that already holds results and never finalizes its CSV staging into parquet or collection_meta.yaml. - SGLang mHC/MLA cleanup resets the 0.5.21 expert-location resource (runtime_compat.reset_expert_location_metadata). - collect_campaign passes AISIM_COLLECTOR_REF (clean-tree HEAD, 40-64 hex) to every shard. Kernel-limit guards: - SGLang MoE: the FlashInfer CUTLASS BF16 %8, SM90 DSV4 FP4 %128 and Triton fp8_block %128 raises now cite their 0.5.21 / FlashInfer 0.6.18 source lines. The int4 group-size raise has no source backing, so it loses the prefix and carries FIXME(re-verify) with the kernels' real limits. - The TRT-LLM fp8 GEMM n/k%16 raise is deleted. The checked-in rc29 l40s and rtx_pro_6000 tables hold 120 rows each that it refused, and SM90+ fails naturally with a cheap TORCH_CHECK. - The SGLang GEMM gap note adds FlashInfer split-K for the 68 tuned bf16 shapes and the fp8_block DeepGEMM-to-Triton fallback for N%64 / K%128. Generator: - The rc29 template renders AUTO, the rc29 default, when no MoE backend is set. - hardware.yaml keeps WIDEEP for Blackwell, and apply_moe_backend drops it from TRT-LLM 1.3.0rc25, the first release without the literal. - 64-token rounding of caller-evaluated limits (preserve_engine_limits) logs the change. - The Sweeper (benchmark rule) renders the MiniMax-M3 sparse-attention algorithm/implementation keys. Other: - A test asserts every standards exception's verified_on equals the collector pin of its tables. - The GLM-5.3-Flash SGLang plan no longer schedules kda/mhc_module. - kernel_source_backends.yaml cites the retired 0520 file at c540ff90a. - Stale hca_attn/csa_attn Rust comments are updated. - The PR244 audit excludes the retired hca table, and its manifest is regenerated. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Tianhao Xu --- benchmarks/evidence/collector/pr244/README.md | 13 +- .../evidence/collector/pr244/manifest.json | 1118 ++--------------- crates/core/src/perfmodel/operators/dsv4.rs | 4 +- .../core/src/perfmodel/perf_database/dsv4.rs | 6 +- .../src/perfmodel/perf_database/table_view.rs | 6 +- ...lm5NextForConditionalGeneration_cases.yaml | 13 +- python/aisimulate/collector/collect.py | 37 +- python/aisimulate/collector/fullnode.py | 9 + python/aisimulate/collector/helper.py | 51 + .../collector/kernel_source_backends.yaml | 8 +- .../collector/sglang/collect_gemm.py | 22 +- .../collector/sglang/collect_mhc_module.py | 12 +- .../collector/sglang/collect_mla_module.py | 47 +- .../collector/sglang/collect_moe.py | 31 +- .../collector/sglang/collect_msa_module.py | 24 +- .../collector/trtllm/collect_gemm.py | 11 - .../collector/trtllm/collect_moe.py | 85 +- .../collector/vllm/collect_mla_module.py | 47 +- python/aisimulate/collector/vllm/utils.py | 19 +- .../extra_engine_args.1.3.0rc29.yaml.j2 | 6 +- .../src/aisimulate/generator/facts/apply.py | 24 +- .../aisimulate/generator/facts/hardware.yaml | 34 +- .../aisimulate/generator/rendering/engine.py | 2 +- .../generator/rendering/rule_engine.py | 17 +- .../aisimulate/generator/request/sweeper.py | 12 + .../rule_plugin/benchmark/trtllm.rule | 7 + .../sglang/test_collect_mla_module.py | 36 +- .../test_collect_mla_module_native_quant.py | 3 +- .../unit/collector/test_collect_replay.py | 11 + .../tests/unit/collector/test_fullnode.py | 65 + .../test_glm5next_collection_plan.py | 13 +- .../collector/test_measurement_subprocess.py | 48 + .../test_standards_exceptions_verified_on.py | 38 + .../unit/collector/test_vllm_kv_block_size.py | 75 ++ .../collector/test_vllm_mla_graph_timing.py | 2 +- .../trtllm/test_collect_gemm_op_cache.py | 7 - .../collector/trtllm/test_moe_serving_gate.py | 78 ++ .../test_engine_limit_preservation.py | 12 + .../generator/test_facts_hardware_l40s.py | 2 +- .../unit/generator/test_sweeper_request.py | 26 + .../test_trtllm_moe_backend_default.py | 70 ++ .../tests/unit/tools/test_collect_campaign.py | 31 +- .../tools/perf_database/collect_campaign.py | 75 +- scripts/audit_pr244_data.py | 8 +- 44 files changed, 1101 insertions(+), 1164 deletions(-) create mode 100644 python/aisimulate/tests/unit/collector/test_measurement_subprocess.py create mode 100644 python/aisimulate/tests/unit/collector/test_standards_exceptions_verified_on.py create mode 100644 python/aisimulate/tests/unit/collector/test_vllm_kv_block_size.py create mode 100644 python/aisimulate/tests/unit/collector/trtllm/test_moe_serving_gate.py create mode 100644 python/aisimulate/tests/unit/generator/test_trtllm_moe_backend_default.py diff --git a/benchmarks/evidence/collector/pr244/README.md b/benchmarks/evidence/collector/pr244/README.md index 76be4c224b..f0caba714e 100644 --- a/benchmarks/evidence/collector/pr244/README.md +++ b/benchmarks/evidence/collector/pr244/README.md @@ -6,8 +6,17 @@ SPDX-License-Identifier: Apache-2.0 # Collection audit evidence This non-packaged bundle verifies the retained vLLM 0.25.0 collection outcomes -and the 85 shipped Parquet tables. It contains case IDs, status vectors, -checksums, and table metadata, not diagnostic logs or machine paths. +and the 80 Parquet tables still shipped from it (85 originally). It contains case +IDs, status vectors, checksums, and table metadata, not diagnostic logs or +machine paths. + +The five `dsv4_hca_attn_module_perf` tables (one per system) were removed with +the hca_attn sidecar's SDK reader in #649. The case-outcome ledger keeps their +reported hashes, which record what was collected. The audit's `RETIRED_TABLES` +excludes them from the inventory check. The manifest was regenerated: it +drops their entries and carries the new hashes of the five 0.25.0 +`collection_meta.yaml` sidecars that listed them. Every other table entry is +unchanged. Run from the repository root in the development environment: diff --git a/benchmarks/evidence/collector/pr244/manifest.json b/benchmarks/evidence/collector/pr244/manifest.json index 960572fcfa..190122b1f9 100644 --- a/benchmarks/evidence/collector/pr244/manifest.json +++ b/benchmarks/evidence/collector/pr244/manifest.json @@ -253,8 +253,8 @@ "failed": 10739, "unattempted": 0 }, - "rows": 420111, - "table_count": 17, + "rows": 416193, + "table_count": 16, "tables": { "context_attention_perf.parquet": { "anomalies": { @@ -569,7 +569,7 @@ ], "rows": 5840, "sha256": "63e9b6b89418403e1998e691e2f93a5be67fd930a7997dac0d248ba0f577d7c3", - "sidecar_sha256": "bebbf880c0df4effe3b4d384e67984ddd36e254a83de2f0504e792646d6c8061" + "sidecar_sha256": "1ec714136ae98e456690f8773a44feba01be9e9a6f89f549b312a57596174af3" }, "dsv4_csa_generation_module_perf.parquet": { "anomalies": { @@ -731,177 +731,7 @@ ], "rows": 1544, "sha256": "632a9063ce5f857ab95731449176045fbde3551efcaefb35a93feea29549e5da", - "sidecar_sha256": "bebbf880c0df4effe3b4d384e67984ddd36e254a83de2f0504e792646d6c8061" - }, - "dsv4_hca_attn_module_perf.parquet": { - "anomalies": { - "duplicate_physical_keys": 0, - "nonfinite_latency": 0, - "nonpositive_latency": 0, - "null_cells": 0 - }, - "columns": { - "architecture": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "DeepseekV4ForCausalLM" - ] - }, - "batch_size": { - "distinct": 11, - "max": 1024, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "compress_ratio": { - "distinct": 1, - "max": 128, - "min": 128, - "nulls": 0, - "type": "int64" - }, - "device": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "NVIDIA B300 SXM6 AC" - ] - }, - "framework": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "VLLM" - ] - }, - "gemm_type": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8_block" - ] - }, - "isl": { - "distinct": 16, - "max": 8192, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "kernel_source": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "FLASHMLA_SPARSE_DSV4" - ] - }, - "kv_cache_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8" - ] - }, - "latency": { - "distinct": 1955, - "max": 5.2688, - "min": 0.0086, - "nulls": 0, - "type": "double" - }, - "local_num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "mla_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "bfloat16" - ] - }, - "model": { - "distinct": 2, - "nulls": 0, - "type": "string", - "values": [ - "sgl-project/DeepSeek-V4-Flash-FP8", - "sgl-project/DeepSeek-V4-Pro-FP8" - ] - }, - "num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "op_name": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "dsv4_hca_attn_module" - ] - }, - "step": { - "distinct": 26, - "max": 1048575, - "min": 0, - "nulls": 0, - "type": "int64" - }, - "tp_size": { - "distinct": 1, - "max": 1, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "version": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "0.25.0" - ] - } - }, - "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/b300_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet", - "physical_key_columns": [ - "framework", - "version", - "op_name", - "kernel_source", - "model", - "architecture", - "mla_dtype", - "kv_cache_dtype", - "gemm_type", - "num_heads", - "local_num_heads", - "batch_size", - "isl", - "tp_size", - "step", - "compress_ratio" - ], - "rows": 3918, - "sha256": "cc616dbae378df19da3788d0d0b632b220144a60fb6067bbcb95cc94afe79513", - "sidecar_sha256": "bebbf880c0df4effe3b4d384e67984ddd36e254a83de2f0504e792646d6c8061" + "sidecar_sha256": "1ec714136ae98e456690f8773a44feba01be9e9a6f89f549b312a57596174af3" }, "dsv4_hca_context_module_perf.parquet": { "anomalies": { @@ -1063,7 +893,7 @@ ], "rows": 5840, "sha256": "fe349c63ed6445f442f2298e47456c37ffef5f1c43d458cf5d57538d282e581e", - "sidecar_sha256": "bebbf880c0df4effe3b4d384e67984ddd36e254a83de2f0504e792646d6c8061" + "sidecar_sha256": "1ec714136ae98e456690f8773a44feba01be9e9a6f89f549b312a57596174af3" }, "dsv4_hca_generation_module_perf.parquet": { "anomalies": { @@ -1225,7 +1055,7 @@ ], "rows": 1544, "sha256": "cdc48ea8a89e6b5543720df991f5f11d691b56349b1ab6a042bf6cbb550e6f62", - "sidecar_sha256": "bebbf880c0df4effe3b4d384e67984ddd36e254a83de2f0504e792646d6c8061" + "sidecar_sha256": "1ec714136ae98e456690f8773a44feba01be9e9a6f89f549b312a57596174af3" }, "dsv4_paged_mqa_logits_module_perf.parquet": { "anomalies": { @@ -1395,7 +1225,7 @@ ], "rows": 3918, "sha256": "64de2420ed80948ca068da0fc1e014734e53e6a77d71ab224e566b350be2280f", - "sidecar_sha256": "bebbf880c0df4effe3b4d384e67984ddd36e254a83de2f0504e792646d6c8061" + "sidecar_sha256": "1ec714136ae98e456690f8773a44feba01be9e9a6f89f549b312a57596174af3" }, "encoder_attention_perf.parquet": { "anomalies": { @@ -3034,8 +2864,8 @@ "failed": 10766, "unattempted": 0 }, - "rows": 420134, - "table_count": 17, + "rows": 416216, + "table_count": 16, "tables": { "context_attention_perf.parquet": { "anomalies": { @@ -3350,7 +3180,7 @@ ], "rows": 5840, "sha256": "76719d88e0481b5153c89ef0e1024435597f660de305bdb9f01750e9273189d3", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "dsv4_csa_generation_module_perf.parquet": { "anomalies": { @@ -3512,9 +3342,9 @@ ], "rows": 1544, "sha256": "b1a09a8e004ebed907e1d6d85379c51f5ec4ca2c5812bb59a62327c99f2dc505", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, - "dsv4_hca_attn_module_perf.parquet": { + "dsv4_hca_context_module_perf.parquet": { "anomalies": { "duplicate_physical_keys": 0, "nonfinite_latency": 0, @@ -3569,8 +3399,8 @@ ] }, "isl": { - "distinct": 16, - "max": 8192, + "distinct": 21, + "max": 65536, "min": 1, "nulls": 0, "type": "int64" @@ -3592,19 +3422,12 @@ ] }, "latency": { - "distinct": 1985, - "max": 5.0301, - "min": 0.0106, + "distinct": 4931, + "max": 120.7941, + "min": 0.0436, "nulls": 0, "type": "double" }, - "local_num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, "mla_dtype": { "distinct": 1, "nulls": 0, @@ -3623,9 +3446,9 @@ ] }, "num_heads": { - "distinct": 2, + "distinct": 5, "max": 128, - "min": 64, + "min": 8, "nulls": 0, "type": "int64" }, @@ -3634,19 +3457,19 @@ "nulls": 0, "type": "string", "values": [ - "dsv4_hca_attn_module" + "dsv4_hca_context_module" ] }, "step": { - "distinct": 26, - "max": 1048575, + "distinct": 24, + "max": 65535, "min": 0, "nulls": 0, "type": "int64" }, "tp_size": { - "distinct": 1, - "max": 1, + "distinct": 4, + "max": 8, "min": 1, "nulls": 0, "type": "int64" @@ -3661,7 +3484,7 @@ } }, "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet", + "path": "python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.25.0/dsv4_hca_context_module_perf.parquet", "physical_key_columns": [ "framework", "version", @@ -3673,18 +3496,17 @@ "kv_cache_dtype", "gemm_type", "num_heads", - "local_num_heads", "batch_size", "isl", "tp_size", "step", "compress_ratio" ], - "rows": 3918, - "sha256": "0abf339525484c011c1c57508bbf01d1338f98f93cbc3d0debc6b2b383b2d0dc", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "rows": 5840, + "sha256": "633e6a30fc69935c0edd9fb672d26a02ba3d13a36e5412f5dab57571b37c7ec1", + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, - "dsv4_hca_context_module_perf.parquet": { + "dsv4_hca_generation_module_perf.parquet": { "anomalies": { "duplicate_physical_keys": 0, "nonfinite_latency": 0, @@ -3739,8 +3561,8 @@ ] }, "isl": { - "distinct": 21, - "max": 65536, + "distinct": 1, + "max": 1, "min": 1, "nulls": 0, "type": "int64" @@ -3762,9 +3584,9 @@ ] }, "latency": { - "distinct": 4931, - "max": 120.7941, - "min": 0.0436, + "distinct": 740, + "max": 0.5519, + "min": 0.0421, "nulls": 0, "type": "double" }, @@ -3797,13 +3619,13 @@ "nulls": 0, "type": "string", "values": [ - "dsv4_hca_context_module" + "dsv4_hca_generation_module" ] }, "step": { - "distinct": 24, - "max": 65535, - "min": 0, + "distinct": 21, + "max": 65536, + "min": 1, "nulls": 0, "type": "int64" }, @@ -3824,7 +3646,7 @@ } }, "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.25.0/dsv4_hca_context_module_perf.parquet", + "path": "python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.25.0/dsv4_hca_generation_module_perf.parquet", "physical_key_columns": [ "framework", "version", @@ -3842,11 +3664,11 @@ "step", "compress_ratio" ], - "rows": 5840, - "sha256": "633e6a30fc69935c0edd9fb672d26a02ba3d13a36e5412f5dab57571b37c7ec1", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "rows": 1544, + "sha256": "9807d59d5b0e6a14a5d82feb7b81c8529fbe2d0d65120fd32706a7b68fe4cf2a", + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, - "dsv4_hca_generation_module_perf.parquet": { + "dsv4_paged_mqa_logits_module_perf.parquet": { "anomalies": { "duplicate_physical_keys": 0, "nonfinite_latency": 0, @@ -3871,8 +3693,8 @@ }, "compress_ratio": { "distinct": 1, - "max": 128, - "min": 128, + "max": 4, + "min": 4, "nulls": 0, "type": "int64" }, @@ -3901,8 +3723,8 @@ ] }, "isl": { - "distinct": 1, - "max": 1, + "distinct": 16, + "max": 8192, "min": 1, "nulls": 0, "type": "int64" @@ -3912,7 +3734,7 @@ "nulls": 0, "type": "string", "values": [ - "FLASHMLA_SPARSE_DSV4" + "vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits" ] }, "kv_cache_dtype": { @@ -3924,18 +3746,25 @@ ] }, "latency": { - "distinct": 740, - "max": 0.5519, - "min": 0.0421, + "distinct": 992, + "max": 11.271, + "min": 0.0062, "nulls": 0, "type": "double" }, + "local_num_heads": { + "distinct": 2, + "max": 128, + "min": 64, + "nulls": 0, + "type": "int64" + }, "mla_dtype": { "distinct": 1, "nulls": 0, "type": "string", "values": [ - "bfloat16" + "fp8_e4m3" ] }, "model": { @@ -3948,9 +3777,9 @@ ] }, "num_heads": { - "distinct": 5, + "distinct": 2, "max": 128, - "min": 8, + "min": 64, "nulls": 0, "type": "int64" }, @@ -3959,176 +3788,7 @@ "nulls": 0, "type": "string", "values": [ - "dsv4_hca_generation_module" - ] - }, - "step": { - "distinct": 21, - "max": 65536, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "tp_size": { - "distinct": 4, - "max": 8, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "version": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "0.25.0" - ] - } - }, - "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/gb200/sparse_attention/vllm/0.25.0/dsv4_hca_generation_module_perf.parquet", - "physical_key_columns": [ - "framework", - "version", - "op_name", - "kernel_source", - "model", - "architecture", - "mla_dtype", - "kv_cache_dtype", - "gemm_type", - "num_heads", - "batch_size", - "isl", - "tp_size", - "step", - "compress_ratio" - ], - "rows": 1544, - "sha256": "9807d59d5b0e6a14a5d82feb7b81c8529fbe2d0d65120fd32706a7b68fe4cf2a", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" - }, - "dsv4_paged_mqa_logits_module_perf.parquet": { - "anomalies": { - "duplicate_physical_keys": 0, - "nonfinite_latency": 0, - "nonpositive_latency": 0, - "null_cells": 0 - }, - "columns": { - "architecture": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "DeepseekV4ForCausalLM" - ] - }, - "batch_size": { - "distinct": 11, - "max": 1024, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "compress_ratio": { - "distinct": 1, - "max": 4, - "min": 4, - "nulls": 0, - "type": "int64" - }, - "device": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "NVIDIA GB200" - ] - }, - "framework": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "VLLM" - ] - }, - "gemm_type": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8_block" - ] - }, - "isl": { - "distinct": 16, - "max": 8192, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "kernel_source": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "vllm.utils.deep_gemm.fp8_fp4_paged_mqa_logits" - ] - }, - "kv_cache_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8" - ] - }, - "latency": { - "distinct": 992, - "max": 11.271, - "min": 0.0062, - "nulls": 0, - "type": "double" - }, - "local_num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "mla_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8_e4m3" - ] - }, - "model": { - "distinct": 2, - "nulls": 0, - "type": "string", - "values": [ - "sgl-project/DeepSeek-V4-Flash-FP8", - "sgl-project/DeepSeek-V4-Pro-FP8" - ] - }, - "num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "op_name": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "dsv4_paged_mqa_logits_module" + "dsv4_paged_mqa_logits_module" ] }, "step": { @@ -4176,7 +3836,7 @@ ], "rows": 3918, "sha256": "e62d12cc3c8a125b66ad066de2abfda101c06a6acdbf9f7d0e9bebd4f442f4c3", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "encoder_attention_perf.parquet": { "anomalies": { @@ -5815,8 +5475,8 @@ "failed": 10699, "unattempted": 0 }, - "rows": 420226, - "table_count": 17, + "rows": 416308, + "table_count": 16, "tables": { "context_attention_perf.parquet": { "anomalies": { @@ -6131,7 +5791,7 @@ ], "rows": 5840, "sha256": "81276eea4e424b2146b58256cd1697529a3a00c8bf3ee7c6613d54d6094972fc", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "dsv4_csa_generation_module_perf.parquet": { "anomalies": { @@ -6293,177 +5953,7 @@ ], "rows": 1544, "sha256": "eb54cfe271dc3d15d0849745c17cb1979da4ac4afc63504acece52bcf4482d08", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" - }, - "dsv4_hca_attn_module_perf.parquet": { - "anomalies": { - "duplicate_physical_keys": 0, - "nonfinite_latency": 0, - "nonpositive_latency": 0, - "null_cells": 0 - }, - "columns": { - "architecture": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "DeepseekV4ForCausalLM" - ] - }, - "batch_size": { - "distinct": 11, - "max": 1024, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "compress_ratio": { - "distinct": 1, - "max": 128, - "min": 128, - "nulls": 0, - "type": "int64" - }, - "device": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "NVIDIA GB300" - ] - }, - "framework": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "VLLM" - ] - }, - "gemm_type": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8_block" - ] - }, - "isl": { - "distinct": 16, - "max": 8192, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "kernel_source": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "FLASHMLA_SPARSE_DSV4" - ] - }, - "kv_cache_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8" - ] - }, - "latency": { - "distinct": 1920, - "max": 4.9961, - "min": 0.0106, - "nulls": 0, - "type": "double" - }, - "local_num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "mla_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "bfloat16" - ] - }, - "model": { - "distinct": 2, - "nulls": 0, - "type": "string", - "values": [ - "sgl-project/DeepSeek-V4-Flash-FP8", - "sgl-project/DeepSeek-V4-Pro-FP8" - ] - }, - "num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "op_name": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "dsv4_hca_attn_module" - ] - }, - "step": { - "distinct": 26, - "max": 1048575, - "min": 0, - "nulls": 0, - "type": "int64" - }, - "tp_size": { - "distinct": 1, - "max": 1, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "version": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "0.25.0" - ] - } - }, - "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/gb300/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet", - "physical_key_columns": [ - "framework", - "version", - "op_name", - "kernel_source", - "model", - "architecture", - "mla_dtype", - "kv_cache_dtype", - "gemm_type", - "num_heads", - "local_num_heads", - "batch_size", - "isl", - "tp_size", - "step", - "compress_ratio" - ], - "rows": 3918, - "sha256": "49ecc0d7e34e1503f7bdbea1c3599bb20a219488786a22fef43cf216de641383", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "dsv4_hca_context_module_perf.parquet": { "anomalies": { @@ -6625,7 +6115,7 @@ ], "rows": 5840, "sha256": "56d026f2e2b5b843c0afbe6b6c0d9b3489abf5db256d5d61ee065f7bcd72d15f", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "dsv4_hca_generation_module_perf.parquet": { "anomalies": { @@ -6787,7 +6277,7 @@ ], "rows": 1544, "sha256": "3dee3c879fe7fdd9fd1e111d2f8df79cf15d8e635ebe32f123b94adc441fc31e", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "dsv4_paged_mqa_logits_module_perf.parquet": { "anomalies": { @@ -6957,7 +6447,7 @@ ], "rows": 3918, "sha256": "12f3aa8b290193bf97e2f144c9bdda347eeba8ab10698669bc41d1fa617280fd", - "sidecar_sha256": "ff4b25e84cb1c03a7c2b6bec9adc91e731a2dd57bb96d72089e13a4440d0d674" + "sidecar_sha256": "7fa57194fb7ad77faea184f07ef6c07ed293371aa4ac9429a9becf2fb2b4782d" }, "encoder_attention_perf.parquet": { "anomalies": { @@ -8586,8 +8076,8 @@ "failed": 3495, "unattempted": 0 }, - "rows": 353449, - "table_count": 17, + "rows": 349531, + "table_count": 16, "tables": { "context_attention_perf.parquet": { "anomalies": { @@ -8710,200 +8200,38 @@ "values": [ "0.25.0" ] - }, - "window_size": { - "distinct": 6, - "max": 8192, - "min": 0, - "nulls": 0, - "type": "int64" - } - }, - "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/attention/vllm/0.25.0/context_attention_perf.parquet", - "physical_key_columns": [ - "framework", - "version", - "op_name", - "kernel_source", - "batch_size", - "isl", - "num_heads", - "num_key_value_heads", - "head_dim", - "beam_width", - "attn_dtype", - "kv_cache_dtype", - "step", - "window_size" - ], - "rows": 53771, - "sha256": "7bca62aa7d140efcd54dadb60ff7582de130f7693265c26a53a817e92a601f9a", - "sidecar_sha256": "9c22cf1a4cb9d50fad792d7e319e933b10cf57feae22c0fb90a46127ce12fc46" - }, - "dsv4_csa_context_module_perf.parquet": { - "anomalies": { - "duplicate_physical_keys": 0, - "nonfinite_latency": 0, - "nonpositive_latency": 0, - "null_cells": 0 - }, - "columns": { - "architecture": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "DeepseekV4ForCausalLM" - ] - }, - "batch_size": { - "distinct": 11, - "max": 1024, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "compress_ratio": { - "distinct": 1, - "max": 4, - "min": 4, - "nulls": 0, - "type": "int64" - }, - "device": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "NVIDIA H100 80GB HBM3" - ] - }, - "framework": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "VLLM" - ] - }, - "gemm_type": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8_block" - ] - }, - "isl": { - "distinct": 21, - "max": 65536, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "kernel_source": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "FLASHMLA_SPARSE_DSV4" - ] - }, - "kv_cache_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8" - ] - }, - "latency": { - "distinct": 5417, - "max": 357.5721, - "min": 0.0702, - "nulls": 0, - "type": "double" - }, - "mla_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "bfloat16" - ] - }, - "model": { - "distinct": 2, - "nulls": 0, - "type": "string", - "values": [ - "sgl-project/DeepSeek-V4-Flash-FP8", - "sgl-project/DeepSeek-V4-Pro-FP8" - ] - }, - "num_heads": { - "distinct": 5, - "max": 128, - "min": 8, - "nulls": 0, - "type": "int64" - }, - "op_name": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "dsv4_csa_context_module" - ] - }, - "step": { - "distinct": 24, - "max": 65535, - "min": 0, - "nulls": 0, - "type": "int64" - }, - "tp_size": { - "distinct": 4, - "max": 8, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "version": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "0.25.0" - ] + }, + "window_size": { + "distinct": 6, + "max": 8192, + "min": 0, + "nulls": 0, + "type": "int64" } }, "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_csa_context_module_perf.parquet", + "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/attention/vllm/0.25.0/context_attention_perf.parquet", "physical_key_columns": [ "framework", "version", "op_name", "kernel_source", - "model", - "architecture", - "mla_dtype", - "kv_cache_dtype", - "gemm_type", - "num_heads", "batch_size", "isl", - "tp_size", + "num_heads", + "num_key_value_heads", + "head_dim", + "beam_width", + "attn_dtype", + "kv_cache_dtype", "step", - "compress_ratio" + "window_size" ], - "rows": 5789, - "sha256": "fd4664e02b104be261b238db5dd571544dc35018ffa6e5fdda700f68e0503a79", - "sidecar_sha256": "19017dfd3c09a7df0dc5dd4d47e01ff0d9324998f3fe4f93b999effe36bfe8aa" + "rows": 53771, + "sha256": "7bca62aa7d140efcd54dadb60ff7582de130f7693265c26a53a817e92a601f9a", + "sidecar_sha256": "9c22cf1a4cb9d50fad792d7e319e933b10cf57feae22c0fb90a46127ce12fc46" }, - "dsv4_csa_generation_module_perf.parquet": { + "dsv4_csa_context_module_perf.parquet": { "anomalies": { "duplicate_physical_keys": 0, "nonfinite_latency": 0, @@ -8958,8 +8286,8 @@ ] }, "isl": { - "distinct": 1, - "max": 1, + "distinct": 21, + "max": 65536, "min": 1, "nulls": 0, "type": "int64" @@ -8981,9 +8309,9 @@ ] }, "latency": { - "distinct": 1031, - "max": 1.3585, - "min": 0.0695, + "distinct": 5417, + "max": 357.5721, + "min": 0.0702, "nulls": 0, "type": "double" }, @@ -9016,13 +8344,13 @@ "nulls": 0, "type": "string", "values": [ - "dsv4_csa_generation_module" + "dsv4_csa_context_module" ] }, "step": { - "distinct": 21, - "max": 65536, - "min": 1, + "distinct": 24, + "max": 65535, + "min": 0, "nulls": 0, "type": "int64" }, @@ -9043,7 +8371,7 @@ } }, "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_csa_generation_module_perf.parquet", + "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_csa_context_module_perf.parquet", "physical_key_columns": [ "framework", "version", @@ -9061,11 +8389,11 @@ "step", "compress_ratio" ], - "rows": 1544, - "sha256": "c9a6ff6f081dee79ff3053d32a8e85d550ee7aaab0c3d080bb2d33d7ca4ab839", - "sidecar_sha256": "19017dfd3c09a7df0dc5dd4d47e01ff0d9324998f3fe4f93b999effe36bfe8aa" + "rows": 5789, + "sha256": "fd4664e02b104be261b238db5dd571544dc35018ffa6e5fdda700f68e0503a79", + "sidecar_sha256": "c17befda9060286fc7bcbe28d086e8964c47644c700e1f60d604ccbed20d6486" }, - "dsv4_hca_attn_module_perf.parquet": { + "dsv4_csa_generation_module_perf.parquet": { "anomalies": { "duplicate_physical_keys": 0, "nonfinite_latency": 0, @@ -9090,8 +8418,8 @@ }, "compress_ratio": { "distinct": 1, - "max": 128, - "min": 128, + "max": 4, + "min": 4, "nulls": 0, "type": "int64" }, @@ -9120,8 +8448,8 @@ ] }, "isl": { - "distinct": 16, - "max": 8192, + "distinct": 1, + "max": 1, "min": 1, "nulls": 0, "type": "int64" @@ -9143,19 +8471,12 @@ ] }, "latency": { - "distinct": 2358, - "max": 14.2497, - "min": 0.01, + "distinct": 1031, + "max": 1.3585, + "min": 0.0695, "nulls": 0, "type": "double" }, - "local_num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, "mla_dtype": { "distinct": 1, "nulls": 0, @@ -9174,9 +8495,9 @@ ] }, "num_heads": { - "distinct": 2, + "distinct": 5, "max": 128, - "min": 64, + "min": 8, "nulls": 0, "type": "int64" }, @@ -9185,19 +8506,19 @@ "nulls": 0, "type": "string", "values": [ - "dsv4_hca_attn_module" + "dsv4_csa_generation_module" ] }, "step": { - "distinct": 26, - "max": 1048575, - "min": 0, + "distinct": 21, + "max": 65536, + "min": 1, "nulls": 0, "type": "int64" }, "tp_size": { - "distinct": 1, - "max": 1, + "distinct": 4, + "max": 8, "min": 1, "nulls": 0, "type": "int64" @@ -9212,7 +8533,7 @@ } }, "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet", + "path": "python/aisimulate/src/aisimulate_core/systems/data/h100_sxm/sparse_attention/vllm/0.25.0/dsv4_csa_generation_module_perf.parquet", "physical_key_columns": [ "framework", "version", @@ -9224,16 +8545,15 @@ "kv_cache_dtype", "gemm_type", "num_heads", - "local_num_heads", "batch_size", "isl", "tp_size", "step", "compress_ratio" ], - "rows": 3918, - "sha256": "86ca415600d3f4b468d683a689403ca298739cc65967c22c66fc2108ae6aa2fe", - "sidecar_sha256": "19017dfd3c09a7df0dc5dd4d47e01ff0d9324998f3fe4f93b999effe36bfe8aa" + "rows": 1544, + "sha256": "c9a6ff6f081dee79ff3053d32a8e85d550ee7aaab0c3d080bb2d33d7ca4ab839", + "sidecar_sha256": "c17befda9060286fc7bcbe28d086e8964c47644c700e1f60d604ccbed20d6486" }, "dsv4_hca_context_module_perf.parquet": { "anomalies": { @@ -9395,7 +8715,7 @@ ], "rows": 5793, "sha256": "9b0d69b9a9634ffd2d58b8e38d402db292ab003f5d4a59cab77a15c983f13d0d", - "sidecar_sha256": "19017dfd3c09a7df0dc5dd4d47e01ff0d9324998f3fe4f93b999effe36bfe8aa" + "sidecar_sha256": "c17befda9060286fc7bcbe28d086e8964c47644c700e1f60d604ccbed20d6486" }, "dsv4_hca_generation_module_perf.parquet": { "anomalies": { @@ -9557,7 +8877,7 @@ ], "rows": 1544, "sha256": "ddeed06756d6e93d3afa9d372587ed19edae41f6051435816b27fc265e7689a8", - "sidecar_sha256": "19017dfd3c09a7df0dc5dd4d47e01ff0d9324998f3fe4f93b999effe36bfe8aa" + "sidecar_sha256": "c17befda9060286fc7bcbe28d086e8964c47644c700e1f60d604ccbed20d6486" }, "dsv4_paged_mqa_logits_module_perf.parquet": { "anomalies": { @@ -9727,7 +9047,7 @@ ], "rows": 3918, "sha256": "8eaca4d81931d542e879f8e1cd9728dda466ecd5d9c19449eef7be5876e90270", - "sidecar_sha256": "19017dfd3c09a7df0dc5dd4d47e01ff0d9324998f3fe4f93b999effe36bfe8aa" + "sidecar_sha256": "c17befda9060286fc7bcbe28d086e8964c47644c700e1f60d604ccbed20d6486" }, "encoder_attention_perf.parquet": { "anomalies": { @@ -11343,8 +10663,8 @@ "failed": 3394, "unattempted": 0 }, - "rows": 353658, - "table_count": 17, + "rows": 349740, + "table_count": 16, "tables": { "context_attention_perf.parquet": { "anomalies": { @@ -11658,7 +10978,7 @@ ], "rows": 5840, "sha256": "8fec19f573b830fd854b0feb199cc6841f8070a86308dc892eec80afdaf93c29", - "sidecar_sha256": "23f9cb39768514c0ef6f079e9bc43abc2719cd464638e371fda85f2922ce7bc1" + "sidecar_sha256": "2c8891395cac294aadefdfe36aa81da7915eb5fb14a581f6a1cc5722957c6d43" }, "dsv4_csa_generation_module_perf.parquet": { "anomalies": { @@ -11820,177 +11140,7 @@ ], "rows": 1544, "sha256": "a4c1dc94f505ba0360125de51735948185b0e9bd44128a42668613875ae7741a", - "sidecar_sha256": "23f9cb39768514c0ef6f079e9bc43abc2719cd464638e371fda85f2922ce7bc1" - }, - "dsv4_hca_attn_module_perf.parquet": { - "anomalies": { - "duplicate_physical_keys": 0, - "nonfinite_latency": 0, - "nonpositive_latency": 0, - "null_cells": 0 - }, - "columns": { - "architecture": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "DeepseekV4ForCausalLM" - ] - }, - "batch_size": { - "distinct": 11, - "max": 1024, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "compress_ratio": { - "distinct": 1, - "max": 128, - "min": 128, - "nulls": 0, - "type": "int64" - }, - "device": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "NVIDIA H200" - ] - }, - "framework": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "VLLM" - ] - }, - "gemm_type": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8_block" - ] - }, - "isl": { - "distinct": 16, - "max": 8192, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "kernel_source": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "FLASHMLA_SPARSE_DSV4" - ] - }, - "kv_cache_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "fp8" - ] - }, - "latency": { - "distinct": 2307, - "max": 14.4426, - "min": 0.0096, - "nulls": 0, - "type": "double" - }, - "local_num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "mla_dtype": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "bfloat16" - ] - }, - "model": { - "distinct": 2, - "nulls": 0, - "type": "string", - "values": [ - "sgl-project/DeepSeek-V4-Flash-FP8", - "sgl-project/DeepSeek-V4-Pro-FP8" - ] - }, - "num_heads": { - "distinct": 2, - "max": 128, - "min": 64, - "nulls": 0, - "type": "int64" - }, - "op_name": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "dsv4_hca_attn_module" - ] - }, - "step": { - "distinct": 26, - "max": 1048575, - "min": 0, - "nulls": 0, - "type": "int64" - }, - "tp_size": { - "distinct": 1, - "max": 1, - "min": 1, - "nulls": 0, - "type": "int64" - }, - "version": { - "distinct": 1, - "nulls": 0, - "type": "string", - "values": [ - "0.25.0" - ] - } - }, - "latency_unit": "ms", - "path": "python/aisimulate/src/aisimulate_core/systems/data/h200_sxm/sparse_attention/vllm/0.25.0/dsv4_hca_attn_module_perf.parquet", - "physical_key_columns": [ - "framework", - "version", - "op_name", - "kernel_source", - "model", - "architecture", - "mla_dtype", - "kv_cache_dtype", - "gemm_type", - "num_heads", - "local_num_heads", - "batch_size", - "isl", - "tp_size", - "step", - "compress_ratio" - ], - "rows": 3918, - "sha256": "3b0d87916ae3f6414cbe92966632b9133fb162a596ffce6b44cfeb916da3a4af", - "sidecar_sha256": "23f9cb39768514c0ef6f079e9bc43abc2719cd464638e371fda85f2922ce7bc1" + "sidecar_sha256": "2c8891395cac294aadefdfe36aa81da7915eb5fb14a581f6a1cc5722957c6d43" }, "dsv4_hca_context_module_perf.parquet": { "anomalies": { @@ -12152,7 +11302,7 @@ ], "rows": 5840, "sha256": "39fa711b357b5c72251862cc741ae81e6a486f6ad0adb2ae29e25565d99642b1", - "sidecar_sha256": "23f9cb39768514c0ef6f079e9bc43abc2719cd464638e371fda85f2922ce7bc1" + "sidecar_sha256": "2c8891395cac294aadefdfe36aa81da7915eb5fb14a581f6a1cc5722957c6d43" }, "dsv4_hca_generation_module_perf.parquet": { "anomalies": { @@ -12314,7 +11464,7 @@ ], "rows": 1544, "sha256": "d9683b5d8a43df2f023bfb4ac50d38f8717b0c447d2ed3962b3ae5eaa64fbf00", - "sidecar_sha256": "23f9cb39768514c0ef6f079e9bc43abc2719cd464638e371fda85f2922ce7bc1" + "sidecar_sha256": "2c8891395cac294aadefdfe36aa81da7915eb5fb14a581f6a1cc5722957c6d43" }, "dsv4_paged_mqa_logits_module_perf.parquet": { "anomalies": { @@ -12484,7 +11634,7 @@ ], "rows": 3918, "sha256": "ad63707a693a23b190807366e1c6eb65cacc33ef42a92b070c23f6985ed0514b", - "sidecar_sha256": "23f9cb39768514c0ef6f079e9bc43abc2719cd464638e371fda85f2922ce7bc1" + "sidecar_sha256": "2c8891395cac294aadefdfe36aa81da7915eb5fb14a581f6a1cc5722957c6d43" }, "encoder_attention_perf.parquet": { "anomalies": { diff --git a/crates/core/src/perfmodel/operators/dsv4.rs b/crates/core/src/perfmodel/operators/dsv4.rs index 4e5ef60039..6a1eda6b83 100644 --- a/crates/core/src/perfmodel/operators/dsv4.rs +++ b/crates/core/src/perfmodel/operators/dsv4.rs @@ -1057,8 +1057,8 @@ mod tests { ); } - /// Real-DB fail-loud: b200_sxm/sglang/0.5.10 now retains only the - /// hca_attn table, so a CSA CP context query must fail LOUD with the + /// Real-DB fail-loud: b200_sxm/sglang/0.5.10 ships no DSV4 module tables + /// (its hca_attn sidecar was retired), so a CSA CP context query must fail LOUD with the /// actionable no-module-rows message (names the missing parquet) (never a fabricated value). /// The calib-specific missing-top_last branch is pinned synthetically in /// `perf_database::dsv4` (absent calib -> None -> operator fail-loud). diff --git a/crates/core/src/perfmodel/perf_database/dsv4.rs b/crates/core/src/perfmodel/perf_database/dsv4.rs index 9434a7a260..7bc151aa2b 100644 --- a/crates/core/src/perfmodel/perf_database/dsv4.rs +++ b/crates/core/src/perfmodel/perf_database/dsv4.rs @@ -537,9 +537,9 @@ impl Dsv4Table { /// table. Mirror of Python /// `ContextDeepSeekV4AttentionModule._lookup_sparse_kernel` with the /// kernel FIXED to `"paged_mqa_logits"` — the quadratic pair-count SOL - /// below is valid only for the causal indexer logits; the windowed - /// sidecars (hca_attn / csa_attn) must NOT route here (Python raises on - /// any other kernel name; Rust simply has no other entry point). + /// below is valid only for the causal indexer logits, the one sparse + /// kernel table left (the windowed hca_attn / csa_attn sidecars were + /// retired 2026-10-15; Rust has no other entry point). /// /// Table shape `[native_heads][tp][past_kv][isl][bs]`; `tp` falls back to /// 1 (the kernel is collected at tp=1 only and its work is diff --git a/crates/core/src/perfmodel/perf_database/table_view.rs b/crates/core/src/perfmodel/perf_database/table_view.rs index ffaa5f831d..78e04fa8bf 100644 --- a/crates/core/src/perfmodel/perf_database/table_view.rs +++ b/crates/core/src/perfmodel/perf_database/table_view.rs @@ -2448,9 +2448,9 @@ pub const TABLE_VIEW_ATTRIBUTES: &[(&str, &[&str])] = &[ /// The engine-side table view: `attribute` is the PerfDatabase attribute name /// the retired Python loader used to fill (`"_gemm_data"`, ...). Returns /// `Ok(None)` exactly when that loader returned `None` (every source path -/// missing), and the loader-shaped nested JSON otherwise. The three DSV4 -/// sparse sub-tables are addressed as -/// `"_dsv4_sparse_kernel_data."`. +/// missing), and the loader-shaped nested JSON otherwise. The DSV4 sparse +/// sub-table is addressed as `"_dsv4_sparse_kernel_data.paged_mqa_logits"` +/// (the hca_attn / csa_attn sidecars were retired 2026-10-15). pub fn table_view_json(tables: &PerfTables, attribute: &str) -> Result, AicError> { let src = |basename: &str| { tables diff --git a/python/aisimulate/collector/cases/models/Glm5NextForConditionalGeneration_cases.yaml b/python/aisimulate/collector/cases/models/Glm5NextForConditionalGeneration_cases.yaml index 52ae01e1a0..68ed3fa4ba 100644 --- a/python/aisimulate/collector/cases/models/Glm5NextForConditionalGeneration_cases.yaml +++ b/python/aisimulate/collector/cases/models/Glm5NextForConditionalGeneration_cases.yaml @@ -24,8 +24,9 @@ description: > Checkpoints: zai-org/GLM-5.3-Flash (FP8 e4m3 128x128 block for dense MLP, shared + routed experts and MLA q_a/q_b/kv_a/o; BF16 elsewhere) and nvidia/GLM-5.3-Flash-NVFP4 (NVFP4 routed experts + dense MLP; everything - else, including the shared expert, BF16). Runtimes are model-pinned in - framework_manifest.yaml (stock vLLM 0.31.0, SGLang 0.5.20). + else, including the shared expert, BF16). KDA / mHC are audited on stock + vLLM 0.31.0 only: the SGLang 0.5.20 lane retired (#649), and 0.5.21 needs a + port of both paths before SGLang collects them again (the 0.5.20 tables stay). Generic ops land in the generic tables (gemm_perf, moe_perf, kda_perf, mhc_module_perf, comm); only the GLM attention layer gets a GLM-specific module table (DeepSeek-V4.1 precedent). @@ -204,8 +205,6 @@ framework_specific_op_cases: sglang: moe: cases: all - # SGLang 0.5.20 is the audited GLM KDA / mHC runtime. - kda: - cases: all - mhc_module: - cases: all + # No kda / mhc_module: SGLang 0.5.21 is not an audited GLM KDA / mHC runtime + # (collect_kda.py _KDA_ARCHITECTURE_COMPAT, collect_mhc_module.py + # _ARCHITECTURE_COMPAT); scheduling them would only log certain failures. diff --git a/python/aisimulate/collector/collect.py b/python/aisimulate/collector/collect.py index 4f4f846e32..175d2fc515 100644 --- a/python/aisimulate/collector/collect.py +++ b/python/aisimulate/collector/collect.py @@ -1152,10 +1152,12 @@ class ReplaySpec: as ``task_params`` — never by substring and never by reconstructing tuples, so the requested set is unambiguous. Nothing else about the run changes: same getter, same run_func, same dispatch, same measurement, so a replay observes exactly what the - campaign observes. The run leaves ``collector_replay.json`` in the output root; - packaging must keep such a result out of shipped tables (it is a diagnosis, not a - dataset). This is a selection mechanism like ``--case-filter``: it decides WHICH - planned cases run, never whether a case may be skipped once queued. + campaign observes. The run leaves ``collector_replay.json`` in the output root. It is + a diagnosis, not a dataset: main() refuses an output root that already holds results + and never finalizes a replay's CSV staging into parquet or ``collection_meta.yaml``, + so its rows cannot replace or extend a table. This is a selection mechanism like + ``--case-filter``: it decides WHICH planned cases run, never whether a case may be + skipped once queued. """ MARKER = "collector_replay.json" @@ -1237,6 +1239,20 @@ def _load_replay_spec(args, log) -> "ReplaySpec | None": return spec +def _require_replay_output_root_without_results( + spec: "ReplaySpec | None", output_root: Path, existing_perf_outputs +) -> None: + """A replay re-measures cases an earlier run already wrote; in that run's output root its + rows would replace or extend the dataset, so it needs a directory without results.""" + if spec is None: + return + if existing_perf_outputs or (output_root / "collection_meta.yaml").exists() or any(output_root.glob("*.parquet")): + raise SystemExit( + f"--cases-from: {output_root} already holds collector results; run a replay in an empty " + "directory so a diagnosis can never overwrite or extend a dataset" + ) + + def _finish_replay(spec: "ReplaySpec | None", output_root: Path, log) -> None: if spec is None: return @@ -6318,8 +6334,9 @@ def main(): metavar="FILE", help="Failure replay: run exactly the cases listed in FILE — an errors_*.json[.gz] or " "collection_summary_*.json[.gz] from an earlier run, a JSON list of case strings, or one case per " - "line — matched by exact case string. Writes collector_replay.json beside the results; such a run " - "is a diagnosis and must never ship.", + "line — matched by exact case string. Writes collector_replay.json beside the results. A replay " + "is a diagnosis: it needs an output directory without collector results and keeps its rows as CSV " + "staging (no parquet, no collection_meta.yaml).", ) parser.add_argument( "--replay-control", @@ -6555,6 +6572,7 @@ def was_touched_by_run(path: Path) -> bool: return resolved not in existing_perf_outputs or path.stat().st_mtime_ns != existing_perf_outputs[resolved] replay = _load_replay_spec(args, logger) + _require_replay_output_root_without_results(replay, output_root, existing_perf_outputs) # Use profiling context manager with ProfilerContext(args.backend, enabled=args.profile): @@ -6574,7 +6592,12 @@ def was_touched_by_run(path: Path) -> bool: _finish_replay(replay, output_root, logger) converted: list[Path] = [] - if args.keep_csv: + if replay is not None: + logger.info( + f"Replay run: keeping collector CSV staging files; a replay never finalizes into parquet or " + f"collection_meta.yaml ({ReplaySpec.MARKER})" + ) + elif args.keep_csv: logger.info("Keeping collector CSV staging files because --keep-csv was passed") else: touched_perf_outputs = {path for path in find_perf_csv_outputs(output_root) if was_touched_by_run(path)} diff --git a/python/aisimulate/collector/fullnode.py b/python/aisimulate/collector/fullnode.py index 08762a85ca..6f0d5402e9 100644 --- a/python/aisimulate/collector/fullnode.py +++ b/python/aisimulate/collector/fullnode.py @@ -75,6 +75,7 @@ def collect_sglang_fullnode_op( get_test_cases_for_model, resume_checkpoint_cls, logger, + replay=None, ) -> list[dict]: """Run one SGLang full-node op with normal V3 checkpoint evidence.""" op = collection["type"] @@ -107,6 +108,14 @@ def collect_sglang_fullnode_op( before_count = len(cases) cases = [case for case in cases if any(fragment in str(case) for fragment in case_filters)] logger.info(f"{op}: --case-filter kept {len(cases)}/{before_count} cases") + if replay is not None: + # same selection, in the same order, as the pool ops (collect.py get_func_with_limit) + before_count = len(cases) + cases = replay.select(op, cases, random.Random(shuffle_seed)) + logger.info( + f"{op}: replay selected {len(replay.matched[op])} requested + " + f"{replay.controls[op]} control of {before_count} cases" + ) if shuffle: rng = random.Random(shuffle_seed) rng.shuffle(cases) diff --git a/python/aisimulate/collector/helper.py b/python/aisimulate/collector/helper.py index 8b29efd589..f503fb7e97 100644 --- a/python/aisimulate/collector/helper.py +++ b/python/aisimulate/collector/helper.py @@ -24,6 +24,7 @@ import shutil import signal import stat +import subprocess import sys import tempfile import threading @@ -111,6 +112,56 @@ def worker_heartbeat() -> None: pass +def _die_with_worker_preexec(): + """A Popen preexec_fn that gives the child SIGKILL when the launching worker dies + (Linux PR_SET_PDEATHSIG), or None elsewhere. prctl is resolved here, in the parent, + so the forked child only makes the call.""" + if not sys.platform.startswith("linux"): + return None + prctl = ctypes.CDLL(None, use_errno=True).prctl + pr_set_pdeathsig = 1 + + def _preexec(): + prctl(pr_set_pdeathsig, signal.SIGKILL) + + return _preexec + + +def run_measurement_subprocess(argv, *, env, cwd, timeout=None, poll_sec: float = 60.0) -> tuple[int, bytes]: + """Run a measurement child; beat the executor's stall watchdog while it makes progress. + + Progress is the child's combined stdout/stderr growing between polls: the module + collectors print every point, and PYTHONUNBUFFERED keeps that output flowing. A child + that prints nothing for the watchdog's STALL_TIMEOUT_SEC is wedged, and the watchdog + reclaims the worker; the child dies with it, so it never keeps measuring next to the + replacement worker's child. Returns ``(returncode, output)``; raises + ``subprocess.TimeoutExpired`` (with the output so far) after ``timeout`` seconds. + """ + env = {**env, "PYTHONUNBUFFERED": "1"} + with tempfile.TemporaryFile() as out: + proc = subprocess.Popen( + argv, env=env, cwd=cwd, stdout=out, stderr=subprocess.STDOUT, preexec_fn=_die_with_worker_preexec() + ) + deadline = None if timeout is None else time.monotonic() + timeout + seen = 0 + while True: + try: + proc.wait(timeout=poll_sec) + break + except subprocess.TimeoutExpired: + if deadline is not None and time.monotonic() >= deadline: + proc.kill() + proc.wait() + out.seek(0) + raise subprocess.TimeoutExpired(argv, timeout, output=out.read()) from None + size = os.fstat(out.fileno()).st_size + if size > seen: + seen = size + worker_heartbeat() + out.seek(0) + return proc.returncode, out.read() + + def _parse_bool_env(env_var: str, default: bool = False) -> bool: """ Robustly parse boolean environment variables. diff --git a/python/aisimulate/collector/kernel_source_backends.yaml b/python/aisimulate/collector/kernel_source_backends.yaml index 62912b0dac..b400b426d8 100644 --- a/python/aisimulate/collector/kernel_source_backends.yaml +++ b/python/aisimulate/collector/kernel_source_backends.yaml @@ -140,10 +140,10 @@ mappings: - {framework: sglang, kernel_source: sglang_flashinfer_bmm_fp8, backend: cublas, source: "collector/sglang/collect_mla_bmm.py:22-26 (sglang>=0.5.21: fp8_utils.bmm_fp8 wraps flashinfer.bmm_fp8 as custom op flashinfer_bmm_fp8_batched, docstring 'via flashinfer's cuBLAS backend', fp8_utils.py:246-273 @0.5.21); flashinfer csrc/bmm_fp8.cu bmm_fp8_internal_cublaslt"} - {framework: sglang, kernel_source: sglang_sgl_kernel_fp8_scaled_mm, backend: cutlass, source: "PR#1342 collect_gemm.py (sgl_kernel fp8_scaled_mm)"} - {framework: sglang, kernel_source: sglang_deepgemm_gemm_nt_f8f8bf16, backend: deepgemm, source: "PR#1342 collect_gemm.py"} - - {framework: sglang, kernel_source: sglang_triton_w8a8_block_fp8, backend: triton, source: "sglang/collect_gemm_0520.py (fp8_utils.py:1094-1107 @v0.5.20 N%64/K%128 fallback)"} - - {framework: sglang, kernel_source: sglang_cutedsl_tgv_bf16, backend: cutlass, source: "sglang/collect_gemm_0520.py (CuTe DSL TGV bf16, cutedsl_bf16_gemm.py @v0.5.20)"} - - {framework: sglang, kernel_source: sglang_flashinfer_bf16_splitk, backend: flashinfer, source: "sglang/collect_gemm_0520.py (unquant.py split-K @v0.5.20)"} - - {framework: sglang, kernel_source: sglang_flashinfer_bf16_direct, backend: flashinfer, source: "sglang/collect_gemm_0520.py (unquant.py direct @v0.5.20)"} + - {framework: sglang, kernel_source: sglang_triton_w8a8_block_fp8, backend: triton, source: "sglang/collect_gemm_0520.py @c540ff90a, retired by #649; labels the shipped 0.5.20 tables (fp8_utils.py:1094-1107 @v0.5.20 N%64/K%128 fallback)"} + - {framework: sglang, kernel_source: sglang_cutedsl_tgv_bf16, backend: cutlass, source: "sglang/collect_gemm_0520.py @c540ff90a, retired by #649; labels the shipped 0.5.20 tables (CuTe DSL TGV bf16, cutedsl_bf16_gemm.py @v0.5.20)"} + - {framework: sglang, kernel_source: sglang_flashinfer_bf16_splitk, backend: flashinfer, source: "sglang/collect_gemm_0520.py @c540ff90a, retired by #649; labels the shipped 0.5.20 tables (unquant.py split-K @v0.5.20)"} + - {framework: sglang, kernel_source: sglang_flashinfer_bf16_direct, backend: flashinfer, source: "sglang/collect_gemm_0520.py @c540ff90a, retired by #649; labels the shipped 0.5.20 tables (unquant.py direct @v0.5.20)"} - {framework: sglang, kernel_source: sglang_flashinfer_cutlass_nvfp4, backend: flashinfer_cutlass, source: "PR#1342 collect_gemm.py (flashinfer mm_fp4 cutlass)"} # MoE labels name the constructed serving method + runner. - {framework: sglang, kernel_source: sglang_flashinfer_trtllm_moe, backend: flashinfer_trtllm_gen, source: "PR#1342 collect_moe.py"} diff --git a/python/aisimulate/collector/sglang/collect_gemm.py b/python/aisimulate/collector/sglang/collect_gemm.py index 9b99936b5a..f44b0af77b 100644 --- a/python/aisimulate/collector/sglang/collect_gemm.py +++ b/python/aisimulate/collector/sglang/collect_gemm.py @@ -43,16 +43,24 @@ # One release per file (owner policy 2026-09-20); earlier code is `git log -- ` away. # The SGLang 0.5.20 fork (collect_gemm_0520.py, serving-built linear layers) # retired with the 0.5.20 lane 2026-10-30; it is git history (main at c540ff90a (before #649)). -# FIXME(re-verify, 2026-10-30): two independent SM100/103 serving gaps, accepted -# by the owner for the 0.5.21 tables (no re-collection). Re-check at the next bump. -# 1. bf16 GEMM kernel: serving resolves the auto bf16 backend to CuTe DSL -# (quantization/unquant.py:216@v0.5.21) and runs the TGV kernel only where +# FIXME(re-verify, 2026-10-30): independent serving gaps, accepted by the owner for +# the 0.5.21 tables (no re-collection). Re-check at the next bump. +# 1. bf16 GEMM on SM100/103: serving resolves the auto bf16 backend to CuTe DSL +# (quantization/unquant.py:216@v0.5.21). It first sends the 68 shapes in +# _BF16_SPLITK_TUNED_TACTICS (unquant.py:105-178: m <= 32 at GB300 TP16 and +# B300 Qwen4-Exp TP4 widths) to FlashInfer split-K or direct +# (SGLANG_ENABLE_BF16_SPLITK_GEMM defaults True, environ.py:1175; dispatch at +# unquant.py:323-347), then runs the TGV kernel only where # use_cutedsl_bf16_gemm(m, n, k) picks it (kernels/ops/gemm/ # cutedsl_bf16_gemm.py:1357-1387: small m, 2048 <= k <= 6144, n >= 1024); # everywhere else serving runs F.linear like this module. -# 2. nvfp4 activation quant: serving quantizes with FlashInfer's cute-dsl backend -# (quantization/fp4_utils.py:24@v0.5.21); this module's flashinfer_fp4_quantize -# uses the default cuda backend. The mm_fp4 GEMM itself matches. +# 2. nvfp4 activation quant on SM100/103: serving quantizes with FlashInfer's +# cute-dsl backend (quantization/fp4_utils.py:24@v0.5.21); this module's +# flashinfer_fp4_quantize uses the default cuda backend. The mm_fp4 GEMM itself matches. +# 3. fp8_block on SM90 and SM100/103: serving's DeepGEMM path falls back to Triton +# (triton_w8a8_block_fp8_linear) unless weight N % 64 == 0 and K % 128 == 0 +# (quantization/fp8_utils.py:1228-1243@v0.5.21; e.g. GLM-5.3-Flash widths +# 288/3336/6416/12576/19360), while this module times DeepGEMM for every shape. __compat__ = "sglang==0.5.21" import os diff --git a/python/aisimulate/collector/sglang/collect_mhc_module.py b/python/aisimulate/collector/sglang/collect_mhc_module.py index 1224d7ebd9..e308ebeeb3 100644 --- a/python/aisimulate/collector/sglang/collect_mhc_module.py +++ b/python/aisimulate/collector/sglang/collect_mhc_module.py @@ -556,11 +556,11 @@ def run_mhc_module( gc.collect() finally: del model_runner + from collector.sglang.runtime_compat import reset_expert_location_metadata from sglang.srt.distributed.parallel_state import ( destroy_distributed_environment, destroy_model_parallel, ) - from sglang.srt.eplb import expert_location as _expert_location # Mirror SGLang 0.5.14 cleanup_dist_env_and_memory: destroying only the # torch process group leaves parallel_state._WORLD pointing at a dead @@ -571,11 +571,11 @@ def run_mhc_module( # and fail the worker rather than hiding retained groups. destroy_model_parallel() destroy_distributed_environment() - # SGLang has no public reset for this module global (its serving - # process never re-creates a ModelRunner); set_global_... asserts None, - # so a second in-worker task fails init unless it is returned to the - # module's pre-init state here. - _expert_location._global_expert_location_metadata = None + # SGLang has no public reset for the expert-location metadata (its + # serving process never re-creates a ModelRunner); set_global_... asserts + # None, so a second in-worker task fails init unless it is returned to + # its pre-init state here. + reset_expert_location_metadata() torch.cuda.empty_cache() gc.collect() summary = f"ok={len(results)} error={error_count} skip=0 total={len(results) + error_count}" diff --git a/python/aisimulate/collector/sglang/collect_mla_module.py b/python/aisimulate/collector/sglang/collect_mla_module.py index 950e7b4e04..96b8f7430a 100644 --- a/python/aisimulate/collector/sglang/collect_mla_module.py +++ b/python/aisimulate/collector/sglang/collect_mla_module.py @@ -50,10 +50,24 @@ from collector.sglang.runtime_compat import resolved_arg # 0.5.21: derived server args live behind the resolving view try: - from helper import PhaseClock, benchmark_with_power, get_sm_version, log_perf, render_phase_totals + from helper import ( + PhaseClock, + benchmark_with_power, + get_sm_version, + log_perf, + render_phase_totals, + run_measurement_subprocess, + ) except ModuleNotFoundError: sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - from helper import PhaseClock, benchmark_with_power, get_sm_version, log_perf, render_phase_totals + from helper import ( + PhaseClock, + benchmark_with_power, + get_sm_version, + log_perf, + render_phase_totals, + run_measurement_subprocess, + ) try: from collector.case_generator import ( @@ -3056,30 +3070,29 @@ def _run_mla_subprocess( f"chunked_prefill_size={chunked_prefill_size!r})\n" ) - proc = subprocess.Popen( - [sys.executable, "-c", code], - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=os.path.dirname(os.path.abspath(__file__)), - ) - # Per-subprocess timeout is OFF by default. A DSA context subprocess sweeps # the full isl x prefix grid for one batch_size and can legitimately run for # many minutes; a wall-clock cap silently truncated the sweep (killed mid-grid, # kept the partial rows, logged a WARNING not an error) and left the perf DB - # with short prefix coverage, forcing AIC to extrapolate. Unset = - # communicate(timeout=None) = block until the subprocess finishes on its own. + # with short prefix coverage, forcing AIC to extrapolate. Unset = block until + # the subprocess finishes on its own; the executor's stall watchdog still sees + # liveness, because the wait beats while the child prints its points, and a + # reclaimed worker takes the child down with it (helper.run_measurement_subprocess). # Users can still opt into a cap by setting AIC_MLA_MODULE_SUBPROCESS_TIMEOUT_SEC. _timeout_env = os.environ.get("AIC_MLA_MODULE_SUBPROCESS_TIMEOUT_SEC") subprocess_timeout = int(_timeout_env) if _timeout_env else None try: - stdout, _ = proc.communicate(timeout=subprocess_timeout) + returncode, stdout = run_measurement_subprocess( + [sys.executable, "-c", code], + env=env, + cwd=os.path.dirname(os.path.abspath(__file__)), + timeout=subprocess_timeout, + ) if stdout: print(stdout.decode("utf-8", errors="replace")) except subprocess.TimeoutExpired as exc: - proc.kill() - proc.wait() + if exc.output: + print(exc.output.decode("utf-8", errors="replace")) raise RuntimeError( f"{attn_type.upper()} module {phase} subprocess timed out " f"after {subprocess_timeout}s " @@ -3087,7 +3100,7 @@ def _run_mla_subprocess( f"gemm={gemm_type}, target_tp={target_tp_size})" ) from exc - if proc.returncode != 0: + if returncode != 0: # Include last lines of subprocess output in the error so the cause # is visible in the error group (subprocess stderr is merged into stdout). tail = "" @@ -3095,7 +3108,7 @@ def _run_mla_subprocess( lines = stdout.decode("utf-8", errors="replace").strip().splitlines() tail = "\n".join(lines[-30:]) # last 30 lines should include traceback raise RuntimeError( - f"{attn_type.upper()} module {phase} subprocess failed (exit code {proc.returncode})\n" + f"{attn_type.upper()} module {phase} subprocess failed (exit code {returncode})\n" f"--- subprocess output (last 30 lines) ---\n{tail}" ) diff --git a/python/aisimulate/collector/sglang/collect_moe.py b/python/aisimulate/collector/sglang/collect_moe.py index d16a3a80dc..441b280abf 100644 --- a/python/aisimulate/collector/sglang/collect_moe.py +++ b/python/aisimulate/collector/sglang/collect_moe.py @@ -1483,9 +1483,14 @@ def run_moe_torch( and moe_backend == "flashinfer_cutlass" and (hidden_size % 8 != 0 or local_inter_size % 8 != 0) ): + # FlashInfer 0.6.18 (pinned by SGLang 0.5.21, python/pyproject.toml:42) + # flashinfer_cutlass_fused_moe_binding.cu:386-394: min_alignment = + # 128 / (8 * min(act_bytes, weight_bytes)) = 8 for BF16, ICHECKed on both + # hidden_size and inter_size; SGLang does not pad this path (unquant.py:648-711). raise ValueError( "FIXME(kernel-limit): SGLang FlashInfer CUTLASS BF16 MoE requires hidden_size and local_inter_size " - f"to be divisible by 8, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" + "to be divisible by 8 (flashinfer 0.6.18 flashinfer_cutlass_fused_moe_binding.cu:386-394), " + f"got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) activation_vector_size = 16 if sm_version >= 100 else 8 if ( @@ -1511,9 +1516,14 @@ def run_moe_torch( and sm_version == 90 and (hidden_size % 128 != 0 or local_inter_size % 128 != 0) ): + # SGLang 0.5.21 selects Mxfp4FlashinferCutlassMoEMethod on SM90 (fp8.py:448-455), + # which raises unless both widths are multiples of 128 + # (mxfp4_flashinfer_cutlass_moe.py:67); the 256 round-up in layer.py:437-443 + # applies only to the serialized-mxfp4 quant config, not to these FP4 experts. raise ValueError( "FIXME(kernel-limit): SGLang SM90 DeepSeek-V4 W4A16 FP4 experts require hidden_size and local_inter_size " - f"to be divisible by 128, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" + "to be divisible by 128 (mxfp4_flashinfer_cutlass_moe.py:67 @0.5.21), " + f"got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) if ( moe_type == "w4a8_mxfp4_mxfp8" @@ -1566,17 +1576,30 @@ def run_moe_torch( int4_config = get_moe_quantization_module_config("sglang", moe_type, model_name=model_name) int4_group_size = int(int4_config.get("group_size", 128)) if hidden_size % int4_group_size != 0 or local_inter_size % int4_group_size != 0: + # FIXME(re-verify): no 0.5.21 source line rejects K % group_size. The kernels' + # own limits are stricter and fire on their own: SM100/103 MxInt4 packs + # block_k=128 bytes (compressed_tensors_w4a4_mxint4_moe.py:248-254 -> flashinfer + # 0.6.18 fused_moe/core.py:581, hidden/local_inter % 256), SM90 Marlin tiles + # (moe_wna16_marlin.cuh:140-152,265,747: hidden % 128, local_inter % 64). Marlin + # floor-divides the groups (compressed_tensors_wNa16_moe.py:158-159), so an + # unaligned K may run with a different group size than the row says — not shown + # either way, so this stays an unclassified failure until a run decides it. raise ValueError( - "FIXME(kernel-limit): SGLang INT4-WO group quantization requires hidden_size and local_inter_size " + "SGLang INT4-WO group quantization requires hidden_size and local_inter_size " f"to be divisible by group_size={int4_group_size}, got " f"hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) block_shape = [0, int4_group_size] elif moe_type == "fp8_block": if moe_backend == "triton" and (hidden_size % 128 != 0 or local_inter_size % 128 != 0): + # Both GEMM inputs are per-token-group quantized at block_k=128 + # (fused_moe_triton_kernels.py:866 -> fp8_kernel.py:577 + # `assert x.shape[-1] % group_size == 0` @0.5.21): K = hidden_size for the + # first GEMM, local_inter_size for the second. raise ValueError( "FIXME(kernel-limit): SGLang Triton fp8_block requires hidden_size and local_inter_size " - f"to be divisible by 128, got hidden_size={hidden_size}, local_inter_size={local_inter_size}" + "to be divisible by 128 (fp8_kernel.py:577 via fused_moe_triton_kernels.py:866 @0.5.21), " + f"got hidden_size={hidden_size}, local_inter_size={local_inter_size}" ) block_shape = [128, 128] else: diff --git a/python/aisimulate/collector/sglang/collect_msa_module.py b/python/aisimulate/collector/sglang/collect_msa_module.py index 86feacd8b6..a9a495a1ca 100644 --- a/python/aisimulate/collector/sglang/collect_msa_module.py +++ b/python/aisimulate/collector/sglang/collect_msa_module.py @@ -84,6 +84,7 @@ get_sm_version, log_perf, render_phase_totals, + run_measurement_subprocess, ) try: @@ -1432,34 +1433,33 @@ def _run_msa_subprocess( f"{batch_repr}, {target_tp_size}, None, {inner_manifest!r})\n" ) - proc = subprocess.Popen( - [sys.executable, "-c", code], - env=env, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - cwd=os.path.dirname(os.path.abspath(__file__)), - ) _timeout_env = os.environ.get("AIC_MLA_MODULE_SUBPROCESS_TIMEOUT_SEC") subprocess_timeout = int(_timeout_env) if _timeout_env else None try: - stdout, _ = proc.communicate(timeout=subprocess_timeout) + # beats the stall watchdog while the child prints progress; the child dies with the worker + returncode, stdout = run_measurement_subprocess( + [sys.executable, "-c", code], + env=env, + cwd=os.path.dirname(os.path.abspath(__file__)), + timeout=subprocess_timeout, + ) if stdout: print(stdout.decode("utf-8", errors="replace")) except subprocess.TimeoutExpired as exc: - proc.kill() - proc.wait() + if exc.output: + print(exc.output.decode("utf-8", errors="replace")) raise RuntimeError( f"MSA module {phase} subprocess timed out after {subprocess_timeout}s " f"(heads={num_heads}, model={model_path}, kv={kv_cache_dtype})" ) from exc - if proc.returncode != 0: + if returncode != 0: tail = "" if stdout: lines = stdout.decode("utf-8", errors="replace").strip().splitlines() tail = "\n".join(lines[-30:]) raise RuntimeError( - f"MSA module {phase} subprocess failed (exit code {proc.returncode})\n" + f"MSA module {phase} subprocess failed (exit code {returncode})\n" f"--- subprocess output (last 30 lines) ---\n{tail}" ) diff --git a/python/aisimulate/collector/trtllm/collect_gemm.py b/python/aisimulate/collector/trtllm/collect_gemm.py index deec6122a3..c65b3fabae 100644 --- a/python/aisimulate/collector/trtllm/collect_gemm.py +++ b/python/aisimulate/collector/trtllm/collect_gemm.py @@ -196,17 +196,6 @@ def run_gemm(gemm_type, m, n, k, *, perf_filename, device="cuda:0"): torch.set_default_device(device) sm_version = get_sm_version() - if gemm_type == "fp8" and (n % 16 or k % 16): - # FP8QDQLinearMethod.apply -> torch.ops.trtllm.cublas_scaled_mm / cuda_scaled_mm, - # which TORCH_CHECK the 16-element alignment of the output row stride (= n) - # and of mat_b's column stride (= k): cpp/tensorrt_llm/thop/cublasScaledMM.cpp:260 - # and cutlassScaledMM.cpp:152 @1.3.0rc29. The shared GEMM grid carries - # n in {1, 4, 8, 12, 24} (1,110 cases per system on gb200/gb300). - raise RuntimeError( - "FIXME(kernel-limit): TRT-LLM fp8 GEMM (cublas/cutlass scaled_mm) requires n and k to be " - f"multiples of 16 (out.strides()[0] % 16 == 0 && mat_b.strides()[1] % 16 == 0), got n={n}, k={k}" - ) - dtype = torch.bfloat16 x = torch.randn((m, k), dtype=dtype).to(torch.device(device)) diff --git a/python/aisimulate/collector/trtllm/collect_moe.py b/python/aisimulate/collector/trtllm/collect_moe.py index 163f9815bb..8fd38cc2bb 100755 --- a/python/aisimulate/collector/trtllm/collect_moe.py +++ b/python/aisimulate/collector/trtllm/collect_moe.py @@ -332,6 +332,73 @@ def _declared_moe_routing(model_name: str): return None +def _zero_routing_bias(gate) -> None: + """Serving gates allocate e_score_correction_bias with torch.empty and only + load_weights fills it (DeepseekV3Gate, modeling_deepseekv3.py:884,926-935 + @1.3.0rc29; DeepseekV4Gate and MiniMaxM3Gate likewise). Their top-k ranks + score + bias, and the collector's balanced / power-law logits separate the + intended experts by ~1e-3 after the sigmoid, so whatever that memory holds + would pick the experts instead of the row's distribution. Zero it, as + vllm/collect_moe.py and wideep/trtllm/collect_moe_a2a.py do.""" + with torch.no_grad(): + gate.e_score_correction_bias.zero_() + + +def _serving_moe_gate(declared, hidden_size, num_experts, topk, dtype, moe_backend): + """The gate serving builds for this architecture, with a zeroed bias. + + DeepSeek-V3-family / GLM / Kimi / Nemotron-H: DeepseekV3Gate + (modeling_deepseekv3.py:979-994 @1.3.0rc29). DeepSeek-V4: DeepseekV4Gate, + whose routing runs separately (gate_forward, sqrt-softplus scoring; + modeling_deepseekv4.py:1533-1546, moe/fused_moe/routing.py:524-614). The + collected layer is a scored one: the first n_hash_layers route by token id + (tid2eid) and have no logit distribution to label. MiniMax-M3: + MiniMaxM3Gate, MiniMax2 routing (modeling_minimaxm3.py:641-646, + routing.py:697-760). Serving resolves moe_backend upper-case; DeepseekV3Gate + keeps a bf16 bias only for 'TRTLLM' (modeling_deepseekv3.py:880-883). + """ + n_group = int(declared.sglang_moe_num_expert_group or 1) + topk_group = int(declared.sglang_moe_topk_group or 1) + routed_scaling_factor = float(declared.sglang_moe_routed_scaling_factor or 1.0) + backend = str(moe_backend).upper() + if declared.architecture == "DeepseekV4ForCausalLM": + from tensorrt_llm._torch.models.modeling_deepseekv4 import DeepseekV4Gate + + gate = DeepseekV4Gate( + hidden_size, + num_experts, + top_k=topk, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=routed_scaling_factor, + is_hashed=False, + dtype=dtype, + moe_backend=backend, + ) + elif declared.architecture == "MiniMaxM3ForCausalLM": + from tensorrt_llm._torch.models.modeling_minimaxm3 import MiniMaxM3Gate + + gate = MiniMaxM3Gate( + hidden_size=hidden_size, + num_experts=num_experts, + top_k=topk, + routed_scaling_factor=routed_scaling_factor, + ) + else: + gate = DeepseekV3Gate( + hidden_size, + num_experts, + top_k=topk, + n_group=n_group, + topk_group=topk_group, + routed_scaling_factor=routed_scaling_factor, + dtype=dtype, + moe_backend=backend, + ) + _zero_routing_bias(gate) + return gate + + def _run_moe_case( moe_type, num_tokens_lists, @@ -570,7 +637,7 @@ def _run_moe_case( topk_group = 4 routed_scaling_factor = 2.5 - routing_method = DeepseekV3Gate( + min_latency_gate = DeepseekV3Gate( hidden_size, num_experts, top_k=topk, @@ -579,7 +646,9 @@ def _run_moe_case( routed_scaling_factor=routed_scaling_factor, dtype=dtype, moe_backend="TRTLLM", - ).routing_method + ) + _zero_routing_bias(min_latency_gate) + routing_method = min_latency_gate.routing_method router_logits_dtype = torch.float32 else: declared = _declared_moe_routing(model_name) @@ -595,15 +664,9 @@ def _run_moe_case( # Mixtral, ...), and running it for DeepSeek-V3 diverged the # moe_fp8block gate on H20 (2026-10-01). The router GEMM that feeds the # gate (dsv3_router_gemm_op) stays outside this op, as before. - routing_method = DeepseekV3Gate( - hidden_size, - num_experts, - top_k=topk, - n_group=int(declared.sglang_moe_num_expert_group or 1), - topk_group=int(declared.sglang_moe_topk_group or 1), - routed_scaling_factor=float(declared.sglang_moe_routed_scaling_factor or 1.0), - dtype=dtype, - moe_backend=model_config.moe_backend, + # DeepSeek-V4 and MiniMax-M3 build their own gates (_serving_moe_gate). + routing_method = _serving_moe_gate( + declared, hidden_size, num_experts, topk, dtype, model_config.moe_backend ).routing_method router_logits_dtype = torch.float32 elif is_gpt_oss: diff --git a/python/aisimulate/collector/vllm/collect_mla_module.py b/python/aisimulate/collector/vllm/collect_mla_module.py index c68c0fc6d1..bc249acac2 100644 --- a/python/aisimulate/collector/vllm/collect_mla_module.py +++ b/python/aisimulate/collector/vllm/collect_mla_module.py @@ -773,6 +773,7 @@ def _create_attention_module( gemm_type: str = "bfloat16", device: str = "cuda:0", is_context: bool = True, + max_model_len: int | None = None, ): """ Create a DeepseekV2MLAAttention module from vLLM's own modeling code. @@ -786,6 +787,9 @@ def _create_attention_module( model_path: HuggingFace model path (e.g. "deepseek-ai/DeepSeek-V3.2"). attn_type: Attention type ("mla" or "dsa"). max_seq_len: Total context length, including the cached prefix. + max_model_len: The engine's max_model_len (default: max_seq_len, at least 4096). + Indexer buffers scale with it, so a series passes the model's + max_position to time every point at the size serving runs. max_query_len: New tokens per request for a context forward pass. use_prefill_fp8: When True, opt in to FP8 prefill query compute via ``attention_config.use_prefill_query_quantization`` plus an @@ -811,7 +815,7 @@ def _create_attention_module( block_size = None # unconstrained selection; _create_kv_cache_and_metadata asks the selected backend for the page # seq_len includes the current token; generation caches only seq_len - 1. # Keep exact-limit models such as Kimi-K2-Instruct at their declared limit. - max_model_len = max(max_seq_len, 4096) + max_model_len = max(max_seq_len, max_model_len or 4096) num_kv_cache_blocks = max( 1 + math.ceil((max_seq_len + 1) / 16) @@ -819,6 +823,13 @@ def _create_attention_module( 8192, ) + # FIXME(gap): the module is built at TP=1 with the per-rank head count, so a check that + # multiplies by the TP world size sees the shard. vLLM 0.30's sparse-MLA masked-MHA prefill + # (SM100, FA4, BF16 KV) admits only model-level head counts 128/64 + # (sparse_mla_attention.py:93-127, called with num_heads * tp at :708), and _use_masked_mha + # reads its thresholds by tensor_parallel_size (mla_attention.py:1844). So for a TP>1 shard, + # e.g. GLM-5.2 TP8 BF16 KV, serving runs masked MHA for pure prefills up to 64K while this + # module times sparse MQA. Affects SM100 BF16-KV DSA context rows below the full head count. # Determine kv cache dtype string for sparse MLA. # For DSA (DeepSeekV3.2), fp8 uses the custom ``fp8_ds_mla`` 656-byte # cache format (512B quantized NoPE + 16B scales + 128B RoPE). @@ -1116,10 +1127,19 @@ def _create_kv_cache_and_metadata( hf_config = vllm_config.model_config.hf_text_config kv_lora_rank = hf_config.kv_lora_rank qk_rope_head_dim = hf_config.qk_rope_head_dim - # page = what the framework would run the SELECTED backend with (serving re-derives it - # from the built model, platforms/interface.py update_block_size_for_backend) + # page = what the framework would run the SELECTED backend with. Serving sizes the cache + # block from the FIRST non-SSM attention layer (platforms/interface.py:588-640 @0.30.0). + # For DSA that is the indexer cache — deepseek_v2.py:1175 builds the Indexer (its + # k_cache registers at :724) before :1211 registers .attn — whose CUDA backend accepts + # only 64 (v1/attention/backends/mla/indexer.py:203-204), so FLASHINFER_MLA_SPARSE + # ([32, 64]) runs at 64, not at its own preferred 32. + from vllm.platforms import current_platform + _attn_layer = vllm_config.compilation_config.static_forward_context["model.layers.0.self_attn.attn"] - block_size = framework_kv_block_size(_attn_layer.get_attn_backend()) + block_size = framework_kv_block_size( + _attn_layer.get_attn_backend(), + cache_backend_cls=current_platform._find_non_ssm_backend(vllm_config), + ) vllm_config.cache_config.block_size = block_size is_dsa = attn_type == "dsa" @@ -1466,11 +1486,15 @@ def _lap(phases: dict[str, float], name: str) -> None: f"compute={compute_dtype}, kv={kv_cache_dtype}, model={model_path}" ) - # 1. Create attention module, sized by the largest point of the series. The - # sizes only reach buffers and capacity declarations (max_model_len, - # max_num_batched_tokens, the top-k index buffer); the backend selection - # is dtype/SM driven and every point's KV cache + metadata is built below - # for that point's own shape. + # 1. Create attention module, sized for the series' largest point and at the + # model's max_position_embeddings as max_model_len — serving's default, and + # independent of the batch: the DSA indexer's decode logits + # [rows, max_model_len] (sparse_attn_indexer.py:711-721 @0.30.0) and prefill + # buffer max_model_len * 40 (mla/indexer.py:722-731) scale with it, so a + # series-max size would bend batch scaling within one table. The other sizes + # only reach capacity declarations (max_num_batched_tokens, the top-k index + # buffer); the backend selection is dtype/SM driven and every point's KV + # cache + metadata is built below for that point's own shape. attn_module, vllm_config = _create_attention_module( model_path=model_path, attn_type=attn_type, @@ -1483,6 +1507,7 @@ def _lap(phases: dict[str, float], name: str) -> None: gemm_type=gemm_type, device=device, is_context=is_context, + max_model_len=_model_max_position_embeddings(model_path), ) _lap(_setup_phase, "module") if attn_type == "dsa" and use_fp8_kv_cache: @@ -1791,6 +1816,10 @@ def measure_and_log(skip_indexer): except Exception as exc: if _is_fatal_cuda_error(exc) or len(points) == 1: raise + # The traceback holds _run_point's frame (KV cache, indexer cache, metadata, + # activations): clear it, or the largest-first series keeps that memory + # pinned and the smaller points that would fit fail too. + traceback.clear_frames(exc.__traceback__) failures.append(((point_seq_len, point_prefix), exc)) print( f" [{variant} module] {phase} b={batch_size}, s={point_seq_len}, prefix={point_prefix}: " diff --git a/python/aisimulate/collector/vllm/utils.py b/python/aisimulate/collector/vllm/utils.py index 2e9880ab29..b19ae0d39d 100644 --- a/python/aisimulate/collector/vllm/utils.py +++ b/python/aisimulate/collector/vllm/utils.py @@ -655,16 +655,18 @@ def wrapper(*args, **kwargs): return wrapper -def framework_kv_block_size(backend_cls, *, user_block_size: int | None = None) -> int: +def framework_kv_block_size(backend_cls, *, cache_backend_cls=None, user_block_size: int | None = None) -> int: """The KV page the FRAMEWORK would run this backend with — asked, not tabled. Mirrors serving (vllm 0.30.0): the cache manager block size is - ``backend.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE)`` unless - the user passed --block-size (platforms/interface.py - update_block_size_for_backend, phase 1), and the kernel page is - ``select_common_block_size(cache_block_size, [backend])`` (v1/worker/utils.py: - the manager size when every backend accepts it, else the largest explicit - size that divides it). The former per-SM table (removed) was the owner's + ``get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE)`` of the model's + FIRST non-SSM attention layer unless the user passed --block-size + (platforms/interface.py:588-640, _find_non_ssm_backend + + update_block_size_for_backend phase 1) — ``cache_backend_cls``, defaulting to + ``backend_cls`` when the measured layer is that first layer — and the kernel + page is ``select_common_block_size(cache_block_size, [backend])`` + (v1/worker/utils.py: the manager size when every backend accepts it, else the + largest explicit size that divides it). The former per-SM table (removed) was the owner's 2026-09-30 interim; the B200 re-run on 2026-10-03 showed it is not equivalent: with mla=32 the bf16 DSA cell selected the TRT-LLM FMHA sparse cubin while serving ran FA-cute (page 64) — the page is a per-(backend, kv @@ -674,7 +676,8 @@ def framework_kv_block_size(backend_cls, *, user_block_size: int | None = None) from vllm.config.cache import CacheConfig from vllm.v1.worker.utils import select_common_block_size + cache_backend = cache_backend_cls or backend_cls cache_block = ( - user_block_size if user_block_size else backend_cls.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE) + user_block_size if user_block_size else cache_backend.get_preferred_block_size(CacheConfig.DEFAULT_BLOCK_SIZE) ) return int(select_common_block_size(int(cache_block), [backend_cls])) diff --git a/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 b/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 index 31e78186df..088fa110d3 100644 --- a/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 +++ b/python/aisimulate/src/aisimulate/generator/config/backend_templates/trtllm/extra_engine_args.1.3.0rc29.yaml.j2 @@ -15,8 +15,10 @@ moe_tensor_parallel_size: {{ moe_tensor_parallel_size }} moe_config: {# moe_config.backend (facts / --generator-set params.agg.moe_config.backend) wins; the generic `moe_backend` param (backend_config_mapping.yaml, the sglang --moe-runner-backend spelling) used to be silently dropped for trtllm — - it now feeds the same field, upper-cased to the TRT-LLM enum. #} - backend: {{ moe_config.backend | default((moe_backend | upper) if moe_backend is defined and moe_backend else 'CUTLASS') }} # MOE backend to use (CUTLASS, CUTEDSL, WIDEEP, TRTLLM, VANILLA, MARLIN, ...) + it now feeds the same field, upper-cased to the TRT-LLM enum. Unset renders AUTO, rc29's own MoeConfig default + (llmapi/llm_args.py:1657-1659): an explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100 + (findings.yaml sm100_moe_backend_fact_rejected_2026_10_04), and rc29 no longer accepts WIDEEP. #} + backend: {{ moe_config.backend | default((moe_backend | upper) if moe_backend is defined and moe_backend else 'AUTO') }} # MOE backend to use (AUTO, CUTLASS, CUTEDSL, TRTLLM, DEEPGEMM, VANILLA, MARLIN, ...) {% if moe_config.load_balancer is defined %} # Configuration for MoE load balancing load_balancer: {{ moe_config.load_balancer }} # moe load balancer {% endif %} diff --git a/python/aisimulate/src/aisimulate/generator/facts/apply.py b/python/aisimulate/src/aisimulate/generator/facts/apply.py index 8ff8b566b0..7a165255c4 100644 --- a/python/aisimulate/src/aisimulate/generator/facts/apply.py +++ b/python/aisimulate/src/aisimulate/generator/facts/apply.py @@ -131,12 +131,30 @@ def quant_family_of(quantization: dict[str, Any] | None) -> str | None: return method or algo.lower() or None +# TRT-LLM MoeConfig.backend literals a release no longer accepts, with the first release that rejects +# them: WIDEEP is in the literal through 1.3.0rc24 and gone from 1.3.0rc25 (llmapi/llm_args.py MoeConfig). +_TRTLLM_MOE_BACKEND_REJECTED_FROM = {"WIDEEP": "1.3.0rc25"} + + +def _trtllm_rejects_moe_backend(choice: str, backend_version: str | None) -> bool: + from packaging.version import InvalidVersion, Version + + rejected_from = _TRTLLM_MOE_BACKEND_REJECTED_FROM.get(str(choice).upper()) + if rejected_from is None or not backend_version: + return False + try: + return Version(str(backend_version)) >= Version(rejected_from) + except InvalidVersion: + return False + + def apply_moe_backend( context: dict[str, Any], hardware: dict[str, Any] | None, *, backend: str, quantization: dict[str, Any] | None = None, + backend_version: str | None = None, ) -> None: """Apply the hardware-derived ``moe_backend`` selection (fill-if-absent, MoE-only). @@ -153,7 +171,9 @@ def apply_moe_backend( Per-backend seam: * ``trtllm``: sets ``context["moe_config"]["backend"]`` (consumed by the typed - engine-config builder). Only when unset — the builder defaults to CUTLASS. + engine-config builder). Only when unset — the template default applies otherwise + (AUTO from 1.3.0rc29, CUTLASS before). A literal the target ``backend_version`` + rejects (WIDEEP from 1.3.0rc25) is not filled. * ``sglang``: sets ``context["moe_backend"]`` (mapped to ``--moe-runner-backend`` via ``backend_config_mapping.yaml``). Only when unset. * ``vllm``: no-op — vLLM's MoE backend comes from MODEL ``defaults:`` (handled @@ -172,6 +192,8 @@ def apply_moe_backend( if not context.get("is_moe"): return if backend == "trtllm": + if _trtllm_rejects_moe_backend(choice, backend_version): + return moe = context.setdefault("moe_config", {}) # a user/recipe `moe_backend` param (the generic spelling the sglang branch below honours) counts as # "already set" here too: until 2026-10-04 the hardware fill ignored it and the trtllm template never diff --git a/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml b/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml index d22e7620e6..116282dcce 100644 --- a/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml +++ b/python/aisimulate/src/aisimulate/generator/facts/hardware.yaml @@ -102,18 +102,17 @@ profiles: # src/aisimulate_core/systems/data/b200_sxm/moe/trtllm/ # /wideep_moe_perf.parquet data collected, # and cli/exps/deepseek_wideep_trtllm.yaml (gb200/b200 wideep configs). - # trtllm: WIDEEP (disabled, see note below) + trtllm: WIDEEP # SGLang: deepep_moe is the default wideep-compatible MoE runner on Blackwell. # Source: cli/main.py _sglang_moe_backend_override and # cli/exps/deepseek_disagg_sglang.yaml. - # sglang: deepep_moe (disabled, see note below) - # 2026-10-04 (B200 probe, PR #228 sm100 round): NOT filled on Blackwell. - # `WIDEEP` is a TRT-LLM 1.0-era MoeConfig.backend literal; 1.3.0rc29 rejects it - # (pydantic literal_error) and sglang 0.5.21 rejects `--moe-runner-backend deepep_moe` - # (argparse invalid choice) — 156 trtllm + 150 sglang MoE renders failed to boot. - # Wide-EP is an EP-size/load-balancer deployment choice, not a per-layer MoE kernel - # backend; single-node MoE on sm100 boots and picks working kernels under the - # framework's AUTO (A/B: explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100). + # sglang: deepep_moe (not filled: no SGLang 0.5.x --moe-runner-backend choice) + # 2026-10-04 (B200 probe, PR #228 sm100 round): TRT-LLM 1.3.0rc29 rejects `WIDEEP` + # (pydantic literal_error; the literal is gone from 1.3.0rc25) and sglang 0.5.21 rejects + # `--moe-runner-backend deepep_moe` (argparse invalid choice; no 0.5.x lists it) — 156 + # trtllm + 150 sglang MoE renders failed to boot. facts/apply.py therefore drops WIDEEP + # for TRT-LLM >= 1.3.0rc25, where the rc29 template renders the framework's AUTO + # (A/B: explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100); older releases keep it. # See benchmarks/evidence/collector/op-probe-2026-10/findings.yaml sm100_moe_backend_fact_rejected_2026_10_04. # --------------------------------------------------------------------------- @@ -163,18 +162,17 @@ profiles: # (all gb200_wideep_* experiments), # src/aisimulate_core/systems/data/gb200/moe/trtllm/ # /wideep_moe_perf.parquet - # trtllm: WIDEEP (disabled, see note below) + trtllm: WIDEEP # SGLang: deepep_moe for the wideep DeepEP path on GB200 NVLink fabric. # Source: cli/main.py _sglang_moe_backend_override and # cli/exps/deepseek_disagg_sglang.yaml. - # sglang: deepep_moe (disabled, see note below) - # 2026-10-04 (B200 probe, PR #228 sm100 round): NOT filled on Blackwell. - # `WIDEEP` is a TRT-LLM 1.0-era MoeConfig.backend literal; 1.3.0rc29 rejects it - # (pydantic literal_error) and sglang 0.5.21 rejects `--moe-runner-backend deepep_moe` - # (argparse invalid choice) — 156 trtllm + 150 sglang MoE renders failed to boot. - # Wide-EP is an EP-size/load-balancer deployment choice, not a per-layer MoE kernel - # backend; single-node MoE on sm100 boots and picks working kernels under the - # framework's AUTO (A/B: explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100). + # sglang: deepep_moe (not filled: no SGLang 0.5.x --moe-runner-backend choice) + # 2026-10-04 (B200 probe, PR #228 sm100 round): TRT-LLM 1.3.0rc29 rejects `WIDEEP` + # (pydantic literal_error; the literal is gone from 1.3.0rc25) and sglang 0.5.21 rejects + # `--moe-runner-backend deepep_moe` (argparse invalid choice; no 0.5.x lists it) — 156 + # trtllm + 150 sglang MoE renders failed to boot. facts/apply.py therefore drops WIDEEP + # for TRT-LLM >= 1.3.0rc25, where the rc29 template renders the framework's AUTO + # (A/B: explicit CUTLASS fails DeepSeek FP8_BLOCK_SCALES on SM100); older releases keep it. # See benchmarks/evidence/collector/op-probe-2026-10/findings.yaml sm100_moe_backend_fact_rejected_2026_10_04. # --------------------------------------------------------------------------- diff --git a/python/aisimulate/src/aisimulate/generator/rendering/engine.py b/python/aisimulate/src/aisimulate/generator/rendering/engine.py index 7d8185d7be..6c295110be 100644 --- a/python/aisimulate/src/aisimulate/generator/rendering/engine.py +++ b/python/aisimulate/src/aisimulate/generator/rendering/engine.py @@ -462,7 +462,7 @@ def render_backend_templates( quantization = ( _bundled_quantization(str(context.get("model_path") or "")) if keyed and context.get("is_moe") else None ) - apply_moe_backend(context, hardware, backend=backend, quantization=quantization) + apply_moe_backend(context, hardware, backend=backend, quantization=quantization, backend_version=version) # Find template files template_path = Path(templates_dir) diff --git a/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py b/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py index 7e446f1c81..c1faff255e 100644 --- a/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py +++ b/python/aisimulate/src/aisimulate/generator/rendering/rule_engine.py @@ -186,11 +186,14 @@ def _align_token_budgets(param_values: dict[str, Any]) -> None: passes). Applied here, after the rule formulas and before any rendering, so the trtllm engine-yaml template — which the mapping transforms cannot reach — gets it too; the mapping keeps the same rounding as a guard for - values injected past the rule engine. Never rounds down.""" + values injected past the rule engine. Never rounds down. Values a caller + already evaluated (``preserve_engine_limits``, the Sweeper) are rounded too, + with a warning, so the deployed limit is visible next to the simulated one.""" params = param_values.get("params") if not isinstance(params, dict): return - for role in params.values(): + preserved = bool(param_values.get("preserve_engine_limits") or params.get("preserve_engine_limits")) + for role_name, role in params.items(): if not isinstance(role, dict): continue for key in _TOKEN_BUDGET_KEYS: @@ -199,6 +202,16 @@ def _align_token_budgets(param_values: dict[str, Any]) -> None: continue aligned = ((v + _TOKEN_BUDGET_ALIGN - 1) // _TOKEN_BUDGET_ALIGN) * _TOKEN_BUDGET_ALIGN if aligned != v: + if preserved: + logger.warning( + "%s %s=%d was evaluated by the caller; the deployment uses %d (token budgets are " + "rounded up to a multiple of %d on every backend)", + role_name, + key, + v, + aligned, + _TOKEN_BUDGET_ALIGN, + ) role[key] = aligned diff --git a/python/aisimulate/src/aisimulate/generator/request/sweeper.py b/python/aisimulate/src/aisimulate/generator/request/sweeper.py index 9cb2b4458e..6042a0482d 100644 --- a/python/aisimulate/src/aisimulate/generator/request/sweeper.py +++ b/python/aisimulate/src/aisimulate/generator/request/sweeper.py @@ -455,6 +455,18 @@ def from_sweeper_candidate( value = getattr(resolved_model_facts, key) if value is not None: raw[f"ModelConfig.{key}"] = value + if backend == "trtllm": + # The benchmark rule renders the MiniMax-M3 sparse-attention keys from ModelConfig as the + # default path does (module_bridge): TRT-LLM >= 1.3.0rc29 needs `algorithm` to build the M3 + # KV-cache manager on every SM; `implementation` is the SM100-family prescription. + from aisimulate.generator.utils import msa_sparse_algorithm, msa_sparse_implementation + + for key, value in ( + ("msa_sparse_algorithm", msa_sparse_algorithm(backend, model_path)), + ("msa_sparse_implementation", msa_sparse_implementation(backend, model_path, hardware_sku)), + ): + if value and f"ModelConfig.{key}" not in raw: + raw[f"ModelConfig.{key}"] = value request = GeneratorRequest( model=ModelSpec(model_path=model_path), diff --git a/python/aisimulate/src/aisimulate/generator/rule_plugin/benchmark/trtllm.rule b/python/aisimulate/src/aisimulate/generator/rule_plugin/benchmark/trtllm.rule index 55850ed7b7..11a5fa412a 100644 --- a/python/aisimulate/src/aisimulate/generator/rule_plugin/benchmark/trtllm.rule +++ b/python/aisimulate/src/aisimulate/generator/rule_plugin/benchmark/trtllm.rule @@ -53,3 +53,10 @@ when (ModelConfig.nextn or 0) > 0: when kv_cache_dtype in ("float16", "bfloat16"): kv_cache_dtype = "auto" + +# MiniMax-M3 sparse attention, as in the default rule: rc29 requires algorithm on every SM, +# implementation is the SM100-family prescription (utils.msa_sparse_algorithm / _implementation). +when (ModelConfig.msa_sparse_implementation or '') != '': + agg_prefill_decode msa_sparse_implementation = ModelConfig.msa_sparse_implementation +when (ModelConfig.msa_sparse_algorithm or '') != '': + agg_prefill_decode msa_sparse_algorithm = ModelConfig.msa_sparse_algorithm diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py index 5dee5dbcd6..6663fa9dd8 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module.py @@ -9,7 +9,7 @@ from unittest.mock import patch import pytest -from collector.helper import PhaseClock, render_phase_totals +from collector.helper import PhaseClock, render_phase_totals, run_measurement_subprocess @pytest.fixture(autouse=True) @@ -22,6 +22,7 @@ def _mock_helper_imports(monkeypatch): fake_helper.benchmark_with_power = lambda **kw: None fake_helper.PhaseClock = PhaseClock fake_helper.render_phase_totals = render_phase_totals + fake_helper.run_measurement_subprocess = run_measurement_subprocess monkeypatch.setitem(__import__("sys").modules, "helper", fake_helper) fake_torch = types.ModuleType("torch") @@ -699,28 +700,16 @@ def test_zero_logged_rows_raises(self, monkeypatch): attention_backend="fa3", ) - def test_configured_subprocess_timeout_raises(self, monkeypatch): + def test_configured_subprocess_timeout_raises(self, monkeypatch, capsys): mod = _import_module() - class TimedOutProcess: - returncode = None - killed = False - waited = False + def timed_out(argv, *, env, cwd, timeout): + # helper.run_measurement_subprocess kills the child and keeps its output + assert timeout == 7 + raise mod.subprocess.TimeoutExpired(cmd=argv, timeout=timeout, output=b"partial sweep") - def communicate(self, timeout=None): - assert timeout == 7 - raise mod.subprocess.TimeoutExpired(cmd="mla-collector", timeout=timeout) - - def kill(self): - self.killed = True - - def wait(self): - self.waited = True - self.returncode = -9 - - process = TimedOutProcess() monkeypatch.setenv("AIC_MLA_MODULE_SUBPROCESS_TIMEOUT_SEC", "7") - monkeypatch.setattr(mod.subprocess, "Popen", lambda *_args, **_kwargs: process) + monkeypatch.setattr(mod, "run_measurement_subprocess", timed_out) with pytest.raises(RuntimeError, match=r"MLA module context subprocess timed out after 7s"): mod._run_mla_subprocess( @@ -735,8 +724,7 @@ def wait(self): attention_backend="fa3", ) - assert process.killed - assert process.waited + assert "partial sweep" in capsys.readouterr().out class TestOrdinaryMLAPrecision: @@ -776,14 +764,14 @@ def test_worker_chunk_size_reaches_subprocess_call(self, monkeypatch, chunk_size def capture_call(*_args, **kwargs): captured.update(kwargs) - def popen(argv, **_kwargs): + def run_child(argv, **_kwargs): # Execute the generated child call without importing the GPU runtime. tree = ast.parse(argv[2]) call = ast.Expression(tree.body[-1].value) eval(compile(call, "", "eval"), {"run_mla_module": capture_call}) - return types.SimpleNamespace(returncode=0, communicate=lambda timeout: (b"", None)) + return 0, b"" - monkeypatch.setattr(mod.subprocess, "Popen", popen) + monkeypatch.setattr(mod, "run_measurement_subprocess", run_child) mod.run_mla_module_worker( 128, 1, diff --git a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py index e2c189b2fe..db5b095ad5 100644 --- a/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py +++ b/python/aisimulate/tests/unit/collector/sglang/test_collect_mla_module_native_quant.py @@ -15,7 +15,7 @@ from unittest.mock import patch import pytest -from collector.helper import PhaseClock, render_phase_totals +from collector.helper import PhaseClock, render_phase_totals, run_measurement_subprocess pytestmark = pytest.mark.unit @@ -47,6 +47,7 @@ def _mock_helper_imports(monkeypatch): # the collector imports the phase-timing helpers at module load fake_helper.PhaseClock = PhaseClock fake_helper.render_phase_totals = render_phase_totals + fake_helper.run_measurement_subprocess = run_measurement_subprocess monkeypatch.setitem(sys.modules, "helper", fake_helper) fake_torch = types.ModuleType("torch") diff --git a/python/aisimulate/tests/unit/collector/test_collect_replay.py b/python/aisimulate/tests/unit/collector/test_collect_replay.py index 0aca3340da..6c66b576ce 100644 --- a/python/aisimulate/tests/unit/collector/test_collect_replay.py +++ b/python/aisimulate/tests/unit/collector/test_collect_replay.py @@ -98,3 +98,14 @@ def test_finish_replay_writes_a_marker_listing_unmatched_requests_across_ops(tmp marker = json.loads((tmp_path / collect.ReplaySpec.MARKER).read_text()) assert marker["requested"] == 2 and marker["matched"] == 1 and marker["unmatched"] == ["[9, 9, 9]"] assert collect._finish_replay(None, tmp_path, log) is None + + +def test_replay_refuses_an_output_root_that_holds_results(tmp_path: Path): + spec = _spec({"[1, 2]"}) + collect._require_replay_output_root_without_results(spec, tmp_path, {}) # empty root: fine + collect._require_replay_output_root_without_results(None, tmp_path, {tmp_path / "gemm_perf.txt": 1}) + with pytest.raises(SystemExit): + collect._require_replay_output_root_without_results(spec, tmp_path, {tmp_path / "gemm_perf.txt": 1}) + (tmp_path / "gemm_perf.parquet").write_bytes(b"") + with pytest.raises(SystemExit): + collect._require_replay_output_root_without_results(spec, tmp_path, {}) diff --git a/python/aisimulate/tests/unit/collector/test_fullnode.py b/python/aisimulate/tests/unit/collector/test_fullnode.py index 00b5d519cc..f0dea2cc2a 100644 --- a/python/aisimulate/tests/unit/collector/test_fullnode.py +++ b/python/aisimulate/tests/unit/collector/test_fullnode.py @@ -172,6 +172,71 @@ def test_runner_fails_closed_when_failure_list_is_missing(monkeypatch): assert "must return" in errors[0]["error_message"] +def test_replay_selects_full_node_cases_like_the_pool_ops(monkeypatch): + cases = [{"shape": 0}, {"shape": 1}, {"shape": 2}] + ran = [] + + def run_cases(*, perf_filename, limit, cases): + ran.extend(cases) + return {"succeeded": len(cases), "failed": []} + + module_name = "fake_fullnode_collector_replay" + monkeypatch.setitem( + sys.modules, + module_name, + SimpleNamespace(__compat__=">=0.5.0", get_cases=lambda: list(cases), run_cases=run_cases), + ) + monkeypatch.setattr(fullnode, "filter_cases", lambda values, **_kwargs: (values, [])) + replay = collect_mod.ReplaySpec("errors.json", {str(cases[1])}) + + errors = fullnode.collect_sglang_fullnode_op( + { + "name": "fake", + "type": "deepep_ll", + "module": module_name, + "get_func": "get_cases", + "run_func": "run_cases", + "perf_filename": "perf.txt", + }, + runtime_version="0.5.12", + limit=None, + shuffle=False, + shuffle_seed=42, + backend="sglang", + resume_options=None, + model_path=None, + case_plan=None, + sm_version=100, + case_filters=None, + replay=replay, + get_test_cases_for_model=lambda get_func, _model_path: get_func(), + resume_checkpoint_cls=_Checkpoint, + logger=_logger(), + ) + + assert errors == [] + assert ran == [cases[1]] + assert replay.report()["matched_by_op"] == {"deepep_ll": 1} + + +def test_collect_sglang_passes_only_keywords_the_full_node_runner_accepts(): + """collect_sglang runs full-node ops after the pool ops: a keyword the runner does not + accept is a TypeError that loses the run's summary and provenance.""" + import ast + import inspect + + tree = ast.parse(Path(collect_mod.__file__).read_text(encoding="utf-8")) + calls = [ + node + for node in ast.walk(tree) + if isinstance(node, ast.Call) and getattr(node.func, "id", None) == "collect_sglang_fullnode_op" + ] + assert calls + accepted = set(inspect.signature(fullnode.collect_sglang_fullnode_op).parameters) + for call in calls: + assert {keyword.arg for keyword in call.keywords} <= accepted + + REAL_FULLNODE_MODULE = "collector.wideep.sglang.collect_deepep_ll" diff --git a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py index 04d92ee23c..d120d01013 100644 --- a/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py +++ b/python/aisimulate/tests/unit/collector/test_glm5next_collection_plan.py @@ -28,13 +28,20 @@ GLM_OFFGRID_WIDTHS = {8, 16, 32, 288, 3336, 6416, 12576, 19360, 24576, 24896, 38720, 77440, 154880} -@pytest.mark.parametrize("backend", ["vllm", "sglang"]) +@pytest.mark.parametrize( + ("backend", "ops"), + [ + ("vllm", {"gemm", "moe", "kda", "mhc_module"}), + # SGLang 0.5.21 is not an audited GLM KDA / mHC runtime: scheduling them only logs failures + ("sglang", {"gemm", "moe"}), + ], +) @pytest.mark.parametrize("model_path", [FP8, NVFP4]) -def test_plan_collects_gemm_moe_kda_and_mhc(backend, model_path): +def test_plan_collects_the_audited_glm_ops(backend, ops, model_path): plan = build_collection_case_plan(backend=backend, model_path=model_path, gpu_type="gb300") assert plan.model_architecture == "Glm5NextForConditionalGeneration" assert plan.sm_version == 103 - assert set(plan.ops) == {"gemm", "moe", "kda", "mhc_module"} + assert set(plan.ops) == ops def test_trtllm_plan_has_no_glm_moe_lane(): diff --git a/python/aisimulate/tests/unit/collector/test_measurement_subprocess.py b/python/aisimulate/tests/unit/collector/test_measurement_subprocess.py new file mode 100644 index 0000000000..b83b75470f --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_measurement_subprocess.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""helper.run_measurement_subprocess: a child that prints progress keeps its worker alive; +a silent one does not, and a timeout keeps the output for diagnosis.""" + +import os +import subprocess +import sys + +import pytest +from collector import helper + +pytestmark = pytest.mark.unit + + +@pytest.fixture +def beats(monkeypatch): + calls = [] + monkeypatch.setattr(helper, "_WORKER_HEARTBEAT", lambda: calls.append(1)) + return calls + + +def _child(code): + return [sys.executable, "-c", code] + + +def test_progress_beats_and_output_is_returned(beats): + code = "import time\nfor i in range(3):\n print('point', i)\n time.sleep(0.3)\n" + rc, out = helper.run_measurement_subprocess(_child(code), env=dict(os.environ), cwd=None, poll_sec=0.2) + assert rc == 0 + assert out.decode().split() == ["point", "0", "point", "1", "point", "2"] + assert beats # unbuffered prints grew the output between polls + + +def test_a_silent_child_does_not_beat(beats): + rc, _ = helper.run_measurement_subprocess( + _child("import time; time.sleep(0.8)"), env=dict(os.environ), cwd=None, poll_sec=0.2 + ) + assert rc == 0 + assert beats == [] + + +def test_timeout_kills_the_child_and_keeps_its_output(beats): + code = "import time\nprint('started')\ntime.sleep(30)\n" + with pytest.raises(subprocess.TimeoutExpired) as caught: + helper.run_measurement_subprocess(_child(code), env=dict(os.environ), cwd=None, timeout=0.6, poll_sec=0.2) + assert b"started" in caught.value.output diff --git a/python/aisimulate/tests/unit/collector/test_standards_exceptions_verified_on.py b/python/aisimulate/tests/unit/collector/test_standards_exceptions_verified_on.py new file mode 100644 index 0000000000..5f1527e241 --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_standards_exceptions_verified_on.py @@ -0,0 +1,38 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""A standards exception scoped to ``versions: all`` is bounded only by ``verified_on`` +(layer_permissions.md "Governed exceptions" item 3): it must name the collector pin of every +table it covers, so a pin bump that nobody re-verified fails here instead of being covered silently.""" + +from pathlib import Path + +import pytest +import yaml +from collector.framework_manifest import MANIFEST_PATH, _registry_entries, _resolve_from, load_manifest +from collector.op_catalog import CATALOG_PATH, load_family_map + +pytestmark = pytest.mark.unit +EXCEPTIONS = Path(__file__).resolve().parents[3] / "collector" / "evidence_exceptions.yaml" +ENTRIES = yaml.safe_load(EXCEPTIONS.read_text())["standards_exceptions"] + + +def _collector_pins(backend: str, table: str) -> set[str]: + manifest, family_map = load_manifest(MANIFEST_PATH), load_family_map(CATALOG_PATH) + return { + _resolve_from(manifest, family_map, backend, entry).version + for entry in _registry_entries(backend) + if table in {Path(str(name)).stem for name in entry.perf_filenames} + } + + +@pytest.mark.parametrize("entry", ENTRIES, ids=[e["rule"].split("—")[-1].strip() for e in ENTRIES]) +def test_verified_on_is_the_collector_pin_of_every_table_in_scope(entry): + scope = entry["scope"] + for table in scope["tables"]: + pins = _collector_pins(scope["backend"], table) + assert pins, f"{scope['backend']} has no collector for {table}" + assert pins == {entry["verified_on"]}, ( + f"{table}: collected on {sorted(pins)}, verified on {entry['verified_on']}; re-verify the " + "deviation on the new pin and move verified_on, or delete the entry" + ) diff --git a/python/aisimulate/tests/unit/collector/test_vllm_kv_block_size.py b/python/aisimulate/tests/unit/collector/test_vllm_kv_block_size.py new file mode 100644 index 0000000000..f54aee205d --- /dev/null +++ b/python/aisimulate/tests/unit/collector/test_vllm_kv_block_size.py @@ -0,0 +1,75 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""framework_kv_block_size: the cache page comes from the model's first attention layer. + +The fakes reproduce vLLM 0.30.0's rules: AttentionBackend.get_preferred_block_size +(v1/attention/backend.py:150-158) and select_common_block_size (v1/worker/utils.py:330-385). +""" + +import ast +import sys +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit +UTILS = Path(__file__).resolve().parents[3] / "collector" / "vllm" / "utils.py" + + +def _backend(*sizes): + class Backend: + @staticmethod + def get_supported_kernel_block_sizes(): + return list(sizes) + + @classmethod + def get_preferred_block_size(cls, default_block_size): + return default_block_size if default_block_size in sizes else min(sizes) + + return Backend + + +def _select_common_block_size(kv_manager_block_size, backends): + if all(kv_manager_block_size in b.get_supported_kernel_block_sizes() for b in backends): + return kv_manager_block_size + common = set.intersection(*(set(b.get_supported_kernel_block_sizes()) for b in backends)) + return max(size for size in common if kv_manager_block_size % size == 0) + + +@pytest.fixture +def kv_block_size(monkeypatch): + for name, attrs in ( + ("vllm", {}), + ("vllm.config", {}), + ("vllm.config.cache", {"CacheConfig": SimpleNamespace(DEFAULT_BLOCK_SIZE=16)}), + ("vllm.v1", {}), + ("vllm.v1.worker", {}), + ("vllm.v1.worker.utils", {"select_common_block_size": _select_common_block_size}), + ): + module = ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + monkeypatch.setitem(sys.modules, name, module) + tree = ast.parse(UTILS.read_text()) + function = next(n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name == "framework_kv_block_size") + namespace = {} + exec(compile(ast.Module(body=[function], type_ignores=[]), str(UTILS), "exec"), namespace) + return namespace["framework_kv_block_size"] + + +def test_dsa_sparse_mla_runs_at_the_indexer_cache_page(kv_block_size): + flashinfer_mla_sparse = _backend(32, 64) # flashinfer_mla_sparse.py:85-86 @0.30.0 + dsa_indexer = _backend(64) # mla/indexer.py:203-204 @0.30.0 (CUDA) + # alone, the sparse MLA backend would pick its own preferred 32 ... + assert kv_block_size(flashinfer_mla_sparse) == 32 + # ... but serving sizes the cache from the indexer, registered first, and keeps 64 + assert kv_block_size(flashinfer_mla_sparse, cache_backend_cls=dsa_indexer) == 64 + assert kv_block_size(flashinfer_mla_sparse, cache_backend_cls=dsa_indexer, user_block_size=32) == 32 + + +def test_first_layer_backend_is_the_default_cache_backend(kv_block_size): + flash_attn = _backend(16, 32, 64) + assert kv_block_size(flash_attn) == 16 + assert kv_block_size(_backend(64)) == 64 diff --git a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py index 8e76b6e0cf..09a18de6fe 100644 --- a/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py +++ b/python/aisimulate/tests/unit/collector/test_vllm_mla_graph_timing.py @@ -154,7 +154,7 @@ def cleanup(): "log_perf": lambda **kwargs: published.append(kwargs), "_cleanup": cleanup, "vllm_version": "0.25.0", - "traceback": SimpleNamespace(print_exc=lambda: None), + "traceback": SimpleNamespace(print_exc=lambda: None, clear_frames=lambda _tb: None), "time": time, } _load_function(COLLECTOR / "vllm/utils.py", "with_exit_stack", namespace) diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py index eafe60a80f..81345e0749 100644 --- a/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py +++ b/python/aisimulate/tests/unit/collector/trtllm/test_collect_gemm_op_cache.py @@ -169,10 +169,3 @@ def test_fp8_block_post_load_runs_once_per_copy_and_is_tagged_deepgemm_on_sm100( assert [op.kwargs["quant_config"] for op in _Linear.instances] == [expected_quant] * 2 assert [row["kernel_source"] for row in published] == ["deepgemm", "deepgemm"] assert all(row["item_list"][0]["latency"] == pytest.approx(0.5) for row in published) - - -def test_fp8_alignment_limit_still_raises_before_any_module_is_built(): - namespace, built, _, _, _ = _load() - with pytest.raises(RuntimeError, match="FIXME\\(kernel-limit\\)"): - namespace["run_gemm"]("fp8", 16, 24, 4096, perf_filename="gemm_perf.txt", device="cuda:0") - assert built == [] and _Linear.instances == [] diff --git a/python/aisimulate/tests/unit/collector/trtllm/test_moe_serving_gate.py b/python/aisimulate/tests/unit/collector/trtllm/test_moe_serving_gate.py new file mode 100644 index 0000000000..955c13625b --- /dev/null +++ b/python/aisimulate/tests/unit/collector/trtllm/test_moe_serving_gate.py @@ -0,0 +1,78 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""GPU-free: the TRT-LLM MoE collector routes through the architecture's serving gate with a zeroed bias.""" + +import ast +import sys +from contextlib import nullcontext +from pathlib import Path +from types import ModuleType, SimpleNamespace + +import pytest + +pytestmark = pytest.mark.unit +COLLECT_MOE = Path(__file__).resolve().parents[4] / "collector" / "trtllm" / "collect_moe.py" + + +class _Bias: + zeroed = False + + def zero_(self): + self.zeroed = True + + +def _gate(kind): + class Gate: + def __init__(self, *args, **kwargs): + self.kind = kind + self.kwargs = kwargs + self.e_score_correction_bias = _Bias() + + return Gate + + +@pytest.fixture +def serving_gate(monkeypatch): + tree = ast.parse(COLLECT_MOE.read_text()) + helpers = [ + n for n in tree.body if isinstance(n, ast.FunctionDef) and n.name in ("_zero_routing_bias", "_serving_moe_gate") + ] + namespace = {"torch": SimpleNamespace(no_grad=nullcontext), "DeepseekV3Gate": _gate("v3")} + exec(compile(ast.Module(body=helpers, type_ignores=[]), str(COLLECT_MOE), "exec"), namespace) + for name in ("tensorrt_llm", "tensorrt_llm._torch", "tensorrt_llm._torch.models"): + monkeypatch.setitem(sys.modules, name, ModuleType(name)) + for name, attr, kind in ( + ("modeling_deepseekv4", "DeepseekV4Gate", "v4"), + ("modeling_minimaxm3", "MiniMaxM3Gate", "m3"), + ): + module = ModuleType(f"tensorrt_llm._torch.models.{name}") + setattr(module, attr, _gate(kind)) + monkeypatch.setitem(sys.modules, module.__name__, module) + return namespace["_serving_moe_gate"] + + +@pytest.mark.parametrize( + ("architecture", "kind"), + [ + ("DeepseekV3ForCausalLM", "v3"), + ("GlmMoeDsaForCausalLM", "v3"), + ("DeepseekV4ForCausalLM", "v4"), + ("MiniMaxM3ForCausalLM", "m3"), + ], +) +def test_gate_matches_the_architecture_and_its_bias_is_zeroed(serving_gate, architecture, kind): + declared = SimpleNamespace( + architecture=architecture, + sglang_moe_num_expert_group=8, + sglang_moe_topk_group=4, + sglang_moe_routed_scaling_factor=2.5, + ) + gate = serving_gate(declared, 7168, 256, 8, "bf16", "trtllm") + assert gate.kind == kind + assert gate.e_score_correction_bias.zeroed + if kind == "v3": + # serving compares the resolved backend upper-case: 'TRTLLM' keeps a bf16 bias + assert gate.kwargs["moe_backend"] == "TRTLLM" + if kind == "v4": + assert gate.kwargs["is_hashed"] is False diff --git a/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py b/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py index 2c551e508c..ec4521ce65 100644 --- a/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py +++ b/python/aisimulate/tests/unit/generator/test_engine_limit_preservation.py @@ -77,6 +77,18 @@ def test_externally_evaluated_engine_limits_are_preserved(backend: str, rule: st assert actual[key] == expected[key] +@pytest.mark.parametrize("backend", ["vllm", "sglang", "trtllm"]) +def test_preserved_unaligned_budget_is_rounded_with_a_warning(backend, caplog): + """A Sweeper candidate simulated at 5000 tokens deploys 5056; the change is logged, not silent.""" + params = _params(preserve=True, rule="benchmark") + params["params"]["prefill"]["max_num_tokens"] = 5000 + params["params"]["prefill"]["max_prefill_tokens"] = 5000 + with caplog.at_level(logging.WARNING): + result = apply_rule_plugins(copy.deepcopy(params), backend) + assert result["params"]["prefill"]["max_num_tokens"] == 5056 + assert "prefill max_num_tokens=5000 was evaluated by the caller; the deployment uses 5056" in caplog.text + + def test_default_vllm_rules_still_resize_unpinned_requests(): result = apply_rule_plugins(_params(preserve=False), "vllm") diff --git a/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py b/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py index 21bb3a6076..c75f8b5837 100644 --- a/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py +++ b/python/aisimulate/tests/unit/generator/test_facts_hardware_l40s.py @@ -79,7 +79,7 @@ def test_l40s_fills_marlin_for_nvfp4_moe_only(): for quant in (None, {"quant_method": "fp8", "weight_block_size": [128, 128]}, {"quant_method": "mxfp4"}): ctx = {"is_moe": True, "moe_config": {}} apply_moe_backend(ctx, hw, backend="trtllm", quantization=quant) - assert "backend" not in ctx["moe_config"], quant # the template default (CUTLASS) applies + assert "backend" not in ctx["moe_config"], quant # the template default applies (rc29: AUTO -> CUTLASS on SM89) dense = {"is_moe": False, "moe_config": {}} apply_moe_backend(dense, hw, backend="trtllm", quantization={"quant_algo": "NVFP4"}) assert "backend" not in dense["moe_config"] diff --git a/python/aisimulate/tests/unit/generator/test_sweeper_request.py b/python/aisimulate/tests/unit/generator/test_sweeper_request.py index c6149efc44..0be267add1 100644 --- a/python/aisimulate/tests/unit/generator/test_sweeper_request.py +++ b/python/aisimulate/tests/unit/generator/test_sweeper_request.py @@ -815,3 +815,29 @@ def _assert_artifact_block_size(artifacts, backend, role, expected): assert flag not in shlex.split(artifacts[f"cli_args_{role}"]) else: assert _cli_flag_value(artifacts[f"cli_args_{role}"], flag) == str(expected) + + +@pytest.mark.parametrize(("hardware_sku", "implementation"), [("h200_sxm", None), ("b200_sxm", "msa")]) +def test_minimax_m3_trtllm_candidate_renders_its_sparse_attention_keys(hardware_sku, implementation): + """The Sweeper lowers through rule=benchmark; rc29 cannot build the M3 KV-cache manager + without sparse_attention_config.algorithm, on every SM.""" + candidate = _agg_candidate( + model_name="MiniMaxAI/MiniMax-M3", + backend="trtllm", + backend_version="1.3.0rc29", + hardware_sku=hardware_sku, + moe_ep=2, + adapters={}, + ) + request = from_sweeper_candidate( + candidate, + workload={"isl": 4000, "osl": 1000}, + model_facts=ModelFacts(is_moe=True, architecture="MiniMaxM3ForCausalLM"), + generator_overrides={"K8sConfig": {"k8s_image": "example/trtllm:1.3.0rc29"}}, + ) + artifacts = generate_from_request(request) + name = next(name for name in artifacts if "extra_engine_args" in name and "agg" in name) + expected = {"algorithm": "minimax_m3"} + if implementation: + expected["implementation"] = implementation + assert yaml.safe_load(artifacts[name]).get("sparse_attention_config") == expected diff --git a/python/aisimulate/tests/unit/generator/test_trtllm_moe_backend_default.py b/python/aisimulate/tests/unit/generator/test_trtllm_moe_backend_default.py new file mode 100644 index 0000000000..372bf74479 --- /dev/null +++ b/python/aisimulate/tests/unit/generator/test_trtllm_moe_backend_default.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""TRT-LLM MoE backend on Blackwell: WIDEEP only for releases whose MoeConfig accepts it +(through 1.3.0rc24); from rc29 an unset backend renders the framework default AUTO.""" + +from __future__ import annotations + +import pytest +import yaml + +from aisimulate.generator.api import generate_backend_artifacts +from aisimulate.generator.facts.apply import apply_moe_backend + +pytestmark = pytest.mark.unit + + +@pytest.mark.parametrize( + ("version", "expected"), + [("1.3.0rc14", "WIDEEP"), ("1.3.0rc24", "WIDEEP"), ("1.3.0rc25", None), ("1.3.0rc29", None), (None, "WIDEEP")], +) +def test_wideep_is_filled_only_for_releases_that_accept_it(version, expected): + ctx = {"is_moe": True, "moe_config": {}} + apply_moe_backend(ctx, {"moe_backend": {"trtllm": "WIDEEP"}}, backend="trtllm", backend_version=version) + assert ctx["moe_config"].get("backend") == expected + + +def _params(system: str) -> dict: + return { + "ServiceConfig": { + "model_path": "deepseek-ai/DeepSeek-V3", + "served_model_path": "deepseek-ai/DeepSeek-V3", + "served_model_name": "DeepSeek-V3", + "include_frontend": True, + }, + "K8sConfig": {"name_prefix": "test", "k8s_namespace": "default"}, + "DynConfig": {"mode": "agg"}, + "WorkerConfig": {"agg_workers": 1, "agg_gpus_per_worker": 8}, + "NodeConfig": {"num_gpus_per_node": 8, "system_name": system}, + "SlaConfig": {"isl": 1024, "osl": 256}, + "ModelConfig": {"is_moe": True, "prefix": 0}, + "BenchConfig": {}, + "params": { + "agg": { + "tensor_parallel_size": 8, + "pipeline_parallel_size": 1, + "data_parallel_size": 1, + "moe_tensor_parallel_size": 1, + "moe_expert_parallel_size": 8, + "max_batch_size": 32, + "max_num_tokens": 4096, + "max_seq_len": 4096, + }, + }, + } + + +def _moe_backend(version: str, system: str) -> str: + artifacts = generate_backend_artifacts(backend="trtllm", backend_version=version, params=_params(system)) + name = next(name for name in artifacts if "extra_engine_args" in name and "agg" in name) + return yaml.safe_load(artifacts[name])["moe_config"]["backend"] + + +@pytest.mark.parametrize(("version", "expected"), [("1.3.0rc23", "WIDEEP"), ("1.3.0rc29", "AUTO")]) +def test_blackwell_render_keeps_wideep_before_rc25_and_auto_from_rc29(version, expected): + assert _moe_backend(version, "b200_sxm") == expected + + +def test_rc29_hopper_render_keeps_the_hardware_fact(): + assert _moe_backend("1.3.0rc29", "h200_sxm") == "CUTLASS" diff --git a/python/aisimulate/tests/unit/tools/test_collect_campaign.py b/python/aisimulate/tests/unit/tools/test_collect_campaign.py index 5e66800bcd..1b0724b9af 100644 --- a/python/aisimulate/tests/unit/tools/test_collect_campaign.py +++ b/python/aisimulate/tests/unit/tools/test_collect_campaign.py @@ -46,10 +46,39 @@ def test_one_namespace_per_shard_and_readonly_checkout(tmp_path): plan = c.load_plan(_plan(tmp_path, [{"op": "gemm", "name": "bf16"}, {"op": "gemm", "name": "fp8"}])) dirs = {c.shard_dir(tmp_path, s) for s in plan.shards} assert len(dirs) == 2 + ref = "0123456789abcdef0123456789abcdef01234567" argv = c.docker_argv( - plan, plan.shards[0], image="img", gpu=3, checkout=Path("/ck"), shard_dir=dirs.pop(), container_name="x" + plan, + plan.shards[0], + image="img", + gpu=3, + checkout=Path("/ck"), + shard_dir=dirs.pop(), + container_name="x", + collector_ref=ref, ) assert '"device=3"' in argv and "/ck:/ais:ro" in argv and argv[argv.index("-w") + 1] == "/out" + # the mount has no .git and the images no git: provenance gets the commit from the launcher + assert f"AISIM_COLLECTOR_REF={ref}" in argv + + +def test_collector_ref_is_the_clean_checkout_commit(tmp_path): + import subprocess + + c = _load() + + def git(*args): + subprocess.run(["git", "-C", str(tmp_path), *args], check=True, capture_output=True) + + git("init", "-q") + (tmp_path / "collect.py").write_text("v1\n") + git("add", "collect.py") + git("-c", "user.name=t", "-c", "user.email=t@example.com", "commit", "-q", "-m", "v1") + ref = c.collector_ref(tmp_path) + assert len(ref) == 40 and int(ref, 16) >= 0 + (tmp_path / "collect.py").write_text("v2\n") # uncommitted: the rows would name the wrong code + with pytest.raises(SystemExit, match="uncommitted"): + c.collector_ref(tmp_path) def test_duplicate_shard_keys_are_rejected(tmp_path): diff --git a/python/aisimulate/tools/perf_database/collect_campaign.py b/python/aisimulate/tools/perf_database/collect_campaign.py index 44a52a4c66..c6a07e4d20 100644 --- a/python/aisimulate/tools/perf_database/collect_campaign.py +++ b/python/aisimulate/tools/perf_database/collect_campaign.py @@ -47,6 +47,7 @@ import argparse import json +import re import shlex import subprocess import sys @@ -160,8 +161,41 @@ def collector_argv(plan: Plan, shard: Shard) -> list[str]: return argv +def collector_ref(checkout: Path) -> str: + """The commit the mounted checkout runs, for every shard's provenance. + + The container mounts only python/aisimulate (no .git) and the framework images carry no + git, so collect.py's own lookup would record "unknown"; it reads AISIM_COLLECTOR_REF + instead. A dirty tree is refused: its rows would name a commit that did not produce them. + """ + + def git(*args: str) -> str: + return subprocess.run( + ["git", "-C", str(checkout), *args], capture_output=True, text=True, check=True + ).stdout.strip() + + try: + ref = git("rev-parse", "HEAD") + dirty = git("status", "--porcelain", "--untracked-files=no", "--", ".") + except (OSError, subprocess.CalledProcessError) as error: + raise SystemExit(f"{checkout}: cannot resolve the collector commit ({error})") from error + if dirty: + raise SystemExit(f"{checkout}: uncommitted changes; commit them so the rows record the code that ran") + if not re.fullmatch(r"[0-9a-f]{40,64}", ref): + raise SystemExit(f"{checkout}: `git rev-parse HEAD` gave {ref!r}, not a full commit hash") + return ref + + def docker_argv( - plan: Plan, shard: Shard, *, image: str, gpu: int, checkout: Path, shard_dir: Path, container_name: str + plan: Plan, + shard: Shard, + *, + image: str, + gpu: int, + checkout: Path, + shard_dir: Path, + container_name: str, + collector_ref: str, ) -> list[str]: return [ "docker", @@ -180,6 +214,8 @@ def docker_argv( "-e", f"PYTHONPATH={CONTAINER_CHECKOUT}", "-e", + f"AISIM_COLLECTOR_REF={collector_ref}", + "-e", f"TRITON_CACHE_DIR={CONTAINER_OUT}/.cache/triton", "-e", f"DG_JIT_CACHE_DIR={CONTAINER_OUT}/.cache/deep_gemm", @@ -220,11 +256,30 @@ def _log(root: Path, record: dict) -> None: fh.write(json.dumps({"ts": time.strftime("%Y-%m-%dT%H:%M:%S"), **record}) + "\n") -def run_shard(plan: Plan, shard: Shard, *, root: Path, gpu: int, checkout: Path, image: str, mode: str = "run") -> int: +def run_shard( + plan: Plan, + shard: Shard, + *, + root: Path, + gpu: int, + checkout: Path, + image: str, + collector_ref: str, + mode: str = "run", +) -> int: d = shard_dir(root, shard) (d / ".cache").mkdir(parents=True, exist_ok=True) name = f"{'f' if mode == 'finalize' else 'c'}_{plan.backend}_{shard.op}_{shard.name}" - cmd = docker_argv(plan, shard, image=image, gpu=gpu, checkout=checkout, shard_dir=d, container_name=name) + cmd = docker_argv( + plan, + shard, + image=image, + gpu=gpu, + checkout=checkout, + shard_dir=d, + container_name=name, + collector_ref=collector_ref, + ) _log( root, { @@ -234,6 +289,7 @@ def run_shard(plan: Plan, shard: Shard, *, root: Path, gpu: int, checkout: Path, "gpu": gpu, "image": image, "image_digest": image_digest(image), + "collector_ref": collector_ref, "argv": collector_argv(plan, shard), }, ) @@ -271,6 +327,7 @@ def run_campaign( continue todo.append(s) print(f"{mode}: {len(todo)}/{len(plan.shards)} shards on gpus {gpus}") + ref = collector_ref(checkout) # once, before any shard starts: every shard records the same commit images = {s.op: (image_override or collect_image_tag(pinned_image(plan.backend, s.op))) for s in todo} for img in sorted(set(images.values())): if image_digest(img) is None: @@ -304,6 +361,8 @@ def run_campaign( str(checkout), "--image", images[s.op], + "--collector-ref", + ref, "--mode", mode, ] @@ -349,13 +408,21 @@ def main() -> int: h.add_argument("--gpu", required=True, type=int) h.add_argument("--shard", required=True) h.add_argument("--image", required=True) + h.add_argument("--collector-ref", required=True) h.add_argument("--mode", required=True) args = ap.parse_args() if args.cmd == "_shard": plan = load_plan(args.plan) shard = next(s for s in plan.shards if s.key == args.shard) return run_shard( - plan, shard, root=args.root, gpu=args.gpu, checkout=args.checkout, image=args.image, mode=args.mode + plan, + shard, + root=args.root, + gpu=args.gpu, + checkout=args.checkout, + image=args.image, + collector_ref=args.collector_ref, + mode=args.mode, ) if args.cmd == "build-image": base = pinned_image(args.backend, args.op) diff --git a/scripts/audit_pr244_data.py b/scripts/audit_pr244_data.py index d79e54eabd..c588451aa3 100644 --- a/scripts/audit_pr244_data.py +++ b/scripts/audit_pr244_data.py @@ -20,6 +20,11 @@ EVIDENCE = ROOT / "benchmarks/evidence/collector/pr244" SOURCE = "1f29ee459882796db312b8a288c040b83edbb211" SYSTEMS = ("b300_sxm", "gb200", "gb300", "h100_sxm", "h200_sxm") +# Tables PR #244 shipped that were later removed from the tree. The ledger keeps their reported +# hashes (it records what was collected); the inventory check compares the remaining tables. +RETIRED_TABLES = { + "dsv4_hca_attn_module_perf.parquet": "hca_attn FMLA sidecar removed with its SDK reader (#649)", +} def sha256(data): @@ -165,7 +170,8 @@ def build_manifest(): systems = summarize_cases(ledger) for system, summary in systems.items(): tables = {path.name: summarize_table(path) for path in sorted((DATA / system).glob("*/vllm/0.25.0/*.parquet"))} - reported = ledger["systems"][system]["reported_tables"] + reported_tables = ledger["systems"][system]["reported_tables"] + reported = {name: table for name, table in reported_tables.items() if name not in RETIRED_TABLES} require(set(tables) == set(reported), f"Table inventory drift: {system}") for name, table in tables.items(): require(