From 337c591b7a958d2efd6b595b2741f26e1deb6b18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Quang=20H=C3=B2a?= Date: Tue, 26 May 2026 13:31:43 +0700 Subject: [PATCH] =?UTF-8?q?fix(techniques):=20rename=20mtp=5Flayers=20?= =?UTF-8?q?=E2=86=92=20num=5Fspeculative=5Ftokens,=20add=20max=5Fnum=5Fbat?= =?UTF-8?q?ched=5Ftokens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The benchmark artifact on vngcloud/InferenceX renamed the per-row layer count field to the canonical vLLM/SGLang name `num_speculative_tokens` (now applies to any draft method, not just MTP). It also gained a new top-level `max_num_batched_tokens` field. Mirroring both on the dashboard side. ETL (normalizers.ts): - TECHNIQUE_KEYS picks up num_speculative_tokens + max_num_batched_tokens - parseTechniques now reads top-level fields too (the new artifact emits them flat, not nested under a `techniques` object); nested-object form is also accepted for forward-compat - PREFIX_TO_NUM_SPEC_TOKENS replaces PREFIX_TO_MTP_LAYERS for the legacy gemma4n4/n6 fallback path Migration 008: - Drop benchmark_results_mtp_layers_idx, add expression indexes for num_speculative_tokens and max_num_batched_tokens - TRUNCATE so re-ingest starts under the new shape UI (recipe-data.ts): - describeTechniques renders `MTP×6` / `EAGLE×3` based on techniques.num_speculative_tokens Tests updated. db 214/214 + app recipe-data 8/8 green. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/evaluation/recipe-data.test.ts | 25 ++++---- .../src/components/evaluation/recipe-data.ts | 13 ++-- ...e_mtp_layers_to_num_speculative_tokens.sql | 28 ++++++++ packages/db/src/etl/benchmark-mapper.test.ts | 8 +-- packages/db/src/etl/normalizers.ts | 64 ++++++++++++------- 5 files changed, 96 insertions(+), 42 deletions(-) create mode 100644 packages/db/migrations/008_rename_mtp_layers_to_num_speculative_tokens.sql diff --git a/packages/app/src/components/evaluation/recipe-data.test.ts b/packages/app/src/components/evaluation/recipe-data.test.ts index 8f4840ece..fc27500f2 100644 --- a/packages/app/src/components/evaluation/recipe-data.test.ts +++ b/packages/app/src/components/evaluation/recipe-data.test.ts @@ -40,13 +40,14 @@ describe('describeTechniques', () => { expect(describeTechniques({})).toBe('baseline'); }); - it('labels mtp + layer count compactly', () => { - expect(describeTechniques({ spec_method: 'mtp', mtp_layers: 4 })).toBe('MTP×4'); - expect(describeTechniques({ spec_method: 'mtp', mtp_layers: 6 })).toBe('MTP×6'); + it('labels spec_method + token count compactly', () => { + expect(describeTechniques({ spec_method: 'mtp', num_speculative_tokens: 4 })).toBe('MTP×4'); + expect(describeTechniques({ spec_method: 'mtp', num_speculative_tokens: 6 })).toBe('MTP×6'); + expect(describeTechniques({ spec_method: 'eagle', num_speculative_tokens: 3 })).toBe('EAGLE×3'); }); - it('falls back to capitalized spec_method', () => { - expect(describeTechniques({ spec_method: 'eagle' })).toBe('Eagle'); + it('falls back to formatted spec_method', () => { + expect(describeTechniques({ spec_method: 'eagle' })).toBe('EAGLE'); }); }); @@ -56,7 +57,7 @@ describe('buildRecipeRows', () => { [ bmk({ techniques: {}, metrics: { tput_per_gpu: 100, median_tpot: 0.01 } }), bmk({ - techniques: { spec_method: 'mtp', mtp_layers: 4 }, + techniques: { spec_method: 'mtp', num_speculative_tokens: 4 }, metrics: { tput_per_gpu: 150, median_tpot: 0.0067 }, }), ], @@ -75,11 +76,11 @@ describe('buildRecipeRows', () => { const rows = buildRecipeRows( [ bmk({ - techniques: { spec_method: 'mtp', mtp_layers: 4 }, + techniques: { spec_method: 'mtp', num_speculative_tokens: 4 }, metrics: { tput_per_gpu: 120 }, }), bmk({ - techniques: { spec_method: 'mtp', mtp_layers: 6 }, + techniques: { spec_method: 'mtp', num_speculative_tokens: 6 }, metrics: { tput_per_gpu: 140 }, }), ], @@ -96,10 +97,10 @@ describe('buildRecipeRows', () => { const rows = buildRecipeRows( [ bmk({ techniques: {} }), - bmk({ techniques: { spec_method: 'mtp', mtp_layers: 4 } }), - bmk({ techniques: { spec_method: 'mtp', mtp_layers: 6 } }), + bmk({ techniques: { spec_method: 'mtp', num_speculative_tokens: 4 } }), + bmk({ techniques: { spec_method: 'mtp', num_speculative_tokens: 6 } }), // Different conc → different group, no shared baseline. - bmk({ techniques: { spec_method: 'mtp', mtp_layers: 4 }, conc: 16 }), + bmk({ techniques: { spec_method: 'mtp', num_speculative_tokens: 4 }, conc: 16 }), ], [], ); @@ -145,7 +146,7 @@ describe('buildRecipeRows', () => { const rows = buildRecipeRows( [ bmk({ - techniques: { spec_method: 'mtp', mtp_layers: 4 }, + techniques: { spec_method: 'mtp', num_speculative_tokens: 4 }, metrics: { tput_per_gpu: 150, median_acceptance_rate: 0.78 }, }), ], diff --git a/packages/app/src/components/evaluation/recipe-data.ts b/packages/app/src/components/evaluation/recipe-data.ts index 056d93fd2..ecfefe72a 100644 --- a/packages/app/src/components/evaluation/recipe-data.ts +++ b/packages/app/src/components/evaluation/recipe-data.ts @@ -67,16 +67,21 @@ function accuracyKey(r: { * short, human-readable summary. Insertion order is preserved from * `parseTechniques()` so the label is deterministic. */ +function specMethodLabel(sm: string): string { + // Short codes (mtp, eagle, ntp, medusa) → ALL CAPS; longer names get title-cased. + return sm.length <= 6 ? sm.toUpperCase() : sm.charAt(0).toUpperCase() + sm.slice(1); +} + export function describeTechniques(t: Record): string { const keys = Object.keys(t); if (keys.length === 0) return 'baseline'; const sm = t.spec_method; - const mtp = t.mtp_layers; - if (typeof sm === 'string' && sm === 'mtp' && typeof mtp === 'number') { - return `MTP×${mtp}`; + const n = t.num_speculative_tokens; + if (typeof sm === 'string' && sm !== 'none' && typeof n === 'number') { + return `${specMethodLabel(sm)}×${n}`; } if (typeof sm === 'string' && sm !== 'none') { - return sm.charAt(0).toUpperCase() + sm.slice(1); + return specMethodLabel(sm); } return keys.map((k) => `${k}=${t[k]}`).join(' · '); } diff --git a/packages/db/migrations/008_rename_mtp_layers_to_num_speculative_tokens.sql b/packages/db/migrations/008_rename_mtp_layers_to_num_speculative_tokens.sql new file mode 100644 index 000000000..59305b844 --- /dev/null +++ b/packages/db/migrations/008_rename_mtp_layers_to_num_speculative_tokens.sql @@ -0,0 +1,28 @@ +-- ============================================================ +-- Migration 008 — rename techniques.mtp_layers → techniques.num_speculative_tokens +-- and add max_num_batched_tokens +-- ============================================================ +-- +-- The benchmark artifact on the vngcloud/InferenceX side renamed its "MTP +-- layer count" field to the canonical vLLM/SGLang name `num_speculative_tokens` +-- (since it now applies to any draft method, not just MTP). It also gained a +-- new `max_num_batched_tokens` field per row. +-- +-- We mirror both on the dashboard side. Drop the old expression index that +-- pointed at `mtp_layers`; add a new one for the renamed key plus one for the +-- new field. Re-truncate so re-ingest under the new shape starts clean. + +drop index if exists benchmark_results_mtp_layers_idx; + +create index benchmark_results_num_speculative_tokens_idx + on benchmark_results ((techniques->>'num_speculative_tokens')) where error is null; + +create index benchmark_results_max_num_batched_tokens_idx + on benchmark_results ((techniques->>'max_num_batched_tokens')) where error is null; + +-- Wipe existing rows (they have `mtp_layers` keys that wouldn't be touched by +-- the renamed parser anyway). Auto-ingest cron repopulates. +truncate benchmark_results, eval_results, eval_samples, run_stats, + availability, changelog_entries, server_logs, workflow_runs + restart identity cascade; +truncate configs restart identity cascade; diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 016bc6968..7bd57d805 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -349,19 +349,19 @@ describe('mapBenchmarkRow', () => { it('reads new-shape techniques object directly', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow( - makeV1Row({ techniques: { spec_method: 'mtp', mtp_layers: 4 } }), + makeV1Row({ techniques: { spec_method: 'mtp', num_speculative_tokens: 4 } }), tracker, ); - expect(result!.techniques).toEqual({ spec_method: 'mtp', mtp_layers: 4 }); + expect(result!.techniques).toEqual({ spec_method: 'mtp', num_speculative_tokens: 4 }); }); - it('infers mtp_layers from gemma4n4/gemma4n6 model prefix (legacy)', () => { + it('infers num_speculative_tokens from gemma4n4/gemma4n6 model prefix (legacy)', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow(makeV1Row({ infmax_model_prefix: 'gemma4n6' }), tracker); expect(result!.config.model).toBe('gemma4'); - expect(result!.techniques.mtp_layers).toBe(6); + expect(result!.techniques.num_speculative_tokens).toBe(6); expect(result!.techniques.spec_method).toBe('mtp'); }); }); diff --git a/packages/db/src/etl/normalizers.ts b/packages/db/src/etl/normalizers.ts index 648c468a2..62fd013a6 100644 --- a/packages/db/src/etl/normalizers.ts +++ b/packages/db/src/etl/normalizers.ts @@ -41,14 +41,16 @@ const PRECISION_SUFFIX = /-(?:fp4|fp8|mxfp4|nvfp4)(?:-.*)?$/iu; const PREFIX_ALIASES: Record = { gptoss: 'gptoss120b', dsv4pro: 'dsv4', - // Gemma-4 MTP variants encode layer count in the prefix on the benchmark side. - // Layer count itself lands in techniques.mtp_layers (see parseTechniques). + // Legacy Gemma-4 MTP variants encoded layer count in the prefix + // (`gemma4n4`/`gemma4n6`). New artifacts emit `gemma4` + a separate + // `num_speculative_tokens` field; we keep the prefix aliases here as a + // fallback for older runs (see parseTechniques). gemma4n4: 'gemma4', gemma4n6: 'gemma4', }; -/** Prefix → mtp_layers count, when the variant smuggles it into the model prefix. */ -const PREFIX_TO_MTP_LAYERS: Record = { +/** Prefix → num_speculative_tokens, when the legacy artifact smuggled N into the model prefix. */ +const PREFIX_TO_NUM_SPEC_TOKENS: Record = { gemma4n4: 4, gemma4n6: 6, }; @@ -183,18 +185,26 @@ export function normalizeSpecMethod(spec: any): string { } /** Known technique keys we promote into the techniques jsonb explicitly. */ -const TECHNIQUE_KEYS = ['spec_method', 'mtp_layers', 'kv_cache_dtype', 'prefix_cache'] as const; +const TECHNIQUE_KEYS = [ + 'spec_method', + 'num_speculative_tokens', + 'max_num_batched_tokens', + 'kv_cache_dtype', + 'prefix_cache', +] as const; /** * Build the per-measurement `techniques` jsonb from a raw artifact row. * - * New-shape artifacts emit a top-level `techniques` object directly: - * { "techniques": { "spec_method": "mtp", "mtp_layers": 4 } } + * The new-shape artifact emits the technique fields at the **top level** of + * each row (not nested in a `techniques` object), e.g.: + * { "spec_decoding": "mtp", "num_speculative_tokens": 6, "max_num_batched_tokens": 4096, … } * - * Legacy-shape artifacts emit a top-level `spec_decoding` field and encode - * the MTP layer count in the model prefix (`gemma4n4`, `gemma4n6`); both are - * coerced into the new shape here so the rest of the pipeline only sees one - * format. Empty/absent values are dropped (no `spec_method: 'none'` keys). + * For forward-compat we also accept a nested `techniques: { ... }` object. + * + * Legacy artifacts (pre-rename) smuggled the layer count into the model prefix + * (`gemma4n4`, `gemma4n6`); we keep that fallback so older runs still ingest. + * Empty/absent values are dropped (no `spec_method: 'none'` keys). * * @param row - Raw artifact dict. * @returns Techniques jsonb. May be empty (`{}`) when no knobs were tuned. @@ -202,16 +212,26 @@ const TECHNIQUE_KEYS = ['spec_method', 'mtp_layers', 'kv_cache_dtype', 'prefix_c export function parseTechniques(row: Record): Record { const out: Record = {}; - // New-shape: top-level techniques object. + const setIfPresent = (key: (typeof TECHNIQUE_KEYS)[number], v: any) => { + if (v === undefined || v === null || v === '') return; + out[key] = typeof v === 'number' ? v : String(v).toLowerCase(); + }; + + // Forward-compat: nested `techniques` object. if (row.techniques && typeof row.techniques === 'object') { for (const k of TECHNIQUE_KEYS) { - const v = (row.techniques as Record)[k]; - if (v === undefined || v === null || v === '') continue; - out[k] = typeof v === 'number' ? v : String(v).toLowerCase(); + setIfPresent(k, (row.techniques as Record)[k]); } } - // Legacy fallback: spec_decoding at top level → techniques.spec_method. + // New-shape: technique fields at top level. + setIfPresent('num_speculative_tokens', row.num_speculative_tokens); + setIfPresent('max_num_batched_tokens', row.max_num_batched_tokens); + setIfPresent('kv_cache_dtype', row.kv_cache_dtype); + setIfPresent('prefix_cache', row.prefix_cache); + + // spec_decoding is the canonical top-level field name; map to spec_method + // unless we already pulled one out of the nested techniques object. if (!('spec_method' in out)) { const legacy = row.spec_decoding; if (legacy && legacy !== '' && String(legacy).toLowerCase() !== 'none') { @@ -219,13 +239,13 @@ export function parseTechniques(row: Record): Record