From 0e35d686dcd5f144f3e05bbe771a8c8a92f8c636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Quang=20H=C3=B2a?= Date: Tue, 26 May 2026 16:07:53 +0700 Subject: [PATCH] feat(evaluation): add topology grouping + technique-category filter MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two fixes after the first real recipe-comparison ingest of gemma4 + the two new sweeps (1×H100 mnbt sweep, 2×H100 spec-decoding): 1. Topology was collapsing. groupKey only included (model, hw, framework, precision, isl, osl, conc) — so 1× H100 (TP=1) and 2× H100 (TP=2) rows merged into a single "h100" row group and the user couldn't tell them apart. Now the groupKey also includes num_prefill_gpu, num_decode_gpu, prefill_tp, decode_tp, disagg. New "Topology" column renders "1× (TP=1)" / "2× (TP=2)" for non-disagg and "{P}P+{D}D (TP=p/d)" for disagg layouts. 2. Spec-decoding and batch-size tuning runs were mixing in the same view. Added a filter chip group at the top of the page driven by `categorizeTechniques`: - baseline (techniques = {}) - spec-decoding (spec_method set) - batch-size (max_num_batched_tokens set) - kv-cache (kv_cache_dtype set) - prefix-cache (prefix_cache set) - other (anything else) Only chips with at least one row are rendered. When a non-"all" chip is active, baseline rows still show through so the user keeps a reference variant. Tests: 10/10 pass (added 2 new — topology-split + categorize). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../evaluation/RecipeComparison.tsx | 74 ++++++++++-- .../components/evaluation/recipe-data.test.ts | 39 ++++++- .../src/components/evaluation/recipe-data.ts | 105 +++++++++++++++++- 3 files changed, 202 insertions(+), 16 deletions(-) diff --git a/packages/app/src/components/evaluation/RecipeComparison.tsx b/packages/app/src/components/evaluation/RecipeComparison.tsx index fd2202505..dd961a9d3 100644 --- a/packages/app/src/components/evaluation/RecipeComparison.tsx +++ b/packages/app/src/components/evaluation/RecipeComparison.tsx @@ -1,14 +1,20 @@ 'use client'; import { ExternalLink } from 'lucide-react'; -import { useMemo } from 'react'; +import { useMemo, useState } from 'react'; import { DataTable, type DataTableColumn } from '@/components/ui/data-table'; import { useGlobalFilters } from '@/components/GlobalFilterContext'; import { useBenchmarks } from '@/hooks/api/use-benchmarks'; import { useEvaluations } from '@/hooks/api/use-evaluations'; +import { track } from '@/lib/analytics'; -import { buildRecipeRows, type RecipeRow } from './recipe-data'; +import { + buildRecipeRows, + TECHNIQUE_CATEGORIES, + type RecipeRow, + type TechniqueCategory, +} from './recipe-data'; function fmtNum(n: number | null, digits = 2, suffix = ''): string { if (n === null || !Number.isFinite(n)) return '—'; @@ -32,6 +38,12 @@ const COLUMNS: DataTableColumn[] = [ cell: (r) => r.hardware, sortValue: (r) => r.hardware, }, + { + header: 'Topology', + align: 'left', + cell: (r) => {r.topology}, + sortValue: (r) => `${r.numPrefillGpu + r.numDecodeGpu}_${r.prefillTp}_${r.disagg ? 1 : 0}`, + }, { header: 'Precision', align: 'left', @@ -132,6 +144,7 @@ const COLUMNS: DataTableColumn[] = [ export default function RecipeComparison() { const { selectedModel, selectedRunDate } = useGlobalFilters(); + const [category, setCategory] = useState('all'); const { data: benchmarks, @@ -140,7 +153,7 @@ export default function RecipeComparison() { } = useBenchmarks(selectedModel, selectedRunDate || undefined); const { data: evals, isLoading: evalLoading, error: evalError } = useEvaluations(); - const rows = useMemo(() => { + const allRows = useMemo(() => { if (!benchmarks || benchmarks.length === 0) return []; const built = buildRecipeRows(benchmarks, evals ?? []); // Sort: group rows together, baseline first within each group, then by speedup descending. @@ -151,18 +164,59 @@ export default function RecipeComparison() { }); }, [benchmarks, evals]); + // Only show category chips for axes that actually have rows. Avoids empty + // chips that lead to a blank table when selected. + const availableCategories = useMemo(() => { + const seen = new Set(allRows.map((r) => r.category)); + return TECHNIQUE_CATEGORIES.filter((c) => c.value === 'all' || seen.has(c.value)); + }, [allRows]); + + const rows = useMemo(() => { + if (category === 'all') return allRows; + // Always include baseline rows in any non-"all" filter so the user has a + // reference variant on the page if one exists in the group. + return allRows.filter((r) => r.category === category || r.isBaseline); + }, [allRows, category]); + const loading = bmkLoading || evalLoading; const error = bmkError ?? evalError; return (
-
-

Recipe Comparison

-

- Compare runtime knobs (speculative decoding, MTP layers, …) on the same deployment. - Speedup is computed against the variant whose techniques is empty within the - same (model × hw × precision × isl/osl × conc) group. -

+
+
+

Recipe Comparison

+

+ Compare runtime knobs (speculative decoding, batch size, …) on the same deployment. + Speedup is computed against the variant whose techniques is empty within + the same (model × topology × precision × isl/osl × conc) group. +

+
+ {availableCategories.length > 1 && ( +
+ Filter by knob: + {availableCategories.map((c) => { + const active = category === c.value; + return ( + + ); + })} +
+ )}
{error && ( diff --git a/packages/app/src/components/evaluation/recipe-data.test.ts b/packages/app/src/components/evaluation/recipe-data.test.ts index fc27500f2..44f2692ae 100644 --- a/packages/app/src/components/evaluation/recipe-data.test.ts +++ b/packages/app/src/components/evaluation/recipe-data.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import type { BenchmarkRow, EvalRow } from '@/lib/api'; -import { buildRecipeRows, describeTechniques } from './recipe-data'; +import { buildRecipeRows, categorizeTechniques, describeTechniques } from './recipe-data'; function bmk(overrides: Partial): BenchmarkRow { return { @@ -142,6 +142,43 @@ describe('buildRecipeRows', () => { expect(rows[0]?.accuracy).toBeCloseTo(0.842, 5); }); + it('separates 1× and 2× H100 into distinct groups via topology in groupKey', () => { + const rows = buildRecipeRows( + [ + bmk({ + num_prefill_gpu: 1, + num_decode_gpu: 1, + prefill_tp: 1, + decode_tp: 1, + techniques: { max_num_batched_tokens: 4096 }, + }), + bmk({ + num_prefill_gpu: 2, + num_decode_gpu: 2, + prefill_tp: 2, + decode_tp: 2, + techniques: { spec_method: 'mtp', num_speculative_tokens: 4 }, + }), + ], + [], + ); + expect(new Set(rows.map((r) => r.groupKey)).size).toBe(2); + const oneXh = rows.find((r) => r.numPrefillGpu === 1)!; + const twoXh = rows.find((r) => r.numPrefillGpu === 2)!; + expect(oneXh.topology).toBe('1× (TP=1)'); + expect(twoXh.topology).toBe('2× (TP=2)'); + }); + + it('tags rows with technique category for filter chips', () => { + expect(categorizeTechniques({})).toBe('baseline'); + expect(categorizeTechniques({ spec_method: 'mtp', num_speculative_tokens: 4 })).toBe( + 'spec-decoding', + ); + expect(categorizeTechniques({ max_num_batched_tokens: 4096 })).toBe('batch-size'); + expect(categorizeTechniques({ kv_cache_dtype: 'fp8' })).toBe('kv-cache'); + expect(categorizeTechniques({ prefix_cache: 'true' })).toBe('prefix-cache'); + }); + it('reads acceptance rate from variant metrics', () => { const rows = buildRecipeRows( [ diff --git a/packages/app/src/components/evaluation/recipe-data.ts b/packages/app/src/components/evaluation/recipe-data.ts index ecfefe72a..4b13010be 100644 --- a/packages/app/src/components/evaluation/recipe-data.ts +++ b/packages/app/src/components/evaluation/recipe-data.ts @@ -17,6 +17,28 @@ import type { BenchmarkRow, EvalRow } from '@/lib/api'; const ACCURACY_KEY = 'em_strict'; +/** + * Recipe-comparison filter axes. Each row is tagged with one category so the + * UI can show e.g. only spec-decoding variants without mixing in batch-size + * sweeps. + */ +export type TechniqueCategory = + | 'baseline' + | 'spec-decoding' + | 'batch-size' + | 'kv-cache' + | 'prefix-cache' + | 'other'; + +export const TECHNIQUE_CATEGORIES: { value: TechniqueCategory | 'all'; label: string }[] = [ + { value: 'all', label: 'All' }, + { value: 'spec-decoding', label: 'Spec decoding' }, + { value: 'batch-size', label: 'Batch size' }, + { value: 'kv-cache', label: 'KV cache' }, + { value: 'prefix-cache', label: 'Prefix cache' }, + { value: 'baseline', label: 'Baseline only' }, +]; + export interface RecipeRow { groupKey: string; /** Identity (constant within a group). */ @@ -24,12 +46,21 @@ export interface RecipeRow { hardware: string; framework: string; precision: string; + /** Parallelism dims that distinguish 1× vs 2× topology, disagg, etc. */ + numPrefillGpu: number; + numDecodeGpu: number; + prefillTp: number; + decodeTp: number; + disagg: boolean; + /** Pretty-printed topology, e.g. "2× (TP=2)", "1P+1D (TP=2/2)". */ + topology: string; isl: number; osl: number; conc: number; /** Variant. */ techniques: Record; variantLabel: string; + category: TechniqueCategory; isBaseline: boolean; /** Measurements. */ tputPerGpu: number; @@ -48,9 +79,65 @@ export interface RecipeRow { } function groupKey( - r: Pick, + r: Pick< + BenchmarkRow, + | 'model' + | 'hardware' + | 'framework' + | 'precision' + | 'isl' + | 'osl' + | 'conc' + | 'num_prefill_gpu' + | 'num_decode_gpu' + | 'prefill_tp' + | 'decode_tp' + | 'disagg' + >, ): string { - return `${r.model}|${r.hardware}|${r.framework}|${r.precision}|${r.isl}|${r.osl}|${r.conc}`; + // Topology dims (num_*_gpu, *_tp, disagg) are part of the deployment + // context — a 1× H100 and a 2× H100 run are different deployments and must + // not collapse into the same row group. + return [ + r.model, + r.hardware, + r.framework, + r.precision, + r.isl, + r.osl, + r.conc, + r.num_prefill_gpu, + r.num_decode_gpu, + r.prefill_tp, + r.decode_tp, + r.disagg ? '1' : '0', + ].join('|'); +} + +function topologyLabel(r: { + num_prefill_gpu: number; + num_decode_gpu: number; + prefill_tp: number; + decode_tp: number; + disagg: boolean; +}): string { + if (r.disagg) { + return `${r.num_prefill_gpu}P+${r.num_decode_gpu}D (TP=${r.prefill_tp}/${r.decode_tp})`; + } + return `${r.num_prefill_gpu}× (TP=${r.prefill_tp})`; +} + +/** + * Tag a techniques jsonb with the category that best describes the knob being + * tuned. Used to drive the filter chip group on /evaluation. + */ +export function categorizeTechniques(t: Record): TechniqueCategory { + if (Object.keys(t).length === 0) return 'baseline'; + if (typeof t.spec_method === 'string' && t.spec_method !== 'none') return 'spec-decoding'; + if (t.max_num_batched_tokens !== undefined) return 'batch-size'; + if (t.kv_cache_dtype !== undefined) return 'kv-cache'; + if (t.prefix_cache !== undefined) return 'prefix-cache'; + return 'other'; } function accuracyKey(r: { @@ -130,18 +217,26 @@ export function buildRecipeRows(benchmarks: BenchmarkRow[], evals: EvalRow[]): R baseline && baseline.metrics?.median_tpot ? tpot / Number(baseline.metrics.median_tpot) : null; + const techniques = r.techniques ?? {}; out.push({ groupKey: k, model: r.model, hardware: r.hardware, framework: r.framework, precision: r.precision, + numPrefillGpu: r.num_prefill_gpu, + numDecodeGpu: r.num_decode_gpu, + prefillTp: r.prefill_tp, + decodeTp: r.decode_tp, + disagg: r.disagg, + topology: topologyLabel(r), isl: r.isl, osl: r.osl, conc: r.conc, - techniques: r.techniques ?? {}, - variantLabel: describeTechniques(r.techniques ?? {}), - isBaseline: isBaseline(r.techniques ?? {}), + techniques, + variantLabel: describeTechniques(techniques), + category: categorizeTechniques(techniques), + isBaseline: isBaseline(techniques), tputPerGpu: tput, medianTpot: tpot, medianIntvty: intvty,