diff --git a/packages/app/src/app/(dashboard)/evaluation/page.tsx b/packages/app/src/app/(dashboard)/evaluation/page.tsx index 649ce6202..72cef52bd 100644 --- a/packages/app/src/app/(dashboard)/evaluation/page.tsx +++ b/packages/app/src/app/(dashboard)/evaluation/page.tsx @@ -1,7 +1,6 @@ import type { Metadata } from 'next'; -import { EvaluationProvider } from '@/components/evaluation/EvaluationContext'; -import EvaluationChartDisplay from '@/components/evaluation/ui/ChartDisplay'; +import RecipeComparison from '@/components/evaluation/RecipeComparison'; import { NudgeEngine } from '@/components/nudge-engine'; import { tabMetadata } from '@/lib/tab-meta'; @@ -9,9 +8,9 @@ export const metadata: Metadata = tabMetadata('evaluation'); export default function EvaluationPage() { return ( - - + <> + - + ); } diff --git a/packages/app/src/components/evaluation/RecipeComparison.tsx b/packages/app/src/components/evaluation/RecipeComparison.tsx new file mode 100644 index 000000000..fd2202505 --- /dev/null +++ b/packages/app/src/components/evaluation/RecipeComparison.tsx @@ -0,0 +1,201 @@ +'use client'; + +import { ExternalLink } from 'lucide-react'; +import { useMemo } from 'react'; + +import { DataTable, type DataTableColumn } from '@/components/ui/data-table'; +import { useGlobalFilters } from '@/components/GlobalFilterContext'; +import { useBenchmarks } from '@/hooks/api/use-benchmarks'; +import { useEvaluations } from '@/hooks/api/use-evaluations'; + +import { buildRecipeRows, type RecipeRow } from './recipe-data'; + +function fmtNum(n: number | null, digits = 2, suffix = ''): string { + if (n === null || !Number.isFinite(n)) return '—'; + return `${n.toFixed(digits)}${suffix}`; +} + +function fmtPct(n: number | null): string { + if (n === null || !Number.isFinite(n)) return '—'; + return `${(n * 100).toFixed(1)}%`; +} + +function fmtSpeedup(n: number | null): string { + if (n === null || !Number.isFinite(n)) return '—'; + return `${n.toFixed(2)}×`; +} + +const COLUMNS: DataTableColumn[] = [ + { + header: 'HW', + align: 'left', + cell: (r) => r.hardware, + sortValue: (r) => r.hardware, + }, + { + header: 'Precision', + align: 'left', + cell: (r) => r.precision, + sortValue: (r) => r.precision, + }, + { + header: 'Seq', + align: 'left', + cell: (r) => `${r.isl / 1024}k/${r.osl / 1024}k`, + sortValue: (r) => `${r.isl}_${r.osl}`, + }, + { + header: 'Conc', + align: 'right', + cell: (r) => String(r.conc), + sortValue: (r) => r.conc, + }, + { + header: 'Variant', + align: 'left', + cell: (r) => ( + + {r.variantLabel} + + ), + sortValue: (r) => r.variantLabel, + }, + { + header: 'Speedup', + align: 'right', + cell: (r) => { + if (r.speedup === null) return —; + const better = r.speedup > 1.01; + const worse = r.speedup < 0.99; + const cls = better ? 'text-green-500' : worse ? 'text-red-500' : ''; + return {fmtSpeedup(r.speedup)}; + }, + sortValue: (r) => r.speedup ?? -Infinity, + }, + { + header: 'tput/GPU', + align: 'right', + cell: (r) => fmtNum(r.tputPerGpu, 0), + sortValue: (r) => r.tputPerGpu, + }, + { + header: 'TPOT (s)', + align: 'right', + cell: (r) => fmtNum(r.medianTpot, 4), + sortValue: (r) => r.medianTpot, + }, + { + header: 'Accept Rate', + align: 'right', + cell: (r) => fmtPct(r.acceptanceRate), + sortValue: (r) => r.acceptanceRate ?? -Infinity, + }, + { + header: 'Accuracy', + align: 'right', + cell: (r) => { + if (r.accuracy === null) return —; + const delta = r.accuracyDelta; + return ( + + {fmtPct(r.accuracy)} + {delta !== null && Math.abs(delta) >= 0.001 && ( + + {delta > 0 ? '+' : ''} + {(delta * 100).toFixed(2)}pp + + )} + + ); + }, + sortValue: (r) => r.accuracy ?? -Infinity, + }, + { + header: 'Run', + align: 'center', + cell: (r) => + r.runUrl ? ( + + + + ) : ( + — + ), + }, +]; + +export default function RecipeComparison() { + const { selectedModel, selectedRunDate } = useGlobalFilters(); + + const { + data: benchmarks, + isLoading: bmkLoading, + error: bmkError, + } = useBenchmarks(selectedModel, selectedRunDate || undefined); + const { data: evals, isLoading: evalLoading, error: evalError } = useEvaluations(); + + const rows = useMemo(() => { + if (!benchmarks || benchmarks.length === 0) return []; + const built = buildRecipeRows(benchmarks, evals ?? []); + // Sort: group rows together, baseline first within each group, then by speedup descending. + return built.toSorted((a, b) => { + if (a.groupKey !== b.groupKey) return a.groupKey.localeCompare(b.groupKey); + if (a.isBaseline !== b.isBaseline) return a.isBaseline ? -1 : 1; + return (b.speedup ?? 0) - (a.speedup ?? 0); + }); + }, [benchmarks, evals]); + + const loading = bmkLoading || evalLoading; + const error = bmkError ?? evalError; + + return ( +
+
+

Recipe Comparison

+

+ Compare runtime knobs (speculative decoding, MTP layers, …) on the same deployment. + Speedup is computed against the variant whose techniques is empty within the + same (model × hw × precision × isl/osl × conc) group. +

+
+ + {error && ( +
+ {error.message} +
+ )} + + {loading && ( +
Loading benchmarks and evaluations…
+ )} + + {!loading && rows.length === 0 && ( +
+ No data for {selectedModel || '(no model)'} + {selectedRunDate ? ( + <> + {' '} + on {selectedRunDate} + + ) : null} + . Pick a different model or date. +
+ )} + + {!loading && rows.length > 0 && ( + + )} +
+ ); +} diff --git a/packages/app/src/components/evaluation/recipe-data.test.ts b/packages/app/src/components/evaluation/recipe-data.test.ts new file mode 100644 index 000000000..8f4840ece --- /dev/null +++ b/packages/app/src/components/evaluation/recipe-data.test.ts @@ -0,0 +1,156 @@ +import { describe, expect, it } from 'vitest'; + +import type { BenchmarkRow, EvalRow } from '@/lib/api'; + +import { buildRecipeRows, describeTechniques } from './recipe-data'; + +function bmk(overrides: Partial): BenchmarkRow { + return { + hardware: 'h100', + framework: 'vllm', + model: 'gemma4', + precision: 'fp8', + spec_method: 'none', + techniques: {}, + disagg: false, + is_multinode: false, + prefill_tp: 2, + prefill_ep: 1, + prefill_dp_attention: false, + prefill_num_workers: 0, + decode_tp: 2, + decode_ep: 1, + decode_dp_attention: false, + decode_num_workers: 0, + num_prefill_gpu: 2, + num_decode_gpu: 2, + isl: 1024, + osl: 1024, + conc: 8, + image: null, + metrics: { tput_per_gpu: 100, median_tpot: 0.01 }, + date: '2026-05-25', + run_url: 'https://github.com/vngcloud/InferenceX/actions/runs/1', + ...overrides, + }; +} + +describe('describeTechniques', () => { + it('labels empty object as baseline', () => { + expect(describeTechniques({})).toBe('baseline'); + }); + + it('labels mtp + layer count compactly', () => { + expect(describeTechniques({ spec_method: 'mtp', mtp_layers: 4 })).toBe('MTP×4'); + expect(describeTechniques({ spec_method: 'mtp', mtp_layers: 6 })).toBe('MTP×6'); + }); + + it('falls back to capitalized spec_method', () => { + expect(describeTechniques({ spec_method: 'eagle' })).toBe('Eagle'); + }); +}); + +describe('buildRecipeRows', () => { + it('computes speedup vs baseline within a group', () => { + const rows = buildRecipeRows( + [ + bmk({ techniques: {}, metrics: { tput_per_gpu: 100, median_tpot: 0.01 } }), + bmk({ + techniques: { spec_method: 'mtp', mtp_layers: 4 }, + metrics: { tput_per_gpu: 150, median_tpot: 0.0067 }, + }), + ], + [], + ); + expect(rows).toHaveLength(2); + const baseline = rows.find((r) => r.isBaseline)!; + const mtp = rows.find((r) => !r.isBaseline)!; + expect(baseline.speedup).toBe(1); + expect(mtp.speedup).toBeCloseTo(1.5, 3); + expect(mtp.tpotRatio).toBeCloseTo(0.67, 2); + }); + + it('returns null speedup when no baseline exists in the group', () => { + // Two MTP variants, no spec_method=none baseline. + const rows = buildRecipeRows( + [ + bmk({ + techniques: { spec_method: 'mtp', mtp_layers: 4 }, + metrics: { tput_per_gpu: 120 }, + }), + bmk({ + techniques: { spec_method: 'mtp', mtp_layers: 6 }, + metrics: { tput_per_gpu: 140 }, + }), + ], + [], + ); + for (const r of rows) { + expect(r.isBaseline).toBe(false); + expect(r.speedup).toBeNull(); + expect(r.tpotRatio).toBeNull(); + } + }); + + it('groups by (model, hw, framework, precision, isl, osl, conc) and not by techniques', () => { + const rows = buildRecipeRows( + [ + bmk({ techniques: {} }), + bmk({ techniques: { spec_method: 'mtp', mtp_layers: 4 } }), + bmk({ techniques: { spec_method: 'mtp', mtp_layers: 6 } }), + // Different conc → different group, no shared baseline. + bmk({ techniques: { spec_method: 'mtp', mtp_layers: 4 }, conc: 16 }), + ], + [], + ); + expect(rows).toHaveLength(4); + const groupKeys = new Set(rows.map((r) => r.groupKey)); + expect(groupKeys.size).toBe(2); + }); + + it('joins accuracy from eval rows at the (model, hw, framework, precision) level', () => { + const ev: EvalRow = { + id: 1, + config_id: 1, + hardware: 'h100', + framework: 'vllm', + model: 'gemma4', + precision: 'fp8', + spec_method: 'none', + techniques: {}, + disagg: false, + is_multinode: false, + prefill_tp: 2, + prefill_ep: 1, + prefill_dp_attention: false, + prefill_num_workers: 0, + decode_tp: 2, + decode_ep: 1, + decode_dp_attention: false, + decode_num_workers: 0, + num_prefill_gpu: 2, + num_decode_gpu: 2, + task: 'gsm8k', + date: '2026-05-25', + conc: null, + metrics: { em_strict: 0.842 }, + timestamp: '2026-05-25T00:00:00Z', + run_url: null, + }; + const rows = buildRecipeRows([bmk({ techniques: {} })], [ev]); + expect(rows[0]?.accuracy).toBeCloseTo(0.842, 5); + }); + + it('reads acceptance rate from variant metrics', () => { + const rows = buildRecipeRows( + [ + bmk({ + techniques: { spec_method: 'mtp', mtp_layers: 4 }, + metrics: { tput_per_gpu: 150, median_acceptance_rate: 0.78 }, + }), + ], + [], + ); + expect(rows[0]?.acceptanceRate).toBeCloseTo(0.78, 5); + }); +}); diff --git a/packages/app/src/components/evaluation/recipe-data.ts b/packages/app/src/components/evaluation/recipe-data.ts new file mode 100644 index 000000000..056d93fd2 --- /dev/null +++ b/packages/app/src/components/evaluation/recipe-data.ts @@ -0,0 +1,154 @@ +/** + * Transform benchmark + eval rows into "recipe comparison" rows for the + * /evaluation table. + * + * Each row is one (deployment context, technique-variant) pair. Within a + * deployment context, the row whose `techniques` is empty (or whose + * spec_method='none') is treated as the baseline; sibling rows show speedup + * and TPOT-ratio deltas against it. When no baseline exists in the group, the + * deltas are null and the UI renders them as "—". + * + * Accuracy is joined from eval rows at the (model, hardware, framework, + * precision) level — coarser than benchmarks because lm-eval is run once per + * model+config, not per concurrency point. + */ + +import type { BenchmarkRow, EvalRow } from '@/lib/api'; + +const ACCURACY_KEY = 'em_strict'; + +export interface RecipeRow { + groupKey: string; + /** Identity (constant within a group). */ + model: string; + hardware: string; + framework: string; + precision: string; + isl: number; + osl: number; + conc: number; + /** Variant. */ + techniques: Record; + variantLabel: string; + isBaseline: boolean; + /** Measurements. */ + tputPerGpu: number; + medianTpot: number; + medianIntvty: number; + /** Deltas vs the baseline in the same group. null when no baseline exists. */ + speedup: number | null; + tpotRatio: number | null; + /** Joined eval. null when no matching eval row. */ + accuracy: number | null; + accuracyDelta: number | null; + /** Acceptance rate from this variant's metrics (spec-dec only). */ + acceptanceRate: number | null; + /** Drill-down to the benchmark run. */ + runUrl: string | null; +} + +function groupKey( + r: Pick, +): string { + return `${r.model}|${r.hardware}|${r.framework}|${r.precision}|${r.isl}|${r.osl}|${r.conc}`; +} + +function accuracyKey(r: { + model: string; + hardware: string; + framework: string; + precision: string; +}): string { + return `${r.model}|${r.hardware}|${r.framework}|${r.precision}`; +} + +/** + * Stable label for a techniques jsonb. Empty object → "baseline"; otherwise a + * short, human-readable summary. Insertion order is preserved from + * `parseTechniques()` so the label is deterministic. + */ +export function describeTechniques(t: Record): string { + const keys = Object.keys(t); + if (keys.length === 0) return 'baseline'; + const sm = t.spec_method; + const mtp = t.mtp_layers; + if (typeof sm === 'string' && sm === 'mtp' && typeof mtp === 'number') { + return `MTP×${mtp}`; + } + if (typeof sm === 'string' && sm !== 'none') { + return sm.charAt(0).toUpperCase() + sm.slice(1); + } + return keys.map((k) => `${k}=${t[k]}`).join(' · '); +} + +function isBaseline(t: Record): boolean { + if (Object.keys(t).length === 0) return true; + return t.spec_method === 'none'; +} + +export function buildRecipeRows(benchmarks: BenchmarkRow[], evals: EvalRow[]): RecipeRow[] { + // Index eval rows by accuracy key → latest em_strict. + const accuracyByKey = new Map(); + for (const e of evals) { + const v = e.metrics?.[ACCURACY_KEY]; + if (typeof v !== 'number') continue; + accuracyByKey.set(accuracyKey(e), v); + } + + // Bucket benchmark rows by deployment-context groupKey. + const groups = new Map(); + for (const b of benchmarks) { + const k = groupKey(b); + const existing = groups.get(k); + if (existing) existing.push(b); + else groups.set(k, [b]); + } + + const out: RecipeRow[] = []; + for (const [k, rows] of groups) { + const baseline = rows.find((r) => isBaseline(r.techniques)) ?? null; + const baselineAccuracy = baseline ? (accuracyByKey.get(accuracyKey(baseline)) ?? null) : null; + for (const r of rows) { + const tput = Number(r.metrics?.tput_per_gpu ?? 0); + const tpot = Number(r.metrics?.median_tpot ?? 0); + const intvty = Number(r.metrics?.median_intvty ?? 0); + const acc = accuracyByKey.get(accuracyKey(r)) ?? null; + const accept = + typeof r.metrics?.median_acceptance_rate === 'number' + ? r.metrics.median_acceptance_rate + : null; + const speedup = + baseline && baseline.metrics?.tput_per_gpu + ? tput / Number(baseline.metrics.tput_per_gpu) + : null; + const tpotRatio = + baseline && baseline.metrics?.median_tpot + ? tpot / Number(baseline.metrics.median_tpot) + : null; + out.push({ + groupKey: k, + model: r.model, + hardware: r.hardware, + framework: r.framework, + precision: r.precision, + isl: r.isl, + osl: r.osl, + conc: r.conc, + techniques: r.techniques ?? {}, + variantLabel: describeTechniques(r.techniques ?? {}), + isBaseline: isBaseline(r.techniques ?? {}), + tputPerGpu: tput, + medianTpot: tpot, + medianIntvty: intvty, + speedup, + tpotRatio, + accuracy: acc, + accuracyDelta: acc !== null && baselineAccuracy !== null ? acc - baselineAccuracy : null, + acceptanceRate: accept, + runUrl: r.run_url, + }); + } + } + + return out; +} diff --git a/packages/app/src/components/landing/landing-page.tsx b/packages/app/src/components/landing/landing-page.tsx index d07c3daff..18333d12d 100644 --- a/packages/app/src/components/landing/landing-page.tsx +++ b/packages/app/src/components/landing/landing-page.tsx @@ -24,9 +24,9 @@ const WORKFLOW: WorkflowEntry[] = [ }, { href: '/evaluation', - label: 'Accuracy Evals', + label: 'Recipe Compare', description: - 'Check accuracy of an FP8 / MTP / quantized variant against the baseline. gsm8k and friends, side-by-side with throughput.', + 'Compare runtime knobs (MTP layers, speculative decoding, kv-cache dtype, …) on the same deployment. Speedup, TPOT, acceptance rate, accuracy delta side-by-side.', }, { href: '/historical', diff --git a/packages/app/src/components/tab-nav.tsx b/packages/app/src/components/tab-nav.tsx index 52c43c61b..13e611156 100644 --- a/packages/app/src/components/tab-nav.tsx +++ b/packages/app/src/components/tab-nav.tsx @@ -72,7 +72,7 @@ function useFeatureGate(): boolean { const VISIBLE_TABS = [ { href: '/inference', label: 'Inference Performance', testId: 'tab-trigger-inference' }, - { href: '/evaluation', label: 'Accuracy Evals', testId: 'tab-trigger-evaluation' }, + { href: '/evaluation', label: 'Recipe Compare', testId: 'tab-trigger-evaluation' }, { href: '/historical', label: 'Historical Trends', testId: 'tab-trigger-historical' }, { href: '/calculator', label: 'TCO Calculator', testId: 'tab-trigger-calculator' }, { href: '/gpu-specs', label: 'GPU Specs', testId: 'tab-trigger-gpu-specs' }, diff --git a/packages/app/src/lib/tab-meta.ts b/packages/app/src/lib/tab-meta.ts index 9145532b2..b0834a339 100644 --- a/packages/app/src/lib/tab-meta.ts +++ b/packages/app/src/lib/tab-meta.ts @@ -31,9 +31,9 @@ export const TAB_META: Record = 'Compare AI inference latency, throughput, and time-to-first-token across GPUs and providers. Real benchmarks on NVIDIA GB200, H100, AMD MI355X, and more.', }, evaluation: { - title: 'LLM Evaluation Results', + title: 'Recipe Comparison', description: - 'LLM evaluation scores and accuracy benchmarks. Compare model quality across providers with standardized evaluation metrics.', + 'Compare runtime serving knobs (speculative decoding, MTP layer count, kv-cache dtype, …) on the same deployment. Speedup, TPOT, acceptance rate, accuracy delta side-by-side.', }, historical: { title: 'Historical Inference Trends',