From e176d0361e51b6d55e93cee27e300a6895fb2306 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Quang=20H=C3=B2a?= Date: Tue, 26 May 2026 11:04:27 +0700 Subject: [PATCH 1/3] feat(db): demote spec_method to per-measurement techniques jsonb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `configs` now describes the deployment (model, hw, framework, precision, parallelism). Per-run knobs (spec_method, mtp_layers, future kv_cache_dtype, chunked_prefill, prefix_cache, …) live in a new `techniques` jsonb on benchmark_results and eval_results, so we can absorb new dimensions without DDL or natural-key bloat. Motivated by the gemma4 MTP work: the benchmark side encoded the MTP layer count in the model prefix (gemma4n4 / gemma4n6), which never matched a model key and silently dropped 40 rows on yesterday's ingest. With techniques.mtp_layers as a measurement-level dimension, the model collapses to gemma4 and the layer count rides on the row. Schema (migration 006): - configs drops spec_method (column + lowercase check + natural key) - benchmark_results + eval_results gain `techniques jsonb default {}` - latest_benchmarks MV adds techniques to row identity to keep same-config-different-technique pairs distinct - Expression indexes on techniques->>'spec_method' and ->>'mtp_layers' - Full-rewrite truncate: re-ingest from artifacts after deploy ETL: - parseTechniques() reads new-shape `techniques` object, falls back to legacy `spec_decoding` field, infers mtp_layers from gemma4n4/n6 prefix aliases - benchmark-mapper / eval-mapper drop specMethod from ConfigParams, add techniques to BenchmarkParams / EvalParams - config-cache key is now 16-col (down from 17) - bulk insert / upsert paths write techniques; availability denormalizes spec_method via the same projection Queries: - All five query files SELECT COALESCE(br.techniques->>'spec_method', 'none') as spec_method + raw techniques jsonb. Downstream readers keep working unchanged because the projection name is preserved. Constants: add `gemma4` model key, register `*_acceptance_rate` and `*_tokens_per_step` metric keys for upcoming spec-dec measurements. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/constants/src/metric-keys.ts | 8 +++ packages/constants/src/models.ts | 1 + packages/db/migrations/006_skinny_configs.sql | 72 +++++++++++++++++++ packages/db/src/apply-overrides.ts | 10 ++- packages/db/src/etl/benchmark-ingest.ts | 24 +++++-- packages/db/src/etl/benchmark-mapper.ts | 9 ++- packages/db/src/etl/config-cache.ts | 11 ++- packages/db/src/etl/eval-ingest.ts | 8 ++- packages/db/src/etl/eval-mapper.ts | 23 +++--- packages/db/src/etl/normalizers.ts | 61 ++++++++++++++++ packages/db/src/ingest-ci-run.ts | 4 +- packages/db/src/ingest-gcs-backup.ts | 4 +- packages/db/src/ingest-supplemental.ts | 23 ++++-- packages/db/src/json-provider.ts | 20 ++++-- packages/db/src/queries/benchmarks.ts | 15 ++-- packages/db/src/queries/evaluations.ts | 5 +- packages/db/src/queries/latest-images.ts | 41 ++++++----- packages/db/src/queries/submissions.ts | 4 +- packages/db/src/queries/workflow-info.ts | 2 +- packages/db/src/verify-db.ts | 14 ++-- 20 files changed, 274 insertions(+), 85 deletions(-) create mode 100644 packages/db/migrations/006_skinny_configs.sql diff --git a/packages/constants/src/metric-keys.ts b/packages/constants/src/metric-keys.ts index cf2c4d0be..073c4045f 100644 --- a/packages/constants/src/metric-keys.ts +++ b/packages/constants/src/metric-keys.ts @@ -43,4 +43,12 @@ export const METRIC_KEYS = new Set([ 'p99_intvty', 'p99.9_intvty', 'std_intvty', + // speculative-decoding acceptance — fraction of draft tokens accepted [0,1] + 'median_acceptance_rate', + 'mean_acceptance_rate', + 'p99_acceptance_rate', + // speculative-decoding tokens per forward step — 1.0 when spec-dec is off + 'median_tokens_per_step', + 'mean_tokens_per_step', + 'p99_tokens_per_step', ]); diff --git a/packages/constants/src/models.ts b/packages/constants/src/models.ts index c75034c71..99d3f185d 100644 --- a/packages/constants/src/models.ts +++ b/packages/constants/src/models.ts @@ -18,6 +18,7 @@ export const DB_MODEL_TO_DISPLAY: Record = { glm5: 'GLM-5', 'glm5.1': 'GLM-5', dsv4: 'DeepSeek-V4-Pro', + gemma4: 'Gemma-4-31B-it', }; /** diff --git a/packages/db/migrations/006_skinny_configs.sql b/packages/db/migrations/006_skinny_configs.sql new file mode 100644 index 000000000..a74c15fa2 --- /dev/null +++ b/packages/db/migrations/006_skinny_configs.sql @@ -0,0 +1,72 @@ +-- ============================================================ +-- Migration 006 — skinny configs, techniques per measurement +-- ============================================================ +-- +-- Demotes spec_method from a config dimension to a measurement-level +-- "technique flag" stored in a jsonb bag. Same shape will absorb any future +-- runtime knob (mtp_layers, kv_cache_dtype, chunked_prefill, prefix_cache, …) +-- without further DDL — each lands as a key in techniques. +-- +-- configs now describes the DEPLOYMENT (model, hw, framework, precision, +-- parallelism). techniques describes HOW THE RECIPE WAS TUNED for that run. +-- +-- Full-rewrite migration: data tables are TRUNCATEd and re-ingested from +-- artifacts. Safe because ingest is idempotent and (for vngcloud fork) very +-- little production data exists. + +-- 1. Drop spec_method from configs +alter table configs drop constraint configs_natural_key; +alter table configs drop constraint configs_spec_method_lowercase; +alter table configs drop column spec_method; +alter table configs add constraint configs_natural_key unique ( + hardware, framework, model, precision, + disagg, is_multinode, + prefill_tp, prefill_ep, prefill_dp_attention, prefill_num_workers, + decode_tp, decode_ep, decode_dp_attention, decode_num_workers, + num_prefill_gpu, num_decode_gpu +); + +-- 2. Add techniques jsonb on the two measurement tables +alter table benchmark_results add column techniques jsonb not null default '{}'::jsonb; +alter table eval_results add column techniques jsonb not null default '{}'::jsonb; + +-- 3. Expression indexes on the two hot technique keys +create index benchmark_results_spec_method_idx + on benchmark_results ((techniques->>'spec_method')) where error is null; +create index benchmark_results_mtp_layers_idx + on benchmark_results ((techniques->>'mtp_layers')) where error is null; + +-- 4. Rebuild latest_benchmarks MV with techniques as part of row identity. +-- Without this, two formerly-different configs (same hw/fw/precision but +-- different spec_method) now share one config_id and would collide on the +-- MV's unique PK at (config_id, conc, isl, osl). +drop materialized view latest_benchmarks; + +create materialized view latest_benchmarks as +select distinct on (br.config_id, br.conc, br.isl, br.osl, br.techniques) + br.* +from benchmark_results br +join latest_workflow_runs wr on wr.id = br.workflow_run_id +where br.error is null +order by br.config_id, br.conc, br.isl, br.osl, br.techniques, + br.date desc, wr.run_started_at desc nulls last; + +create unique index latest_benchmarks_pk + on latest_benchmarks (config_id, conc, isl, osl, techniques); +create index latest_benchmarks_model_idx on latest_benchmarks (config_id); + +-- 5. Rebuild the seq-history covering index to include techniques so +-- historical-trends queries can still get an index-only scan when grouping +-- by technique. +drop index if exists benchmark_results_seq_history_idx; +create index benchmark_results_seq_history_idx + on benchmark_results (isl, osl, date, config_id, conc, techniques) + include (image, metrics) + where error is null; + +-- 6. Full-rewrite TRUNCATE. Re-ingest from artifacts after deploy. +-- configs is also truncated because its natural key changed shape. +truncate benchmark_results, eval_results, eval_samples, run_stats, + availability, changelog_entries, server_logs, workflow_runs + restart identity cascade; +truncate configs restart identity cascade; diff --git a/packages/db/src/apply-overrides.ts b/packages/db/src/apply-overrides.ts index e84078ff2..4259354ec 100644 --- a/packages/db/src/apply-overrides.ts +++ b/packages/db/src/apply-overrides.ts @@ -150,10 +150,13 @@ async function purge(wrIds: number[]): Promise { await sql.begin(async (_tx) => { const tx = _tx as unknown as Sql; - // Capture availability keys + server_log_ids before deleting benchmarks + // Capture availability keys + server_log_ids before deleting benchmarks. + // spec_method is now a measurement-level technique flag (see migration 006). const availKeys = await tx` SELECT DISTINCT c.model, br.isl, br.osl, c.precision, c.hardware, - c.framework, c.spec_method, c.disagg, br.date::text AS date + c.framework, + COALESCE(br.techniques->>'spec_method', 'none') AS spec_method, + c.disagg, br.date::text AS date FROM benchmark_results br JOIN configs c ON c.id = br.config_id WHERE br.workflow_run_id = ANY(${wrIds}) @@ -210,7 +213,8 @@ async function purge(wrIds: number[]): Promise { JOIN configs c ON c.id = br.config_id WHERE c.model = a.model AND br.isl = a.isl AND br.osl = a.osl AND c.precision = a.precision AND c.hardware = a.hardware - AND c.framework = a.framework AND c.spec_method = a.spec_method + AND c.framework = a.framework + AND COALESCE(br.techniques->>'spec_method', 'none') = a.spec_method AND c.disagg = a.disagg AND br.date = a.date AND br.error IS NULL ) `; diff --git a/packages/db/src/etl/benchmark-ingest.ts b/packages/db/src/etl/benchmark-ingest.ts index 67173c648..f61fc1a8e 100644 --- a/packages/db/src/etl/benchmark-ingest.ts +++ b/packages/db/src/etl/benchmark-ingest.ts @@ -40,11 +40,12 @@ export async function bulkIngestBenchmarkRows( const concs = deduped.map((r) => r.conc); const images = deduped.map((r) => r.image); const metricsJsons = deduped.map((r) => JSON.stringify(r.metrics)); + const techniquesJsons = deduped.map((r) => JSON.stringify(r.techniques)); const result = await sql<{ inserted: boolean; id: number }[]>` insert into benchmark_results ( workflow_run_id, config_id, benchmark_type, date, - isl, osl, conc, image, metrics + isl, osl, conc, image, metrics, techniques ) select ${workflowRunId}, @@ -55,11 +56,13 @@ export async function bulkIngestBenchmarkRows( unnest(${sql.array(osls)}::int[]), unnest(${sql.array(concs)}::int[]), unnest(${sql.array(images)}), - unnest(${sql.array(metricsJsons)}::jsonb[]) + unnest(${sql.array(metricsJsons)}::jsonb[]), + unnest(${sql.array(techniquesJsons)}::jsonb[]) on conflict (workflow_run_id, config_id, benchmark_type, isl, osl, conc) do update set metrics = excluded.metrics, - image = excluded.image + image = excluded.image, + techniques = excluded.techniques returning (xmax = 0) as inserted, id `; @@ -143,6 +146,14 @@ export async function bulkIngestRunStats( * Rows are deduplicated within the batch before sending. ON CONFLICT DO NOTHING * makes re-runs idempotent. */ +/** + * availability.spec_method is a denormalized projection of techniques.spec_method + * (kept for filter ergonomics on the date picker). Default to 'none' when absent. + */ +function specMethodFor(t: Record): string { + return typeof t.spec_method === 'string' ? t.spec_method : 'none'; +} + export async function bulkUpsertAvailability( sql: Sql, rows: { @@ -152,7 +163,7 @@ export async function bulkUpsertAvailability( precision: string; hardware: string; framework: string; - specMethod: string; + techniques: Record; disagg: boolean; }[], date: string, @@ -162,7 +173,8 @@ export async function bulkUpsertAvailability( const seen = new Set(); const unique: typeof rows = []; for (const r of rows) { - const key = `${r.model}|${r.isl}|${r.osl}|${r.precision}|${r.hardware}|${r.framework}|${r.specMethod}|${r.disagg}|${date}`; + const sm = specMethodFor(r.techniques); + const key = `${r.model}|${r.isl}|${r.osl}|${r.precision}|${r.hardware}|${r.framework}|${sm}|${r.disagg}|${date}`; if (!seen.has(key)) { seen.add(key); unique.push(r); @@ -178,7 +190,7 @@ export async function bulkUpsertAvailability( unnest(${sql.array(unique.map((r) => r.precision))}::text[]), unnest(${sql.array(unique.map((r) => r.hardware))}::text[]), unnest(${sql.array(unique.map((r) => r.framework))}::text[]), - unnest(${sql.array(unique.map((r) => r.specMethod))}::text[]), + unnest(${sql.array(unique.map((r) => specMethodFor(r.techniques)))}::text[]), unnest(${sql.array(unique.map((r) => r.disagg))}::bool[]), ${date}::date on conflict do nothing diff --git a/packages/db/src/etl/benchmark-mapper.ts b/packages/db/src/etl/benchmark-mapper.ts index 7d78e1755..d95ff220a 100644 --- a/packages/db/src/etl/benchmark-mapper.ts +++ b/packages/db/src/etl/benchmark-mapper.ts @@ -11,7 +11,7 @@ import { hwToGpuKey, normalizeFramework, normalizePrecision, - normalizeSpecMethod, + parseTechniques, parseBool, parseNum, parseInt2, @@ -41,7 +41,9 @@ const NON_METRIC_KEYS = new Set([ 'image', 'disagg', 'is_multinode', + // techniques (legacy + new-shape) — extracted into BenchmarkParams.techniques 'spec_decoding', + 'techniques', // v1 parallelism 'tp', 'ep', @@ -75,6 +77,7 @@ export interface BenchmarkParams { conc: number; image: string | null; metrics: Record; + techniques: Record; } /** @@ -128,7 +131,7 @@ export function mapBenchmarkRow( if (!PRECISION_KEYS.has(precision)) { tracker.unmappedPrecisions.add(precision); } - const specMethod = normalizeSpecMethod(row.spec_decoding); + const techniques = parseTechniques(row); let prefillTp: number, prefillEp: number, prefillDpAttn: boolean, prefillNumWorkers: number; let decodeTp: number, decodeEp: number, decodeDpAttn: boolean, decodeNumWorkers: number; @@ -191,7 +194,6 @@ export function mapBenchmarkRow( framework, model: modelKey, precision, - specMethod, disagg, isMultinode, prefillTp, @@ -210,5 +212,6 @@ export function mapBenchmarkRow( conc, image, metrics, + techniques, }; } diff --git a/packages/db/src/etl/config-cache.ts b/packages/db/src/etl/config-cache.ts index 4986ae350..9ec4b1034 100644 --- a/packages/db/src/etl/config-cache.ts +++ b/packages/db/src/etl/config-cache.ts @@ -14,7 +14,6 @@ export interface ConfigParams { framework: string; model: string; precision: string; - specMethod: string; disagg: boolean; isMultinode: boolean; prefillTp: number; @@ -43,7 +42,6 @@ export function configCacheKey(p: ConfigParams): string { p.framework, p.model, p.precision, - p.specMethod, p.disagg, p.isMultinode, p.prefillTp, @@ -85,20 +83,20 @@ export function createConfigCache(sql: Sql) { const [row] = await sql` insert into configs ( - hardware, framework, model, precision, spec_method, + hardware, framework, model, precision, disagg, is_multinode, prefill_tp, prefill_ep, prefill_dp_attention, prefill_num_workers, decode_tp, decode_ep, decode_dp_attention, decode_num_workers, num_prefill_gpu, num_decode_gpu ) values ( - ${p.hardware}, ${p.framework}, ${p.model}, ${p.precision}, ${p.specMethod}, + ${p.hardware}, ${p.framework}, ${p.model}, ${p.precision}, ${p.disagg}, ${p.isMultinode}, ${p.prefillTp}, ${p.prefillEp}, ${p.prefillDpAttn}, ${p.prefillNumWorkers}, ${p.decodeTp}, ${p.decodeEp}, ${p.decodeDpAttn}, ${p.decodeNumWorkers}, ${p.numPrefillGpu}, ${p.numDecodeGpu} ) on conflict ( - hardware, framework, model, precision, spec_method, + hardware, framework, model, precision, disagg, is_multinode, prefill_tp, prefill_ep, prefill_dp_attention, prefill_num_workers, decode_tp, decode_ep, decode_dp_attention, decode_num_workers, @@ -119,7 +117,7 @@ export function createConfigCache(sql: Sql) { */ async function preloadConfigs(): Promise { const rows = await sql` - select id, hardware, framework, model, precision, spec_method, + select id, hardware, framework, model, precision, disagg, is_multinode, prefill_tp, prefill_ep, prefill_dp_attention, prefill_num_workers, decode_tp, decode_ep, decode_dp_attention, decode_num_workers, @@ -132,7 +130,6 @@ export function createConfigCache(sql: Sql) { framework: r.framework, model: r.model, precision: r.precision, - specMethod: r.spec_method, disagg: r.disagg, isMultinode: r.is_multinode, prefillTp: r.prefill_tp, diff --git a/packages/db/src/etl/eval-ingest.ts b/packages/db/src/etl/eval-ingest.ts index 475b5b753..c01715014 100644 --- a/packages/db/src/etl/eval-ingest.ts +++ b/packages/db/src/etl/eval-ingest.ts @@ -32,14 +32,16 @@ export async function ingestEvalRow( const [row] = await sql<{ inserted: boolean; id: number }[]>` insert into eval_results ( workflow_run_id, config_id, task, date, - isl, osl, conc, lm_eval_version, metrics + isl, osl, conc, lm_eval_version, metrics, techniques ) values ( ${workflowRunId}, ${configId}, ${p.task}, ${date}, ${p.isl}, ${p.osl}, ${p.conc}, ${p.lmEvalVersion}, - ${sql.json(p.metrics)} + ${sql.json(p.metrics)}, ${sql.json(p.techniques)} ) on conflict (workflow_run_id, config_id, task, isl, osl, conc) - do update set metrics = excluded.metrics + do update set + metrics = excluded.metrics, + techniques = excluded.techniques returning id, (xmax = 0) as inserted `; return { outcome: row.inserted ? 'new' : 'dup', id: row.id }; diff --git a/packages/db/src/etl/eval-mapper.ts b/packages/db/src/etl/eval-mapper.ts index 5ef580159..9c4592d46 100644 --- a/packages/db/src/etl/eval-mapper.ts +++ b/packages/db/src/etl/eval-mapper.ts @@ -13,7 +13,7 @@ import { hwToGpuKey, normalizeFramework, normalizePrecision, - normalizeSpecMethod, + parseTechniques, parseBool, parseNum, parseInt2, @@ -39,6 +39,7 @@ export interface EvalParams { conc: number | null; lmEvalVersion: string | null; metrics: Record; + techniques: Record; } /** @@ -91,17 +92,9 @@ export function mapEvalRow( if (!PRECISION_KEYS.has(precision)) { tracker.unmappedPrecisions.add(precision); } - const specMethod = normalizeSpecMethod(meta.spec_decoding); + const techniques = parseTechniques(meta); const lmEvalVersion = results.lm_eval_version ? String(results.lm_eval_version) : null; - const config = buildEvalConfig( - meta, - gpuKey, - framework, - modelKey, - precision, - specMethod, - disaggFromFw, - ); + const config = buildEvalConfig(meta, gpuKey, framework, modelKey, precision, disaggFromFw); const nSamples = results['n-samples'] as Record | undefined; @@ -127,6 +120,7 @@ export function mapEvalRow( conc: parseInt2(meta.conc) ?? null, lmEvalVersion, metrics, + techniques, }; }); } @@ -174,7 +168,7 @@ export function mapAggEvalRow(row: Record, tracker: SkipTracker): E if (!PRECISION_KEYS.has(precision)) { tracker.unmappedPrecisions.add(precision); } - const specMethod = normalizeSpecMethod(row.spec_decoding); + const techniques = parseTechniques(row); // ISL/OSL is encoded in the source path (e.g. "eval_dsr1_1k8k_.../results_..."). const islOsl = row.source ? parseIslOsl(String(row.source)) : null; @@ -193,13 +187,14 @@ export function mapAggEvalRow(row: Record, tracker: SkipTracker): E add('score_se', row.score_se); return { - config: buildEvalConfig(row, gpuKey, framework, modelKey, precision, specMethod, disaggFromFw), + config: buildEvalConfig(row, gpuKey, framework, modelKey, precision, disaggFromFw), task, isl: islOsl?.isl ?? null, osl: islOsl?.osl ?? null, conc: parseInt2(row.conc) ?? null, lmEvalVersion: null, metrics, + techniques, }; } @@ -225,7 +220,6 @@ function buildEvalConfig( framework: string, model: string, precision: string, - specMethod: string, disaggFromFw: boolean, ): ConfigParams { const isMultinode = parseBool(src.is_multinode); @@ -270,7 +264,6 @@ function buildEvalConfig( framework, model, precision, - specMethod, disagg, isMultinode, prefillTp, diff --git a/packages/db/src/etl/normalizers.ts b/packages/db/src/etl/normalizers.ts index d42429c9d..648c468a2 100644 --- a/packages/db/src/etl/normalizers.ts +++ b/packages/db/src/etl/normalizers.ts @@ -41,6 +41,16 @@ const PRECISION_SUFFIX = /-(?:fp4|fp8|mxfp4|nvfp4)(?:-.*)?$/iu; const PREFIX_ALIASES: Record = { gptoss: 'gptoss120b', dsv4pro: 'dsv4', + // Gemma-4 MTP variants encode layer count in the prefix on the benchmark side. + // Layer count itself lands in techniques.mtp_layers (see parseTechniques). + gemma4n4: 'gemma4', + gemma4n6: 'gemma4', +}; + +/** Prefix → mtp_layers count, when the variant smuggles it into the model prefix. */ +const PREFIX_TO_MTP_LAYERS: Record = { + gemma4n4: 4, + gemma4n6: 6, }; function resolvePrefixToKey(prefix: string): string | null { @@ -172,6 +182,57 @@ export function normalizeSpecMethod(spec: any): string { return String(spec).toLowerCase(); } +/** Known technique keys we promote into the techniques jsonb explicitly. */ +const TECHNIQUE_KEYS = ['spec_method', 'mtp_layers', 'kv_cache_dtype', 'prefix_cache'] as const; + +/** + * Build the per-measurement `techniques` jsonb from a raw artifact row. + * + * New-shape artifacts emit a top-level `techniques` object directly: + * { "techniques": { "spec_method": "mtp", "mtp_layers": 4 } } + * + * Legacy-shape artifacts emit a top-level `spec_decoding` field and encode + * the MTP layer count in the model prefix (`gemma4n4`, `gemma4n6`); both are + * coerced into the new shape here so the rest of the pipeline only sees one + * format. Empty/absent values are dropped (no `spec_method: 'none'` keys). + * + * @param row - Raw artifact dict. + * @returns Techniques jsonb. May be empty (`{}`) when no knobs were tuned. + */ +export function parseTechniques(row: Record): Record { + const out: Record = {}; + + // New-shape: top-level techniques object. + if (row.techniques && typeof row.techniques === 'object') { + for (const k of TECHNIQUE_KEYS) { + const v = (row.techniques as Record)[k]; + if (v === undefined || v === null || v === '') continue; + out[k] = typeof v === 'number' ? v : String(v).toLowerCase(); + } + } + + // Legacy fallback: spec_decoding at top level → techniques.spec_method. + if (!('spec_method' in out)) { + const legacy = row.spec_decoding; + if (legacy && legacy !== '' && String(legacy).toLowerCase() !== 'none') { + out.spec_method = String(legacy).toLowerCase(); + } + } + + // Legacy fallback: MTP layer count smuggled into the model prefix. + if (!('mtp_layers' in out)) { + const rawPrefix = String(row.infmax_model_prefix ?? row.model_prefix ?? '').toLowerCase(); + const layers = PREFIX_TO_MTP_LAYERS[rawPrefix]; + if (layers !== undefined) { + out.mtp_layers = layers; + // If spec_method wasn't set elsewhere but we have MTP layers, infer it. + if (!('spec_method' in out)) out.spec_method = 'mtp'; + } + } + + return out; +} + /** * Coerce a loosely-typed value to a boolean. * Accepts JS `true`, the string `'true'`, and the Python-style string `'True'`. diff --git a/packages/db/src/ingest-ci-run.ts b/packages/db/src/ingest-ci-run.ts index c345e6624..e9c003fdb 100644 --- a/packages/db/src/ingest-ci-run.ts +++ b/packages/db/src/ingest-ci-run.ts @@ -299,7 +299,7 @@ async function main(): Promise { precision: string; hardware: string; framework: string; - specMethod: string; + techniques: Record; disagg: boolean; }[] = []; @@ -413,7 +413,7 @@ async function main(): Promise { precision: r.config.precision, hardware: r.config.hardware, framework: r.config.framework, - specMethod: r.config.specMethod, + techniques: r.techniques, disagg: r.config.disagg, }); } diff --git a/packages/db/src/ingest-gcs-backup.ts b/packages/db/src/ingest-gcs-backup.ts index 9c17bfafd..1934181bc 100644 --- a/packages/db/src/ingest-gcs-backup.ts +++ b/packages/db/src/ingest-gcs-backup.ts @@ -626,7 +626,7 @@ async function main(): Promise { precision: string; hardware: string; framework: string; - specMethod: string; + techniques: Record; disagg: boolean; }[] = []; for (const r of allInserted) { @@ -637,7 +637,7 @@ async function main(): Promise { precision: r.config.precision, hardware: r.config.hardware, framework: r.config.framework, - specMethod: r.config.specMethod, + techniques: r.techniques, disagg: r.config.disagg, }); } diff --git a/packages/db/src/ingest-supplemental.ts b/packages/db/src/ingest-supplemental.ts index a3b62fe03..9ac5b5d4e 100644 --- a/packages/db/src/ingest-supplemental.ts +++ b/packages/db/src/ingest-supplemental.ts @@ -22,6 +22,15 @@ import { normalizeSpecMethod, parseBool, } from './etl/normalizers'; + +/** + * Build a techniques jsonb from the legacy supplemental-data `spec_decoding` + * field. Returns an empty object when the value is absent/none. + */ +function techniquesFromSpec(spec: string): Record { + const sm = normalizeSpecMethod(spec); + return sm === 'none' ? {} : { spec_method: sm }; +} import { bulkIngestBenchmarkRows, bulkUpsertAvailability } from './etl/benchmark-ingest'; import { ingestEvalRow } from './etl/eval-ingest'; @@ -99,7 +108,7 @@ async function ingestSupplementalEvals( continue; } const { framework, disagg } = normalizeFramework(entry.framework, false); - const specMethod = normalizeSpecMethod(entry.spec_decoding); + const techniques = techniquesFromSpec(entry.spec_decoding); const dpAttn = parseBool(entry.dp_attention); try { @@ -108,7 +117,6 @@ async function ingestSupplementalEvals( framework, model: modelKey, precision: entry.precision, - specMethod, disagg, isMultinode: false, prefillTp: entry.tp, @@ -142,6 +150,7 @@ async function ingestSupplementalEvals( score: entry.score, score_se: entry.score_se, }, + techniques, }, workflowRunId, date, @@ -224,6 +233,7 @@ async function ingestSupplementalBmk( conc: number; image: string | null; metrics: Record; + techniques: Record; }[] = []; for (const entry of entries) { @@ -245,7 +255,7 @@ async function ingestSupplementalBmk( entry.framework, entry.disagg ?? false, ); - const specMethod = normalizeSpecMethod(entry.spec_decoding); + const techniques = techniquesFromSpec(entry.spec_decoding); const dpAttn = parseBool(entry.dp_attention); const disagg = entry.disagg ?? frameworkDisagg; @@ -254,7 +264,6 @@ async function ingestSupplementalBmk( framework, model: modelKey, precision: entry.precision, - specMethod, disagg, isMultinode: entry.is_multinode ?? false, prefillTp: entry.tp, @@ -276,6 +285,7 @@ async function ingestSupplementalBmk( conc: entry.conc, image: entry.image, metrics: entry.metrics, + techniques, }); } @@ -299,7 +309,7 @@ async function ingestSupplementalBmk( precision: string; hardware: string; framework: string; - specMethod: string; + techniques: Record; disagg: boolean; }[] = []; for (const entry of entries) { @@ -307,7 +317,6 @@ async function ingestSupplementalBmk( const hw = hwToGpuKey(entry.hw); if (!modelKey || !hw) continue; const { framework, disagg } = normalizeFramework(entry.framework, entry.disagg ?? false); - const specMethod = normalizeSpecMethod(entry.spec_decoding); availRows.push({ model: modelKey, isl: entry.isl, @@ -315,7 +324,7 @@ async function ingestSupplementalBmk( precision: entry.precision, hardware: hw, framework, - specMethod, + techniques: techniquesFromSpec(entry.spec_decoding), disagg, }); } diff --git a/packages/db/src/json-provider.ts b/packages/db/src/json-provider.ts index 25525e04f..1030fcfee 100644 --- a/packages/db/src/json-provider.ts +++ b/packages/db/src/json-provider.ts @@ -31,7 +31,6 @@ interface RawConfig { framework: string; model: string; precision: string; - spec_method: string; disagg: boolean; is_multinode: boolean; prefill_tp: number; @@ -72,6 +71,7 @@ interface RawBenchmarkResult { conc: number; image: string | null; metrics: Record; + techniques: Record; error: string | null; server_log_id: number | null; } @@ -96,6 +96,7 @@ interface RawEvalResult { conc: number | null; lm_eval_version: string | null; metrics: Record; + techniques: Record; } interface RawChangelogEntry { @@ -272,12 +273,15 @@ function toBenchmarkRow( wr: RawWorkflowRun, metrics?: Record, ): BenchmarkRow { + const techniques = br.techniques ?? {}; + const specMethod = typeof techniques.spec_method === 'string' ? techniques.spec_method : 'none'; return { hardware: c.hardware, framework: c.framework, model: c.model, precision: c.precision, - spec_method: c.spec_method, + spec_method: specMethod, + techniques, disagg: c.disagg, is_multinode: c.is_multinode, prefill_tp: c.prefill_tp, @@ -446,6 +450,9 @@ export function getAllEvalResults(): EvalRow[] { const wr = s.latestRunsById.get(er.workflow_run_id); if (!wr) continue; + const techniques = er.techniques ?? {}; + const specMethod = typeof techniques.spec_method === 'string' ? techniques.spec_method : 'none'; + rows.push({ id: er.id, config_id: er.config_id, @@ -453,7 +460,8 @@ export function getAllEvalResults(): EvalRow[] { framework: c.framework, model: c.model, precision: c.precision, - spec_method: c.spec_method, + spec_method: specMethod, + techniques, disagg: c.disagg, is_multinode: c.is_multinode, prefill_tp: c.prefill_tp, @@ -548,7 +556,9 @@ export function getDateConfigs(date: string): DateConfigRow[] { const c = s.configs.get(br.config_id); if (!c) continue; - const key = `${c.model}|${br.isl}|${br.osl}|${c.precision}|${c.hardware}|${c.framework}|${c.spec_method}|${c.disagg}`; + const techniques = br.techniques ?? {}; + const specMethod = typeof techniques.spec_method === 'string' ? techniques.spec_method : 'none'; + const key = `${c.model}|${br.isl}|${br.osl}|${c.precision}|${c.hardware}|${c.framework}|${specMethod}|${c.disagg}`; if (seen.has(key)) continue; seen.add(key); @@ -559,7 +569,7 @@ export function getDateConfigs(date: string): DateConfigRow[] { precision: c.precision, hardware: c.hardware, framework: c.framework, - spec_method: c.spec_method, + spec_method: specMethod, disagg: c.disagg, }); } diff --git a/packages/db/src/queries/benchmarks.ts b/packages/db/src/queries/benchmarks.ts index 1c30b1fd6..541c79eff 100644 --- a/packages/db/src/queries/benchmarks.ts +++ b/packages/db/src/queries/benchmarks.ts @@ -5,7 +5,9 @@ export interface BenchmarkRow { framework: string; model: string; precision: string; + /** Derived from techniques.spec_method via SQL COALESCE; 'none' when absent. */ spec_method: string; + techniques: Record; disagg: boolean; is_multinode: boolean; prefill_tp: number; @@ -50,12 +52,13 @@ export async function getLatestBenchmarks( // exact=false (default): return latest data as of this date (for main chart) const dateFilter = exact ? sql`br.date = ${date}::date` : sql`br.date <= ${date}::date`; const rows = await sql` - SELECT DISTINCT ON (br.config_id, br.conc, br.isl, br.osl) + SELECT DISTINCT ON (br.config_id, br.conc, br.isl, br.osl, br.techniques) c.hardware, c.framework, c.model, c.precision, - c.spec_method, + COALESCE(br.techniques->>'spec_method', 'none') AS spec_method, + br.techniques, c.disagg, c.is_multinode, c.prefill_tp, @@ -81,7 +84,7 @@ export async function getLatestBenchmarks( WHERE c.model = ANY(${modelKeys}) AND br.error IS NULL AND ${dateFilter} - ORDER BY br.config_id, br.conc, br.isl, br.osl, br.date DESC + ORDER BY br.config_id, br.conc, br.isl, br.osl, br.techniques, br.date DESC `; return rows as unknown as BenchmarkRow[]; } @@ -93,7 +96,8 @@ export async function getLatestBenchmarks( c.framework, c.model, c.precision, - c.spec_method, + COALESCE(lb.techniques->>'spec_method', 'none') AS spec_method, + lb.techniques, c.disagg, c.is_multinode, c.prefill_tp, @@ -140,7 +144,8 @@ export async function getAllBenchmarksForHistory( c.framework, c.model, c.precision, - c.spec_method, + COALESCE(br.techniques->>'spec_method', 'none') AS spec_method, + br.techniques, c.disagg, c.is_multinode, c.prefill_tp, diff --git a/packages/db/src/queries/evaluations.ts b/packages/db/src/queries/evaluations.ts index 15304569c..220853dc0 100644 --- a/packages/db/src/queries/evaluations.ts +++ b/packages/db/src/queries/evaluations.ts @@ -7,7 +7,9 @@ export interface EvalRow { framework: string; model: string; precision: string; + /** Derived from techniques.spec_method via SQL COALESCE; 'none' when absent. */ spec_method: string; + techniques: Record; disagg: boolean; is_multinode: boolean; prefill_tp: number; @@ -38,7 +40,8 @@ export async function getAllEvalResults(sql: DbClient): Promise { c.framework, c.model, c.precision, - c.spec_method, + COALESCE(er.techniques->>'spec_method', 'none') AS spec_method, + er.techniques, c.disagg, c.is_multinode, c.prefill_tp, diff --git a/packages/db/src/queries/latest-images.ts b/packages/db/src/queries/latest-images.ts index 9200d3e23..7a8694d09 100644 --- a/packages/db/src/queries/latest-images.ts +++ b/packages/db/src/queries/latest-images.ts @@ -5,6 +5,7 @@ export interface LatestImageRow { hardware: string; framework: string; precision: string; + /** Derived from techniques.spec_method via SQL COALESCE; 'none' when absent. */ spec_method: string; isl: number; osl: number; @@ -13,25 +14,33 @@ export interface LatestImageRow { } /** - * Fetch the latest non-null image tag per unique (model, hardware, framework, precision, spec_method, isl, osl). - * Uses the latest_benchmarks materialized view for fast lookups. + * Fetch the latest non-null image tag per unique + * (model, hardware, framework, precision, spec_method, isl, osl). + * Uses the latest_benchmarks materialized view for fast lookups. spec_method is + * projected from techniques->>'spec_method' since it was demoted from configs in + * migration 006. */ export async function getLatestImages(sql: DbClient): Promise { const rows = await sql` - SELECT DISTINCT ON (c.model, c.hardware, c.framework, c.precision, c.spec_method, lb.isl, lb.osl) - c.model, - c.hardware, - c.framework, - c.precision, - c.spec_method, - lb.isl, - lb.osl, - lb.image, - lb.date::text - FROM latest_benchmarks lb - JOIN configs c ON c.id = lb.config_id - WHERE lb.image IS NOT NULL - ORDER BY c.model, c.hardware, c.framework, c.precision, c.spec_method, lb.isl, lb.osl, lb.date DESC + WITH latest AS ( + SELECT + c.model, + c.hardware, + c.framework, + c.precision, + COALESCE(lb.techniques->>'spec_method', 'none') AS spec_method, + lb.isl, + lb.osl, + lb.image, + lb.date::text AS date + FROM latest_benchmarks lb + JOIN configs c ON c.id = lb.config_id + WHERE lb.image IS NOT NULL + ) + SELECT DISTINCT ON (model, hardware, framework, precision, spec_method, isl, osl) + model, hardware, framework, precision, spec_method, isl, osl, image, date + FROM latest + ORDER BY model, hardware, framework, precision, spec_method, isl, osl, date DESC `; return rows as unknown as LatestImageRow[]; } diff --git a/packages/db/src/queries/submissions.ts b/packages/db/src/queries/submissions.ts index 0971695e0..12b5d754d 100644 --- a/packages/db/src/queries/submissions.ts +++ b/packages/db/src/queries/submissions.ts @@ -37,7 +37,7 @@ export async function getSubmissionSummary(sql: DbClient): Promise>'spec_method', 'none') AS spec_method, c.disagg, c.is_multinode, c.num_prefill_gpu, @@ -56,7 +56,7 @@ export async function getSubmissionSummary(sql: DbClient): Promise>'spec_method'), c.disagg, c.is_multinode, c.num_prefill_gpu, c.num_decode_gpu, c.prefill_tp, c.prefill_ep, c.decode_tp, c.decode_ep, br.date ORDER BY br.date DESC, COUNT(*) DESC `; return rows as unknown as SubmissionSummaryRow[]; diff --git a/packages/db/src/queries/workflow-info.ts b/packages/db/src/queries/workflow-info.ts index b4e4f2550..deda5a124 100644 --- a/packages/db/src/queries/workflow-info.ts +++ b/packages/db/src/queries/workflow-info.ts @@ -75,7 +75,7 @@ export async function getDateConfigs(sql: DbClient, date: string): Promise>'spec_method', 'none') AS spec_method, c.disagg FROM benchmark_results br JOIN configs c ON c.id = br.config_id diff --git a/packages/db/src/verify-db.ts b/packages/db/src/verify-db.ts index ff2c3ed49..cd251843a 100644 --- a/packages/db/src/verify-db.ts +++ b/packages/db/src/verify-db.ts @@ -141,12 +141,12 @@ async function verify(): Promise { sub('spec_method'); const specs = await sql` - select c.spec_method, - count(distinct c.id)::int as configs, + select coalesce(br.techniques->>'spec_method', 'none') as spec_method, + count(distinct br.config_id)::int as configs, count(br.id)::int as bmk_rows - from configs c - left join benchmark_results br on br.config_id = c.id - group by c.spec_method order by c.spec_method + from benchmark_results br + group by (br.techniques->>'spec_method') + order by 1 `; console.log(` ${'SPEC_METHOD'.padEnd(10)} ${'configs'.padStart(8)} ${'bmk_rows'.padStart(10)}`); specs.forEach((r) => @@ -316,13 +316,13 @@ async function verify(): Promise { sub('configs (5 most recent)'); const sampleConfigs = await sql` - select id, hardware, framework, model, precision, spec_method, disagg, + select id, hardware, framework, model, precision, disagg, prefill_tp, prefill_ep, num_prefill_gpu, decode_tp, decode_ep, num_decode_gpu from configs order by id desc limit 5 `; sampleConfigs.forEach((r) => console.log( - ` [${r.id}] ${r.hardware} ${r.framework} ${r.model} ${r.precision} ${r.spec_method} disagg=${r.disagg} tp=${r.prefill_tp}/${r.decode_tp}`, + ` [${r.id}] ${r.hardware} ${r.framework} ${r.model} ${r.precision} disagg=${r.disagg} tp=${r.prefill_tp}/${r.decode_tp}`, ), ); From bea1abb96a57709a0c1f7c54c5bb0a3f9ee596a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Quang=20H=C3=B2a?= Date: Tue, 26 May 2026 11:04:56 +0700 Subject: [PATCH 2/3] chore(app): plumb techniques through client-side row types MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/api.ts BenchmarkRow + EvalRow gain a required `techniques` field mirroring what the API actually returns now. spec_method stays as a derived projection so every downstream consumer (charts, CSV export, TCO calculator, eval drawer, …) keeps working unchanged. unofficial-run/route.ts (artifact ZIP overlay path) reads spec_method from params.techniques.spec_method and carries the raw techniques jsonb through, matching the SQL row shape. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../app/src/app/api/unofficial-run/route.ts | 21 ++++++++++++++----- packages/app/src/lib/api.ts | 4 ++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/packages/app/src/app/api/unofficial-run/route.ts b/packages/app/src/app/api/unofficial-run/route.ts index a90e26fc7..493350e45 100644 --- a/packages/app/src/app/api/unofficial-run/route.ts +++ b/packages/app/src/app/api/unofficial-run/route.ts @@ -32,12 +32,15 @@ export function normalizeArtifactRows( const params = mapBenchmarkRow(raw as Record, tracker); if (!params) continue; const { config } = params; + const specMethod = + typeof params.techniques.spec_method === 'string' ? params.techniques.spec_method : 'none'; results.push({ hardware: config.hardware, framework: config.framework, model: config.model, precision: config.precision, - spec_method: config.specMethod, + spec_method: specMethod, + techniques: params.techniques, disagg: config.disagg, is_multinode: config.isMultinode, prefill_tp: config.prefillTp, @@ -62,13 +65,17 @@ export function normalizeArtifactRows( return results; } -function evalConfigKey(config: EvalParams['config']): string { +function evalConfigKey( + config: EvalParams['config'], + techniques: Record, +): string { return [ config.hardware, config.framework, config.model, config.precision, - config.specMethod, + typeof techniques.spec_method === 'string' ? techniques.spec_method : 'none', + typeof techniques.mtp_layers === 'number' ? String(techniques.mtp_layers) : 'none', config.disagg ? '1' : '0', config.prefillTp, config.prefillEp, @@ -107,7 +114,7 @@ export function normalizeEvalArtifactRows( const params = mapAggEvalRow(raw as Record, tracker); if (!params) continue; - const key = evalConfigKey(params.config); + const key = evalConfigKey(params.config, params.techniques); let localId = configIds.get(key); if (!localId) { localId = nextLocalId; @@ -115,6 +122,9 @@ export function normalizeEvalArtifactRows( nextLocalId += 1; } + const specMethod = + typeof params.techniques.spec_method === 'string' ? params.techniques.spec_method : 'none'; + rows.push({ // Synthetic id — unofficial rows are never persisted to eval_results, so // there's no real PK to surface. -1 signals "no DB-side row" to the @@ -125,7 +135,8 @@ export function normalizeEvalArtifactRows( framework: params.config.framework, model: params.config.model, precision: params.config.precision, - spec_method: params.config.specMethod, + spec_method: specMethod, + techniques: params.techniques, disagg: params.config.disagg, is_multinode: params.config.isMultinode, prefill_tp: params.config.prefillTp, diff --git a/packages/app/src/lib/api.ts b/packages/app/src/lib/api.ts index 1d7e8a3f5..832ac925c 100644 --- a/packages/app/src/lib/api.ts +++ b/packages/app/src/lib/api.ts @@ -10,7 +10,9 @@ export interface BenchmarkRow { framework: string; model: string; precision: string; + /** Derived from techniques.spec_method by the API; 'none' when absent. */ spec_method: string; + techniques: Record; disagg: boolean; is_multinode: boolean; prefill_tp: number; @@ -83,7 +85,9 @@ export interface EvalRow { framework: string; model: string; precision: string; + /** Derived from techniques.spec_method by the API; 'none' when absent. */ spec_method: string; + techniques: Record; disagg: boolean; is_multinode: boolean; prefill_tp: number; From 66b2a55f1b7576cc3d9395cbaea741218f536828 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Quang=20H=C3=B2a?= Date: Tue, 26 May 2026 11:05:07 +0700 Subject: [PATCH 3/3] test: update unit tests for skinny configs + techniques jsonb MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DB-side mappers / config-cache: - benchmark-mapper: techniques.spec_method assertions, plus a new test for the gemma4n4/n6 → mtp_layers inference path and one for the new-shape `techniques` object on the artifact - eval-mapper: techniques.spec_method assertion - config-cache: 16-col key, updated golden string App-side mocks: - compare-pair-defaults, benchmark-transform, evaluation/chart-data, unofficial-run-provider: add `techniques: {}` to BenchmarkRow / EvalRow stubs so they satisfy the updated row shape Co-Authored-By: Claude Opus 4.7 (1M context) --- .../components/evaluation/chart-data.test.ts | 1 + .../unofficial-run-provider.test.ts | 1 + .../app/src/lib/benchmark-transform.test.ts | 1 + .../app/src/lib/compare-pair-defaults.test.ts | 1 + packages/db/src/etl/benchmark-mapper.test.ts | 35 ++++++++++++++----- packages/db/src/etl/config-cache.test.ts | 10 +++--- packages/db/src/etl/eval-mapper.test.ts | 2 +- 7 files changed, 36 insertions(+), 15 deletions(-) diff --git a/packages/app/src/components/evaluation/chart-data.test.ts b/packages/app/src/components/evaluation/chart-data.test.ts index 47cf9af85..28be4ee54 100644 --- a/packages/app/src/components/evaluation/chart-data.test.ts +++ b/packages/app/src/components/evaluation/chart-data.test.ts @@ -18,6 +18,7 @@ function evalRow(overrides: Partial = {}): EvalRow { model: 'dsr1', precision: 'fp8', spec_method: 'none', + techniques: {}, disagg: false, is_multinode: false, prefill_tp: 8, diff --git a/packages/app/src/components/unofficial-run-provider.test.ts b/packages/app/src/components/unofficial-run-provider.test.ts index 1863060d9..0112a9a48 100644 --- a/packages/app/src/components/unofficial-run-provider.test.ts +++ b/packages/app/src/components/unofficial-run-provider.test.ts @@ -17,6 +17,7 @@ function stubRow(overrides: Partial = {}): BenchmarkRow { model: 'dsr1', precision: 'fp8', spec_method: 'none', + techniques: {}, disagg: false, is_multinode: false, prefill_tp: 8, diff --git a/packages/app/src/lib/benchmark-transform.test.ts b/packages/app/src/lib/benchmark-transform.test.ts index be76438e1..8e830aeaf 100644 --- a/packages/app/src/lib/benchmark-transform.test.ts +++ b/packages/app/src/lib/benchmark-transform.test.ts @@ -11,6 +11,7 @@ function makeRow(overrides: Partial = {}): BenchmarkRow { model: 'dsr1', precision: 'fp8', spec_method: 'none', + techniques: {}, disagg: false, is_multinode: false, prefill_tp: 8, diff --git a/packages/app/src/lib/compare-pair-defaults.test.ts b/packages/app/src/lib/compare-pair-defaults.test.ts index f0f1ef5b3..d411e7183 100644 --- a/packages/app/src/lib/compare-pair-defaults.test.ts +++ b/packages/app/src/lib/compare-pair-defaults.test.ts @@ -11,6 +11,7 @@ function makeRow(overrides: Partial): BenchmarkRow { model: 'dsr1', precision: 'fp8', spec_method: 'none', + techniques: {}, disagg: false, is_multinode: false, prefill_tp: 1, diff --git a/packages/db/src/etl/benchmark-mapper.test.ts b/packages/db/src/etl/benchmark-mapper.test.ts index 6c233de43..016bc6968 100644 --- a/packages/db/src/etl/benchmark-mapper.test.ts +++ b/packages/db/src/etl/benchmark-mapper.test.ts @@ -58,7 +58,7 @@ describe('mapBenchmarkRow', () => { expect(result!.config.framework).toBe('vllm'); expect(result!.config.model).toBe('dsr1'); expect(result!.config.precision).toBe('fp8'); - expect(result!.config.specMethod).toBe('none'); + expect(result!.techniques).toEqual({}); expect(result!.config.disagg).toBe(false); expect(result!.config.isMultinode).toBe(false); expect(result!.isl).toBe(1024); @@ -286,11 +286,11 @@ describe('mapBenchmarkRow', () => { expect(result!.config.disagg).toBe(true); }); - it('handles spec_decoding=mtp', () => { + it('handles spec_decoding=mtp (legacy field → techniques.spec_method)', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow(makeV1Row({ spec_decoding: 'mtp' }), tracker); - expect(result!.config.specMethod).toBe('mtp'); + expect(result!.techniques.spec_method).toBe('mtp'); }); }); @@ -331,19 +331,38 @@ describe('mapBenchmarkRow', () => { }); }); - describe('spec_decoding', () => { - it('normalizes spec_decoding to lowercase', () => { + describe('techniques (spec_decoding legacy + new shape)', () => { + it('normalizes legacy spec_decoding to lowercase under techniques.spec_method', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow(makeV1Row({ spec_decoding: 'Eagle' }), tracker); - expect(result!.config.specMethod).toBe('eagle'); + expect(result!.techniques.spec_method).toBe('eagle'); }); - it('defaults to none when absent', () => { + it('omits spec_method when absent (techniques bag stays empty)', () => { const tracker = createSkipTracker(); const result = mapBenchmarkRow(makeV1Row(), tracker); - expect(result!.config.specMethod).toBe('none'); + expect(result!.techniques).toEqual({}); + }); + + it('reads new-shape techniques object directly', () => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow( + makeV1Row({ techniques: { spec_method: 'mtp', mtp_layers: 4 } }), + tracker, + ); + + expect(result!.techniques).toEqual({ spec_method: 'mtp', mtp_layers: 4 }); + }); + + it('infers mtp_layers from gemma4n4/gemma4n6 model prefix (legacy)', () => { + const tracker = createSkipTracker(); + const result = mapBenchmarkRow(makeV1Row({ infmax_model_prefix: 'gemma4n6' }), tracker); + + expect(result!.config.model).toBe('gemma4'); + expect(result!.techniques.mtp_layers).toBe(6); + expect(result!.techniques.spec_method).toBe('mtp'); }); }); diff --git a/packages/db/src/etl/config-cache.test.ts b/packages/db/src/etl/config-cache.test.ts index 9ac5f010a..6544f6bfe 100644 --- a/packages/db/src/etl/config-cache.test.ts +++ b/packages/db/src/etl/config-cache.test.ts @@ -7,7 +7,6 @@ function makeConfig(overrides: Partial = {}): ConfigParams { framework: 'vllm', model: 'dsr1', precision: 'fp8', - specMethod: 'none', disagg: false, isMultinode: false, prefillTp: 8, @@ -27,7 +26,7 @@ function makeConfig(overrides: Partial = {}): ConfigParams { describe('configCacheKey', () => { it('produces a colon-joined string of all config fields', () => { const key = configCacheKey(makeConfig()); - expect(key).toBe('h200:vllm:dsr1:fp8:none:false:false:8:1:false:0:8:1:false:0:8:8'); + expect(key).toBe('h200:vllm:dsr1:fp8:false:false:8:1:false:0:8:1:false:0:8:8'); }); it('is deterministic — same input produces same output', () => { @@ -44,7 +43,6 @@ describe('configCacheKey', () => { expect(configCacheKey(makeConfig({ framework: 'sglang' }))).not.toBe(base); expect(configCacheKey(makeConfig({ model: 'llama70b' }))).not.toBe(base); expect(configCacheKey(makeConfig({ precision: 'fp4' }))).not.toBe(base); - expect(configCacheKey(makeConfig({ specMethod: 'eagle' }))).not.toBe(base); expect(configCacheKey(makeConfig({ disagg: true }))).not.toBe(base); expect(configCacheKey(makeConfig({ isMultinode: true }))).not.toBe(base); expect(configCacheKey(makeConfig({ prefillTp: 4 }))).not.toBe(base); @@ -59,10 +57,10 @@ describe('configCacheKey', () => { expect(configCacheKey(makeConfig({ numDecodeGpu: 4 }))).not.toBe(base); }); - it('includes all 17 fields in the key', () => { + it('includes all 16 fields in the key (spec_method moved to techniques)', () => { const key = configCacheKey(makeConfig()); - // 17 fields = 16 colons - expect(key.split(':').length).toBe(17); + // 16 fields = 15 colons + expect(key.split(':').length).toBe(16); }); it('handles boolean fields correctly', () => { diff --git a/packages/db/src/etl/eval-mapper.test.ts b/packages/db/src/etl/eval-mapper.test.ts index ef85bbfd8..858c5c1ee 100644 --- a/packages/db/src/etl/eval-mapper.test.ts +++ b/packages/db/src/etl/eval-mapper.test.ts @@ -70,7 +70,7 @@ describe('mapEvalRow', () => { expect(ev.config.framework).toBe('vllm'); expect(ev.config.model).toBe('dsr1'); expect(ev.config.precision).toBe('fp8'); - expect(ev.config.specMethod).toBe('none'); + expect(ev.techniques).toEqual({}); expect(ev.config.disagg).toBe(false); expect(ev.task).toBe('gsm8k'); expect(ev.isl).toBe(1024);