From 7113e980ff4cfd5c9e156cf42365b8e73264c443 Mon Sep 17 00:00:00 2001 From: kesslerio Date: Wed, 19 Aug 2026 15:12:28 -0700 Subject: [PATCH] feat(bench): model picker for multi-model routers Adds a model picker dropdown to the benchmark dialog for multi-model routers like LiteLLM. When /v1/models returns multiple models, the benchmark uses the first model by default and the user can pick from the full list. - LlmProbe: detect multi-model routers, expose models array and benchmarkModel field - BenchmarkDialog: model picker dropdown with .bench-select styling - LlmPanel: pass benchmarkModel and models to BenchmarkDialog, use benchmarkModel in showcase URL params - types.ts: add benchmarkModel and models fields to LlmMetrics Re-submission of closed PR #51, rebased on current upstream/main. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- server/collectors/LlmProbe.js | 37 +++++++++++---- src/api/types.ts | 4 ++ src/components/SparkPage/BenchmarkDialog.tsx | 50 ++++++++++++++++++-- src/components/SparkPage/LlmPanel.tsx | 6 ++- src/index.css | 28 +++++++++++ 5 files changed, 112 insertions(+), 13 deletions(-) diff --git a/server/collectors/LlmProbe.js b/server/collectors/LlmProbe.js index 7c32312..9b2a22b 100644 --- a/server/collectors/LlmProbe.js +++ b/server/collectors/LlmProbe.js @@ -70,6 +70,8 @@ export class LlmProbe { this.stepId = 0; this.modelId = null; this.modelPath = null; + this.benchmarkModel = null; + this.models = []; this.contextLength = null; this.gpuMemoryUtilization = null; this.slotsActive = 0; @@ -226,6 +228,8 @@ export class LlmProbe { this.authOpen = null; this.modelId = null; this.modelPath = null; + this.benchmarkModel = null; + this.models = []; this.generationTps = 0; this.prefillTps = 0; this.cachedPrefillTps = null; @@ -386,14 +390,28 @@ export class LlmProbe { if (auth === "ok") { modelsOk = true; const modelsData = await modelsRes.json(); - const model = modelsData?.data?.[0]; - this.modelId = normalizeModelId(model?.id || null); - // Drop HF hub cache paths from modelPath if /v1/models id was a cache dir - if (isHfHubCachePath(model?.id)) this.modelPath = null; - // ds4-server uses context_length; vLLM uses max_model_len - this.contextLength = - model?.max_model_len ?? model?.context_length ?? this.contextLength; - owned = model?.owned_by; + const models = Array.isArray(modelsData?.data) ? modelsData.data : []; + if (models.length > 1) { + // Multi-model router (e.g. LiteLLM): show count, list IDs in modelPath. + // Store first model for benchmark/showcase requests. + this.models = models.map((m) => m?.id).filter(Boolean); + this.modelId = `${this.models.length} models`; + this.modelPath = this.models.join(", "); + this.benchmarkModel = this.models[0] || null; + this.contextLength = null; + owned = models[0]?.owned_by; + } else { + const model = models[0]; + this.modelId = normalizeModelId(model?.id || null); + this.benchmarkModel = this.modelId; + this.models = this.modelId ? [this.modelId] : []; + // Drop HF hub cache paths from modelPath if /v1/models id was a cache dir + if (isHfHubCachePath(model?.id)) this.modelPath = null; + // ds4-server uses context_length; vLLM uses max_model_len + this.contextLength = + model?.max_model_len ?? model?.context_length ?? this.contextLength; + owned = model?.owned_by; + } } } catch {} @@ -1253,6 +1271,8 @@ export class LlmProbe { backend: this.backendType, modelId: this.modelId || null, modelPath: this.modelPath || null, + benchmarkModel: this.benchmarkModel || null, + models: this.models, contextLength: this.contextLength, gpuMemoryUtilization: this.gpuMemoryUtilization, slotsActive: this.slotsActive, @@ -1282,6 +1302,7 @@ export class LlmProbe { backend: this.backendType, modelId: null, modelPath: null, + models: [], contextLength: null, gpuMemoryUtilization: null, slotsActive: 0, diff --git a/src/api/types.ts b/src/api/types.ts index 7e42fd1..72d90b5 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -284,6 +284,10 @@ export interface LlmMetrics { backend: "vllm" | "llama.cpp" | "sglang" | "ds4" | null; modelId: string | null; modelPath: string | null; + /** Model ID to use for benchmark/showcase requests. Falls back to modelId for single-model backends. */ + benchmarkModel?: string | null; + /** All model IDs from /v1/models. Single-element for single-model backends, multi-element for routers like LiteLLM. */ + models?: string[]; contextLength: number | null; /** GPU memory utilization for the LLM engine (0–1), e.g. 0.9. Only from vLLM internal info. */ gpuMemoryUtilization: number | null; diff --git a/src/components/SparkPage/BenchmarkDialog.tsx b/src/components/SparkPage/BenchmarkDialog.tsx index 8dbbf4a..9b42054 100644 --- a/src/components/SparkPage/BenchmarkDialog.tsx +++ b/src/components/SparkPage/BenchmarkDialog.tsx @@ -20,6 +20,8 @@ interface BenchmarkDialogProps { sparkId: string; llmPort: number; modelId: string | null; + /** All model IDs from /v1/models. Undefined for backends that don't report a list. */ + models?: string[]; } function useEscape(onClose: () => void, enabled: boolean) { @@ -144,6 +146,7 @@ export function BenchmarkDialog({ sparkId, llmPort, modelId, + models, }: BenchmarkDialogProps) { const [selected, setSelected] = useState([...DEFAULT_SELECTED]); const [maxTokensDraft, setMaxTokensDraft] = useState(String(DEFAULT_MAX_TOKENS)); @@ -152,9 +155,20 @@ export function BenchmarkDialog({ const [starting, setStarting] = useState(false); const [loadingLast, setLoadingLast] = useState(false); const [copied, setCopied] = useState(false); + const [selectedModel, setSelectedModel] = useState(null); const pollRef = useRef | null>(null); const copyResetRef = useRef | null>(null); + const availableModels = models ?? []; + const isMultiModel = availableModels.length > 1; + + // Sync selected model when the prop changes or dialog opens. + useEffect(() => { + if (open) { + setSelectedModel(modelId); + } + }, [open, modelId]); + const stopPoll = useCallback(() => { if (pollRef.current != null) { clearInterval(pollRef.current); @@ -308,7 +322,7 @@ export function BenchmarkDialog({ port: llmPort, concurrencies: selected, maxTokens, - modelId: modelId || undefined, + modelId: selectedModel || undefined, }); setJob(started); startPolling(started.benchId); @@ -338,7 +352,7 @@ export function BenchmarkDialog({ const handleCopyResults = async () => { if (!job || job.results.length === 0) return; - const text = buildShareText(job, modelId); + const text = buildShareText(job, selectedModel); try { if (navigator.clipboard?.writeText) { await navigator.clipboard.writeText(text); @@ -411,7 +425,11 @@ export function BenchmarkDialog({

Port {llmPort} - {modelId ? ` · ${modelId}` : ""} + {isMultiModel + ? ` · ${availableModels.length} models` + : modelId + ? ` · ${modelId}` + : ""}