Skip to content

feat(bench): model picker for multi-model routers #56

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kesslerio wants to merge 1 commit into
base: main
Choose a base branch
from
Open

feat(bench): model picker for multi-model routers #56

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 29 additions & 8 deletions server/collectors/LlmProbe.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,6 +70,8 @@ export class LlmProbe {
this.stepId = 0;
this.modelId = null;
this.modelPath = null;
this.benchmarkModel = null;
this.models = [];
this.contextLength = null;
this.gpuMemoryUtilization = null;
this.slotsActive = 0;
Expand Down Expand Up @@ -226,6 +228,8 @@ export class LlmProbe {
this.authOpen = null;
this.modelId = null;
this.modelPath = null;
this.benchmarkModel = null;
this.models = [];
this.generationTps = 0;
this.prefillTps = 0;
this.cachedPrefillTps = null;
Expand Down Expand Up @@ -386,14 +390,28 @@ export class LlmProbe {
if (auth === "ok") {
modelsOk = true;
const modelsData = await modelsRes.json();
const model = modelsData?.data?.[0];
this.modelId = normalizeModelId(model?.id || null);
// Drop HF hub cache paths from modelPath if /v1/models id was a cache dir
if (isHfHubCachePath(model?.id)) this.modelPath = null;
// ds4-server uses context_length; vLLM uses max_model_len
this.contextLength =
model?.max_model_len ?? model?.context_length ?? this.contextLength;
owned = model?.owned_by;
const models = Array.isArray(modelsData?.data) ? modelsData.data : [];
if (models.length > 1) {
// Multi-model router (e.g. LiteLLM): show count, list IDs in modelPath.
// Store first model for benchmark/showcase requests.
this.models = models.map((m) => m?.id).filter(Boolean);
this.modelId = `${this.models.length} models`;
this.modelPath = this.models.join(", ");
this.benchmarkModel = this.models[0] || null;
this.contextLength = null;
owned = models[0]?.owned_by;
} else {
const model = models[0];
this.modelId = normalizeModelId(model?.id || null);
this.benchmarkModel = this.modelId;
this.models = this.modelId ? [this.modelId] : [];
// Drop HF hub cache paths from modelPath if /v1/models id was a cache dir
if (isHfHubCachePath(model?.id)) this.modelPath = null;
// ds4-server uses context_length; vLLM uses max_model_len
this.contextLength =
model?.max_model_len ?? model?.context_length ?? this.contextLength;
owned = model?.owned_by;
}
}
} catch {}

Expand Down Expand Up @@ -1253,6 +1271,8 @@ export class LlmProbe {
backend: this.backendType,
modelId: this.modelId || null,
modelPath: this.modelPath || null,
benchmarkModel: this.benchmarkModel || null,
models: this.models,
contextLength: this.contextLength,
gpuMemoryUtilization: this.gpuMemoryUtilization,
slotsActive: this.slotsActive,
Expand Down Expand Up @@ -1282,6 +1302,7 @@ export class LlmProbe {
backend: this.backendType,
modelId: null,
modelPath: null,
models: [],
contextLength: null,
gpuMemoryUtilization: null,
slotsActive: 0,
Expand Down
4 changes: 4 additions & 0 deletions src/api/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -284,6 +284,10 @@ export interface LlmMetrics {
backend: "vllm" | "llama.cpp" | "sglang" | "ds4" | null;
modelId: string | null;
modelPath: string | null;
/** Model ID to use for benchmark/showcase requests. Falls back to modelId for single-model backends. */
benchmarkModel?: string | null;
/** All model IDs from /v1/models. Single-element for single-model backends, multi-element for routers like LiteLLM. */
models?: string[];
contextLength: number | null;
/** GPU memory utilization for the LLM engine (0–1), e.g. 0.9. Only from vLLM internal info. */
gpuMemoryUtilization: number | null;
Expand Down
50 changes: 47 additions & 3 deletions src/components/SparkPage/BenchmarkDialog.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ interface BenchmarkDialogProps {
sparkId: string;
llmPort: number;
modelId: string | null;
/** All model IDs from /v1/models. Undefined for backends that don't report a list. */
models?: string[];
}

function useEscape(onClose: () => void, enabled: boolean) {
Expand Down Expand Up @@ -144,6 +146,7 @@ export function BenchmarkDialog({
sparkId,
llmPort,
modelId,
models,
}: BenchmarkDialogProps) {
const [selected, setSelected] = useState<number[]>([...DEFAULT_SELECTED]);
const [maxTokensDraft, setMaxTokensDraft] = useState(String(DEFAULT_MAX_TOKENS));
Expand All @@ -152,9 +155,20 @@ export function BenchmarkDialog({
const [starting, setStarting] = useState(false);
const [loadingLast, setLoadingLast] = useState(false);
const [copied, setCopied] = useState(false);
const [selectedModel, setSelectedModel] = useState<string | null>(null);
const pollRef = useRef<ReturnType<typeof setInterval> | null>(null);
const copyResetRef = useRef<ReturnType<typeof setTimeout> | null>(null);

const availableModels = models ?? [];
const isMultiModel = availableModels.length > 1;

// Sync selected model when the prop changes or dialog opens.
useEffect(() => {
if (open) {
setSelectedModel(modelId);
}
}, [open, modelId]);

const stopPoll = useCallback(() => {
if (pollRef.current != null) {
clearInterval(pollRef.current);
Expand Down Expand Up @@ -308,7 +322,7 @@ export function BenchmarkDialog({
port: llmPort,
concurrencies: selected,
maxTokens,
modelId: modelId || undefined,
modelId: selectedModel || undefined,
});
setJob(started);
startPolling(started.benchId);
Expand Down Expand Up @@ -338,7 +352,7 @@ export function BenchmarkDialog({

const handleCopyResults = async () => {
if (!job || job.results.length === 0) return;
const text = buildShareText(job, modelId);
const text = buildShareText(job, selectedModel);
try {
if (navigator.clipboard?.writeText) {
await navigator.clipboard.writeText(text);
Expand Down Expand Up @@ -411,7 +425,11 @@ export function BenchmarkDialog({
</h2>
<p className="bench-sheet__subtitle">
Port {llmPort}
{modelId ? ` · ${modelId}` : ""}
{isMultiModel
? ` · ${availableModels.length} models`
: modelId
? ` · ${modelId}`
: ""}
</p>
</div>
<button
Expand All @@ -431,6 +449,32 @@ export function BenchmarkDialog({

{showConfig && (
<section className="bench-sheet__section">
{isMultiModel && (
<div className="bench-field">
<div className="bench-field__head">
<label htmlFor="bench-model-select" className="bench-sheet__section-title">
Model
</label>
<p className="bench-sheet__hint">
Select which model the benchmark targets.
</p>
</div>
<select
id="bench-model-select"
disabled={isRunning || starting}
value={selectedModel ?? ""}
onChange={(e) => setSelectedModel(e.target.value || null)}
className="bench-select"
>
{availableModels.map((m) => (
<option key={m} value={m}>
{m}
</option>
))}
</select>
</div>
)}

<div className="bench-field">
<div className="bench-field__head">
<h3 className="bench-sheet__section-title">Concurrency</h3>
Expand Down
6 changes: 4 additions & 2 deletions src/components/SparkPage/LlmPanel.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -719,7 +719,8 @@ export function LlmPanel({
onClick={() => {
const params = new URLSearchParams();
if (llmPort) params.set("port", String(llmPort));
if (llm?.modelId) params.set("model", llm.modelId);
const benchModel = llm?.benchmarkModel ?? llm?.modelId ?? null;
if (benchModel) params.set("model", benchModel);
const q = params.toString() ? `?${params.toString()}` : "";
window.open(
`/showcase/${encodeURIComponent(sparkId)}${q}`,
Expand All @@ -740,7 +741,8 @@ export function LlmPanel({
onClose={() => setBenchOpen(false)}
sparkId={sparkId}
llmPort={llmPort}
modelId={llm?.modelId ?? null}
modelId={llm?.benchmarkModel ?? llm?.modelId ?? null}
models={llm?.models ?? undefined}
/>
</Panel>
);
Expand Down
28 changes: 28 additions & 0 deletions src/index.css
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1258,6 +1258,34 @@ input[type="checkbox"] {
opacity: 0.5;
}

.bench-select {
display: block;
box-sizing: border-box;
width: 100%;
max-width: 24rem;
min-height: 2.25rem;
padding: 0.4rem 0.65rem;
border-radius: 0.5rem;
border: 1px solid var(--color-border);
background: #ffffff;
color: var(--color-text);
font-size: 0.8125rem;
outline: none;
cursor: pointer;
transition: border-color 0.15s ease;
}
[data-theme="dark"] .bench-select,
[data-theme="oled"] .bench-select {
background: var(--color-surface-elevated);
}
.bench-select:focus {
border-color: var(--color-accent);
}
.bench-select:disabled {
opacity: 0.5;
cursor: not-allowed;
}

.bench-progress {
display: flex;
flex-direction: column;
Expand Down