From a436c60f58f4d3413ca717e3d96481ac05eba535 Mon Sep 17 00:00:00 2001 From: y1618 Date: Thu, 20 Aug 2026 00:20:18 +0900 Subject: [PATCH 1/3] fix: send `thinking` chat_template_kwarg for DeepSeek models MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The decode bench and showcase disable thinking via `chat_template_kwargs.enable_thinking`, but DeepSeek V4 templates only honor `thinking` and silently ignore `enable_thinking`. With a server default of `{"thinking":true,"reasoning_effort":"max"}` the whole bench window is spent inside ``, where MTP acceptance is much lower — on a 2x DGX Spark DeepSeek-V4-Flash stack the c=1 decode bench read 44.1 tok/s; with thinking actually off it reads 70.4 tok/s. Map deepseek model ids to `thinking` in applyThinkingFlags (keeping enable_thinking for everyone else), strip it in stripThinkingFlags for the 400-retry path, and add unit tests. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01NRSGQ8VRdFgB76tz9JbtMX --- server/collectors/LlmStreaming.js | 8 +++- .../__tests__/LlmStreaming.thinking.test.js | 40 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 server/collectors/__tests__/LlmStreaming.thinking.test.js diff --git a/server/collectors/LlmStreaming.js b/server/collectors/LlmStreaming.js index 529faa8..11d14e2 100644 --- a/server/collectors/LlmStreaming.js +++ b/server/collectors/LlmStreaming.js @@ -206,7 +206,8 @@ function extractDeltaPieces(choice) { /** * Apply per-model thinking flags so reasoning models don't 400. - * MiniMax-M3 needs `thinking_mode`; most others use `enable_thinking`. + * MiniMax-M3 needs `thinking_mode`; DeepSeek V4 needs `thinking`; + * most others use `enable_thinking`. * * @param {Record} body * @param {string | null | undefined} modelId @@ -226,6 +227,10 @@ export function applyThinkingFlags(body, modelId, think = true) { if (id.includes("minimax") || /(^|[^a-z])m3([^a-z]|$)/.test(id)) { ctk.thinking_mode = think ? "enabled" : "disabled"; } + // DeepSeek V4 templates use `thinking` and ignore enable_thinking + if (id.includes("deepseek")) { + ctk.thinking = think; + } body.chat_template_kwargs = ctk; return body; } @@ -240,6 +245,7 @@ export function stripThinkingFlags(body) { const ctk = { ...body.chat_template_kwargs }; delete ctk.enable_thinking; delete ctk.thinking_mode; + delete ctk.thinking; if (Object.keys(ctk).length) body.chat_template_kwargs = ctk; else delete body.chat_template_kwargs; } diff --git a/server/collectors/__tests__/LlmStreaming.thinking.test.js b/server/collectors/__tests__/LlmStreaming.thinking.test.js new file mode 100644 index 0000000..6d86d2e --- /dev/null +++ b/server/collectors/__tests__/LlmStreaming.thinking.test.js @@ -0,0 +1,40 @@ +/** + * Unit tests for per-model thinking flag mapping. + * Run: npm test + */ +import { test } from "node:test"; +import { strict as assert } from "node:assert"; +import { applyThinkingFlags, stripThinkingFlags } from "../LlmStreaming.js"; + +test("applyThinkingFlags: default models get enable_thinking only", () => { + const body = applyThinkingFlags({}, "qwen3-32b", false); + assert.deepEqual(body.chat_template_kwargs, { enable_thinking: false }); +}); + +test("applyThinkingFlags: MiniMax gets thinking_mode", () => { + const body = applyThinkingFlags({}, "minimax-m3", false); + assert.equal(body.chat_template_kwargs.thinking_mode, "disabled"); +}); + +test("applyThinkingFlags: DeepSeek gets thinking (enable_thinking is ignored by its template)", () => { + const off = applyThinkingFlags({}, "deepseek-v4-flash-0731", false); + assert.equal(off.chat_template_kwargs.thinking, false); + const on = applyThinkingFlags({}, "DeepSeek-V4-Flash-0731", true); + assert.equal(on.chat_template_kwargs.thinking, true); +}); + +test("applyThinkingFlags: preserves existing chat_template_kwargs", () => { + const body = applyThinkingFlags( + { chat_template_kwargs: { reasoning_effort: "low" } }, + "deepseek-v4-flash-0731", + false, + ); + assert.equal(body.chat_template_kwargs.reasoning_effort, "low"); + assert.equal(body.chat_template_kwargs.thinking, false); +}); + +test("stripThinkingFlags: removes thinking key too", () => { + const body = { chat_template_kwargs: { thinking: false, enable_thinking: false } }; + stripThinkingFlags(body); + assert.equal(body.chat_template_kwargs, undefined); +}); From 2a1325ff85a345133bfb236baa0f31269a3faef8 Mon Sep 17 00:00:00 2001 From: y1618 Date: Sun, 23 Aug 2026 11:51:52 +0900 Subject: [PATCH 2/3] docs: add CLAUDE.md with repo guidance for Claude Code Commands, server/client architecture map, and the on-purpose rules (broadcast diff cache, hot vs restart config paths, backend probe quirks, theme tokens, shared llmPrompts.js) that are not obvious from any one file. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HgmJYczuwSYo7RGBMwvsTQ --- CLAUDE.md | 84 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 CLAUDE.md diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..c9f6979 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,84 @@ +# CLAUDE.md + +This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. + +## What this is + +sparkDash is a real-time monitoring dashboard for multiple NVIDIA DGX Spark (GB10) units plus "dedicated GPU host" Linux boxes. A Node/Express/`ws` backend (plain JS ESM, **no TypeScript on the server**) polls hosts (local sysfs/proc/`nvidia-smi`, or remote over SSH) and LLM servers (llama.cpp / vLLM / SGLang / ds4), and streams snapshots over WebSocket to a React 19 + TypeScript + Vite + Tailwind v4 SPA. Production target is an arm64 Docker container. README.md is thorough on features, REST routes, env vars, and security model — read it for user-facing behavior; this file covers what you need to change code safely. + +## Commands + +```bash +npm install # needed for typecheck and the ComfyProbe test (imports `ws`); most tests run without deps +npm run dev # Vite :5173 (proxies /api and /ws → 127.0.0.1:5555) + `node --watch server/index.js` :5555 +npm run dev:server # server only +npm run dev:client # Vite only +npm run build # frontend → dist/ (server serves dist/ + SPA fallback in prod) +npm start # node server/index.js (serves API, WS, and dist/) +npm run typecheck # tsc --noEmit — covers src/ only (server/ and config/ are excluded) +npm test # node --test server/collectors/__tests__/*.test.js server/sparks/__tests__/*.test.js +node --test server/collectors/__tests__/LlmProbe.sglang.test.js # single file +node --test --test-name-pattern "sticky" server/collectors/__tests__/*.test.js # by test name +``` + +- There is no linter/formatter config and no frontend test harness; `npm test` is server-only and the glob list in `package.json` is explicit — **a new `__tests__` directory is not picked up unless added there**. +- Docker: `docker compose up --build -d` (prod; `network_mode: host`, `privileged`, `pid: host`), `docker compose -f docker-compose.dev.yml up --build` (dev with HMR), `./deploy.sh` (down → build → up). The prod compose bind-mounts `./server` and `./src/shared` and runs `node --watch`, so **server edits hot-reload inside the running container, but frontend edits need an image rebuild**. +- Release convention (see CHANGELOG.md header): bump `package.json` version, add a section at the top of `CHANGELOG.md`, and replace the README "Latest version changelog" block (README shows only the current release). + +## Architecture + +### One Spark model, N instances + +Every unit is a record in `config/sparks.json` with `kind: "spark" | "host"` and `role: "head" | "worker" | "standalone"`. The same `SparkMonitor` / `SystemCollector` / `LlmProbe` code runs for all of them — prefer extending the shared model over per-unit special cases. + +### Server data flow (`server/`) + +``` +SparkRegistry (config + encrypted secrets, change events) + └─ SparkMonitor (one per spark; per-domain setInterval + liveness timer; owns rate baselines + _metrics cache) + ├─ SystemCollector local: /host/proc, /host/sys, nvidia-smi via `nsenter --mount`; remote: one batched SSH cmd per domain, split on '---' + ├─ LlmProbe[] one per configured LLM port; backend autodetect + counter-diff tok/s + ├─ ComfyProbe opt-in; + ComfyProgressSocket (ws client to Comfy) + └─ HermesProbe opt-in; `hermes update --check` over SSH / setpriv-as-host-user locally + → monitor.snapshot() → orderedSnapshots() (registry tab order) → WS broadcast +``` + +- `server/index.js` is the whole HTTP layer: Express routes, the single `WebSocketServer` on `/ws`, `monitors: Map`, `startBroadcast()`. Boot: `loadSettings()` → `startBroadcast()` → `listen` → `startAllMonitors()`. +- **WS protocol is one message type**, `{ type: "snapshot", sparks: SparkSnapshot[], refreshInterval }`, pushed every `settings.pollIntervalMs`. No client→server WS messages — all mutations are REST. The broadcaster **skips the tick if the JSON payload is byte-identical to the last one**; that is why `snapshot()` deliberately has no `timestamp` field (see the NOTE in `SparkMonitor.js`). Use `forceBroadcast()` after out-of-band state changes (Hermes check/update, manual refresh). +- Poll loops run with no clients connected so rate metrics (tok/s, bytes/s, disk I/O) keep valid baselines. `_pollDomain()` has in-flight guards and re-checks `this._running` after every `await` — keep that pattern when adding a domain. +- Hot vs restart: `PATCH /api/sparks/:id` stops and restarts the monitor; password-only, disabled-devices/interfaces, and llm-port routes call `monitor.updateConfig()` (comment `// hot — no monitor restart`) so baselines survive. Settings hot-reload in-process; only `pollIntervalMs` triggers `restartBroadcast()`. +- Route order matters in `index.js`: batch routes (`/api/sparks/shutdown-all`, `/wake-all`, `/hermes/update-all`) and `PUT /api/sparks/order` are registered **before** `/api/sparks/:id/*`. SPA fallback uses Express 5 syntax `app.get("*splat", …)`. +- Every route accepting a host/user/id goes through `server/validate.js` (`validateSparkTarget`, `isAllowedTargetHost`, `isValidSshUser`, `isValidSparkId`). The API is intentionally unauthenticated (LAN-trusted) — don't add features that assume otherwise without saying so. +- `server/collectors/ssh.js` `sshExec()` is the only SSH path: `execFile` argv (no shell), whitelisted child env, password via `sshpass -e` + `SSHPASS` env (never argv). Remote collectors batch commands and parse text; add to the batch rather than opening more SSH sessions. +- Local Hermes must run as the host user (`setpriv` inside `nsenter`), never container root — root previously corrupted installs. Keep `chooseLocalInvocation()` semantics if touching HermesProbe. + +### Persistence (`config/`, bind-mounted volume) + +All writers go through `server/util/atomicWrite.js` (tmp + rename, with fallbacks for root-owned files from Docker). `sparks.json` never contains secrets; SSH passwords and LLM API keys are AES-256-GCM in `sparks-secrets.json` keyed by `.secrets-key` (or `SPARKDASH_SECRETS_KEY`). `registry.toPublic()` strips secrets for every API response (`ssh.hasPassword`, `llmApiKeyPorts` only). `bench-active.json` checkpoints running decode benchmarks so a `node --watch` reload doesn't 404 the dialog. `gpu-memory.json` is written by the host cron `config/gpu-memory.sh` and is only used as a fallback when the live `nvidia-smi --query-compute-apps` call failed. + +### LLM subsystem (`server/collectors/LlmProbe.js`, `LlmStreaming.js`, `DecodeBench.js`, `ShowcaseManager.js`) + +- `LlmProbe` detects backend via `/slots` (llama.cpp) then `/v1/models` + `owned_by` / Prometheus `ds4_*` / `/get_server_info` → `vllm | sglang | ds4 | lmstudio`. Once known to be OpenAI-compatible it stops probing `/slots` (avoids 404 spam). LM Studio (`owned_by: organization_owner`) answers *every* unknown path with 200 + `{"error"}` and logs it as an ERROR, so it is classified from `owned_by` alone, never probed for `/metrics`/`/get_server_info`, and its model list is read only every 10 s (`_probeLmStudio`); `_looksLikeServerInfo()` keeps such error envelopes from reading as SGLang. Several tests lock in backend quirks: SGLang `last_gen_throughput` is a sticky gauge (treat as live only while it changes); never use SGLang `max_total_num_tokens` as context length; never use ds4 `*_tok_s` window gauges for live tok/s; HF hub cache paths normalize to `org/name` via `normalizeModelId()`. +- `LlmStreaming.js` is the shared SSE layer for both DecodeBench and Showcase: splits `delta.content` from `delta.reasoning`/`reasoning_content`, measures decode tok/s first-visible→last-visible token, and `applyThinkingFlags()` writes model-family-specific `chat_template_kwargs` (`enable_thinking`, MiniMax `thinking_mode`, DeepSeek `thinking`) with a one-shot retry on HTTP 400 after stripping them. +- `src/shared/llmPrompts.js` is the prompt catalog imported by **both** the server (`../../src/shared/llmPrompts.js`) and the client (`../../shared/llmPrompts.js`). `showcasePrompts.test.js` reads the *source text* of DecodeBench, ShowcaseManager, llmPrompts and the client re-export and regex-asserts request bodies and import paths — a behavior-preserving refactor of those request bodies can still fail that test. +- Decode bench and Showcase are mutually exclusive per spark; Showcase sessions are kept alive by client `GET` heartbeats and auto-cancel after 5 s without one. + +### Client (`src/`) + +- No router library: `hooks/useRoute.ts` does History-API path routing — `/` Overview, `/spark/:id` detail, `/showcase/:id` a separate full-screen mode rendered by `App.tsx` with no dashboard chrome (opened via `window.open` from `LlmPanel`). `OVERVIEW_ID = "__overview__"` (`constants.ts`) is a reserved pseudo-tab id; the server rejects it as a spark id by hand-duplicated check in `validate.js`. +- **No React Context.** Live data is a module-level store + `useSyncExternalStore`: `hooks/useSnapshot.ts` owns the WS (fixed 2 s reconnect) and calls `ingestSnapshots()` from `hooks/metricsStore.ts`, which keeps client-side rolling series (`HISTORY_MAX = 1800`, precomputed `SPARKLINE_TAIL = 30`) keyed `"${sparkId}:${metric}"`. Panels take `sparkId` and read their own series via `useMetricsHistoryTail()` — don't thread history through props. The store header documents a reference-stability contract (unchanged keys keep the same array ref); preserve it. `hooks/useHermesUpdateDialog.ts` uses the same module-store pattern for a global dialog. +- REST lives in `api/client.ts` behind one `apiFetch()` wrapper (relative URLs; throws `Error(body.error)`); wire types are hand-mirrored from the server in `api/types.ts` — **no codegen**, keep both sides in sync manually. +- Always resolve roles via `api/sparkRole.ts` (`resolveSparkRole`, `isLlmMonitoringEnabled`) — it handles the legacy `workerNode` boolean; don't read `role`/`workerNode` directly. +- Cross-tier shared code goes in `src/shared/*.js` (plain ESM so Node can import it) with a sibling `.d.ts` — never `.ts`. +- Theming is a `data-theme` attribute on `` (`white | light | dark | oled`, localStorage key `sparkdash-theme`), not Tailwind `dark:`. Tokens are CSS custom properties defined per `[data-theme]` block in `index.css` and re-exported through an `@theme { --color-*: var(--color-*) }` block so Tailwind utilities (`bg-surface`, `text-muted`, `text-accent`, …) resolve to the live theme. **A new color token must be added in every `[data-theme]` block and in `@theme`.** Density is a second attribute, `data-density="compact|comfortable"`, driven by server-persisted settings and consumed via `--density-*` vars. +- Every metric card is a `ui/Panel`; `ui/MetricBar` (`bandColor`), `ui/Sparkline`, and hand-inlined `ui/icons.tsx` are the primitives (no icon library). Dialogs use `createPortal` + `hooks/useModalPresence` for enter/exit transitions. +- `SparkPage.tsx` layout is conditional on `kind`: hosts render GPU (left, row-span-3) + RAM → Network → Storage; Sparks render GPU (row-span-2) + Storage → Network with **no RamPanel** (unified memory lives in GpuPanel). `metrics.llm[i]` is index-matched to `llmPorts[i]`. +- `SparkTabs.tsx` (@dnd-kit): the drag handle is intentionally outside the memoized tab label so `useSortable` listeners stay fresh (see comment near line 95); `App.tsx` keeps an `orderOverride` so reorders don't flicker against the next WS frame. +- The `@/` alias exists in both `vite.config.ts` and `tsconfig.json` but is unused — code uses relative imports; match that. + +## Conventions + +- Server: ESM, `.js` extensions on every import, `__dirname` via `fileURLToPath`, JSDoc for types. Client: strict TS, `import type` for type-only imports, `console.error` as the failure path (the toast system was removed in 1.8.0). +- Tests use `node:test` + `node:assert` with no mocking library: swap `globalThis.fetch` with a URL-matching stub (restore in `finally`), monkey-patch instance methods (`probe._run = async () => …`), or `Object.create(Class.prototype)` to call pure private methods without touching disk. +- Collectors must degrade gracefully — catch and return zero/default metrics rather than throwing out of a poll loop; sustained liveness failure (10 s grace) marks the spark offline. +- Honor the explicit "on purpose" comments when you meet them (no snapshot `timestamp`, gpu-memory.json fallback-only, SGLang/ds4 gauge rules, Hermes privilege drop, `ComfyPanel.comfyOpenUrl()` preferring `lanIp` over the dashboard origin). From d56b6ab6ab93c751cb48b80e9d46a827af736381 Mon Sep 17 00:00:00 2001 From: y1618 Date: Sun, 23 Aug 2026 11:51:52 +0900 Subject: [PATCH 3/3] feat(llm): LM Studio backend; stop probing its non-existent endpoints LM Studio answers every unknown path with HTTP 200 + {"error": ...} and logs each hit as an ERROR. The SGLang check treated any 200 JSON object as server info, so LM Studio was misdetected as sglang and every 2 s poll hit /get_server_info, /metrics, /get_model_info and /model_info; the Showcase rate poller added /metrics + /get_server_info every 400 ms. - New `lmstudio` backend, classified from `owned_by: organization_owner` before any network probe. `_probeLmStudio()` reads the loaded model and context from the native /api/v0/models list (fallback /v1/models) at most every 10 s and serves the cached snapshot in between; never touches /slots, /metrics or the SGLang info endpoints. Live tok/s stays 0 (no counters); decode bench and Showcase are unaffected. - `_looksLikeServerInfo()` rejects bare error envelopes so a 200 error body can no longer promote an unknown backend to sglang. - `pollServerGenerationRates()` gives up after three initial misses on counter-less backends (also quiets llama.cpp 404s during a showcase). - Client: `lmstudio` in LlmMetrics.backend, "LM Studio" badge/label. - Tests: LlmProbe.lmstudio.test.js (detection, forbidden paths, throttle, redetect, fallback, 401, disconnect, sglang false-positive, poller). - Docs: README probe section, CHANGELOG [Unreleased], CLAUDE.md. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01HgmJYczuwSYo7RGBMwvsTQ --- CHANGELOG.md | 10 + README.md | 3 +- server/collectors/LlmProbe.js | 152 +++++++++- server/collectors/LlmStreaming.js | 12 + .../__tests__/LlmProbe.lmstudio.test.js | 263 ++++++++++++++++++ src/api/types.ts | 2 +- src/components/OverviewPage/OverviewPage.tsx | 4 +- src/components/SparkPage/LlmPanel.tsx | 1 + 8 files changed, 431 insertions(+), 16 deletions(-) create mode 100644 server/collectors/__tests__/LlmProbe.lmstudio.test.js diff --git a/CHANGELOG.md b/CHANGELOG.md index f03714a..2271586 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ Format: version sections are listed newest first. --- +## [Unreleased] + +### Added +- **LM Studio backend** — OpenAI-compatible servers reporting `owned_by: organization_owner` are detected as `lmstudio` (badge **LM Studio**). Model + context come from LM Studio's native `/api/v0/models` (first *loaded* LLM/VLM, `loaded_context_length`; falls back to `/v1/models`), read at most every 10 s. No live tok/s (LM Studio has no counters); decode benchmark and Prompt Showcase are unaffected. + +### Fixed +- **LM Studio error-log spam / misdetection as SGLang** — LM Studio answers every unknown path with HTTP 200 + `{"error": …}`, which the SGLang check read as "server info", so each 2 s poll hit `/get_server_info`, `/metrics`, `/get_model_info`, `/model_info` (each logged as an ERROR in LM Studio's Developer Logs), and the Showcase rate poller hit `/metrics` + `/get_server_info` every 400 ms. The SGLang check now rejects bare error envelopes, LM Studio is classified from `owned_by` before any network probe, and the showcase poller stops after three initial misses on counter-less backends (also quiets llama.cpp 404s). + +--- + ## [1.8.0] — 2026-08-15 ### Added diff --git a/README.md b/README.md index 5605f4e..ccca07d 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,7 @@ Full history: [CHANGELOG.md](./CHANGELOG.md) | **Non-Spark GPU hosts** | Linux boxes with a dedicated NVIDIA GPU are first-class units: same `nvidia-smi` collectors over SSH, detected hardware summary, and separate **RAM** / **VRAM** panels. Detail page: GPU (left) + **RAM → Network → Storage** (right column); Overview cards show RAM and VRAM bars | | **Live streaming** | WebSocket metrics with configurable poll intervals; central history store for sparklines across tab switches | | **Local + remote** | Host metrics via sysfs/proc/`nvidia-smi`; remotes over SSH (key or password) | -| **LLM probe** | Auto-detects llama.cpp, vLLM, sglang, or ds4-server; live tok/s per server | +| **LLM probe** | Auto-detects llama.cpp, vLLM, sglang, ds4-server, or LM Studio; live tok/s per server (LM Studio: model + context only) | | **ComfyUI** | Opt-in probe: queue/jobs, progress, cancel, Open link, inventory, overview chip | | **Hermes Agent** | Opt-in per unit: background update check (10 min), status badges, one-click or batch `hermes update` | | **Decode benchmark** | Multi-concurrency streaming decode tok/s (server + per-stream), persisted last run | @@ -448,6 +448,7 @@ Each configured LLM port gets its own `LlmProbe` instance running in parallel. P - **llama.cpp** — `/slots` for live decode rates; model from `/props` - **ds4-server** (Entrpi/ds4-on-spark) — `/v1/models` (`owned_by: ds4.c`) + Prometheus `ds4_*` token counters for live tok/s +- **LM Studio** — `/v1/models` (`owned_by: organization_owner`); loaded model + context from the native `/api/v0/models` list, read every 10 s. LM Studio exposes no live token counters and answers every unknown path with `200 {"error"}` (logged as an ERROR on its side), so sparkDash never probes `/slots`, `/metrics`, or the SGLang info endpoints on it — live tok/s stays 0; the decode benchmark and Prompt Showcase measure client-side and work normally - **vLLM / sglang** — `/v1/models`; sglang via `/get_server_info` (`last_gen_throughput` when metrics off), vLLM via Prometheus `/metrics` counters (scientific notation supported) Rates are derived from per-probe cumulative counter diffs (or SGLang sticky throughput while it moves). Multiple ports can be added or removed at runtime without restarting the monitor. diff --git a/server/collectors/LlmProbe.js b/server/collectors/LlmProbe.js index 02ef237..f457f68 100644 --- a/server/collectors/LlmProbe.js +++ b/server/collectors/LlmProbe.js @@ -2,6 +2,10 @@ * LlmProbe — probes an LLM server on port 8888, auto-detects backend, * computes live tokens/sec (generation + prefill). * + * Backends: llama.cpp (native /slots), vLLM / SGLang / ds4-server + * (OpenAI-compatible + live counters), LM Studio (OpenAI-compatible, no live + * counters — model list only, polled slowly). + * * Ported from legacy `probeLlamaServerType` and `_getLlamaMetricsFor`. */ import { LLM_PROBE_TIMEOUT_MS } from "../config.js"; @@ -16,6 +20,17 @@ const REDETECT_INTERVAL_MS = 60_000; * expire back to 0 if it stops changing. */ const SGLANG_STICKY_TPS_LIVE_MS = 6_000; +/** + * LM Studio exposes no live token counters (no /metrics, /slots, /get_server_info) + * and answers *every* unknown path with HTTP 200 + {"error": ...}, logging each + * hit as an ERROR in its Developer Logs. Once detected, read only its model list, + * and only this often — there is nothing live to pick up between polls. + */ +const LMSTUDIO_POLL_INTERVAL_MS = 10_000; +/** `owned_by` that LM Studio's /v1/models reports for every model. */ +const LMSTUDIO_OWNED_BY = "organization_owner"; +/** Keys of a bare error envelope ({"error": "..."}) — not server info. */ +const ERROR_ENVELOPE_KEYS = new Set(["error", "message", "detail", "code", "status"]); /** * Prefer a short model id when the server returns a Hugging Face hub cache path. @@ -63,7 +78,7 @@ export class LlmProbe { this.baseUrl = `http://${llmProbeHost(spark)}:${port}`; // State - this.backendType = null; // 'vllm' | 'llama.cpp' | 'sglang' | 'ds4' | null + this.backendType = null; // 'vllm' | 'llama.cpp' | 'sglang' | 'ds4' | 'lmstudio' | null this.serverIsOpenAI = null; // true = OpenAI-compatible /** Whether /v1/models (or /slots) answered without credentials. null = unknown. */ this.authOpen = null; @@ -110,6 +125,8 @@ export class LlmProbe { this._lastDetectAt = 0; /** @type {{ value: number, liveUntil: number } | null} */ this._sglangStickyTps = null; + /** Last successful LM Studio model-list read (ms). 0 = never / reset. */ + this._lmStudioLastOkAt = 0; } /** @@ -156,7 +173,10 @@ export class LlmProbe { this._noteSuccess(); return snap; } else if (this.serverIsOpenAI === true) { - const snap = await this._probeOpenAICompatible(); + const snap = + this.backendType === "lmstudio" + ? await this._probeLmStudio() + : await this._probeOpenAICompatible(); this._noteSuccess(); return snap; } else { @@ -209,6 +229,7 @@ export class LlmProbe { this.lastTtftSum = null; this.lastIterSum = null; this._sglangStickyTps = null; + this._lmStudioLastOkAt = 0; } /** Note auth from an HTTP status on an unauthenticated probe request. */ @@ -227,14 +248,16 @@ export class LlmProbe { // ─── Server type detection ─────────────────────────────── async _detectServerType() { // Skip the llama.cpp /slots probe once we've positively identified an - // OpenAI-compatible backend. vLLM / sglang / ds4-server have no /slots, - // so re-probing it on every re-detect cycle just spams 404s in the - // backend's access log (#15). Still probe /slots on first contact, when - // the type is unknown, or when the backend was previously llama.cpp. + // OpenAI-compatible backend. vLLM / sglang / ds4-server / LM Studio have + // no /slots, so re-probing it on every re-detect cycle just spams 404s + // (or, for LM Studio, ERROR log lines) in the backend's log (#15). Still + // probe /slots on first contact, when the type is unknown, or when the + // backend was previously llama.cpp. if ( this.backendType !== "vllm" && this.backendType !== "sglang" && - this.backendType !== "ds4" + this.backendType !== "ds4" && + this.backendType !== "lmstudio" ) { const slotUrl = `${this.baseUrl}/slots`; try { @@ -281,12 +304,16 @@ export class LlmProbe { } /** - * Classify an OpenAI-compatible server: ds4-server, SGLang, or vLLM (default). + * Classify an OpenAI-compatible server: LM Studio, ds4-server, SGLang, or + * vLLM (default). LM Studio is decided from `owned_by` alone — it answers + * 200 to any path, so the ds4 / SGLang network probes would misfire on it + * (and every probe lands in its error log). * @param {unknown} ownedBy - * @returns {Promise<"ds4" | "sglang" | "vllm">} + * @returns {Promise<"lmstudio" | "ds4" | "sglang" | "vllm">} */ async _classifyOpenAIBackend(ownedBy) { if (typeof ownedBy === "string") { + if (LlmProbe._isLmStudioOwner(ownedBy)) return "lmstudio"; if (/ds4/i.test(ownedBy)) return "ds4"; if (/sglang/i.test(ownedBy)) return "sglang"; } @@ -295,14 +322,21 @@ export class LlmProbe { return "vllm"; } - /** True when SGLang native server-info endpoints respond. */ + /** @param {unknown} ownedBy */ + static _isLmStudioOwner(ownedBy) { + return ( + typeof ownedBy === "string" && ownedBy.trim().toLowerCase() === LMSTUDIO_OWNED_BY + ); + } + + /** True when SGLang native server-info endpoints respond with real info. */ async _probeIsSglang() { for (const path of ["/get_server_info", "/server_info"]) { try { const res = await this._fetch(`${this.baseUrl}${path}`); if (!res.ok) continue; const data = await res.json().catch(() => null); - if (data && typeof data === "object" && !Array.isArray(data)) return true; + if (LlmProbe._looksLikeServerInfo(data)) return true; } catch { /* try next */ } @@ -310,6 +344,19 @@ export class LlmProbe { return false; } + /** + * True for a JSON object that carries real server info — not a bare error + * envelope. Some servers (LM Studio) answer unknown paths with + * 200 + {"error": "..."}; that must not read as "SGLang answered". + * @param {unknown} data + */ + static _looksLikeServerInfo(data) { + if (!data || typeof data !== "object" || Array.isArray(data)) return false; + const keys = Object.keys(data); + if (!keys.length) return false; + return keys.some((k) => !ERROR_ENVELOPE_KEYS.has(k)); + } + /** True when Prometheus /metrics exposes ds4-server series (ds4-on-spark). */ async _probeIsDs4() { try { @@ -373,9 +420,13 @@ export class LlmProbe { try { const sgRes = await this._fetch(`${this.baseUrl}/get_server_info`); if (sgRes.ok) { - this.backendType = "sglang"; const sgData = await sgRes.json(); - this._applySglangServerInfo(sgData, dtSec); + // Only *promote* an unknown backend on real server info — a 200 + // error envelope (LM Studio-style) is not SGLang. + if (this.backendType === "sglang" || LlmProbe._looksLikeServerInfo(sgData)) { + this.backendType = "sglang"; + this._applySglangServerInfo(sgData, dtSec); + } } } catch {} } @@ -786,6 +837,81 @@ export class LlmProbe { } } + // ─── LM Studio path ─────────────────────────────────────── + /** + * LM Studio: OpenAI-compatible, but with no live counters anywhere, and it + * answers every unknown path with 200 + {"error"} and logs it as an ERROR. + * So: never touch /slots, /metrics, /get_server_info, /get_model_info here. + * Read the loaded model + context from its native REST list (/api/v0/models, + * falling back to /v1/models) at most once per LMSTUDIO_POLL_INTERVAL_MS and + * re-serve the last snapshot in between. Live tok/s stays 0 by design — the + * decode bench / showcase measure client-side. + */ + async _probeLmStudio() { + const now = Date.now(); + if (this._lmStudioLastOkAt && now - this._lmStudioLastOkAt < LMSTUDIO_POLL_INTERVAL_MS) { + return this._getSnapshot(); + } + this.lastProbeTime = now; + + let applied = false; + try { + const res = await this._fetch(`${this.baseUrl}/api/v0/models`); + const auth = this._noteAuthStatus(res.status); + if (auth === "auth") return this._getSnapshot(); + if (auth === "ok") { + const data = await res.json().catch(() => null); + const list = Array.isArray(data?.data) ? data.data : null; + // Shape check: the native list carries `state`; a 200 error envelope doesn't. + if (list && list.some((m) => m && typeof m === "object" && "state" in m)) { + this._applyLmStudioModels(list); + applied = true; + } + } + } catch { + /* fall back to the OpenAI list */ + } + + if (!applied) { + const res = await this._fetch(`${this.baseUrl}/v1/models`); + const auth = this._noteAuthStatus(res.status); + if (auth === "auth") return this._getSnapshot(); + if (auth !== "ok") throw new Error("LM Studio /v1/models unreachable"); + const data = await res.json(); + const model = data?.data?.[0]; + this.modelId = normalizeModelId(model?.id || null); + this.modelPath = null; + } + + // No live counters on this backend — never show a stale rate. + this.generationTps = 0; + this.prefillTps = 0; + this._lmStudioLastOkAt = Date.now(); + return this._getSnapshot(); + } + + /** + * Pick the model to show from LM Studio's native list: the first *loaded* + * LLM/VLM (what a request would hit), else the first LLM (JIT-loadable) so + * the bench / showcase still have a model id to send. Embedding models are + * never picked. + * @param {Array>} list + */ + _applyLmStudioModels(list) { + const isLlm = (m) => m?.type == null || m.type === "llm" || m.type === "vlm"; + const loaded = list.filter((m) => m && m.state === "loaded" && isLlm(m)); + const pick = loaded[0] ?? list.find((m) => m && isLlm(m)) ?? null; + this.modelId = normalizeModelId(pick?.id || null); + this.modelPath = null; + this.contextLength = + LlmProbe._positiveNumber(pick?.loaded_context_length) ?? + LlmProbe._positiveNumber(pick?.max_context_length) ?? + null; + // Running / queued request counts are not exposed by LM Studio. + this.slotsActive = 0; + this.slotsTotal = 0; + } + // ─── llama.cpp native path ──────────────────────────────── async _probeLlamaCpp() { const now = Date.now(); diff --git a/server/collectors/LlmStreaming.js b/server/collectors/LlmStreaming.js index 11d14e2..4494c74 100644 --- a/server/collectors/LlmStreaming.js +++ b/server/collectors/LlmStreaming.js @@ -11,6 +11,12 @@ const DEBUG_HEADER_RE = /** Truncate streamed content previews stored for debugging. */ export const CONTENT_PREVIEW_CHARS = 160; +/** + * pollServerGenerationRates: stop after this many *initial* null reads. + * Backends without counters (llama.cpp, LM Studio, …) answer null forever — + * and LM Studio logs every /metrics + /get_server_info miss as an ERROR. + */ +const SERVER_RATE_INITIAL_MISS_LIMIT = 3; export function round2(n) { return Math.round(n * 100) / 100; @@ -274,8 +280,12 @@ export async function pollServerGenerationRates( let lastTokens = await readServerGenerationTokens(baseUrl, { apiKey: apiKey || null }); let lastT = performance.now(); const onSample = typeof opts.onSample === "function" ? opts.onSample : null; + // Give up on backends that never answer with counters instead of hammering + // them every `intervalMs` for the whole session (see SERVER_RATE_INITIAL_MISS_LIMIT). + let initialMisses = lastTokens == null ? 1 : 0; while (!signal.aborted) { + if (initialMisses >= SERVER_RATE_INITIAL_MISS_LIMIT) break; try { await sleep(intervalMs, signal); } catch { @@ -287,6 +297,8 @@ export async function pollServerGenerationRates( if (tokens != null) { lastTokens = tokens; lastT = now; + } else if (lastTokens == null) { + initialMisses += 1; } continue; } diff --git a/server/collectors/__tests__/LlmProbe.lmstudio.test.js b/server/collectors/__tests__/LlmProbe.lmstudio.test.js new file mode 100644 index 0000000..40a9981 --- /dev/null +++ b/server/collectors/__tests__/LlmProbe.lmstudio.test.js @@ -0,0 +1,263 @@ +/** + * LM Studio (OpenAI-compatible, no live counters). + * + * Real behavior (LM Studio 0.4.21): /v1/models reports `owned_by: + * "organization_owner"`, the native list lives at /api/v0/models (with + * `state`, `max_context_length`, `loaded_context_length`), and EVERY unknown + * path answers HTTP 200 + {"error": "Unexpected endpoint or method. (...)"} — + * which LM Studio logs as an ERROR. Before this backend existed, sparkDash + * read that 200 as "SGLang answered" and hit /get_server_info, /metrics, + * /get_model_info and /model_info every 2 s (plus /metrics + /get_server_info + * every 400 ms during a showcase). These tests lock in: detect from owned_by, + * never touch those paths, poll the model list slowly. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import { LlmProbe } from "../LlmProbe.js"; +import { pollServerGenerationRates } from "../LlmStreaming.js"; + +function jsonRes(data, status = 200) { + return { + ok: status >= 200 && status < 300, + status, + json: async () => data, + text: async () => JSON.stringify(data), + }; +} + +const OPENAI_MODELS = { + data: [ + { id: "ornith-1.5-35b-a3b@q5_k_m", object: "model", owned_by: "organization_owner" }, + { id: "qwen/qwen3.6-27b", object: "model", owned_by: "organization_owner" }, + ], + object: "list", +}; + +const V0_MODELS = { + data: [ + { + id: "text-embedding-nomic-embed-text-v1.5", + object: "model", + type: "embeddings", + state: "loaded", + max_context_length: 2048, + }, + { + id: "ornith-1.5-35b-a3b@q4_k_m", + object: "model", + type: "vlm", + state: "not-loaded", + max_context_length: 262144, + }, + { + id: "ornith-1.5-35b-a3b@q5_k_m", + object: "model", + type: "vlm", + state: "loaded", + max_context_length: 262144, + loaded_context_length: 131072, + }, + ], + object: "list", +}; + +const NEVER = ["/metrics", "/get_server_info", "/server_info", "/get_model_info", "/model_info"]; + +/** LM Studio-shaped fetch stub: unknown paths → 200 + error envelope. */ +function lmStudioFetch(hits, overrides = {}) { + return async (url) => { + const p = String(url).replace(/^https?:\/\/[^/]+/, ""); + hits.push(p); + if (p in overrides) return overrides[p]; + if (p === "/v1/models") return jsonRes(OPENAI_MODELS); + if (p === "/api/v0/models") return jsonRes(V0_MODELS); + return jsonRes({ error: `Unexpected endpoint or method. (GET ${p})` }); + }; +} + +test("detect: owned_by organization_owner → lmstudio with no ds4/sglang probes", async () => { + const probe = new LlmProbe({ lanIp: "192.168.50.76" }, 1234); + const hits = []; + probe._fetch = lmStudioFetch(hits); + await probe._detectServerType(); + assert.equal(probe.serverIsOpenAI, true); + assert.equal(probe.backendType, "lmstudio"); + for (const p of NEVER) assert.ok(!hits.includes(p), `must not probe ${p}`); +}); + +test("probe(): loaded LLM + loaded context from /api/v0/models; sglang/metrics paths never hit", async () => { + const probe = new LlmProbe({ lanIp: "192.168.50.76" }, 1234); + const hits = []; + probe._fetch = lmStudioFetch(hits); + + const snap = await probe.probe(); + assert.equal(snap.available, true); + assert.equal(snap.backend, "lmstudio"); + // First *loaded* LLM/VLM — not the loaded embedding model, not the unloaded q4. + assert.equal(snap.modelId, "ornith-1.5-35b-a3b@q5_k_m"); + assert.equal(snap.contextLength, 131072); + assert.equal(snap.generationTps, 0); + assert.equal(snap.prefillTps, 0); + assert.equal(snap.error, null); + + for (const p of NEVER) assert.ok(!hits.includes(p), `must not hit ${p}`); + // /slots is allowed once on first contact (type unknown), never again. + assert.ok(hits.filter((h) => h === "/slots").length <= 1); + assert.ok(hits.includes("/api/v0/models")); + + // Throttled: polls inside the window make no HTTP calls at all. + const before = hits.length; + await probe.probe(); + await probe.probe(); + assert.equal(hits.length, before, "no HTTP while inside the LM Studio poll window"); + assert.equal((await probe.probe()).modelId, "ornith-1.5-35b-a3b@q5_k_m"); + + // Window elapsed → exactly one model-list read. + probe._lmStudioLastOkAt = Date.now() - 60_000; + await probe.probe(); + assert.deepEqual(hits.slice(before), ["/api/v0/models"]); + + // Periodic re-detect (60 s) re-reads /v1/models only — no /slots, no probes. + const beforeRedetect = hits.length; + probe._lastDetectAt = 0; + probe._lmStudioLastOkAt = Date.now(); + await probe.probe(); + assert.deepEqual(hits.slice(beforeRedetect), ["/v1/models"]); + assert.equal(probe.backendType, "lmstudio"); +}); + +test("probe(): falls back to /v1/models when /api/v0/models is not the native list", async () => { + const probe = new LlmProbe({ lanIp: "192.168.50.76" }, 1234); + const hits = []; + probe._fetch = lmStudioFetch(hits, { + "/api/v0/models": jsonRes({ error: "Unexpected endpoint or method. (GET /api/v0/models)" }), + }); + const snap = await probe.probe(); + assert.equal(snap.backend, "lmstudio"); + assert.equal(snap.modelId, "ornith-1.5-35b-a3b@q5_k_m"); + assert.equal(snap.available, true); + for (const p of NEVER) assert.ok(!hits.includes(p), `must not hit ${p}`); +}); + +test("probe(): nothing loaded → first LLM id (JIT-loadable), never an embedding model", async () => { + const probe = new LlmProbe({ lanIp: "192.168.50.76" }, 1234); + const hits = []; + probe._fetch = lmStudioFetch(hits, { + "/api/v0/models": jsonRes({ + data: [ + { id: "text-embedding-nomic-embed-text-v1.5", type: "embeddings", state: "not-loaded" }, + { id: "qwen/qwen3.6-27b", type: "llm", state: "not-loaded", max_context_length: 40960 }, + ], + }), + }); + const snap = await probe.probe(); + assert.equal(snap.modelId, "qwen/qwen3.6-27b"); + assert.equal(snap.contextLength, 40960); +}); + +test("probe(): 401 on the model list → protected posture, not available", async () => { + const probe = new LlmProbe({ lanIp: "192.168.50.76" }, 1234); + probe.serverIsOpenAI = true; + probe.backendType = "lmstudio"; + probe._lastDetectAt = Date.now(); + probe._fetch = async () => jsonRes({ error: "unauthorized" }, 401); + const snap = await probe.probe(); + assert.equal(snap.available, false); + assert.equal(snap.posture?.auth, "protected"); +}); + +test("probe(): server gone → failure path (not the cached snapshot forever)", async () => { + const probe = new LlmProbe({ lanIp: "192.168.50.76" }, 1234); + const hits = []; + probe._fetch = lmStudioFetch(hits); + await probe.probe(); + probe._lmStudioLastOkAt = Date.now() - 60_000; + probe._fetch = async () => { + throw new Error("ECONNREFUSED"); + }; + const snap = await probe.probe(); + assert.equal(snap.available, false); + assert.match(String(snap.error), /ECONNREFUSED/); +}); + +test("_probeIsSglang: a 200 error envelope is not SGLang", async () => { + const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 30000); + probe._fetch = async (url) => + jsonRes({ error: `Unexpected endpoint or method. (GET ${String(url)})` }); + assert.equal(await probe._probeIsSglang(), false); + // Real server info still counts. + probe._fetch = async () => jsonRes({ version: "0.5.0", model_path: "org/model" }); + assert.equal(await probe._probeIsSglang(), true); +}); + +test("_looksLikeServerInfo: envelopes and empties are rejected, real info accepted", () => { + assert.equal(LlmProbe._looksLikeServerInfo({ error: "x" }), false); + assert.equal(LlmProbe._looksLikeServerInfo({ error: "x", message: "y" }), false); + assert.equal(LlmProbe._looksLikeServerInfo({}), false); + assert.equal(LlmProbe._looksLikeServerInfo([]), false); + assert.equal(LlmProbe._looksLikeServerInfo(null), false); + assert.equal(LlmProbe._looksLikeServerInfo({ version: "0.5.0" }), true); + assert.equal(LlmProbe._looksLikeServerInfo({ model_path: "org/m", error: null }), true); +}); + +test("unknown OpenAI backend + 200 error envelope on /get_server_info must not become sglang", async () => { + const probe = new LlmProbe({ lanIp: "10.0.0.1" }, 8000); + probe.serverIsOpenAI = true; + probe.backendType = null; + probe._lastDetectAt = Date.now(); + probe._fetch = async (url) => { + const p = String(url).replace(/^https?:\/\/[^/]+/, ""); + if (p === "/v1/models") return jsonRes({ data: [{ id: "some/model", owned_by: "acme" }] }); + return jsonRes({ error: `Unexpected endpoint or method. (GET ${p})` }); + }; + const snap = await probe.probe(); + assert.notEqual(snap.backend, "sglang"); +}); + +// ─── Showcase server-rate poller ───────────────────────────── + +test("pollServerGenerationRates: gives up after a few initial misses on a counter-less backend", async () => { + const calls = []; + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const p = String(url).replace(/^https?:\/\/[^/]+/, ""); + calls.push(p); + return jsonRes({ error: `Unexpected endpoint or method. (GET ${p})` }); + }; + try { + const ac = new AbortController(); + const safety = setTimeout(() => ac.abort(), 3_000); + const res = await pollServerGenerationRates("http://192.168.50.76:1234", ac.signal, 5); + clearTimeout(safety); + assert.equal(ac.signal.aborted, false, "returned on its own, not via the safety abort"); + assert.equal(res.samples, 0); + assert.equal(res.median, null); + // 3 reads × (/metrics + /get_server_info) — then silence. + assert.equal(calls.length, 6, `expected 6 requests, got ${calls.length}: ${calls.join(",")}`); + } finally { + globalThis.fetch = realFetch; + } +}); + +test("pollServerGenerationRates: keeps sampling when counters exist (vLLM regression)", async () => { + let gen = 1000; + const realFetch = globalThis.fetch; + globalThis.fetch = async (url) => { + const p = String(url).replace(/^https?:\/\/[^/]+/, ""); + if (p === "/metrics") { + gen += 50; + const txt = `vllm:generation_tokens_total{engine="0"} ${gen}.0\n`; + return { ok: true, status: 200, text: async () => txt, json: async () => ({}) }; + } + return jsonRes({}, 404); + }; + try { + const ac = new AbortController(); + setTimeout(() => ac.abort(), 120); + const res = await pollServerGenerationRates("http://10.0.0.1:8000", ac.signal, 5); + assert.ok(res.samples >= 1, `expected samples, got ${res.samples}`); + assert.ok(res.median > 0); + } finally { + globalThis.fetch = realFetch; + } +}); diff --git a/src/api/types.ts b/src/api/types.ts index 40f29a7..d5742f0 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -277,7 +277,7 @@ export interface UnifiedMemoryMetrics { // ─── LLM metrics ───────────────────────────────────────── export interface LlmMetrics { available: boolean; - backend: "vllm" | "llama.cpp" | "sglang" | "ds4" | null; + backend: "vllm" | "llama.cpp" | "sglang" | "ds4" | "lmstudio" | null; modelId: string | null; modelPath: string | null; contextLength: number | null; diff --git a/src/components/OverviewPage/OverviewPage.tsx b/src/components/OverviewPage/OverviewPage.tsx index 0eed217..135c55c 100644 --- a/src/components/OverviewPage/OverviewPage.tsx +++ b/src/components/OverviewPage/OverviewPage.tsx @@ -315,7 +315,9 @@ function SparkCard({ ? "ds4" : llm.backend === "sglang" ? "sgLang" - : llm.backend ?? "LLM" + : llm.backend === "lmstudio" + ? "LM Studio" + : llm.backend ?? "LLM" } value={llm.modelId ?? "unknown"} tone="accent" diff --git a/src/components/SparkPage/LlmPanel.tsx b/src/components/SparkPage/LlmPanel.tsx index 237d007..9b9e90c 100644 --- a/src/components/SparkPage/LlmPanel.tsx +++ b/src/components/SparkPage/LlmPanel.tsx @@ -45,6 +45,7 @@ function BackendBadge({ backend }: { backend: string | null }) { "llama.cpp": "llama.cpp", sglang: "sgLang", ds4: "ds4", + lmstudio: "LM Studio", }; return (