From afd7493dc7607e2e6fb2da945e174addb7840836 Mon Sep 17 00:00:00 2001 From: yr75myjnqx-tech Date: Thu, 6 Aug 2026 17:10:09 +1000 Subject: [PATCH 1/7] feat: collect nvErrNoMemory from local kernel journal --- server/collectors/SystemCollector.js | 17 +++++++++ .../__tests__/SystemCollector.nvErr.test.js | 38 +++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100644 server/collectors/__tests__/SystemCollector.nvErr.test.js diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index 15edcbe..ea3bcf2 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -897,6 +897,21 @@ export class SystemCollector { } } catch {} + // NV_ERR_NO_MEMORY kernel error count from the NVRM driver (host journal). + let nvErrNoMemory = 0; + try { + const js = "journalctl -k --no-pager 2>/dev/null | grep -c \"NV_ERR_NO_MEMORY\" || true"; + const out = this._hasHostProc() ? await this._execOnHost(js) : await this._exec(js); + const n = Number(String(out).trim()); + if (!Number.isNaN(n) && n > 0) nvErrNoMemory = Math.round(n); + } catch { + try { + const out = await this._exec("dmesg 2>/dev/null | grep -c \"NV_ERR_NO_MEMORY\" || true"); + const n = Number(String(out).trim()); + if (!Number.isNaN(n) && n > 0) nvErrNoMemory = Math.round(n); + } catch {} + } + return { total: totalMB, gpuUsed: gpuUsedMB, @@ -906,6 +921,7 @@ export class SystemCollector { percentage, oomRisk, bandwidth, + nvErrNoMemory, }; } @@ -1411,6 +1427,7 @@ export class SystemCollector { percentage: 0, oomRisk: "low", bandwidth: { current: 0, peak: 0 }, + nvErrNoMemory: 0, }; } diff --git a/server/collectors/__tests__/SystemCollector.nvErr.test.js b/server/collectors/__tests__/SystemCollector.nvErr.test.js new file mode 100644 index 0000000..0bdbe0a --- /dev/null +++ b/server/collectors/__tests__/SystemCollector.nvErr.test.js @@ -0,0 +1,38 @@ +import { test, mock } from "node:test"; +import { strict as assert } from "node:assert"; +import { SystemCollector } from "../SystemCollector.js"; + +const MEMINFO = + "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 2000000 kB\n"; + +function makeLocalCollector() { + const c = new SystemCollector({ id: "gx10", isLocal: true }); + mock.method(c, "_readHostFile", async () => MEMINFO); + mock.method(c, "_parseMemTotal", (raw) => { + const m = raw.match(/MemTotal:\s+(\d+)\s+kB/); + return m ? parseInt(m[1], 10) : 0; + }); + mock.method(c, "_readGpuMemoryFile", () => 0); + mock.method(c, "_nvidiaSmi", async () => ""); + mock.method(c, "_hasHostProc", () => false); + mock.method(c, "_exec", async () => "12"); + mock.method(c, "_execOnHost", async () => "12"); + return c; +} + +test("nvErrNoMemory: local collection returns the journal count", async () => { + const c = makeLocalCollector(); + const result = await c._getUnifiedMemory(); + assert.equal(typeof result.nvErrNoMemory, "number"); + assert.equal(result.nvErrNoMemory, 12); +}); + +test("nvErrNoMemory: defaults to 0 when the journal command fails", async () => { + const c = makeLocalCollector(); + mock.method(c, "_hasHostProc", () => false); + mock.method(c, "_exec", async () => { + throw new Error("no journal"); + }); + const result = await c._getUnifiedMemory(); + assert.equal(result.nvErrNoMemory, 0); +}); From afe0e5d2a4a3fe28f4f27ecf2ebef4e45dd901ff Mon Sep 17 00:00:00 2001 From: yr75myjnqx-tech Date: Thu, 6 Aug 2026 17:24:48 +1000 Subject: [PATCH 2/7] feat: collect nvErrNoMemory over SSH for remote sparks --- package.json | 2 +- server/collectors/SystemCollector.js | 12 +++++++ .../__tests__/SystemCollector.nvErr.test.js | 31 ++++++++++++++++++- 3 files changed, 43 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index f946648..062b35b 100644 --- a/package.json +++ b/package.json @@ -9,7 +9,7 @@ "dev:client": "vite", "build": "vite build", "typecheck": "tsc --noEmit", - "test": "node --test server/collectors/__tests__/*.test.js server/sparks/__tests__/*.test.js", + "test": "node --experimental-test-module-mocks --test server/collectors/__tests__/*.test.js server/sparks/__tests__/*.test.js", "preview": "vite preview", "start": "node server/index.js", "docker:up": "docker compose up -d", diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index ea3bcf2..a3dddae 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -1264,6 +1264,17 @@ export class SystemCollector { const percentage = totalMB > 0 ? Math.round((usedMB / totalMB) * 100) : 0; const oomRisk = percentage > 85 ? "high" : percentage > 60 ? "medium" : "low"; + // NV_ERR_NO_MEMORY kernel error count from the NVRM driver (remote host). + let nvErrNoMemory = 0; + try { + const countRaw = await sshExec( + this.spark, + "journalctl -k --no-pager 2>/dev/null | grep -c \"NV_ERR_NO_MEMORY\" || true" + ); + const n = Number(String(countRaw).trim()); + if (!Number.isNaN(n) && n > 0) nvErrNoMemory = Math.round(n); + } catch {} + return { total: totalMB, gpuUsed: gpuUsedMB, @@ -1273,6 +1284,7 @@ export class SystemCollector { percentage, oomRisk, bandwidth: { current: 0, peak: 400 }, + nvErrNoMemory, }; } catch (err) { console.error(`[SystemCollector] Remote Unified Memory error for ${this.spark.id}:`, err.message); diff --git a/server/collectors/__tests__/SystemCollector.nvErr.test.js b/server/collectors/__tests__/SystemCollector.nvErr.test.js index 0bdbe0a..f5bced2 100644 --- a/server/collectors/__tests__/SystemCollector.nvErr.test.js +++ b/server/collectors/__tests__/SystemCollector.nvErr.test.js @@ -1,6 +1,24 @@ import { test, mock } from "node:test"; import { strict as assert } from "node:assert"; -import { SystemCollector } from "../SystemCollector.js"; + +// The remote unified-memory path calls the module-level sshExec() imported by +// SystemCollector.js. ESM namespace bindings are non-configurable, so we mock +// the whole ssh.js module via mock.module() (requires --experimental-test-module-mocks). +let journalCount = 0; +mock.module("../ssh.js", { + exports: { + sshExec: async (_spark, cmd) => { + if (cmd.includes("journalctl")) { + journalCount += 1; + return "43"; + } + return "MemTotal: 16000000 kB\nMemAvailable: 4000000 kB\n---\n"; + }, + }, +}); + +// Import after registering the mock so SystemCollector picks up the stubbed sshExec. +const { SystemCollector } = await import("../SystemCollector.js"); const MEMINFO = "MemTotal: 8000000 kB\nMemFree: 1000000 kB\nMemAvailable: 2000000 kB\n"; @@ -36,3 +54,14 @@ test("nvErrNoMemory: defaults to 0 when the journal command fails", async () => const result = await c._getUnifiedMemory(); assert.equal(result.nvErrNoMemory, 0); }); + +test("nvErrNoMemory: remote collection includes the count over SSH", async () => { + journalCount = 0; + + const c = new SystemCollector({ id: "gx11", isLocal: false, lanIp: "10.200.0.2" }); + const result = await c._getRemoteUnifiedMemory(); + + assert.ok(journalCount === 1, "journalctl count command should be issued once"); + assert.equal(typeof result.nvErrNoMemory, "number"); + assert.equal(result.nvErrNoMemory, 43); +}); From e90ed0c631c2f76a977e4a10b34e0fff36fdd7f9 Mon Sep 17 00:00:00 2001 From: yr75myjnqx-tech Date: Thu, 6 Aug 2026 17:36:51 +1000 Subject: [PATCH 3/7] feat: type nvErrNoMemory on UnifiedMemoryMetrics --- src/api/types.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/api/types.ts b/src/api/types.ts index e44b219..c80198e 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -175,6 +175,8 @@ export interface UnifiedMemoryMetrics { used: number; available: number; percentage: number; + /** Cumulative count of NV_ERR_NO_MEMORY kernel driver errors. */ + nvErrNoMemory: number; oomRisk: "low" | "medium" | "high"; bandwidth: { current: number; From 1272dcd5270519871d8084638177a57fb5ddf52b Mon Sep 17 00:00:00 2001 From: yr75myjnqx-tech Date: Thu, 6 Aug 2026 18:34:11 +1000 Subject: [PATCH 4/7] feat: show NV_ERR_NO_MEMORY counter with reset baseline in CPU panel --- src/components/SparkPage/CpuPanel.tsx | 58 +++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/components/SparkPage/CpuPanel.tsx b/src/components/SparkPage/CpuPanel.tsx index 9eb2922..8057999 100644 --- a/src/components/SparkPage/CpuPanel.tsx +++ b/src/components/SparkPage/CpuPanel.tsx @@ -1,3 +1,4 @@ +import { useState, useEffect } from "react"; import type { CpuMetrics, RamMetrics, UnifiedMemoryMetrics } from "../../api/types"; import { Sparkline } from "../ui/Sparkline"; import { Panel } from "../ui/Panel"; @@ -17,6 +18,25 @@ function formatMb(mb: number): string { return `${Math.round(mb)} MB`; } +const NV_ERR_STORAGE_KEY = "nvErrBaseline"; + +function getNvErrBaseline(sparkId: string): number { + try { + const raw = localStorage.getItem(`${NV_ERR_STORAGE_KEY}.${sparkId}`); + return raw ? parseInt(raw, 10) || 0 : 0; + } catch { + return 0; + } +} + +function setNvErrBaseline(sparkId: string, value: number) { + try { + localStorage.setItem(`${NV_ERR_STORAGE_KEY}.${sparkId}`, String(value)); + } catch { + /* localStorage unavailable */ + } +} + function MetricRow({ label, spark, @@ -51,6 +71,22 @@ export function CpuPanel({ cpu, ram, sparkId, unifiedMemory }: CpuPanelProps) { const ramPct = ram?.percentage ?? 0; const ramAvail = ramTotal > 0 ? ramTotal - ramUsed : 0; + const nvErrRaw = unifiedMemory?.nvErrNoMemory ?? 0; + const [nvErrBaseline, setNvErrBaselineState] = useState(() => + getNvErrBaseline(sparkId) + ); + const [nvErrSinceReset, setNvErrSinceReset] = useState(0); + + useEffect(() => { + setNvErrSinceReset(Math.max(0, nvErrRaw - nvErrBaseline)); + }, [nvErrRaw, nvErrBaseline]); + + const handleResetNvErr = () => { + setNvErrBaseline(sparkId, nvErrRaw); + setNvErrBaselineState(nvErrRaw); + setNvErrSinceReset(0); + }; + return ( } className="panel-cpu" bodyClassName="space-y-3" accent> )} + {unifiedMemory !== null && ( +
+ NV_ERR_NO_MEMORY +
+ 0 ? "text-danger font-semibold" : "text-text" + }`} + > + {nvErrSinceReset} + + +
+
+ )} )} From 971107528663a7c4a3e111b85b7f88fecbc2ca22 Mon Sep 17 00:00:00 2001 From: yr75myjnqx-tech Date: Thu, 6 Aug 2026 18:42:26 +1000 Subject: [PATCH 5/7] fix: remount CpuPanel per spark (cross-spark nvErr baseline); align dmesg fallback to host journal; test remote default-0 --- server/collectors/SystemCollector.js | 3 ++- .../__tests__/SystemCollector.nvErr.test.js | 12 ++++++++++++ src/components/SparkPage/SparkPage.tsx | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index a3dddae..51d80fb 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -906,7 +906,8 @@ export class SystemCollector { if (!Number.isNaN(n) && n > 0) nvErrNoMemory = Math.round(n); } catch { try { - const out = await this._exec("dmesg 2>/dev/null | grep -c \"NV_ERR_NO_MEMORY\" || true"); + const dmesg = "dmesg 2>/dev/null | grep -c \"NV_ERR_NO_MEMORY\" || true"; + const out = this._hasHostProc() ? await this._execOnHost(dmesg) : await this._exec(dmesg); const n = Number(String(out).trim()); if (!Number.isNaN(n) && n > 0) nvErrNoMemory = Math.round(n); } catch {} diff --git a/server/collectors/__tests__/SystemCollector.nvErr.test.js b/server/collectors/__tests__/SystemCollector.nvErr.test.js index f5bced2..b915bbb 100644 --- a/server/collectors/__tests__/SystemCollector.nvErr.test.js +++ b/server/collectors/__tests__/SystemCollector.nvErr.test.js @@ -5,9 +5,11 @@ import { strict as assert } from "node:assert"; // SystemCollector.js. ESM namespace bindings are non-configurable, so we mock // the whole ssh.js module via mock.module() (requires --experimental-test-module-mocks). let journalCount = 0; +let failSsh = false; mock.module("../ssh.js", { exports: { sshExec: async (_spark, cmd) => { + if (failSsh) throw new Error("ssh exec failed"); if (cmd.includes("journalctl")) { journalCount += 1; return "43"; @@ -65,3 +67,13 @@ test("nvErrNoMemory: remote collection includes the count over SSH", async () => assert.equal(typeof result.nvErrNoMemory, "number"); assert.equal(result.nvErrNoMemory, 43); }); + +test("nvErrNoMemory: defaults to 0 when the remote SSH journal call throws", async () => { + failSsh = true; + + const c = new SystemCollector({ id: "gx11", isLocal: false, lanIp: "10.200.0.2" }); + const result = await c._getRemoteUnifiedMemory(); + + failSsh = false; + assert.equal(result.nvErrNoMemory, 0); +}); diff --git a/src/components/SparkPage/SparkPage.tsx b/src/components/SparkPage/SparkPage.tsx index dc6bc55..b482509 100644 --- a/src/components/SparkPage/SparkPage.tsx +++ b/src/components/SparkPage/SparkPage.tsx @@ -96,7 +96,7 @@ export function SparkPage({ spark, temperatureUnit, onEdit }: SparkPageProps) {
- + Date: Sun, 9 Aug 2026 17:09:48 +1000 Subject: [PATCH 6/7] feat: fleet storage tiers and model placement view Adds a Storage view that classifies each Spark's mounted disks into hot/warm/cold tiers (root NVMe vs other local disks vs NAS cifs/nfs mounts), scans configured model directories for weight files over a new models metrics domain, and shows both resident (local copy) and CX7/ConnectX-fabric model placement. Per-Spark modelDirs/tierPaths can override the heuristics in config/sparks.json. --- CHANGELOG.md | 11 + README.md | 8 + server/collectors/SystemCollector.js | 191 ++++++++++++++++- .../__tests__/collectModels.test.js | 75 +++++++ .../collectors/__tests__/tierClassify.test.js | 49 +++++ server/config.js | 17 ++ server/sparks/SparkMonitor.js | 11 + server/sparks/SparkRegistry.js | 28 +++ src/App.tsx | 9 +- src/api/types.ts | 28 +++ .../FleetStoragePage/FleetStoragePage.tsx | 195 ++++++++++++++++++ .../FleetStoragePage/fleetPlacement.ts | 81 ++++++++ src/components/SparkPage/StoragePanel.tsx | 19 ++ src/components/SparkTabs.tsx | 37 +++- src/constants.ts | 5 +- src/hooks/useRoute.ts | 25 ++- 16 files changed, 772 insertions(+), 17 deletions(-) create mode 100644 server/collectors/__tests__/collectModels.test.js create mode 100644 server/collectors/__tests__/tierClassify.test.js create mode 100644 src/components/FleetStoragePage/FleetStoragePage.tsx create mode 100644 src/components/FleetStoragePage/fleetPlacement.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 5c6aaf4..8483f27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,17 @@ Format: version sections are listed newest first. --- +## [Unreleased] + +### Added +- **Fleet model storage tiers** — a dedicated **Storage** view (fleet-wide) plus per-Spark storage cards and a model-placement table +- **Tier classification** — each mounted disk is labeled **Hot** (root NVMe), **Warm** (other local disks), or **Cold** (NAS `/mnt/modelshelf` and `cifs`/`smb`/`nfs` mounts), driven by `tierPaths` overrides, filesystem type, and mount prefix +- **Model inventory** — scans configured model directories (`.safetensors` / `.gguf` / `.bin` / `.pt` / `.pth` / `.ckpt` weight files) per tier, local and over SSH, with a new `models` metrics domain (`POLL_INTERVAL_MODELS`, default 30 s) +- **Dual placement** — a model is shown as **resident** on the Sparks holding a local copy, and as served **over the CX7/ConnectX fabric** on a peer that has it loaded via its LLM probe without a local copy +- Optional per-Spark `modelDirs` / `tierPaths` in `config/sparks.json` to override which directories map to which tier + +--- + ## [1.5.0] — 2026-08-03 ### Added diff --git a/README.md b/README.md index f85a49b..1851981 100644 --- a/README.md +++ b/README.md @@ -76,6 +76,7 @@ Full history: [CHANGELOG.md](./CHANGELOG.md) | **Secrets** | SSH passwords AES-256-GCM encrypted; never in `sparks.json` or API responses | | **Docker-first** | Single privileged container for host metrics; prod and dev Compose files | | **Hot config** | Add / edit / remove / reorder Sparks from the UI with no process restart | +| **Storage tiers** | Fleet **Storage** view — disks classified **Hot / Warm / Cold** (NAS/nfs/cifs vs local), model inventory, and resident-vs-CX7-fabric placement | --- @@ -228,6 +229,7 @@ Copy `.env.example` to `.env` if needed: | `POLL_INTERVAL_LLM` | `2000` | LLM probe poll (ms) | | `POLL_INTERVAL_BANDWIDTH` | `2000` | Memory bandwidth / dmon poll (ms) | | `POLL_INTERVAL_LIVENESS` | `5000` | Online/SSH liveness check (ms) | +| `POLL_INTERVAL_MODELS` | `30000` | Model inventory / tier scan (ms) | | `SPARKDASH_SECRETS_KEY` | _(auto)_ | Passphrase or 64-char hex for secret encryption | | `HOST_PROC_PATH` | `/host/proc` | Host proc mount inside container | | `HOST_SYS_PATH` | `/host/sys` | Host sys mount | @@ -323,6 +325,12 @@ Each configured LLM port gets its own `LlmProbe` instance running in parallel. P Rates are derived from per-probe cumulative counter diffs (or SGLang sticky throughput while it moves). Multiple ports can be added or removed at runtime without restarting the monitor. +### Storage tiers & model placement + +Each mounted disk is assigned a tier — **Hot** (root NVMe), **Warm** (other local disks), or **Cold** (NAS: `/mnt/modelshelf`, `/media`, `/Volumes`, `/mnt`, or any `cifs`/`smb`/`nfs` mount). Per-Spark `tierPaths` in `config/sparks.json` override the heuristic per mount. The collector scans the configured model directories (grouped by tier via `modelDirs`) for weight files (`.safetensors`, `.gguf`, `.bin`, `.pt`, `.pth`, `.ckpt`) and reports each model by name, size, and tier. + +A model is **resident** on the Sparks holding a local copy. Because a peer with the model loaded in its LLM probe (matched by name) serves it to the fleet over the CX7/ConnectX fabric, such a Spark is shown as placing the model **over the fabric** even with no local copy. The fleet **Storage** view rolls these up per tier and lists every model with its resident and fabric placement. + --- ## Contributing diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index 51d80fb..a067456 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -1,9 +1,72 @@ import fs from "fs"; import path from "path"; -import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS } from "../config.js"; +import { + HOST_PATHS, + GPU_MEMORY_JSON_PATH, + DGX_SPARK, + HARDWARE_DEFAULTS, + TIER_DEFAULTS, + WEIGHT_EXTENSIONS, +} from "../config.js"; import { normalizeMac, WOL_INTERFACE } from "../wol.js"; import { sshExec } from "./ssh.js"; +/** + * Classify a storage device into a tier: "hot" | "warm" | "cold". + * Pure (no `this`) so it is unit-testable in isolation. + * + * @param {object} params + * @param {string} params.device lsblk/df device name (e.g. "nvme0n1p2", "sdb1") + * @param {string} params.mount mount path (host-style, e.g. "/", "/mnt/modelshelf") + * @param {string} [params.fstype] filesystem type (may be "") + * @param {object} [params.tierPaths] per-Spark override { hot?, warm?, cold?: string[] } + * @returns {"hot"|"warm"|"cold"} + */ +export function classifyTier(device, mount, fstype = "", tierPaths = {}) { + const overrides = tierPaths || {}; + // Explicit per-Spark overrides win — match by mount-prefix or device name. + for (const tier of ["hot", "warm", "cold"]) { + const paths = overrides[tier]; + if (Array.isArray(paths)) { + for (const p of paths) { + if (!p) continue; + if (mount === p || mount.startsWith(p + "/") || device === p) return tier; + } + } + } + // Cold: NAS/library share — by fstype or well-known mount prefix. + const ft = (fstype || "").toLowerCase(); + if (TIER_DEFAULTS.COLD_FSTYPES.includes(ft)) return "cold"; + if ( + mount && + TIER_DEFAULTS.COLD_MOUNT_PREFIXES.some((p) => mount === p || mount.startsWith(p + "/")) + ) { + return "cold"; + } + // Hot: the primary internal disk — root mount or the GB10 root partition. + if (mount === "/" || device === "nvme0n1p2") return "hot"; + // Everything else that is a real local mount is warm. + return "warm"; +} + +/** True when a file name carries a model-weight extension. */ +export function isWeightFile(name) { + const dot = name.lastIndexOf("."); + if (dot < 0) return false; + return WEIGHT_EXTENSIONS.has(name.slice(dot).toLowerCase()); +} + +/** Strip the weight extension from a file name (e.g. "model.gguf" -> "model"). */ +export function stripWeightExt(name) { + const dot = name.lastIndexOf("."); + return dot > 0 ? name.slice(0, dot) : name; +} + +/** find -name args for weight extensions (used by the remote scan). */ +export const WEIGHT_ARGS = Array.from(WEIGHT_EXTENSIONS) + .map((ext) => `-name '*${ext}'`) + .join(" "); + /** * SystemCollector — collects hardware metrics for a Spark. * In Phase 2, this is the LOCAL path only (no SSH). @@ -96,6 +159,130 @@ export class SystemCollector { } } + /** + * Collect the Spark's resident model inventory, grouped by storage tier. + * Which dirs are scanned come from the per-Spark `modelDirs` config + * ({ hot?, warm?, cold?: string[] }); a tier with no configured dir + * contributes nothing. With no `modelDirs` set, returns []. + */ + async collectModels() { + try { + const dirs = this.spark.modelDirs || {}; + const models = []; + const seen = new Set(); + for (const tier of ["hot", "warm", "cold"]) { + const roots = dirs[tier]; + if (!Array.isArray(roots) || roots.length === 0) continue; + for (const root of roots) { + if (!root) continue; + const found = this.spark.isLocal + ? await this._scanLocalRoot(root) + : await this._scanRemoteRoot(root); + for (const m of found) { + if (!m?.name || !m?.sizeBytes || seen.has(m.name)) continue; + seen.add(m.name); + models.push({ name: m.name, sizeBytes: m.sizeBytes, tier }); + } + } + } + return models; + } catch (err) { + console.error(`[SystemCollector] Models error for ${this.spark.id}:`, err.message); + return []; + } + } + + /** + * Scan one model dir locally. A top-level entry is a model if it is a loose + * weight file, or a subdirectory whose immediate children include a weight + * file. Size is on-disk bytes (dir size recursed; loose-file size direct). + * @returns {Promise>} + */ + async _scanLocalRoot(root) { + let entries; + try { + entries = await fs.promises.readdir(root, { withFileTypes: true }); + } catch { + return []; + } + const out = []; + for (const entry of entries) { + const full = path.join(root, entry.name); + try { + if (entry.isFile() && isWeightFile(entry.name)) { + const st = await fs.promises.stat(full); + out.push({ name: stripWeightExt(entry.name), sizeBytes: st.size }); + } else if (entry.isDirectory() && (await this._dirHasWeights(full))) { + out.push({ name: entry.name, sizeBytes: await this._dirSizeBytes(full) }); + } + } catch { + // Unreadable entry — skip (matches the storage collector's per-mount catch) + } + } + return out; + } + + /** True when the immediate children of `dir` include a weight file. */ + async _dirHasWeights(dir) { + try { + const names = await fs.promises.readdir(dir); + return names.some((n) => isWeightFile(n)); + } catch { + return false; + } + } + + /** Recursive on-disk size of a directory (bytes). */ + async _dirSizeBytes(dir) { + let total = 0; + const stack = [dir]; + while (stack.length > 0) { + const cur = stack.pop(); + let names; + try { + names = await fs.promises.readdir(cur, { withFileTypes: true }); + } catch { + continue; + } + for (const e of names) { + const full = path.join(cur, e.name); + if (e.isDirectory()) stack.push(full); + else if (e.isFile()) { + try { + total += (await fs.promises.stat(full)).size; + } catch {} + } + } + } + return total; + } + + /** + * Scan one model dir over SSH (single call per root). Prints one + * "namebytes" line per model so parsing stays trivial; a subdir only + * qualifies when it immediately contains a weight file; loose weight files + * are reported by file stem. + * @returns {Promise>} + */ + async _scanRemoteRoot(root) { + const cmd = + `find '${root}' -maxdepth 1 -type f \( ${WEIGHT_ARGS} \) -printf '%f\t%s\n' 2>/dev/null; ` + + `for d in '${root}'/*/; do [ -d "$d" ] || continue; ` + + `if find "$d" -maxdepth 1 -type f \( ${WEIGHT_ARGS} \) -print -quit 2>/dev/null | grep -q .; then ` + + `echo "$(basename "$d")\t$(du -sb "$d" 2>/dev/null | cut -f1)"; fi; done`; + const output = await sshExec(this.spark, cmd, { timeoutMs: 15000 }); + const models = []; + for (const line of output.split("\n")) { + const m = line.match(/^(.+?)\t(\d+)$/); + if (!m) continue; + const name = String(m[1]).replace(/^\//, ""); + const sizeBytes = parseInt(m[2], 10) || 0; + if (!name || sizeBytes <= 0) continue; + models.push({ name, sizeBytes }); + } + return models; + } + /** Collect network metrics (interfaces, speeds). */ async collectNetwork() { if (!this.spark.isLocal) return this._getRemoteNetwork(); @@ -596,6 +783,7 @@ export class SystemCollector { readSpeed: io.readSpeed, writeSpeed: io.writeSpeed, disabled: isDisabled, + tier: classifyTier(name, displayMount, fstype, this.spark.tierPaths), }); } catch (err) { console.warn( @@ -1101,6 +1289,7 @@ export class SystemCollector { readSpeed: 0, writeSpeed: 0, disabled: isDisabled, + tier: classifyTier(device, mount, type, this.spark.tierPaths), }); } diff --git a/server/collectors/__tests__/collectModels.test.js b/server/collectors/__tests__/collectModels.test.js new file mode 100644 index 0000000..4aa32c0 --- /dev/null +++ b/server/collectors/__tests__/collectModels.test.js @@ -0,0 +1,75 @@ +import { test } from "node:test"; +import { strict as assert } from "node:assert"; +import fs from "fs"; +import os from "os"; +import path from "path"; +import { SystemCollector } from "../SystemCollector.js"; + +/** Build a local SystemCollector whose modelDirs point at a temp tree. */ +function makeTempHost() { + const base = fs.mkdtempSync(path.join(os.tmpdir(), "modelshelf-")); + const hot = path.join(base, "hot"); + const warm = path.join(base, "warm"); + fs.mkdirSync(path.join(hot, "llama-3-8b"), { recursive: true }); + fs.writeFileSync(path.join(hot, "llama-3-8b", "model.safetensors"), Buffer.alloc(4096)); + fs.writeFileSync(path.join(hot, "llama-3-8b", "config.json"), Buffer.alloc(128)); + fs.mkdirSync(warm, { recursive: true }); + fs.writeFileSync(path.join(warm, "embedding.gguf"), Buffer.alloc(2048)); + // Non-model dir: only config.json, no weights — must be ignored. + fs.mkdirSync(path.join(hot, "not-a-model")); + fs.writeFileSync(path.join(hot, "not-a-model", "config.json"), Buffer.alloc(64)); + return { base, hot, warm }; +} + +test("collectModels: scans configured modelDirs and catalogues resident models", async () => { + const { base, hot, warm } = makeTempHost(); + try { + const collector = new SystemCollector({ + id: "gx10", + isLocal: true, + modelDirs: { hot: [hot], warm: [warm] }, + }); + const models = await collector.collectModels(); + const byName = new Map(models.map((m) => [m.name, m])); + // Model inside hot dir (subdir containing a weight file) -> size recursed. + const llama = byName.get("llama-3-8b"); + assert.ok(llama, "expected llama-3-8b dir to be catalogued"); + assert.equal(llama.tier, "hot"); + assert.equal(llama.sizeBytes, 4096 + 128); // model.safetensors + config.json + // Loose weight file in warm -> reported by file stem. + const emb = byName.get("embedding"); + assert.ok(emb, "expected loose embedding.gguf to be catalogued"); + assert.equal(emb.tier, "warm"); + assert.equal(emb.sizeBytes, 2048); + // Non-model dir (no weights) must be absent. + assert.ok(!byName.has("not-a-model"), "non-model dir must not be catalogued"); + } finally { + fs.rmSync(base, { recursive: true, force: true }); + } +}); + +test("collectModels: empty modelDirs returns []", async () => { + const collector = new SystemCollector({ id: "gx10", isLocal: true }); + const models = await collector.collectModels(); + assert.deepEqual(models, []); +}); + +test("collectModels: dedupes by name, first tier wins", async () => { + const { base, hot, warm } = makeTempHost(); + try { + // Same model name in both hot and warm dirs. + fs.mkdirSync(path.join(warm, "llama-3-8b"), { recursive: true }); + fs.writeFileSync(path.join(warm, "llama-3-8b", "model.gguf"), Buffer.alloc(512)); + const collector = new SystemCollector({ + id: "gx10", + isLocal: true, + modelDirs: { hot: [hot], warm: [warm] }, + }); + const models = await collector.collectModels(); + const hits = models.filter((m) => m.name === "llama-3-8b"); + assert.equal(hits.length, 1); + assert.equal(hits[0].tier, "hot"); // hot scanned first + } finally { + fs.rmSync(base, { recursive: true, force: true }); + } +}); diff --git a/server/collectors/__tests__/tierClassify.test.js b/server/collectors/__tests__/tierClassify.test.js new file mode 100644 index 0000000..4718d1e --- /dev/null +++ b/server/collectors/__tests__/tierClassify.test.js @@ -0,0 +1,49 @@ +import { test } from "node:test"; +import { strict as assert } from "node:assert"; +import { classifyTier, isWeightFile, stripWeightExt } from "../SystemCollector.js"; + +test("classifyTier: root NVMe mount is hot", () => { + assert.equal(classifyTier("nvme0n1p2", "/", "ext4"), "hot"); + assert.equal(classifyTier("nvme0n1p2", "/", ""), "hot"); +}); + +test("classifyTier: NAS cifs/nfs mount is cold", () => { + assert.equal(classifyTier("//192.168.68.58/modelshelf", "/mnt/modelshelf", "cifs"), "cold"); + assert.equal(classifyTier("nfs.example:/data", "/mnt/models", "nfs4"), "cold"); + assert.equal(classifyTier("somedev", "/media/hot4tb", "ext4"), "cold"); + assert.equal(classifyTier("somedev", "/Volumes/T5", "exfat"), "cold"); +}); + +test("classifyTier: other real local mounts are warm", () => { + assert.equal(classifyTier("sdb1", "/data/usb", "ext4"), "warm"); + assert.equal(classifyTier("xdpu0", "/mnt/xdp", "xfs"), "cold"); // prefix override +}); + +test("classifyTier: tierPaths override wins", () => { + const tierPaths = { cold: ["/mnt/models"] }; + assert.equal(classifyTier("nvme0n1p2", "/", "ext4", tierPaths), "hot"); + assert.equal(classifyTier("nvme0n1p2", "/mnt/models", "ext4", tierPaths), "cold"); + assert.equal(classifyTier("somedev", "/mnt/models/lora", "ext4", tierPaths), "cold"); +}); + +test("classifyTier: empty mount/device defaults to warm", () => { + assert.equal(classifyTier("", ""), "warm"); + assert.equal(classifyTier("sdb1", ""), "warm"); +}); + +test("isWeightFile: recognizes weight extensions, case-insensitive, rejects others", () => { + assert.equal(isWeightFile("model.safetensors"), true); + assert.equal(isWeightFile("model.gguf"), true); + assert.equal(isWeightFile("model.bin"), true); + assert.equal(isWeightFile("model.pt"), true); + assert.equal(isWeightFile("model.GGUF"), true); + assert.equal(isWeightFile("README.md"), false); + assert.equal(isWeightFile("config.json"), false); + assert.equal(isWeightFile("tokenizer"), false); +}); + +test("stripWeightExt: removes extension and keeps the stem", () => { + assert.equal(stripWeightExt("llama-3.gguf"), "llama-3"); + assert.equal(stripWeightExt("model.safetensors"), "model"); + assert.equal(stripWeightExt("noext"), "noext"); +}); diff --git a/server/config.js b/server/config.js index fbb558c..8ce2e1d 100644 --- a/server/config.js +++ b/server/config.js @@ -30,6 +30,8 @@ const POLL_INTERVAL_LLM = parseInt(process.env.POLL_INTERVAL_LLM || "2000", 10); const POLL_INTERVAL_BANDWIDTH = parseInt(process.env.POLL_INTERVAL_BANDWIDTH || "2000", 10); // Dedicated liveness (sshTest / local ping) cadence — not a metric domain. const POLL_INTERVAL_LIVENESS = parseInt(process.env.POLL_INTERVAL_LIVENESS || "5000", 10); +// Model-inventory scan cadence (heavier dir walk than storage; keep slow). +const POLL_INTERVAL_MODELS = parseInt(process.env.POLL_INTERVAL_MODELS || "30000", 10); // ─── Port ──────────────────────────────────────────────── const PORT = parseInt(process.env.PORT || "5555", 10); @@ -71,6 +73,18 @@ const HOST_PATHS = { ROOT: process.env.HOST_ROOT_PATH || "/host/root", }; +// ─── Model tier storage ────────────────────────────────── +// Classifies each Spark storage device into hot/warm/cold. These are the +// self-contained defaults; a Spark can override them via `tierPaths`. +const TIER_DEFAULTS = { + /** Cold = the shared library/NAS. Matched by fstype or by mount prefix. */ + COLD_FSTYPES: ["cifs", "smb", "smb3", "nfs", "nfs4"], + COLD_MOUNT_PREFIXES: ["/mnt/modelshelf", "/media", "/Volumes", "/mnt"], +}; + +/** Weight file extensions that make a directory count as a model. */ +const WEIGHT_EXTENSIONS = new Set([".safetensors", ".gguf", ".bin", ".pt", ".pth", ".ckpt"]); + export { SPARKS_JSON_PATH, GPU_MEMORY_JSON_PATH, @@ -85,11 +99,14 @@ export { POLL_INTERVAL_LLM, POLL_INTERVAL_BANDWIDTH, POLL_INTERVAL_LIVENESS, + POLL_INTERVAL_MODELS, PORT, LLM_PORT, DGX_SPARK, UNIT_CONVERSION, HARDWARE_DEFAULTS, HOST_PATHS, + TIER_DEFAULTS, + WEIGHT_EXTENSIONS, ROOT, }; \ No newline at end of file diff --git a/server/sparks/SparkMonitor.js b/server/sparks/SparkMonitor.js index 5d65a06..ee9b592 100644 --- a/server/sparks/SparkMonitor.js +++ b/server/sparks/SparkMonitor.js @@ -11,6 +11,7 @@ import { POLL_INTERVAL_LLM, POLL_INTERVAL_BANDWIDTH, POLL_INTERVAL_LIVENESS, + POLL_INTERVAL_MODELS, LLM_PORT, HOST_PATHS, } from "../config.js"; @@ -52,6 +53,7 @@ export class SparkMonitor { cpu: this.collector._defaultCpu(), ram: this.collector._defaultRam(), storage: [], + models: [], network: this.collector._defaultNetwork(), unifiedMemory: this.collector._defaultUnifiedMemory(), llm: [], @@ -145,6 +147,7 @@ export class SparkMonitor { this._intervals.push(setInterval(() => this._pollDomain("cpu"), POLL_INTERVAL_CPU)); this._intervals.push(setInterval(() => this._pollDomain("network"), POLL_INTERVAL_NETWORK)); this._intervals.push(setInterval(() => this._pollDomain("storage"), POLL_INTERVAL_STORAGE)); + this._intervals.push(setInterval(() => this._pollDomain("models"), POLL_INTERVAL_MODELS)); this._intervals.push(setInterval(() => this._pollDomain("ram"), POLL_INTERVAL_CPU)); this._intervals.push(setInterval(() => this._pollDomain("memory"), POLL_INTERVAL_BANDWIDTH)); this._restartLlmPollInterval(); @@ -199,6 +202,7 @@ export class SparkMonitor { cpu: this._metrics.cpu, ram: this._metrics.ram, storage: this._metrics.storage, + models: this._metrics.models, network: this._metrics.network, unifiedMemory: this._metrics.unifiedMemory, llm: this._metrics.llm, @@ -266,6 +270,7 @@ export class SparkMonitor { this._pollDomain("cpu"), this._pollDomain("network"), this._pollDomain("storage"), + this._pollDomain("models"), this._pollDomain("ram"), this._pollDomain("memory"), this._pollDomain("llm"), @@ -297,6 +302,9 @@ export class SparkMonitor { case "storage": result = await this.collector.collectStorage(); break; + case "models": + result = await this.collector.collectModels(); + break; case "memory": result = await this.collector.collectUnifiedMemory(); break; @@ -336,6 +344,9 @@ export class SparkMonitor { case "storage": this._metrics.storage = result; break; + case "models": + this._metrics.models = result; + break; case "memory": this._metrics.unifiedMemory = result; break; diff --git a/server/sparks/SparkRegistry.js b/server/sparks/SparkRegistry.js index f10e1cc..2547118 100644 --- a/server/sparks/SparkRegistry.js +++ b/server/sparks/SparkRegistry.js @@ -508,9 +508,37 @@ export class SparkRegistry { disabledDevices: Array.isArray(config.disabledDevices) ? config.disabledDevices : [], disabledInterfaces: Array.isArray(config.disabledInterfaces) ? config.disabledInterfaces : [], storagePollDisabled: Boolean(config.storagePollDisabled), + /** + * Optional storage-tier override: { hot?: string[], warm?: string[], + * cold?: string[] } of mount-prefix lists. When empty/absent, tier + * classification falls back to TIER_DEFAULTS heuristics. + */ + tierPaths: this._normalizeTierMap(config.tierPaths), + /** + * Optional per-tier model directories to scan: + * { hot?: string[], warm?: string[], cold?: string[] }. When empty, + * scan roots are derived from the classified tier mounts. + */ + modelDirs: this._normalizeTierMap(config.modelDirs), }; } + /** Normalize { tier: string[] } to a clean { hot?, warm?, cold? } map. */ + _normalizeTierMap(value) { + if (!value || typeof value !== "object") return {}; + const out = {}; + for (const tier of ["hot", "warm", "cold"]) { + const raw = value[tier]; + if (Array.isArray(raw)) { + const paths = raw + .map((p) => (typeof p === "string" ? p.trim() : "")) + .filter(Boolean); + if (paths.length > 0) out[tier] = paths; + } + } + return out; + } + /** Normalize role; legacy workerNode=true → worker. */ _normalizeRole(config) { const coerced = this._coerceRole(config?.role); diff --git a/src/App.tsx b/src/App.tsx index 07e8a81..b287cc3 100644 --- a/src/App.tsx +++ b/src/App.tsx @@ -7,11 +7,12 @@ import { AddSparkDialog } from "./components/AddSparkDialog"; import { EditSparkDialog } from "./components/EditSparkDialog"; import { SparkPage } from "./components/SparkPage/SparkPage"; import { OverviewPage } from "./components/OverviewPage/OverviewPage"; +import { FleetStoragePage } from "./components/FleetStoragePage/FleetStoragePage"; import { ShowcasePage } from "./components/ShowcasePage/ShowcasePage"; import { ThemeSwitch } from "./components/ThemeSwitch"; import { SettingsDialog } from "./components/SettingsDialog"; import { GearIcon, BoltIcon } from "./components/ui/icons"; -import { OVERVIEW_ID } from "./constants"; +import { OVERVIEW_ID, FLEET_STORAGE_ID } from "./constants"; import type { Settings, SparkSnapshot } from "./api/types"; function placeholderSnapshot( @@ -70,6 +71,7 @@ function placeholderSnapshot( cpu: null, ram: null, storage: [], + models: [], network: null, unifiedMemory: null, llm: [], @@ -115,6 +117,7 @@ function DashboardApp() { }, [liveSparks, orderOverride]); const isOverview = activeId === OVERVIEW_ID; + const isFleetStorage = activeId === FLEET_STORAGE_ID; const displayActive = isOverview ? null : displaySparks.find((s) => s.id === activeId) || displaySparks[0] || activeSpark || null; @@ -235,7 +238,9 @@ function DashboardApp() {
- {isOverview ? ( + {isFleetStorage ? ( + + ) : isOverview ? ( >; + /** + * Optional per-tier model directories to scan for the model inventory: + * { hot?, warm?, cold?: string[] }. When empty, no models are reported. + */ + modelDirs?: Partial>; } export type SparkRole = "head" | "worker" | "standalone"; @@ -144,6 +154,22 @@ export interface StorageMetrics { writeSpeed: number; /** Present when device is in disabledDevices; still returned for Settings UI */ disabled?: boolean; + /** Storage tier: hot (internal NVMe) / warm (USB) / cold (NAS library). */ + tier?: "hot" | "warm" | "cold"; +} + +/** A storage tier label. */ +export type ModelTier = "hot" | "warm" | "cold"; + +/** + * A resident model on a Spark, from the per-Spark scanner. Placement + * (replicated across Sparks vs served over the CX7 fabric from a peer) is + * derived fleet-wide in the frontend from each Spark's models + loaded llm ids. + */ +export interface ModelTierMetrics { + name: string; + sizeBytes: number; + tier: ModelTier; } // ─── Network metrics ───────────────────────────────────── @@ -244,6 +270,8 @@ export interface SparkMetrics { cpu: CpuMetrics | null; ram: RamMetrics | null; storage: StorageMetrics[]; + /** Resident models from the per-Spark scanner (tier + size). */ + models: ModelTierMetrics[]; network: NetworkMetrics | null; unifiedMemory: UnifiedMemoryMetrics | null; /** Array of LLM metrics, one per configured port. Empty array when no ports. */ diff --git a/src/components/FleetStoragePage/FleetStoragePage.tsx b/src/components/FleetStoragePage/FleetStoragePage.tsx new file mode 100644 index 0000000..7a0d087 --- /dev/null +++ b/src/components/FleetStoragePage/FleetStoragePage.tsx @@ -0,0 +1,195 @@ +import { useMemo } from "react"; +import type { ModelTier, SparkSnapshot, StorageMetrics } from "../../api/types"; +import { MetricBar } from "../ui/MetricBar"; +import { Panel } from "../ui/Panel"; +import { DiskIcon } from "../ui/icons"; +import { resolvePlacement, rollupTierGb } from "./fleetPlacement"; + +interface FleetStoragePageProps { + sparks: SparkSnapshot[]; +} + +const TIERS: ModelTier[] = ["hot", "warm", "cold"]; + +const TIER_TONE: Record = { + hot: "bg-accent text-accent", + warm: "bg-warning text-black", + cold: "bg-muted text-black", +}; + +function fmtGb(mb: number): string { + const gb = mb / 1024; + return gb >= 100 ? `${Math.round(gb)} GB` : `${gb.toFixed(1)} GB`; +} + +function fmtBytes(bytes: number): string { + const gb = bytes / 1024 / 1024 / 1024; + if (gb >= 1) return `${gb.toFixed(1)} GB`; + const m = bytes / 1024 / 1024; + return `${Math.round(m)} MB`; +} + +function TierChip({ tier }: { tier: ModelTier }) { + const label = tier === "hot" ? "Hot" : tier === "warm" ? "Warm" : "Cold"; + return ( + + {label} + + ); +} + +export function FleetStoragePage({ sparks }: FleetStoragePageProps) { + const tierGb = useMemo(() => rollupTierGb(sparks), [sparks]); + const placement = useMemo(() => resolvePlacement(sparks), [sparks]); + const replicated = placement.filter((p) => p.resident.length > 1).length; + const fabric = placement.filter((p) => p.fabric.length > 0).length; + + return ( +
+
+

+ Storage +

+
+ {TIERS.map((t) => ( + + + {fmtGb(tierGb[t])} + used + + ))} + + {replicated} replicated + + + {fabric} on CX7 fabric + +
+
+ + {/* Per-Spark tier cards */} +
+ {sparks.map((spark) => ( + + ))} +
+ + {/* Model placement table */} + } className="panel-storage"> + {placement.length === 0 ? ( +

+ No models discovered. Configure modelDirs per tier for each Spark in sparks.json. +

+ ) : ( + + + + + + + + + + {placement.map((p) => ( + + + + + + ))} + +
ModelSizePlacement
+ {p.name} + + {fmtBytes(p.resident[0]?.tier ? byteSizeFor(p, sparks) : 0)} + +
+ {p.resident.map((r) => ( + + {r.sparkName} + + + ))} + {p.fabric.map((f) => ( + + {f.sparkName} · fabric + + ))} +
+
+ )} +

+ Resident = local disk copy. “fabric” = this Spark loads the model over the CX7 link from a peer’s copy (no local copy). +

+ +
+ ); +} + +function byteSizeFor( + p: { name: string }, + sparks: SparkSnapshot[] +): number { + for (const s of sparks) { + const m = (Array.isArray(s.metrics?.models) ? s.metrics.models : []).find( + (x) => x.name === p.name + ); + if (m) return m.sizeBytes; + } + return 0; +} + +function SparkTierCard({ spark }: { spark: SparkSnapshot }) { + const disks: StorageMetrics[] = Array.isArray(spark.metrics?.storage) + ? spark.metrics.storage.filter((d) => !d.disabled) + : []; + const tierAgg: Record = { + hot: { used: 0, total: 0 }, + warm: { used: 0, total: 0 }, + cold: { used: 0, total: 0 }, + }; + for (const d of disks) { + const t = d.tier; + if (t === "hot" || t === "warm" || t === "cold") { + tierAgg[t].used += d.used || 0; + tierAgg[t].total += d.total || 0; + } + } + return ( +
+
+ + {spark.name} + {spark.online ? "online" : "offline"} +
+
+ {TIERS.map((t) => + tierAgg[t].total > 0 ? ( +
+
+ + + + + {fmtGb(tierAgg[t].used)} / {fmtGb(tierAgg[t].total)} + +
+ +
+ ) : null + )} + {TIERS.every((t) => tierAgg[t].total === 0) && ( +

No tiered storage detected

+ )} +
+
+ ); +} diff --git a/src/components/FleetStoragePage/fleetPlacement.ts b/src/components/FleetStoragePage/fleetPlacement.ts new file mode 100644 index 0000000..cffd90b --- /dev/null +++ b/src/components/FleetStoragePage/fleetPlacement.ts @@ -0,0 +1,81 @@ +import type { ModelTier, SparkSnapshot, StorageMetrics } from "../../api/types"; + +/** A Spark's view of where a model lives, given the whole fleet's snapshots. */ +export interface ModelPlacement { + name: string; + /** Resident copies: which Spark(s) hold a local copy, and in which tier. */ + resident: Array<{ sparkId: string; sparkName: string; tier: ModelTier }>; + /** Sparks serving this model over the CX7 fabric (loading but not resident). */ + fabric: Array<{ sparkId: string; sparkName: string }>; +} + +/** Loose model-name match: exact, or normalize HF-hub-style basenames. */ +function modelIdMatches(llmModelId: string | null | undefined, modelName: string): boolean { + if (!llmModelId) return false; + const a = llmModelId.toLowerCase(); + const b = modelName.toLowerCase(); + if (a === b) return true; + // HF hub cache path like ~/.cache/huggingface/hub/models--org--name vs resident name. + const last = a.split("/").pop() ?? ""; + return last === b || a.endsWith("/" + b) || a.includes("models--" + b.replace(/\s+/g, "-")); +} + +/** + * Resolve per-model placement across the fleet. A model is: + * - "resident" on a Spark that holds a local disk copy (tier from its scan); + * - served "fabric" on a Spark that is currently *loading* it in unified + * memory (llm modelId) but holds no local copy, while at least one peer is + * resident — i.e. the peer's hot copy is reached over the CX7 interconnect. + * Pure (no React) so it is directly unit-testable. + */ +export function resolvePlacement(sparks: SparkSnapshot[]): ModelPlacement[] { + const byName = new Map(); + for (const spark of sparks) { + const resident = Array.isArray(spark.metrics?.models) ? spark.metrics.models : []; + for (const m of resident) { + let entry = byName.get(m.name); + if (!entry) { + entry = { name: m.name, resident: [], fabric: [] }; + byName.set(m.name, entry); + } + entry.resident.push({ sparkId: spark.id, sparkName: spark.name, tier: m.tier }); + } + } + // Fabric: a Spark is loading a model it does not own, while a peer owns it. + for (const spark of sparks) { + const loaded = (Array.isArray(spark.metrics?.llm) ? spark.metrics.llm : []) + .map((l) => l?.modelId) + .filter(Boolean); + for (const modelName of Array.from(byName.keys())) { + const entry = byName.get(modelName)!; + if (entry.resident.some((r) => r.sparkId === spark.id)) continue; // resident here + if (entry.resident.length === 0) continue; // nobody owns it locally + const isLoaded = loaded.some((id) => modelIdMatches(id, modelName)); + if (isLoaded) { + // Avoid duplicate fabric markers per spark/model. + if (!entry.fabric.some((f) => f.sparkId === spark.id)) { + entry.fabric.push({ sparkId: spark.id, sparkName: spark.name }); + } + } + } + } + return Array.from(byName.values()).sort((a, b) => a.name.localeCompare(b.name)); +} + +/** Fleet-wide tier usage (GB) summed across all Sparks' storage devices. */ +export function rollupTierGb(sparks: SparkSnapshot[]): Record { + const out: Record = { hot: 0, warm: 0, cold: 0 }; + for (const spark of sparks) { + const disks: StorageMetrics[] = Array.isArray(spark.metrics?.storage) + ? spark.metrics.storage + : []; + for (const d of disks) { + if (d?.disabled) continue; + const tier = d.tier; + if (tier === "hot" || tier === "warm" || tier === "cold") { + out[tier] += (d.used || 0) / 1024; + } + } + } + return out; +} diff --git a/src/components/SparkPage/StoragePanel.tsx b/src/components/SparkPage/StoragePanel.tsx index dc8fb98..1af493b 100644 --- a/src/components/SparkPage/StoragePanel.tsx +++ b/src/components/SparkPage/StoragePanel.tsx @@ -25,6 +25,24 @@ function formatGb(mb: number): string { return `${Math.round(mb)} MB`; } +function TierBadge({ tier }: { tier?: "hot" | "warm" | "cold" }) { + if (tier !== "hot" && tier !== "warm" && tier !== "cold") return null; + const label = tier === "hot" ? "Hot" : tier === "warm" ? "Warm" : "Cold"; + const cls = + tier === "hot" + ? "bg-accent text-black" + : tier === "warm" + ? "bg-warning text-black" + : "bg-muted text-black"; + return ( + + {label} + + ); +} + function MetricBar({ value, max }: { value: number; max: number }) { const pct = max > 0 ? Math.min(100, Math.round((value / max) * 100)) : 0; const barColor = pct > 85 ? "bg-danger" : pct > 60 ? "bg-warning" : "bg-accent"; @@ -213,6 +231,7 @@ export function StoragePanel({
{disk.label} + {disk.device}
{pct}% diff --git a/src/components/SparkTabs.tsx b/src/components/SparkTabs.tsx index 5afe485..eaf66bf 100644 --- a/src/components/SparkTabs.tsx +++ b/src/components/SparkTabs.tsx @@ -19,8 +19,8 @@ import { } from "@dnd-kit/sortable"; import { CSS } from "@dnd-kit/utilities"; import type { SparkSnapshot } from "../api/types"; -import { PlusIcon, GridIcon } from "./ui/icons"; -import { OVERVIEW_ID } from "../constants"; +import { PlusIcon, GridIcon, DiskIcon } from "./ui/icons"; +import { OVERVIEW_ID, FLEET_STORAGE_ID } from "../constants"; interface SparkTabsProps { sparks: SparkSnapshot[]; @@ -326,6 +326,7 @@ export function SparkTabs({ return (