diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bbf4ec..a9a4961 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ Format: version sections are listed newest first. ### Added - **Tailnet monitoring** — opt-in per unit (`tailscaleMonitoring`, default **off**); `tailscale status --json` on the host and a Tailnet card under Resources. Flags a unit that is healthy on the LAN but off its tailnet. ([#43](https://github.com/MiaAI-Lab/sparkDash/pull/43)) +- **NV_ERR_NO_MEMORY on the GPU panel** — count of NVRM `NV_ERR_NO_MEMORY` kernel log lines since boot (shown when > 0). Journal is scanned at most once a minute, not on the 2s poll. Replaces the approach in [#40](https://github.com/MiaAI-Lab/sparkDash/pull/40). ### Security - **`BIND_HOST` now defaults to `127.0.0.1` (loopback) instead of `0.0.0.0`** — the dashboard is unauthenticated and can SSH into and power off Sparks, so it is no longer reachable on the LAN by default. Set `BIND_HOST` to the host's LAN IP (or `0.0.0.0`) to opt in to remote access. **Migration:** if you access sparkDash from another machine via bare-metal `npm start`, set `BIND_HOST` explicitly. Production and dev Compose both set `BIND_HOST=0.0.0.0` (`network_mode: host`). Startup now also warns when bound to a non-loopback address. ([#35](https://github.com/MiaAI-Lab/sparkDash/pull/35)) diff --git a/README.md b/README.md index 17a358c..ef50fd6 100644 --- a/README.md +++ b/README.md @@ -396,6 +396,7 @@ Copy `.env.example` to `.env` if needed: | `POLL_INTERVAL_HERMES` | `600000` | Hermes Agent update check poll (ms) | | `POLL_INTERVAL_TAILSCALE` | `30000` | Tailnet probe poll (ms) | | `TAILSCALE_PROBE_TIMEOUT_MS` | `8000` | Timeout for `tailscale status --json` (ms) | +| `POLL_INTERVAL_NVERR` | `60000` | Kernel journal scan for NVRM `NV_ERR_NO_MEMORY` (ms) | | `HERMES_UPDATE_TIMEOUT_MS` | `600000` | Hard timeout for running `hermes update` over SSH (ms) | | `POLL_INTERVAL_LIVENESS` | `5000` | Online/SSH liveness check (ms) | | `SPARKDASH_SECRETS_KEY` | _(auto)_ | Passphrase or 64-char hex for secret encryption | diff --git a/server/collectors/SystemCollector.js b/server/collectors/SystemCollector.js index ecba788..5f5a534 100644 --- a/server/collectors/SystemCollector.js +++ b/server/collectors/SystemCollector.js @@ -1,9 +1,24 @@ import fs from "fs"; import path from "path"; -import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS } from "../config.js"; +import { HOST_PATHS, GPU_MEMORY_JSON_PATH, DGX_SPARK, HARDWARE_DEFAULTS, POLL_INTERVAL_NVERR } from "../config.js"; import { normalizeMac, WOL_INTERFACE } from "../wol.js"; import { sshExec } from "./ssh.js"; +const NVERR_JOURNAL_CMD = + 'journalctl -k --no-pager -q --grep=NV_ERR_NO_MEMORY 2>/dev/null | grep -c NV_ERR_NO_MEMORY || true'; + +/** + * Parse `grep -c` stdout into a non-negative integer. Exported for tests. + * @param {unknown} raw + * @returns {number} + */ +export function parseNvErrNoMemoryCount(raw) { + const line = String(raw ?? "").trim().split("\n").pop() ?? ""; + const n = Number.parseInt(line, 10); + if (!Number.isFinite(n) || n < 0) return 0; + return n; +} + /** * SystemCollector — collects hardware metrics for a Spark. * In Phase 2, this is the LOCAL path only (no SSH). @@ -32,6 +47,8 @@ export class SystemCollector { // Cached hardware info this._hardwareInfo = null; + /** Cached NVRM NV_ERR_NO_MEMORY count (slow journal scan). */ + this._nvErrCache = { count: 0, at: 0 }; } /** Collect GPU metrics (temperature, usage, power, VRAM). */ @@ -157,6 +174,7 @@ export class SystemCollector { vram, processes, throttle: gpu.throttle, + nvErrNoMemory: await this._nvErrNoMemory(), }; } @@ -994,6 +1012,7 @@ export class SystemCollector { vram: { used: usedMB, total: totalMB, percentage, available: availableMB }, processes, throttle: gpu.throttle, + nvErrNoMemory: await this._nvErrNoMemory(), }; } catch (err) { console.error(`[SystemCollector] Remote GPU error for ${this.spark.id}:`, err.message); @@ -1494,6 +1513,34 @@ export class SystemCollector { return fs.promises.statfs(dir); } + /** + * Count NVRM `NV_ERR_NO_MEMORY` lines in the kernel journal since boot. + * Cached for POLL_INTERVAL_NVERR — never on the 2s GPU/memory loop uncached. + * @returns {Promise} + */ + async _nvErrNoMemory() { + const now = Date.now(); + if (this._nvErrCache.at > 0 && now - this._nvErrCache.at < POLL_INTERVAL_NVERR) { + return this._nvErrCache.count; + } + try { + let out; + if (this.spark.isLocal) { + out = this._hasHostProc() + ? await this._execOnHost(NVERR_JOURNAL_CMD) + : await this._exec(NVERR_JOURNAL_CMD); + } else { + out = await sshExec(this.spark, NVERR_JOURNAL_CMD, { timeoutMs: 8000 }); + } + const count = parseNvErrNoMemoryCount(out); + this._nvErrCache = { count, at: now }; + return count; + } catch { + this._nvErrCache.at = now; + return this._nvErrCache.count; + } + } + // ─── Default metrics ───────────────────────────────────── _defaultGpu() { return { @@ -1503,6 +1550,7 @@ export class SystemCollector { vram: { used: 0, total: 0, percentage: 0, available: 0 }, processes: [], throttle: this._defaultThrottle(), + nvErrNoMemory: 0, }; } diff --git a/server/collectors/__tests__/SystemCollector.nvErr.test.js b/server/collectors/__tests__/SystemCollector.nvErr.test.js new file mode 100644 index 0000000..5a4036d --- /dev/null +++ b/server/collectors/__tests__/SystemCollector.nvErr.test.js @@ -0,0 +1,16 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { parseNvErrNoMemoryCount } from "../SystemCollector.js"; + +test("parseNvErrNoMemoryCount reads grep -c output", () => { + assert.equal(parseNvErrNoMemoryCount("12"), 12); + assert.equal(parseNvErrNoMemoryCount("0"), 0); + assert.equal(parseNvErrNoMemoryCount(" 43\n"), 43); +}); + +test("parseNvErrNoMemoryCount defaults invalid input to 0", () => { + assert.equal(parseNvErrNoMemoryCount(""), 0); + assert.equal(parseNvErrNoMemoryCount("not-a-number"), 0); + assert.equal(parseNvErrNoMemoryCount(undefined), 0); + assert.equal(parseNvErrNoMemoryCount("-3"), 0); +}); diff --git a/server/config.js b/server/config.js index 5297cc6..2bbf2aa 100644 --- a/server/config.js +++ b/server/config.js @@ -34,6 +34,8 @@ const POLL_INTERVAL_LLM = parseInt(process.env.POLL_INTERVAL_LLM || "2000", 10); const POLL_INTERVAL_COMFY = parseInt(process.env.POLL_INTERVAL_COMFY || "2000", 10); // Tailnet membership changes slowly; each poll is an SSH round-trip. const POLL_INTERVAL_TAILSCALE = parseInt(process.env.POLL_INTERVAL_TAILSCALE || "30000", 10); +// Kernel journal scan for NV_ERR_NO_MEMORY — not on the 2s GPU loop. +const POLL_INTERVAL_NVERR = parseInt(process.env.POLL_INTERVAL_NVERR || "60000", 10); // dmon -c 1 -d 1 blocks ~1s; default 2s avoids stacking with in-flight guards const POLL_INTERVAL_BANDWIDTH = parseInt(process.env.POLL_INTERVAL_BANDWIDTH || "2000", 10); // Dedicated liveness (sshTest / local ping) cadence — not a metric domain. @@ -106,6 +108,7 @@ export { POLL_INTERVAL_LLM, POLL_INTERVAL_COMFY, POLL_INTERVAL_TAILSCALE, + POLL_INTERVAL_NVERR, POLL_INTERVAL_BANDWIDTH, POLL_INTERVAL_LIVENESS, POLL_INTERVAL_HERMES, diff --git a/src/api/types.ts b/src/api/types.ts index 7e42fd1..3b4a9f6 100644 --- a/src/api/types.ts +++ b/src/api/types.ts @@ -211,6 +211,8 @@ export interface GpuMetrics { processes?: Array<{ pid: number; name: string; vramMB: number }>; /** NVIDIA clock throttle / thermal slowdown state from nvidia-smi. */ throttle?: GpuThrottle | null; + /** Kernel NVRM NV_ERR_NO_MEMORY count since boot (cached ~60s). */ + nvErrNoMemory?: number; } // ─── CPU metrics ───────────────────────────────────────── diff --git a/src/components/SparkPage/GpuPanel.tsx b/src/components/SparkPage/GpuPanel.tsx index 56c5a82..60cc67d 100644 --- a/src/components/SparkPage/GpuPanel.tsx +++ b/src/components/SparkPage/GpuPanel.tsx @@ -178,6 +178,18 @@ export function GpuPanel({ gpu, sparkId, temperatureUnit, className }: GpuPanelP )} + {(gpu?.nvErrNoMemory ?? 0) > 0 && ( +
+ NV_ERR_NO_MEMORY + + {gpu?.nvErrNoMemory} + +
+ )} + {/* Top GPU processes by VRAM usage */} {gpu && gpu.processes && gpu.processes.length > 0 && (