diff --git a/CHANGELOG.md b/CHANGELOG.md index 7385ba0..efb82bb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,9 @@ has a matching GitHub release whose body comes from its section below. `berth status` and its JSON show it as passed through to a VM, and waiters are told why nothing is granted. While a VFIO hold is granted, berthd does not open the card, so a driver switch can run. `card.passthrough` and `card.returned` events mark each change. +- The harbor shows a card passed through to a VM (experimental): a GPU IN VM tag on every tab, the + card's panel and the departures board marked, the change named in the log, and the session left as a + gap in the tide. - `berth vfio status` and `berth vfio run` (experimental, Linux only) for passing the governed card to a VM. `status` shows IOMMU, the card's IOMMU group with each function's driver, and the processes holding the card. `run -- CMD` drains the card, stops berthd's own probe for the hold, refuses while diff --git a/docs/harbor.md b/docs/harbor.md index f537cbe..0956d56 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -74,7 +74,11 @@ network you trust. its own probe would block the driver switch; it reads the driver from sysfs instead. Nothing new is granted while the card is away, and each waiter's reason says so ("gpu0 is passed through to a VM (vfio-pci); it comes back when the VM's hold ends"). A card switched by hand, with no hold, shows the - same state instead of a probe-failed warning. + same state instead of a probe-failed warning. In the harbor, a GPU IN VM tag sits beside the state pill + on every tab. The card's panel shows the passthrough state in place of its memory figures, with the last + reading greyed and labelled with its age. The departures board marks the hold IN VM, the log names each + change in plain words, and the tide leaves the session as a gap. Engines stopped for the session do not + turn the state DEGRADED. - **Alerts:** one threshold table in `berth.toml` (`[alerts]`): warn and crit per key (`queue`, `ttft`, `tpot`, `p95`, `cache`, `reingest`, `vram`, `temp`, `power`, `kv`). The node evaluates it on its 5 s sample cycle, fires a desktop notification and webhook diff --git a/internal/harbor/events_test.go b/internal/harbor/events_test.go index 1c9707c..f7ac989 100644 --- a/internal/harbor/events_test.go +++ b/internal/harbor/events_test.go @@ -28,6 +28,7 @@ type fakeNode struct { sock string lastEvent uint64 evts []proto.Event + st *proto.Status // canned OpStatus reply when set; otherwise an empty status ln net.Listener mu sync.Mutex @@ -79,7 +80,10 @@ func (f *fakeNode) serveConn(c net.Conn) { } switch m.Op { case proto.OpStatus: - st := &proto.Status{API: proto.APIVersion, LastEvent: f.lastEvent} + st := f.st + if st == nil { + st = &proto.Status{API: proto.APIVersion, LastEvent: f.lastEvent} + } _ = conn.Send(proto.Msg{Op: proto.OpStatus, Status: st}) case proto.OpEvents: for i := range f.evts { diff --git a/internal/harbor/harbor_test.go b/internal/harbor/harbor_test.go index a3cddd7..7bdf1b0 100644 --- a/internal/harbor/harbor_test.go +++ b/internal/harbor/harbor_test.go @@ -187,6 +187,54 @@ func TestListenMode(t *testing.T) { } } +// TestStatusPassthroughReachesThePage: a node that reports the governed card +// in a VM passes the passthrough object and the card's figures through +// /v1/status unshaped, so the page's panels have the state and the last +// reading's age to show. +func TestStatusPassthroughReachesThePage(t *testing.T) { + n := newFakeNode(t, 0, nil) + n.st = &proto.Status{ + API: proto.APIVersion, + GPUs: []proto.GPU{{ + Index: 0, Name: "NVIDIA GeForce RTX 4090", Scheduled: true, + Total: 24 << 30, Used: 20 << 30, Free: 4 << 30, Baseline: 2 << 30, Age: 300, + }}, + Passthrough: &proto.Passthrough{State: proto.CardPassthrough, Driver: "vfio-pci"}, + } + srv := newEventsHarbor(t, n) + resp, err := http.Get(srv.URL + "/v1/status") + if err != nil { + t.Fatal(err) + } + b, _ := io.ReadAll(resp.Body) + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("/v1/status: %s %s", resp.Status, b) + } + body := string(b) + for _, want := range []string{ + `"passthrough":{"state":"passthrough","driver":"vfio-pci"}`, + `"scheduled":true`, + `"age_seconds":300`, + } { + if !strings.Contains(body, want) { + t.Errorf("/v1/status does not carry %s:\n%s", want, body) + } + } + + // detaching reads as its own state, a switch under way, not passthrough + n.st.Passthrough = &proto.Passthrough{State: proto.CardDetaching, Driver: "nvidia"} + resp, err = http.Get(srv.URL + "/v1/status") + if err != nil { + t.Fatal(err) + } + b, _ = io.ReadAll(resp.Body) + resp.Body.Close() + if !strings.Contains(string(b), `"passthrough":{"state":"detaching","driver":"nvidia"}`) { + t.Fatalf("detaching: %s", b) + } +} + // logBuf is a bytes.Buffer safe to poll from the test while Serve logs to it. type logBuf struct { mu sync.Mutex diff --git a/internal/harbor/web/board.js b/internal/harbor/web/board.js index bd2146d..41e8bfb 100644 --- a/internal/harbor/web/board.js +++ b/internal/harbor/web/board.js @@ -202,7 +202,9 @@ export class Board { } const l = entry.lease; const st = entry.isWaiting ? "waiting" : l.state; - const word = statusText(st); + // a VFIO hold (group "vfio") is a VM session holding the card, not an + // engine's berth: the status word says so + const word = !entry.isWaiting && l.group === "vfio" && st === "active" ? "IN VM" : statusText(st); if (!isNew && this.prevState.get(entry.key) !== word) flash(row, "row-changed"); this.prevState.set(entry.key, word); const tone = { active: "cyan", draining: "amber", stuck: "magenta", waiting: "amber", parking: "amber", parked: "green" }[st] || "dim"; @@ -219,12 +221,12 @@ export class Board { updateCell(c.hold, fit(hold, WIDTH.hold), ""); // status cell: flip tiles, then the WARM and CONTESTED flags syncSi(c.status, st); - updateCell(c.status, fit(statusText(st), WIDTH.status), tone); + updateCell(c.status, fit(word, WIDTH.status), tone); syncWarm(row, entry); syncQueued(row, entry); syncContest(c.status, l.contested); updateCell(c.time, fit(ageText(l.age_seconds), WIDTH.time), ""); - row.setAttribute("aria-label", "open ticket " + l.id + ", " + l.engine + ", " + statusText(st).toLowerCase()); + row.setAttribute("aria-label", "open ticket " + l.id + ", " + l.engine + ", " + word.toLowerCase()); const title = rowTitle(entry); if (title) row.title = title; else row.removeAttribute("title"); @@ -311,6 +313,7 @@ function syncQueued(row, entry) { function rowTitle(entry) { const l = entry.lease; const bits = []; + if (l.group === "vfio") bits.push("this hold pins the card for a virtual machine session"); if (l.contested > 0) bits.push("granted while holders without a lease kept " + fmtBytes(l.contested) + " on this card"); if (!entry.isWaiting && l.state === "parked") { const ram = entry.fp > 0 ? "about " + gib(entry.fp) : "an unknown amount"; diff --git a/internal/harbor/web/charts.js b/internal/harbor/web/charts.js index 238cfac..d4c980e 100644 --- a/internal/harbor/web/charts.js +++ b/internal/harbor/web/charts.js @@ -223,17 +223,22 @@ export function sparkline(series, opts = {}) { if (sig === ctl._sig) return; ctl._sig = sig; let lo = o.min; - let hi = o.max == null ? Math.max(...data, o.min) : o.max; + let hi = o.max == null ? Math.max(...data.filter((v) => v != null), o.min) : o.max; if (hi <= lo) hi = lo + 1; const step = (w - o.pad * 2) / (n - 1); const y = (v) => h - o.pad - ((v - lo) / (hi - lo)) * (h - o.pad * 2); + // null = no data (the card was in a VM): the run breaks, it never + // draws as zero let d = ""; + let run = false; for (let i = 0; i < n; i++) { + if (data[i] == null) { run = false; continue; } const x = o.pad + i * step; - d += (i === 0 ? "M" : "L") + x.toFixed(2) + " " + y(data[i]).toFixed(2); + d += (run ? "L" : "M") + x.toFixed(2) + " " + y(data[i]).toFixed(2); + run = true; } line.setAttribute("d", d); - if (o.area) area.setAttribute("d", d + "L" + (o.pad + (n - 1) * step).toFixed(2) + " " + (h - o.pad) + "L" + o.pad + " " + (h - o.pad) + "Z"); + if (o.area && !data.some((v) => v == null)) area.setAttribute("d", d + "L" + (o.pad + (n - 1) * step).toFixed(2) + " " + (h - o.pad) + "L" + o.pad + " " + (h - o.pad) + "Z"); else area.setAttribute("d", ""); }, }; @@ -402,7 +407,7 @@ export function stackedArea(data, opts = {}) { const n = t.length; if (n < 2) { gBands.replaceChildren(); gGrid.replaceChildren(); gEvents.replaceChildren(); frame.clear(); state = null; return; } const total = data.total || 0; - state = { t, t0: t[0], t1: t[n - 1], total, series: data.series || [], events: data.events || [] }; + state = { t, t0: t[0], t1: t[n - 1], total, series: data.series || [], events: data.events || [], holes: data.holes || [] }; // y grid in the svg, its numbers, the x times and the total in the overlay gGrid.replaceChildren(); frame.clear(); @@ -422,30 +427,37 @@ export function stackedArea(data, opts = {}) { totalLine.setAttribute("y1", gyTotal); totalLine.setAttribute("y2", gyTotal); frame.axisX("TOTAL " + (Math.round(total * 10) / 10) + "G", padL + 4, gyTotal - 4, "total-label"); - // stacked bands, bottom to top + // stacked bands, bottom to top. A hole (the card was in a VM, no data) + // breaks every band: each gap-free run of samples closes over itself, so + // the gap reads as a gap and no frozen reading draws across it. gBands.replaceChildren(); const series = state.series; + const hole = (i) => series.some((b) => (b.data || [])[i] == null); + const runs = []; + let runStart = -1; + for (let i = 0; i < n; i++) { + if (!hole(i) && runStart < 0) runStart = i; + if (hole(i) && runStart >= 0) { runs.push([runStart, i - 1]); runStart = -1; } + } + if (runStart >= 0) runs.push([runStart, n - 1]); const cum = new Array(n).fill(0); - const lower = new Array(n).fill(0); for (const band of series) { const dataArr = band.data || []; - const lo = new Array(n).fill(0); - for (let i = 0; i < n; i++) { - lo[i] = cum[i]; - cum[i] += dataArr[i] || 0; - } - let d = ""; - for (let i = 0; i < n; i++) { - const yTop = padT + plotH - (cum[i] / (total || 1)) * plotH; - d += (i === 0 ? "M" : "L") + xOf(t[i]).toFixed(1) + " " + yTop.toFixed(1); - } - for (let i = n - 1; i >= 0; i--) { - const yBot = padT + plotH - (lo[i] / (total || 1)) * plotH; - d += "L" + xOf(t[i]).toFixed(1) + " " + yBot.toFixed(1); - } - d += "Z"; const isSquat = band.kind === "squat"; const isBase = band.kind === "baseline"; + let d = ""; + const edge = (lo, hi) => { // one gap-free run: top edge forward, bottom edge back, closed + let p = ""; + for (let i = lo; i <= hi; i++) { + p += (i === lo ? "M" : "L") + xOf(t[i]).toFixed(1) + " " + (padT + plotH - (cum[i] / (total || 1)) * plotH).toFixed(1); + } + for (let i = hi; i >= lo; i--) { + p += "L" + xOf(t[i]).toFixed(1) + " " + (padT + plotH - ((cum[i] - (dataArr[i] || 0)) / (total || 1)) * plotH).toFixed(1); + } + return p + "Z"; + }; + for (let i = 0; i < n; i++) cum[i] += dataArr[i] || 0; + for (const [lo, hi] of runs) d += edge(lo, hi); const path = S("path", { d, "vector-effect": "non-scaling-stroke" }); path.style.fill = isSquat ? "url(#" + pid + ")" : band.color || PAL.baseline; // translucent bands with a crisp edge read as layers, not slabs of paint @@ -486,6 +498,12 @@ export function stackedArea(data, opts = {}) { cross.setAttribute("x1", px); cross.setAttribute("x2", px); cross.setAttribute("visibility", "visible"); + if (state.holes[idx]) { + const html = tipBox(tipRow(o.fmtTip(state.t[idx]), "", null)); + html.append(tipRow("", "no data, the card was in a VM", null)); + showTip(tip, wrap, ev, html); + return; + } let used = 0; const html = tipBox(tipRow(o.fmtTip(state.t[idx]), "", null)); for (const band of state.series) { diff --git a/internal/harbor/web/harbor.css b/internal/harbor/web/harbor.css index d45ac1f..46f2683 100644 --- a/internal/harbor/web/harbor.css +++ b/internal/harbor/web/harbor.css @@ -395,6 +395,9 @@ body.mode-basic .adv-only { display: none !important; } /* the demo page has no live node, so the state pill says DEMO */ .conn.state.state-demo { color: var(--strip-demo-ink); border-color: var(--magenta-dim); } .conn.state.state-demo .dot { background: var(--magenta); } +/* the GPU IN VM tag rides beside the state pill on every tab: the governed card + is bound to vfio-pci, a known state, not a fault */ +.conn.vm-tag { color: var(--amber); border-color: var(--amber-dim); background: var(--amber-wash); } /* ---------- the status indicator ---------- */ /* One vocabulary for every state word: an 8px shape in the word's own colour, @@ -917,6 +920,11 @@ body.ready #panel-engines > :nth-child(n+2) { animation-delay: 60ms; } .berth-name { font-size: var(--fs-small); color: var(--dim); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; } .berth-note { font-size: var(--fs-micro); letter-spacing: 1px; color: var(--dim); } .berth.is-scheduled .berth-note { color: var(--cyan); } +/* the card in a VM: the passthrough word reads as the card's state, and the + last memory reading beside it is greyed and labelled as its age, never current */ +.berth.is-vfio .memnums { grid-template-columns: repeat(2, minmax(0, 1fr)); } +.berth.is-vfio .memnums .n:last-child .v, +.berth.is-vfio .memnums .n:last-child .l { color: var(--dim-2); } .membar { display: flex; height: 24px; diff --git a/internal/harbor/web/harbor.js b/internal/harbor/web/harbor.js index ed12d3d..a664694 100644 --- a/internal/harbor/web/harbor.js +++ b/internal/harbor/web/harbor.js @@ -80,6 +80,7 @@ const state = { logQuery: "", // the log toolbar's text filter, matched case-insensitively storageFailed: false, // the store read failed: the storage panel says so lastSeq: 0, + vmFirstSeen: 0, // first poll that read the card as passthrough, for the gap the event ring did not keep demoHist: null, // ?demo: the recorded 1 h window the longer ranges are synthesized from boardSort: null, // departures board sort: null | "vessel" | "hold" | "status" | "time" alertSeen: {}, // key -> first-seen ms, so a sticky alert names its age @@ -123,7 +124,7 @@ function initialLbRange() { } let board, ticket, engineView, residency, priorityView, libraryView; -let connPill, statePill, nodeVersion, lastUpdate, demoStrip, alertStripEl; +let connPill, statePill, vmPill, nodeVersion, lastUpdate, demoStrip, alertStripEl; let odometerReels, counterSince, counterMeta; let ctiles; let logList, logNote, logMeta, refusalTallyEl, logCountEl, logWrapEl, logJumpEl, logSearchInput; @@ -147,6 +148,7 @@ let histSeries = {}; // precomputed per-history sparkline series function main() { connPill = $("conn-pill"); statePill = $("state-pill"); + vmPill = $("vm-pill"); alertStripEl = $("alerts"); nodeVersion = $("node-version"); lastUpdate = $("last-update"); @@ -749,8 +751,26 @@ function paintConnAge() { function refreshConn() { setConn(); paintOverall(); + paintVM(); paintLastUpdate(); } +// The VM tag: the governed card is not on the node, it is bound to vfio-pci or +// switching to or from that. It reads beside the state pill on every tab, so +// the card's absence is visible without opening a panel. Hidden when the node +// reports no passthrough. +let vmSig = null; +function paintVM() { + const p = state.status && state.status.passthrough; + const sig = p ? p.state + "|" + p.driver : ""; + if (sig === vmSig || !vmPill) return; + vmSig = sig; + if (!p) { vmPill.hidden = true; return; } + vmPill.hidden = false; + setText(vmPill, p.state === "detaching" ? "GPU TO VM" : "GPU IN VM"); + vmPill.title = p.state === "detaching" + ? "a VFIO hold is granted and the card's driver is switching, driver " + (p.driver || "unbound") + : "the governed card is bound to " + p.driver + " and serves a virtual machine"; +} function paintLastUpdate() { if (!lastUpdate) return; lastUpdate.textContent = state.lastGood ? "updated " + relTime(state.lastGood) : "never read"; @@ -765,18 +785,33 @@ function overallState() { if (state.demo) return "demo"; const c = connState(); if (c === "down") return "offline"; - const lv = (state.alerts && state.alerts.levels) || {}; - const latched = (state.status && state.status.engines || []).some((e) => e.state === "latched"); + const lv = alertLevels(); + // While the card is in a VM no engine can hold it, so an engine stopped for + // the session is expected, not a fault. + const inVM = !!(state.status && state.status.passthrough); + const latched = !inVM && (state.status && state.status.engines || []).some((e) => e.state === "latched"); if (Object.keys(lv).length > 0 || latched) return "degraded"; if (c === "stale") return "degraded"; // no fresh status within 10 s: not provably live return "live"; } +// alertLevels is the page's view of the node's sticky levels. While the card is +// in a VM (or switching to one) its probe failure is the known passthrough +// state, not a fault, so the probe level reaches neither pill nor strip. +function alertLevels() { + const lv = (state.alerts && state.alerts.levels) || {}; + if (state.status && state.status.passthrough && lv.probe) { + const rest = { ...lv }; + delete rest.probe; + return rest; + } + return lv; +} // the state pill is a live region too: rebuild it only when the state or its // explanation changes, not on every poll let stateSig = null; function paintOverall() { const s = overallState(); - const lv = (state.alerts && state.alerts.levels) || {}; + const lv = alertLevels(); const sorted = alertKeys(lv); const title = sorted.length ? alertMsg(sorted[0], lv[sorted[0]], state.status) : ""; const sig = s + "|" + title; @@ -808,6 +843,7 @@ async function fetchStatus() { } else if (res.ok) { state.status = await res.json(); state.alerts = state.status.alerts || null; + state.vmFirstSeen = state.status.passthrough ? (state.vmFirstSeen || Date.now()) : 0; state.nodeDown = null; state.fetchFailing = false; state.lastGood = Date.now(); @@ -1374,7 +1410,8 @@ function paintAlerts(s) { const now = Date.now(); for (const k of Object.keys(lv)) if (!state.alertSeen[k]) state.alertSeen[k] = now; for (const k of Object.keys(state.alertSeen)) if (!lv[k]) { delete state.alertSeen[k]; delete state.alertDismissed[k]; } - const keys = alertKeys(lv).filter((k) => state.alertDismissed[k] !== lv[k]); + const keys = alertKeys(lv).filter((k) => + state.alertDismissed[k] !== lv[k] && !(s.passthrough && k === "probe")); const sig = keys.map((k) => k + ":" + lv[k]).join("|") + "#" + keys.map((k) => alertMsg(k, lv[k], s)).join("|"); if (sig === lastAlertSig) return; lastAlertSig = sig; @@ -1403,6 +1440,42 @@ function paintAlerts(s) { } // ---------------- the tide ---------------- +// The card's windows in a VM: [start, end] pairs in epoch ms, recovered from +// the node's card events, so the tide draws the session as a gap in the data +// and never as the frozen last reading. An open window is one with no end: +// the card is still in the VM right now, or the page saw a passthrough status +// whose start event has fallen out of the event ring. +function vmWindows(now) { + const out = []; + let open = null; + for (const ev of state.events) { + if (ev.kind === "card.passthrough") { + if (open == null) open = new Date(ev.at).getTime(); + } else if (ev.kind === "card.returned" && open != null) { + out.push([open, new Date(ev.at).getTime()]); + open = null; + } + } + if (open != null) { + out.push([open, now]); + return out; + } + return state.status && state.status.passthrough && state.vmFirstSeen + ? [[state.vmFirstSeen, now]] + : out; +} +function inVmWindow(t, win) { + for (const [a, b] of win) if (t >= a && t <= b) return true; + return false; +} +// The card events' plain words in the log and on the tide's event ticks: the +// node's own detail is the driver, the word is the state. +function eventCardText(e) { + const g = schedGpu(state.status); + const card = g && g.index != null ? "gpu" + g.index : "the card"; + const bare = e.kind === "card.passthrough"; + return bare ? card + " passed through to a VM" : card + " back from the VM"; +} // The axis reads the current range when it paints, so one chart object // serves every range: clock times within a day, short dates for 7d and 30d. function xAxisLabel(d) { @@ -1447,6 +1520,11 @@ function tideSeries(s) { const samples = state.history && state.history.samples ? state.history.samples : []; const n = samples.length; if (!sched || n < 2) return null; + // samples inside a VM passthrough window carry the frozen pre-hold reading: + // they are holes, not data + const win = vmWindows(Date.now()); + const holes = samples.map((sm) => inVmWindow(new Date(sm.t).getTime(), win)); + const nHoles = holes.filter(Boolean).length; const total = sched.total / GIB; const base = (sched.baseline || 0) / GIB; const byName = new Map(); @@ -1462,11 +1540,17 @@ function tideSeries(s) { const series = [{ label: "baseline", kind: "baseline", color: PAL.baseline, data: new Array(n).fill(base) }]; names.forEach((eng, i) => series.push({ label: eng, kind: "holder", color: HOLDER_PALETTE[i % HOLDER_PALETTE.length], data: byName.get(eng).data })); series.push({ label: "squatter", kind: "squat", color: PAL.rose, data: samples.map((sm) => (sm.squat || 0) / GIB) }); + if (nHoles) for (const band of series) band.data = band.data.map((v, i) => (holes[i] ? null : v)); const events = state.events .filter((ev) => EVENT_TONE[ev.kind]) .map((ev) => ({ t: new Date(ev.at).getTime(), tone: EVENT_TONE[ev.kind], kind: ev.kind, engine: ev.engine || "", detail: ev.detail || "" })); const t = samples.map((sm) => new Date(sm.t).getTime()); - return { total, base, t, series, events }; + // the card events ride the tide too: a hole sticks out, its boundary says why + for (const ev of state.events) { + if (ev.kind !== "card.passthrough" && ev.kind !== "card.returned") continue; + events.push({ t: new Date(ev.at).getTime(), tone: "stuck", kind: kindWord(ev.kind), engine: "", detail: eventCardText(ev) }); + } + return { total, base, t, series, events, holes, nHoles }; } function updateTide(s) { buildTide(); @@ -1487,10 +1571,10 @@ function updateTide(s) { tideMeta.textContent = meta; setAttr(tideChart.el, "aria-label", "scheduled card memory by holder, " + RANGE_WORD[state.range] + ". Focus shows a data point."); const lastHolder = data.series[1] ? (data.series[1].data[data.t.length - 1] || 0).toFixed(1) : "0"; - const sig = data.total + "|" + data.t[0] + "|" + data.t[data.t.length - 1] + "|" + data.series.map((b) => b.label).join(",") + "|" + lastHolder + "|" + data.events.length; + const sig = data.total + "|" + data.t[0] + "|" + data.t[data.t.length - 1] + "|" + data.series.map((b) => b.label).join(",") + "|" + lastHolder + "|" + data.events.length + "|" + data.nHoles; if (sig === tideChart._sig) { paintTideLegend(data.series); return; } tideChart._sig = sig; - tideChart.set({ t: data.t, total: data.total, series: data.series, events: data.events }); + tideChart.set({ t: data.t, total: data.total, series: data.series, events: data.events, holes: data.holes }); paintTideLegend(data.series); } function firstSched(s) { @@ -1693,6 +1777,10 @@ function renderBerths(s) { function updateBerth(card, g, s) { card.root.classList.toggle("is-scheduled", !!g.scheduled); card.root.classList.toggle("inventory", !g.scheduled); + // the card in a VM: its status names the state, its figures are the last + // probe read before the hold, which the card does not serve now + const pt = g.scheduled && s.passthrough ? s.passthrough : null; + card.root.classList.toggle("is-vfio", !!pt); // the vram level tints the whole card; it comes from the node's sticky // levels, never re-derived here const alv = (s.alerts && s.alerts.levels) || {}; @@ -1731,12 +1819,24 @@ function updateBerth(card, g, s) { if (node.style.width !== w) node.style.width = w; if (b.title && node.title !== b.title) node.title = b.title; }); - setNums(card.memnums, [ - [fmtGiB(g.used) + " / " + fmtGiB(g.total) + " GiB", "USED / TOTAL"], - [fmtGiB(g.free) + " GiB", "FREE"], - [fmtGiB(g.baseline) + " GiB", "BASELINE"], - [fmtAge(g.age_seconds), "READING AGE"], - ]); + if (pt) { + // the passthrough word reads as the card's state in place of the figures; + // the last reading keeps its cell, greyed by .is-vfio, labelled as its age + const word = pt.state === "detaching" + ? "being passed through to a VM" + (pt.driver ? " (driver " + pt.driver + ")" : "") + : "passed through to a VM (" + pt.driver + ")"; + setNums(card.memnums, [ + [word, "CARD STATE"], + [fmtGiB(g.used) + " / " + fmtGiB(g.total) + " GiB", "LAST READING, READ " + fmtAge(g.age_seconds) + " AGO"], + ]); + } else { + setNums(card.memnums, [ + [fmtGiB(g.used) + " / " + fmtGiB(g.total) + " GiB", "USED / TOTAL"], + [fmtGiB(g.free) + " GiB", "FREE"], + [fmtGiB(g.baseline) + " GiB", "BASELINE"], + [fmtAge(g.age_seconds), "READING AGE"], + ]); + } paintTelemetry(card, g, s); // occupants card.occ.replaceChildren(); @@ -1751,7 +1851,8 @@ function updateBerth(card, g, s) { if (l.parked_since != null) ram += ", parked " + fmtAge(l.parked_since); } const cls = "occ" + (l.state === "stuck" ? " stuck" : l.state === "parked" ? " warm" : ""); - card.occ.append(el("div", cls, el("span", "who", tier + " " + l.engine + " " + (l.id || "")), el("span", null, l.state === "parked" ? ram.trim() : l.used > 0 ? fmtGiB(l.used) + " GiB" : "draining"))); + const sits = l.group === "vfio" ? "in the VM" : l.state === "parked" ? ram.trim() : l.used > 0 ? fmtGiB(l.used) + " GiB" : "draining"; + card.occ.append(el("div", cls, el("span", "who", tier + " " + l.engine + " " + (l.id || "")), el("span", null, sits))); } for (const q of s.squatters || []) card.occ.append(el("div", "occ squatter", el("span", "who", "STOWAWAY " + q.name), el("span", null, fmtGiB(q.used) + " GiB"))); if (card.occ.childElementCount === 0) card.occ.append(el("div", "occ", el("span", "who", "all berths free"))); @@ -1826,17 +1927,28 @@ function updateBerthCharts(s) { if (!card.g || !card.g.scheduled) continue; card.charts.replaceChildren(); if (samples.length < 2) { card.charts.append(el("div", "empty-note", "no history yet")); continue; } - const pick = (fn) => samples.map((sm) => { const gs = (sm.gpus || []).find((x) => x.index === idx); return gs ? gs[fn] || 0 : 0; }); + // samples inside a VM passthrough window are no data, never a frozen + // reading drawn as if the card answered + const win = vmWindows(Date.now()); + const pick = (fn) => samples.map((sm) => { + if (inVmWindow(new Date(sm.t).getTime(), win)) return null; + const gs = (sm.gpus || []).find((x) => x.index === idx); + return gs ? gs[fn] || 0 : 0; + }); const mk = (label, data, color, unit) => { const box = el("div", "mini-chart"); box.append(el("div", "mc-label", label)); const c = sparkline(data, { w: 100, h: 22, color, area: true, min: 0, max: maxOf(data) || 1 }); box.append(c.el); - box.append(el("div", "mc-val", fmtNow(data) + unit)); + box.append(el("div", "mc-val", lastOf(data, unit || ""))); return box; }; const maxOf = (a) => a.reduce((m, v) => Math.max(m, v || 0), 0); - const fmtNow = (a) => { const v = a[a.length - 1] || 0; return v >= 1000 ? (v / 1000).toFixed(1) + "k" : v.toFixed(0); }; + const lastOf = (a, unit) => { + const v = a[a.length - 1]; + if (v == null) return "no data"; + return (v >= 1000 ? (v / 1000).toFixed(1) + "k" : v.toFixed(0)) + unit; + }; card.charts.append(mk("UTIL %", pick("util_pct"), PAL.cyan, "%")); card.charts.append(mk("TEMP °C", pick("temp_c"), PAL.amber, "°")); card.charts.append(mk("POWER W", pick("power_w"), PAL.green, "W")); @@ -2112,8 +2224,13 @@ function makeLogRow(e) { const at = new Date(e.at); const row = el("div", "log-row" + (kindCss(e.kind) ? " " + kindCss(e.kind) : "")); const detail = el("span", "ld", e.detail || ""); + if (e.kind === "card.passthrough" || e.kind === "card.returned") { + // the card events speak in plain words; the driver rides in the title + detail.textContent = eventCardText(e); + detail.title = eventCardText(e) + ", device driver " + (e.detail || "unknown"); + } // the phone's 2-line row ellipsises both words, so the whole of each rides on title - detail.title = e.detail || ""; + detail.title = detail.title || e.detail || ""; if (e.kind === "lease.juggle") { const why = el("button", "log-why", "why"); why.type = "button"; diff --git a/internal/harbor/web/index.html b/internal/harbor/web/index.html index d9652a6..ceae2f3 100644 --- a/internal/harbor/web/index.html +++ b/internal/harbor/web/index.html @@ -33,6 +33,7 @@
+ CONNECTING diff --git a/internal/harbor/web/util.js b/internal/harbor/web/util.js index 6faab94..9915c10 100644 --- a/internal/harbor/web/util.js +++ b/internal/harbor/web/util.js @@ -107,6 +107,8 @@ export function kindWord(kind) { case "probe.recovered": return "PROBE UP"; case "alert.raised": return "ALERT"; case "alert.cleared": return "ALERT CLEAR"; + case "card.passthrough": return "GPU TO VM"; + case "card.returned": return "GPU BACK"; default: return String(kind || "EVENT").toUpperCase(); } } @@ -174,6 +176,10 @@ const KIND_SI = { "squatter.gone": "adrift", "probe.failed": "crit", "probe.recovered": "up", + // the card going to and coming back from a VM reads like the lease states: + // away = aground (attention, the card does not answer), back = up + "card.passthrough": "aground", + "card.returned": "up", "alert.raised": "warn", "alert.cleared": "adrift", };