VFIO (experimental): the harbor shows a card passed through to a VM

CHANGELOG.md

@@ -12,6 +12,9 @@ has a matching GitHub release whose body comes from its section below.
   `berth status` and its JSON show it as passed through to a VM, and waiters are told why nothing is
   granted. While a VFIO hold is granted, berthd does not open the card, so a driver switch can run.
   `card.passthrough` and `card.returned` events mark each change.
+- The harbor shows a card passed through to a VM (experimental): a GPU IN VM tag on every tab, the
+  card's panel and the departures board marked, the change named in the log, and the session left as a
+  gap in the tide.
 - `berth vfio status` and `berth vfio run` (experimental, Linux only) for passing the governed card to
   a VM. `status` shows IOMMU, the card's IOMMU group with each function's driver, and the processes
   holding the card. `run -- CMD` drains the card, stops berthd's own probe for the hold, refuses while

docs/harbor.md

@@ -74,7 +74,11 @@ network you trust.
   its own probe would block the driver switch; it reads the driver from sysfs instead. Nothing new is
   granted while the card is away, and each waiter's reason says so ("gpu0 is passed through to a VM
   (vfio-pci); it comes back when the VM's hold ends"). A card switched by hand, with no hold, shows the
-  same state instead of a probe-failed warning.
+  same state instead of a probe-failed warning. In the harbor, a GPU IN VM tag sits beside the state pill
+  on every tab. The card's panel shows the passthrough state in place of its memory figures, with the last
+  reading greyed and labelled with its age. The departures board marks the hold IN VM, the log names each
+  change in plain words, and the tide leaves the session as a gap. Engines stopped for the session do not
+  turn the state DEGRADED.
 - **Alerts:** one threshold table in `berth.toml` (`[alerts]`): warn and crit per key
   (`queue`, `ttft`, `tpot`, `p95`, `cache`, `reingest`, `vram`, `temp`, `power`, `kv`).
   The node evaluates it on its 5 s sample cycle, fires a desktop notification and webhook

internal/harbor/events_test.go

@@ -28,6 +28,7 @@ type fakeNode struct {
 	sock      string
 	lastEvent uint64
 	evts      []proto.Event
+	st        *proto.Status // canned OpStatus reply when set; otherwise an empty status
 
 	ln      net.Listener
 	mu      sync.Mutex
@@ -79,7 +80,10 @@ func (f *fakeNode) serveConn(c net.Conn) {
 	}
 	switch m.Op {
 	case proto.OpStatus:
-		st := &proto.Status{API: proto.APIVersion, LastEvent: f.lastEvent}
+		st := f.st
+		if st == nil {
+			st = &proto.Status{API: proto.APIVersion, LastEvent: f.lastEvent}
+		}
 		_ = conn.Send(proto.Msg{Op: proto.OpStatus, Status: st})
 	case proto.OpEvents:
 		for i := range f.evts {

internal/harbor/harbor_test.go

@@ -187,6 +187,54 @@ func TestListenMode(t *testing.T) {
 	}
 }
 
+// TestStatusPassthroughReachesThePage: a node that reports the governed card
+// in a VM passes the passthrough object and the card's figures through
+// /v1/status unshaped, so the page's panels have the state and the last
+// reading's age to show.
+func TestStatusPassthroughReachesThePage(t *testing.T) {
+	n := newFakeNode(t, 0, nil)
+	n.st = &proto.Status{
+		API: proto.APIVersion,
+		GPUs: []proto.GPU{{
+			Index: 0, Name: "NVIDIA GeForce RTX 4090", Scheduled: true,
+			Total: 24 << 30, Used: 20 << 30, Free: 4 << 30, Baseline: 2 << 30, Age: 300,
+		}},
+		Passthrough: &proto.Passthrough{State: proto.CardPassthrough, Driver: "vfio-pci"},
+	}
+	srv := newEventsHarbor(t, n)
+	resp, err := http.Get(srv.URL + "/v1/status")
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := io.ReadAll(resp.Body)
+	resp.Body.Close()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("/v1/status: %s %s", resp.Status, b)
+	}
+	body := string(b)
+	for _, want := range []string{
+		`"passthrough":{"state":"passthrough","driver":"vfio-pci"}`,
+		`"scheduled":true`,
+		`"age_seconds":300`,
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("/v1/status does not carry %s:\n%s", want, body)
+		}
+	}
+
+	// detaching reads as its own state, a switch under way, not passthrough
+	n.st.Passthrough = &proto.Passthrough{State: proto.CardDetaching, Driver: "nvidia"}
+	resp, err = http.Get(srv.URL + "/v1/status")
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ = io.ReadAll(resp.Body)
+	resp.Body.Close()
+	if !strings.Contains(string(b), `"passthrough":{"state":"detaching","driver":"nvidia"}`) {
+		t.Fatalf("detaching: %s", b)
+	}
+}
+
 // logBuf is a bytes.Buffer safe to poll from the test while Serve logs to it.
 type logBuf struct {
 	mu sync.Mutex

internal/harbor/web/board.js

@@ -202,7 +202,9 @@ export class Board {
       }
       const l = entry.lease;
       const st = entry.isWaiting ? "waiting" : l.state;
-      const word = statusText(st);
+      // a VFIO hold (group "vfio") is a VM session holding the card, not an
+      // engine's berth: the status word says so
+      const word = !entry.isWaiting && l.group === "vfio" && st === "active" ? "IN VM" : statusText(st);
       if (!isNew && this.prevState.get(entry.key) !== word) flash(row, "row-changed");
       this.prevState.set(entry.key, word);
       const tone = { active: "cyan", draining: "amber", stuck: "magenta", waiting: "amber", parking: "amber", parked: "green" }[st] || "dim";
@@ -219,12 +221,12 @@ export class Board {
       updateCell(c.hold, fit(hold, WIDTH.hold), "");
       // status cell: flip tiles, then the WARM and CONTESTED flags
       syncSi(c.status, st);
-      updateCell(c.status, fit(statusText(st), WIDTH.status), tone);
+      updateCell(c.status, fit(word, WIDTH.status), tone);
       syncWarm(row, entry);
       syncQueued(row, entry);
       syncContest(c.status, l.contested);
       updateCell(c.time, fit(ageText(l.age_seconds), WIDTH.time), "");
-      row.setAttribute("aria-label", "open ticket " + l.id + ", " + l.engine + ", " + statusText(st).toLowerCase());
+      row.setAttribute("aria-label", "open ticket " + l.id + ", " + l.engine + ", " + word.toLowerCase());
       const title = rowTitle(entry);
       if (title) row.title = title;
       else row.removeAttribute("title");
@@ -311,6 +313,7 @@ function syncQueued(row, entry) {
 function rowTitle(entry) {
   const l = entry.lease;
   const bits = [];
+  if (l.group === "vfio") bits.push("this hold pins the card for a virtual machine session");
   if (l.contested > 0) bits.push("granted while holders without a lease kept " + fmtBytes(l.contested) + " on this card");
   if (!entry.isWaiting && l.state === "parked") {
     const ram = entry.fp > 0 ? "about " + gib(entry.fp) : "an unknown amount";

internal/harbor/web/charts.js

@@ -223,17 +223,22 @@ export function sparkline(series, opts = {}) {
       if (sig === ctl._sig) return;
       ctl._sig = sig;
       let lo = o.min;
-      let hi = o.max == null ? Math.max(...data, o.min) : o.max;
+      let hi = o.max == null ? Math.max(...data.filter((v) => v != null), o.min) : o.max;
       if (hi <= lo) hi = lo + 1;
       const step = (w - o.pad * 2) / (n - 1);
       const y = (v) => h - o.pad - ((v - lo) / (hi - lo)) * (h - o.pad * 2);
+      // null = no data (the card was in a VM): the run breaks, it never
+      // draws as zero
       let d = "";
+      let run = false;
       for (let i = 0; i < n; i++) {
+        if (data[i] == null) { run = false; continue; }
         const x = o.pad + i * step;
-        d += (i === 0 ? "M" : "L") + x.toFixed(2) + " " + y(data[i]).toFixed(2);
+        d += (run ? "L" : "M") + x.toFixed(2) + " " + y(data[i]).toFixed(2);
+        run = true;
       }
       line.setAttribute("d", d);
-      if (o.area) area.setAttribute("d", d + "L" + (o.pad + (n - 1) * step).toFixed(2) + " " + (h - o.pad) + "L" + o.pad + " " + (h - o.pad) + "Z");
+      if (o.area && !data.some((v) => v == null)) area.setAttribute("d", d + "L" + (o.pad + (n - 1) * step).toFixed(2) + " " + (h - o.pad) + "L" + o.pad + " " + (h - o.pad) + "Z");
       else area.setAttribute("d", "");
     },
   };
@@ -402,7 +407,7 @@ export function stackedArea(data, opts = {}) {
     const n = t.length;
     if (n < 2) { gBands.replaceChildren(); gGrid.replaceChildren(); gEvents.replaceChildren(); frame.clear(); state = null; return; }
     const total = data.total || 0;
-    state = { t, t0: t[0], t1: t[n - 1], total, series: data.series || [], events: data.events || [] };
+    state = { t, t0: t[0], t1: t[n - 1], total, series: data.series || [], events: data.events || [], holes: data.holes || [] };
     // y grid in the svg, its numbers, the x times and the total in the overlay
     gGrid.replaceChildren();
     frame.clear();
@@ -422,30 +427,37 @@ export function stackedArea(data, opts = {}) {
     totalLine.setAttribute("y1", gyTotal);
     totalLine.setAttribute("y2", gyTotal);
     frame.axisX("TOTAL " + (Math.round(total * 10) / 10) + "G", padL + 4, gyTotal - 4, "total-label");
-    // stacked bands, bottom to top
+    // stacked bands, bottom to top. A hole (the card was in a VM, no data)
+    // breaks every band: each gap-free run of samples closes over itself, so
+    // the gap reads as a gap and no frozen reading draws across it.
     gBands.replaceChildren();
     const series = state.series;
+    const hole = (i) => series.some((b) => (b.data || [])[i] == null);
+    const runs = [];
+    let runStart = -1;
+    for (let i = 0; i < n; i++) {
+      if (!hole(i) && runStart < 0) runStart = i;
+      if (hole(i) && runStart >= 0) { runs.push([runStart, i - 1]); runStart = -1; }
+    }
+    if (runStart >= 0) runs.push([runStart, n - 1]);
     const cum = new Array(n).fill(0);
-    const lower = new Array(n).fill(0);
     for (const band of series) {
       const dataArr = band.data || [];
-      const lo = new Array(n).fill(0);
-      for (let i = 0; i < n; i++) {
-        lo[i] = cum[i];
-        cum[i] += dataArr[i] || 0;
-      }
-      let d = "";
-      for (let i = 0; i < n; i++) {
-        const yTop = padT + plotH - (cum[i] / (total || 1)) * plotH;
-        d += (i === 0 ? "M" : "L") + xOf(t[i]).toFixed(1) + " " + yTop.toFixed(1);
-      }
-      for (let i = n - 1; i >= 0; i--) {
-        const yBot = padT + plotH - (lo[i] / (total || 1)) * plotH;
-        d += "L" + xOf(t[i]).toFixed(1) + " " + yBot.toFixed(1);
-      }
-      d += "Z";
       const isSquat = band.kind === "squat";
       const isBase = band.kind === "baseline";
+      let d = "";
+      const edge = (lo, hi) => { // one gap-free run: top edge forward, bottom edge back, closed
+        let p = "";
+        for (let i = lo; i <= hi; i++) {
+          p += (i === lo ? "M" : "L") + xOf(t[i]).toFixed(1) + " " + (padT + plotH - (cum[i] / (total || 1)) * plotH).toFixed(1);
+        }
+        for (let i = hi; i >= lo; i--) {
+          p += "L" + xOf(t[i]).toFixed(1) + " " + (padT + plotH - ((cum[i] - (dataArr[i] || 0)) / (total || 1)) * plotH).toFixed(1);
+        }
+        return p + "Z";
+      };
+      for (let i = 0; i < n; i++) cum[i] += dataArr[i] || 0;
+      for (const [lo, hi] of runs) d += edge(lo, hi);
       const path = S("path", { d, "vector-effect": "non-scaling-stroke" });
       path.style.fill = isSquat ? "url(#" + pid + ")" : band.color || PAL.baseline;
       // translucent bands with a crisp edge read as layers, not slabs of paint
@@ -486,6 +498,12 @@ export function stackedArea(data, opts = {}) {
       cross.setAttribute("x1", px);
       cross.setAttribute("x2", px);
       cross.setAttribute("visibility", "visible");
+      if (state.holes[idx]) {
+        const html = tipBox(tipRow(o.fmtTip(state.t[idx]), "", null));
+        html.append(tipRow("", "no data, the card was in a VM", null));
+        showTip(tip, wrap, ev, html);
+        return;
+      }
       let used = 0;
       const html = tipBox(tipRow(o.fmtTip(state.t[idx]), "", null));
       for (const band of state.series) {

internal/harbor/web/harbor.css

@@ -395,6 +395,9 @@ body.mode-basic .adv-only { display: none !important; }
 /* the demo page has no live node, so the state pill says DEMO */
 .conn.state.state-demo { color: var(--strip-demo-ink); border-color: var(--magenta-dim); }
 .conn.state.state-demo .dot { background: var(--magenta); }
+/* the GPU IN VM tag rides beside the state pill on every tab: the governed card
+   is bound to vfio-pci, a known state, not a fault */
+.conn.vm-tag { color: var(--amber); border-color: var(--amber-dim); background: var(--amber-wash); }
 
 /* ---------- the status indicator ---------- */
 /* One vocabulary for every state word: an 8px shape in the word's own colour,
@@ -917,6 +920,11 @@ body.ready #panel-engines > :nth-child(n+2) { animation-delay: 60ms; }
 .berth-name { font-size: var(--fs-small); color: var(--dim); white-space: nowrap; overflow: hidden; text-overflow: ellipsis; }
 .berth-note { font-size: var(--fs-micro); letter-spacing: 1px; color: var(--dim); }
 .berth.is-scheduled .berth-note { color: var(--cyan); }
+/* the card in a VM: the passthrough word reads as the card's state, and the
+   last memory reading beside it is greyed and labelled as its age, never current */
+.berth.is-vfio .memnums { grid-template-columns: repeat(2, minmax(0, 1fr)); }
+.berth.is-vfio .memnums .n:last-child .v,
+.berth.is-vfio .memnums .n:last-child .l { color: var(--dim-2); }
 .membar {
   display: flex;
   height: 24px;