VFIO (experimental): berthd shows a card passed through to a VM and stays off it during a switch

CHANGELOG.md

@@ -8,6 +8,10 @@ has a matching GitHub release whose body comes from its section below.
 
 ### Added
 
+- Card passed through to a VM (experimental). While the governed card is bound to `vfio-pci`,
+  `berth status` and its JSON show it as passed through to a VM, and waiters are told why nothing is
+  granted. While a VFIO hold is granted, berthd does not open the card, so a driver switch can run.
+  `card.passthrough` and `card.returned` events mark each change.
 - Gateway fallback routing: an `[[engine]]` may name a `fallback`, another engine in the file that
   needs no card (a CPU llama-server, an Ollama on another box), and `fallback_after` (10 s by
   default). A request that has waited that long for its own engine is proxied to the fallback when

docs/harbor.md

@@ -66,6 +66,15 @@ network you trust.
   frees: the measured starts in the waiter's chain, one per seat ahead plus its own. An
   unmeasured start says unmeasured and adds no number, and the holder's drain is never in
   the figure, because a request runs as long as it runs.
+- **A card passed through to a VM (experimental).** When the governed card is bound to `vfio-pci`,
+  `berth status` shows "passed through to a VM (vfio-pci)" in place of its memory figures, and the JSON
+  has a `passthrough` object with the state and the bound driver. The state reads `detaching` while a
+  switch is under way. `card.passthrough` and `card.returned` mark each change once in the event log.
+  While a VFIO hold is granted (`berth vfio run`), berthd does not open the card, because
+  its own probe would block the driver switch; it reads the driver from sysfs instead. Nothing new is
+  granted while the card is away, and each waiter's reason says so ("gpu0 is passed through to a VM
+  (vfio-pci); it comes back when the VM's hold ends"). A card switched by hand, with no hold, shows the
+  same state instead of a probe-failed warning.
 - **Alerts:** one threshold table in `berth.toml` (`[alerts]`): warn and crit per key
   (`queue`, `ttft`, `tpot`, `p95`, `cache`, `reingest`, `vram`, `temp`, `power`, `kv`).
   The node evaluates it on its 5 s sample cycle, fires a desktop notification and webhook

internal/broker/broker.go

@@ -331,6 +331,14 @@ type Broker struct {
 	mu       sync.Mutex
 	snap     gpu.Snapshot
 	probeErr error
+	// cardAddr is the governed card's PCI function in sysfs form
+	// ("0000:01:00.0"), remembered from the last successful snapshot and
+	// found by scan when the probe fails and no reading ever came.
+	// cardState is "" normal, CardPassthrough or CardDetaching, and
+	// cardDriver is the driver bound at the last look.
+	cardAddr   string
+	cardState  string
+	cardDriver string
 	// squatClamped is whether the squatter total was last capped at the
 	// card's used figure (boundSquat); a change of that state logs once.
 	squatClamped bool
@@ -613,7 +621,16 @@ func (b *Broker) Tick(ctx context.Context) {
 	defer b.persistFootprints()
 	defer func() { b.persistUsage(b.now(), false) }()
 	read := b.reads.Add(1)
-	snap, err := b.probe.Read(ctx)
+	b.mu.Lock()
+	held := b.vfioHold()
+	b.mu.Unlock()
+	var snap gpu.Snapshot
+	var err error
+	if held == nil {
+		// While a VFIO hold is granted the operator is switching the card,
+		// and the probe opens the device; it does not run at all.
+		snap, err = b.probe.Read(ctx)
+	}
 	var rs []runs.Record
 	if b.cfg.RunsDir != "" {
 		rs = runs.List(b.cfg.RunsDir, b.tree.StartTime)
@@ -625,7 +642,20 @@ func (b *Broker) Tick(ctx context.Context) {
 	b.runs = rs
 	b.accountUsage(now)
 	b.pruneVerdicts(now)
+	if held != nil {
+		// The switch is berthd's to watch, not to touch: read the driver
+		// from sysfs, grant nothing, keep the hold HOT.
+		b.holdTick()
+		return
+	}
 	if err != nil {
+		if d := b.vfioBound(); d != "" {
+			// The card is in a VM, switched by hand: no probe-failed
+			// warning, the card state says it instead.
+			b.setCard(proto.CardPassthrough, d)
+			b.schedule(now)
+			return
+		}
 		if b.probeErr == nil {
 			b.log.Warn("gpu probe failed; scheduling on the lease table alone", "err", err)
 			b.emit(proto.Event{Kind: proto.EvProbeFailed, Detail: err.Error()})
@@ -639,6 +669,14 @@ func (b *Broker) Tick(ctx context.Context) {
 		b.emit(proto.Event{Kind: proto.EvProbeRecovered})
 	}
 	b.probeErr = nil
+	if a := gpu.SysfsAddr(gpu.BusID(snap)); a != "" {
+		b.cardAddr = a
+	}
+	if b.cardState != "" {
+		// The probe answers again, so the card is back; the driver it reads
+		// now is the one the returned event names.
+		b.setCard("", gpu.DriverFor(b.cardAddr))
+	}
 	b.snap = snap
 	b.recordDescendants()
 	b.refreshScopes()
@@ -892,6 +930,13 @@ func (b *Broker) allowed(name string) bool {
 // holders to drain for the first waiter that does not fit. Nobody behind a
 // blocked waiter jumps it, so a big request is not starved by small ones.
 func (b *Broker) schedule(now time.Time) {
+	if b.cardState != "" {
+		// The card does not serve the node, so nothing new is granted, and
+		// every waiter reads the same reason. The VFIO hold itself keeps
+		// the card.
+		b.notifyWait(b.queue, b.cardWhy())
+		return
+	}
 	sort.SliceStable(b.queue, func(i, j int) bool {
 		if b.queue[i].prio != b.queue[j].prio {
 			return b.queue[i].prio > b.queue[j].prio

internal/broker/status.go

@@ -51,6 +51,9 @@ func (b *Broker) Status() proto.Status {
 	if b.probeErr != nil {
 		st.ProbeErr = b.probeErr.Error()
 	}
+	if b.cardState != "" {
+		st.Passthrough = &proto.Passthrough{State: b.cardState, Driver: b.cardDriver}
+	}
 	devs := b.snap.Devices
 	if len(devs) == 0 && b.snap.Total > 0 {
 		devs = []gpu.Device{{Index: b.snap.Index, Name: b.snap.Name, Total: b.snap.Total, Used: b.snap.Used, Free: b.snap.Free,

internal/broker/vfio.go

@@ -0,0 +1,88 @@
+// Copyright 2026 Trevin Tindall (T-Crypt)
+// SPDX-License-Identifier: Apache-2.0
+
+package broker
+
+// The card's VM passthrough state. While the governed card is in a VM (or
+// on its way to or from one) the probe cannot answer, so the driver bound
+// to the card's PCI function says where the card is. Everything here is
+// read-only against the machine: the switch itself is the operator's,
+// guarded by a VFIO hold, a pinned acquire in Group "vfio".
+
+import (
+	"fmt"
+
+	"github.com/T-Crypt/berth/internal/gpu"
+	"github.com/T-Crypt/berth/internal/proto"
+)
+
+// vfioHold is the granted lease in Group "vfio" that guards a switch, or
+// nil. A releasing or stuck hold has ended; the probe reads the card again.
+// Called with b.mu held.
+func (b *Broker) vfioHold() *lease {
+	for _, l := range b.leases {
+		if l.group == proto.GroupVFIO && l.state != releasing && l.state != stuck {
+			return l
+		}
+	}
+	return nil
+}
+
+// holdTick is the tick while a VFIO hold is granted: the probe is never
+// called, the card state comes from the driver bound to the function, and
+// schedule grants nothing new. The hold itself stays HOT. Called with b.mu
+// held.
+func (b *Broker) holdTick() {
+	if b.cardAddr == "" {
+		// berthd never saw the card: it can still find the one function
+		// the VM's passthrough bound, the same scan a probe failure runs.
+		b.cardAddr = gpu.FindVFIOCard()
+	}
+	if d := gpu.DriverFor(b.cardAddr); d == gpu.VFIODriver {
+		b.setCard(proto.CardPassthrough, d)
+	} else {
+		b.setCard(proto.CardDetaching, d)
+	}
+	b.schedule(b.now())
+}
+
+// vfioBound reads the governed card's driver after a probe failure:
+// vfio-pci when the card is in a VM (switched by hand), "" otherwise. When
+// no reading ever named the card's address, it is found by scan and
+// remembered. Called with b.mu held.
+func (b *Broker) vfioBound() string {
+	if b.cardAddr == "" {
+		b.cardAddr = gpu.FindVFIOCard()
+	}
+	if b.cardAddr == "" || gpu.DriverFor(b.cardAddr) != gpu.VFIODriver {
+		return ""
+	}
+	return gpu.VFIODriver
+}
+
+// setCard records the card state and emits one event per transition into
+// or out of passthrough. Called with b.mu held.
+func (b *Broker) setCard(state, driver string) {
+	if b.cardState == state && b.cardDriver == driver {
+		return
+	}
+	switch {
+	case state == proto.CardPassthrough && b.cardState != proto.CardPassthrough:
+		b.log.Info("card passed through to a VM", "driver", driver, "addr", b.cardAddr)
+		b.emit(proto.Event{Kind: proto.EvCardPassthrough, Detail: driver})
+	case b.cardState == proto.CardPassthrough && state != proto.CardPassthrough:
+		b.log.Info("card returned from the VM", "driver", driver)
+		b.emit(proto.Event{Kind: proto.EvCardReturned, Detail: driver})
+	}
+	b.cardState, b.cardDriver = state, driver
+}
+
+// cardWhy is the reason every waiter reads while the card does not serve
+// the node: where the card is and what brings it back. Called with b.mu
+// held.
+func (b *Broker) cardWhy() string {
+	if b.cardState == proto.CardDetaching {
+		return fmt.Sprintf("gpu%d is being passed through to a VM; it comes back when the VM's hold ends", b.snap.Index)
+	}
+	return fmt.Sprintf("gpu%d is passed through to a VM (%s); it comes back when the VM's hold ends", b.snap.Index, b.cardDriver)
+}