diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f6ffb..73e4b8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ has a matching GitHub release whose body comes from its section below. ### Added +- Card passed through to a VM (experimental). While the governed card is bound to `vfio-pci`, + `berth status` and its JSON show it as passed through to a VM, and waiters are told why nothing is + granted. While a VFIO hold is granted, berthd does not open the card, so a driver switch can run. + `card.passthrough` and `card.returned` events mark each change. - Gateway fallback routing: an `[[engine]]` may name a `fallback`, another engine in the file that needs no card (a CPU llama-server, an Ollama on another box), and `fallback_after` (10 s by default). A request that has waited that long for its own engine is proxied to the fallback when diff --git a/docs/harbor.md b/docs/harbor.md index a88e471..34af109 100644 --- a/docs/harbor.md +++ b/docs/harbor.md @@ -66,6 +66,15 @@ network you trust. frees: the measured starts in the waiter's chain, one per seat ahead plus its own. An unmeasured start says unmeasured and adds no number, and the holder's drain is never in the figure, because a request runs as long as it runs. +- **A card passed through to a VM (experimental).** When the governed card is bound to `vfio-pci`, + `berth status` shows "passed through to a VM (vfio-pci)" in place of its memory figures, and the JSON + has a `passthrough` object with the state and the bound driver. The state reads `detaching` while a + switch is under way. `card.passthrough` and `card.returned` mark each change once in the event log. + While a VFIO hold is granted (`berth vfio run`), berthd does not open the card, because + its own probe would block the driver switch; it reads the driver from sysfs instead. Nothing new is + granted while the card is away, and each waiter's reason says so ("gpu0 is passed through to a VM + (vfio-pci); it comes back when the VM's hold ends"). A card switched by hand, with no hold, shows the + same state instead of a probe-failed warning. - **Alerts:** one threshold table in `berth.toml` (`[alerts]`): warn and crit per key (`queue`, `ttft`, `tpot`, `p95`, `cache`, `reingest`, `vram`, `temp`, `power`, `kv`). The node evaluates it on its 5 s sample cycle, fires a desktop notification and webhook diff --git a/internal/broker/broker.go b/internal/broker/broker.go index d3a9974..045330c 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -331,6 +331,14 @@ type Broker struct { mu sync.Mutex snap gpu.Snapshot probeErr error + // cardAddr is the governed card's PCI function in sysfs form + // ("0000:01:00.0"), remembered from the last successful snapshot and + // found by scan when the probe fails and no reading ever came. + // cardState is "" normal, CardPassthrough or CardDetaching, and + // cardDriver is the driver bound at the last look. + cardAddr string + cardState string + cardDriver string // squatClamped is whether the squatter total was last capped at the // card's used figure (boundSquat); a change of that state logs once. squatClamped bool @@ -613,7 +621,16 @@ func (b *Broker) Tick(ctx context.Context) { defer b.persistFootprints() defer func() { b.persistUsage(b.now(), false) }() read := b.reads.Add(1) - snap, err := b.probe.Read(ctx) + b.mu.Lock() + held := b.vfioHold() + b.mu.Unlock() + var snap gpu.Snapshot + var err error + if held == nil { + // While a VFIO hold is granted the operator is switching the card, + // and the probe opens the device; it does not run at all. + snap, err = b.probe.Read(ctx) + } var rs []runs.Record if b.cfg.RunsDir != "" { rs = runs.List(b.cfg.RunsDir, b.tree.StartTime) @@ -625,7 +642,20 @@ func (b *Broker) Tick(ctx context.Context) { b.runs = rs b.accountUsage(now) b.pruneVerdicts(now) + if held != nil { + // The switch is berthd's to watch, not to touch: read the driver + // from sysfs, grant nothing, keep the hold HOT. + b.holdTick() + return + } if err != nil { + if d := b.vfioBound(); d != "" { + // The card is in a VM, switched by hand: no probe-failed + // warning, the card state says it instead. + b.setCard(proto.CardPassthrough, d) + b.schedule(now) + return + } if b.probeErr == nil { b.log.Warn("gpu probe failed; scheduling on the lease table alone", "err", err) b.emit(proto.Event{Kind: proto.EvProbeFailed, Detail: err.Error()}) @@ -639,6 +669,14 @@ func (b *Broker) Tick(ctx context.Context) { b.emit(proto.Event{Kind: proto.EvProbeRecovered}) } b.probeErr = nil + if a := gpu.SysfsAddr(gpu.BusID(snap)); a != "" { + b.cardAddr = a + } + if b.cardState != "" { + // The probe answers again, so the card is back; the driver it reads + // now is the one the returned event names. + b.setCard("", gpu.DriverFor(b.cardAddr)) + } b.snap = snap b.recordDescendants() b.refreshScopes() @@ -892,6 +930,13 @@ func (b *Broker) allowed(name string) bool { // holders to drain for the first waiter that does not fit. Nobody behind a // blocked waiter jumps it, so a big request is not starved by small ones. func (b *Broker) schedule(now time.Time) { + if b.cardState != "" { + // The card does not serve the node, so nothing new is granted, and + // every waiter reads the same reason. The VFIO hold itself keeps + // the card. + b.notifyWait(b.queue, b.cardWhy()) + return + } sort.SliceStable(b.queue, func(i, j int) bool { if b.queue[i].prio != b.queue[j].prio { return b.queue[i].prio > b.queue[j].prio diff --git a/internal/broker/status.go b/internal/broker/status.go index d82feba..ee90fe8 100644 --- a/internal/broker/status.go +++ b/internal/broker/status.go @@ -51,6 +51,9 @@ func (b *Broker) Status() proto.Status { if b.probeErr != nil { st.ProbeErr = b.probeErr.Error() } + if b.cardState != "" { + st.Passthrough = &proto.Passthrough{State: b.cardState, Driver: b.cardDriver} + } devs := b.snap.Devices if len(devs) == 0 && b.snap.Total > 0 { devs = []gpu.Device{{Index: b.snap.Index, Name: b.snap.Name, Total: b.snap.Total, Used: b.snap.Used, Free: b.snap.Free, diff --git a/internal/broker/vfio.go b/internal/broker/vfio.go new file mode 100644 index 0000000..32a5096 --- /dev/null +++ b/internal/broker/vfio.go @@ -0,0 +1,88 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package broker + +// The card's VM passthrough state. While the governed card is in a VM (or +// on its way to or from one) the probe cannot answer, so the driver bound +// to the card's PCI function says where the card is. Everything here is +// read-only against the machine: the switch itself is the operator's, +// guarded by a VFIO hold, a pinned acquire in Group "vfio". + +import ( + "fmt" + + "github.com/T-Crypt/berth/internal/gpu" + "github.com/T-Crypt/berth/internal/proto" +) + +// vfioHold is the granted lease in Group "vfio" that guards a switch, or +// nil. A releasing or stuck hold has ended; the probe reads the card again. +// Called with b.mu held. +func (b *Broker) vfioHold() *lease { + for _, l := range b.leases { + if l.group == proto.GroupVFIO && l.state != releasing && l.state != stuck { + return l + } + } + return nil +} + +// holdTick is the tick while a VFIO hold is granted: the probe is never +// called, the card state comes from the driver bound to the function, and +// schedule grants nothing new. The hold itself stays HOT. Called with b.mu +// held. +func (b *Broker) holdTick() { + if b.cardAddr == "" { + // berthd never saw the card: it can still find the one function + // the VM's passthrough bound, the same scan a probe failure runs. + b.cardAddr = gpu.FindVFIOCard() + } + if d := gpu.DriverFor(b.cardAddr); d == gpu.VFIODriver { + b.setCard(proto.CardPassthrough, d) + } else { + b.setCard(proto.CardDetaching, d) + } + b.schedule(b.now()) +} + +// vfioBound reads the governed card's driver after a probe failure: +// vfio-pci when the card is in a VM (switched by hand), "" otherwise. When +// no reading ever named the card's address, it is found by scan and +// remembered. Called with b.mu held. +func (b *Broker) vfioBound() string { + if b.cardAddr == "" { + b.cardAddr = gpu.FindVFIOCard() + } + if b.cardAddr == "" || gpu.DriverFor(b.cardAddr) != gpu.VFIODriver { + return "" + } + return gpu.VFIODriver +} + +// setCard records the card state and emits one event per transition into +// or out of passthrough. Called with b.mu held. +func (b *Broker) setCard(state, driver string) { + if b.cardState == state && b.cardDriver == driver { + return + } + switch { + case state == proto.CardPassthrough && b.cardState != proto.CardPassthrough: + b.log.Info("card passed through to a VM", "driver", driver, "addr", b.cardAddr) + b.emit(proto.Event{Kind: proto.EvCardPassthrough, Detail: driver}) + case b.cardState == proto.CardPassthrough && state != proto.CardPassthrough: + b.log.Info("card returned from the VM", "driver", driver) + b.emit(proto.Event{Kind: proto.EvCardReturned, Detail: driver}) + } + b.cardState, b.cardDriver = state, driver +} + +// cardWhy is the reason every waiter reads while the card does not serve +// the node: where the card is and what brings it back. Called with b.mu +// held. +func (b *Broker) cardWhy() string { + if b.cardState == proto.CardDetaching { + return fmt.Sprintf("gpu%d is being passed through to a VM; it comes back when the VM's hold ends", b.snap.Index) + } + return fmt.Sprintf("gpu%d is passed through to a VM (%s); it comes back when the VM's hold ends", b.snap.Index, b.cardDriver) +} diff --git a/internal/broker/vfio_test.go b/internal/broker/vfio_test.go new file mode 100644 index 0000000..96c8906 --- /dev/null +++ b/internal/broker/vfio_test.go @@ -0,0 +1,376 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package broker + +import ( + "context" + "errors" + "io" + "log/slog" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/gpu" + "github.com/T-Crypt/berth/internal/procs" + "github.com/T-Crypt/berth/internal/proto" +) + +// vfioBroker is a broker over the fake probe, configured the way the hold +// tests set theirs up. +func vfioBroker(t *testing.T, p gpu.Probe) *Broker { + t.Helper() + return New(Config{ + ReleaseGrace: time.Hour, + DefaultWait: 2 * time.Second, + JuggleWindow: -1, + Log: slog.New(slog.NewTextHandler(io.Discard, nil)), + }, p, procs.Map{}) +} + +// vfioRequest is the acquire V2's command makes, pinned at hold priority +// in the group the broker matches on. +func vfioRequest() Request { + return Request{Engine: "vfio:galaxy", Group: "vfio", Priority: Hold, Pinned: true} +} + +// fakeProbe is a card reader that counts its calls and fails on demand, +// so a test can watch whether the broker opens the device at all. +type fakeProbe struct { + mu sync.Mutex + calls int + fail bool + busID string +} + +func (f *fakeProbe) set(fail bool, busID string) { + f.mu.Lock() + defer f.mu.Unlock() + f.fail, f.busID = fail, busID +} + +func (f *fakeProbe) reads() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.calls +} + +func (f *fakeProbe) Read(context.Context) (gpu.Snapshot, error) { + f.mu.Lock() + f.calls++ + fail, busID := f.fail, f.busID + f.mu.Unlock() + if fail { + return gpu.Snapshot{}, errors.New("device gone") + } + return gpu.Snapshot{ + Index: 0, Name: "RTX 4090", Total: 24 * GiB, + Devices: []gpu.Device{{Index: 0, Name: "RTX 4090", BusID: busID, Total: 24 * GiB}}, + }, nil +} + +// sysfsCard builds a fake sysfs root with the PCI function at addr bound +// to driver ("" unbound), and points gpu.SysfsRoot at it for the test. The +// root is returned for later rebinds. +func sysfsCard(t *testing.T, addr, driver string) string { + t.Helper() + root := t.TempDir() + bindCard(t, root, addr, driver) + gpu.SysfsRoot = root + t.Cleanup(func() { gpu.SysfsRoot = "/sys" }) + return root +} + +// bindCard points the function's driver symlink at driver, "" unbinding it. +func bindCard(t *testing.T, root, addr, driver string) { + t.Helper() + base := filepath.Join(root, "bus", "pci", "devices", addr) + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Remove(filepath.Join(base, "driver")); err != nil && !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + if driver == "" { + return + } + dst := filepath.Join(root, "bus", "pci", "drivers", driver) + if err := os.MkdirAll(dst, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(dst, filepath.Join(base, "driver")); err != nil { + t.Fatal(err) + } +} + +// displayAttrs writes the vendor and class a discovery scan reads. +func displayAttrs(t *testing.T, root, addr string) { + t.Helper() + base := filepath.Join(root, "bus", "pci", "devices", addr) + for name, value := range map[string]string{"vendor": "0x10de", "class": "0x030000"} { + if err := os.WriteFile(filepath.Join(base, name), []byte(value+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } +} + +// skipNonLinux guards the behavior tests: the card state reads sysfs, +// which only Linux has. +func skipNonLinux(t *testing.T) { + t.Helper() + if runtime.GOOS != "linux" { + t.Skip("the card state reads sysfs") + } +} + +// kinds counts the broker's events by kind. +func kinds(b *Broker) map[string]int { + m := map[string]int{} + evs, _, stop := b.Events(0, false) + defer stop() + for _, e := range evs { + m[e.Kind]++ + } + return m +} + +// lastWaiting is the newest lease.waiting reason. +func lastWaiting(b *Broker) string { + why := "" + evs, _, stop := b.Events(0, false) + defer stop() + for _, e := range evs { + if e.Kind == proto.EvWaiting { + why = e.Detail + } + } + return why +} + +// While a VFIO hold is granted, Tick never calls the probe. +func TestVFIOHoldSkipsProbe(t *testing.T) { + skipNonLinux(t) + sysfsCard(t, "0000:01:00.0", "nvidia") + p := &fakeProbe{busID: "00000000:01:00.0"} + b := vfioBroker(t, p) + b.Tick(context.Background()) + hold := must(t, acquire(b, vfioRequest())) + defer hold.Release() + + b.Tick(context.Background()) + b.Tick(context.Background()) + + if n := p.reads(); n != 1 { + t.Fatalf("probe ran %d times, want 1 (the tick before the hold)", n) + } + if card := b.Status().Passthrough; card == nil || card.State != proto.CardDetaching || card.Driver != "nvidia" { + t.Fatalf("card state = %+v, want detaching on driver nvidia", card) + } + st := b.Status() + if len(st.Leases) != 1 || st.Leases[0].State != "active" || st.Leases[0].Group != "vfio" { + t.Fatalf("leases = %+v, want the VFIO hold HOT", st.Leases) + } +} + +// Detaching, then passthrough, then returned: one event per transition and +// no probe-failed warning anywhere. +func TestVFIOTransitions(t *testing.T) { + skipNonLinux(t) + root := sysfsCard(t, "0000:01:00.0", "nvidia") + p := &fakeProbe{busID: "00000000:01:00.0"} + b := vfioBroker(t, p) + b.Tick(context.Background()) + hold := must(t, acquire(b, vfioRequest())) + + b.Tick(context.Background()) + if card := b.Status().Passthrough; card == nil || card.State != proto.CardDetaching { + t.Fatalf("card state = %+v, want detaching while the hold is on", card) + } + + bindCard(t, root, "0000:01:00.0", "vfio-pci") + b.Tick(context.Background()) + if card := b.Status().Passthrough; card == nil || card.State != proto.CardPassthrough || card.Driver != "vfio-pci" { + t.Fatalf("card state = %+v, want passthrough on vfio-pci", card) + } + if n := p.reads(); n != 1 { + t.Fatalf("probe ran %d times under the hold, want 1", n) + } + + hold.Release() + bindCard(t, root, "0000:01:00.0", "nvidia") + b.Tick(context.Background()) + if card := b.Status().Passthrough; card != nil { + t.Fatalf("card state = %+v, want none once the probe answers again", card) + } + + k := kinds(b) + if k[proto.EvCardPassthrough] != 1 || k[proto.EvCardReturned] != 1 { + t.Fatalf("events = %v, want one passthrough and one returned", k) + } + if k[proto.EvProbeFailed] != 0 { + t.Fatalf("probe.failed fired %d times, want 0", k[proto.EvProbeFailed]) + } +} + +// A waiter under passthrough is told why in plain words and is not granted. +func TestWaiterToldTheCardIsPassthrough(t *testing.T) { + skipNonLinux(t) + sysfsCard(t, "0000:01:00.0", "vfio-pci") + p := &fakeProbe{busID: "00000000:01:00.0"} + b := vfioBroker(t, p) + b.Tick(context.Background()) + hold := must(t, acquire(b, vfioRequest())) + defer hold.Release() + b.Tick(context.Background()) + if card := b.Status().Passthrough; card == nil || card.State != proto.CardPassthrough { + t.Fatalf("card state = %+v, want passthrough", card) + } + + waiter := acquire(b, Request{Engine: "llama", Priority: Foreground, Needs: 4 * GiB}) + pending(t, waiter) + b.Tick(context.Background()) + + st := b.Status() + if len(st.Leases) != 1 { + t.Fatalf("granted under passthrough: %+v", st.Leases) + } + if len(st.Waiting) != 1 || st.Waiting[0].Engine != "llama" || st.Waiting[0].Position != 1 { + t.Fatalf("waiting = %+v, want llama next in line", st.Waiting) + } + select { + case <-waiter: + t.Fatal("the waiter was granted a card that is in a VM") + default: + } + why := lastWaiting(b) + if !strings.Contains(why, "next in line: gpu0 is passed through to a VM (vfio-pci); it comes back when the VM's hold ends") { + t.Fatalf("wait reason = %q", why) + } +} + +// A probe failure with the card bound to vfio-pci and no hold is a hand +// switch: the card reads passthrough, the probe stays silent, and the card +// is reported returned once the probe answers again. +func TestHandSwitchedPassthrough(t *testing.T) { + skipNonLinux(t) + root := sysfsCard(t, "0000:01:00.0", "vfio-pci") + displayAttrs(t, root, "0000:01:00.0") + p := &fakeProbe{} + p.set(true, "") + b := vfioBroker(t, p) + b.Tick(context.Background()) + st := b.Status() + if card := st.Passthrough; card == nil || card.State != proto.CardPassthrough || card.Driver != "vfio-pci" { + t.Fatalf("card state = %+v, want passthrough on vfio-pci", card) + } + if st.ProbeErr != "" { + t.Fatalf("probe error = %q, want none in passthrough", st.ProbeErr) + } + + p.set(false, "00000000:01:00.0") + b.Tick(context.Background()) + if card := b.Status().Passthrough; card != nil { + t.Fatalf("card state = %+v, want none after the card returns", card) + } + k := kinds(b) + if k[proto.EvCardPassthrough] != 1 || k[proto.EvCardReturned] != 1 || k[proto.EvProbeFailed] != 0 { + t.Fatalf("events = %v, want one passthrough, one returned, no probe failure", k) + } +} + +// Without exactly one vfio-bound card to find, a probe failure keeps +// today's warning. +func TestProbeFailureWithoutOneVFIOCardWarns(t *testing.T) { + for _, tc := range []struct { + name string + binding []string // one driver name per card, in address order + }{ + {"no display function bound", nil}, + {"card on its host driver", []string{"nvidia"}}, + {"two vfio cards", []string{"vfio-pci", "vfio-pci"}}, + } { + t.Run(tc.name, func(t *testing.T) { + skipNonLinux(t) + root := t.TempDir() + gpu.SysfsRoot = root + t.Cleanup(func() { gpu.SysfsRoot = "/sys" }) + for i, addr := range []string{"0000:01:00.0", "0000:02:00.0"} { + driver := "" + if i < len(tc.binding) { + driver = tc.binding[i] + } + if driver != "" { + bindCard(t, root, addr, driver) + displayAttrs(t, root, addr) + } + } + p := &fakeProbe{} + p.set(true, "") + b := vfioBroker(t, p) + b.Tick(context.Background()) + + if card := b.Status().Passthrough; card != nil { + t.Fatalf("card state = %+v, want none", card) + } + if k := kinds(b); k[proto.EvProbeFailed] != 1 { + t.Fatalf("probe.failed fired %d times, want today's warning once", k[proto.EvProbeFailed]) + } + }) + } +} + +// A card remembered on nvidia whose probe then fails is an ordinary probe +// failure: the warning stands. +func TestProbeFailureOnNvidiaWarns(t *testing.T) { + skipNonLinux(t) + root := sysfsCard(t, "0000:01:00.0", "nvidia") + p := &fakeProbe{} + p.set(false, "00000000:01:00.0") + b := vfioBroker(t, p) + b.Tick(context.Background()) + p.set(true, "") + b.Tick(context.Background()) + + if card := b.Status().Passthrough; card != nil { + t.Fatalf("card state = %+v, want none", card) + } + if k := kinds(b); k[proto.EvProbeFailed] != 1 { + t.Fatalf("probe.failed fired %d times, want once", k[proto.EvProbeFailed]) + } + _ = root +} + +// A VFIO hold asked for while an engine holds the card drains it and is +// granted like any hold. Until the grant the card serves the node: the +// probe runs and the queue is scheduled, or the hold could never be granted. +func TestVFIOHoldDrainsTheHolderFirst(t *testing.T) { + skipNonLinux(t) + sysfsCard(t, "0000:01:00.0", "nvidia") + p := &fakeProbe{busID: "00000000:01:00.0"} + b := vfioBroker(t, p) + b.Tick(context.Background()) + engine := must(t, acquire(b, Request{Engine: "llama", Priority: Interactive})) + + req := acquire(b, vfioRequest()) + pending(t, req) + drained(t, engine) + b.Tick(context.Background()) + if card := b.Status().Passthrough; card != nil { + t.Fatalf("card state = %+v while the hold still waits, want none", card) + } + engine.Release() + for i := 0; i < 3; i++ { + b.Tick(context.Background()) + } + hold := must(t, req) + defer hold.Release() + b.Tick(context.Background()) + if card := b.Status().Passthrough; card == nil || card.State != proto.CardDetaching { + t.Fatalf("card state = %+v after the grant, want detaching", card) + } +} diff --git a/internal/cli/status.go b/internal/cli/status.go index 9cf516f..82a99da 100644 --- a/internal/cli/status.go +++ b/internal/cli/status.go @@ -28,9 +28,17 @@ func (v statusView) Human(c *Context, ok bool) error { c.Printf("%s\n", c.Style.Dim(fmt.Sprintf("berthd %s on %s (api %d)", st.Version, v.sock, st.API))) if len(st.GPUs) == 0 { - c.Printf("gpu no reading yet\n") + if p := st.Passthrough; p != nil { + c.Printf("gpu0 %s\n", cardLine(p)) + } else { + c.Printf("gpu no reading yet\n") + } } for _, g := range st.GPUs { + if g.Scheduled && st.Passthrough != nil { + c.Printf("gpu%d %s: %s, memory read %s ago\n", g.Index, g.Name, cardLine(st.Passthrough), age(g.Age)) + continue + } role := "inventory only" if g.Scheduled { role = "leased by berth, baseline " + server.MiB(g.Baseline) @@ -200,6 +208,18 @@ func (v statusView) Human(c *Context, ok bool) error { return nil } +// cardLine is what the scheduled card's gpu line shows instead of memory +// figures while the card is in a VM or switching to or from one. +func cardLine(p *proto.Passthrough) string { + if p.State == proto.CardDetaching { + if p.Driver == "" { + return "being passed through to a VM" + } + return "being passed through to a VM (driver " + p.Driver + ")" + } + return "passed through to a VM (" + p.Driver + ")" +} + // leaseState paints the one word a reader looks for, and keeps the contested // flag beside it. func leaseState(c *Context, l proto.Lease) string { diff --git a/internal/cli/status_test.go b/internal/cli/status_test.go index 612d8ba..e99f63e 100644 --- a/internal/cli/status_test.go +++ b/internal/cli/status_test.go @@ -230,6 +230,44 @@ func TestStatusShowsTheTurnColumn(t *testing.T) { } } +// The scheduled card's gpu line names the VM instead of memory figures +// while the card is passed through, and the others keep their memory. +func TestStatusShowsTheCardPassthrough(t *testing.T) { + run := func(p *proto.Passthrough) string { + t.Helper() + var out, errOut bytes.Buffer + c := &Context{Out: &out, Err: &errOut, Format: FormatHuman, Style: newStyle(&out), errStyle: newStyle(&errOut)} + st := &proto.Status{ + API: proto.APIVersion, Version: "test", + GPUs: []proto.GPU{ + {Index: 0, Name: "NVIDIA GeForce RTX 4090", Scheduled: true, Total: 24 << 30}, + {Index: 1, Name: "GT 710"}, + }, + Passthrough: p, + } + if err := (statusView{st: st}).Human(c, true); err != nil { + t.Fatal(err) + } + return out.String() + } + s := run(&proto.Passthrough{State: proto.CardPassthrough, Driver: "vfio-pci"}) + for _, line := range strings.Split(s, "\n") { + if !strings.HasPrefix(line, "gpu0") { + continue + } + if !strings.Contains(line, "passed through to a VM (vfio-pci)") || strings.Contains(line, " used of ") { + t.Errorf("the scheduled card's line = %q, want the VM and no memory figures", line) + } + } + if !strings.Contains(s, "inventory only") { + t.Errorf("stdout %q lacks the other card's memory line", s) + } + s = run(&proto.Passthrough{State: proto.CardDetaching, Driver: "nvidia"}) + if !strings.Contains(s, "being passed through to a VM (driver nvidia)") { + t.Errorf("stdout %q lacks the switching state", s) + } +} + func TestStatusYAML(t *testing.T) { isolate(t) sock := startDaemon(t) diff --git a/internal/gpu/gpu.go b/internal/gpu/gpu.go index 5bc1b55..00c0129 100644 --- a/internal/gpu/gpu.go +++ b/internal/gpu/gpu.go @@ -166,8 +166,8 @@ type dedicatedUsage struct { Bytes uint64 } -// busID is the governed card's pci.bus_id, from this read's device list. -func busID(s Snapshot) string { +// BusID is the governed card's pci.bus_id, from this read's device list. +func BusID(s Snapshot) string { for _, d := range s.Devices { if d.Index == s.Index { return d.BusID diff --git a/internal/gpu/gpu_test.go b/internal/gpu/gpu_test.go index 71b355c..88f6d91 100644 --- a/internal/gpu/gpu_test.go +++ b/internal/gpu/gpu_test.go @@ -79,11 +79,11 @@ func TestBusID(t *testing.T) { {Index: 1, Name: "B", BusID: "00000000:02:00.0"}, }, } - if got := busID(s); got != "00000000:02:00.0" { - t.Fatalf("busID = %q, want the governed card's id", got) + if got := BusID(s); got != "00000000:02:00.0" { + t.Fatalf("BusID = %q, want the governed card's id", got) } - if got := busID(Snapshot{Index: 9}); got != "" { - t.Fatalf("busID for a missing card = %q, want empty", got) + if got := BusID(Snapshot{Index: 9}); got != "" { + t.Fatalf("BusID for a missing card = %q, want empty", got) } } diff --git a/internal/gpu/pdh_windows.go b/internal/gpu/pdh_windows.go index 297d2bc..349b63e 100644 --- a/internal/gpu/pdh_windows.go +++ b/internal/gpu/pdh_windows.go @@ -297,7 +297,7 @@ func enumerateAdaptersCached() ([]adapter, error) { // adapters on the box are dropped. It reports an error when no single adapter // matches, which the caller takes as "keep every adapter". func targetLUID(s Snapshot) (map[luid]bool, error) { - id := busID(s) + id := BusID(s) bus, dev, fn, ok := parseBusID(id) if !ok { return nil, fmt.Errorf("the governed card's pci.bus_id %q does not parse", id) diff --git a/internal/gpu/vfio.go b/internal/gpu/vfio.go new file mode 100644 index 0000000..6c5aa80 --- /dev/null +++ b/internal/gpu/vfio.go @@ -0,0 +1,111 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package gpu + +// Reading the kernel driver a PCI function is bound to. This is the +// read-only half of the VFIO story: while the card is in a VM the probe +// cannot answer, and the driver binding is what says so. Nothing here +// writes to sysfs, unbinds a driver or loads a module. + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strconv" + "strings" +) + +// SysfsRoot is where the PCI tree is mounted. Tests point it at a temp dir. +var SysfsRoot = "/sys" + +// VFIODriver is the kernel driver a VM passthrough binds the card to. +const VFIODriver = "vfio-pci" + +// DriverFor is the kernel driver bound to the PCI function named by addr +// ("0000:01:00.0"): the basename of its driver symlink. "" when the +// function is unbound, the address is unknown, the tree is unreadable, or +// the machine is not Linux. +func DriverFor(addr string) string { + if runtime.GOOS != "linux" || addr == "" { + return "" + } + dst, err := os.Readlink(filepath.Join(SysfsRoot, "bus", "pci", "devices", addr, "driver")) + if err != nil { + return "" + } + return filepath.Base(dst) +} + +// SysfsAddr turns nvidia-smi's bus id ("00000000:01:00.0") into the device +// name sysfs uses ("0000:01:00.0"): the domain is read as hex and printed +// at least four digits wide, and the rest is lowercased, as the kernel +// names devices. "" when the bus id is not a PCI address with a domain. +func SysfsAddr(busID string) string { + dom, rest, ok := strings.Cut(busID, ":") + if !ok { + return "" + } + d, err := strconv.ParseUint(dom, 16, 32) + if err != nil { + return "" + } + bus, tail, ok := strings.Cut(rest, ":") + if !ok { + return "" + } + dev, fn, ok := strings.Cut(tail, ".") + if !ok { + return "" + } + for _, part := range []string{bus, dev, fn} { + if _, err := strconv.ParseUint(part, 16, 32); err != nil { + return "" + } + } + return fmt.Sprintf("%04x:%s", d, strings.ToLower(rest)) +} + +// FindVFIOCard names the one PCI display function bound to vfio-pci, by its +// sysfs device name: vendor 0x10de (NVIDIA) and class 0x03xxxx (display). +// "" when none or several match, so a guess never decides. Off Linux: "". +func FindVFIOCard() string { + if runtime.GOOS != "linux" { + return "" + } + devs, err := os.ReadDir(filepath.Join(SysfsRoot, "bus", "pci", "devices")) + if err != nil { + return "" + } + found := "" + for _, d := range devs { + addr := d.Name() + base := filepath.Join(SysfsRoot, "bus", "pci", "devices", addr) + if DriverFor(addr) != VFIODriver { + continue + } + vendor, err := sysfsHex(filepath.Join(base, "vendor")) + if err != nil || vendor != 0x10de { + continue + } + class, err := sysfsHex(filepath.Join(base, "class")) + if err != nil || class&0xFF0000 != 0x030000 { + continue + } + if found != "" { + return "" + } + found = addr + } + return found +} + +// sysfsHex reads one sysfs attribute like "0x030000" as a number. +func sysfsHex(path string) (uint64, error) { + b, err := os.ReadFile(path) + if err != nil { + return 0, err + } + return strconv.ParseUint(strings.TrimSpace(string(b)), 0, 32) +} diff --git a/internal/gpu/vfio_test.go b/internal/gpu/vfio_test.go new file mode 100644 index 0000000..2465168 --- /dev/null +++ b/internal/gpu/vfio_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package gpu + +import ( + "os" + "path/filepath" + "runtime" + "testing" +) + +// setSysfs points SysfsRoot at a temp root and restores the old value. +func setSysfs(t *testing.T, root string) { + t.Helper() + old := SysfsRoot + SysfsRoot = root + t.Cleanup(func() { SysfsRoot = old }) +} + +// pciFunction builds one PCI function in the sysfs root: an address dir, a +// driver dir whose symlink binds it (driver "" leaves it unbound), and the +// given vendor and class attributes when the strings are set. +func pciFunction(t *testing.T, root, addr, driver, vendor, class string) { + t.Helper() + base := filepath.Join(root, "bus", "pci", "devices", addr) + if err := os.MkdirAll(base, 0o755); err != nil { + t.Fatal(err) + } + if driver != "" { + dst := filepath.Join(root, "bus", "pci", "drivers", driver) + if err := os.MkdirAll(dst, 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(dst, filepath.Join(base, "driver")); err != nil { + t.Fatal(err) + } + } + for name, value := range map[string]string{"vendor": vendor, "class": class} { + if value == "" { + continue + } + if err := os.WriteFile(filepath.Join(base, name), []byte(value+"\n"), 0o644); err != nil { + t.Fatal(err) + } + } +} + +func TestSysfsAddr(t *testing.T) { + for _, tc := range []struct { + in, want string + }{ + {"00000000:01:00.0", "0000:01:00.0"}, + {"00000000:1A:00.0", "0000:1a:00.0"}, + {"0000C101:01:00.0", "c101:01:00.0"}, + {"", ""}, + {"01:00.0", ""}, + {"domain/01:00.0", ""}, + } { + if got := SysfsAddr(tc.in); got != tc.want { + t.Errorf("SysfsAddr(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestDriverFor(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sysfs is a Linux view") + } + root := t.TempDir() + setSysfs(t, root) + pciFunction(t, root, "0000:01:00.0", "vfio-pci", "", "") + pciFunction(t, root, "0000:02:00.0", "nvidia", "", "") + pciFunction(t, root, "0000:03:00.0", "", "", "") + + if got := DriverFor("0000:01:00.0"); got != "vfio-pci" { + t.Errorf("bound function: DriverFor = %q, want vfio-pci", got) + } + if got := DriverFor("0000:02:00.0"); got != "nvidia" { + t.Errorf("host driver: DriverFor = %q, want nvidia", got) + } + if got := DriverFor("0000:03:00.0"); got != "" { + t.Errorf("unbound function: DriverFor = %q, want empty", got) + } + if got := DriverFor("0000:04:00.0"); got != "" { + t.Errorf("missing function: DriverFor = %q, want empty", got) + } + if got := DriverFor(""); got != "" { + t.Errorf("empty address: DriverFor = %q, want empty", got) + } +} + +func TestFindVFIOCard(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip("sysfs is a Linux view") + } + t.Run("one match", func(t *testing.T) { + root := t.TempDir() + setSysfs(t, root) + pciFunction(t, root, "0000:01:00.0", "vfio-pci", "0x10de", "0x030000") + if got := FindVFIOCard(); got != "0000:01:00.0" { + t.Fatalf("FindVFIOCard() = %q, want 0000:01:00.0", got) + } + }) + t.Run("bound to another driver", func(t *testing.T) { + root := t.TempDir() + setSysfs(t, root) + pciFunction(t, root, "0000:01:00.0", "nvidia", "0x10de", "0x030000") + if got := FindVFIOCard(); got != "" { + t.Fatalf("FindVFIOCard() = %q, want empty", got) + } + }) + t.Run("not a display function", func(t *testing.T) { + root := t.TempDir() + setSysfs(t, root) + pciFunction(t, root, "0000:01:00.0", "vfio-pci", "0x10de", "0x020000") + if got := FindVFIOCard(); got != "" { + t.Fatalf("FindVFIOCard() = %q, want empty", got) + } + }) + t.Run("another vendor", func(t *testing.T) { + root := t.TempDir() + setSysfs(t, root) + pciFunction(t, root, "0000:01:00.0", "vfio-pci", "0x8086", "0x030000") + if got := FindVFIOCard(); got != "" { + t.Fatalf("FindVFIOCard() = %q, want empty", got) + } + }) + t.Run("two matches", func(t *testing.T) { + root := t.TempDir() + setSysfs(t, root) + pciFunction(t, root, "0000:01:00.0", "vfio-pci", "0x10de", "0x030000") + pciFunction(t, root, "0000:02:00.0", "vfio-pci", "0x10de", "0x030000") + if got := FindVFIOCard(); got != "" { + t.Fatalf("FindVFIOCard() = %q, want empty", got) + } + }) +} diff --git a/internal/proto/proto.go b/internal/proto/proto.go index 6e6395b..7d4f9de 100644 --- a/internal/proto/proto.go +++ b/internal/proto/proto.go @@ -149,6 +149,22 @@ const ( // EvFootprint: a freed run measured far more VRAM than the engine's // learned footprint, so the estimate a next run is offered grows. EvFootprint = "engine.footprint" + // EvCardPassthrough: the governed card's PCI function is bound to + // vfio-pci, a VM holds the card. EvCardReturned: it is back on its + // host driver. Each fires once per transition. + EvCardPassthrough = "card.passthrough" + EvCardReturned = "card.returned" +) + +// Card states, the values Status.Passthrough.State takes. Normal is the +// zero state: no Passthrough in the status at all. +const ( + // CardPassthrough: the card is in a VM, its PCI function is bound to + // vfio-pci. + CardPassthrough = "passthrough" + // CardDetaching: a VFIO hold is granted and the function is not bound + // to vfio-pci (yet or any more); the switch is in progress. + CardDetaching = "detaching" ) // Counters are the collisions board. Every count is an event berthd can @@ -207,6 +223,19 @@ type Status struct { // RAM their warm copies are expected to hold. A reading, never a // reservation. ParkedBytes uint64 `json:"parked_bytes,omitempty"` + // Passthrough is the governed card's VM state while it does not serve + // the node: bound to vfio-pci, or switching to or from that. Omitted + // when the card is normal. + Passthrough *Passthrough `json:"passthrough,omitempty"` +} + +// Passthrough is the card's VM passthrough state. +type Passthrough struct { + // State is CardPassthrough or CardDetaching. + State string `json:"state"` + // Driver is the kernel driver bound to the card's PCI function at the + // last look: vfio-pci, the host driver mid-switch, or "" unbound. + Driver string `json:"driver"` } // EngineFootprint is what the broker learned from one engine's leases: the @@ -592,3 +621,8 @@ func (c *Conn) Recv() (Msg, error) { } return m, json.Unmarshal(line, &m) } + +// GroupVFIO is the lease group of a VFIO hold: a pinned acquire at hold +// priority, Engine "vfio:", made by `berth vfio run`. While one is +// granted berthd does not open the card, so the driver switch can run. +const GroupVFIO = "vfio"