From 1554cd414071b5a199c45f0715a2d26ed23feb25 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 15:56:32 -0600 Subject: [PATCH 1/9] vfio feat(vfio): read a card's IOMMU group, drivers and openers --- internal/runner/runner.go | 21 +++ internal/vfio/linux.go | 216 +++++++++++++++++++++++++ internal/vfio/other.go | 21 +++ internal/vfio/vfio.go | 62 ++++++++ internal/vfio/vfio_linux_test.go | 260 +++++++++++++++++++++++++++++++ 5 files changed, 580 insertions(+) create mode 100644 internal/vfio/linux.go create mode 100644 internal/vfio/other.go create mode 100644 internal/vfio/vfio.go create mode 100644 internal/vfio/vfio_linux_test.go diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 83d0ab6..6750039 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -640,6 +640,27 @@ func command(o Options, out io.Writer) *exec.Cmd { return cmd } +// Isolate, SignalGroup, ForwardSignals and ExitCode are the wrapped-command +// pieces the vfio wrapper reuses: like berth run, it starts the command in +// its own process group with the terminal attached, forwards stop signals +// to it, and exits with its code. The vfio flow itself (the hold, the +// opener check, the driver wait) stays in the cli package. + +// Isolate puts the command in its own process group (a job object on +// Windows), so a stop reaches everything it forked. +func Isolate(cmd *exec.Cmd) { isolate(cmd) } + +// SignalGroup sends one stop signal to the command's process group. +func SignalGroup(cmd *exec.Cmd, s os.Signal) { signalGroup(cmd, s) } + +// ForwardSignals passes stop signals to the command's process group until +// done closes. +func ForwardSignals(cmd *exec.Cmd, done <-chan struct{}) { forwardSignals(cmd, done) } + +// ExitCode is the command's exit status: 128 plus the signal number when it +// was signalled. +func ExitCode(cmd *exec.Cmd, err error) int { return exitCode(cmd, err) } + // forwardSignals passes stop signals (llama-swap stopping a model, Ctrl-C) // to the engine's whole process group. func forwardSignals(cmd *exec.Cmd, done <-chan struct{}) { diff --git a/internal/vfio/linux.go b/internal/vfio/linux.go new file mode 100644 index 0000000..35a8a81 --- /dev/null +++ b/internal/vfio/linux.go @@ -0,0 +1,216 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package vfio + +import ( + "fmt" + "os" + "path/filepath" + "sort" + "strconv" + "strings" +) + +func (r Reader) sysfs() string { + if r.Sysfs == "" { + return "/sys" + } + return r.Sysfs +} + +func (r Reader) proc() string { + if r.Proc == "" { + return "/proc" + } + return r.Proc +} + +// Supported is nil on Linux, where the reader's shapes exist. +func (r Reader) Supported() error { return nil } + +// IOMMUOn reports whether the kernel exposes IOMMU groups, which VFIO +// needs: /sys/kernel/iommu_groups has one entry per group when VT-d or +// AMD-Vi is on (iommu=on iommu=pt, or the platform's own form). +func (r Reader) IOMMUOn() (bool, error) { + entries, err := os.ReadDir(filepath.Join(r.sysfs(), "kernel", "iommu_groups")) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, fmt.Errorf("read the IOMMU groups: %w", err) + } + return len(entries) > 0, nil +} + +// Group reads the card's IOMMU group: the functions in it, each with the +// driver bound to it now, sorted by address. +func (r Reader) Group(addr string) ([]Function, error) { + dir := filepath.Join(r.sysfs(), "bus", "pci", "devices", addr, "iommu_group", "devices") + entries, err := os.ReadDir(dir) + if err != nil { + return nil, fmt.Errorf("read the IOMMU group of %s: %w", addr, err) + } + fs := make([]Function, 0, len(entries)) + for _, e := range entries { + d, err := r.Driver(e.Name()) + if err != nil { + return nil, err + } + fs = append(fs, Function{Addr: e.Name(), Driver: d}) + } + sort.Slice(fs, func(i, j int) bool { return fs[i].Addr < fs[j].Addr }) + return fs, nil +} + +// Driver is the driver bound to one PCI function: the basename of its +// driver symlink, "" when none is bound. +func (r Reader) Driver(addr string) (string, error) { + t, err := os.Readlink(filepath.Join(r.sysfs(), "bus", "pci", "devices", addr, "driver")) + if err != nil { + if os.IsNotExist(err) { + return "", nil + } + return "", fmt.Errorf("read the driver of %s: %w", addr, err) + } + return filepath.Base(t), nil +} + +// Openers lists the processes holding a device node of the card open: every +// /dev/nvidia* and every DRM node of the card under /dev/dri, one entry per +// process, sorted by PID. Berth's own probe (an nvidia-smi under the berth +// process) is named as such, so the vfio commands can tell it apart from a +// real holder. +func (r Reader) Openers(addr string) ([]Opener, error) { + drm, err := r.drmNodes(addr) + if err != nil { + return nil, err + } + entries, err := os.ReadDir(r.proc()) + if err != nil { + return nil, fmt.Errorf("read %s: %w", r.proc(), err) + } + var out []Opener + at := map[int]int{} + for _, e := range entries { + pid, err := strconv.Atoi(e.Name()) + if err != nil { + continue + } + // A process that exited between the listing and now has no fd + // directory left: it holds nothing and is not listed. + if _, err := os.Stat(filepath.Join(r.proc(), e.Name())); err != nil { + continue + } + fds := filepath.Join(r.proc(), e.Name(), "fd") + list, err := os.ReadDir(fds) + if err != nil { + // Permission or a process that died mid-walk: listed as + // unreadable, never skipped. + out = append(out, Opener{PID: pid, Name: r.comm(pid), Unreadable: true}) + continue + } + for _, f := range list { + t, err := os.Readlink(filepath.Join(fds, f.Name())) + if err != nil { + continue // the fd closed between listing and reading + } + if !holdsNode(t, drm) { + continue + } + i, ok := at[pid] + if !ok { + i = len(out) + at[pid] = i + out = append(out, Opener{PID: pid, Name: r.comm(pid), Probe: r.isProbe(pid)}) + } + out[i].Nodes = append(out[i].Nodes, t) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].PID < out[j].PID }) + return out, nil +} + +// drmNodes is the card's DRM node names under /dev/dri, from the device's +// drm directory (card0, renderD128). +func (r Reader) drmNodes(addr string) (map[string]bool, error) { + dir := filepath.Join(r.sysfs(), "bus", "pci", "devices", addr, "drm") + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, fmt.Errorf("read the DRM nodes of %s: %w", addr, err) + } + nodes := make(map[string]bool, len(entries)) + for _, e := range entries { + nodes[e.Name()] = true + } + return nodes, nil +} + +// holdsNode reports whether an open fd's target is a device node of the +// card: any /dev/nvidia* (nvidia0, nvidiactl, nvidia-modeset), or the +// card's own DRM node under /dev/dri. +func holdsNode(target string, drm map[string]bool) bool { + if strings.HasPrefix(target, "/dev/nvidia") { + return true + } + if filepath.Dir(target) == "/dev/dri" && drm[filepath.Base(target)] { + return true + } + return false +} + +// comm is the process's command name; empty when unreadable. +func (r Reader) comm(pid int) string { + b, err := os.ReadFile(filepath.Join(r.proc(), strconv.Itoa(pid), "comm")) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +// isProbe reports whether the process is berthd's own GPU probe: an +// nvidia-smi running under the berth process. +func (r Reader) isProbe(pid int) bool { + if r.comm(pid) != "nvidia-smi" { + return false + } + p := pid + for i := 0; i < 64; i++ { + parent, ok := r.parent(p) + if !ok { + return false + } + if r.comm(parent) == "berth" { + return true + } + p = parent + } + return false +} + +// parent is the process's parent PID from /proc//stat. +func (r Reader) parent(pid int) (int, bool) { + b, err := os.ReadFile(filepath.Join(r.proc(), strconv.Itoa(pid), "stat")) + if err != nil { + return 0, false + } + s := string(b) + i := strings.LastIndexByte(s, ')') + if i < 0 { + return 0, false + } + f := strings.Fields(s[i+1:]) + if len(f) < 2 { + return 0, false + } + ppid, err := strconv.Atoi(f[1]) + if err != nil { + return 0, false + } + return ppid, true +} diff --git a/internal/vfio/other.go b/internal/vfio/other.go new file mode 100644 index 0000000..5d3cf21 --- /dev/null +++ b/internal/vfio/other.go @@ -0,0 +1,21 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build !linux + +package vfio + +import "errors" + +// ErrLinuxOnly is what the reader reports off Linux: the sysfs and /proc +// shapes it reads belong to Linux. The vfio commands refuse with it and +// change nothing. +var ErrLinuxOnly = errors.New("the VFIO state is readable on Linux only in this release") + +// Supported is nil on Linux, ErrLinuxOnly elsewhere. +func (r Reader) Supported() error { return ErrLinuxOnly } + +func (r Reader) IOMMUOn() (bool, error) { return false, ErrLinuxOnly } +func (r Reader) Group(addr string) ([]Function, error) { return nil, ErrLinuxOnly } +func (r Reader) Driver(addr string) (string, error) { return "", ErrLinuxOnly } +func (r Reader) Openers(addr string) ([]Opener, error) { return nil, ErrLinuxOnly } diff --git a/internal/vfio/vfio.go b/internal/vfio/vfio.go new file mode 100644 index 0000000..36fac41 --- /dev/null +++ b/internal/vfio/vfio.go @@ -0,0 +1,62 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +// Package vfio reads the VFIO state of one PCI card: whether IOMMU is on, +// the card's IOMMU group, the driver bound to each function, and the +// processes holding the card's device nodes open. Berth reads this state to +// coordinate a passthrough; it never writes sysfs, never binds or unbinds a +// driver, and never stops a process. +// +// The reader's roots are parameters, so a test points it at a fake tree. +package vfio + +import "strings" + +// Reader reads the card's VFIO state from the machine's sysfs and /proc. +// Empty fields mean the real roots; tests name their own. +type Reader struct { + Sysfs string + Proc string +} + +// Function is one PCI function of the card's IOMMU group and the driver +// bound to it right now. +type Function struct { + Addr string `json:"addr"` // 0000:01:00.0 + Driver string `json:"driver,omitempty"` // nvidia, vfio-pci, ...; "" when none is bound +} + +// Opener is one process holding a device node of the card open, read from +// /proc//fd. A process whose fd directory cannot be read is listed as +// unreadable, never skipped: the reader cannot prove it holds nothing. +type Opener struct { + PID int `json:"pid"` + Name string `json:"name,omitempty"` // the command name, from /proc//comm + Nodes []string `json:"nodes,omitempty"` // /dev/nvidia0, /dev/dri/renderD128 + Probe bool `json:"probe,omitempty"` // berthd's own nvidia-smi probe + Unreadable bool `json:"unreadable,omitempty"` // /proc//fd could not be read +} + +// SysAddr turns nvidia-smi's pci.bus_id form ("00000000:01:00.0") into the +// form sysfs uses for the same function ("0000:01:00.0"). +func SysAddr(busID string) string { + busID = strings.TrimSpace(busID) + domain, fn, ok := strings.Cut(busID, ":") + if !ok { + return busID + } + domain = strings.TrimLeft(domain, "0") + for len(domain) < 4 { + domain = "0" + domain + } + return domain + ":" + fn +} + +// CardFunction reports whether addr is one of the card's own functions: it +// shares the card's bus and device (0000:01:00.1 with 0000:01:00.0), so the +// whole card passes through. Any other function in the group is another +// device, and a group that holds one blocks the passthrough. +func CardFunction(addr, card string) bool { + i, j := strings.LastIndexByte(addr, '.'), strings.LastIndexByte(card, '.') + return i >= 0 && j >= 0 && addr[:i] == card[:j] +} diff --git a/internal/vfio/vfio_linux_test.go b/internal/vfio/vfio_linux_test.go new file mode 100644 index 0000000..71501e0 --- /dev/null +++ b/internal/vfio/vfio_linux_test.go @@ -0,0 +1,260 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package vfio + +import ( + "fmt" + "os" + "path/filepath" + "testing" +) + +// tree is a fake sysfs and /proc for one card at 0000:01:00.0. +type tree struct { + t *testing.T + sysfs string + proc string +} + +func newTree(t *testing.T) tree { + t.Helper() + return tree{t: t, sysfs: t.TempDir(), proc: t.TempDir()} +} + +func (tr tree) reader() Reader { return Reader{Sysfs: tr.sysfs, Proc: tr.proc} } + +// device makes one PCI function with a driver ("" = unbound) and links it +// into the card's IOMMU group, the way sysfs lists the group's members. +func (tr tree) device(addr, driver string) { + tr.t.Helper() + d := filepath.Join(tr.sysfs, "bus", "pci", "devices", addr) + if err := os.MkdirAll(d, 0o755); err != nil { + tr.t.Fatal(err) + } + if driver != "" { + if err := os.Symlink(filepath.Join(tr.sysfs, "bus", "pci", "drivers", driver), filepath.Join(d, "driver")); err != nil { + tr.t.Fatal(err) + } + } + group := filepath.Join(tr.sysfs, "kernel", "iommu_groups", "12") + if err := os.MkdirAll(filepath.Join(group, "devices"), 0o755); err != nil { + tr.t.Fatal(err) + } + if err := os.Symlink(group, filepath.Join(d, "iommu_group")); err != nil { + tr.t.Fatal(err) + } + if err := os.Symlink(d, filepath.Join(group, "devices", addr)); err != nil { + tr.t.Fatal(err) + } +} + +// drm adds the card's DRM node names to sysfs. +func (tr tree) drm(names ...string) { + tr.t.Helper() + dir := filepath.Join(tr.sysfs, "bus", "pci", "devices", cardAddr, "drm") + for _, n := range names { + if err := os.MkdirAll(filepath.Join(dir, n), 0o755); err != nil { + tr.t.Fatal(err) + } + } +} + +// process makes one /proc entry: a command name and a parent, and fd links +// for every node given. +func (tr tree) process(pid int, comm string, parent int, nodes ...string) { + tr.t.Helper() + p := filepath.Join(tr.proc, fmt.Sprintf("%d", pid)) + if err := os.MkdirAll(p, 0o755); err != nil { + tr.t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(p, "comm"), []byte(comm+"\n"), 0o644); err != nil { + tr.t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(p, "stat"), []byte(fmt.Sprintf("%d (%s) S %d 0 0 0 0", pid, comm, parent)), 0o644); err != nil { + tr.t.Fatal(err) + } + fds := filepath.Join(p, "fd") + if err := os.MkdirAll(fds, 0o755); err != nil { + tr.t.Fatal(err) + } + for i, n := range nodes { + if err := os.Symlink(n, filepath.Join(fds, fmt.Sprintf("%d", i))); err != nil { + tr.t.Fatal(err) + } + } +} + +// locked closes one process's fd directory to everyone. +func (tr tree) locked(pid int) { + tr.t.Helper() + if err := os.Chmod(filepath.Join(tr.proc, fmt.Sprintf("%d", pid), "fd"), 0o000); err != nil { + tr.t.Fatal(err) + } +} + +const cardAddr = "0000:01:00.0" + +// TestSysAddrConvertsNvidiaSmisForm covers the bus ids nvidia-smi prints. +func TestSysAddrConvertsNvidiaSmisForm(t *testing.T) { + cases := []struct{ in, want string }{ + {"00000000:01:00.0", "0000:01:00.0"}, + {"00000000:81:00.0", "0000:81:00.0"}, + {"0000:01:00.0", "0000:01:00.0"}, + {" 00000000:01:00.0 ", "0000:01:00.0"}, + } + for _, tc := range cases { + if got := SysAddr(tc.in); got != tc.want { + t.Errorf("SysAddr(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// TestCardFunctionSplitsTheCardFromItsGroup covers the rule that decides a +// passthrough blocker. +func TestCardFunctionSplitsTheCardFromItsGroup(t *testing.T) { + cases := []struct { + addr, card string + want bool + }{ + {"0000:01:00.0", "0000:01:00.0", true}, + {"0000:01:00.1", "0000:01:00.0", true}, // the card's audio function + {"0000:06:00.0", "0000:01:00.0", false}, // another device in the group + {"0000:02:00.0", "0000:01:00.0", false}, // another bus + {"", "0000:01:00.0", false}, + } + for _, tc := range cases { + if got := CardFunction(tc.addr, tc.card); got != tc.want { + t.Errorf("CardFunction(%q, %q) = %v, want %v", tc.addr, tc.card, got, tc.want) + } + } +} + +// TestIOMMUOnReadsTheKernelGroups: no groups means off, entries mean on. +func TestIOMMUOnReadsTheKernelGroups(t *testing.T) { + t.Run("off when the kernel has no groups", func(t *testing.T) { + on, err := Reader{Sysfs: t.TempDir()}.IOMMUOn() + if err != nil || on { + t.Fatalf("IOMMUOn() = %v, %v", on, err) + } + }) + t.Run("on when the kernel has groups", func(t *testing.T) { + root := t.TempDir() + if err := os.MkdirAll(filepath.Join(root, "kernel", "iommu_groups", "12"), 0o755); err != nil { + t.Fatal(err) + } + on, err := Reader{Sysfs: root}.IOMMUOn() + if err != nil || !on { + t.Fatalf("IOMMUOn() = %v, %v", on, err) + } + }) +} + +// TestGroupReadsMembersAndTheirDrivers: the group lists every function and +// the driver each is bound to, unbound reads "", missing group errors. +func TestGroupReadsMembersAndTheirDrivers(t *testing.T) { + tt := newTree(t) + tt.device(cardAddr, "nvidia") + tt.device("0000:01:00.1", "snd_hda_intel") + tt.device("0000:06:00.0", "") + + g, err := tt.reader().Group(cardAddr) + if err != nil { + t.Fatal(err) + } + want := []Function{ + {Addr: cardAddr, Driver: "nvidia"}, + {Addr: "0000:01:00.1", Driver: "snd_hda_intel"}, + {Addr: "0000:06:00.0", Driver: ""}, + } + if len(g) != len(want) { + t.Fatalf("group = %+v, want %+v", g, want) + } + for i, f := range g { + if f != want[i] { + t.Errorf("group[%d] = %+v, want %+v", i, f, want[i]) + } + } + if _, err := tt.reader().Group("0000:09:00.0"); err == nil { + t.Error("a card with no IOMMU group read without error") + } +} + +// TestDriverReadsBoundUnboundAndMissing covers the driver symlink's states. +func TestDriverReadsBoundUnboundAndMissing(t *testing.T) { + tt := newTree(t) + tt.device(cardAddr, "nvidia") + r := tt.reader() + if d, err := r.Driver(cardAddr); err != nil || d != "nvidia" { + t.Fatalf("Driver = %q, %v", d, err) + } + if d, err := r.Driver("0000:01:00.1"); err != nil || d != "" { + t.Fatalf("unbound Driver = %q, %v", d, err) + } + if d, err := r.Driver("0000:09:00.0"); err != nil || d != "" { + t.Fatalf("missing device Driver = %q, %v", d, err) + } +} + +// TestOpenersFindsTheCardHoldersAndNamesBerthdsProbe: /dev/nvidia* and the +// card's DRM nodes are found, one entry per process, the nvidia-smi under +// the berth process is the probe, an unreadable fd directory is listed as +// unreadable, and an unrelated fd is not. +func TestOpenersFindsTheCardHoldersAndNamesBerthdsProbe(t *testing.T) { + tt := newTree(t) + tt.device(cardAddr, "nvidia") + tt.drm("card1", "renderD128") + tt.process(100, "app", 1, "/dev/nvidia0", "/dev/dri/renderD128") + tt.process(101, "other", 1, "/dev/tty1") // not the card's + tt.process(102, "nvidia-smi", 99, "/dev/nvidiactl") + tt.process(104, "app", 1, "/dev/dri/card1") + tt.process(103, "locked", 1) + tt.locked(103) + tt.process(99, "berth", 1) + tt.process(98, "filler", 1) + + ops, err := tt.reader().Openers(cardAddr) + if err != nil { + t.Fatal(err) + } + want := []Opener{ + {PID: 100, Name: "app", Nodes: []string{"/dev/nvidia0", "/dev/dri/renderD128"}}, + {PID: 102, Name: "nvidia-smi", Nodes: []string{"/dev/nvidiactl"}, Probe: true}, + {PID: 103, Name: "locked", Unreadable: true}, + {PID: 104, Name: "app", Nodes: []string{"/dev/dri/card1"}}, + } + if len(ops) != len(want) { + t.Fatalf("openers = %+v, want %+v", ops, want) + } + for i, o := range ops { + if o.PID != want[i].PID || o.Name != want[i].Name || o.Probe != want[i].Probe || o.Unreadable != want[i].Unreadable { + t.Errorf("openers[%d] = %+v, want %+v", i, o, want[i]) + } + if len(o.Nodes) != len(want[i].Nodes) { + t.Errorf("openers[%d].Nodes = %v, want %v", i, o.Nodes, want[i].Nodes) + continue + } + for j, n := range o.Nodes { + if n != want[i].Nodes[j] { + t.Errorf("openers[%d].Nodes[%d] = %q, want %q", i, j, n, want[i].Nodes[j]) + } + } + } +} + +// TestOpenersWithoutDrm: a card whose drm directory is absent reads with no +// error and finds the nvidia holders. +func TestOpenersWithoutDrm(t *testing.T) { + tt := newTree(t) + tt.device(cardAddr, "nvidia") + tt.process(100, "app", 1, "/dev/nvidia0") + ops, err := tt.reader().Openers(cardAddr) + if err != nil { + t.Fatal(err) + } + if len(ops) != 1 || ops[0].PID != 100 || len(ops[0].Nodes) != 1 || ops[0].Nodes[0] != "/dev/nvidia0" { + t.Fatalf("openers = %+v", ops) + } +} From 42e86443f424febc34e5cbcddc1989b93de03ab0 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:10:13 -0600 Subject: [PATCH 2/9] vfio feat(cli): berth vfio status and berth vfio run --- internal/cli/commands.go | 47 ++++ internal/cli/vfio.go | 321 +++++++++++++++++++++++ internal/cli/vfio_run.go | 392 ++++++++++++++++++++++++++++ internal/cli/vfio_test.go | 532 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 1292 insertions(+) create mode 100644 internal/cli/vfio.go create mode 100644 internal/cli/vfio_run.go create mode 100644 internal/cli/vfio_test.go diff --git a/internal/cli/commands.go b/internal/cli/commands.go index eca9641..9e19445 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -561,6 +561,53 @@ same reference again repairs only what does not match.`, Formats: []Format{FormatHuman, FormatJSON, FormatYAML}, Run: runVersion, }, + { + Name: "vfio", + Summary: "pass the card to a VM through VFIO: watch it, wrap the switch script", + Help: `Berth drains the engines, holds the card while your own script switches it to +vfio-pci and runs the VM, and refuses to let the next engine load until the card +is back on its host driver and answering nvidia-smi. Berth never switches a +driver itself, never asks for a password, and never stops a process: your script +(sudo, virsh) owns the switch and the VM. Experimental, Linux only; +docs/vfio.md has the requirements and an example script.`, + Sub: []*Command{ + { + Name: "run", + Summary: "hold the card, run the switch script, wait for the card back", + Args: "[--reason NAME] [--wait D] -- CMD [ARGS...]", + Help: `Acquire the VFIO hold the way berth hold does: engines drain at their +boundaries and none starts. Then the script runs with the terminal attached, so +its sudo prompt reaches you, and with BERTH_VFIO_DEVICES set to the card's PCI +functions, so it hardcodes no address. + +Before the script starts, the openers are re-read: anyone but berthd's own +probe holding /dev/nvidia* or the card's DRM nodes refuses the run, names every +PID and node, releases the hold and exits 1. Berth never stops a process. + +When the script exits, the hold ends only when every function of the card is +off vfio-pci and the card answers nvidia-smi again (up to a minute). Until +then the hold stays, the message names the function that is out and what to do, +and Ctrl-C rechecks: it releases only when the card is back, so the next engine +never loads onto a card that is out.`, + Run: runVfioRun, + }, + { + Name: "status", + Summary: "the card's VFIO state: IOMMU, group, drivers, openers, berthd's view", + Args: "[ADDR] [--socket P] [--format F]", + Help: `Print the card's VFIO state and change nothing: whether IOMMU is on, the card's +IOMMU group and the driver each function in it is bound to (anything but the +card's own functions blocks the passthrough), the processes holding the card's +device nodes open, and what berthd reports about the card. + +ADDR is the card's PCI address, as in 0000:01:00.0. Without it the card berthd +leases is used, which needs nvidia-smi to answer; nvidia-smi cannot see a card +bound to vfio-pci, so pass the address while the VM runs.`, + Formats: []Format{FormatHuman, FormatJSON, FormatYAML}, + Run: runVfioStatus, + }, + }, + }, { Name: "wire", Summary: "make another engine's whole roster run under berth", diff --git a/internal/cli/vfio.go b/internal/cli/vfio.go new file mode 100644 index 0000000..4da7ef5 --- /dev/null +++ b/internal/cli/vfio.go @@ -0,0 +1,321 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "context" + "fmt" + "os/exec" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/server" + "github.com/T-Crypt/berth/internal/vfio" +) + +// vfioCard is the card Berth governs: berthd's scheduled index, and its PCI +// address in both forms nvidia-smi and sysfs use. +type vfioCard struct { + Index int // gpu index nvidia-smi answers with + BusID string // 00000000:01:00.0, the form nvidia-smi's -i takes + Addr string // 0000:01:00.0, the form sysfs uses + Daemon *proto.Status + DialErr error // why Daemon is nil +} + +// vfioCardAt resolves the governed card. With an explicit address (status +// takes one as its argument) no daemon or nvidia-smi is asked at all, so the +// state stays readable while the card is bound to vfio-pci and nvidia-smi +// cannot see it. Otherwise the status berthd reports names the scheduled +// index, and nvidia-smi names that card's bus id; when berthd is down or +// has no reading yet, index 0. +func vfioCardAt(sock, explicit, smiBin string) (vfioCard, *Error) { + if explicit != "" { + addr := vfio.SysAddr(explicit) + return vfioCard{Addr: addr, BusID: explicitNvidiaForm(addr)}, nil + } + var card vfioCard + st, err := vfioStatusOf(sock) + card.Daemon, card.DialErr = st, err + if err == nil { + card.Index = vfioIndex(st) + } + addr, busID, err := vfioBusID(smiBin, card.Index) + if err != nil { + return card, Systemf("cannot read the card's PCI address: %v", err). + WithHint("pass the card's PCI address yourself: nvidia-smi cannot see a card bound to vfio-pci (for example: berth vfio status 0000:01:00.0)") + } + card.Addr, card.BusID = addr, busID + return card, nil +} + +// explicitNvidiaForm is an address in nvidia-smi's domain form +// ("0000:01:00.0" -> "00000000:01:00.0"). +func explicitNvidiaForm(addr string) string { + domain, rest, ok := strings.Cut(addr, ":") + if !ok { + return addr + } + for len(domain) < 8 { + domain = "0" + domain + } + return domain + ":" + rest +} + +// vfioStatusOf asks berthd for its view of the node. +func vfioStatusOf(sock string) (*proto.Status, error) { + conn, raw, err := server.Dial(sock) + if err != nil { + return nil, err + } + defer raw.Close() + if err := conn.Send(proto.Msg{Op: proto.OpStatus}); err != nil { + return nil, err + } + m, err := conn.Recv() + if err != nil || m.Status == nil { + return nil, fmt.Errorf("no status from berthd: %w", err) + } + return m.Status, nil +} + +// vfioIndex is the card berthd leases: the scheduled GPU's index, 0 when +// berthd has no reading yet. +func vfioIndex(st *proto.Status) int { + for _, g := range st.GPUs { + if g.Scheduled { + return g.Index + } + } + return 0 +} + +// vfioBusID asks nvidia-smi for one card's PCI bus id, in both forms: +// sysfs's, which the reader takes, and nvidia-smi's own, which -i takes. +func vfioBusID(bin string, index int) (addr, busID string, err error) { + out, err := vfioSMI(bin, "-i", strconv.Itoa(index), "--query-gpu=pci.bus_id", "--format=csv,noheader") + if err != nil { + return "", "", err + } + busID = strings.SplitN(strings.TrimSpace(out), "\n", 2)[0] + return vfio.SysAddr(busID), strings.TrimSpace(busID), nil +} + +// vfioAnswers reports whether the card answers nvidia-smi again: its driver +// is bound and talking. +func vfioAnswers(bin, busID, addr string) error { + out, err := vfioSMI(bin, "-i", busID, "--query-gpu=pci.bus_id", "--format=csv,noheader") + if err != nil { + return err + } + if got := vfio.SysAddr(strings.SplitN(strings.TrimSpace(out), "\n", 2)[0]); got != addr { + return fmt.Errorf("nvidia-smi answered with %s, not %s", got, addr) + } + return nil +} + +// vfioSMI runs nvidia-smi once and returns its stdout. The binary is a +// parameter, so tests point it at a stand-in. +func vfioSMI(bin string, args ...string) (string, error) { + if bin == "" { + bin = "nvidia-smi" + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + out, err := exec.CommandContext(ctx, bin, args...).Output() + if err != nil { + return "", fmt.Errorf("%s %s: %w", bin, strings.Join(args, " "), err) + } + return string(out), nil +} + +// vfioStatusDoc is what `berth vfio status --format json` prints: the +// reading, and what berthd reports about the card when it answers. +type vfioStatusDoc struct { + Addr string `json:"addr"` + IOMMU bool `json:"iommu"` + Group []vfio.Function `json:"group"` + Blockers []string `json:"blockers,omitempty"` // group members that are not the card's + Passed bool `json:"passed_through"` // the card is bound to vfio-pci + Openers []vfio.Opener `json:"openers"` + Berthd *vfioBerthdDoc `json:"berthd,omitempty"` +} + +type vfioBerthdDoc struct { + Reachable bool `json:"reachable"` + ProbeErr string `json:"probe_error,omitempty"` + GPU *vfioGPUDoc `json:"gpu,omitempty"` + Hold string `json:"hold,omitempty"` // the granted vfio hold, lease and engine +} + +type vfioGPUDoc struct { + Index int `json:"index"` + Name string `json:"name"` + Used uint64 `json:"used"` + Total uint64 `json:"total"` + Age float64 `json:"age_seconds"` +} + +type vfioStatusView struct{ doc vfioStatusDoc } + +func (v vfioStatusView) Data() any { return v.doc } + +func (v vfioStatusView) Human(c *Context, ok bool) error { + d := v.doc + state := fmt.Sprintf("on %s", d.Addr) + if d.Passed { + state = fmt.Sprintf("on %s: passed through (vfio-pci)", d.Addr) + } else { + for _, f := range d.Group { + if f.Addr == d.Addr && f.Driver != "" { + state = fmt.Sprintf("on %s: bound to %s", d.Addr, f.Driver) + } + } + } + c.Printf("vfio %s\n", state) + c.Printf("iommu %s\n", onOff(d.IOMMU)) + tw := tabwriter.NewWriter(c.Out, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, " \tFUNCTION\tDRIVER\t") + for _, f := range d.Group { + role := "not the card's; blocks the passthrough" + if vfio.CardFunction(f.Addr, d.Addr) { + role = "the card" + } + fmt.Fprintf(tw, " \t%s\t%s\t%s\n", f.Addr, orDash(f.Driver), c.Style.Dim(role)) + } + tw.Flush() + if len(d.Openers) == 0 { + c.Printf("openers none\n") + } else { + c.Printf("openers %d\n", len(d.Openers)) + for _, o := range d.Openers { + who := strconv.Itoa(o.PID) + if o.Name != "" { + who += " " + o.Name + } + if o.Unreadable { + c.Sub("pid %s: %s (its fd directory is unreadable; it may hold the card)", who, c.Style.Warn("/proc unreadable")) + continue + } + if o.Probe { + c.Sub("pid %s holds %s (berthd's own probe)", who, strings.Join(o.Nodes, ", ")) + continue + } + c.Sub("pid %s holds %s", who, strings.Join(o.Nodes, ", ")) + } + } + if d.Berthd == nil { + return nil + } + if !d.Berthd.Reachable { + c.Printf("berthd %s\n", c.Style.Dim("unreachable; what it reports is not shown")) + return nil + } + if g := d.Berthd.GPU; g != nil { + c.Printf("berthd gpu%d %s: %s used of %s (read %.0f s ago)\n", + g.Index, g.Name, server.MiB(g.Used), server.MiB(g.Total), g.Age) + } + if d.Berthd.ProbeErr != "" { + c.Printf("probe %s\n", c.Style.Err(d.Berthd.ProbeErr)) + } + if d.Berthd.Hold != "" { + c.Printf("hold %s granted\n", d.Berthd.Hold) + } + return nil +} + +func orDash(s string) string { + if s == "" { + return "-" + } + return s +} + +func onOff(b bool) string { + if b { + return "on" + } + return "off" +} + +// runVfioStatus prints the card's VFIO state. Read only: it changes nothing. +func runVfioStatus(c *Context) int { + r := vfio.Reader{} + if err := r.Supported(); err != nil { + return c.fail(Systemf("%v", err)) + } + return vfioStatus(c, r, "") +} + +// vfioStatus is the status flow; a test names its own reader roots and +// nvidia-smi stand-in. +func vfioStatus(c *Context, r vfio.Reader, smiBin string) int { + fs := c.FlagSet() + sock := fs.String("socket", defaultSocket(), "berthd socket ($BERTH_SOCKET)") + if code, done := c.Parse(fs); done { + return code + } + if len(c.Args) > 1 { + return c.fail(Systemf("unexpected argument %q", c.Args[1]). + WithHint("the optional argument is the card's PCI address, as in: berth vfio status 0000:01:00.0")) + } + explicit := "" + if len(c.Args) == 1 { + explicit = c.Args[0] + } + card, e := vfioCardAt(*sock, explicit, smiBin) + if e != nil { + return c.fail(e) + } + on, err := r.IOMMUOn() + if err != nil { + return c.fail(Systemf("%v", err)) + } + group, err := r.Group(card.Addr) + if err != nil { + return c.fail(Systemf("%v", err)) + } + openers, err := r.Openers(card.Addr) + if err != nil { + return c.fail(Systemf("%v", err)) + } + if openers == nil { + openers = []vfio.Opener{} + } + doc := vfioStatusDoc{Addr: card.Addr, IOMMU: on, Group: group, Openers: openers} + for _, f := range group { + if !vfio.CardFunction(f.Addr, card.Addr) { + doc.Blockers = append(doc.Blockers, f.Addr) + } + if f.Addr == card.Addr && f.Driver == "vfio-pci" { + doc.Passed = true + } + } + doc.Berthd = vfioBerthdView(card) + return c.Write(vfioStatusView{doc: doc}, nil) +} + +// vfioBerthdView is the card state berthd reports: the scheduled card, its +// probe, and the vfio hold it holds. berthd being down is a report line, +// not a failure: the reading above is the command's work. +func vfioBerthdView(card vfioCard) *vfioBerthdDoc { + if card.Daemon == nil { + return &vfioBerthdDoc{} + } + b := &vfioBerthdDoc{Reachable: true, ProbeErr: card.Daemon.ProbeErr} + for _, g := range card.Daemon.GPUs { + if g.Index == card.Index { + b.GPU = &vfioGPUDoc{Index: g.Index, Name: g.Name, Used: g.Used, Total: g.Total, Age: g.Age} + } + } + for _, l := range card.Daemon.Leases { + if l.Group == "vfio" { + b.Hold = l.Engine + " " + l.ID + } + } + return b +} diff --git a/internal/cli/vfio_run.go b/internal/cli/vfio_run.go new file mode 100644 index 0000000..6607693 --- /dev/null +++ b/internal/cli/vfio_run.go @@ -0,0 +1,392 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +package cli + +import ( + "fmt" + "os" + "os/exec" + "os/signal" + "strings" + "syscall" + "time" + + "github.com/T-Crypt/berth/internal/proto" + "github.com/T-Crypt/berth/internal/runner" + "github.com/T-Crypt/berth/internal/server" + "github.com/T-Crypt/berth/internal/vfio" +) + +// vfioSettle is how long the run waits after the grant for berthd's own +// probe to go quiet (a daemon that knows VFIO pauses it; an older one +// finishes its nvidia-smi in that time), before the opener check decides. +const vfioSettle = 5 * time.Second + +// vfioBack is how long the card may take to come back after the script +// exits: the driver rebind takes seconds. +const vfioBack = 60 * time.Second + +// vfioPoll is how often the driver state is re-read while the card comes +// back. +const vfioPoll = 2 * time.Second + +// vfioRunOpts carries what a test replaces: the reader's roots, the +// nvidia-smi stand-in, the waits, and the signal source. +type vfioRunOpts struct { + reader vfio.Reader + smiBin string + settle time.Duration // 0 = vfioSettle + back time.Duration // 0 = vfioBack + poll time.Duration // 0 = vfioPoll + sigs <-chan os.Signal // nil: the real signals +} + +func (o vfioRunOpts) orSettle() time.Duration { + if o.settle > 0 { + return o.settle + } + return vfioSettle +} + +func (o vfioRunOpts) orBack() time.Duration { + if o.back > 0 { + return o.back + } + return vfioBack +} + +func (o vfioRunOpts) orPoll() time.Duration { + if o.poll > 0 { + return o.poll + } + return vfioPoll +} + +// ctxCanceled is closed by the wait goroutine when a signal arrived during +// the wait for the grant. +func ctxCanceled(canceled <-chan struct{}) bool { + select { + case <-canceled: + return true + default: + return false + } +} + +// gotSignal says whether a signal is waiting on the channel: during the +// settle it means the operator asked out. +func gotSignal(sigs <-chan os.Signal) bool { + select { + case <-sigs: + return true + default: + return false + } +} + +// drainSignals drops any signal the CMD phase buffered: it was meant for +// the script, and the release loops below recheck on the next one. +func drainSignals(sigs <-chan os.Signal) { + for gotSignal(sigs) { + } +} + +// runVfioRun holds the card for a passthrough and runs the operator's own +// switch script with the terminal attached. The script (sudo, virsh) owns +// the switch and the VM; Berth owns the drain, the proof the device is +// free, and the wait for the card to come back. +func runVfioRun(c *Context) int { + r := vfio.Reader{} + if err := r.Supported(); err != nil { + return c.fail(Systemf("%v", err)) + } + return vfioRun(c, vfioRunOpts{reader: r}) +} + +// vfioRun is the run flow; a test names its own reader roots, nvidia-smi +// stand-in, waits and signals. +func vfioRun(c *Context, o vfioRunOpts) int { + say := func(format string, a ...any) { fmt.Fprintf(c.Err, "berth: "+format+"\n", a...) } + fs := c.FlagSet() + sock := fs.String("socket", defaultSocket(), "berthd socket ($BERTH_SOCKET)") + reason := fs.String("reason", "", "shown on the lease, e.g. galaxy (default: passthrough)") + wait := fs.Duration("wait", 10*time.Minute, "how long to wait for engines to drain; 0 = berthd's default; negative = no deadline") + if code, done := c.Parse(fs); done { + return code + } + if len(c.Args) == 0 { + return c.fail(Systemf("the command to run is required"). + WithHint("usage: berth vfio run [--reason NAME] -- CMD [ARGS...]")) + } + argv := c.Args + + // The card, by way of berthd's scheduled index and nvidia-smi. + card, e := vfioCardAt(*sock, "", o.smiBin) + if e != nil { + return c.fail(e) + } + if card.Daemon == nil { + return c.fail(Systemf("berthd unreachable on %s: %v", *sock, card.DialErr). + WithHint("nothing is held: start berthd, or the card is not coordinated at all")) + } + + // Preflight, before anything is asked of the engines: without IOMMU, or + // with devices that are not the card's in its group, the passthrough + // cannot work and the script would fail halfway. + funcs, e := vfioPreflight(o.reader, card) + if e != nil { + return c.fail(e) + } + + name := "vfio" + if r := strings.TrimSpace(*reason); r != "" { + name = "vfio:" + strings.ReplaceAll(r, " ", "-") + } + + // The signals: the operator's Ctrl-C. During the wait for the grant a + // signal cancels it; from the grant on, this run's own loops read them. + var sigs <-chan os.Signal + if o.sigs != nil { + sigs = o.sigs + } else { + ch := make(chan os.Signal, 4) + signal.Notify(ch, os.Interrupt, syscall.SIGTERM, syscall.SIGHUP) + defer signal.Stop(ch) + sigs = ch + } + + // The hold: engines drain at their boundaries, none starts, until this + // process ends. Pinned at hold priority, like berth hold. + conn, raw, err := server.Dial(*sock) + if err != nil { + return c.fail(Systemf("berthd unreachable on %s: %v", *sock, err). + WithHint("nothing is held: start berthd, or the card is not coordinated at all")) + } + defer raw.Close() + stopped := make(chan struct{}) + canceled := make(chan struct{}) + go func() { + select { + case <-stopped: + case <-sigs: + close(canceled) + raw.Close() + } + }() + if err := conn.Send(proto.Msg{Op: proto.OpAcquire, Engine: name, Group: "vfio", Priority: "hold", + Wait: wait.Seconds(), Pinned: true}); err != nil { + return c.fail(Systemf("berthd on %s: %v", *sock, err)) + } + lease := "" + for granted := false; !granted; { + m, err := conn.Recv() + if err != nil { + if ctxCanceled(canceled) { + fmt.Fprintln(c.Err, "berth: vfio hold cancelled before the card was free") + return 0 + } + return c.fail(Systemf("berthd went away: %v", err)) + } + switch m.Op { + case proto.OpWaiting: + fmt.Fprintf(c.Err, "berth: waiting: %s\n", m.Reason) + case proto.OpRefused: + return c.fail(Failf("vfio hold refused: %s", m.Reason)) + case proto.OpGranted: + lease = m.Lease + granted = true + } + } + close(stopped) + if ctxCanceled(canceled) { + fmt.Fprintln(c.Err, "berth: vfio hold cancelled before the card was free") + return 0 + } + + say("card held as %s (%s); engines are off the card", lease, name) + + // berthd's probe opens the card's device every tick. Give it the settle + // time to go quiet (V1 pauses it for a vfio hold), then decide on one + // reading: anyone else holding a node keeps the card where it is. + time.Sleep(o.orSettle()) + if ctxCanceled(canceled) || gotSignal(sigs) { + fmt.Fprintln(c.Err, "berth: vfio hold cancelled before the card was free") + return 0 + } + openers, err := o.reader.Openers(card.Addr) + if err != nil { + return c.fail(Systemf("read the openers: %v", err)) + } + if bad := vfioBadOpeners(openers); len(bad) > 0 { + raw.Close() // the hold is not earned: the next engine may load + e := Failf("the card's device nodes are in use; not holding %s", card.Addr) + for _, b := range bad { + e = e.Detail("%s", b) + } + e = e.WithHint("stop the holders yourself (Berth never stops a process); berth vfio status names them") + return c.fail(e) + } + + // The script runs with the terminal attached, so its sudo prompt and + // the VM's console reach it, and with the card's PCI functions in + // BERTH_VFIO_DEVICES, so it hardcodes no address. + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr + runner.Isolate(cmd) + devices := vfioFunctionList(funcs, card.Addr) + cmd.Env = append(os.Environ(), "BERTH_VFIO_DEVICES="+devices) + if err := cmd.Start(); err != nil { + return c.fail(Systemf("start %s: %v", argv[0], err)) + } + say("running %s with %s", argv[0], devices) + done := make(chan struct{}) + go runner.ForwardSignals(cmd, done) + waitErr := cmd.Wait() + close(done) + code := runner.ExitCode(cmd, waitErr) + if code != 0 { + say("%s exited with %d", argv[0], code) + } + + // The hold ends only when the card is back for the next engine: every + // function of the card off vfio-pci, and the card answering nvidia-smi. + drainSignals(sigs) + answers := func() error { return vfioAnswers(o.smiBin, card.BusID, card.Addr) } + if back, _ := vfioCardBack(o.reader, funcs, card.Addr, answers); back { + say("the card is back; hold released") + return code + } + // Poll the driver for the rebind the script's tail does; a signal + // during the wait goes to the manual wait below, which releases only + // once the card really is back. + until := time.Now().Add(o.orBack()) +poll: + for { + select { + case <-time.After(o.orPoll()): + case <-sigs: + break poll + } + if back, _ := vfioCardBack(o.reader, funcs, card.Addr, answers); back { + say("the card is back; hold released") + return code + } + if time.Now().After(until) { + break + } + } + // Not back on its own: say which function and what to do about it, and + // keep the hold. Ctrl-C rechecks; it releases only when the card is + // back, so the next engine never loads onto a card that is out. + vfioWaitForTheCard(o.reader, funcs, card.Addr, answers, sigs, say) + return code +} + +// vfioWaitForTheCard says what is not back and what to do about it, then +// waits for a signal; on each one it rechecks and releases only when the +// card is back. +func vfioWaitForTheCard(r vfio.Reader, funcs []vfio.Function, addr string, answers func() error, + sigs <-chan os.Signal, say func(string, ...any)) { + for { + back, why := vfioCardBack(r, funcs, addr, answers) + if back { + say("the card is back; hold released") + return + } + say("%s; the hold stays", why) + <-sigs + } +} + +// vfioPreflight refuses a passthrough that cannot work, before anything is +// asked of the engines. It returns the card's own functions, which the run +// passes to the script and watches until each is back. +func vfioPreflight(r vfio.Reader, card vfioCard) ([]vfio.Function, *Error) { + on, err := r.IOMMUOn() + if err != nil { + return nil, Systemf("%v", err) + } + if !on { + return nil, Failf("IOMMU is off; the card cannot pass through"). + WithHint("turn VT-d (or AMD-Vi) on in the firmware and add intel_iommu=on iommu=pt (or its AMD form) to the kernel command line") + } + group, err := r.Group(card.Addr) + if err != nil { + return nil, Systemf("%v", err) + } + var funcs []vfio.Function + var others []string + for _, f := range group { + if vfio.CardFunction(f.Addr, card.Addr) { + funcs = append(funcs, f) + continue + } + others = append(others, fmt.Sprintf("%s (%s)", f.Addr, orDash(f.Driver))) + } + if len(others) > 0 { + return nil, Failf("the IOMMU group of %s holds %s, and the whole group goes with the card", card.Addr, strings.Join(others, ", ")). + WithHint("pass the other device through too, or move the card to its own group (ACS)") + } + return funcs, nil +} + +// vfioBadOpeners is the openers that keep the card where it is: every +// process that is not berthd's own probe, including one whose fd directory +// could not be read (it may hold the card; Berth cannot prove otherwise). +func vfioBadOpeners(openers []vfio.Opener) []string { + var bad []string + for _, o := range openers { + switch { + case o.Probe: + case o.Unreadable: + bad = append(bad, fmt.Sprintf("pid %d%s: /proc unreadable; it may hold the card", o.PID, vfioNamed(o.Name))) + default: + bad = append(bad, fmt.Sprintf("pid %d%s holds %s", o.PID, vfioNamed(o.Name), strings.Join(o.Nodes, ", "))) + } + } + return bad +} + +func vfioNamed(name string) string { + if name == "" { + return "" + } + return " " + name +} + +// vfioCardBack reports whether the card is back for the next engine: every +// function of the card on a host driver (anything but vfio-pci, and not +// unbound), and the card answering nvidia-smi. why names the first thing +// that is not, and what to do about it. +func vfioCardBack(r vfio.Reader, funcs []vfio.Function, addr string, answers func() error) (bool, string) { + for _, f := range funcs { + d, err := r.Driver(f.Addr) + if err != nil { + return false, fmt.Sprintf("%s: %v", f.Addr, err) + } + if d == "vfio-pci" { + return false, fmt.Sprintf("%s is still bound to vfio-pci; stop the VM and bind the function back to its host driver", f.Addr) + } + if d == "" { + return false, fmt.Sprintf("%s is bound to no driver; bind it back to its host driver", f.Addr) + } + } + if err := answers(); err != nil { + return false, fmt.Sprintf("%s is off vfio-pci but does not answer nvidia-smi (%v); the driver is still coming up", addr, err) + } + return true, "" +} + +// vfioFunctionList is the card's functions, space-separated, for +// BERTH_VFIO_DEVICES. The card's own function first, so a single-function +// card reads one plain address. +func vfioFunctionList(funcs []vfio.Function, card string) string { + out := []string{card} + for _, f := range funcs { + if f.Addr != card { + out = append(out, f.Addr) + } + } + return strings.Join(out, " ") +} diff --git a/internal/cli/vfio_test.go b/internal/cli/vfio_test.go new file mode 100644 index 0000000..35e10ea --- /dev/null +++ b/internal/cli/vfio_test.go @@ -0,0 +1,532 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build linux + +package cli + +// berth vfio against a fake sysfs, a fake /proc and a live daemon: the +// reading, the refusals with their holds released, the happy path with a +// script that flips the fake drivers and back, and the hold kept while the +// card is still out. + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/T-Crypt/berth/internal/broker" + "github.com/T-Crypt/berth/internal/procs" + "github.com/T-Crypt/berth/internal/server" + "github.com/T-Crypt/berth/internal/vfio" +) + +const ( + vfioAddr = "0000:01:00.0" + vfioNvidiaBus = "00000000:01:00.0" +) + +// vfioTree is the fake sysfs and /proc the tests build. +type vfioTree struct { + t *testing.T + sysfs string + proc string +} + +func newVfioTree(t *testing.T) *vfioTree { + t.Helper() + return &vfioTree{t: t, sysfs: t.TempDir(), proc: t.TempDir()} +} + +// function makes one PCI function with a driver ("" = unbound). +func (tr *vfioTree) function(addr, driver string) { + tr.t.Helper() + d := filepath.Join(tr.sysfs, "bus", "pci", "devices", addr) + if err := os.MkdirAll(d, 0o755); err != nil { + tr.t.Fatal(err) + } + if driver != "" { + if err := os.Symlink(filepath.Join(tr.sysfs, "bus", "pci", "drivers", driver), filepath.Join(d, "driver")); err != nil { + tr.t.Fatal(err) + } + } +} + +// group is the card's IOMMU group: every function made so far is linked +// into it, the way sysfs lists a group's members. +func (tr *vfioTree) group() { + tr.t.Helper() + group := filepath.Join(tr.sysfs, "kernel", "iommu_groups", "12") + if err := os.MkdirAll(filepath.Join(group, "devices"), 0o755); err != nil { + tr.t.Fatal(err) + } + entries, err := os.ReadDir(filepath.Join(tr.sysfs, "bus", "pci", "devices")) + if err != nil { + tr.t.Fatal(err) + } + for _, e := range entries { + d := filepath.Join(tr.sysfs, "bus", "pci", "devices", e.Name()) + if err := os.Symlink(group, filepath.Join(d, "iommu_group")); err != nil { + tr.t.Fatal(err) + } + if err := os.Symlink(d, filepath.Join(group, "devices", e.Name())); err != nil { + tr.t.Fatal(err) + } + } +} + +// opener records a process holding device nodes, the way /proc does. +func (tr *vfioTree) opener(pid int, comm string, parent int, nodes ...string) { + tr.t.Helper() + p := filepath.Join(tr.proc, fmt.Sprintf("%d", pid)) + if err := os.MkdirAll(p, 0o755); err != nil { + tr.t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(p, "comm"), []byte(comm+"\n"), 0o644); err != nil { + tr.t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(p, "stat"), []byte(fmt.Sprintf("%d (%s) S %d 0 0 0 0", pid, comm, parent)), 0o644); err != nil { + tr.t.Fatal(err) + } + fds := filepath.Join(p, "fd") + if err := os.MkdirAll(fds, 0o755); err != nil { + tr.t.Fatal(err) + } + for i, n := range nodes { + if err := os.Symlink(n, filepath.Join(fds, fmt.Sprintf("%d", i))); err != nil { + tr.t.Fatal(err) + } + } +} + +// unreadable closes one process's fd directory to everyone. +func (tr *vfioTree) unreadable(pid int) { + tr.t.Helper() + if err := os.Chmod(filepath.Join(tr.proc, fmt.Sprintf("%d", pid), "fd"), 0o000); err != nil { + tr.t.Fatal(err) + } +} + +// smi writes the nvidia-smi stand-in: it prints the bus id when the card's +// driver is back on nvidia and fails otherwise, which is what the initial +// address query and the answers check each need. +func (tr *vfioTree) smi() string { + tr.t.Helper() + bin := filepath.Join(tr.t.TempDir(), "nvidia-smi") + script := fmt.Sprintf(`#!/bin/sh +d=$(basename "$(readlink "%s/bus/pci/devices/%s/driver" 2>/dev/null)" 2>/dev/null) +if [ "$d" = nvidia ]; then + echo %s + exit 0 +fi +echo "nvidia-smi: no device" >&2 +exit 1 +`, tr.sysfs, vfioAddr, vfioNvidiaBus) + if err := os.WriteFile(bin, []byte(script), 0o755); err != nil { + tr.t.Fatal(err) + } + return bin +} + +// script writes a command the run executes. +func (tr *vfioTree) script(name, body string) string { + tr.t.Helper() + bin := filepath.Join(tr.t.TempDir(), name) + if err := os.WriteFile(bin, []byte("#!/bin/sh\n"+body), 0o755); err != nil { + tr.t.Fatal(err) + } + return bin +} + +// flip is the shell for one driver change on every function of the card. +func (tr *vfioTree) flip(to string) string { + return fmt.Sprintf(`for f in $BERTH_VFIO_DEVICES; do + rm "%s/bus/pci/devices/$f/driver" + ln -s "%s/bus/pci/drivers/%s" "%s/bus/pci/devices/$f/driver" +done +`, tr.sysfs, tr.sysfs, to, tr.sysfs) +} + +// driver reads one function's driver back from the fake tree. +func (tr *vfioTree) driver(addr string) string { + tr.t.Helper() + t, err := os.Readlink(filepath.Join(tr.sysfs, "bus", "pci", "devices", addr, "driver")) + if err != nil { + return "" + } + return filepath.Base(t) +} + +// bind puts a function back on its driver by hand, as the operator's script +// or hand does between two checks. +func (tr *vfioTree) bind(addr, driver string) { + tr.t.Helper() + p := filepath.Join(tr.sysfs, "bus", "pci", "devices", addr, "driver") + os.Remove(p) + if err := os.Symlink(filepath.Join(tr.sysfs, "bus", "pci", "drivers", driver), p); err != nil { + tr.t.Fatal(err) + } +} + +// runVfioStatusIn runs the status flow against the given roots. +func runVfioStatusIn(t *testing.T, tr *vfioTree, smiBin string, args ...string) (int, string, string) { + t.Helper() + var out, errOut bytes.Buffer + c := &Context{Out: &out, Err: &errOut, Version: "test", Format: FormatHuman, + Style: newStyle(&out), errStyle: newStyle(&errOut), + Cmd: lookup(lookup(commands, "vfio").Sub, "status"), Args: args} + code := vfioStatus(c, vfio.Reader{Sysfs: tr.sysfs, Proc: tr.proc}, smiBin) + return code, out.String(), errOut.String() +} + +// runVfioRunIn runs the flow against the given roots and waits. +func runVfioRunIn(t *testing.T, tr *vfioTree, smiBin string, o vfioRunOpts, args ...string) (int, string, string) { + t.Helper() + o.reader = vfio.Reader{Sysfs: tr.sysfs, Proc: tr.proc} + o.smiBin = smiBin + var out, errOut bytes.Buffer + c := &Context{Out: &out, Err: &errOut, Version: "test", Format: FormatHuman, + Style: newStyle(&out), errStyle: newStyle(&errOut), + Cmd: lookup(lookup(commands, "vfio").Sub, "run"), Args: args} + code := vfioRun(c, o) + return code, out.String(), errOut.String() +} + +// startVfioDaemon is berthd on a temp socket with an empty card: the hold +// grants at once and the lease table is observable. +func startVfioDaemon(t *testing.T) (string, *broker.Broker) { + t.Helper() + sock := filepath.Join(t.TempDir(), "berth.sock") + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + c := &procCard{} + b := broker.New(broker.Config{ + Poll: 10 * time.Millisecond, DefaultWait: 5 * time.Second, JuggleWindow: -1, + Version: "test", Log: log, + }, c, procs.Map{}) + ln, err := server.Listen(sock) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + b.Tick(ctx) + go b.Run(ctx) + go server.Serve(ctx, ln, b, server.Extras{}, log) + return sock, b +} + +// waitUntil polls until f is true, so async daemon-side releases settle. +func waitUntil(t *testing.T, f func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !f() { + if time.Now().After(deadline) { + t.Fatal("the condition never became true") + } + time.Sleep(5 * time.Millisecond) + } +} + +// none is true when the daemon holds no lease. +func none(b *broker.Broker) func() bool { + return func() bool { return len(b.Status().Leases) == 0 } +} + +// TestVfioStatusReadsTheFakeTree: the reading, no daemon needed, exit 0. +func TestVfioStatusReadsTheFakeTree(t *testing.T) { + isolate(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + code, out, errOut := runVfioStatusIn(t, tr, "", "--socket", "/nonexistent/berth.sock") + if code != ExitOK { + t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) + } + if !strings.Contains(out, "bound to nvidia") || !strings.Contains(out, "iommu on") { + t.Errorf("the reading is not what the tree says: %q", out) + } + if !strings.Contains(out, "unreachable") { + t.Errorf("the report does not say berthd is unreachable: %q", out) + } +} + +// TestVfioStatusJSONIsTheDocument covers the machine view of the reading. +func TestVfioStatusJSONIsTheDocument(t *testing.T) { + isolate(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.function("0000:01:00.1", "snd_hda_intel") + tr.function("0000:06:00.0", "xhci_hcd") + tr.group() + tr.opener(100, "app", 1, "/dev/nvidia0") + tr.opener(101, "nvidia-smi", 99, "/dev/nvidiactl") + tr.opener(99, "berth", 1) + code, out, errOut := runVfioStatusIn(t, tr, "", "--socket", "/nonexistent/berth.sock", "--format", "json") + if code != ExitOK { + t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) + } + var doc struct { + Addr string `json:"addr"` + IOMMU bool `json:"iommu"` + Blockers []string `json:"blockers"` + Passed bool `json:"passed_through"` + Openers []struct { + PID int `json:"pid"` + Name string `json:"name"` + Nodes []string `json:"nodes"` + Probe bool `json:"probe"` + } `json:"openers"` + } + if err := json.Unmarshal([]byte(out), &doc); err != nil { + t.Fatalf("json %q: %v", out, err) + } + if doc.Addr != vfioAddr || !doc.IOMMU || doc.Passed { + t.Errorf("doc = %+v", doc) + } + if len(doc.Blockers) != 1 || doc.Blockers[0] != "0000:06:00.0" { + t.Errorf("blockers = %v", doc.Blockers) + } + if len(doc.Openers) != 2 || doc.Openers[0].PID != 100 || doc.Openers[0].Name != "app" || + len(doc.Openers[0].Nodes) != 1 || doc.Openers[0].Nodes[0] != "/dev/nvidia0" { + t.Errorf("openers = %+v", doc.Openers) + } + if !doc.Openers[1].Probe { + t.Errorf("the nvidia-smi opener is not named as the probe: %+v", doc.Openers[1]) + } +} + +// TestVfioStatusReportsPassthroughWithAnAddress: nvidia-smi sees nothing +// once the card is on vfio-pci, and an explicit address still reads. +func TestVfioStatusReportsPassthroughWithAnAddress(t *testing.T) { + isolate(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "vfio-pci") + tr.group() + code, out, errOut := runVfioStatusIn(t, tr, "", "--socket", "/nonexistent/berth.sock", vfioAddr) + if code != ExitOK { + t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) + } + if !strings.Contains(out, "passed through") { + t.Errorf("the report does not say the card is passed through: %q", out) + } +} + +// TestVfioRunRefusesIOMMUOff: exit 1 with the hint, and nothing is acquired. +func TestVfioRunRefusesIOMMUOff(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + smi := tr.smi() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--", "/bin/true") + if code != ExitFail { + t.Fatalf("exit = %d, want 1 (stderr %q)", code, errOut) + } + if !strings.Contains(errOut, "IOMMU is off") || !strings.Contains(errOut, "intel_iommu=on") { + t.Errorf("the refusal does not say why and how: %q", errOut) + } + if !none(b)() { + t.Error("the refusal acquired a hold") + } +} + +// TestVfioRunRefusesAnExtraGroupMember: the whole IOMMU group goes with the +// card, so a stranger in it refuses the run with its name and driver. +func TestVfioRunRefusesAnExtraGroupMember(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.function("0000:06:00.0", "xhci_hcd") + tr.group() + smi := tr.smi() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--", "/bin/true") + if code != ExitFail { + t.Fatalf("exit = %d, want 1 (stderr %q)", code, errOut) + } + if !strings.Contains(errOut, "0000:06:00.0") || !strings.Contains(errOut, "xhci_hcd") { + t.Errorf("the refusal does not name the stranger: %q", errOut) + } + if !none(b)() { + t.Error("the refusal acquired a hold") + } +} + +// TestVfioRunRefusesOpenersAndReleasesTheHold: a holder outside Berth keeps +// the card where it is, the report names it, and the hold is not kept. +func TestVfioRunRefusesOpenersAndReleasesTheHold(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + tr.opener(100, "qemu", 1, "/dev/nvidia0") + smi := tr.smi() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--", "/bin/true") + if code != ExitFail { + t.Fatalf("exit = %d, want 1 (stderr %q)", code, errOut) + } + if !strings.Contains(errOut, "pid 100 qemu holds /dev/nvidia0") { + t.Errorf("the refusal does not name the holder: %q", errOut) + } + waitUntil(t, none(b)) +} + +// TestVfioRunRefusesAnUnreadableProc: a process whose fds cannot be read is +// listed as unreadable, never skipped. +func TestVfioRunRefusesAnUnreadableProc(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + tr.opener(100, "locked", 1) + tr.unreadable(100) + smi := tr.smi() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--", "/bin/true") + if code != ExitFail { + t.Fatalf("exit = %d, want 1 (stderr %q)", code, errOut) + } + if !strings.Contains(errOut, "pid 100") || !strings.Contains(errOut, "unreadable") { + t.Errorf("the refusal does not name the unreadable process: %q", errOut) + } + waitUntil(t, none(b)) +} + +// TestVfioRunHappyPath: the hold is granted, the script runs and its exit +// code passes through once the card is back on its host driver. +func TestVfioRunHappyPath(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + smi := tr.smi() + cmd := tr.script("switch", tr.flip("vfio-pci")+tr.flip("nvidia")+"exit 7\n") + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond, poll: 5 * time.Millisecond}, + "--socket", sock, "--", cmd) + if code != 7 { + t.Fatalf("exit = %d, want the script's 7 (stderr %q)", code, errOut) + } + if !strings.Contains(errOut, "the card is back; hold released") { + t.Errorf("the report does not say the card came back: %q", errOut) + } + waitUntil(t, none(b)) +} + +// TestVfioRunSetsTheDevicesEnv: the script sees the card's PCI functions +// without hardcoding them, and the hold covers the whole card. +func TestVfioRunSetsTheDevicesEnv(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.function("0000:01:00.1", "snd_hda_intel") + tr.group() + smi := tr.smi() + seen := filepath.Join(tr.t.TempDir(), "seen") + cmd := tr.script("switch", fmt.Sprintf("echo \"$BERTH_VFIO_DEVICES\" > %s\n%s\nexit 0\n", seen, tr.flip("nvidia"))) + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond, poll: 5 * time.Millisecond}, + "--socket", sock, "--", cmd) + if code != ExitOK { + t.Fatalf("exit = %d (stderr %q)", code, errOut) + } + wrote, err := os.ReadFile(seen) + if err != nil { + t.Fatal(err) + } + if got := strings.TrimSpace(string(wrote)); got != vfioAddr+" 0000:01:00.1" { + t.Errorf("BERTH_VFIO_DEVICES = %q, want %q", got, vfioAddr+" 0000:01:00.1") + } + waitUntil(t, none(b)) +} + +// TestVfioRunKeepsTheHoldWhileTheCardIsOut: the script exits with the card +// still on vfio-pci, the message names the function, and the hold releases +// only once the card is really back. +func TestVfioRunKeepsTheHoldWhileTheCardIsOut(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + smi := tr.smi() + cmd := tr.script("switch", tr.flip("vfio-pci")+"exit 3\n") + + sigs := make(chan os.Signal, 4) + var mu sync.Mutex + var errOut bytes.Buffer + code := make(chan int, 1) + go func() { + o := vfioRunOpts{settle: time.Millisecond, poll: 5 * time.Millisecond, + back: 100 * time.Millisecond, sigs: sigs} + o.reader = vfio.Reader{Sysfs: tr.sysfs, Proc: tr.proc} + o.smiBin = smi + c := &Context{Out: io.Discard, Err: errWriter{&mu, &errOut}, Version: "test", Format: FormatHuman, + Style: newStyle(io.Discard), errStyle: newStyle(io.Discard), + Cmd: lookup(lookup(commands, "vfio").Sub, "run"), + Args: []string{"--socket", sock, "--", cmd}} + code <- vfioRun(c, o) + }() + waitFor(t, &mu, &errOut, func(said string) bool { + return strings.Contains(said, vfioAddr+" is still bound to vfio-pci") + }) + // Ctrl-C while the card is out: it must not release the hold. + sigs <- os.Interrupt + time.Sleep(50 * time.Millisecond) + if leases := len(b.Status().Leases); leases != 1 { + t.Fatalf("Ctrl-C released the hold while the card was out (%d leases)", leases) + } + // The card comes back (the script's tail, or the operator by hand): the + // next check releases. + tr.bind(vfioAddr, "nvidia") + sigs <- os.Interrupt + if got := <-code; got != 3 { + t.Fatalf("exit = %d, want the script's 3", got) + } + waitUntil(t, none(b)) +} + +// waitFor polls the stderr buffer until its text satisfies f. +func waitFor(t *testing.T, mu *sync.Mutex, b *bytes.Buffer, f func(string) bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + mu.Lock() + said := b.String() + mu.Unlock() + if f(said) { + return + } + if time.Now().After(deadline) { + t.Fatalf("the message never came: %q", said) + } + time.Sleep(5 * time.Millisecond) + } +} + +// errWriter is a stderr that stays safe when the flow and the test read it +// from different goroutines. +type errWriter struct { + mu *sync.Mutex + b *bytes.Buffer +} + +func (w errWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + return w.b.Write(p) +} From 2dd61fbcaa19b8e08967e1036b7a992ac0fb8c75 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:12:25 -0600 Subject: [PATCH 3/9] vfio docs: the VFIO passthrough page, experimental and Linux only --- CHANGELOG.md | 14 ++++++ README.md | 1 + docs/README.md | 1 + docs/vfio.md | 114 +++++++++++++++++++++++++++++++++++++++++++++++++ mkdocs.yml | 1 + 5 files changed, 131 insertions(+) create mode 100644 docs/vfio.md diff --git a/CHANGELOG.md b/CHANGELOG.md index d1f6ffb..b405901 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,20 @@ has a matching GitHub release whose body comes from its section below. ### Added +- `berth vfio status` and `berth vfio run` (experimental, Linux only): the user side of passing the governed + card to a virtual machine through VFIO. `status` reads the card's VFIO state (IOMMU on or off, the IOMMU + group and the driver bound to each function, the processes holding the card's device nodes, what berthd + reports about the card) and changes nothing. `run -- CMD` acquires the VFIO hold the way `berth hold` does + (a pinned lease at hold priority, group `vfio`, engine `vfio:`), refuses when IOMMU is off, when + the IOMMU group holds devices that are not the card's, or when a process other than berthd's own probe + still holds a device node of the card (named by PID and node, hold released, exit 1; Berth never stops a + process), and then runs the operator's own switch script with the terminal attached and + `BERTH_VFIO_DEVICES` set to the card's PCI functions, so its sudo prompt reaches the terminal and the + script hardcodes no address. When the script exits, the hold ends only when every function is off + `vfio-pci` and the card answers `nvidia-smi` again; until then the hold stays and Ctrl-C releases it only + once the card is back, so the next engine never loads onto a card that is out. Berth never binds or + unbinds a driver, never writes to sysfs, and never asks for a password; the script (sudo, virsh) owns the + switch and libvirt owns the VM. See docs/vfio.md. - Gateway fallback routing: an `[[engine]]` may name a `fallback`, another engine in the file that needs no card (a CPU llama-server, an Ollama on another box), and `fallback_after` (10 s by default). A request that has waited that long for its own engine is proxied to the fallback when diff --git a/README.md b/README.md index 022ffad..461af29 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,7 @@ The same pages are a site with search at . | [Install](docs/install.md) | The agent prompt, manual install, `berth.toml`, uninstall | | [Engines](docs/engines.md) | Every engine Berth reads, API keys, ComfyUI as a suite | | [Leasing, games and holds](docs/leasing.md) | Priorities, drains, stuck releases, the juggle guard, minimum turns, claims and `berth hold` | +| [VFIO passthrough](docs/vfio.md) (experimental, Linux) | `berth vfio status`, `berth vfio run`, the IOMMU requirements, an example switch script | | [Models](docs/models.md) | `berth pull`, integrity checks, shelves, `berth storage`, the page cache and `berth prefetch` | | [Gateway](docs/gateway.md) | One OpenAI-compatible port, cold starts, fallbacks, KV snapshots, switch costs, wake pages, park in RAM | | [The harbor and telemetry](docs/harbor.md) | Stowaways, the collisions board, alerts, every harbor panel, what Berth measures | diff --git a/docs/README.md b/docs/README.md index f3bd318..461041f 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,6 +9,7 @@ has the overview, the quick start and screenshots. These pages are the reference | [Install](install.md) | The agent prompt, manual install, `berth.toml`, uninstall | | [Engines](engines.md) | Every engine Berth reads, API keys, ComfyUI as a suite | | [Leasing, games and holds](leasing.md) | Priorities, drains, stuck releases, the juggle guard, minimum turns, claims and `berth hold` | +| [VFIO passthrough](vfio.md) (experimental, Linux) | `berth vfio status`, `berth vfio run`, the IOMMU requirements, an example switch script | | [Models](models.md) | `berth pull`, integrity checks, shelves, `berth storage`, the page cache and `berth prefetch` | | [Gateway](gateway.md) | One OpenAI-compatible port, cold starts, fallbacks, KV snapshots, switch costs, wake pages, park in RAM | | [The harbor and telemetry](harbor.md) | Stowaways, the collisions board, alerts, every harbor panel, what Berth measures | diff --git a/docs/vfio.md b/docs/vfio.md new file mode 100644 index 0000000..fe9133f --- /dev/null +++ b/docs/vfio.md @@ -0,0 +1,114 @@ +# VFIO passthrough (experimental) + +Linux only, and experimental: it has run on one machine (the reference RTX 4090 +box) and nowhere else. The API and the messages can still change. + +When you pass the governed card to a virtual machine through VFIO, the card +leaves the host for the whole VM session: the NVIDIA driver lets go, `vfio-pci` +takes the function, and nothing on the host may touch it until the VM is done. +Berth coordinates its side of that handover. It drains the engines first, holds +the card for you, wraps your own switch script, and does not let the next +engine load until the card is back on its host driver. + +## What Berth does, and what it does not + +Berth does: + +- Drain every engine on the card at its own boundary, the way any lease does. +- Hold the card for the whole passthrough: a pinned lease at hold priority + (group `vfio`), so nothing else starts on it and nothing drains it. +- Check before the switch that nothing but the driver holds the card: the + processes with `/dev/nvidia*` or the card's DRM nodes open. Anyone but + berthd's own probe refuses the run, by PID and device node. +- Run your script with the terminal attached, so its sudo prompt and the VM's + console reach you, with `BERTH_VFIO_DEVICES` set to the card's PCI functions. +- Wait after the script exits until every function of the card is off + `vfio-pci` and the card answers `nvidia-smi` again, and only then release. +- Refuse to release a card that is not back. Until the card is back, the hold + stays; Ctrl-C rechecks and releases only when it is. + +Berth does not: + +- Bind or unbind a driver, or write to sysfs. Your script owns the switch. +- Ask for a password or keep one. Your script calls sudo itself, and the + prompt reaches your terminal. +- Manage the VM, libvirt, QEMU or any other device in the machine. Your + script and libvirt own those. +- Stop a process that holds the card. It names the holders and refuses; you + decide what to do with them. + +## Requirements + +- A machine with an IOMMU: VT-d or AMD-Vi on in the firmware, and + `intel_iommu=on iommu=pt` (or the AMD form) on the kernel command line. + `berth vfio status` says whether IOMMU is on. +- A second GPU or an iGPU for the host display, since the passed-through card + is out of the host's hands. +- The card's IOMMU group holding only the card's own functions (for a GPU, + its audio function is fine). A group with anything else in it blocks the + passthrough; `berth vfio status` names the members and their drivers, and + `berth vfio run` refuses until the group is clean. +- An NVIDIA card. AMD cards are out of scope in this release. + +## The commands + +`berth vfio status` reads the card's state and changes nothing: IOMMU on or +off, the IOMMU group and the driver each function is bound to, the processes +holding the card's device nodes, and what berthd reports about the card. It +takes the card's PCI address when you have it (`berth vfio status +0000:01:00.0`), which is the only way to read the state while the card is +bound to `vfio-pci`, because `nvidia-smi` cannot see the card then. + +`berth vfio run [--reason NAME] -- CMD` holds the card and runs your script: + + berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy + +Engines drain first. When the card is granted, berthd's probe is given a few +seconds to go quiet, then the openers are read again: anyone else holding a +device node refuses the run, names itself, and releases the hold. Berth never +stops it. When the script exits, the hold ends only when the card is back; +otherwise the hold stays and the message names the function that is still on +`vfio-pci`. + +The exit code is your script's exit code. + +## An example script to adapt + +This is an example to adapt to your own machine, not something Berth installs. +It uses `BERTH_VFIO_DEVICES`, so no PCI address is hardcoded: + +```sh +#!/bin/sh +# gpu-to-galaxy: pass the card to the VM, run it, bring the card back. +# Berth holds the card around this whole script; run it through +# berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy +set -eu +DEVS="${BERTH_VFIO_DEVICES:?run through: berth vfio run -- ...}" + +# The switch asks for your password once, through sudo. +for d in $DEVS; do + echo "$d" | sudo tee /sys/bus/pci/drivers/nvidia/unbind > /dev/null + echo "$d" | sudo tee /sys/bus/pci/drivers/vfio-pci/bind > /dev/null +done + +# The VM holds the card until it is shut down. +virsh start galaxy +while [ "$(virsh domstate galaxy 2>/dev/null || echo gone)" != "shut off" ]; do + sleep 5 +done + +for d in $DEVS; do + echo "$d" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null + echo "$d" | sudo tee /sys/bus/pci/drivers/nvidia/bind > /dev/null +done +``` + +A real script keeps the VM lifecycle its own way (a managed `virt-manager` +session, a `virsh shutdown` wait, hooks inside the libvirt domain); the shape +Berth needs is only this: switch, run the VM, switch back, and exit. If the +script dies with the card still on `vfio-pci`, the hold stays until the card +is back, and the next engine waits. + +While a VFIO hold is granted, berthd pauses its own GPU probe (so its +nvidia-smi child never blocks the unbind) and reports the card as passed +through to a VM; waiters are told so instead of seeing a probe failure. diff --git a/mkdocs.yml b/mkdocs.yml index 15fee83..1d81044 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -27,6 +27,7 @@ nav: - Install: install.md - Engines: engines.md - Leasing, games and holds: leasing.md + - VFIO passthrough (experimental): vfio.md - Models: models.md - Gateway: gateway.md - The harbor and telemetry: harbor.md From 83f85ab3e1073bccf1b8adb77192dae83b6af858 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:13:28 -0600 Subject: [PATCH 4/9] vfio test(cli): drop the unused tree helper --- internal/cli/vfio_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/internal/cli/vfio_test.go b/internal/cli/vfio_test.go index 35e10ea..da8ea57 100644 --- a/internal/cli/vfio_test.go +++ b/internal/cli/vfio_test.go @@ -156,16 +156,6 @@ done `, tr.sysfs, tr.sysfs, to, tr.sysfs) } -// driver reads one function's driver back from the fake tree. -func (tr *vfioTree) driver(addr string) string { - tr.t.Helper() - t, err := os.Readlink(filepath.Join(tr.sysfs, "bus", "pci", "devices", addr, "driver")) - if err != nil { - return "" - } - return filepath.Base(t) -} - // bind puts a function back on its driver by hand, as the operator's script // or hand does between two checks. func (tr *vfioTree) bind(addr, driver string) { From 5c1a843be27a62a011d0d5bb12580334fa8a1d21 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:17:06 -0600 Subject: [PATCH 5/9] vfio fix(cli): unreadable fd directories warn instead of refusing; dedupe nodes; status stays readable with IOMMU off --- internal/cli/vfio.go | 48 ++++++++++++++++++++++++++++++++------- internal/cli/vfio_run.go | 26 +++++++++++++++++---- internal/cli/vfio_test.go | 15 ++++++------ internal/vfio/linux.go | 6 ++++- 4 files changed, 74 insertions(+), 21 deletions(-) diff --git a/internal/cli/vfio.go b/internal/cli/vfio.go index 4da7ef5..733d7e7 100644 --- a/internal/cli/vfio.go +++ b/internal/cli/vfio.go @@ -191,22 +191,46 @@ func (v vfioStatusView) Human(c *Context, ok bool) error { if len(d.Openers) == 0 { c.Printf("openers none\n") } else { - c.Printf("openers %d\n", len(d.Openers)) + confirmed, unreadable := 0, 0 for _, o := range d.Openers { - who := strconv.Itoa(o.PID) - if o.Name != "" { - who += " " + o.Name + if o.Unreadable { + unreadable++ + } else { + confirmed++ } + } + c.Printf("openers %d\n", confirmed+unreadable) + for _, o := range d.Openers { if o.Unreadable { - c.Sub("pid %s: %s (its fd directory is unreadable; it may hold the card)", who, c.Style.Warn("/proc unreadable")) continue } + who := strconv.Itoa(o.PID) + if o.Name != "" { + who += " " + o.Name + } if o.Probe { c.Sub("pid %s holds %s (berthd's own probe)", who, strings.Join(o.Nodes, ", ")) continue } c.Sub("pid %s holds %s", who, strings.Join(o.Nodes, ", ")) } + if unreadable > 0 { + names := make([]string, 0, 2) + for _, o := range d.Openers { + if !o.Unreadable { + continue + } + who := "pid " + strconv.Itoa(o.PID) + if o.Name != "" { + who += " " + o.Name + } + if len(names) < 2 { + names = append(names, who) + } + } + c.Sub("%s and %d more: %s (their fd directories could not be read; they are not proven clear)", + strings.Join(names, ", "), unreadable-len(names), c.Style.Warn("/proc unreadable")) + } } if d.Berthd == nil { return nil @@ -275,9 +299,17 @@ func vfioStatus(c *Context, r vfio.Reader, smiBin string) int { if err != nil { return c.fail(Systemf("%v", err)) } - group, err := r.Group(card.Addr) - if err != nil { - return c.fail(Systemf("%v", err)) + var group []vfio.Function + if on { + // Without IOMMU the card has no group to read; the report says the + // state that blocks the passthrough and stays readable. + group, err = r.Group(card.Addr) + if err != nil { + return c.fail(Systemf("%v", err)) + } + } + if group == nil { + group = []vfio.Function{} } openers, err := r.Openers(card.Addr) if err != nil { diff --git a/internal/cli/vfio_run.go b/internal/cli/vfio_run.go index 6607693..d1c79b1 100644 --- a/internal/cli/vfio_run.go +++ b/internal/cli/vfio_run.go @@ -218,6 +218,12 @@ func vfioRun(c *Context, o vfioRunOpts) int { if err != nil { return c.fail(Systemf("read the openers: %v", err)) } + if bad := vfioUnreadable(openers); len(bad) > 0 { + for _, b := range bad { + c.Warnf("%s", b) + } + c.Warnf("the fd directories above are unreadable, so they are not proven clear; the driver unbind fails if one of them holds the card") + } if bad := vfioBadOpeners(openers); len(bad) > 0 { raw.Close() // the hold is not earned: the next engine may load e := Failf("the card's device nodes are in use; not holding %s", card.Addr) @@ -332,15 +338,12 @@ func vfioPreflight(r vfio.Reader, card vfioCard) ([]vfio.Function, *Error) { } // vfioBadOpeners is the openers that keep the card where it is: every -// process that is not berthd's own probe, including one whose fd directory -// could not be read (it may hold the card; Berth cannot prove otherwise). +// process that is not berthd's own probe and that the reader could read. func vfioBadOpeners(openers []vfio.Opener) []string { var bad []string for _, o := range openers { switch { - case o.Probe: - case o.Unreadable: - bad = append(bad, fmt.Sprintf("pid %d%s: /proc unreadable; it may hold the card", o.PID, vfioNamed(o.Name))) + case o.Probe, o.Unreadable: default: bad = append(bad, fmt.Sprintf("pid %d%s holds %s", o.PID, vfioNamed(o.Name), strings.Join(o.Nodes, ", "))) } @@ -348,6 +351,19 @@ func vfioBadOpeners(openers []vfio.Opener) []string { return bad } +// vfioUnreadable is the processes whose fd directories could not be read. +// They are reported, never refused: an unreadable list is no proof of a +// holder, and the unbind the script does fails where there is one. +func vfioUnreadable(openers []vfio.Opener) []string { + var bad []string + for _, o := range openers { + if o.Unreadable { + bad = append(bad, fmt.Sprintf("pid %d%s: /proc unreadable", o.PID, vfioNamed(o.Name))) + } + } + return bad +} + func vfioNamed(name string) string { if name == "" { return "" diff --git a/internal/cli/vfio_test.go b/internal/cli/vfio_test.go index da8ea57..f6dd5d1 100644 --- a/internal/cli/vfio_test.go +++ b/internal/cli/vfio_test.go @@ -374,9 +374,10 @@ func TestVfioRunRefusesOpenersAndReleasesTheHold(t *testing.T) { waitUntil(t, none(b)) } -// TestVfioRunRefusesAnUnreadableProc: a process whose fds cannot be read is -// listed as unreadable, never skipped. -func TestVfioRunRefusesAnUnreadableProc(t *testing.T) { +// TestVfioRunWarnsOnAnUnreadableProc: a process whose fds cannot be read is +// listed as unreadable, never skipped; it is a warning, not a refusal, +// because an unreadable list is no proof of a holder. +func TestVfioRunWarnsOnAnUnreadableProc(t *testing.T) { isolate(t) sock, b := startVfioDaemon(t) tr := newVfioTree(t) @@ -385,13 +386,13 @@ func TestVfioRunRefusesAnUnreadableProc(t *testing.T) { tr.opener(100, "locked", 1) tr.unreadable(100) smi := tr.smi() - code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond, poll: 5 * time.Millisecond}, "--socket", sock, "--", "/bin/true") - if code != ExitFail { - t.Fatalf("exit = %d, want 1 (stderr %q)", code, errOut) + if code != ExitOK { + t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) } if !strings.Contains(errOut, "pid 100") || !strings.Contains(errOut, "unreadable") { - t.Errorf("the refusal does not name the unreadable process: %q", errOut) + t.Errorf("the warning does not name the unreadable process: %q", errOut) } waitUntil(t, none(b)) } diff --git a/internal/vfio/linux.go b/internal/vfio/linux.go index 35a8a81..81e2dd5 100644 --- a/internal/vfio/linux.go +++ b/internal/vfio/linux.go @@ -9,6 +9,7 @@ import ( "fmt" "os" "path/filepath" + "slices" "sort" "strconv" "strings" @@ -126,7 +127,10 @@ func (r Reader) Openers(addr string) ([]Opener, error) { at[pid] = i out = append(out, Opener{PID: pid, Name: r.comm(pid), Probe: r.isProbe(pid)}) } - out[i].Nodes = append(out[i].Nodes, t) + // One node opened many times reads once. + if !slices.Contains(out[i].Nodes, t) { + out[i].Nodes = append(out[i].Nodes, t) + } } } sort.Slice(out, func(i, j int) bool { return out[i].PID < out[j].PID }) From d5b64dcfa994bb31a6d48525d3e8f64bc18ae69f Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:17:42 -0600 Subject: [PATCH 6/9] vfio docs: spell out the unreadable-opener rule --- docs/vfio.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/vfio.md b/docs/vfio.md index fe9133f..3cb8884 100644 --- a/docs/vfio.md +++ b/docs/vfio.md @@ -66,9 +66,11 @@ bound to `vfio-pci`, because `nvidia-smi` cannot see the card then. Engines drain first. When the card is granted, berthd's probe is given a few seconds to go quiet, then the openers are read again: anyone else holding a device node refuses the run, names itself, and releases the hold. Berth never -stops it. When the script exits, the hold ends only when the card is back; -otherwise the hold stays and the message names the function that is still on -`vfio-pci`. +stops it. A process whose fd directory cannot be read is named as unreadable; +it is a warning, not a refusal, and the unbind fails where it holds the card. +When the script exits, the hold ends only when the card is back; otherwise the +hold stays and the message names the function that is still on `vfio-pci`, and +Ctrl-C rechecks and releases only once the card is back. The exit code is your script's exit code. From 39550b4662fe71a9ddcd03dfa37298b8d80aff1f Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:28:50 -0600 Subject: [PATCH 7/9] vfio: the script runs in the foreground process group; docs rewritten A script in its own process group that reads the terminal is stopped by SIGTTIN, so sudo's password prompt would hang (reproduced with a pty: the child sat in state T). The script now stays in berth's process group, and a Ctrl-C reaches it from the terminal. Only ExitCode stays exported from the runner. The docs no longer claim a real run, and the example script moves each function with driver_override, which the card's audio function needs. --- CHANGELOG.md | 19 ++--- docs/vfio.md | 170 +++++++++++++++++--------------------- internal/cli/vfio_run.go | 11 +-- internal/runner/runner.go | 21 +---- 4 files changed, 89 insertions(+), 132 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b405901..143f1f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,20 +8,11 @@ has a matching GitHub release whose body comes from its section below. ### Added -- `berth vfio status` and `berth vfio run` (experimental, Linux only): the user side of passing the governed - card to a virtual machine through VFIO. `status` reads the card's VFIO state (IOMMU on or off, the IOMMU - group and the driver bound to each function, the processes holding the card's device nodes, what berthd - reports about the card) and changes nothing. `run -- CMD` acquires the VFIO hold the way `berth hold` does - (a pinned lease at hold priority, group `vfio`, engine `vfio:`), refuses when IOMMU is off, when - the IOMMU group holds devices that are not the card's, or when a process other than berthd's own probe - still holds a device node of the card (named by PID and node, hold released, exit 1; Berth never stops a - process), and then runs the operator's own switch script with the terminal attached and - `BERTH_VFIO_DEVICES` set to the card's PCI functions, so its sudo prompt reaches the terminal and the - script hardcodes no address. When the script exits, the hold ends only when every function is off - `vfio-pci` and the card answers `nvidia-smi` again; until then the hold stays and Ctrl-C releases it only - once the card is back, so the next engine never loads onto a card that is out. Berth never binds or - unbinds a driver, never writes to sysfs, and never asks for a password; the script (sudo, virsh) owns the - switch and libvirt owns the VM. See docs/vfio.md. +- `berth vfio status` and `berth vfio run` (experimental, Linux only) for passing the governed card to a + VM. `status` shows IOMMU, the card's IOMMU group with each function's driver, and the processes holding + the card. `run -- CMD` drains the card, refuses while another process holds it, runs your own switch + script with `BERTH_VFIO_DEVICES` set, and releases the hold only when the card is back on its host + driver. Berth never switches drivers or asks for passwords. See docs/vfio.md. - Gateway fallback routing: an `[[engine]]` may name a `fallback`, another engine in the file that needs no card (a CPU llama-server, an Ollama on another box), and `fallback_after` (10 s by default). A request that has waited that long for its own engine is proxied to the fallback when diff --git a/docs/vfio.md b/docs/vfio.md index 3cb8884..d864a84 100644 --- a/docs/vfio.md +++ b/docs/vfio.md @@ -1,116 +1,98 @@ # VFIO passthrough (experimental) -Linux only, and experimental: it has run on one machine (the reference RTX 4090 -box) and nowhere else. The API and the messages can still change. - -When you pass the governed card to a virtual machine through VFIO, the card -leaves the host for the whole VM session: the NVIDIA driver lets go, `vfio-pci` -takes the function, and nothing on the host may touch it until the VM is done. -Berth coordinates its side of that handover. It drains the engines first, holds -the card for you, wraps your own switch script, and does not let the next -engine load until the card is back on its host driver. - -## What Berth does, and what it does not - -Berth does: - -- Drain every engine on the card at its own boundary, the way any lease does. -- Hold the card for the whole passthrough: a pinned lease at hold priority - (group `vfio`), so nothing else starts on it and nothing drains it. -- Check before the switch that nothing but the driver holds the card: the - processes with `/dev/nvidia*` or the card's DRM nodes open. Anyone but - berthd's own probe refuses the run, by PID and device node. -- Run your script with the terminal attached, so its sudo prompt and the VM's - console reach you, with `BERTH_VFIO_DEVICES` set to the card's PCI functions. -- Wait after the script exits until every function of the card is off - `vfio-pci` and the card answers `nvidia-smi` again, and only then release. -- Refuse to release a card that is not back. Until the card is back, the hold - stays; Ctrl-C rechecks and releases only when it is. - -Berth does not: - -- Bind or unbind a driver, or write to sysfs. Your script owns the switch. -- Ask for a password or keep one. Your script calls sudo itself, and the - prompt reaches your terminal. -- Manage the VM, libvirt, QEMU or any other device in the machine. Your - script and libvirt own those. -- Stop a process that holds the card. It names the holders and refuses; you - decide what to do with them. +Linux only and experimental. It has passed its tests but has not yet run on a real passthrough, so the +commands and messages can still change. + +Passing the governed card to a virtual machine through VFIO takes it away from the host for the whole +VM session. The NVIDIA driver lets go, `vfio-pci` takes the card, and nothing on the host may touch it +until the VM is done. Berth coordinates its side of that handover. It drains your engines, holds the +card, runs your own switch script, and keeps the next engine waiting until the card is back on its host +driver. + +## What Berth does + +- Drains every engine on the card at its boundary, as for any other lease. +- Holds the card for the whole session with a pinned lease at hold priority, so nothing else starts on + it and nothing drains it. +- Checks, before your script runs, which processes hold `/dev/nvidia*` or the card's DRM nodes. Any + holder besides berthd's own probe stops the run, and Berth names it by PID and device node. +- Runs your script with the terminal attached, so its sudo prompt and the VM's console reach you, and + with `BERTH_VFIO_DEVICES` set to the card's PCI functions. +- After the script exits, waits until every function of the card is off `vfio-pci` and the card + answers `nvidia-smi`, and only then releases the hold. + +## What Berth leaves to you + +- The driver switch. Berth never binds or unbinds a driver and never writes to sysfs. +- Passwords. Your script calls sudo, and the prompt comes to your terminal. +- The VM, libvirt, QEMU and every other device you pass through. +- Processes that hold the card. Berth names them and refuses; it never stops them. ## Requirements -- A machine with an IOMMU: VT-d or AMD-Vi on in the firmware, and - `intel_iommu=on iommu=pt` (or the AMD form) on the kernel command line. - `berth vfio status` says whether IOMMU is on. -- A second GPU or an iGPU for the host display, since the passed-through card - is out of the host's hands. -- The card's IOMMU group holding only the card's own functions (for a GPU, - its audio function is fine). A group with anything else in it blocks the - passthrough; `berth vfio status` names the members and their drivers, and - `berth vfio run` refuses until the group is clean. -- An NVIDIA card. AMD cards are out of scope in this release. +- An IOMMU. Turn on VT-d (Intel) or AMD-Vi in the firmware, and add `intel_iommu=on iommu=pt` (or + `amd_iommu=on iommu=pt`) to the kernel command line. +- A second GPU or an iGPU for the host's display, because the passed-through card leaves the host. +- A clean IOMMU group. The card's group may hold only the card's own functions, such as its audio + function. `berth vfio status` lists the group's members and their drivers, and `berth vfio run` + refuses while anything else is in the group. +- An NVIDIA card. AMD cards are not covered yet. ## The commands -`berth vfio status` reads the card's state and changes nothing: IOMMU on or -off, the IOMMU group and the driver each function is bound to, the processes -holding the card's device nodes, and what berthd reports about the card. It -takes the card's PCI address when you have it (`berth vfio status -0000:01:00.0`), which is the only way to read the state while the card is -bound to `vfio-pci`, because `nvidia-smi` cannot see the card then. +`berth vfio status` reads the card's state and changes nothing. It shows whether IOMMU is on, the IOMMU +group and the driver bound to each function, the processes holding the card's device nodes, and what +berthd reports. While the card is bound to `vfio-pci`, `nvidia-smi` cannot see it, so pass the card's +address yourself (`berth vfio status 0000:01:00.0`). `berth vfio run [--reason NAME] -- CMD` holds the card and runs your script: - berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy +```sh +berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy +``` -Engines drain first. When the card is granted, berthd's probe is given a few -seconds to go quiet, then the openers are read again: anyone else holding a -device node refuses the run, names itself, and releases the hold. Berth never -stops it. A process whose fd directory cannot be read is named as unreadable; -it is a warning, not a refusal, and the unbind fails where it holds the card. -When the script exits, the hold ends only when the card is back; otherwise the -hold stays and the message names the function that is still on `vfio-pci`, and -Ctrl-C rechecks and releases only once the card is back. +Engines drain first. Once the card is granted, berthd's probe gets a few seconds to go quiet and Berth +reads the device holders again. If any process besides the probe still holds the card, Berth names it, +releases the hold and exits 1. A process whose open files Berth cannot read (another user's, usually +root's) gets a warning instead, because Berth cannot prove it clear. Such a process can block the unbind. -The exit code is your script's exit code. +When your script exits, Berth releases the hold only if the card is back. Otherwise it keeps the hold, +names the function still on `vfio-pci`, and waits. Each Ctrl-C checks again and releases only once the +card is back. `berth vfio run` exits with your script's exit code. + +While the hold is granted, berthd stops opening the card and `berth status` shows it as passed through +to a VM. Waiting requests are told why. ## An example script to adapt -This is an example to adapt to your own machine, not something Berth installs. -It uses `BERTH_VFIO_DEVICES`, so no PCI address is hardcoded: +This is an example, not something Berth installs. It moves every function the generic way +(`driver_override`), so it works for the card's audio function too, and it reads the addresses from +`BERTH_VFIO_DEVICES`. If your libvirt domain marks the card's `hostdev` as `managed='yes'`, libvirt +switches the drivers itself, and the script only starts the VM and waits. ```sh #!/bin/sh -# gpu-to-galaxy: pass the card to the VM, run it, bring the card back. -# Berth holds the card around this whole script; run it through -# berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy +# gpu-to-galaxy: give the card to the VM, run it, take the card back. +# Run it through: berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy set -eu -DEVS="${BERTH_VFIO_DEVICES:?run through: berth vfio run -- ...}" - -# The switch asks for your password once, through sudo. -for d in $DEVS; do - echo "$d" | sudo tee /sys/bus/pci/drivers/nvidia/unbind > /dev/null - echo "$d" | sudo tee /sys/bus/pci/drivers/vfio-pci/bind > /dev/null -done - -# The VM holds the card until it is shut down. -virsh start galaxy -while [ "$(virsh domstate galaxy 2>/dev/null || echo gone)" != "shut off" ]; do - sleep 5 -done - -for d in $DEVS; do - echo "$d" | sudo tee /sys/bus/pci/drivers/vfio-pci/unbind > /dev/null - echo "$d" | sudo tee /sys/bus/pci/drivers/nvidia/bind > /dev/null -done +devs="${BERTH_VFIO_DEVICES:?run this through berth vfio run}" + +to() { # to DRIVER: move every function of the card to DRIVER ("" = its own driver) + for d in $devs; do + echo "$1" | sudo tee "/sys/bus/pci/devices/$d/driver_override" >/dev/null + if [ -e "/sys/bus/pci/devices/$d/driver" ]; then + echo "$d" | sudo tee "/sys/bus/pci/devices/$d/driver/unbind" >/dev/null + fi + echo "$d" | sudo tee /sys/bus/pci/drivers_probe >/dev/null + done +} + +sudo modprobe vfio-pci +to vfio-pci +sudo virsh start galaxy +while [ "$(sudo virsh domstate galaxy)" != "shut off" ]; do sleep 5; done +to "" ``` -A real script keeps the VM lifecycle its own way (a managed `virt-manager` -session, a `virsh shutdown` wait, hooks inside the libvirt domain); the shape -Berth needs is only this: switch, run the VM, switch back, and exit. If the -script dies with the card still on `vfio-pci`, the hold stays until the card -is back, and the next engine waits. - -While a VFIO hold is granted, berthd pauses its own GPU probe (so its -nvidia-smi child never blocks the unbind) and reports the card as passed -through to a VM; waiters are told so instead of seeing a probe failure. +If the script dies with the card still on `vfio-pci`, the hold stays, and the next engine waits until +the card is back. diff --git a/internal/cli/vfio_run.go b/internal/cli/vfio_run.go index d1c79b1..da326d2 100644 --- a/internal/cli/vfio_run.go +++ b/internal/cli/vfio_run.go @@ -236,20 +236,21 @@ func vfioRun(c *Context, o vfioRunOpts) int { // The script runs with the terminal attached, so its sudo prompt and // the VM's console reach it, and with the card's PCI functions in - // BERTH_VFIO_DEVICES, so it hardcodes no address. + // BERTH_VFIO_DEVICES, so it hardcodes no address. It stays in this + // process group, unlike an engine under berth run: a child in its own + // group that reads the terminal is stopped by SIGTTIN, and sudo would + // hang at its password prompt. A Ctrl-C reaches the script from the + // terminal directly, so nothing is forwarded; signals meant for this + // run are drained once the script exits. cmd := exec.Command(argv[0], argv[1:]...) cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr - runner.Isolate(cmd) devices := vfioFunctionList(funcs, card.Addr) cmd.Env = append(os.Environ(), "BERTH_VFIO_DEVICES="+devices) if err := cmd.Start(); err != nil { return c.fail(Systemf("start %s: %v", argv[0], err)) } say("running %s with %s", argv[0], devices) - done := make(chan struct{}) - go runner.ForwardSignals(cmd, done) waitErr := cmd.Wait() - close(done) code := runner.ExitCode(cmd, waitErr) if code != 0 { say("%s exited with %d", argv[0], code) diff --git a/internal/runner/runner.go b/internal/runner/runner.go index 6750039..7261625 100644 --- a/internal/runner/runner.go +++ b/internal/runner/runner.go @@ -640,25 +640,8 @@ func command(o Options, out io.Writer) *exec.Cmd { return cmd } -// Isolate, SignalGroup, ForwardSignals and ExitCode are the wrapped-command -// pieces the vfio wrapper reuses: like berth run, it starts the command in -// its own process group with the terminal attached, forwards stop signals -// to it, and exits with its code. The vfio flow itself (the hold, the -// opener check, the driver wait) stays in the cli package. - -// Isolate puts the command in its own process group (a job object on -// Windows), so a stop reaches everything it forked. -func Isolate(cmd *exec.Cmd) { isolate(cmd) } - -// SignalGroup sends one stop signal to the command's process group. -func SignalGroup(cmd *exec.Cmd, s os.Signal) { signalGroup(cmd, s) } - -// ForwardSignals passes stop signals to the command's process group until -// done closes. -func ForwardSignals(cmd *exec.Cmd, done <-chan struct{}) { forwardSignals(cmd, done) } - -// ExitCode is the command's exit status: 128 plus the signal number when it -// was signalled. +// ExitCode is a wrapped command's exit status, 128 plus the signal number +// when it was signalled. berth vfio run exits with its script's code. func ExitCode(cmd *exec.Cmd, err error) int { return exitCode(cmd, err) } // forwardSignals passes stop signals (llama-swap stopping a model, Ctrl-C) From 17e1221b7cfe8375b324f7ffa3b12a8c15df6db7 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:34:11 -0600 Subject: [PATCH 8/9] vfio: berthd stops its nvidia-smi loop for a VFIO hold; quieter openers berthd's probe is one long-running nvidia-smi -lms, not a call per tick, so skipping the read during a hold still left berthd holding the card open, and the driver unbind would block on berthd itself. Found by running berth vfio status on aphotic. Stream.Pause kills the loop and keeps it stopped until the hold ends; the broker calls it every tick. berth vfio run now refuses a probe still open after the settle time, from a berthd that does not pause. Kernel threads (a readable, empty cmdline) are skipped in the openers scan: on aphotic they were 400 of the 430 unreadable entries. --- CHANGELOG.md | 2 +- docs/vfio.md | 11 ++--- internal/broker/broker.go | 5 +++ internal/broker/vfio_test.go | 41 +++++++++++++++++ internal/cli/vfio.go | 2 +- internal/cli/vfio_run.go | 18 +++++--- internal/cli/vfio_test.go | 23 ++++++++++ internal/gpu/gpu.go | 6 +++ internal/gpu/stream.go | 46 ++++++++++++++++++- internal/gpu/stream_pause_unix_test.go | 61 ++++++++++++++++++++++++++ internal/gpu/stream_test.go | 2 +- internal/vfio/linux.go | 5 +++ internal/vfio/vfio_linux_test.go | 12 +++++ 13 files changed, 219 insertions(+), 15 deletions(-) create mode 100644 internal/gpu/stream_pause_unix_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a25d4f8..0d98087 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ has a matching GitHub release whose body comes from its section below. `card.passthrough` and `card.returned` events mark each change. - `berth vfio status` and `berth vfio run` (experimental, Linux only) for passing the governed card to a VM. `status` shows IOMMU, the card's IOMMU group with each function's driver, and the processes holding - the card. `run -- CMD` drains the card, refuses while another process holds it, runs your own switch + the card. `run -- CMD` drains the card, stops berthd's own probe for the hold, refuses while any process holds it, runs your own switch script with `BERTH_VFIO_DEVICES` set, and releases the hold only when the card is back on its host driver. Berth never switches drivers or asks for passwords. See docs/vfio.md. - Gateway fallback routing: an `[[engine]]` may name a `fallback`, another engine in the file that diff --git a/docs/vfio.md b/docs/vfio.md index d864a84..c441031 100644 --- a/docs/vfio.md +++ b/docs/vfio.md @@ -14,8 +14,9 @@ driver. - Drains every engine on the card at its boundary, as for any other lease. - Holds the card for the whole session with a pinned lease at hold priority, so nothing else starts on it and nothing drains it. -- Checks, before your script runs, which processes hold `/dev/nvidia*` or the card's DRM nodes. Any - holder besides berthd's own probe stops the run, and Berth names it by PID and device node. +- Stops berthd's own `nvidia-smi` probe for the whole hold, then checks which processes still hold + `/dev/nvidia*` or the card's DRM nodes. Any holder stops the run, and Berth names it by PID and + device node. - Runs your script with the terminal attached, so its sudo prompt and the VM's console reach you, and with `BERTH_VFIO_DEVICES` set to the card's PCI functions. - After the script exits, waits until every function of the card is off `vfio-pci` and the card @@ -51,9 +52,9 @@ address yourself (`berth vfio status 0000:01:00.0`). berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy ``` -Engines drain first. Once the card is granted, berthd's probe gets a few seconds to go quiet and Berth -reads the device holders again. If any process besides the probe still holds the card, Berth names it, -releases the hold and exits 1. A process whose open files Berth cannot read (another user's, usually +Engines drain first. Once the card is granted, berthd stops its probe, and after a few seconds Berth +reads the device holders. If any process still holds the card, Berth names it, releases the hold and +exits 1. A process whose open files Berth cannot read (another user's, usually root's) gets a warning instead, because Berth cannot prove it clear. Such a process can block the unbind. When your script exits, Berth releases the hold only if the card is back. Otherwise it keeps the hold, diff --git a/internal/broker/broker.go b/internal/broker/broker.go index 045330c..772d12c 100644 --- a/internal/broker/broker.go +++ b/internal/broker/broker.go @@ -624,6 +624,11 @@ func (b *Broker) Tick(ctx context.Context) { b.mu.Lock() held := b.vfioHold() b.mu.Unlock() + if p, ok := b.probe.(gpu.Pauser); ok { + // A probe that keeps nvidia-smi running would hold the card open + // through the driver switch, so it stops for the whole hold. + p.Pause(held != nil) + } var snap gpu.Snapshot var err error if held == nil { diff --git a/internal/broker/vfio_test.go b/internal/broker/vfio_test.go index 96c8906..2dda0c4 100644 --- a/internal/broker/vfio_test.go +++ b/internal/broker/vfio_test.go @@ -374,3 +374,44 @@ func TestVFIOHoldDrainsTheHolderFirst(t *testing.T) { t.Fatalf("card state = %+v after the grant, want detaching", card) } } + +// pausingProbe records what the broker asked of a probe that keeps a +// process on the card between reads. +type pausingProbe struct { + fakeProbe + mu sync.Mutex + paused []bool +} + +func (p *pausingProbe) Pause(v bool) { + p.mu.Lock() + defer p.mu.Unlock() + if n := len(p.paused); n == 0 || p.paused[n-1] != v { + p.paused = append(p.paused, v) + } +} + +func (p *pausingProbe) changes() []bool { + p.mu.Lock() + defer p.mu.Unlock() + return append([]bool(nil), p.paused...) +} + +// A VFIO hold stops the probe's own process on the card for the whole hold, +// and the probe starts again once the hold ends. +func TestVFIOHoldPausesTheProbe(t *testing.T) { + skipNonLinux(t) + sysfsCard(t, "0000:01:00.0", "nvidia") + p := &pausingProbe{fakeProbe: fakeProbe{busID: "00000000:01:00.0"}} + b := vfioBroker(t, p) + b.Tick(context.Background()) + hold := must(t, acquire(b, vfioRequest())) + b.Tick(context.Background()) + hold.Release() + for i := 0; i < 3; i++ { + b.Tick(context.Background()) + } + if got := p.changes(); len(got) != 3 || got[0] || !got[1] || got[2] { + t.Fatalf("pause calls = %v, want running, paused for the hold, running again", got) + } +} diff --git a/internal/cli/vfio.go b/internal/cli/vfio.go index 578d2f8..8b09772 100644 --- a/internal/cli/vfio.go +++ b/internal/cli/vfio.go @@ -199,7 +199,7 @@ func (v vfioStatusView) Human(c *Context, ok bool) error { confirmed++ } } - c.Printf("openers %d\n", confirmed+unreadable) + c.Printf("openers %d\n", confirmed) for _, o := range d.Openers { if o.Unreadable { continue diff --git a/internal/cli/vfio_run.go b/internal/cli/vfio_run.go index 17cf6a0..a6a8793 100644 --- a/internal/cli/vfio_run.go +++ b/internal/cli/vfio_run.go @@ -219,10 +219,12 @@ func vfioRun(c *Context, o vfioRunOpts) int { return c.fail(Systemf("read the openers: %v", err)) } if bad := vfioUnreadable(openers); len(bad) > 0 { - for _, b := range bad { - c.Warnf("%s", b) + named := strings.Join(bad[:min(len(bad), 3)], "; ") + if len(bad) > 3 { + named += "; ..." } - c.Warnf("the fd directories above are unreadable, so they are not proven clear; the driver unbind fails if one of them holds the card") + c.Warnf("%d processes could not be read, so they are not proven clear (%s); if one holds the card, the driver unbind blocks", + len(bad), named) } if bad := vfioBadOpeners(openers); len(bad) > 0 { raw.Close() // the hold is not earned: the next engine may load @@ -344,7 +346,13 @@ func vfioBadOpeners(openers []vfio.Opener) []string { var bad []string for _, o := range openers { switch { - case o.Probe, o.Unreadable: + case o.Unreadable: + case o.Probe: + // berthd stops its probe for a VFIO hold; one still open after + // the settle time is a berthd that does not, and it would block + // the unbind like any other holder. + bad = append(bad, fmt.Sprintf("pid %d%s holds %s: berthd's own probe, still running; upgrade berthd so it stops the probe for a VFIO hold", + o.PID, vfioNamed(o.Name), strings.Join(o.Nodes, ", "))) default: bad = append(bad, fmt.Sprintf("pid %d%s holds %s", o.PID, vfioNamed(o.Name), strings.Join(o.Nodes, ", "))) } @@ -354,7 +362,7 @@ func vfioBadOpeners(openers []vfio.Opener) []string { // vfioUnreadable is the processes whose fd directories could not be read. // They are reported, never refused: an unreadable list is no proof of a -// holder, and the unbind the script does fails where there is one. +// holder, and the unbind the script does blocks where there is one. func vfioUnreadable(openers []vfio.Opener) []string { var bad []string for _, o := range openers { diff --git a/internal/cli/vfio_test.go b/internal/cli/vfio_test.go index f6dd5d1..3e9797f 100644 --- a/internal/cli/vfio_test.go +++ b/internal/cli/vfio_test.go @@ -374,6 +374,29 @@ func TestVfioRunRefusesOpenersAndReleasesTheHold(t *testing.T) { waitUntil(t, none(b)) } +// TestVfioRunRefusesALingeringProbe: berthd stops its probe for a VFIO hold; +// an nvidia-smi of berthd's still open after the settle time is a berthd +// that does not, and it would block the unbind like any other holder. +func TestVfioRunRefusesALingeringProbe(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + tr.opener(99, "berth", 1) + tr.opener(102, "nvidia-smi", 99, "/dev/nvidiactl") + smi := tr.smi() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--", "/bin/true") + if code != ExitFail { + t.Fatalf("exit = %d, want 1 (stderr %q)", code, errOut) + } + if !strings.Contains(errOut, "berthd's own probe, still running") { + t.Errorf("the refusal does not name berthd's probe: %q", errOut) + } + waitUntil(t, none(b)) +} + // TestVfioRunWarnsOnAnUnreadableProc: a process whose fds cannot be read is // listed as unreadable, never skipped; it is a warning, not a refusal, // because an unreadable list is no proof of a holder. diff --git a/internal/gpu/gpu.go b/internal/gpu/gpu.go index 00c0129..dd0221f 100644 --- a/internal/gpu/gpu.go +++ b/internal/gpu/gpu.go @@ -287,6 +287,12 @@ type Probe interface { Read(ctx context.Context) (Snapshot, error) } +// Pauser is a probe that keeps a process on the card between reads (the +// nvidia-smi loop) and can stop it while the card's driver is switched. +type Pauser interface { + Pause(bool) +} + // logger returns l, or the default logger when l is nil, so a probe without a // configured logger still records its PDH notes. func logger(l *slog.Logger) *slog.Logger { diff --git a/internal/gpu/stream.go b/internal/gpu/stream.go index b06cb0e..f15ca69 100644 --- a/internal/gpu/stream.go +++ b/internal/gpu/stream.go @@ -35,6 +35,32 @@ type Stream struct { procs []Proc procsAt time.Time procsFor uint64 // the scheduled card's used bytes when procs was read + + // paused keeps nvidia-smi off the card (a VFIO switch): stop ends the + // running loop, and resume wakes Run when the pause ends. + paused bool + stop context.CancelFunc + resume chan struct{} +} + +// Pause stops the nvidia-smi loop at once and keeps it stopped until +// Pause(false), so nothing of berthd's holds the card open while the +// operator switches its driver. Calling it again with the same value does +// nothing. +func (s *Stream) Pause(p bool) { + s.mu.Lock() + defer s.mu.Unlock() + if p == s.paused { + return + } + s.paused = p + if p && s.stop != nil { + s.stop() + } + if !p && s.resume != nil { + close(s.resume) + s.resume = nil + } } func (s *Stream) oneShot() NvidiaSMI { return NvidiaSMI{Index: s.Index, Bin: s.Bin, Log: s.Log} } @@ -46,10 +72,26 @@ func (s *Stream) interval() time.Duration { return s.Interval } -// Run feeds the stream until ctx ends, restarting nvidia-smi if it exits. +// Run feeds the stream until ctx ends, restarting nvidia-smi if it exits, +// and starts none while paused. func (s *Stream) Run(ctx context.Context) { for ctx.Err() == nil { - s.follow(ctx) + s.mu.Lock() + if s.paused { + wake := make(chan struct{}) + s.resume = wake + s.mu.Unlock() + select { + case <-ctx.Done(): + case <-wake: + } + continue + } + fctx, cancel := context.WithCancel(ctx) + s.stop = cancel + s.mu.Unlock() + s.follow(fctx) + cancel() select { case <-ctx.Done(): case <-time.After(2 * time.Second): diff --git a/internal/gpu/stream_pause_unix_test.go b/internal/gpu/stream_pause_unix_test.go new file mode 100644 index 0000000..0d4bdb9 --- /dev/null +++ b/internal/gpu/stream_pause_unix_test.go @@ -0,0 +1,61 @@ +// Copyright 2026 Trevin Tindall (T-Crypt) +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package gpu + +import ( + "context" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" +) + +func loops(dir string) []int { + b, _ := os.ReadFile(filepath.Join(dir, "loops")) + var pids []int + for _, f := range strings.Fields(string(b)) { + if n, err := strconv.Atoi(f); err == nil { + pids = append(pids, n) + } + } + return pids +} + +// A paused stream has no nvidia-smi open on the card, so a driver switch can +// run: Pause stops the loop at once and nothing restarts it until Resume. +func TestStreamPauseStopsTheLoop(t *testing.T) { + bin, dir := fakeSMI(t) + s := &Stream{Bin: bin, Interval: 50 * time.Millisecond} + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + go s.Run(ctx) + waitUntil(t, func() bool { return len(loops(dir)) == 1 }) + first := loops(dir)[0] + + s.Pause(true) + waitUntil(t, func() bool { return syscall.Kill(first, 0) != nil }) + time.Sleep(3 * time.Second) // past Run's 2 s restart delay + if n := len(loops(dir)); n != 1 { + t.Fatalf("%d nvidia-smi loops started while paused, want 1", n) + } + + s.Pause(false) + waitUntil(t, func() bool { return len(loops(dir)) == 2 }) +} + +func waitUntil(t *testing.T, ok func() bool) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for !ok() { + if time.Now().After(deadline) { + t.Fatal("condition not met within 5 s") + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/internal/gpu/stream_test.go b/internal/gpu/stream_test.go index ea88bbe..85b95ff 100644 --- a/internal/gpu/stream_test.go +++ b/internal/gpu/stream_test.go @@ -25,7 +25,7 @@ func fakeSMI(t *testing.T) (bin, dir string) { D=` + dir + ` case "$*" in *compute-apps*) echo x >> $D/calls; cat $D/apps; exit 0 ;; -*-lms*) while :; do echo "0, Fake 4090, 24564, $(cat $D/used), 100"; sleep 0.05; done ;; +*-lms*) echo $$ >> $D/loops; while :; do echo "0, Fake 4090, 24564, $(cat $D/used), 100"; sleep 0.05; done ;; *) echo "0, Fake 4090, 24564, $(cat $D/used), 100" ;; esac ` diff --git a/internal/vfio/linux.go b/internal/vfio/linux.go index 81e2dd5..ee91e11 100644 --- a/internal/vfio/linux.go +++ b/internal/vfio/linux.go @@ -105,6 +105,11 @@ func (r Reader) Openers(addr string) ([]Opener, error) { if _, err := os.Stat(filepath.Join(r.proc(), e.Name())); err != nil { continue } + // A kernel thread (a readable, empty cmdline) holds no files; its fd + // directory is root's alone, so reading it would only add noise. + if b, err := os.ReadFile(filepath.Join(r.proc(), e.Name(), "cmdline")); err == nil && len(b) == 0 { + continue + } fds := filepath.Join(r.proc(), e.Name(), "fd") list, err := os.ReadDir(fds) if err != nil { diff --git a/internal/vfio/vfio_linux_test.go b/internal/vfio/vfio_linux_test.go index 71501e0..591e532 100644 --- a/internal/vfio/vfio_linux_test.go +++ b/internal/vfio/vfio_linux_test.go @@ -87,6 +87,17 @@ func (tr tree) process(pid int, comm string, parent int, nodes ...string) { } } +// kthread adds a kernel thread: an empty cmdline and an fd directory no +// one but root reads, as /proc shows them to a user. +func (tr tree) kthread(pid int, comm string) { + tr.t.Helper() + tr.process(pid, comm, 2) + if err := os.WriteFile(filepath.Join(tr.proc, fmt.Sprintf("%d", pid), "cmdline"), nil, 0o644); err != nil { + tr.t.Fatal(err) + } + tr.locked(pid) +} + // locked closes one process's fd directory to everyone. func (tr tree) locked(pid int) { tr.t.Helper() @@ -214,6 +225,7 @@ func TestOpenersFindsTheCardHoldersAndNamesBerthdsProbe(t *testing.T) { tt.locked(103) tt.process(99, "berth", 1) tt.process(98, "filler", 1) + tt.kthread(97, "kworker/0:1") // holds no files: never listed ops, err := tt.reader().Openers(cardAddr) if err != nil { From 5cec580af9686f21ef2b52875fbd31004dac032e Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 16:36:49 -0600 Subject: [PATCH 9/9] vfio test(cli): status tests use the nvidia-smi stand-in They passed on a machine with nvidia-smi and failed on CI without one. --- internal/cli/vfio_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/internal/cli/vfio_test.go b/internal/cli/vfio_test.go index 3e9797f..667e545 100644 --- a/internal/cli/vfio_test.go +++ b/internal/cli/vfio_test.go @@ -237,7 +237,7 @@ func TestVfioStatusReadsTheFakeTree(t *testing.T) { tr := newVfioTree(t) tr.function(vfioAddr, "nvidia") tr.group() - code, out, errOut := runVfioStatusIn(t, tr, "", "--socket", "/nonexistent/berth.sock") + code, out, errOut := runVfioStatusIn(t, tr, tr.smi(), "--socket", "/nonexistent/berth.sock") if code != ExitOK { t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) } @@ -260,7 +260,7 @@ func TestVfioStatusJSONIsTheDocument(t *testing.T) { tr.opener(100, "app", 1, "/dev/nvidia0") tr.opener(101, "nvidia-smi", 99, "/dev/nvidiactl") tr.opener(99, "berth", 1) - code, out, errOut := runVfioStatusIn(t, tr, "", "--socket", "/nonexistent/berth.sock", "--format", "json") + code, out, errOut := runVfioStatusIn(t, tr, tr.smi(), "--socket", "/nonexistent/berth.sock", "--format", "json") if code != ExitOK { t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) } @@ -301,7 +301,7 @@ func TestVfioStatusReportsPassthroughWithAnAddress(t *testing.T) { tr := newVfioTree(t) tr.function(vfioAddr, "vfio-pci") tr.group() - code, out, errOut := runVfioStatusIn(t, tr, "", "--socket", "/nonexistent/berth.sock", vfioAddr) + code, out, errOut := runVfioStatusIn(t, tr, tr.smi(), "--socket", "/nonexistent/berth.sock", vfioAddr) if code != ExitOK { t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) }