From 07a4b8e9c981b0c4ae6bd08913ad57c8de22dac3 Mon Sep 17 00:00:00 2001 From: T-Crypt <132922589+T-Crypt@users.noreply.github.com> Date: Wed, 21 Oct 2026 17:42:44 -0600 Subject: [PATCH] vfio: --before and --after around the session, and a libvirt example A VM session has to stop the operator's own services that hold the card (a monitor running nvidia-smi in a loop, a desktop widget) after the engines drain and before the check that nothing holds it. Without a hook, a launcher had to take a plain hold, stop them, queue the VFIO hold and release the plain one. --before runs right after the grant; --after runs once the hold has ended, whenever --before ran, so what it stopped always comes back and never while the card is in the VM. A script that ran and exited 127 no longer reads as one that never started: the start error is its own value, so the card-back check always runs. contrib/vfio has a launcher and the domain's managed hostdev entries. --- CHANGELOG.md | 13 +++-- contrib/vfio/README.md | 21 ++++++++ contrib/vfio/domain-hostdev.xml | 22 +++++++++ contrib/vfio/vm-session.sh | 47 ++++++++++++++++++ docs/vfio.md | 27 +++++++--- internal/cli/commands.go | 13 +++-- internal/cli/vfio_run.go | 63 +++++++++++++++++------ internal/cli/vfio_test.go | 88 +++++++++++++++++++++++++++++++++ 8 files changed, 263 insertions(+), 31 deletions(-) create mode 100644 contrib/vfio/README.md create mode 100644 contrib/vfio/domain-hostdev.xml create mode 100755 contrib/vfio/vm-session.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d98087..7385ba0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,11 +12,14 @@ has a matching GitHub release whose body comes from its section below. `berth status` and its JSON show it as passed through to a VM, and waiters are told why nothing is granted. While a VFIO hold is granted, berthd does not open the card, so a driver switch can run. `card.passthrough` and `card.returned` events mark each change. -- `berth vfio status` and `berth vfio run` (experimental, Linux only) for passing the governed card to a - VM. `status` shows IOMMU, the card's IOMMU group with each function's driver, and the processes holding - the card. `run -- CMD` drains the card, stops berthd's own probe for the hold, refuses while any process holds it, runs your own switch - script with `BERTH_VFIO_DEVICES` set, and releases the hold only when the card is back on its host - driver. Berth never switches drivers or asks for passwords. See docs/vfio.md. +- `berth vfio status` and `berth vfio run` (experimental, Linux only) for passing the governed card to + a VM. `status` shows IOMMU, the card's IOMMU group with each function's driver, and the processes + holding the card. `run -- CMD` drains the card, stops berthd's own probe for the hold, refuses while + any process holds it, runs your own switch script with `BERTH_VFIO_DEVICES` set, and releases the + hold only when the card is back on its host driver. `--before` and `--after` run your own commands + around the session, for example to stop a service that holds the card and start it again once the + card is back. Berth never switches drivers or asks for passwords. See docs/vfio.md; `contrib/vfio` + has an example launcher and the libvirt device entries. - Gateway fallback routing: an `[[engine]]` may name a `fallback`, another engine in the file that needs no card (a CPU llama-server, an Ollama on another box), and `fallback_after` (10 s by default). A request that has waited that long for its own engine is proxied to the fallback when diff --git a/contrib/vfio/README.md b/contrib/vfio/README.md new file mode 100644 index 0000000..eef97e2 --- /dev/null +++ b/contrib/vfio/README.md @@ -0,0 +1,21 @@ +# VFIO examples + +Examples to adapt for passing the GPU Berth governs to a libvirt VM. Berth installs none of them. +[docs/vfio.md](../../docs/vfio.md) explains what Berth does during a passthrough and what it leaves to you. + +| File | What it is | +|---|---| +| [vm-session.sh](vm-session.sh) | A launcher. `berth vfio run` drains the card, `--before` stops your own services that hold it, libvirt boots the VM, and Berth gives the card back to your engines once the VM is off | +| [domain-hostdev.xml](domain-hostdev.xml) | The GPU part of a libvirt domain, with managed host devices so libvirt switches the drivers | + +To try it: + +1. Check the card can pass through: `berth vfio status` should show IOMMU on and an IOMMU group that + holds only the card's own functions. +2. Add the two `` entries to your domain with `virsh edit`, using the addresses + `berth vfio status` printed. +3. Copy `vm-session.sh`, set `DOMAIN` and `UNITS` at the top, and run it from a terminal first, so you + see sudo's prompt and Berth's messages. + +Run `berth vfio status` again while the VM is up: any process it lists as holding the card belongs in +`UNITS`. diff --git a/contrib/vfio/domain-hostdev.xml b/contrib/vfio/domain-hostdev.xml new file mode 100644 index 0000000..e2b2e8c --- /dev/null +++ b/contrib/vfio/domain-hostdev.xml @@ -0,0 +1,22 @@ + + + +
+ + + + +
+ + diff --git a/contrib/vfio/vm-session.sh b/contrib/vfio/vm-session.sh new file mode 100755 index 0000000..34267af --- /dev/null +++ b/contrib/vfio/vm-session.sh @@ -0,0 +1,47 @@ +#!/bin/sh +# Copyright 2026 Trevin Tindall (T-Crypt) +# SPDX-License-Identifier: Apache-2.0 +# +# vm-session: give the governed GPU to a libvirt VM for one session, and give it +# back to your engines when the VM shuts down. An example to adapt; Berth does +# not install it. Guide: docs/vfio.md. +# +# vm-session start the VM session (drains the card first) +# vm-session session the part berth vfio run runs; do not call it yourself +# +# What happens: +# 1. berth vfio run drains your engines at their boundaries and holds the card. +# 2. --before stops the services that keep the card open without a lease +# (a monitor running nvidia-smi in a loop, a desktop widget). +# 3. Berth checks that nothing holds the card any more, then runs `session`: +# libvirt switches the card to vfio-pci (managed hostdevs, see +# domain-hostdev.xml), the VM boots, and the session waits for it to shut +# down. sudo asks for your password. +# 4. Berth releases the card only once it is back on its host driver, then +# --after starts those services again. +set -eu + +# Change these three for your machine. +DOMAIN=${VM_DOMAIN:-win11} +UNITS=${VM_STOP_UNITS:-"llama-swap.service"} # user units to stop for the session +BERTH=${BERTH:-berth} # a full path when run from a desktop keybind + +case ${1:-start} in +start) + exec "$BERTH" vfio run --reason "$DOMAIN" \ + --before "systemctl --user stop $UNITS" \ + --after "systemctl --user start $UNITS" \ + -- "$0" session + ;; +session) + echo "devices for $DOMAIN: $BERTH_VFIO_DEVICES" + sudo virsh start "$DOMAIN" + while [ "$(sudo virsh domstate "$DOMAIN")" != "shut off" ]; do + sleep 5 + done + ;; +*) + echo "usage: vm-session [start]" >&2 + exit 2 + ;; +esac diff --git a/docs/vfio.md b/docs/vfio.md index c441031..86dc7e6 100644 --- a/docs/vfio.md +++ b/docs/vfio.md @@ -46,25 +46,40 @@ group and the driver bound to each function, the processes holding the card's de berthd reports. While the card is bound to `vfio-pci`, `nvidia-smi` cannot see it, so pass the card's address yourself (`berth vfio status 0000:01:00.0`). -`berth vfio run [--reason NAME] -- CMD` holds the card and runs your script: +`berth vfio run [--reason NAME] [--before CMD] [--after CMD] -- CMD` holds the card and runs your +script: ```sh -berth vfio run --reason galaxy -- ~/bin/gpu-to-galaxy +berth vfio run --reason galaxy \ + --before "systemctl --user stop llama-swap.service" \ + --after "systemctl --user start llama-swap.service" \ + -- ~/bin/gpu-to-galaxy ``` -Engines drain first. Once the card is granted, berthd stops its probe, and after a few seconds Berth -reads the device holders. If any process still holds the card, Berth names it, releases the hold and -exits 1. A process whose open files Berth cannot read (another user's, usually +Engines drain first. Once the card is granted, Berth runs `--before`, if you gave one. Use it to stop +your own services that keep the card open without a lease, such as a monitor that runs `nvidia-smi` in +a loop. berthd stops its own probe for the hold. After a few seconds Berth reads the device holders. If +any process still holds the card, Berth names it, releases the hold and exits 1. A process whose open files Berth cannot read (another user's, usually root's) gets a warning instead, because Berth cannot prove it clear. Such a process can block the unbind. When your script exits, Berth releases the hold only if the card is back. Otherwise it keeps the hold, names the function still on `vfio-pci`, and waits. Each Ctrl-C checks again and releases only once the card is back. `berth vfio run` exits with your script's exit code. +`--after` runs once the hold has ended, whenever `--before` ran. That covers the card coming back, a +refusal and a failed `--before`. What `--before` stopped always comes back, and never while the card +is still in the VM. Both run with `sh -c`, the terminal attached and `BERTH_VFIO_DEVICES` set. + While the hold is granted, berthd stops opening the card and `berth status` shows it as passed through to a VM. Waiting requests are told why. -## An example script to adapt +## Examples to adapt + +[contrib/vfio](https://github.com/T-Crypt/berth/tree/main/contrib/vfio) has a complete launcher for a +libvirt VM (`vm-session.sh`) and the GPU part of the domain (`domain-hostdev.xml`). With managed host +devices, libvirt switches the drivers, and the launcher only starts the VM and waits for it. + +Without libvirt's managed devices, your script switches the drivers itself, as below. This is an example, not something Berth installs. It moves every function the generic way (`driver_override`), so it works for the card's audio function too, and it reads the addresses from diff --git a/internal/cli/commands.go b/internal/cli/commands.go index 9e19445..7dfc9be 100644 --- a/internal/cli/commands.go +++ b/internal/cli/commands.go @@ -574,15 +574,20 @@ docs/vfio.md has the requirements and an example script.`, { Name: "run", Summary: "hold the card, run the switch script, wait for the card back", - Args: "[--reason NAME] [--wait D] -- CMD [ARGS...]", + Args: "[--reason NAME] [--wait D] [--before CMD] [--after CMD] -- CMD [ARGS...]", Help: `Acquire the VFIO hold the way berth hold does: engines drain at their boundaries and none starts. Then the script runs with the terminal attached, so its sudo prompt reaches you, and with BERTH_VFIO_DEVICES set to the card's PCI functions, so it hardcodes no address. -Before the script starts, the openers are re-read: anyone but berthd's own -probe holding /dev/nvidia* or the card's DRM nodes refuses the run, names every -PID and node, releases the hold and exits 1. Berth never stops a process. +--before runs once the engines are off the card: stop your own services that +hold it there. --after runs once the hold has ended, whenever --before ran, +so what --before stopped always comes back, never while the card is out. + +Before the script starts, berthd stops its own probe and the openers are +re-read: any process holding /dev/nvidia* or the card's DRM nodes refuses the +run, names every PID and node, releases the hold and exits 1. Berth never stops +a process. When the script exits, the hold ends only when every function of the card is off vfio-pci and the card answers nvidia-smi again (up to a minute). Until diff --git a/internal/cli/vfio_run.go b/internal/cli/vfio_run.go index a6a8793..5ec901d 100644 --- a/internal/cli/vfio_run.go +++ b/internal/cli/vfio_run.go @@ -112,6 +112,8 @@ func vfioRun(c *Context, o vfioRunOpts) int { sock := fs.String("socket", defaultSocket(), "berthd socket ($BERTH_SOCKET)") reason := fs.String("reason", "", "shown on the lease, e.g. galaxy (default: passthrough)") wait := fs.Duration("wait", 10*time.Minute, "how long to wait for engines to drain; 0 = berthd's default; negative = no deadline") + before := fs.String("before", "", "shell command run once the engines are off the card, before the check that nothing holds it (stop your own services here)") + after := fs.String("after", "", "shell command run once the hold has ended, whenever --before ran (start those services again)") if code, done := c.Parse(fs); done { return code } @@ -205,6 +207,29 @@ func vfioRun(c *Context, o vfioRunOpts) int { } say("card held as %s (%s); engines are off the card", lease, name) + devices := vfioFunctionList(funcs, card.Addr) + + // --before stops what the operator runs beside the engines (a monitor + // loop, a desktop shell) while the hold keeps the engines off; --after + // starts it again once the hold has ended, whatever happened between. + // The hold is closed first: a deferred --after must not run while the + // card is still held. + if *before != "" { + defer func() { + raw.Close() + if *after == "" { + return + } + say("running --after: %s", *after) + if code, err := vfioExec([]string{"/bin/sh", "-c", *after}, devices); err != nil || code != 0 { + c.Warnf("--after did not finish cleanly (exit %d, %v)", code, err) + } + }() + say("running --before: %s", *before) + if code, err := vfioExec([]string{"/bin/sh", "-c", *before}, devices); err != nil || code != 0 { + return c.fail(Failf("--before did not finish cleanly (exit %d, %v); not holding %s", code, err, card.Addr)) + } + } // berthd's probe opens the card's device every tick. Give it the settle // time to go quiet (V1 pauses it for a vfio hold), then decide on one @@ -236,24 +261,12 @@ func vfioRun(c *Context, o vfioRunOpts) int { return c.fail(e) } - // The script runs with the terminal attached, so its sudo prompt and - // the VM's console reach it, and with the card's PCI functions in - // BERTH_VFIO_DEVICES, so it hardcodes no address. It stays in this - // process group, unlike an engine under berth run: a child in its own - // group that reads the terminal is stopped by SIGTTIN, and sudo would - // hang at its password prompt. A Ctrl-C reaches the script from the - // terminal directly, so nothing is forwarded; signals meant for this - // run are drained once the script exits. - cmd := exec.Command(argv[0], argv[1:]...) - cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr - devices := vfioFunctionList(funcs, card.Addr) - cmd.Env = append(os.Environ(), "BERTH_VFIO_DEVICES="+devices) - if err := cmd.Start(); err != nil { + say("running %s with %s", argv[0], devices) + code, err := vfioExec(argv, devices) + if err != nil { + // It never started, so the card never left: nothing to wait for. return c.fail(Systemf("start %s: %v", argv[0], err)) } - say("running %s with %s", argv[0], devices) - waitErr := cmd.Wait() - code := runner.ExitCode(cmd, waitErr) if code != 0 { say("%s exited with %d", argv[0], code) } @@ -415,3 +428,21 @@ func vfioFunctionList(funcs []vfio.Function, card string) string { } return strings.Join(out, " ") } + +// vfioExec runs one of the operator's commands with the terminal attached, +// so a sudo prompt and a VM console reach it, and with the card's PCI +// functions in BERTH_VFIO_DEVICES, so it hardcodes no address. It stays in +// this process group, unlike an engine under berth run: a child in its own +// group that reads the terminal is stopped by SIGTTIN, and sudo would hang +// at its password prompt. A Ctrl-C reaches it from the terminal directly, +// so nothing is forwarded. err is set only when the command did not start; +// a command that ran reports its exit code, whatever it is. +func vfioExec(argv []string, devices string) (int, error) { + cmd := exec.Command(argv[0], argv[1:]...) + cmd.Stdin, cmd.Stdout, cmd.Stderr = os.Stdin, os.Stdout, os.Stderr + cmd.Env = append(os.Environ(), "BERTH_VFIO_DEVICES="+devices) + if err := cmd.Start(); err != nil { + return 0, err + } + return runner.ExitCode(cmd, cmd.Wait()), nil +} diff --git a/internal/cli/vfio_test.go b/internal/cli/vfio_test.go index 667e545..ff1297c 100644 --- a/internal/cli/vfio_test.go +++ b/internal/cli/vfio_test.go @@ -544,3 +544,91 @@ func (w errWriter) Write(p []byte) (int, error) { defer w.mu.Unlock() return w.b.Write(p) } + +// TestVfioRunBeforeStopsAHolderAndAfterStartsItAgain: --before runs once the +// engines are off the card, so it can stop a service that holds the card; +// the check then passes, the script runs, and --after runs once the hold +// has ended. +func TestVfioRunBeforeStopsAHolderAndAfterStartsItAgain(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + tr.opener(100, "monitor", 1, "/dev/nvidia0") + smi := tr.smi() + marks := t.TempDir() + before := "rm -rf " + filepath.Join(tr.proc, "100") + " && touch " + filepath.Join(marks, "before") + after := "touch " + filepath.Join(marks, "after") + cmd := tr.script("switch", tr.flip("vfio-pci")+tr.flip("nvidia")) + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond, poll: 5 * time.Millisecond}, + "--socket", sock, "--before", before, "--after", after, "--", cmd) + if code != ExitOK { + t.Fatalf("exit = %d, want 0 (stderr %q)", code, errOut) + } + for _, m := range []string{"before", "after"} { + if _, err := os.Stat(filepath.Join(marks, m)); err != nil { + t.Errorf("--%s did not run: %v", m, err) + } + } + waitUntil(t, none(b)) +} + +// TestVfioRunAfterRunsWhenTheCheckRefuses: --before ran and stopped something, +// the check still found a holder, so the run is refused; --after puts back +// what --before stopped. +func TestVfioRunAfterRunsWhenTheCheckRefuses(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + tr.opener(100, "qemu", 1, "/dev/nvidia0") + smi := tr.smi() + marks := t.TempDir() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--before", "true", "--after", "touch "+filepath.Join(marks, "after"), "--", "/bin/true") + if code != ExitFail || !strings.Contains(errOut, "pid 100 qemu holds /dev/nvidia0") { + t.Fatalf("exit = %d, want the opener refusal (stderr %q)", code, errOut) + } + if _, err := os.Stat(filepath.Join(marks, "after")); err != nil { + t.Errorf("--after did not run after the refusal: %v", err) + } + waitUntil(t, none(b)) +} + +// TestVfioRunBeforeFailureReleases: a --before that fails ends the run with +// the card still on the host, and the hold is released. +func TestVfioRunBeforeFailureReleases(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + smi := tr.smi() + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond}, + "--socket", sock, "--before", "exit 4", "--", "/bin/true") + if code != ExitFail || !strings.Contains(errOut, "--before did not finish cleanly (exit 4") { + t.Fatalf("exit = %d, want the --before refusal (stderr %q)", code, errOut) + } + waitUntil(t, none(b)) +} + +// TestVfioRunScriptExit127StillWaitsForTheCard: a script that ran and exited +// 127 (a typo inside it) may still have moved the card; the run keeps the +// hold until the card is back, as for any other exit code. +func TestVfioRunScriptExit127StillWaitsForTheCard(t *testing.T) { + isolate(t) + sock, b := startVfioDaemon(t) + tr := newVfioTree(t) + tr.function(vfioAddr, "nvidia") + tr.group() + smi := tr.smi() + cmd := tr.script("switch", tr.flip("vfio-pci")+tr.flip("nvidia")+"exit 127\n") + code, _, errOut := runVfioRunIn(t, tr, smi, vfioRunOpts{settle: time.Millisecond, poll: 5 * time.Millisecond}, + "--socket", sock, "--", cmd) + if code != 127 || !strings.Contains(errOut, "the card is back; hold released") { + t.Fatalf("exit = %d, want 127 after the card-back check (stderr %q)", code, errOut) + } + waitUntil(t, none(b)) +}