diff --git a/.github/workflows/manager.yml b/.github/workflows/manager.yml index 6f16fc3..62702e3 100644 --- a/.github/workflows/manager.yml +++ b/.github/workflows/manager.yml @@ -50,21 +50,23 @@ jobs: sudo apt-get update sudo apt-get install -y busybox-static cpio gzip e2fsprogs + - name: Fetch the build-lite kernel and its modules + run: | + # The kernel (vmlinuz-lite-qemu) is produced by the build-lite + # workflow. The e1000 and qemu_fw_cfg modules come from the same + # artifact's initramfs, so they are built for that kernel. + curl -fsSL -o qemu-assets.zip \ + "https://nightly.link/AMDL-Web/wrapper/workflows/build-lite/lite/qemu-assets.zip" + unzip -o -q qemu-assets.zip vmlinuz-lite-qemu lite-initramfs.cpio.gz -d kernel-assets/ + - name: Build guest assets + env: + KERNEL_ASSETS: kernel-assets run: | chmod +x guest/build.sh guest/init ./guest/build.sh dist-guest CREATE_DATA_IMG=1 ./guest/build.sh dist-guest - - name: Fetch upstream kernel - run: | - # The kernel (vmlinuz-lite-qemu) is produced by the upstream wrapper - # build-lite workflow; fetch it via nightly.link so the guest image - # package is complete. - curl -sL -o qemu-assets.zip \ - "https://nightly.link/AMDL-Web/wrapper/workflows/build-lite/lite/qemu-assets.zip" - unzip -o -q qemu-assets.zip vmlinuz-lite-qemu -d dist-guest/ - - name: Upload guest assets uses: actions/upload-artifact@v4 with: diff --git a/manager/README.md b/manager/README.md index 1bb06c3..1d4bcae 100644 --- a/manager/README.md +++ b/manager/README.md @@ -276,13 +276,13 @@ The launcher is modeled after upstream # 1. the launcher (any platform) go build -o wrapper-manager-qemu ./cmd/wrapper-manager-qemu -# 2. the guest initramfs (on Linux; see guest/build.sh) -./guest/build.sh out # builds wrapper-manager-initramfs.cpio.gz -CREATE_DATA_IMG=1 ./guest/build.sh out # also create a fresh data.img - -# 3. the kernel is reused from the upstream wrapper qemu-assets artifact: -# https://nightly.link/AMDL-Web/wrapper/workflows/build-lite/lite/qemu-assets.zip -# (vmlinuz-lite-qemu). Put it next to the initramfs in the assets dir. +# 2. the guest (on Linux; see guest/README.md). The kernel and its e1000 and +# qemu_fw_cfg modules come from the build-lite qemu-assets artifact. +curl -fsSL -o qemu-assets.zip \ + "https://nightly.link/AMDL-Web/wrapper/workflows/build-lite/lite/qemu-assets.zip" +unzip -o qemu-assets.zip vmlinuz-lite-qemu lite-initramfs.cpio.gz -d kernel-assets/ +KERNEL_ASSETS=kernel-assets ./guest/build.sh out # kernel + initramfs +KERNEL_ASSETS=kernel-assets CREATE_DATA_IMG=1 ./guest/build.sh out # also a fresh data.img ``` Prebuilt artifacts (launchers for linux/windows/macos + guest image) are diff --git a/manager/guest/README.md b/manager/guest/README.md index 1fcff95..e5240d9 100644 --- a/manager/guest/README.md +++ b/manager/guest/README.md @@ -8,10 +8,7 @@ runs on Windows / macOS / any Linux host (see the parent README). ```text guest/ ├── build.sh # builds wrapper-manager-initramfs.cpio.gz (+ optional data.img) -├── init # guest init script (busybox sh) -└── modules/ # kernel modules reused from the upstream wrapper initramfs - ├── e1000.ko - └── qemu_fw_cfg.ko +└── init # guest init script (busybox sh) ``` `overlay/` is a build output (gitignored): build.sh rebuilds it from scratch @@ -22,9 +19,12 @@ certificates and the kernel modules. ## Build ```shell -# requires: go, cpio, gzip, e2fsprogs, glibc + ca-certificates (any Debian/Ubuntu) -./guest/build.sh out # initramfs only -CREATE_DATA_IMG=1 ./guest/build.sh out # also create a fresh 512MB data.img +# requires: go, cpio, gzip, e2fsprogs, file, kmod, glibc + ca-certificates (any Debian/Ubuntu) +curl -fsSL -o qemu-assets.zip \ + "https://nightly.link/AMDL-Web/wrapper/workflows/build-lite/lite/qemu-assets.zip" +unzip -o qemu-assets.zip vmlinuz-lite-qemu lite-initramfs.cpio.gz -d kernel-assets/ +KERNEL_ASSETS=kernel-assets ./guest/build.sh out # kernel + initramfs +KERNEL_ASSETS=kernel-assets CREATE_DATA_IMG=1 ./guest/build.sh out # also a fresh 512MB data.img ``` ## Kernel @@ -32,16 +32,12 @@ CREATE_DATA_IMG=1 ./guest/build.sh out # also create a fresh 512MB data.img `vmlinuz-lite-qemu` is **not** built here. It is the kernel produced by the [AMDL-Web/wrapper](https://github.com/AMDL-Web/wrapper) (a fork of [WorldObservationLog/wrapper](https://github.com/WorldObservationLog/wrapper)) -`build-lite` workflow and is distributed in its `qemu-assets` artifact: - -```shell -curl -sL -o qemu-assets.zip \ - "https://nightly.link/AMDL-Web/wrapper/workflows/build-lite/lite/qemu-assets.zip" -unzip -o qemu-assets.zip vmlinuz-lite-qemu -d out/ -``` - -Copy it next to the initramfs (the launcher's `--assets-dir`, default -`/guest`). +`build-lite` workflow and is distributed in its `qemu-assets` artifact. +build.sh copies it next to the initramfs, and takes `e1000.ko` and +`qemu_fw_cfg.ko` from the same artifact's `lite-initramfs.cpio.gz`: modules +built for another kernel do not load, which leaves the guest without network +and without its fw_cfg arguments. build.sh fails when their vermagic differs +from the kernel's version. ## What the guest does on boot (init) @@ -49,7 +45,11 @@ Copy it next to the initramfs (the launcher's `--assets-dir`, default (10.0.2.15, DNS 10.0.2.3) and writes `/etc/resolv.conf`. 2. Mounts the ext4 data disk (`/dev/vda` → `/data`); formats it with `mke2fs` on first boot when it is a raw empty image. -3. Executes `/wrapper-manager -host 0.0.0.0 -port 8080` from `/data`, so all +3. Runs `/wrapper-manager -host 0.0.0.0 -port 8080` from `/data`, so all manager state (wrapper-lite payload, accounts, tokens) persists in `data.img`. Optional manager args can be supplied via qemu fw_cfg (`manager_args`) or the kernel cmdline (`manager_args_b64=`). +4. Powers the guest off when the manager exits, so QEMU exits with it rather + than hanging on a kernel panic. The manager itself retries a failed + wrapper-lite download with backoff instead of exiting; only a download + that fails its pinned digest ends it. diff --git a/manager/guest/build.sh b/manager/guest/build.sh index 22b36d0..0d05b6b 100755 --- a/manager/guest/build.sh +++ b/manager/guest/build.sh @@ -10,12 +10,15 @@ # - cpio, gzip, e2fsprogs (mke2fs) # # The kernel (vmlinuz-lite-qemu) is NOT built here - it is reused from the -# upstream wrapper qemu-assets artifact (see guest/README.md). -# -# Kernel modules (e1000.ko / qemu_fw_cfg.ko) live in guest/modules/ and are -# copied verbatim from the upstream wrapper initramfs. +# build-lite qemu-assets artifact (see guest/README.md). KERNEL_ASSETS names +# the unzipped artifact: the kernel and the e1000 / qemu_fw_cfg modules both +# come from it, so they always match. A module built for another kernel fails +# to load, and the guest boots without network or fw_cfg arguments. set -euo pipefail +KERNEL_ASSETS="${KERNEL_ASSETS:?set KERNEL_ASSETS to the unzipped build-lite qemu-assets artifact}" +KERNEL_ASSETS="$(cd "$KERNEL_ASSETS" && pwd)" + REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" GUEST_DIR="$REPO_DIR/guest" OVERLAY="$GUEST_DIR/overlay" @@ -70,9 +73,21 @@ if [ ! -f "$OVERLAY/etc/ssl/certs/ca-certificates.crt" ]; then fi fi -# 5. Kernel modules. -echo "[build] staging kernel modules..." -cp "$GUEST_DIR"/modules/*.ko "$OVERLAY/" 2>/dev/null || true +# 5. Kernel and the modules built for it, from the same qemu-assets artifact. +echo "[build] staging the kernel and its modules..." +KERNEL_VER="$(file -bL "$KERNEL_ASSETS/vmlinuz-lite-qemu" | sed -n 's/.*version \([^ ]*\).*/\1/p')" +[ -n "$KERNEL_VER" ] || { echo "cannot read the version of $KERNEL_ASSETS/vmlinuz-lite-qemu"; exit 1; } +LITE_ROOT="$(mktemp -d)" +trap 'rm -rf "$LITE_ROOT"' EXIT +(cd "$LITE_ROOT" && gzip -dc "$KERNEL_ASSETS/lite-initramfs.cpio.gz" | cpio -id --quiet) +for mod in e1000 qemu_fw_cfg; do + [ -f "$LITE_ROOT/$mod.ko" ] || { echo "$mod.ko is missing from the qemu-assets initramfs"; exit 1; } + cp "$LITE_ROOT/$mod.ko" "$OVERLAY/$mod.ko" + magic="$(modinfo -F vermagic "$OVERLAY/$mod.ko" | cut -d' ' -f1)" + [ "$magic" = "$KERNEL_VER" ] || { echo "$mod.ko is built for $magic, the kernel is $KERNEL_VER"; exit 1; } +done +cp "$KERNEL_ASSETS/vmlinuz-lite-qemu" "$OUT_DIR/vmlinuz-lite-qemu" +echo "[build] kernel and modules: $KERNEL_VER" # 6. Init script at the initramfs root (required by the kernel). cp "$GUEST_DIR/init" "$OVERLAY/init" @@ -96,5 +111,4 @@ if [ "${CREATE_DATA_IMG:-0}" = "1" ]; then echo "[build] data.img created: $DATA_IMG" fi -echo "[build] done. Copy vmlinuz-lite-qemu next to the initramfs:" -echo " cp /vmlinuz-lite-qemu $OUT_DIR/" +echo "[build] done." diff --git a/manager/guest/init b/manager/guest/init index 3c2c69f..7903feb 100755 --- a/manager/guest/init +++ b/manager/guest/init @@ -12,7 +12,7 @@ /bin/busybox hostname wrapper-manager # --- network (QEMU user-mode NAT) --- -/bin/busybox insmod /e1000.ko 2>/dev/null || true +/bin/busybox insmod /e1000.ko || echo "[init] e1000.ko did not load: no network" /bin/busybox ip link set lo up /bin/busybox ip link set eth0 up 2>/dev/null || true @@ -45,17 +45,19 @@ fi # read it if present. The content is split on whitespace into arguments, so # no argument may contain a space. MANAGER_ARGS="-host 0.0.0.0 -port 8080" -/bin/busybox insmod /qemu_fw_cfg.ko 2>/dev/null || true +FROM_FW_CFG=0 +/bin/busybox insmod /qemu_fw_cfg.ko || echo "[init] qemu_fw_cfg.ko did not load" if [ -f /sys/firmware/qemu_fw_cfg/by_name/manager_args/raw ]; then echo "[init] reading manager args from fw_cfg" ARGS_LINE=$(/bin/busybox cat /sys/firmware/qemu_fw_cfg/by_name/manager_args/raw 2>/dev/null) if [ -n "$ARGS_LINE" ]; then MANAGER_ARGS="$ARGS_LINE" + FROM_FW_CFG=1 fi fi # Fallback: kernel cmdline base64 (works without qemu_fw_cfg driver) -if [ -z "$(echo "$MANAGER_ARGS" | /bin/busybox grep -c 'fw_cfg')" ]; then +if [ "$FROM_FW_CFG" = 0 ]; then for w in $(/bin/busybox cat /proc/cmdline); do case "$w" in manager_args_b64=*) @@ -72,5 +74,9 @@ echo "[init] eth0 carrier: $(cat /sys/class/net/eth0/carrier 2>/dev/null)" echo "[init] dns test: $(/bin/busybox nslookup nightly.link 10.0.2.3 2>&1 | head -3 | tr '\n' '|')" /bin/busybox chmod +x /wrapper-manager cd /data +# Not exec: the kernel panics when init exits, and a panicked guest keeps +# QEMU running. Powering off ends QEMU, so the host sees the manager stop. # shellcheck disable=SC2086 -exec /wrapper-manager $MANAGER_ARGS +/wrapper-manager $MANAGER_ARGS +echo "[init] wrapper-manager exited with status $?, powering off" +/bin/busybox poweroff -f diff --git a/manager/guest/modules/e1000.ko b/manager/guest/modules/e1000.ko deleted file mode 100644 index 7474925..0000000 Binary files a/manager/guest/modules/e1000.ko and /dev/null differ diff --git a/manager/guest/modules/qemu_fw_cfg.ko b/manager/guest/modules/qemu_fw_cfg.ko deleted file mode 100644 index cad36c5..0000000 Binary files a/manager/guest/modules/qemu_fw_cfg.ko and /dev/null differ diff --git a/manager/main.go b/manager/main.go index 0cb7351..fa6ee57 100644 --- a/manager/main.go +++ b/manager/main.go @@ -87,7 +87,10 @@ func main() { if !wrapperPayloadInstalled() { log.Warn("wrapper-lite does not exist, downloading...") } - PrepareWrapper(*mirror) + if err := PrepareWrapper(*mirror); err != nil { + log.Errorf("wrapper-lite cannot be installed: %v", err) + os.Exit(1) + } } if *prepare { diff --git a/manager/payload_source_test.go b/manager/payload_source_test.go index e01d1c3..759a141 100644 --- a/manager/payload_source_test.go +++ b/manager/payload_source_test.go @@ -11,7 +11,9 @@ import ( "path/filepath" "runtime" "strings" + "sync/atomic" "testing" + "time" ) // writePayloadZip builds a zip with the launcher and lite binary laid out like @@ -88,6 +90,13 @@ func TestPayloadFromAnotherSourceIsNotReady(t *testing.T) { // servePayload serves a payload zip and returns its URL and sha256. func servePayload(t *testing.T) (url, sum string) { + t.Helper() + return servePayloadAfter(t, 0) +} + +// servePayloadAfter serves the payload once failures requests have been +// answered 503. +func servePayloadAfter(t *testing.T, failures int32) (url, sum string) { t.Helper() zipPath := filepath.Join(t.TempDir(), "p.zip") writePayloadZip(t, zipPath) @@ -95,7 +104,14 @@ func servePayload(t *testing.T) (url, sum string) { if err != nil { t.Fatal(err) } - srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { _, _ = w.Write(body) })) + var seen atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if seen.Add(1) <= failures { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + _, _ = w.Write(body) + })) t.Cleanup(srv.Close) h := sha256.Sum256(body) return srv.URL + "/p.zip", hex.EncodeToString(h[:]) @@ -148,7 +164,7 @@ func TestPayloadSHA256RefusesAMismatchAndInstallsNothing(t *testing.T) { func TestPayloadSHA256MismatchDoesNotKeepAnInstalledPayload(t *testing.T) { t.Chdir(t.TempDir()) // An installed payload from another source is normally kept when the - // replacement fails; a digest mismatch must panic instead. + // replacement fails; a digest mismatch must fail instead. for _, p := range []string{launcherPath(), filepath.Join(wrapperDir, "rootfs", "system", "bin", "lite")} { if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { t.Fatal(err) @@ -159,10 +175,49 @@ func TestPayloadSHA256MismatchDoesNotKeepAnInstalledPayload(t *testing.T) { } url, _ := servePayload(t) setPin(t, url, strings.Repeat("0", 64)) - defer func() { - if r := recover(); r == nil { - t.Fatal("PrepareWrapper kept the unverified payload") + if err := prepareWrapper(false); !errors.Is(err, errPayloadSHA256) { + t.Fatalf("prepareWrapper kept the unverified payload: %v", err) + } + if wrapperPayloadReady() { + t.Fatal("the unverified payload reported ready") + } +} + +func TestPrepareWrapperGivesUpOnADigestMismatch(t *testing.T) { + t.Chdir(t.TempDir()) + url, _ := servePayload(t) + setPin(t, url, strings.Repeat("0", 64)) + done := make(chan error, 1) + go func() { done <- PrepareWrapper(false) }() + select { + case err := <-done: + if !errors.Is(err, errPayloadSHA256) { + t.Fatalf("PrepareWrapper: %v", err) } - }() - PrepareWrapper(false) + case <-time.After(10 * time.Second): + t.Fatal("PrepareWrapper retried a pin that cannot match") + } +} + +func TestPrepareWrapperRetriesUntilThePayloadArrives(t *testing.T) { + t.Chdir(t.TempDir()) + first, max := prepareRetryFirst, prepareRetryMax + prepareRetryFirst, prepareRetryMax = time.Millisecond, 2*time.Millisecond + t.Cleanup(func() { prepareRetryFirst, prepareRetryMax = first, max }) + url, sum := servePayloadAfter(t, 3) + setPin(t, url, sum) + done := make(chan struct{}) + var err error + go func() { err = PrepareWrapper(false); close(done) }() + select { + case <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(10 * time.Second): + t.Fatal("PrepareWrapper did not install the payload once it was served") + } + if !wrapperPayloadReady() { + t.Fatal("payload not ready after PrepareWrapper returned") + } } diff --git a/manager/wrapper.go b/manager/wrapper.go index 7ed7c26..bb81dcb 100644 --- a/manager/wrapper.go +++ b/manager/wrapper.go @@ -252,12 +252,39 @@ func describeInstall(res installResult) { } } -// PrepareWrapper downloads and extracts the wrapper-lite native package when +// prepareRetryFirst and prepareRetryMax bound the wait between attempts to +// install the payload. +var ( + prepareRetryFirst = 5 * time.Second + prepareRetryMax = time.Minute +) + +// PrepareWrapper installs the wrapper-lite payload, retrying a failed +// download with backoff until it succeeds. Nothing can be served without it, +// and exiting is no better: in the QEMU guest the manager is what init runs, +// so its exit stops the guest, and a host that sees a guest stop at once may +// blame its accelerator for a network that is merely down. A download that +// fails its pinned digest is returned instead: the same pin would fail again. +func PrepareWrapper(mirror bool) error { + delay := prepareRetryFirst + for { + err := prepareWrapper(mirror) + if err == nil || errors.Is(err, errPayloadSHA256) { + return err + } + log.Errorf("wrapper-lite is not ready: %v; trying again in %s", err, delay) + time.Sleep(delay) + delay = min(2*delay, prepareRetryMax) + } +} + +// prepareWrapper downloads and extracts the wrapper-lite native package when // missing or installed from another source. mirror routes the download through -// gh-proxy.com. Failing to replace an installed payload keeps it. -func PrepareWrapper(mirror bool) { +// gh-proxy.com. Failing to replace an installed payload keeps it, unless the +// download failed its pinned digest. +func prepareWrapper(mirror bool) error { if wrapperPayloadReady() { - return + return nil } installed := wrapperPayloadInstalled() if installed { @@ -271,12 +298,13 @@ func PrepareWrapper(mirror bool) { if err != nil { if installed && !errors.Is(err, errPayloadSHA256) { log.Warnf("keeping the installed wrapper-lite: %v", err) - return + return nil } - panic(err) + return err } log.Info("wrapper-lite ready") describeInstall(res) + return nil } // UpdateWrapper force-reinstalls the latest wrapper-lite payload from