From 4a12f9b02b34b98623bdc76aed22c40c4d800f2e Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:29:32 +0100 Subject: [PATCH 1/4] Pin the whole NVIDIA userspace to the host's driver version libcuda1 needs libnvidia-ptxjitcompiler1, libnvidia-gpucomp and libnvidia-pkcs11-openssl3 at exactly its own version. Only the four top-level packages were requested at the host's version, so apt resolved the rest to the newest release in the CUDA repo and the install failed (#209). A version stanza now covers the whole libnvidia-* family. It has to come before the origin pin: apt takes the first stanza that matches a version, so behind the origin pin it is never read. The fallback that kept only libcuda1 pinned could not work either -- libnvcuvid1 at the newest release depends on libcuda1 at that release -- so it is replaced by an error that carries apt's own reason. setup_hwaccel no longer prints a green line when no GPU ended up usable, and an AMD APU now reports amd instead of none. --- .github/workflows/verify-split-files.yml | 5 ++ lib/hwaccel.func | 109 +++++++++++++++-------- tests/nvidia-version-pin.sh | 63 +++++++++++++ 3 files changed, 140 insertions(+), 37 deletions(-) create mode 100644 tests/nvidia-version-pin.sh diff --git a/.github/workflows/verify-split-files.yml b/.github/workflows/verify-split-files.yml index 955961e..64d9ff3 100644 --- a/.github/workflows/verify-split-files.yml +++ b/.github/workflows/verify-split-files.yml @@ -30,6 +30,7 @@ on: - "tests/uv-required-version.sh" - "tests/docker-lxcfs.sh" - "tests/java-memory.sh" + - "tests/nvidia-version-pin.sh" - "tests/external-installer.sh" - "tests/forge-version-key.sh" - "tests/nodejs-key-refresh.sh" @@ -85,6 +86,7 @@ on: - "tests/uv-required-version.sh" - "tests/docker-lxcfs.sh" - "tests/java-memory.sh" + - "tests/nvidia-version-pin.sh" - "tests/external-installer.sh" - "tests/forge-version-key.sh" - "tests/nodejs-key-refresh.sh" @@ -164,6 +166,9 @@ jobs: - name: Java memory drop-in tests run: bash tests/java-memory.sh + - name: NVIDIA userspace version pin tests + run: bash tests/nvidia-version-pin.sh + - name: External installer confirmation tests run: bash tests/external-installer.sh diff --git a/lib/hwaccel.func b/lib/hwaccel.func index 3793a8b..7c0fcc1 100644 --- a/lib/hwaccel.func +++ b/lib/hwaccel.func @@ -338,7 +338,7 @@ setup_hwaccel() { fi if [[ "$HWACCEL_VENDOR" == "none" ]]; then case "$_gpu_type" in - AMD) HWACCEL_VENDOR="amd" ;; + AMD*) HWACCEL_VENDOR="amd" ;; INTEL*) HWACCEL_VENDOR="intel" ;; esac fi @@ -346,7 +346,11 @@ setup_hwaccel() { export HWACCEL_VENDOR cache_installed_version "hwaccel" "1.0" - msg_ok "Setup Hardware Acceleration" + if [[ "$HWACCEL_VENDOR" == "none" ]]; then + msg_warn "Hardware acceleration is not available - see the messages above" + else + msg_ok "Setup Hardware Acceleration (${HWACCEL_VENDOR})" + fi } # ══════════════════════════════════════════════════════════════════════════════ @@ -663,6 +667,57 @@ ENVEOF # ══════════════════════════════════════════════════════════════════════════════ # NVIDIA GPU Setup # ══════════════════════════════════════════════════════════════════════════════ + +# Pick the host's exact version when the repo has it, and only then settle +# for another release in the same series. Taking the newest of the series +# unconditionally is the wrong direction: a host on 580.126 would be handed +# 580.178, which is newer than the loaded kernel module and therefore fails +# the same way a whole major mismatch does. +_cuda_pick_version() { + local host="$1" madison exact + madison=$(apt-cache madison libcuda1 2>/dev/null | awk '{print $3}') || return 0 + # Dots are escaped: unescaped they are regex wildcards, so 580.178.04 + # would also match a hypothetical 580x178x04. + exact=$(printf '%s\n' "$madison" | grep -E "^${host//./\\.}(-|$)" | head -1 || true) + if [[ -n "$exact" ]]; then + printf '%s\n' "$exact" + return 0 + fi + printf '%s\n' "$madison" | grep -E "^${host%%.*}\." | head -1 || true +} + +# libcuda1 needs libnvidia-ptxjitcompiler1, libnvidia-gpucomp and usually +# libnvidia-pkcs11-openssl3 at exactly its own version; without a version pin +# apt resolves them to the newest release and the install fails. Pinned rather +# than listed: the repo carries libcuda1 builds with and without the pkcs11 +# dependency. The priority has to stay above the origin pin's. +_nvidia_version_pin() { + cat </etc/apt/preferences.d/nvidia-cuda-pin -Package: libcuda* libnvcuvid* libnvidia-* nvidia-smi nvidia-utils-* cuda-* -Pin: origin developer.download.nvidia.com -Pin-Priority: 1001 -NVIDIA_PIN + local nvidia_pin_file="/etc/apt/preferences.d/nvidia-cuda-pin" + _nvidia_apt_pins >"$nvidia_pin_file" # Extract major version for flexible matching (580.126.09 -> 580) local nvidia_major_version="${nvidia_host_version%%.*}" local cuda_list="/etc/apt/sources.list.d/cuda-community-scripts.list" local chosen_repo="" available_version="" cuda_repo="" - # Pick the host's exact version when the repo has it, and only then settle - # for another release in the same series. Taking the newest of the series - # unconditionally is the wrong direction: a host on 580.126 would be handed - # 580.178, which is newer than the loaded kernel module and therefore fails - # the same way a whole major mismatch does. - _cuda_pick_version() { - local madison exact - madison=$(apt-cache madison libcuda1 2>/dev/null | awk '{print $3}') || return 0 - # Dots are escaped: unescaped they are regex wildcards, so 580.178.04 - # would also match a hypothetical 580x178x04. - exact=$(printf '%s\n' "$madison" | grep -E "^${nvidia_host_version//./\\.}(-|$)" | head -1 || true) - if [[ -n "$exact" ]]; then - printf '%s\n' "$exact" - return 0 - fi - printf '%s\n' "$madison" | grep -E "^${nvidia_major_version}\." | head -1 || true - } - for cuda_repo in "${cuda_repos[@]}"; do # The keyring package drops its own list file for whichever repo it came # from. Left in place it would keep a rejected candidate active alongside @@ -783,7 +811,7 @@ NVIDIA_PIN echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${cuda_repo}/x86_64/ /" >"$cuda_list" $STD apt-get -y update 2>/dev/null || true - available_version="$(_cuda_pick_version)" + available_version="$(_cuda_pick_version "$nvidia_host_version")" if [[ -n "$available_version" ]]; then chosen_repo="$cuda_repo" break @@ -795,12 +823,12 @@ NVIDIA_PIN # Last resort: the distribution's own non-free packages. rm -f "$cuda_list" $STD apt-get -y update 2>/dev/null || true - available_version="$(_cuda_pick_version)" + available_version="$(_cuda_pick_version "$nvidia_host_version")" [[ -n "$available_version" ]] && chosen_repo="Debian non-free" fi - unset -f _cuda_pick_version if [[ -n "$chosen_repo" ]]; then + _nvidia_apt_pins "$available_version" >"$nvidia_pin_file" msg_info "Installing NVIDIA userspace ${available_version} from ${chosen_repo}" local nvidia_pkgs="libcuda1=${available_version} libnvcuvid1=${available_version} libnvidia-encode1=${available_version} libnvidia-ml1=${available_version}" if $STD apt-get -y -o Dpkg::Options::="--force-confold" install --no-install-recommends $nvidia_pkgs 2>/dev/null; then @@ -815,9 +843,16 @@ NVIDIA_PIN msg_warn "Same driver series, different release. CUDA workloads may still fail on this." fi else - msg_warn "Version-pinned install failed - keeping libcuda1 at ${available_version}, the rest unpinned" - $STD apt-get -y -o Dpkg::Options::="--force-confold" install --no-install-recommends \ - "libcuda1=${available_version}" libnvcuvid1 libnvidia-encode1 libnvidia-ml1 2>/dev/null || true + # No retry with the rest unpinned: libnvcuvid1 at the newest release + # depends on libcuda1 (= that release), so it cannot sit next to a + # libcuda1 held at the host's. Say what apt refused instead. + msg_error "NVIDIA userspace ${available_version} would not install from ${chosen_repo}" + local apt_reason="" reason_line + apt_reason="$(apt-get -s install --no-install-recommends $nvidia_pkgs 2>&1 | grep -E 'Depends:|^E: ' | sed 's/^ *//' | head -3 || true)" + while IFS= read -r reason_line; do + if [[ -n "$reason_line" ]]; then msg_warn "apt: ${reason_line}"; fi + done <<<"$apt_reason" + _nvidia_apt_pins >"$nvidia_pin_file" fi else # Deliberately install nothing. The previous code fell back to whatever diff --git a/tests/nvidia-version-pin.sh b/tests/nvidia-version-pin.sh new file mode 100644 index 0000000..8d96516 --- /dev/null +++ b/tests/nvidia-version-pin.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +set -euo pipefail + +CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +for fn in _cuda_pick_version _nvidia_version_pin _nvidia_apt_pins; do + eval "$(awk "/^${fn}[(][)] [{]/,/^}/" "$CORE_DIR/lib/hwaccel.func")" +done +passed=0 failed=0 + +check() { + if [[ "$3" == "$2" ]]; then + echo "PASS $1" + passed=$((passed + 1)) + else + echo "FAIL $1: expected <$2>, got <$3>" >&2 + failed=$((failed + 1)) + fi +} + +# madison lists newest first, as the NVIDIA debian13 repo did for issue #209. +apt-cache() { + local v + for v in 620.70.06-1 595.112.07-1 595.71.05-1 590.48.01-1; do + printf ' libcuda1 | %s | https://developer.download.nvidia.com/compute/cuda/repos/debian13/x86_64 Packages\n' "$v" + done +} + +check "the host's exact release wins" 595.112.07-1 "$(_cuda_pick_version 595.112.07)" +check "a mid-series host is not handed a newer release" 595.71.05-1 "$(_cuda_pick_version 595.71.05)" +check "a missing release falls back to the series" 595.112.07-1 "$(_cuda_pick_version 595.80.01)" +check "another series is never taken" "" "$(_cuda_pick_version 580.126.09)" + +pins="$(_nvidia_apt_pins 595.112.07-1)" +check "apt reads the version stanza first" "Pin: version 595.112.07-1" "$(grep -m1 '^Pin:' <<<"$pins")" +check "the stanzas are separated by a blank line" 1 "$(grep -c '^$' <<<"$pins")" + +priority() { awk -v pin="$2" '/^Pin:/ {hit = index($0, pin)} hit && /^Pin-Priority:/ {print $2; exit}' <<<"$1"; } +version_prio="$(priority "$pins" "Pin: version")" +origin_prio="$(priority "$pins" "Pin: origin")" +check "the version pin outranks the origin pin" yes "$([[ "$version_prio" -gt "$origin_prio" ]] && echo yes || echo no)" + +# Every package of the dependency chain has to fall under the version stanza. +globs="$(grep -m1 '^Package:' <<<"$pins" | cut -d' ' -f2-)" +covered() { + local glob + for glob in $globs; do + # shellcheck disable=SC2053 + [[ "$1" == $glob ]] && return 0 + done + return 1 +} +uncovered=() +set -f +for pkg in libcuda1 libnvcuvid1 libnvidia-encode1 libnvidia-ml1 libnvidia-ptxjitcompiler1 libnvidia-gpucomp libnvidia-pkcs11-openssl3; do + covered "$pkg" || uncovered+=("$pkg") +done +set +f +check "the version pin covers the whole dependency chain" "" "${uncovered[*]}" + +check "without a version only the origin pin is written" "Pin: origin developer.download.nvidia.com" "$(_nvidia_apt_pins | grep '^Pin:')" + +printf '%s passed, %s failed\n' "$passed" "$failed" +((failed == 0)) From eedf7e4d224356d0585fcc0ef0151674d8752d31 Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:41:28 +0100 Subject: [PATCH 2/4] Close the hardware acceleration step when no GPU is usable msg_warn leaves the step, its error message and the failed apt record in place, so a later failure would be reported under hardware acceleration with the NVIDIA error. The success line keeps its old text, since telemetry reports it as the step. --- lib/hwaccel.func | 35 ++++++++++++----------------------- tests/nvidia-version-pin.sh | 3 +-- 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/lib/hwaccel.func b/lib/hwaccel.func index 7c0fcc1..3507e4c 100644 --- a/lib/hwaccel.func +++ b/lib/hwaccel.func @@ -347,9 +347,13 @@ setup_hwaccel() { cache_installed_version "hwaccel" "1.0" if [[ "$HWACCEL_VENDOR" == "none" ]]; then + # Close the step like msg_ok, or a later failure is reported under it. + # shellcheck disable=SC2034 + CS_STEP_DONE="Setup Hardware Acceleration" CS_STEP="" CS_ERROR_MSG="" + _cs_drop_silent_record msg_warn "Hardware acceleration is not available - see the messages above" else - msg_ok "Setup Hardware Acceleration (${HWACCEL_VENDOR})" + msg_ok "Setup Hardware Acceleration" fi } @@ -668,16 +672,11 @@ ENVEOF # NVIDIA GPU Setup # ══════════════════════════════════════════════════════════════════════════════ -# Pick the host's exact version when the repo has it, and only then settle -# for another release in the same series. Taking the newest of the series -# unconditionally is the wrong direction: a host on 580.126 would be handed -# 580.178, which is newer than the loaded kernel module and therefore fails -# the same way a whole major mismatch does. +# The host's exact release first, else its series: a release newer than the +# loaded kernel module fails like a wrong major. _cuda_pick_version() { local host="$1" madison exact madison=$(apt-cache madison libcuda1 2>/dev/null | awk '{print $3}') || return 0 - # Dots are escaped: unescaped they are regex wildcards, so 580.178.04 - # would also match a hypothetical 580x178x04. exact=$(printf '%s\n' "$madison" | grep -E "^${host//./\\.}(-|$)" | head -1 || true) if [[ -n "$exact" ]]; then printf '%s\n' "$exact" @@ -686,11 +685,8 @@ _cuda_pick_version() { printf '%s\n' "$madison" | grep -E "^${host%%.*}\." | head -1 || true } -# libcuda1 needs libnvidia-ptxjitcompiler1, libnvidia-gpucomp and usually -# libnvidia-pkcs11-openssl3 at exactly its own version; without a version pin -# apt resolves them to the newest release and the install fails. Pinned rather -# than listed: the repo carries libcuda1 builds with and without the pkcs11 -# dependency. The priority has to stay above the origin pin's. +# libcuda1 needs its libnvidia-* dependencies at exactly its own version. +# The priority has to stay above the origin pin's. _nvidia_version_pin() { cat < Date: Fri, 6 Nov 2026 16:48:59 +0000 Subject: [PATCH 3/4] Update hardware acceleration API snapshot Co-authored-by: MickLesk <47820557+MickLesk@users.noreply.github.com> --- lib/API.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/API.txt b/lib/API.txt index 1f12aa5..9cd9070 100644 --- a/lib/API.txt +++ b/lib/API.txt @@ -7,6 +7,7 @@ _cs_drop_pm_shim _cs_repair_dpkg _cs_write_cargo_net _cs_write_cargo_profile +_cuda_pick_version _deploy_source_tarball _deploy_unpacked_archive _diagnose_deb_failure @@ -37,6 +38,8 @@ _install_uvx_wrapper _mongodb_in_place_precheck _mongodb_in_place_simulate _nodejs_in_place_simulate +_nvidia_apt_pins +_nvidia_version_pin _purge_is_safe _resolve_igc_tag _setup_amd_apu From fadb493077d882b67820bffae1eb37c83ba307fa Mon Sep 17 00:00:00 2001 From: MickLesk <47820557+MickLesk@users.noreply.github.com> Date: Fri, 6 Nov 2026 17:55:19 +0100 Subject: [PATCH 4/4] Follow a host driver rollback and keep the pin on failure Review findings on the NVIDIA version pin: - apt -y refuses the downgrade a host rollback needs even with the pin in place, so the install passes --allow-downgrades. The simulation that explains a failure runs with the same flags. - A failed run no longer rewrites the pin file without the version stanza, which left an earlier install free to move on the next apt upgrade. The origin-only file is written only when none exists. - AMD_APU reports amd_apu instead of amd: consumers read amd as ROCm, and an APU gets Mesa only. --- lib/hwaccel.func | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/hwaccel.func b/lib/hwaccel.func index 3507e4c..6a830ec 100644 --- a/lib/hwaccel.func +++ b/lib/hwaccel.func @@ -338,7 +338,9 @@ setup_hwaccel() { fi if [[ "$HWACCEL_VENDOR" == "none" ]]; then case "$_gpu_type" in - AMD*) HWACCEL_VENDOR="amd" ;; + AMD) HWACCEL_VENDOR="amd" ;; + # Not amd: consumers read that as ROCm, and an APU gets Mesa only. + AMD_APU) HWACCEL_VENDOR="amd_apu" ;; INTEL*) HWACCEL_VENDOR="intel" ;; esac fi @@ -672,8 +674,8 @@ ENVEOF # NVIDIA GPU Setup # ══════════════════════════════════════════════════════════════════════════════ -# The host's exact release first, else its series: a release newer than the -# loaded kernel module fails like a wrong major. +# The host's exact release first; the series fallback may end up newer than +# the kernel module, which the caller warns about. _cuda_pick_version() { local host="$1" madison exact madison=$(apt-cache madison libcuda1 2>/dev/null | awk '{print $3}') || return 0 @@ -787,7 +789,7 @@ _setup_nvidia_gpu() { fi local nvidia_pin_file="/etc/apt/preferences.d/nvidia-cuda-pin" - _nvidia_apt_pins >"$nvidia_pin_file" + [[ -f "$nvidia_pin_file" ]] || _nvidia_apt_pins >"$nvidia_pin_file" # Extract major version for flexible matching (580.126.09 -> 580) local nvidia_major_version="${nvidia_host_version%%.*}" @@ -822,7 +824,9 @@ _setup_nvidia_gpu() { _nvidia_apt_pins "$available_version" >"$nvidia_pin_file" msg_info "Installing NVIDIA userspace ${available_version} from ${chosen_repo}" local nvidia_pkgs="libcuda1=${available_version} libnvcuvid1=${available_version} libnvidia-encode1=${available_version} libnvidia-ml1=${available_version}" - if $STD apt-get -y -o Dpkg::Options::="--force-confold" install --no-install-recommends $nvidia_pkgs 2>/dev/null; then + # -y refuses the downgrade a host rollback needs, pin or not. + local -a apt_install=(apt-get -y --allow-downgrades -o Dpkg::Options::="--force-confold" install --no-install-recommends) + if $STD "${apt_install[@]}" $nvidia_pkgs 2>/dev/null; then # Say which kind of match this was. NVIDIA wants the full version to # agree; the fallback above only guarantees the series, so 580.178.04 # on the host and 580.126.20 here would both land here. State both and @@ -837,11 +841,10 @@ _setup_nvidia_gpu() { # A partly pinned retry cannot resolve either; report apt's reason. msg_error "NVIDIA userspace ${available_version} would not install from ${chosen_repo}" local apt_reason="" reason_line - apt_reason="$(apt-get -s install --no-install-recommends $nvidia_pkgs 2>&1 | grep -E 'Depends:|^E: ' | sed 's/^ *//' | head -3 || true)" + apt_reason="$("${apt_install[@]}" -s $nvidia_pkgs 2>&1 | grep -E 'Depends:|^E: ' | sed 's/^ *//' | head -3 || true)" while IFS= read -r reason_line; do if [[ -n "$reason_line" ]]; then msg_warn "apt: ${reason_line}"; fi done <<<"$apt_reason" - _nvidia_apt_pins >"$nvidia_pin_file" fi else # Deliberately install nothing. The previous code fell back to whatever