diff --git a/.github/workflows/verify-split-files.yml b/.github/workflows/verify-split-files.yml index 955961e..64d9ff3 100644 --- a/.github/workflows/verify-split-files.yml +++ b/.github/workflows/verify-split-files.yml @@ -30,6 +30,7 @@ on: - "tests/uv-required-version.sh" - "tests/docker-lxcfs.sh" - "tests/java-memory.sh" + - "tests/nvidia-version-pin.sh" - "tests/external-installer.sh" - "tests/forge-version-key.sh" - "tests/nodejs-key-refresh.sh" @@ -85,6 +86,7 @@ on: - "tests/uv-required-version.sh" - "tests/docker-lxcfs.sh" - "tests/java-memory.sh" + - "tests/nvidia-version-pin.sh" - "tests/external-installer.sh" - "tests/forge-version-key.sh" - "tests/nodejs-key-refresh.sh" @@ -164,6 +166,9 @@ jobs: - name: Java memory drop-in tests run: bash tests/java-memory.sh + - name: NVIDIA userspace version pin tests + run: bash tests/nvidia-version-pin.sh + - name: External installer confirmation tests run: bash tests/external-installer.sh diff --git a/lib/API.txt b/lib/API.txt index 1f12aa5..9cd9070 100644 --- a/lib/API.txt +++ b/lib/API.txt @@ -7,6 +7,7 @@ _cs_drop_pm_shim _cs_repair_dpkg _cs_write_cargo_net _cs_write_cargo_profile +_cuda_pick_version _deploy_source_tarball _deploy_unpacked_archive _diagnose_deb_failure @@ -37,6 +38,8 @@ _install_uvx_wrapper _mongodb_in_place_precheck _mongodb_in_place_simulate _nodejs_in_place_simulate +_nvidia_apt_pins +_nvidia_version_pin _purge_is_safe _resolve_igc_tag _setup_amd_apu diff --git a/lib/hwaccel.func b/lib/hwaccel.func index 3793a8b..6a830ec 100644 --- a/lib/hwaccel.func +++ b/lib/hwaccel.func @@ -339,6 +339,8 @@ setup_hwaccel() { if [[ "$HWACCEL_VENDOR" == "none" ]]; then case "$_gpu_type" in AMD) HWACCEL_VENDOR="amd" ;; + # Not amd: consumers read that as ROCm, and an APU gets Mesa only. + AMD_APU) HWACCEL_VENDOR="amd_apu" ;; INTEL*) HWACCEL_VENDOR="intel" ;; esac fi @@ -346,7 +348,15 @@ setup_hwaccel() { export HWACCEL_VENDOR cache_installed_version "hwaccel" "1.0" - msg_ok "Setup Hardware Acceleration" + if [[ "$HWACCEL_VENDOR" == "none" ]]; then + # Close the step like msg_ok, or a later failure is reported under it. + # shellcheck disable=SC2034 + CS_STEP_DONE="Setup Hardware Acceleration" CS_STEP="" CS_ERROR_MSG="" + _cs_drop_silent_record + msg_warn "Hardware acceleration is not available - see the messages above" + else + msg_ok "Setup Hardware Acceleration" + fi } # ══════════════════════════════════════════════════════════════════════════════ @@ -663,6 +673,44 @@ ENVEOF # ══════════════════════════════════════════════════════════════════════════════ # NVIDIA GPU Setup # ══════════════════════════════════════════════════════════════════════════════ + +# The host's exact release first; the series fallback may end up newer than +# the kernel module, which the caller warns about. +_cuda_pick_version() { + local host="$1" madison exact + madison=$(apt-cache madison libcuda1 2>/dev/null | awk '{print $3}') || return 0 + exact=$(printf '%s\n' "$madison" | grep -E "^${host//./\\.}(-|$)" | head -1 || true) + if [[ -n "$exact" ]]; then + printf '%s\n' "$exact" + return 0 + fi + printf '%s\n' "$madison" | grep -E "^${host%%.*}\." | head -1 || true +} + +# libcuda1 needs its libnvidia-* dependencies at exactly its own version. +# The priority has to stay above the origin pin's. +_nvidia_version_pin() { + cat </etc/apt/preferences.d/nvidia-cuda-pin -Package: libcuda* libnvcuvid* libnvidia-* nvidia-smi nvidia-utils-* cuda-* -Pin: origin developer.download.nvidia.com -Pin-Priority: 1001 -NVIDIA_PIN + local nvidia_pin_file="/etc/apt/preferences.d/nvidia-cuda-pin" + [[ -f "$nvidia_pin_file" ]] || _nvidia_apt_pins >"$nvidia_pin_file" # Extract major version for flexible matching (580.126.09 -> 580) local nvidia_major_version="${nvidia_host_version%%.*}" local cuda_list="/etc/apt/sources.list.d/cuda-community-scripts.list" local chosen_repo="" available_version="" cuda_repo="" - # Pick the host's exact version when the repo has it, and only then settle - # for another release in the same series. Taking the newest of the series - # unconditionally is the wrong direction: a host on 580.126 would be handed - # 580.178, which is newer than the loaded kernel module and therefore fails - # the same way a whole major mismatch does. - _cuda_pick_version() { - local madison exact - madison=$(apt-cache madison libcuda1 2>/dev/null | awk '{print $3}') || return 0 - # Dots are escaped: unescaped they are regex wildcards, so 580.178.04 - # would also match a hypothetical 580x178x04. - exact=$(printf '%s\n' "$madison" | grep -E "^${nvidia_host_version//./\\.}(-|$)" | head -1 || true) - if [[ -n "$exact" ]]; then - printf '%s\n' "$exact" - return 0 - fi - printf '%s\n' "$madison" | grep -E "^${nvidia_major_version}\." | head -1 || true - } - for cuda_repo in "${cuda_repos[@]}"; do # The keyring package drops its own list file for whichever repo it came # from. Left in place it would keep a rejected candidate active alongside @@ -783,7 +804,7 @@ NVIDIA_PIN echo "deb [signed-by=/usr/share/keyrings/cuda-archive-keyring.gpg] https://developer.download.nvidia.com/compute/cuda/repos/${cuda_repo}/x86_64/ /" >"$cuda_list" $STD apt-get -y update 2>/dev/null || true - available_version="$(_cuda_pick_version)" + available_version="$(_cuda_pick_version "$nvidia_host_version")" if [[ -n "$available_version" ]]; then chosen_repo="$cuda_repo" break @@ -795,15 +816,17 @@ NVIDIA_PIN # Last resort: the distribution's own non-free packages. rm -f "$cuda_list" $STD apt-get -y update 2>/dev/null || true - available_version="$(_cuda_pick_version)" + available_version="$(_cuda_pick_version "$nvidia_host_version")" [[ -n "$available_version" ]] && chosen_repo="Debian non-free" fi - unset -f _cuda_pick_version if [[ -n "$chosen_repo" ]]; then + _nvidia_apt_pins "$available_version" >"$nvidia_pin_file" msg_info "Installing NVIDIA userspace ${available_version} from ${chosen_repo}" local nvidia_pkgs="libcuda1=${available_version} libnvcuvid1=${available_version} libnvidia-encode1=${available_version} libnvidia-ml1=${available_version}" - if $STD apt-get -y -o Dpkg::Options::="--force-confold" install --no-install-recommends $nvidia_pkgs 2>/dev/null; then + # -y refuses the downgrade a host rollback needs, pin or not. + local -a apt_install=(apt-get -y --allow-downgrades -o Dpkg::Options::="--force-confold" install --no-install-recommends) + if $STD "${apt_install[@]}" $nvidia_pkgs 2>/dev/null; then # Say which kind of match this was. NVIDIA wants the full version to # agree; the fallback above only guarantees the series, so 580.178.04 # on the host and 580.126.20 here would both land here. State both and @@ -815,9 +838,13 @@ NVIDIA_PIN msg_warn "Same driver series, different release. CUDA workloads may still fail on this." fi else - msg_warn "Version-pinned install failed - keeping libcuda1 at ${available_version}, the rest unpinned" - $STD apt-get -y -o Dpkg::Options::="--force-confold" install --no-install-recommends \ - "libcuda1=${available_version}" libnvcuvid1 libnvidia-encode1 libnvidia-ml1 2>/dev/null || true + # A partly pinned retry cannot resolve either; report apt's reason. + msg_error "NVIDIA userspace ${available_version} would not install from ${chosen_repo}" + local apt_reason="" reason_line + apt_reason="$("${apt_install[@]}" -s $nvidia_pkgs 2>&1 | grep -E 'Depends:|^E: ' | sed 's/^ *//' | head -3 || true)" + while IFS= read -r reason_line; do + if [[ -n "$reason_line" ]]; then msg_warn "apt: ${reason_line}"; fi + done <<<"$apt_reason" fi else # Deliberately install nothing. The previous code fell back to whatever diff --git a/tests/nvidia-version-pin.sh b/tests/nvidia-version-pin.sh new file mode 100644 index 0000000..3980043 --- /dev/null +++ b/tests/nvidia-version-pin.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail + +CORE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +for fn in _cuda_pick_version _nvidia_version_pin _nvidia_apt_pins; do + eval "$(awk "/^${fn}[(][)] [{]/,/^}/" "$CORE_DIR/lib/hwaccel.func")" +done +passed=0 failed=0 + +check() { + if [[ "$3" == "$2" ]]; then + echo "PASS $1" + passed=$((passed + 1)) + else + echo "FAIL $1: expected <$2>, got <$3>" >&2 + failed=$((failed + 1)) + fi +} + +# Newest first, like madison. +apt-cache() { + local v + for v in 620.70.06-1 595.112.07-1 595.71.05-1 590.48.01-1; do + printf ' libcuda1 | %s | https://developer.download.nvidia.com/compute/cuda/repos/debian13/x86_64 Packages\n' "$v" + done +} + +check "the host's exact release wins" 595.112.07-1 "$(_cuda_pick_version 595.112.07)" +check "a mid-series host is not handed a newer release" 595.71.05-1 "$(_cuda_pick_version 595.71.05)" +check "a missing release falls back to the series" 595.112.07-1 "$(_cuda_pick_version 595.80.01)" +check "another series is never taken" "" "$(_cuda_pick_version 580.126.09)" + +pins="$(_nvidia_apt_pins 595.112.07-1)" +check "apt reads the version stanza first" "Pin: version 595.112.07-1" "$(grep -m1 '^Pin:' <<<"$pins")" +check "the stanzas are separated by a blank line" 1 "$(grep -c '^$' <<<"$pins")" + +priority() { awk -v pin="$2" '/^Pin:/ {hit = index($0, pin)} hit && /^Pin-Priority:/ {print $2; exit}' <<<"$1"; } +version_prio="$(priority "$pins" "Pin: version")" +origin_prio="$(priority "$pins" "Pin: origin")" +check "the version pin outranks the origin pin" yes "$([[ "$version_prio" -gt "$origin_prio" ]] && echo yes || echo no)" + +globs="$(grep -m1 '^Package:' <<<"$pins" | cut -d' ' -f2-)" +covered() { + local glob + for glob in $globs; do + # shellcheck disable=SC2053 + [[ "$1" == $glob ]] && return 0 + done + return 1 +} +uncovered=() +set -f +for pkg in libcuda1 libnvcuvid1 libnvidia-encode1 libnvidia-ml1 libnvidia-ptxjitcompiler1 libnvidia-gpucomp libnvidia-pkcs11-openssl3; do + covered "$pkg" || uncovered+=("$pkg") +done +set +f +check "the version pin covers the whole dependency chain" "" "${uncovered[*]}" + +check "without a version only the origin pin is written" "Pin: origin developer.download.nvidia.com" "$(_nvidia_apt_pins | grep '^Pin:')" + +printf '%s passed, %s failed\n' "$passed" "$failed" +((failed == 0))