diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 0d86742de72..c6f35ba5b6b 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -95,6 +95,15 @@ installUbuntu2604MinimalBuildDeps() { aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" } +# Shared by VHD production and content validation; independent of the prebake feature flag. +isUbuntuGPUCacheOnlyImage() { + local version="$1" arch="$2" sku="$3" fips="$4" + case "${version}:${arch}:${sku}:${fips,,}" in + 22.04:amd64:22_04-lts-gen2:false|24.04:amd64:server:false) return 0 ;; + *) return 1 ;; + esac +} + installDeps() { wait_for_apt_locks retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT diff --git a/spec/vhdbuilder/packer/gpu_cache_only_spec.sh b/spec/vhdbuilder/packer/gpu_cache_only_spec.sh new file mode 100644 index 00000000000..a497e62cfee --- /dev/null +++ b/spec/vhdbuilder/packer/gpu_cache_only_spec.sh @@ -0,0 +1,337 @@ +#!/bin/bash +# shellcheck disable=SC2329 + +Describe 'shared Ubuntu GPU installer cache' + Include "./parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh" + BeforeAll "eval \"\$(sed -n '/^configureCachedGPUDriverPrerequisites()/,/^}/p; /^buildNVIDIAKernelModule()/,/^}/p' './vhdbuilder/packer/install-dependencies.sh')\"" + BeforeAll "eval \"\$(sed -n '/^testUbuntuGPUCacheOnlyImage()/,/^}/p' './vhdbuilder/packer/test/linux-vhd-content-test.sh')\"" + + setup_cache_only() { + TEST_ROOT=$(mktemp -d) + OS=UBUNTU + UBUNTU_OS_NAME=UBUNTU + OS_SKU=Ubuntu + OS_VERSION=24.04 + CPU_ARCH=amd64 + IMG_SKU=server + ENABLE_FIPS=False + FEATURE_FLAGS=None + COMPONENTS_FILEPATH="./parts/common/components.json" + TEST_KERNEL=6.8.0-test-azure + mkdir -p "$TEST_ROOT/lib/modules/$TEST_KERNEL/build" "$TEST_ROOT/boot" \ + "$TEST_ROOT/etc/modprobe.d" "$TEST_ROOT/var/lib/dkms" "$TEST_ROOT/usr/bin" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/build/Makefile" "$TEST_ROOT/boot/initrd.img-$TEST_KERNEL" + printf 'blacklist nouveau\noptions nouveau modeset=0\n' > "$TEST_ROOT/etc/modprobe.d/blacklist-nouveau.conf" + DKMS_STATUS="" + DKMS_RC=0 + INITRAMFS_RC=0 + INITRAMFS_CONTENTS="etc/modprobe.d/blacklist-nouveau.conf" + MISSING_PACKAGE="" + CACHE_RC=0 + CACHE_MISSING=false + } + cleanup_cache_only() { rm -rf "$TEST_ROOT"; } + BeforeEach 'setup_cache_only' + AfterEach 'cleanup_cache_only' + + getCPUArch() { echo "$CPU_ARCH"; } + isARM64() { if [ "$CPU_ARCH" = arm64 ]; then echo 1; else echo 0; fi; } + err() { echo "$1:Error: $2" >&2; } + uname() { echo "$TEST_KERNEL"; } + dkms() { printf '%s\n' "$DKMS_STATUS"; return "$DKMS_RC"; } + dpkg-query() { + if [ "${*: -1}" = "$MISSING_PACKAGE" ]; then return 1; fi + printf 'install ok installed' + } + lsinitramfs() { printf '%s\n' "$INITRAMFS_CONTENTS"; return "$INITRAMFS_RC"; } + ctr() { + if [ "$CACHE_MISSING" = false ]; then printf '%s\n' "${6#name==}"; fi + return "$CACHE_RC" + } + apt_get_install() { echo "apt_get_install $*"; return "${APT_RC:-0}"; } + update-initramfs() { echo "update-initramfs $*"; return "${UPDATE_INITRAMFS_RC:-0}"; } + retrycmd_if_failure() { echo "unexpected driver build $*" >&2; return 1; } + + Describe 'image policy' + Parameters + 22.04 amd64 22_04-lts-gen2 False 0 + 24.04 amd64 server false 0 + 24.04 arm64 server false 1 + 24.04 amd64 server TRUE 1 + 22.04 amd64 22_04-lts-gen2 true 1 + 24.04 amd64 cvm false 1 + 24.04 amd64 server-gen1 false 1 + 22.04 amd64 22_04-lts false 1 + 26.04 amd64 server false 1 + End + It "classifies version=$1 arch=$2 sku=$3 fips=$4" + When call isUbuntuGPUCacheOnlyImage "$1" "$2" "$3" "$4" + The status should equal "$5" + End + End + + Describe 'build prerequisites' + It 'installs the compiler prerequisites and refreshes every initramfs without building a driver' + When call configureCachedGPUDriverPrerequisites "$TEST_ROOT" + The status should be success + The output should include "apt_get_install 10 2 300 gcc make libc6-dev" + The output should include "update-initramfs -u -k all" + The contents of file "$TEST_ROOT/etc/modprobe.d/blacklist-nouveau.conf" should equal "blacklist nouveau +options nouveau modeset=0" + End + + It 'propagates package installation failure without rebuilding initramfs' + APT_RC=1 + When call configureCachedGPUDriverPrerequisites "$TEST_ROOT" + The status should be failure + The output should include "apt_get_install" + The output should not include "update-initramfs" + End + + It 'propagates initramfs update failure' + UPDATE_INITRAMFS_RC=1 + When call configureCachedGPUDriverPrerequisites "$TEST_ROOT" + The status should be failure + The output should include "update-initramfs -u -k all" + End + + It 'does not execute host build-only with the shared-image default' + When call buildNVIDIAKernelModule + The status should be success + The output should equal "" + The stderr should equal "" + End + End + + Describe 'final-image invariant' + It 'accepts the cache-only image with prebaking disabled' + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be success + The output should include "testUbuntuGPUCacheOnlyImage:Finish" + End + + It 'allows unrelated DKMS registrations and the in-tree framebuffer module' + DKMS_STATUS="other/1.0, $TEST_KERNEL, x86_64: installed" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/nvidiafb.ko" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be success + The output should include ":Finish" + End + + Describe 'stock backlight modules' + Parameters + nvidia-wmi-ec-backlight.ko + nvidia-wmi-ec-backlight.ko.gz + nvidia-wmi-ec-backlight.ko.xz + nvidia-wmi-ec-backlight.ko.zst + End + It "allows $1 in the stock kernel path on disk and in initramfs" + mkdir -p "$TEST_ROOT/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86/$1" + INITRAMFS_CONTENTS="$INITRAMFS_CONTENTS +usr/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be success + The output should include ":Finish" + End + End + + Describe 'effective registrations' + Parameters + "nvidia/580.1, 6.8.0, x86_64: installed" + "nvidia, 580.1, 6.8.0, x86_64: installed" + "nvidia/580.1: added" + "nvidia-current/580.1: added" + End + It "rejects '$1' even without a marker" + DKMS_STATUS="$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "NVIDIA DKMS registration" + End + End + + It 'fails closed when DKMS inspection fails' + DKMS_RC=1 + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Cannot inspect DKMS registration" + End + + It 'rejects dangling DKMS links even when dkms status is empty' + ln -s "$TEST_ROOT/nonexistent" "$TEST_ROOT/var/lib/dkms/nvidia" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected host NVIDIA artifact" + The path "$TEST_ROOT/var/lib/dkms/nvidia" should be symlink + End + + Describe 'host residue' + Parameters + usr/bin/nvidia-modprobe + usr/bin/lib64/libnvidia-ml.so + usr/lib/x86_64-linux-gnu/libcuda.so.1 + opt/azure/aks-gpu/dkms-marker + usr/src/nvidia-580.1/dkms.conf + sys/module/nvidia/refcnt + End + It "rejects $1 without deleting it" + mkdir -p "$(dirname "$TEST_ROOT/$1")" + touch "$TEST_ROOT/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected host NVIDIA artifact" + The path "$TEST_ROOT/$1" should be exist + End + End + + Describe 'installed modules' + Parameters + nvidia.ko + nvidia-modeset.ko.zst + nvidia_uvm.ko.xz + updates/dkms/nvidia-wmi-ec-backlight.ko + kernel/drivers/platform/x86/nvidia.ko.zst + End + It "rejects $1" + mkdir -p "$(dirname "$TEST_ROOT/lib/modules/$TEST_KERNEL/$1")" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected host NVIDIA modules" + End + End + + Describe 'initramfs containing stock backlight and driver residue' + Parameters + nvidia.ko + nvidia.ko.gz + nvidia_uvm.ko.xz + nvidia-peermem.ko.zst + nvidia-wmi-ec-backlight.ko.zst + End + It "still rejects updates/dkms/$1" + INITRAMFS_CONTENTS="$INITRAMFS_CONTENTS +usr/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86/nvidia-wmi-ec-backlight.ko.zst +usr/lib/modules/$TEST_KERNEL/updates/dkms/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected NVIDIA module" + End + End + + It 'rejects NVIDIA embedded in a different installed kernel initramfs' + touch "$TEST_ROOT/boot/initrd.img-another-kernel" + lsinitramfs() { + echo "etc/modprobe.d/blacklist-nouveau.conf" + case "$1" in *another-kernel) echo "usr/lib/modules/old/updates/dkms/nvidia.ko.zst" ;; esac + } + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected NVIDIA module" + End + + It 'rejects driver residue before a large initramfs listing with pipefail enabled' + INITRAMFS_CONTENTS="$INITRAMFS_CONTENTS +usr/lib/modules/$TEST_KERNEL/updates/dkms/nvidia.ko +$(awk 'BEGIN { for (i = 0; i < 4096; i++) print "usr/lib/modules/kernel/other-module-" i ".ko" }')" + check_with_pipefail() { + set -o pipefail + testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + } + When run check_with_pipefail + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected NVIDIA module" + End + + Describe 'missing or unverifiable prerequisites' + Parameters + "missing libc6-dev" "Missing GPU installation prerequisite" + "missing headers" "Missing usable headers" + "missing host blacklist" "Missing nouveau boot configuration" + "missing initramfs" "Missing initramfs" + "unreadable initramfs" "Cannot inspect initramfs" + "missing initramfs blacklist" "Missing nouveau boot configuration in" + "missing cache" "Missing cached GPU installer" + "failed cache inspection" "Missing cached GPU installer" + "invalid components" "Cannot resolve cached GPU installer reference" + End + It "rejects $1" + case "$1" in + "missing libc6-dev") MISSING_PACKAGE=libc6-dev ;; + "missing headers") rm "$TEST_ROOT/lib/modules/$TEST_KERNEL/build/Makefile" ;; + "missing host blacklist") printf '# no blacklist\n' > "$TEST_ROOT/etc/modprobe.d/blacklist-nouveau.conf" ;; + "missing initramfs") rm "$TEST_ROOT/boot/initrd.img-$TEST_KERNEL" ;; + "unreadable initramfs") INITRAMFS_RC=1 ;; + "missing initramfs blacklist") INITRAMFS_CONTENTS="etc/modprobe.d/other.conf" ;; + "missing cache") CACHE_MISSING=true ;; + "failed cache inspection") CACHE_RC=1 ;; + "invalid components") COMPONENTS_FILEPATH="$TEST_ROOT/components.json"; printf '{}\n' > "$COMPONENTS_FILEPATH" ;; + esac + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "$2" + End + End + + It 'cannot bypass the invariant by accidentally restoring the prebake flag' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + DKMS_STATUS="nvidia/580.1: added" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "NVIDIA DKMS registration" + End + End + + Describe 'unaffected image families' + Parameters + UBUNTU Ubuntu 24.04 arm64 server false NVIDIA_GB + UBUNTU Ubuntu 24.04 amd64 cvm false cvm + UBUNTU Ubuntu 22.04 amd64 22_04-lts-gen2 True None + AZURELINUX AzureLinux 3.0 amd64 core false None + End + It "does not configure or inspect OS=$1 arch=$4 sku=$5 fips=$6 flags=$7" + OS=$1 OS_SKU=$2 OS_VERSION=$3 CPU_ARCH=$4 IMG_SKU=$5 ENABLE_FIPS=$6 FEATURE_FLAGS=$7 + DKMS_RC=1 + unaffected() { + configureCachedGPUDriverPrerequisites "$TEST_ROOT" && + testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + } + When call unaffected + The status should be success + The output should equal "" + The stderr should equal "" + End + End + + Describe 'pipeline defaults' + Parameters + .pipelines/.vsts-vhd-builder.yaml build2204gen2containerd + .pipelines/.vsts-vhd-builder.yaml build2404gen2containerd + .pipelines/.vsts-vhd-builder-release.yaml build2204gen2containerd + .pipelines/.vsts-vhd-builder-release.yaml build2404gen2containerd + End + It "keeps $2 cache-only in $1" + build_variables() { + awk -v job="$2" ' + $1 == "-" && $2 == "job:" { if (found) exit; found = ($3 == job); next } + found { print } + ' "$1" + } + When call build_variables "$1" "$2" + The status should be success + The output should include 'variable=FEATURE_FLAGS]None' + The output should not include 'NVIDIA_CUDA_PREBAKE' + End + End +End diff --git a/vhdbuilder/packer/README.md b/vhdbuilder/packer/README.md index 1d23f04ec18..280045620b7 100644 --- a/vhdbuilder/packer/README.md +++ b/vhdbuilder/packer/README.md @@ -17,6 +17,39 @@ external-vhd-src->(func/init: ensure dest existing/static gallery/definition, in # Roadmap Goal1: remove mariner workflow so things will be simplified. +# Shared Ubuntu GPU installer cache + +The non-FIPS Ubuntu 22.04/24.04 amd64 Gen2 shared images cache the CUDA-LTS +installer container without installing its driver into the host. Their build and +release defaults use `FEATURE_FLAGS=None`, not `NVIDIA_CUDA_PREBAKE`. Managed GPU +nodes use the ordinary node-time installer; CPU and GPU-opt-out nodes must not +depend on CSE cleanup to remove an image's NVIDIA DKMS registration. + +The build retains gcc, make, libc6-dev and the nouveau blacklist in all initramfs +images independently of host prebaking. Content validation checks the booted +kernel's headers, installer cache, boot configuration and absence of NVIDIA +registration, modules and known host artifacts, including dangling registration +links and residue without a marker. The guard uses the shared-image identity, +not the disabled feature flag. It fails the build rather than cleaning up a +contaminated image. Stock-kernel framebuffer and WMI backlight modules are not +CUDA/GRID payload; the backlight exception applies only under +`kernel/drivers/platform/x86`, on disk and in initramfs. Copies under DKMS paths +and actual NVIDIA GPU modules remain failures. ARM64/Grace-Blackwell, FIPS, CVM +and other OS images are outside this policy. + +Release the missing-driver validation fast fallback before broad image rollout +where validation-only callers would otherwise spend 115 seconds retrying an +absent `nvidia-modprobe`. Older CSE retains its ordinary installation fallback, +but its provisioning budget still needs verification. Retain legacy cleanup and +PIS caching; do not move driver activation from `nodePrep` into `basePrep`. + +Before promotion, exercise changed Ubuntu images on CPU, GPU-opt-out, CUDA and +GRID nodes, distinct-VM PIS, and security-patched derivatives. Check kernel-update +hooks, GPU workloads, cache availability and provisioning latency. New source +does not repair existing registered bases, derived images, PIS caches or nodes; +refresh those through the supported image/node lifecycle. Re-enabling active +shared-image prebaking is not a safe rollback. + # Linux CSE configuration modules `parts/linux/cloud-init/artifacts/cse_config.sh` is installed as diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 001db64bc34..9989710fcea 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -839,19 +839,24 @@ EOF fi } +configureCachedGPUDriverPrerequisites() { + [ "$OS" = "$UBUNTU_OS_NAME" ] || return 0 + isUbuntuGPUCacheOnlyImage "$OS_VERSION" "$CPU_ARCH" "$IMG_SKU" "$ENABLE_FIPS" || return 0 + local root="${1:-}" + + # Keep the ordinary node-time installer independent of the optional host prebake. + apt_get_install 10 2 300 gcc make libc6-dev || return 1 + mkdir -p "${root}/etc/modprobe.d" || return 1 + printf 'blacklist nouveau\noptions nouveau modeset=0\n' > "${root}/etc/modprobe.d/blacklist-nouveau.conf" || return 1 + update-initramfs -u -k all || return 1 +} + buildNVIDIAKernelModule() { if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now gpu_action="copy" - # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the - # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles - # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, - # performs NO device access (safe on the GPU-less Packer builder), and writes the marker - # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" - # when that marker matches, running only the device-dependent steps. - # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the - # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. - # Dropping the image is a separate, deferred size optimization. + # Experimental host prebaking is not enabled on shared images. It registers NVIDIA with + # DKMS before node GPU policy is known. Keep the cached installer for normal node-time setup. if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install @@ -1193,6 +1198,7 @@ ctr namespace create k8s.io # Running them after the container-image cache and/or concurrently with the BCC build fills the disk # (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". cacheGPUContainerImageComponents +configureCachedGPUDriverPrerequisites "" buildNVIDIAKernelModule capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 66fb3f438a5..b27c17a870a 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -1303,6 +1303,95 @@ testVHDBuildLogsExist() { echo "$test:Finish" } +testUbuntuGPUCacheOnlyImage() { + [ "$OS_SKU" = "Ubuntu" ] || return 0 + isUbuntuGPUCacheOnlyImage "$OS_VERSION" "$(getCPUArch)" "$IMG_SKU" "$ENABLE_FIPS" || return 0 + local test="testUbuntuGPUCacheOnlyImage" root="${1:-}" + local status path modules initrd listing package kernel image cached + echo "$test:Start" + + if ! status=$(dkms status); then + err "$test" "Cannot inspect DKMS registration" + return 1 + fi + if grep -Eq '^[[:space:]]*nvidia([/,[:space:]-]|$)' <<< "$status"; then + err "$test" "Shared image contains NVIDIA DKMS registration: $status" + return 1 + fi + # Include dangling links and residue with a missing marker; do not silently sanitize the image. + for path in "${root}"/var/lib/dkms/nvidia* "${root}"/usr/src/nvidia* \ + "${root}"/usr/bin/nvidia* "${root}"/usr/sbin/nvidia* "${root}"/usr/bin/lib64 \ + "${root}"/usr/lib/x86_64-linux-gnu/libnvidia* "${root}"/usr/lib/x86_64-linux-gnu/libcuda.so* \ + "${root}"/etc/ld.so.conf.d/nvidia.conf "${root}"/opt/azure/aks-gpu/dkms-marker \ + "${root}"/sys/module/nvidia; do + if [ -e "$path" ] || [ -L "$path" ]; then + err "$test" "Unexpected host NVIDIA artifact: $path" + return 1 + fi + done + # The stock kernel's WMI backlight module is not CUDA/GRID driver payload. + if ! modules=$(find "${root}/lib/modules" \( -name 'nvidia.ko*' -o -name 'nvidia-*.ko*' -o -name 'nvidia_*.ko*' \) \ + ! -path '*/kernel/drivers/platform/x86/nvidia-wmi-ec-backlight.ko*' -print); then + err "$test" "Cannot inspect host kernel modules" + return 1 + fi + if [ -n "$modules" ]; then + err "$test" "Unexpected host NVIDIA modules: $modules" + return 1 + fi + + for package in gcc make libc6-dev dkms initramfs-tools; do + if ! status=$(dpkg-query -W -f='${Status}' "$package") || [ "$status" != "install ok installed" ]; then + err "$test" "Missing GPU installation prerequisite: $package" + return 1 + fi + done + kernel=$(uname -r) + if [ ! -f "${root}/lib/modules/${kernel}/build/Makefile" ]; then + err "$test" "Missing usable headers for booted kernel $kernel" + return 1 + fi + path="${root}/etc/modprobe.d/blacklist-nouveau.conf" + if ! grep -Eq '^[[:space:]]*blacklist[[:space:]]+nouveau([[:space:]]|$)' "$path" || + ! grep -Eq '^[[:space:]]*options[[:space:]]+nouveau[[:space:]]+modeset=0([[:space:]]|$)' "$path"; then + err "$test" "Missing nouveau boot configuration" + return 1 + fi + if [ ! -f "${root}/boot/initrd.img-${kernel}" ]; then + err "$test" "Missing initramfs for booted kernel $kernel" + return 1 + fi + for initrd in "${root}"/boot/initrd.img-*; do + if ! listing=$(lsinitramfs "$initrd"); then + err "$test" "Cannot inspect initramfs $initrd" + return 1 + fi + # Consume the filtered listing fully so pipefail cannot hide a match behind SIGPIPE. + if grep -Ev '(^|/)kernel/drivers/platform/x86/nvidia-wmi-ec-backlight\.ko(\.[^/]*)?$' <<< "$listing" | + grep -E '(^|/)nvidia([_-][^/]*)?\.ko(\.[^/]*)?$' >/dev/null; then + err "$test" "Unexpected NVIDIA module in $initrd" + return 1 + fi + if ! grep -Eq '(^|/)etc/modprobe.d/blacklist-nouveau.conf$' <<< "$listing"; then + err "$test" "Missing nouveau boot configuration in $initrd" + return 1 + fi + done + if ! image=$(jq -er ' + [.GPUContainerImages[] | select(.downloadURL == "mcr.microsoft.com/aks/aks-gpu-cuda-lts:*") | .gpuVersion.latestVersion] + | if length == 1 and (.[0] | type == "string" and length > 0) + then "mcr.microsoft.com/aks/aks-gpu-cuda-lts:" + .[0] + else error("expected one CUDA-LTS image version") end' "$COMPONENTS_FILEPATH"); then + err "$test" "Cannot resolve cached GPU installer reference" + return 1 + fi + if ! cached=$(ctr -n k8s.io images ls -q "name==$image") || [ "$cached" != "$image" ]; then + err "$test" "Missing cached GPU installer: $image" + return 1 + fi + echo "$test:Finish" +} + testAzureLinuxNvidiaGPUDriverReleaseNotes() { local test="testAzureLinuxNvidiaGPUDriverReleaseNotes" local enable_fips="${ENABLE_FIPS,,}" @@ -2802,6 +2891,7 @@ testVHDBuildLogsExist testAzureLinuxNvidiaGPUDriverReleaseNotes testCriticalTools testPackagesInstalled +testUbuntuGPUCacheOnlyImage "" testFuseInstalled if [ "$OS_SKU" = "Ubuntu" ]; then testBlobfuse2 "$(getPackageExpectedVersion "blobfuse2")"