From 25ecafeb98581e9f79185840a4347ce27fa4be21 Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan Date: Fri, 2 Oct 2026 15:02:48 -0700 Subject: [PATCH 1/3] fix(gpu): keep shared Ubuntu VHDs cache-only Disable shared-image host prebaking, retain installer prerequisites and fail final-image validation on NVIDIA registration or installed host residue. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61d3dd43-5cc2-49b3-9ce5-fbe943764363 --- .pipelines/.vsts-vhd-builder-release.yaml | 12 +- .pipelines/.vsts-vhd-builder.yaml | 12 +- .../artifacts/ubuntu/cse_install_ubuntu.sh | 9 + spec/vhdbuilder/packer/gpu_cache_only_spec.sh | 283 ++++++++++++++++++ vhdbuilder/packer/README.md | 30 ++ vhdbuilder/packer/install-dependencies.sh | 24 +- .../packer/test/linux-vhd-content-test.sh | 86 ++++++ 7 files changed, 431 insertions(+), 25 deletions(-) create mode 100644 spec/vhdbuilder/packer/gpu_cache_only_spec.sh diff --git a/.pipelines/.vsts-vhd-builder-release.yaml b/.pipelines/.vsts-vhd-builder-release.yaml index 62feb62c71e..db1aeacea5f 100644 --- a/.pipelines/.vsts-vhd-builder-release.yaml +++ b/.pipelines/.vsts-vhd-builder-release.yaml @@ -629,10 +629,8 @@ stages: echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - # NVIDIA_CUDA_PREBAKE bakes the CUDA driver kernel module into this shared x86 gen2 image so GPU - # nodes can later skip the ~80-150s in-CSE DKMS compile via the configGPUDrivers skip-build path - # (PR #8787); non-GPU and --gpu-driver None nodes tear the module down during provisioning. - echo '##vso[task.setvariable variable=FEATURE_FLAGS]NVIDIA_CUDA_PREBAKE' + # Cache the GPU installer without registering NVIDIA on this shared image. + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' @@ -707,10 +705,8 @@ stages: echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - # NVIDIA_CUDA_PREBAKE bakes the CUDA driver kernel module into this shared x86 gen2 image so GPU - # nodes can later skip the ~80-150s in-CSE DKMS compile via the configGPUDrivers skip-build path - # (PR #8787); non-GPU and --gpu-driver None nodes tear the module down during provisioning. - echo '##vso[task.setvariable variable=FEATURE_FLAGS]NVIDIA_CUDA_PREBAKE' + # Cache the GPU installer without registering NVIDIA on this shared image. + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' diff --git a/.pipelines/.vsts-vhd-builder.yaml b/.pipelines/.vsts-vhd-builder.yaml index 46a1bdd108d..7605ac8dc86 100644 --- a/.pipelines/.vsts-vhd-builder.yaml +++ b/.pipelines/.vsts-vhd-builder.yaml @@ -86,10 +86,8 @@ stages: echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - # NVIDIA_CUDA_PREBAKE bakes the CUDA driver kernel module into this shared x86 gen2 image so GPU - # nodes can later skip the ~80-150s in-CSE DKMS compile via the configGPUDrivers skip-build path - # (PR #8787); non-GPU and --gpu-driver None nodes tear the module down during provisioning. - echo '##vso[task.setvariable variable=FEATURE_FLAGS]NVIDIA_CUDA_PREBAKE' + # Cache the GPU installer without registering NVIDIA on this shared image. + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]false' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' @@ -133,10 +131,8 @@ stages: echo '##vso[task.setvariable variable=IMG_VERSION]latest' echo '##vso[task.setvariable variable=HYPERV_GENERATION]V2' echo '##vso[task.setvariable variable=AZURE_VM_SIZE]Standard_D16ds_v5' - # NVIDIA_CUDA_PREBAKE bakes the CUDA driver kernel module into this shared x86 gen2 image so GPU - # nodes can later skip the ~80-150s in-CSE DKMS compile via the configGPUDrivers skip-build path - # (PR #8787); non-GPU and --gpu-driver None nodes tear the module down during provisioning. - echo '##vso[task.setvariable variable=FEATURE_FLAGS]NVIDIA_CUDA_PREBAKE' + # Cache the GPU installer without registering NVIDIA on this shared image. + echo '##vso[task.setvariable variable=FEATURE_FLAGS]None' echo '##vso[task.setvariable variable=ARCHITECTURE]X86_64' echo '##vso[task.setvariable variable=ENABLE_FIPS]False' echo '##vso[task.setvariable variable=ENABLE_TRUSTED_LAUNCH]False' diff --git a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh index 0d86742de72..c6f35ba5b6b 100755 --- a/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh +++ b/parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh @@ -95,6 +95,15 @@ installUbuntu2604MinimalBuildDeps() { aptGetBatchInstallPackagesWithFallback "${pkg_list[@]}" } +# Shared by VHD production and content validation; independent of the prebake feature flag. +isUbuntuGPUCacheOnlyImage() { + local version="$1" arch="$2" sku="$3" fips="$4" + case "${version}:${arch}:${sku}:${fips,,}" in + 22.04:amd64:22_04-lts-gen2:false|24.04:amd64:server:false) return 0 ;; + *) return 1 ;; + esac +} + installDeps() { wait_for_apt_locks retrycmd_silent 120 5 25 curl -fsSL https://packages.microsoft.com/config/ubuntu/${UBUNTU_RELEASE}/packages-microsoft-prod.deb > /tmp/packages-microsoft-prod.deb || exit $ERR_MS_PROD_DEB_DOWNLOAD_TIMEOUT diff --git a/spec/vhdbuilder/packer/gpu_cache_only_spec.sh b/spec/vhdbuilder/packer/gpu_cache_only_spec.sh new file mode 100644 index 00000000000..dfe3e7ae868 --- /dev/null +++ b/spec/vhdbuilder/packer/gpu_cache_only_spec.sh @@ -0,0 +1,283 @@ +#!/bin/bash +# shellcheck disable=SC2329 + +Describe 'shared Ubuntu GPU installer cache' + Include "./parts/linux/cloud-init/artifacts/ubuntu/cse_install_ubuntu.sh" + BeforeAll "eval \"\$(sed -n '/^configureCachedGPUDriverPrerequisites()/,/^}/p; /^buildNVIDIAKernelModule()/,/^}/p' './vhdbuilder/packer/install-dependencies.sh')\"" + BeforeAll "eval \"\$(sed -n '/^testUbuntuGPUCacheOnlyImage()/,/^}/p' './vhdbuilder/packer/test/linux-vhd-content-test.sh')\"" + + setup_cache_only() { + TEST_ROOT=$(mktemp -d) + OS=UBUNTU + UBUNTU_OS_NAME=UBUNTU + OS_SKU=Ubuntu + OS_VERSION=24.04 + CPU_ARCH=amd64 + IMG_SKU=server + ENABLE_FIPS=False + FEATURE_FLAGS=None + COMPONENTS_FILEPATH="./parts/common/components.json" + TEST_KERNEL=6.8.0-test-azure + mkdir -p "$TEST_ROOT/lib/modules/$TEST_KERNEL/build" "$TEST_ROOT/boot" \ + "$TEST_ROOT/etc/modprobe.d" "$TEST_ROOT/var/lib/dkms" "$TEST_ROOT/usr/bin" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/build/Makefile" "$TEST_ROOT/boot/initrd.img-$TEST_KERNEL" + printf 'blacklist nouveau\noptions nouveau modeset=0\n' > "$TEST_ROOT/etc/modprobe.d/blacklist-nouveau.conf" + DKMS_STATUS="" + DKMS_RC=0 + INITRAMFS_RC=0 + INITRAMFS_CONTENTS="etc/modprobe.d/blacklist-nouveau.conf" + MISSING_PACKAGE="" + CACHE_RC=0 + CACHE_MISSING=false + } + cleanup_cache_only() { rm -rf "$TEST_ROOT"; } + BeforeEach 'setup_cache_only' + AfterEach 'cleanup_cache_only' + + getCPUArch() { echo "$CPU_ARCH"; } + isARM64() { if [ "$CPU_ARCH" = arm64 ]; then echo 1; else echo 0; fi; } + err() { echo "$1:Error: $2" >&2; } + uname() { echo "$TEST_KERNEL"; } + dkms() { printf '%s\n' "$DKMS_STATUS"; return "$DKMS_RC"; } + dpkg-query() { + if [ "${*: -1}" = "$MISSING_PACKAGE" ]; then return 1; fi + printf 'install ok installed' + } + lsinitramfs() { printf '%s\n' "$INITRAMFS_CONTENTS"; return "$INITRAMFS_RC"; } + ctr() { + if [ "$CACHE_MISSING" = false ]; then printf '%s\n' "${6#name==}"; fi + return "$CACHE_RC" + } + apt_get_install() { echo "apt_get_install $*"; return "${APT_RC:-0}"; } + update-initramfs() { echo "update-initramfs $*"; return "${UPDATE_INITRAMFS_RC:-0}"; } + retrycmd_if_failure() { echo "unexpected driver build $*" >&2; return 1; } + + Describe 'image policy' + Parameters + 22.04 amd64 22_04-lts-gen2 False 0 + 24.04 amd64 server false 0 + 24.04 arm64 server false 1 + 24.04 amd64 server TRUE 1 + 22.04 amd64 22_04-lts-gen2 true 1 + 24.04 amd64 cvm false 1 + 24.04 amd64 server-gen1 false 1 + 22.04 amd64 22_04-lts false 1 + 26.04 amd64 server false 1 + End + It "classifies version=$1 arch=$2 sku=$3 fips=$4" + When call isUbuntuGPUCacheOnlyImage "$1" "$2" "$3" "$4" + The status should equal "$5" + End + End + + Describe 'build prerequisites' + It 'installs the compiler prerequisites and refreshes every initramfs without building a driver' + When call configureCachedGPUDriverPrerequisites "$TEST_ROOT" + The status should be success + The output should include "apt_get_install 10 2 300 gcc make libc6-dev" + The output should include "update-initramfs -u -k all" + The contents of file "$TEST_ROOT/etc/modprobe.d/blacklist-nouveau.conf" should equal "blacklist nouveau +options nouveau modeset=0" + End + + It 'propagates package installation failure without rebuilding initramfs' + APT_RC=1 + When call configureCachedGPUDriverPrerequisites "$TEST_ROOT" + The status should be failure + The output should include "apt_get_install" + The output should not include "update-initramfs" + End + + It 'propagates initramfs update failure' + UPDATE_INITRAMFS_RC=1 + When call configureCachedGPUDriverPrerequisites "$TEST_ROOT" + The status should be failure + The output should include "update-initramfs -u -k all" + End + + It 'does not execute host build-only with the shared-image default' + When call buildNVIDIAKernelModule + The status should be success + The output should equal "" + The stderr should equal "" + End + End + + Describe 'final-image invariant' + It 'accepts the cache-only image with prebaking disabled' + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be success + The output should include "testUbuntuGPUCacheOnlyImage:Finish" + End + + It 'allows unrelated DKMS registrations and the in-tree framebuffer module' + DKMS_STATUS="other/1.0, $TEST_KERNEL, x86_64: installed" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/nvidiafb.ko" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be success + The output should include ":Finish" + End + + Describe 'effective registrations' + Parameters + "nvidia/580.1, 6.8.0, x86_64: installed" + "nvidia, 580.1, 6.8.0, x86_64: installed" + "nvidia/580.1: added" + "nvidia-current/580.1: added" + End + It "rejects '$1' even without a marker" + DKMS_STATUS="$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "NVIDIA DKMS registration" + End + End + + It 'fails closed when DKMS inspection fails' + DKMS_RC=1 + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Cannot inspect DKMS registration" + End + + It 'rejects dangling DKMS links even when dkms status is empty' + ln -s "$TEST_ROOT/nonexistent" "$TEST_ROOT/var/lib/dkms/nvidia" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected host NVIDIA artifact" + The path "$TEST_ROOT/var/lib/dkms/nvidia" should be symlink + End + + Describe 'host residue' + Parameters + usr/bin/nvidia-modprobe + usr/bin/lib64/libnvidia-ml.so + usr/lib/x86_64-linux-gnu/libcuda.so.1 + opt/azure/aks-gpu/dkms-marker + usr/src/nvidia-580.1/dkms.conf + sys/module/nvidia/refcnt + End + It "rejects $1 without deleting it" + mkdir -p "$(dirname "$TEST_ROOT/$1")" + touch "$TEST_ROOT/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected host NVIDIA artifact" + The path "$TEST_ROOT/$1" should be exist + End + End + + Describe 'installed modules' + Parameters + nvidia.ko + nvidia-modeset.ko.zst + nvidia_uvm.ko.xz + End + It "rejects $1" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected host NVIDIA modules" + End + End + + It 'rejects NVIDIA embedded in a different installed kernel initramfs' + touch "$TEST_ROOT/boot/initrd.img-another-kernel" + lsinitramfs() { + echo "etc/modprobe.d/blacklist-nouveau.conf" + case "$1" in *another-kernel) echo "usr/lib/modules/old/updates/dkms/nvidia.ko.zst" ;; esac + } + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected NVIDIA module" + End + + Describe 'missing or unverifiable prerequisites' + Parameters + "missing libc6-dev" "Missing GPU installation prerequisite" + "missing headers" "Missing usable headers" + "missing host blacklist" "Missing nouveau boot configuration" + "missing initramfs" "Missing initramfs" + "unreadable initramfs" "Cannot inspect initramfs" + "missing initramfs blacklist" "Missing nouveau boot configuration in" + "missing cache" "Missing cached GPU installer" + "failed cache inspection" "Missing cached GPU installer" + "invalid components" "Cannot resolve cached GPU installer reference" + End + It "rejects $1" + case "$1" in + "missing libc6-dev") MISSING_PACKAGE=libc6-dev ;; + "missing headers") rm "$TEST_ROOT/lib/modules/$TEST_KERNEL/build/Makefile" ;; + "missing host blacklist") printf '# no blacklist\n' > "$TEST_ROOT/etc/modprobe.d/blacklist-nouveau.conf" ;; + "missing initramfs") rm "$TEST_ROOT/boot/initrd.img-$TEST_KERNEL" ;; + "unreadable initramfs") INITRAMFS_RC=1 ;; + "missing initramfs blacklist") INITRAMFS_CONTENTS="etc/modprobe.d/other.conf" ;; + "missing cache") CACHE_MISSING=true ;; + "failed cache inspection") CACHE_RC=1 ;; + "invalid components") COMPONENTS_FILEPATH="$TEST_ROOT/components.json"; printf '{}\n' > "$COMPONENTS_FILEPATH" ;; + esac + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "$2" + End + End + + It 'cannot bypass the invariant by accidentally restoring the prebake flag' + FEATURE_FLAGS=NVIDIA_CUDA_PREBAKE + DKMS_STATUS="nvidia/580.1: added" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "NVIDIA DKMS registration" + End + End + + Describe 'unaffected image families' + Parameters + UBUNTU Ubuntu 24.04 arm64 server false NVIDIA_GB + UBUNTU Ubuntu 24.04 amd64 cvm false cvm + UBUNTU Ubuntu 22.04 amd64 22_04-lts-gen2 True None + AZURELINUX AzureLinux 3.0 amd64 core false None + End + It "does not configure or inspect OS=$1 arch=$4 sku=$5 fips=$6 flags=$7" + OS=$1 OS_SKU=$2 OS_VERSION=$3 CPU_ARCH=$4 IMG_SKU=$5 ENABLE_FIPS=$6 FEATURE_FLAGS=$7 + DKMS_RC=1 + unaffected() { + configureCachedGPUDriverPrerequisites "$TEST_ROOT" && + testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + } + When call unaffected + The status should be success + The output should equal "" + The stderr should equal "" + End + End + + Describe 'pipeline defaults' + Parameters + .pipelines/.vsts-vhd-builder.yaml build2204gen2containerd + .pipelines/.vsts-vhd-builder.yaml build2404gen2containerd + .pipelines/.vsts-vhd-builder-release.yaml build2204gen2containerd + .pipelines/.vsts-vhd-builder-release.yaml build2404gen2containerd + End + It "keeps $2 cache-only in $1" + build_variables() { + awk -v job="$2" ' + $1 == "-" && $2 == "job:" { if (found) exit; found = ($3 == job); next } + found { print } + ' "$1" + } + When call build_variables "$1" "$2" + The status should be success + The output should include 'variable=FEATURE_FLAGS]None' + The output should not include 'NVIDIA_CUDA_PREBAKE' + End + End +End diff --git a/vhdbuilder/packer/README.md b/vhdbuilder/packer/README.md index 1d23f04ec18..eaf9a3c1f1a 100644 --- a/vhdbuilder/packer/README.md +++ b/vhdbuilder/packer/README.md @@ -17,6 +17,36 @@ external-vhd-src->(func/init: ensure dest existing/static gallery/definition, in # Roadmap Goal1: remove mariner workflow so things will be simplified. +# Shared Ubuntu GPU installer cache + +The non-FIPS Ubuntu 22.04/24.04 amd64 Gen2 shared images cache the CUDA-LTS +installer container without installing its driver into the host. Their build and +release defaults use `FEATURE_FLAGS=None`, not `NVIDIA_CUDA_PREBAKE`. Managed GPU +nodes use the ordinary node-time installer; CPU and GPU-opt-out nodes must not +depend on CSE cleanup to remove an image's NVIDIA DKMS registration. + +The build retains gcc, make, libc6-dev and the nouveau blacklist in all initramfs +images independently of host prebaking. Content validation checks the booted +kernel's headers, installer cache, boot configuration and absence of NVIDIA +registration, modules and known host artifacts, including dangling registration +links and residue without a marker. The guard uses the shared-image identity, +not the disabled feature flag. It fails publication rather than cleaning up a +contaminated image. ARM64/Grace-Blackwell, FIPS, CVM and other OS images are outside +this policy. + +Release the missing-driver validation fast fallback before broad image rollout +where validation-only callers would otherwise spend 115 seconds retrying an +absent `nvidia-modprobe`. Older CSE retains its ordinary installation fallback, +but its provisioning budget still needs verification. Retain legacy cleanup and +PIS caching; do not move driver activation from `nodePrep` into `basePrep`. + +Before promotion, exercise changed Ubuntu images on CPU, GPU-opt-out, CUDA and +GRID nodes, distinct-VM PIS, and security-patched derivatives. Check kernel-update +hooks, GPU workloads, cache availability and provisioning latency. New source +does not repair existing registered bases, derived images, PIS caches or nodes; +refresh those through the supported image/node lifecycle. Re-enabling active +shared-image prebaking is not a safe rollback. + # Linux CSE configuration modules `parts/linux/cloud-init/artifacts/cse_config.sh` is installed as diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 001db64bc34..51838827ce4 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -839,19 +839,24 @@ EOF fi } +configureCachedGPUDriverPrerequisites() { + [ "$OS" = "$UBUNTU_OS_NAME" ] || return 0 + isUbuntuGPUCacheOnlyImage "$OS_VERSION" "$CPU_ARCH" "$IMG_SKU" "$ENABLE_FIPS" || return 0 + local root="${1:-}" + + # Keep the ordinary node-time installer independent of the optional host prebake. + apt_get_install 10 2 300 gcc make libc6-dev || return 1 + mkdir -p "${root}/etc/modprobe.d" || return 1 + printf 'blacklist nouveau\noptions nouveau modeset=0\n' > "${root}/etc/modprobe.d/blacklist-nouveau.conf" || return 1 + update-initramfs -u -k all || return 1 +} + buildNVIDIAKernelModule() { if [ $OS = $UBUNTU_OS_NAME ] && [ "$(isARM64)" -ne 1 ]; then # No ARM64 SKU with GPU now gpu_action="copy" - # Opt-in: pre-build the NVIDIA kernel module into the VHD so node provisioning skips the - # ~100s in-CSE DKMS compile. The aks-gpu container is run in "build-only" mode: it compiles - # and DKMS-registers the kernel module + stages userspace libs against THIS VHD's kernel, - # performs NO device access (safe on the GPU-less Packer builder), and writes the marker - # /opt/azure/aks-gpu/dkms-marker. At node boot, configGPUDrivers passes "install-skip-build" - # when that marker matches, running only the device-dependent steps. - # The driver image is intentionally LEFT in the VHD: boot-time device init still sources the - # container toolkit debs, fabric manager, containerd runtime config and udev rules from it. - # Dropping the image is a separate, deferred size optimization. + # Experimental host prebaking is not enabled on shared images. It registers NVIDIA with + # DKMS before node GPU policy is known. Keep the cached installer for normal node-time setup. if grep -q "NVIDIA_CUDA_PREBAKE" <<< "$FEATURE_FLAGS"; then echo "Pre-building NVIDIA CUDA kernel module into the VHD (build-only) for kernel $(uname -r)" # nvidia-installer needs gcc/make + libc6-dev to compile; the builder lacks them here, so install @@ -1193,6 +1198,7 @@ ctr namespace create k8s.io # Running them after the container-image cache and/or concurrently with the BCC build fills the disk # (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". cacheGPUContainerImageComponents +configureCachedGPUDriverPrerequisites buildNVIDIAKernelModule capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index 66fb3f438a5..fc22a956b39 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -1303,6 +1303,91 @@ testVHDBuildLogsExist() { echo "$test:Finish" } +testUbuntuGPUCacheOnlyImage() { + [ "$OS_SKU" = "Ubuntu" ] || return 0 + isUbuntuGPUCacheOnlyImage "$OS_VERSION" "$(getCPUArch)" "$IMG_SKU" "$ENABLE_FIPS" || return 0 + local test="testUbuntuGPUCacheOnlyImage" root="${1:-}" + local status path modules initrd listing package kernel image cached + echo "$test:Start" + + if ! status=$(dkms status); then + err "$test" "Cannot inspect DKMS registration" + return 1 + fi + if grep -Eq '^[[:space:]]*nvidia([/,[:space:]-]|$)' <<< "$status"; then + err "$test" "Shared image contains NVIDIA DKMS registration: $status" + return 1 + fi + # Include dangling links and residue with a missing marker; do not silently sanitize the image. + for path in "${root}"/var/lib/dkms/nvidia* "${root}"/usr/src/nvidia* \ + "${root}"/usr/bin/nvidia* "${root}"/usr/sbin/nvidia* "${root}"/usr/bin/lib64 \ + "${root}"/usr/lib/x86_64-linux-gnu/libnvidia* "${root}"/usr/lib/x86_64-linux-gnu/libcuda.so* \ + "${root}"/etc/ld.so.conf.d/nvidia.conf "${root}"/opt/azure/aks-gpu/dkms-marker \ + "${root}"/sys/module/nvidia; do + if [ -e "$path" ] || [ -L "$path" ]; then + err "$test" "Unexpected host NVIDIA artifact: $path" + return 1 + fi + done + if ! modules=$(find "${root}/lib/modules" \( -name 'nvidia.ko*' -o -name 'nvidia-*.ko*' -o -name 'nvidia_*.ko*' \) -print); then + err "$test" "Cannot inspect host kernel modules" + return 1 + fi + if [ -n "$modules" ]; then + err "$test" "Unexpected host NVIDIA modules: $modules" + return 1 + fi + + for package in gcc make libc6-dev dkms initramfs-tools; do + if ! status=$(dpkg-query -W -f='${Status}' "$package") || [ "$status" != "install ok installed" ]; then + err "$test" "Missing GPU installation prerequisite: $package" + return 1 + fi + done + kernel=$(uname -r) + if [ ! -f "${root}/lib/modules/${kernel}/build/Makefile" ]; then + err "$test" "Missing usable headers for booted kernel $kernel" + return 1 + fi + path="${root}/etc/modprobe.d/blacklist-nouveau.conf" + if ! grep -Eq '^[[:space:]]*blacklist[[:space:]]+nouveau([[:space:]]|$)' "$path" || + ! grep -Eq '^[[:space:]]*options[[:space:]]+nouveau[[:space:]]+modeset=0([[:space:]]|$)' "$path"; then + err "$test" "Missing nouveau boot configuration" + return 1 + fi + if [ ! -f "${root}/boot/initrd.img-${kernel}" ]; then + err "$test" "Missing initramfs for booted kernel $kernel" + return 1 + fi + for initrd in "${root}"/boot/initrd.img-*; do + if ! listing=$(lsinitramfs "$initrd"); then + err "$test" "Cannot inspect initramfs $initrd" + return 1 + fi + if grep -Eq '(^|/)nvidia([_-][^/]*)?\.ko(\.[^/]*)?$' <<< "$listing"; then + err "$test" "Unexpected NVIDIA module in $initrd" + return 1 + fi + if ! grep -Eq '(^|/)etc/modprobe.d/blacklist-nouveau.conf$' <<< "$listing"; then + err "$test" "Missing nouveau boot configuration in $initrd" + return 1 + fi + done + if ! image=$(jq -er ' + [.GPUContainerImages[] | select(.downloadURL == "mcr.microsoft.com/aks/aks-gpu-cuda-lts:*") | .gpuVersion.latestVersion] + | if length == 1 and (.[0] | type == "string" and length > 0) + then "mcr.microsoft.com/aks/aks-gpu-cuda-lts:" + .[0] + else error("expected one CUDA-LTS image version") end' "$COMPONENTS_FILEPATH"); then + err "$test" "Cannot resolve cached GPU installer reference" + return 1 + fi + if ! cached=$(ctr -n k8s.io images ls -q "name==$image") || [ "$cached" != "$image" ]; then + err "$test" "Missing cached GPU installer: $image" + return 1 + fi + echo "$test:Finish" +} + testAzureLinuxNvidiaGPUDriverReleaseNotes() { local test="testAzureLinuxNvidiaGPUDriverReleaseNotes" local enable_fips="${ENABLE_FIPS,,}" @@ -2802,6 +2887,7 @@ testVHDBuildLogsExist testAzureLinuxNvidiaGPUDriverReleaseNotes testCriticalTools testPackagesInstalled +testUbuntuGPUCacheOnlyImage testFuseInstalled if [ "$OS_SKU" = "Ubuntu" ]; then testBlobfuse2 "$(getPackageExpectedVersion "blobfuse2")" From cc9b90f20f93b4cd270ed85db0753d0fe61c2057 Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan Date: Fri, 2 Oct 2026 16:01:05 -0700 Subject: [PATCH 2/3] fix: distinguish stock backlight modules from GPU driver residue Exclude only the kernel platform backlight path from cache-only disk and initramfs checks. Keep rejecting DKMS-path copies and test compressed modules plus pipefail behavior. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61d3dd43-5cc2-49b3-9ce5-fbe943764363 --- spec/vhdbuilder/packer/gpu_cache_only_spec.sh | 54 +++++++++++++++++++ vhdbuilder/packer/README.md | 9 ++-- .../packer/test/linux-vhd-content-test.sh | 8 ++- 3 files changed, 66 insertions(+), 5 deletions(-) diff --git a/spec/vhdbuilder/packer/gpu_cache_only_spec.sh b/spec/vhdbuilder/packer/gpu_cache_only_spec.sh index dfe3e7ae868..a497e62cfee 100644 --- a/spec/vhdbuilder/packer/gpu_cache_only_spec.sh +++ b/spec/vhdbuilder/packer/gpu_cache_only_spec.sh @@ -118,6 +118,24 @@ options nouveau modeset=0" The output should include ":Finish" End + Describe 'stock backlight modules' + Parameters + nvidia-wmi-ec-backlight.ko + nvidia-wmi-ec-backlight.ko.gz + nvidia-wmi-ec-backlight.ko.xz + nvidia-wmi-ec-backlight.ko.zst + End + It "allows $1 in the stock kernel path on disk and in initramfs" + mkdir -p "$TEST_ROOT/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86" + touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86/$1" + INITRAMFS_CONTENTS="$INITRAMFS_CONTENTS +usr/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be success + The output should include ":Finish" + End + End + Describe 'effective registrations' Parameters "nvidia/580.1, 6.8.0, x86_64: installed" @@ -176,8 +194,11 @@ options nouveau modeset=0" nvidia.ko nvidia-modeset.ko.zst nvidia_uvm.ko.xz + updates/dkms/nvidia-wmi-ec-backlight.ko + kernel/drivers/platform/x86/nvidia.ko.zst End It "rejects $1" + mkdir -p "$(dirname "$TEST_ROOT/lib/modules/$TEST_KERNEL/$1")" touch "$TEST_ROOT/lib/modules/$TEST_KERNEL/$1" When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" The status should be failure @@ -186,6 +207,25 @@ options nouveau modeset=0" End End + Describe 'initramfs containing stock backlight and driver residue' + Parameters + nvidia.ko + nvidia.ko.gz + nvidia_uvm.ko.xz + nvidia-peermem.ko.zst + nvidia-wmi-ec-backlight.ko.zst + End + It "still rejects updates/dkms/$1" + INITRAMFS_CONTENTS="$INITRAMFS_CONTENTS +usr/lib/modules/$TEST_KERNEL/kernel/drivers/platform/x86/nvidia-wmi-ec-backlight.ko.zst +usr/lib/modules/$TEST_KERNEL/updates/dkms/$1" + When call testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected NVIDIA module" + End + End + It 'rejects NVIDIA embedded in a different installed kernel initramfs' touch "$TEST_ROOT/boot/initrd.img-another-kernel" lsinitramfs() { @@ -198,6 +238,20 @@ options nouveau modeset=0" The stderr should include "Unexpected NVIDIA module" End + It 'rejects driver residue before a large initramfs listing with pipefail enabled' + INITRAMFS_CONTENTS="$INITRAMFS_CONTENTS +usr/lib/modules/$TEST_KERNEL/updates/dkms/nvidia.ko +$(awk 'BEGIN { for (i = 0; i < 4096; i++) print "usr/lib/modules/kernel/other-module-" i ".ko" }')" + check_with_pipefail() { + set -o pipefail + testUbuntuGPUCacheOnlyImage "$TEST_ROOT" + } + When run check_with_pipefail + The status should be failure + The output should not include ":Finish" + The stderr should include "Unexpected NVIDIA module" + End + Describe 'missing or unverifiable prerequisites' Parameters "missing libc6-dev" "Missing GPU installation prerequisite" diff --git a/vhdbuilder/packer/README.md b/vhdbuilder/packer/README.md index eaf9a3c1f1a..280045620b7 100644 --- a/vhdbuilder/packer/README.md +++ b/vhdbuilder/packer/README.md @@ -30,9 +30,12 @@ images independently of host prebaking. Content validation checks the booted kernel's headers, installer cache, boot configuration and absence of NVIDIA registration, modules and known host artifacts, including dangling registration links and residue without a marker. The guard uses the shared-image identity, -not the disabled feature flag. It fails publication rather than cleaning up a -contaminated image. ARM64/Grace-Blackwell, FIPS, CVM and other OS images are outside -this policy. +not the disabled feature flag. It fails the build rather than cleaning up a +contaminated image. Stock-kernel framebuffer and WMI backlight modules are not +CUDA/GRID payload; the backlight exception applies only under +`kernel/drivers/platform/x86`, on disk and in initramfs. Copies under DKMS paths +and actual NVIDIA GPU modules remain failures. ARM64/Grace-Blackwell, FIPS, CVM +and other OS images are outside this policy. Release the missing-driver validation fast fallback before broad image rollout where validation-only callers would otherwise spend 115 seconds retrying an diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index fc22a956b39..dd00b617c18 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -1329,7 +1329,9 @@ testUbuntuGPUCacheOnlyImage() { return 1 fi done - if ! modules=$(find "${root}/lib/modules" \( -name 'nvidia.ko*' -o -name 'nvidia-*.ko*' -o -name 'nvidia_*.ko*' \) -print); then + # The stock kernel's WMI backlight module is not CUDA/GRID driver payload. + if ! modules=$(find "${root}/lib/modules" \( -name 'nvidia.ko*' -o -name 'nvidia-*.ko*' -o -name 'nvidia_*.ko*' \) \ + ! -path '*/kernel/drivers/platform/x86/nvidia-wmi-ec-backlight.ko*' -print); then err "$test" "Cannot inspect host kernel modules" return 1 fi @@ -1364,7 +1366,9 @@ testUbuntuGPUCacheOnlyImage() { err "$test" "Cannot inspect initramfs $initrd" return 1 fi - if grep -Eq '(^|/)nvidia([_-][^/]*)?\.ko(\.[^/]*)?$' <<< "$listing"; then + # Consume the filtered listing fully so pipefail cannot hide a match behind SIGPIPE. + if grep -Ev '(^|/)kernel/drivers/platform/x86/nvidia-wmi-ec-backlight\.ko(\.[^/]*)?$' <<< "$listing" | + grep -E '(^|/)nvidia([_-][^/]*)?\.ko(\.[^/]*)?$' >/dev/null; then err "$test" "Unexpected NVIDIA module in $initrd" return 1 fi From 387b78080999c5ec014f1ccdd4b461d990fd527d Mon Sep 17 00:00:00 2001 From: Ganeshkumar Ashokavardhanan Date: Fri, 2 Oct 2026 17:08:23 -0700 Subject: [PATCH 3/3] fix: make GPU helper production roots explicit Pass an explicit empty filesystem prefix at the two production call sites. Preserve the existing default-root behavior while satisfying CI ShellCheck SC2119/SC2120 for helpers with optional fixture roots. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 61d3dd43-5cc2-49b3-9ce5-fbe943764363 --- vhdbuilder/packer/install-dependencies.sh | 2 +- vhdbuilder/packer/test/linux-vhd-content-test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/vhdbuilder/packer/install-dependencies.sh b/vhdbuilder/packer/install-dependencies.sh index 51838827ce4..9989710fcea 100644 --- a/vhdbuilder/packer/install-dependencies.sh +++ b/vhdbuilder/packer/install-dependencies.sh @@ -1198,7 +1198,7 @@ ctr namespace create k8s.io # Running them after the container-image cache and/or concurrently with the BCC build fills the disk # (worse on 24.04), failing at the nvidia.ko link or the driver lib copy with "No space left on device". cacheGPUContainerImageComponents -configureCachedGPUDriverPrerequisites +configureCachedGPUDriverPrerequisites "" buildNVIDIAKernelModule capture_benchmark "${SCRIPT_NAME}_caching_gpu_container_images_and_build_nvidia_kernel_module" diff --git a/vhdbuilder/packer/test/linux-vhd-content-test.sh b/vhdbuilder/packer/test/linux-vhd-content-test.sh index dd00b617c18..b27c17a870a 100644 --- a/vhdbuilder/packer/test/linux-vhd-content-test.sh +++ b/vhdbuilder/packer/test/linux-vhd-content-test.sh @@ -2891,7 +2891,7 @@ testVHDBuildLogsExist testAzureLinuxNvidiaGPUDriverReleaseNotes testCriticalTools testPackagesInstalled -testUbuntuGPUCacheOnlyImage +testUbuntuGPUCacheOnlyImage "" testFuseInstalled if [ "$OS_SKU" = "Ubuntu" ]; then testBlobfuse2 "$(getPackageExpectedVersion "blobfuse2")"