fix: use GRID v19 LTS for A10

.github/renovate.json

@@ -708,6 +708,7 @@
       ],
       "groupName": "nvidia-gpu-grid",
       "versioning": "regex:^(?<major>\\d+)\\.(?<minor>\\d+)(?:\\.(?<patch>\\d+))?-(?<prerelease>\\d{14})$",
+      "allowedVersions": "/^580\\./",
       "automerge": false,
       "enabled": true,
       "ignoreUnstable": false,
@@ -732,6 +733,7 @@
       ],
       "groupName": "nvidia-gpu-grid-v20",
       "versioning": "regex:^(?<major>\\d+)\\.(?<minor>\\d+)\\.(?<patch>\\d+)-(?<prerelease>\\d{14})$",
+      "allowedVersions": "/^595\\./",
       "automerge": false,
       "enabled": true,
       "ignoreUnstable": false,

.github/workflows/go-test.yml

@@ -57,6 +57,10 @@ jobs:
           (cd aks-live-patching && gocover-cobertura < ../coverage/aks-live-patching.out > ../coverage/aks-live-patching.xml)
         fi
         rm -f coverage/*.out
+    - name: Test GRID driver version validation
+      run: |
+        cd e2e
+        go test -short ./scenario -run '^TestNvidiaDriverVersionValidationScript'
     - name: Upload coverage reports
       uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
       if: (!cancelled())

README.md

@@ -21,6 +21,24 @@ Pull request titles must follow [Conventional Commits](https://www.conventionalc
 
 ## Tests
 
+### Ubuntu GRID driver branches
+
+The Ubuntu A10 GRID path uses `aks-gpu-grid` on the patched R580/vGPU 19.6
+LTS branch. NCv6 RTX PRO 6000 BSE uses `aks-gpu-grid-v20` on the patched
+R595/vGPU 20.2 branch. Driver versions and image tags are pinned in
+`parts/common/components.json`; Renovate is constrained to the corresponding
+branch for each image.
+
+[Azure's Linux GRID support matrix](https://learn.microsoft.com/azure/virtual-machines/linux/n-series-driver-setup#supported-grid-drivers)
+lists v19.6 and v20.2 for NVadsA10_v5, but only v20.2 for NCv6 and NCasT4_v3.
+This does not change existing CUDA selection for compute SKUs, or the separate
+Azure Linux/ACL sysext installation paths. The existing NCads_A10_v4 aliases
+retain their A10 GRID image selection.
+
+The Ubuntu GRID E2E scenarios require the exact configured driver version on
+every GPU, in addition to checking GRID licensing and service health. These
+checks do not establish application-specific renderer compatibility.
+
 ### Shell scripts
 
 For ShellSpec unit test instructions, see the [ShellSpec README](./spec/README.md).

e2e/scenario/scenario.go

@@ -1498,6 +1498,7 @@ var _ = Register(&Scenario{
 			return errors.Join(
 				ValidateNvidiaModProbeInstalled(ctx, s),
 				ValidateNvidiaGRIDLicenseValid(ctx, s),
+				ValidateNvidiaDriverVersion(ctx, s, datamodel.NvidiaGridDriverVersion),
 				ValidateKubeletHasNotStopped(ctx, s),
 				ValidateServicesDoNotRestartKubelet(ctx, s),
 				ValidateNvidiaPersistencedRunning(ctx, s),
@@ -3003,6 +3004,7 @@ func ubuntu2404GRIDScenario(name, vmSize string) *Scenario {
 					// Ensure nvidia-modprobe install does not restart kubelet and temporarily cause node to be unschedulable
 					ValidateNvidiaModProbeInstalled(ctx, s),
 					ValidateNvidiaGRIDLicenseValid(ctx, s),
+					ValidateNvidiaDriverVersion(ctx, s, datamodel.NvidiaGridDriverVersion),
 					ValidateKubeletHasNotStopped(ctx, s),
 					ValidateServicesDoNotRestartKubelet(ctx, s),
 					ValidateNvidiaPersistencedRunning(ctx, s),
@@ -3043,6 +3045,7 @@ var _ = Register(&Scenario{
 				ValidateNvidiaModProbeInstalled(ctx, s),
 				ValidateNvidiaSMIInstalled(ctx, s),
 				ValidateNvidiaGridV20DriverInstalled(ctx, s),
+				ValidateNvidiaDriverVersion(ctx, s, datamodel.NvidiaGridV20DriverVersion),
 				ValidateKubeletHasNotStopped(ctx, s),
 			)
 		},

e2e/scenario/validators.go

@@ -630,6 +630,31 @@ func ValidateNvidiaGridV20DriverInstalled(ctx context.Context, s *Scenario) erro
 	return err
 }
 
+func nvidiaDriverVersionValidationScript(expected string) (string, error) {
+	if !regexp.MustCompile(`^[0-9]+(\.[0-9]+)+$`).MatchString(expected) {
+		return "", fmt.Errorf("invalid expected NVIDIA driver version %q; check GPUContainerImages in components.json", expected)
+	}
+	return strings.Join([]string{
+		"#!/usr/bin/env bash",
+		"set -euo pipefail",
+		"driver_versions=$(sudo nvidia-smi --query-gpu=driver_version --format=csv,noheader)",
+		"echo \"nvidia driver_versions=$driver_versions\"",
+		"while IFS= read -r driver_version; do",
+		"  driver_version=${driver_version//[[:space:]]/}",
+		fmt.Sprintf("  if [ \"$driver_version\" != %q ]; then echo \"expected NVIDIA driver %s, got '$driver_version'\"; exit 1; fi", expected, expected),
+		"done <<< \"$driver_versions\"",
+	}, "\n"), nil
+}
+
+func ValidateNvidiaDriverVersion(ctx context.Context, s *Scenario, expected string) error {
+	script, err := nvidiaDriverVersionValidationScript(expected)
+	if err != nil {
+		return err
+	}
+	_, err = execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0, fmt.Sprintf("expected NVIDIA driver version %s on every GPU", expected))
+	return err
+}
+
 func ValidateNonEmptyDirectory(ctx context.Context, s *Scenario, dirName string) error {
 	command := []string{
 		"set -ex",

e2e/scenario/validators_gpu_driver_test.go

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+package scenario
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNvidiaDriverVersionValidationScript(t *testing.T) {
+	directory := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(directory, "sudo"),
+		[]byte("#!/usr/bin/env bash\nexec \"$@\"\n"), 0o755))
+	require.NoError(t, os.WriteFile(filepath.Join(directory, "nvidia-smi"),
+		[]byte("#!/usr/bin/env bash\nprintf '%s\\n' \"$TEST_DRIVER_VERSIONS\"\nexit \"$TEST_SMI_EXIT_CODE\"\n"), 0o755))
+
+	cases := []struct {
+		name     string
+		expected string
+		versions string
+		smiExit  string
+		wantErr  bool
+	}{
+		{"matching LTS", "580.178.04", "580.178.04", "0", false},
+		{"matching v20", "595.91.07", "595.91.07", "0", false},
+		{"wrong branch", "580.178.04", "595.91.07", "0", true},
+		{"stale LTS patch", "580.178.04", "580.126.09", "0", true},
+		{"matching multiple GPUs", "595.91.07", "595.91.07\n595.91.07", "0", false},
+		{"second GPU mismatches", "595.91.07", "595.91.07\n580.178.04", "0", true},
+		{"empty GPU output", "580.178.04", "", "0", true},
+		{"nvidia-smi failure", "580.178.04", "580.178.04", "1", true},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			script, err := nvidiaDriverVersionValidationScript(tc.expected)
+			require.NoError(t, err)
+			scriptPath := filepath.Join(t.TempDir(), "validate-driver.sh")
+			require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o755))
+			command := exec.Command(scriptPath)
+			command.Env = append(os.Environ(),
+				"PATH="+directory+string(os.PathListSeparator)+os.Getenv("PATH"),
+				"TEST_DRIVER_VERSIONS="+tc.versions,
+				"TEST_SMI_EXIT_CODE="+tc.smiExit)
+			output, err := command.CombinedOutput()
+			if tc.wantErr {
+				require.Error(t, err, string(output))
+			} else {
+				require.NoError(t, err, string(output))
+			}
+		})
+	}
+}
+
+func TestNvidiaDriverVersionValidationScriptRejectsInvalidVersion(t *testing.T) {
+	for _, expected := range []string{"", "580", "580.178.04 invalid"} {
+		t.Run(expected, func(t *testing.T) {
+			script, err := nvidiaDriverVersionValidationScript(expected)
+			require.Error(t, err)
+			require.Empty(t, script)
+		})
+	}
+}

parts/common/components.json

@@ -768,7 +768,7 @@
       "downloadURL": "mcr.microsoft.com/aks/aks-gpu-grid:*",
       "gpuVersion": {
         "renovateTag": "registry=https://mcr.microsoft.com, name=aks/aks-gpu-grid",
-        "latestVersion": "570.237-20260817204535"
+        "latestVersion": "580.178.04-20261006181431"
       }
     },
     {

pkg/agent/datamodel/gpu_components_test.go

@@ -3,6 +3,7 @@ package datamodel
 
 import (
 	"regexp"
+	"strings"
 	"testing"
 )
 
@@ -79,6 +80,15 @@ func TestLoadConfig(t *testing.T) {
 	}
 }
 
+func TestGPUGridDriverBranches(t *testing.T) {
+	if !strings.HasPrefix(NvidiaGridDriverVersion, "580.") {
+		t.Errorf("A10 GRID driver must remain on R580 LTS, got %q", NvidiaGridDriverVersion)
+	}
+	if !strings.HasPrefix(NvidiaGridV20DriverVersion, "595.") {
+		t.Errorf("GRID v20 driver must remain on R595, got %q", NvidiaGridV20DriverVersion)
+	}
+}
+
 // TestGPUImageRepo verifies that the bare repo name is extracted via exact final
 // path segment (tag stripped), so prefix-sharing repos like "aks-gpu-grid" and
 // "aks-gpu-grid-v20" are never confused by substring matching. This guards the

pkg/agent/gpu_driver_selection_test.go

@@ -0,0 +1,81 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT license.
+
+package agent
+
+import (
+	"strings"
+	"testing"
+
+	"github.com/Azure/agentbaker/pkg/agent/datamodel"
+)
+
+func TestGPUDriverSelection(t *testing.T) {
+	cases := []struct {
+		size       string
+		driverType string
+	}{
+		{"Standard_NV6ads_A10_v5", "grid"},
+		{"Standard_NV12ads_A10_v5", "grid"},
+		{"Standard_NV18ads_A10_v5", "grid"},
+		{"Standard_NV36ads_A10_v5", "grid"},
+		{"Standard_NV72ads_A10_v5", "grid"},
+		{"Standard_NV36adms_A10_v5", "grid"},
+		{"Standard_NC8ads_A10_v4", "grid"},
+		{"Standard_NC16ads_A10_v4", "grid"},
+		{"Standard_NC32ads_A10_v4", "grid"},
+		{"Standard_NC24lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC36ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC36lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC72ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC72lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC144ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC144lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC288ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC288lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC128ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC128lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC256ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC256lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC320ds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC320lds_xl_RTXPRO6000BSE_v6", "grid-v20"},
+		{"Standard_NC4as_T4_v3", "cuda-lts"},
+		{"Standard_NC6s_v3", "cuda-lts"},
+		{"Standard_NC24ads_A100_v4", "cuda-lts"},
+		{"Standard_NC40ads_H100_v5", "cuda-lts"},
+		{"Standard_ND128isr_NDR_GB200_v6", "cuda-lts"},
+		{"Standard_NC6", "cuda"},
+	}
+	for _, tc := range cases {
+		t.Run(tc.size, func(t *testing.T) {
+			var expectedVersion, expectedSuffix string
+			switch tc.driverType {
+			case "grid":
+				expectedVersion = datamodel.NvidiaGridDriverVersion
+				expectedSuffix = datamodel.AKSGPUGridVersionSuffix
+			case "grid-v20":
+				expectedVersion = datamodel.NvidiaGridV20DriverVersion
+				expectedSuffix = datamodel.AKSGPUGridV20VersionSuffix
+			case "cuda-lts":
+				expectedVersion = datamodel.NvidiaCudaLTSDriverVersion
+				expectedSuffix = datamodel.AKSGPUCudaLTSVersionSuffix
+			case "cuda":
+				expectedVersion = datamodel.Nvidia470CudaDriverVersion
+				expectedSuffix = datamodel.AKSGPUCudaLTSVersionSuffix
+			default:
+				t.Fatalf("unexpected test driver type %q", tc.driverType)
+			}
+			for _, size := range []string{tc.size, strings.ToLower(tc.size), strings.ToUpper(tc.size)} {
+				if got := GetGPUDriverType(size); got != tc.driverType {
+					t.Errorf("GetGPUDriverType(%q) = %q, want %q", size, got, tc.driverType)
+				}
+				if got := GetGPUDriverVersion(size); got != expectedVersion {
+					t.Errorf("GetGPUDriverVersion(%q) = %q, want %q", size, got, expectedVersion)
+				}
+				if got := GetAKSGPUImageSHA(size); got != expectedSuffix {
+					t.Errorf("GetAKSGPUImageSHA(%q) = %q, want %q", size, got, expectedSuffix)
+				}
+			}
+		})
+	}
+}