diff --git a/.github/renovate.json b/.github/renovate.json index 9cedd5e58b2..f61b27f0f72 100644 --- a/.github/renovate.json +++ b/.github/renovate.json @@ -708,6 +708,7 @@ ], "groupName": "nvidia-gpu-grid", "versioning": "regex:^(?\\d+)\\.(?\\d+)(?:\\.(?\\d+))?-(?\\d{14})$", + "allowedVersions": "/^580\\./", "automerge": false, "enabled": true, "ignoreUnstable": false, @@ -732,6 +733,7 @@ ], "groupName": "nvidia-gpu-grid-v20", "versioning": "regex:^(?\\d+)\\.(?\\d+)\\.(?\\d+)-(?\\d{14})$", + "allowedVersions": "/^595\\./", "automerge": false, "enabled": true, "ignoreUnstable": false, diff --git a/.github/workflows/go-test.yml b/.github/workflows/go-test.yml index b8ee5431bd3..4a55325d386 100644 --- a/.github/workflows/go-test.yml +++ b/.github/workflows/go-test.yml @@ -57,6 +57,10 @@ jobs: (cd aks-live-patching && gocover-cobertura < ../coverage/aks-live-patching.out > ../coverage/aks-live-patching.xml) fi rm -f coverage/*.out + - name: Test GRID driver version validation + run: | + cd e2e + go test -short ./scenario -run '^TestNvidiaDriverVersionValidationScript' - name: Upload coverage reports uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: (!cancelled()) diff --git a/README.md b/README.md index 591b8fe5610..1e44a1837aa 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,24 @@ Pull request titles must follow [Conventional Commits](https://www.conventionalc ## Tests +### Ubuntu GRID driver branches + +The Ubuntu A10 GRID path uses `aks-gpu-grid` on the patched R580/vGPU 19.6 +LTS branch. NCv6 RTX PRO 6000 BSE uses `aks-gpu-grid-v20` on the patched +R595/vGPU 20.2 branch. Driver versions and image tags are pinned in +`parts/common/components.json`; Renovate is constrained to the corresponding +branch for each image. + +[Azure's Linux GRID support matrix](https://learn.microsoft.com/azure/virtual-machines/linux/n-series-driver-setup#supported-grid-drivers) +lists v19.6 and v20.2 for NVadsA10_v5, but only v20.2 for NCv6 and NCasT4_v3. +This does not change existing CUDA selection for compute SKUs, or the separate +Azure Linux/ACL sysext installation paths. The existing NCads_A10_v4 aliases +retain their A10 GRID image selection. + +The Ubuntu GRID E2E scenarios require the exact configured driver version on +every GPU, in addition to checking GRID licensing and service health. These +checks do not establish application-specific renderer compatibility. + ### Shell scripts For ShellSpec unit test instructions, see the [ShellSpec README](./spec/README.md). diff --git a/e2e/scenario/scenario.go b/e2e/scenario/scenario.go index c14bc9dcdcf..474ffd1e09e 100755 --- a/e2e/scenario/scenario.go +++ b/e2e/scenario/scenario.go @@ -1498,6 +1498,7 @@ var _ = Register(&Scenario{ return errors.Join( ValidateNvidiaModProbeInstalled(ctx, s), ValidateNvidiaGRIDLicenseValid(ctx, s), + ValidateNvidiaDriverVersion(ctx, s, datamodel.NvidiaGridDriverVersion), ValidateKubeletHasNotStopped(ctx, s), ValidateServicesDoNotRestartKubelet(ctx, s), ValidateNvidiaPersistencedRunning(ctx, s), @@ -3003,6 +3004,7 @@ func ubuntu2404GRIDScenario(name, vmSize string) *Scenario { // Ensure nvidia-modprobe install does not restart kubelet and temporarily cause node to be unschedulable ValidateNvidiaModProbeInstalled(ctx, s), ValidateNvidiaGRIDLicenseValid(ctx, s), + ValidateNvidiaDriverVersion(ctx, s, datamodel.NvidiaGridDriverVersion), ValidateKubeletHasNotStopped(ctx, s), ValidateServicesDoNotRestartKubelet(ctx, s), ValidateNvidiaPersistencedRunning(ctx, s), @@ -3043,6 +3045,7 @@ var _ = Register(&Scenario{ ValidateNvidiaModProbeInstalled(ctx, s), ValidateNvidiaSMIInstalled(ctx, s), ValidateNvidiaGridV20DriverInstalled(ctx, s), + ValidateNvidiaDriverVersion(ctx, s, datamodel.NvidiaGridV20DriverVersion), ValidateKubeletHasNotStopped(ctx, s), ) }, diff --git a/e2e/scenario/validators.go b/e2e/scenario/validators.go index 8086b4a564d..414cbc43c25 100644 --- a/e2e/scenario/validators.go +++ b/e2e/scenario/validators.go @@ -630,6 +630,31 @@ func ValidateNvidiaGridV20DriverInstalled(ctx context.Context, s *Scenario) erro return err } +func nvidiaDriverVersionValidationScript(expected string) (string, error) { + if !regexp.MustCompile(`^[0-9]+(\.[0-9]+)+$`).MatchString(expected) { + return "", fmt.Errorf("invalid expected NVIDIA driver version %q; check GPUContainerImages in components.json", expected) + } + return strings.Join([]string{ + "#!/usr/bin/env bash", + "set -euo pipefail", + "driver_versions=$(sudo nvidia-smi --query-gpu=driver_version --format=csv,noheader)", + "echo \"nvidia driver_versions=$driver_versions\"", + "while IFS= read -r driver_version; do", + " driver_version=${driver_version//[[:space:]]/}", + fmt.Sprintf(" if [ \"$driver_version\" != %q ]; then echo \"expected NVIDIA driver %s, got '$driver_version'\"; exit 1; fi", expected, expected), + "done <<< \"$driver_versions\"", + }, "\n"), nil +} + +func ValidateNvidiaDriverVersion(ctx context.Context, s *Scenario, expected string) error { + script, err := nvidiaDriverVersionValidationScript(expected) + if err != nil { + return err + } + _, err = execScriptOnVMForScenarioValidateExitCode(ctx, s, script, 0, fmt.Sprintf("expected NVIDIA driver version %s on every GPU", expected)) + return err +} + func ValidateNonEmptyDirectory(ctx context.Context, s *Scenario, dirName string) error { command := []string{ "set -ex", diff --git a/e2e/scenario/validators_gpu_driver_test.go b/e2e/scenario/validators_gpu_driver_test.go new file mode 100644 index 00000000000..d2ba0b3add2 --- /dev/null +++ b/e2e/scenario/validators_gpu_driver_test.go @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package scenario + +import ( + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNvidiaDriverVersionValidationScript(t *testing.T) { + directory := t.TempDir() + require.NoError(t, os.WriteFile(filepath.Join(directory, "sudo"), + []byte("#!/usr/bin/env bash\nexec \"$@\"\n"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(directory, "nvidia-smi"), + []byte("#!/usr/bin/env bash\nprintf '%s\\n' \"$TEST_DRIVER_VERSIONS\"\nexit \"$TEST_SMI_EXIT_CODE\"\n"), 0o755)) + + cases := []struct { + name string + expected string + versions string + smiExit string + wantErr bool + }{ + {"matching LTS", "580.178.04", "580.178.04", "0", false}, + {"matching v20", "595.91.07", "595.91.07", "0", false}, + {"wrong branch", "580.178.04", "595.91.07", "0", true}, + {"stale LTS patch", "580.178.04", "580.126.09", "0", true}, + {"matching multiple GPUs", "595.91.07", "595.91.07\n595.91.07", "0", false}, + {"second GPU mismatches", "595.91.07", "595.91.07\n580.178.04", "0", true}, + {"empty GPU output", "580.178.04", "", "0", true}, + {"nvidia-smi failure", "580.178.04", "580.178.04", "1", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + script, err := nvidiaDriverVersionValidationScript(tc.expected) + require.NoError(t, err) + scriptPath := filepath.Join(t.TempDir(), "validate-driver.sh") + require.NoError(t, os.WriteFile(scriptPath, []byte(script), 0o755)) + command := exec.Command(scriptPath) + command.Env = append(os.Environ(), + "PATH="+directory+string(os.PathListSeparator)+os.Getenv("PATH"), + "TEST_DRIVER_VERSIONS="+tc.versions, + "TEST_SMI_EXIT_CODE="+tc.smiExit) + output, err := command.CombinedOutput() + if tc.wantErr { + require.Error(t, err, string(output)) + } else { + require.NoError(t, err, string(output)) + } + }) + } +} + +func TestNvidiaDriverVersionValidationScriptRejectsInvalidVersion(t *testing.T) { + for _, expected := range []string{"", "580", "580.178.04 invalid"} { + t.Run(expected, func(t *testing.T) { + script, err := nvidiaDriverVersionValidationScript(expected) + require.Error(t, err) + require.Empty(t, script) + }) + } +} diff --git a/parts/common/components.json b/parts/common/components.json index 54982ebc33a..8c4737ff226 100644 --- a/parts/common/components.json +++ b/parts/common/components.json @@ -768,7 +768,7 @@ "downloadURL": "mcr.microsoft.com/aks/aks-gpu-grid:*", "gpuVersion": { "renovateTag": "registry=https://mcr.microsoft.com, name=aks/aks-gpu-grid", - "latestVersion": "570.237-20260817204535" + "latestVersion": "580.178.04-20261006181431" } }, { diff --git a/pkg/agent/datamodel/gpu_components_test.go b/pkg/agent/datamodel/gpu_components_test.go index 16de9ab7fb2..a158167f116 100644 --- a/pkg/agent/datamodel/gpu_components_test.go +++ b/pkg/agent/datamodel/gpu_components_test.go @@ -3,6 +3,7 @@ package datamodel import ( "regexp" + "strings" "testing" ) @@ -79,6 +80,15 @@ func TestLoadConfig(t *testing.T) { } } +func TestGPUGridDriverBranches(t *testing.T) { + if !strings.HasPrefix(NvidiaGridDriverVersion, "580.") { + t.Errorf("A10 GRID driver must remain on R580 LTS, got %q", NvidiaGridDriverVersion) + } + if !strings.HasPrefix(NvidiaGridV20DriverVersion, "595.") { + t.Errorf("GRID v20 driver must remain on R595, got %q", NvidiaGridV20DriverVersion) + } +} + // TestGPUImageRepo verifies that the bare repo name is extracted via exact final // path segment (tag stripped), so prefix-sharing repos like "aks-gpu-grid" and // "aks-gpu-grid-v20" are never confused by substring matching. This guards the diff --git a/pkg/agent/gpu_driver_selection_test.go b/pkg/agent/gpu_driver_selection_test.go new file mode 100644 index 00000000000..d6d16d4d7a9 --- /dev/null +++ b/pkg/agent/gpu_driver_selection_test.go @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT license. + +package agent + +import ( + "strings" + "testing" + + "github.com/Azure/agentbaker/pkg/agent/datamodel" +) + +func TestGPUDriverSelection(t *testing.T) { + cases := []struct { + size string + driverType string + }{ + {"Standard_NV6ads_A10_v5", "grid"}, + {"Standard_NV12ads_A10_v5", "grid"}, + {"Standard_NV18ads_A10_v5", "grid"}, + {"Standard_NV36ads_A10_v5", "grid"}, + {"Standard_NV72ads_A10_v5", "grid"}, + {"Standard_NV36adms_A10_v5", "grid"}, + {"Standard_NC8ads_A10_v4", "grid"}, + {"Standard_NC16ads_A10_v4", "grid"}, + {"Standard_NC32ads_A10_v4", "grid"}, + {"Standard_NC24lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC36ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC36lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC72ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC72lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC144ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC144lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC288ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC288lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC128ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC128lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC256ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC256lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC320ds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC320lds_xl_RTXPRO6000BSE_v6", "grid-v20"}, + {"Standard_NC4as_T4_v3", "cuda-lts"}, + {"Standard_NC6s_v3", "cuda-lts"}, + {"Standard_NC24ads_A100_v4", "cuda-lts"}, + {"Standard_NC40ads_H100_v5", "cuda-lts"}, + {"Standard_ND128isr_NDR_GB200_v6", "cuda-lts"}, + {"Standard_NC6", "cuda"}, + } + for _, tc := range cases { + t.Run(tc.size, func(t *testing.T) { + var expectedVersion, expectedSuffix string + switch tc.driverType { + case "grid": + expectedVersion = datamodel.NvidiaGridDriverVersion + expectedSuffix = datamodel.AKSGPUGridVersionSuffix + case "grid-v20": + expectedVersion = datamodel.NvidiaGridV20DriverVersion + expectedSuffix = datamodel.AKSGPUGridV20VersionSuffix + case "cuda-lts": + expectedVersion = datamodel.NvidiaCudaLTSDriverVersion + expectedSuffix = datamodel.AKSGPUCudaLTSVersionSuffix + case "cuda": + expectedVersion = datamodel.Nvidia470CudaDriverVersion + expectedSuffix = datamodel.AKSGPUCudaLTSVersionSuffix + default: + t.Fatalf("unexpected test driver type %q", tc.driverType) + } + for _, size := range []string{tc.size, strings.ToLower(tc.size), strings.ToUpper(tc.size)} { + if got := GetGPUDriverType(size); got != tc.driverType { + t.Errorf("GetGPUDriverType(%q) = %q, want %q", size, got, tc.driverType) + } + if got := GetGPUDriverVersion(size); got != expectedVersion { + t.Errorf("GetGPUDriverVersion(%q) = %q, want %q", size, got, expectedVersion) + } + if got := GetAKSGPUImageSHA(size); got != expectedSuffix { + t.Errorf("GetAKSGPUImageSHA(%q) = %q, want %q", size, got, expectedSuffix) + } + } + }) + } +}