tests/ai-conformance: use nvidia GPU operator to install GPU drivers

Author: justinsb

tests/e2e/scenarios/ai-conformance/run-test.sh

@@ -64,13 +64,12 @@ kops-acquire-latest
 
 # Cluster Configuration
 # - Networking: Cilium with Gateway API enabled
-# - Nodes: c5.large
-# - Runtime: NVIDIA enabled
+# - Nodes: c5.large (we need some non-GPU nodes for non-GPU workloads)
+# - NVIDIA driver and runtime are managed by GPU Operator (not kOps)
 OVERRIDES="${OVERRIDES-} --networking=cilium"
 OVERRIDES="${OVERRIDES} --set=cluster.spec.networking.cilium.gatewayAPI.enabled=true"
 OVERRIDES="${OVERRIDES} --node-size=c5.large"
 OVERRIDES="${OVERRIDES} --node-count=2"
-OVERRIDES="${OVERRIDES} --set=cluster.spec.containerd.nvidiaGPU.enabled=true"
 
 kops-up
 
@@ -120,30 +119,19 @@ kubectl apply --server-side -f https://github.com/cert-manager/cert-manager/rele
 # kubectl apply -f https://raw.githubusercontent.com/NVIDIA/k8s-device-plugin/refs/tags/v0.17.0/deployments/static/nvidia-device-plugin.yml
 
 # NVIDIA GPU Operator
+# Manages the full NVIDIA stack: kernel driver, container toolkit, device plugin.
+# The driver is installed into /run/nvidia/driver on each node.
 helm upgrade -i nvidia-gpu-operator --wait \
     -n gpu-operator --create-namespace \
     nvidia/gpu-operator \
-    --version=v25.10.1
+    --version=v25.10.1 \
+    --wait
 
-# 1.5 NVIDIA DRA Driver
 
+# NVIDIA DRA Driver
+# Uses the driver installed by GPU Operator at /run/nvidia/driver (the default).
 echo "Installing NVIDIA DRA Driver..."
 
-# cat > values.yaml <<EOF
-# kubeletPlugins:
-#   nodeSelector:
-#     feature.node.kubernetes.io/pci-0302_10de.present: "true"
-# EOF
-
-# helm upgrade -i nvidia-dra-driver-gpu nvidia/nvidia-dra-driver-gpu \
-#   --create-namespace \
-#   --namespace nvidia-dra-driver-gpu \
-#   --version 25.12.0 \
-#   --set resources.gpus.enabled=true \
-#   --set gpuResourcesEnabledOverride=true \
-#   -f values.yaml \
-#   --wait
-
 cat > values.yaml <<EOF
 # The driver daemonset needs a toleration for the nvidia.com/gpu taint
 kubeletPlugin:
@@ -182,9 +170,11 @@ echo "----------------------------------------------------------------"
 # Wait for kOps validation
 "${KOPS}" validate cluster --wait=15m
 
-# Verify Components
-echo "Verifying NVIDIA Device Plugin..."
-#kubectl rollout status daemonset -n kube-system nvidia-device-plugin-daemonset --timeout=5m || echo "Warning: NVIDIA Device Plugin not ready yet"
+echo "Verifying GPU Operator driver..."
+kubectl rollout status daemonset -n gpu-operator nvidia-driver-daemonset --timeout=5m || echo "Warning: GPU Operator driver daemonset not ready yet"
+
+echo "Verifying GPU Operator device plugin..."
+kubectl rollout status daemonset -n gpu-operator nvidia-device-plugin-daemonset --timeout=5m || echo "Warning: GPU Operator device plugin not ready yet"
 
 echo "Verifying Kueue..."
 kubectl rollout status deployment -n kueue-system kueue-controller-manager --timeout=5m || echo "Warning: Kueue not ready yet"