diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py
index d4ce2ad16f3e..cd3205296343 100644
--- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py
+++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py
@@ -229,7 +229,7 @@ class HardwareSpec(BaseModel):
gpuSku: Optional[GPUSKUType] = Field(
default=None,
- description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.",
+ description="GPUSKU selects the GPU type to target. When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.",
)
vramMb: Optional[float] = Field(
default=None,
@@ -237,11 +237,11 @@ class HardwareSpec(BaseModel):
)
totalGpus: Optional[int] = Field(
default=None,
- description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.",
+ description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.",
)
numGpusPerNode: Optional[int] = Field(
default=None,
- description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected from cluster GPU nodes.",
+ description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on.",
)
interconnect: Optional[str] = Field(
default=None,
diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go
index bbdc617b187e..3d4d7daf893f 100644
--- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go
+++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go
@@ -407,8 +407,8 @@ type FeaturesSpec struct {
// If all four fields are set, discovery is skipped.
type HardwareSpec struct {
// GPUSKU selects the GPU type to target.
- // When omitted, auto-detected by selecting the GPU with the highest
- // node count, then highest VRAM. In mixed-GPU clusters, set this to
+ // When omitted, auto-detected as the GPU type of the node with the most GPUs
+ // (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to
// choose which GPU type to use. Discovery and totalGpus are then
// restricted to nodes matching this SKU.
// The legacy value gb200_sxm is deprecated; use gb200 instead.
@@ -423,15 +423,17 @@ type HardwareSpec struct {
// TotalGPUs is the GPU budget for profiling and deployment.
// The profiler uses this to determine parallelism and replica count.
- // When omitted, computed by counting GPUs on discovered nodes
- // (filtered by gpuSku when set), temporarily capped at 32 to
+ // When omitted, computed by summing GPUs across discovered nodes of the
+ // selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to
// limit profiler search space. This cap may be removed in a future
// release. Set this field explicitly to override.
// +optional
TotalGPUs *int32 `json:"totalGpus,omitempty"`
// NumGPUsPerNode is the number of GPUs per node.
- // When omitted, auto-detected from cluster GPU nodes.
+ // When omitted, auto-detected as the highest GPU count on any discovered node
+ // of the selected GPU type. In clusters whose nodes have different GPU counts,
+ // set this explicitly to the node size you intend to schedule on.
// +optional
NumGPUsPerNode *int32 `json:"numGpusPerNode,omitempty"`
// Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml
index 404cc63ec777..437a12d274ea 100644
--- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml
+++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml
@@ -663,8 +663,8 @@ spec:
- mi300
description: |-
GPUSKU selects the GPU type to target.
- When omitted, auto-detected by selecting the GPU with the highest
- node count, then highest VRAM. In mixed-GPU clusters, set this to
+ When omitted, auto-detected as the GPU type of the node with the most GPUs
+ (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead.
@@ -693,7 +693,9 @@ spec:
numGpusPerNode:
description: |-
NumGPUsPerNode is the number of GPUs per node.
- When omitted, auto-detected from cluster GPU nodes.
+ When omitted, auto-detected as the highest GPU count on any discovered node
+ of the selected GPU type. In clusters whose nodes have different GPU counts,
+ set this explicitly to the node size you intend to schedule on.
format: int32
type: integer
rdma:
@@ -721,8 +723,8 @@ spec:
description: |-
TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
- When omitted, computed by counting GPUs on discovered nodes
- (filtered by gpuSku when set), temporarily capped at 32 to
+ When omitted, computed by summing GPUs across discovered nodes of the
+ selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override.
format: int32
diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go
index 7577d49df281..e93462cdf5e7 100644
--- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go
+++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go
@@ -1900,7 +1900,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx
logger.Info("GPU discovery completed successfully",
"gpusPerNode", gpuInfo.GPUsPerNode,
"nodesWithGPUs", gpuInfo.NodesWithGPUs,
- "totalGpus", gpuInfo.GPUsPerNode*gpuInfo.NodesWithGPUs,
+ "totalGpus", gpuInfo.TotalGPUs,
"model", gpuInfo.Model,
"vramMiB", gpuInfo.VRAMPerGPU,
"system", gpuInfo.System,
@@ -1935,14 +1935,14 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx
if hw.TotalGPUs == nil && gpuInfo != nil {
// TODO: This is a temporary limit to prevent the profiler from using too many GPUs.
// Will be removed once a fix is in the Profiler/AIC.
- const defaultMaxAutoGPUs = int32(32)
- total := int32(gpuInfo.GPUsPerNode * gpuInfo.NodesWithGPUs)
+ const defaultMaxAutoGPUs = 32
+ total := gpuInfo.TotalGPUs
if total > defaultMaxAutoGPUs {
logger.Info("Capping auto-discovered TotalGPUs at default limit; set hardware.totalGpus to override",
"discovered", total, "cap", defaultMaxAutoGPUs)
total = defaultMaxAutoGPUs
}
- hw.TotalGPUs = &total
+ hw.TotalGPUs = new(int32(total))
changed = true
}
if hw.Interconnect == "" && gpuInfo != nil && gpuInfo.Interconnect != "" {
diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go
index 7658c07a3b0b..ba02c3b2fdb7 100644
--- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go
+++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go
@@ -23,6 +23,7 @@ import (
"context"
"encoding/json"
"errors"
+ "fmt"
"slices"
"strings"
"time"
@@ -3007,13 +3008,13 @@ spec:
Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed())
defer func() { _ = k8sClient.Delete(ctx, dgdr) }()
- GinkgoT().Log("creating two eight-GPU nodes for node-label discovery")
+ GinkgoT().Log("creating eight-GPU and four-GPU nodes for node-label discovery")
gpuNodes := make([]*corev1.Node, 0, 2)
- for _, nodeName := range []string{"gpu-worker-budget-1", "gpu-worker-budget-2"} {
+ for i, count := range []string{"8", "4"} {
node := &corev1.Node{
- Name: nodeName,
+ Name: fmt.Sprintf("gpu-worker-budget-%d", i),
Labels: map[string]string{
- gpu.LabelGPUCount: "8",
+ gpu.LabelGPUCount: count,
gpu.LabelGPUProduct: "H200-SXM5-141GB",
gpu.LabelGPUMemory: "141312",
},
@@ -3042,7 +3043,7 @@ spec:
Expect(k8sClient.Get(ctx, request.NamespacedName, &persisted)).Should(Succeed())
Expect(persisted.Spec.Hardware).NotTo(BeNil())
Expect(persisted.Spec.Hardware.TotalGPUs).NotTo(BeNil())
- Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(16)))
+ Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(12)))
GinkgoT().Log("creating the profiling Job from the persisted DGDR")
_, err = reconciler.Reconcile(ctx, request)
@@ -3110,6 +3111,7 @@ spec:
VRAMPerGPU: 81920,
System: "h100_sxm",
NodesWithGPUs: 1,
+ TotalGPUs: 8,
}
cache := gpu.NewGPUDiscoveryCache()
cache.Set("", mockGPU, 10*time.Minute)
@@ -3224,6 +3226,7 @@ spec:
VRAMPerGPU: 81920,
System: "h100_sxm",
NodesWithGPUs: 1,
+ TotalGPUs: 8,
}
cache := gpu.NewGPUDiscoveryCache()
cache.Set("", mockGPU, 10*time.Minute)
@@ -3339,6 +3342,7 @@ spec:
VRAMPerGPU: 81920,
System: "h100_sxm",
NodesWithGPUs: 1,
+ TotalGPUs: 8,
}
cache := gpu.NewGPUDiscoveryCache()
cache.Set("", mockGPU, 10*time.Minute)
@@ -3961,6 +3965,7 @@ spec:
VRAMPerGPU: 81920,
System: "a100_sxm",
NodesWithGPUs: 1,
+ TotalGPUs: 8,
}
cache := gpu.NewGPUDiscoveryCache()
cache.Set("", mockGPU, 10*time.Minute)
diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go
index ce7b90a4225f..4d365ca62e29 100644
--- a/deploy/operator/internal/controller/enrich_hardware_test.go
+++ b/deploy/operator/internal/controller/enrich_hardware_test.go
@@ -273,6 +273,46 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) {
}
}
+func TestEnrichHardwareFromDiscovery_TotalGPUs(t *testing.T) {
+ tests := []struct {
+ name string
+ counts []int
+ explicitTotal *int32
+ wantTotal int32
+ }{
+ {name: "heterogeneous nodes", counts: []int{8, 4}, wantTotal: 12},
+ {name: "aggregate below cap despite larger maximum times nodes", counts: []int{8, 8, 8, 4, 2}, wantTotal: 30},
+ {name: "aggregate above cap", counts: []int{8, 8, 8, 8, 4}, wantTotal: 32},
+ {name: "explicit total above cap", counts: []int{8, 4}, explicitTotal: new(int32(64)), wantTotal: 64},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Log("creating H100 nodes and an unrelated A100 node")
+ objects := []client.Object{gpuNode("a100-node", "A100-SXM4-40GB", 8, 40960)}
+ for i, count := range tt.counts {
+ objects = append(objects, gpuNode(fmt.Sprintf("h100-node-%d", i), "H100-SXM5-80GB", count, 81920))
+ }
+ r := newFakeReconciler(objects...)
+ dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{
+ Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{
+ Hardware: &nvidiacomv1beta1.HardwareSpec{
+ GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM,
+ TotalGPUs: tt.explicitTotal,
+ },
+ },
+ }
+
+ t.Log("enriching missing hardware fields while preserving an explicit budget")
+ changed, err := r.enrichHardwareFromDiscovery(t.Context(), dgdr)
+ require.NoError(t, err)
+ require.True(t, changed)
+ require.NotNil(t, dgdr.Spec.Hardware.TotalGPUs)
+ assert.Equal(t, tt.wantTotal, *dgdr.Spec.Hardware.TotalGPUs)
+ assert.Equal(t, int32(8), *dgdr.Spec.Hardware.NumGPUsPerNode)
+ })
+ }
+}
+
func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T) {
r := newFakeReconciler()
r.GPUDiscovery = gpupkg.NewGPUDiscovery(nil)
@@ -281,6 +321,7 @@ func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T
NodeName: "n1",
GPUsPerNode: 8,
NodesWithGPUs: 2,
+ TotalGPUs: 16,
Model: "NVIDIA H100 80GB HBM3",
VRAMPerGPU: 81079,
System: nvidiacomv1beta1.GPUSKUTypeH100PCIe,
@@ -313,6 +354,7 @@ func TestEnrichHardwareFromDiscovery_FillsOptionalMetadataWhenRequiredFieldsSet(
NodeName: "n1",
GPUsPerNode: 8,
NodesWithGPUs: 4,
+ TotalGPUs: 32,
Model: "NVIDIA H100 80GB HBM3",
VRAMPerGPU: 81079,
System: nvidiacomv1beta1.GPUSKUTypeH100PCIe,
@@ -367,6 +409,7 @@ func TestCreateProfilingJobPersistsDiscoveredHardware(t *testing.T) {
NodeName: "n1",
GPUsPerNode: 8,
NodesWithGPUs: 2,
+ TotalGPUs: 16,
Model: "NVIDIA H100 80GB HBM3",
VRAMPerGPU: 81079,
System: nvidiacomv1beta1.GPUSKUTypeH100PCIe,
diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go
index d431fdcce5c9..e553776ba575 100644
--- a/deploy/operator/internal/gpu/discovery.go
+++ b/deploy/operator/internal/gpu/discovery.go
@@ -167,8 +167,9 @@ var gpuRules = []gpuRule{
// GPUInfo contains discovered GPU configuration from cluster nodes
type GPUInfo struct {
NodeName string // Name of the node with this GPU configuration
- GPUsPerNode int // Maximum GPUs per node found in the cluster
- NodesWithGPUs int // Number of nodes that have GPUs
+ GPUsPerNode int // Maximum GPUs per node with the selected SKU
+ NodesWithGPUs int // Number of discovered nodes with the selected SKU
+ TotalGPUs int // Sum of GPUs across discovered nodes with the selected SKU, before any budget cap
Model string // GPU product name (e.g., "H100-SXM5-80GB")
VRAMPerGPU int // VRAM in MiB per GPU
System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown
@@ -259,7 +260,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien
//
// When filterSKU is non-empty, only nodes whose inferred SKU matches are
// considered. When empty, the best node is selected first (highest GPU count,
-// then VRAM) and then only nodes with the same SKU are counted.
+// then VRAM). Each node with the selected SKU contributes once to NodesWithGPUs,
+// and its GPU count is summed into TotalGPUs.
//
// The function performs the following:
//
@@ -271,8 +273,7 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien
// 6. Selects the "best" GPU node (filtered by SKU when set) based on:
// - Highest GPU count
// - Highest VRAM per GPU (tie-breaker)
-// 7. Counts only nodes matching the selected SKU for NodesWithGPUs.
-// 8. Caches the result per SKU for a short duration to avoid repeated scraping.
+// 7. Caches the result per SKU for a short duration to avoid repeated scraping.
//
// Behavior Notes:
//
@@ -280,7 +281,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien
// load-balancing ambiguity in multi-node clusters.
// - If at least one pod is successfully scraped, partial failures are tolerated.
// - If all pods fail to scrape, an aggregated error is returned.
-// - Assumes DCGM exporter runs as a DaemonSet (one pod per GPU node).
+// - Multiple exporters on one node use the report with the highest GPU count,
+// then highest VRAM, for both node selection and aggregation.
//
// Returns:
// - *GPUInfo for the selected node
@@ -353,6 +355,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context,
nodeName string
}
allNodes := make([]nodeInfo, 0, len(dcgmPods))
+ nodeIndexes := make(map[string]int)
var scrapeErrors []error
for _, pod := range dcgmPods {
@@ -366,7 +369,18 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context,
continue
}
- allNodes = append(allNodes, nodeInfo{info: info, sku: InferHardwareSystem(info.Model), nodeName: pod.Spec.NodeName})
+ // Running pods have a scheduled node name; keep its strongest report in first-seen node order.
+ n := nodeInfo{info: info, sku: InferHardwareSystem(info.Model), nodeName: pod.Spec.NodeName}
+ if index, seen := nodeIndexes[n.nodeName]; seen {
+ previous := allNodes[index].info
+ if info.GPUsPerNode > previous.GPUsPerNode ||
+ (info.GPUsPerNode == previous.GPUsPerNode && info.VRAMPerGPU > previous.VRAMPerGPU) {
+ allNodes[index] = n
+ }
+ continue
+ }
+ nodeIndexes[n.nodeName] = len(allNodes)
+ allNodes = append(allNodes, n)
}
if len(allNodes) == 0 {
@@ -399,11 +413,12 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context,
return nil, fmt.Errorf("no GPU metrics could be parsed from any DCGM pod")
}
- // Count only nodes with the same SKU as the selected best node,
+ // Count nodes and GPUs with the same SKU as the selected best node,
// and detect RDMA on matching nodes only. On a cold cache and a no-RDMA
// cluster, this performs one Node read per matching node; that keeps a
// single negative node from masking RDMA on another node.
nodesWithGPUs := 0
+ totalGPUs := 0
var rdmaDetected bool
var rdmaType string
for _, n := range allNodes {
@@ -411,6 +426,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context,
continue
}
nodesWithGPUs++
+ totalGPUs += n.info.GPUsPerNode
if !rdmaDetected {
rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.nodeName)
if rdma {
@@ -435,6 +451,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context,
bestNode.System = bestSKU
bestNode.CloudProvider = cloudProvider
bestNode.NodesWithGPUs = nodesWithGPUs
+ bestNode.TotalGPUs = totalGPUs
bestNode.RDMAEnabled = rdmaDetected
bestNode.RDMAType = rdmaType
@@ -868,16 +885,18 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK
len(nodeList.Items), LabelGPUCount, LabelGPUProduct, LabelGPUMemory)
}
- // Count only nodes with the same SKU as the selected best node,
+ // Count nodes and GPUs with the same SKU as the selected best node,
// and detect RDMA on matching nodes only. On a cold cache and a no-RDMA
// cluster, this performs one Node read per matching node; that keeps a
// single negative node from masking RDMA on another node.
nodesWithGPUs := 0
+ totalGPUs := 0
var rdmaDetected bool
var rdmaType string
for _, n := range allNodes {
if n.sku == bestSKU {
nodesWithGPUs++
+ totalGPUs += n.info.GPUsPerNode
if !rdmaDetected {
rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.info.NodeName)
if rdma {
@@ -898,12 +917,13 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK
bestNode.System = bestSKU
bestNode.NodesWithGPUs = nodesWithGPUs
+ bestNode.TotalGPUs = totalGPUs
bestNode.RDMAEnabled = rdmaDetected
bestNode.RDMAType = rdmaType
logger.Info("GPU discovery completed",
"gpusPerNode", bestNode.GPUsPerNode,
"nodesWithGPUs", bestNode.NodesWithGPUs,
- "totalGpus", bestNode.GPUsPerNode*bestNode.NodesWithGPUs,
+ "totalGpus", bestNode.TotalGPUs,
"model", bestNode.Model,
"vram", bestNode.VRAMPerGPU,
"system", bestNode.System,
diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go
index 10f0c84a91dd..1f2cf1f50fe0 100644
--- a/deploy/operator/internal/gpu/discovery_test.go
+++ b/deploy/operator/internal/gpu/discovery_test.go
@@ -51,6 +51,82 @@ func newFakeClient(objs ...client.Object) client.Reader {
Build()
}
+func TestGPUDiscovery_TotalGPUs(t *testing.T) {
+ for _, source := range []string{"node labels", "DCGM"} {
+ t.Run(source, func(t *testing.T) {
+ tests := []struct {
+ name string
+ h100Counts []int
+ a100Counts []int
+ filterSKU nvidiacomv1beta1.GPUSKUType
+ wantSKU nvidiacomv1beta1.GPUSKUType
+ wantPerNode int
+ wantNodes int
+ wantTotal int
+ }{
+ {name: "heterogeneous nodes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12},
+ {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12},
+ {name: "filter selects other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, filterSKU: "a100_sxm", wantSKU: "a100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 10},
+ {name: "discovery does not cap the total", h100Counts: []int{8, 8, 8, 8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 5, wantTotal: 36},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Log("creating GPU nodes and DCGM exporters for each SKU")
+ var objects []client.Object
+ metrics := make(map[string]GPUInfo)
+ for _, group := range []struct {
+ model string
+ vram int
+ counts []int
+ }{
+ {model: "H100-SXM5-80GB", vram: 81920, counts: tt.h100Counts},
+ {model: "A100-SXM4-40GB", vram: 40960, counts: tt.a100Counts},
+ } {
+ for _, count := range group.counts {
+ name := fmt.Sprintf("gpu-node-%d", len(metrics))
+ ip := fmt.Sprintf("10.0.0.%d", len(metrics)+1)
+ objects = append(objects, &corev1.Node{
+ Name: name,
+ Labels: map[string]string{
+ LabelGPUCount: fmt.Sprint(count), LabelGPUProduct: group.model, LabelGPUMemory: fmt.Sprint(group.vram),
+ },
+ }, &corev1.Pod{
+ Name: name, Namespace: "gpu-operator",
+ Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter},
+ Spec: corev1.PodSpec{NodeName: name},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip},
+ })
+ metrics[buildDCGMEndpoint(ip)] = GPUInfo{NodeName: name, GPUsPerNode: count, Model: group.model, VRAMPerGPU: group.vram}
+ }
+ }
+ k8sClient := newFakeClient(objects...)
+
+ t.Log("discovering the selected SKU's actual GPU total")
+ var info *GPUInfo
+ var err error
+ if source == "DCGM" {
+ discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) {
+ scraped, ok := metrics[endpoint]
+ if !ok {
+ return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint)
+ }
+ return &scraped, nil
+ })
+ info, err = discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, nil, tt.filterSKU)
+ } else {
+ info, err = DiscoverGPUsFiltered(t.Context(), k8sClient, tt.filterSKU)
+ }
+ require.NoError(t, err)
+ assert.Equal(t, tt.wantSKU, info.System)
+ assert.Equal(t, tt.wantPerNode, info.GPUsPerNode)
+ assert.Equal(t, tt.wantNodes, info.NodesWithGPUs)
+ assert.Equal(t, tt.wantTotal, info.TotalGPUs)
+ })
+ }
+ })
+ }
+}
+
func TestDiscoverGPUs_SingleNode(t *testing.T) {
ctx := context.Background()
@@ -70,6 +146,7 @@ func TestDiscoverGPUs_SingleNode(t *testing.T) {
require.NotNil(t, gpuInfo)
assert.Equal(t, 8, gpuInfo.GPUsPerNode)
+ assert.Equal(t, 8, gpuInfo.TotalGPUs)
assert.Equal(t, "H100-SXM5-80GB", gpuInfo.Model)
assert.Equal(t, 81920, gpuInfo.VRAMPerGPU)
assert.Equal(t, "h100_sxm", string(gpuInfo.System))
@@ -103,6 +180,7 @@ func TestDiscoverGPUs_MultipleNodesHomogeneous(t *testing.T) {
require.NotNil(t, gpuInfo)
assert.Equal(t, 8, gpuInfo.GPUsPerNode)
+ assert.Equal(t, 16, gpuInfo.TotalGPUs)
assert.Equal(t, "H100-SXM5-80GB", gpuInfo.Model)
assert.Equal(t, 81920, gpuInfo.VRAMPerGPU)
}
@@ -983,6 +1061,62 @@ func TestScrapeMetricsEndpoint(t *testing.T) {
})
}
+func TestDiscoverGPUsFromDCGM_DeduplicatesNodes(t *testing.T) {
+ tests := []struct {
+ name string
+ counts [2]int
+ vram [2]int
+ failFirstExporter bool
+ wantVRAM int
+ }{
+ {name: "matching reports", counts: [2]int{8, 8}, vram: [2]int{81920, 81920}, wantVRAM: 81920},
+ {name: "first exporter fails", counts: [2]int{8, 8}, vram: [2]int{81920, 81920}, failFirstExporter: true, wantVRAM: 81920},
+ {name: "larger count last", counts: [2]int{4, 8}, vram: [2]int{81920, 40960}, wantVRAM: 40960},
+ {name: "larger count first", counts: [2]int{8, 4}, vram: [2]int{40960, 81920}, wantVRAM: 40960},
+ {name: "higher VRAM last", counts: [2]int{8, 8}, vram: [2]int{40960, 81920}, wantVRAM: 81920},
+ {name: "higher VRAM first", counts: [2]int{8, 8}, vram: [2]int{81920, 40960}, wantVRAM: 81920},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Log("creating two exporters on one node and one on another node")
+ var objects []client.Object
+ metrics := make(map[string]GPUInfo)
+ for i, node := range []string{"node-a", "node-a", "node-b"} {
+ ip := fmt.Sprintf("10.0.0.%d", i+1)
+ count, vram := 4, 81920
+ if i < 2 {
+ count, vram = tt.counts[i], tt.vram[i]
+ }
+ metrics[buildDCGMEndpoint(ip)] = GPUInfo{GPUsPerNode: count, Model: "H100-SXM5-80GB", VRAMPerGPU: vram}
+ objects = append(objects, &corev1.Pod{
+ Name: fmt.Sprintf("dcgm-%d", i), Namespace: "gpu-operator",
+ Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter},
+ Spec: corev1.PodSpec{NodeName: node},
+ Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip},
+ })
+ }
+ discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) {
+ if tt.failFirstExporter && endpoint == buildDCGMEndpoint("10.0.0.1") {
+ return nil, errors.New("exporter unavailable")
+ }
+ scraped, ok := metrics[endpoint]
+ if !ok {
+ return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint)
+ }
+ return &scraped, nil
+ })
+
+ t.Log("counting each successfully discovered node once")
+ info, err := discovery.DiscoverGPUsFromDCGM(t.Context(), newFakeClient(objects...), nil)
+ require.NoError(t, err)
+ assert.Equal(t, 8, info.GPUsPerNode)
+ assert.Equal(t, tt.wantVRAM, info.VRAMPerGPU)
+ assert.Equal(t, 2, info.NodesWithGPUs)
+ assert.Equal(t, 12, info.TotalGPUs)
+ })
+ }
+}
+
const scrapeTestMetrics = "# TYPE DCGM_FI_DEV_GPU_TEMP gauge\n" +
`DCGM_FI_DEV_GPU_TEMP{gpu="0",modelName="NVIDIA A100",Hostname="test-node"} 50` + "\n"
@@ -1118,6 +1252,7 @@ func TestDiscoverGPUsFromDCGM_CacheHit(t *testing.T) {
require.Equal(t, 1, callCount)
require.Equal(t, info1, info2)
+ require.Equal(t, 4, info2.TotalGPUs)
}
func TestDiscoverGPUsFromDCGM_SharesConcurrentScrape(t *testing.T) {
@@ -1216,11 +1351,13 @@ func TestDiscoverGPUsFromDCGMFiltered_MixedSKU(t *testing.T) {
&corev1.Pod{
Name: "dcgm-h100", Namespace: "gpu-operator",
Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter},
+ Spec: corev1.PodSpec{NodeName: "node-h100"},
Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"},
},
&corev1.Pod{
Name: "dcgm-a100", Namespace: "gpu-operator",
Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter},
+ Spec: corev1.PodSpec{NodeName: "node-a100"},
Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.2"},
},
}
diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md
index 3109084e12e0..6f54b7d8d94c 100644
--- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md
+++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md
@@ -2543,10 +2543,10 @@ _Appears in:_
| Field | Description | Default | Validation |
| --- | --- | --- | --- |
-| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the GPU with the highest
node count, then highest VRAM. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
|
+| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected as the GPU type of the node with the most GPUs
(ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
|
| `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
|
-| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by counting GPUs on discovered nodes
(filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
|
-| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
|
+| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs across discovered nodes of the
selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
|
+| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected as the highest GPU count on any discovered node
of the selected GPU type. In clusters whose nodes have different GPU counts,
set this explicitly to the node size you intend to schedule on. | | Optional: \{\}
|
| `interconnect` _string_ | Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT configure or enable any GPU interconnect; it only describes what is available/assumed.
- When omitted, the operator may attempt best-effort discovery (currently distinguishes "nvlink"
vs "pcie" based on DCGM NVLink link count). If discovery is unavailable, it may remain empty.
Impact of wrong / missing values:
- If set more optimistically than reality (e.g., "nvlink" when only PCIe is present), performance
models may overestimate intra-node bandwidth and choose overly aggressive parallelism or layouts,
resulting in degraded performance compared to expectations.
- If set more pessimistically than reality (e.g., "pcie" when NVLink is present), the system may
choose conservative plans and leave performance on the table.
- If unset and undiscovered, consumers should treat the interconnect as unknown and fall back to
conservative assumptions.
Example values: "pcie", "nvlink". Other values may be accepted but may not be auto-detected. | | Optional: \{\}
|
| `rdma` _boolean_ | RDMA indicates whether the cluster has RDMA-capable networking available for Dynamo data movement.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT install, enable, or configure RDMA (e.g., drivers, SR-IOV, NVIDIA network operator,
GPUDirect settings). It only expresses availability/intent.
- When omitted, the operator may attempt best-effort discovery (e.g., via node labels indicating
RDMA/SR-IOV capability and/or presence of NVIDIA network-operator RDMA components). If discovery
is unavailable, it may remain unset.
Impact of wrong / missing values:
- False positive (set true when RDMA is not actually usable end-to-end) may cause plans or
deployments to assume RDMA is available; depending on the runtime transport selection and
fallback behavior, this can lead to connection/setup failures or performance regressions.
- False negative (set false when RDMA is available) will typically avoid RDMA-optimized paths and
fall back to non-RDMA transports, usually remaining functional but potentially slower.
- If unset and undiscovered, consumers should treat RDMA availability as unknown and use
conservative defaults / fallback transports. | | Optional: \{\}
|
diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx
index 286a33378170..4f4a7472b00d 100644
--- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx
+++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx
@@ -142,16 +142,16 @@ service.
- GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes.
+ GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes.
VRAM per GPU in MiB. Auto-detected from cluster GPU nodes when omitted.
- GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. Set explicitly to override the cap.
+ GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (`gpuSku`, or the auto-detected type), temporarily capped at 32 to limit the profiler search space. This reflects discovered GPUs, not currently free capacity. For example, matching nodes with 8 and 4 GPUs yield a budget of 12, while `numGpusPerNode` remains 8. Set explicitly to override the cap.
- GPUs per node. Auto-detected when omitted.
+ GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on.
Primary GPU-to-GPU interconnect within a node, for example `nvlink` or `pcie`. Capability metadata for profiling and planning only — it does not enable any interconnect. Best-effort discovery distinguishes `nvlink` from `pcie` when omitted. An over-optimistic value can lead the planner to overestimate intra-node bandwidth and pick overly aggressive parallelism.
diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx
index 57f507d12171..dfab5f383e68 100644
--- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx
+++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx
@@ -3799,7 +3799,7 @@ If all four fields are set, discovery is skipped.
**Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta1-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta1-dynamographdeploymentrequestspec)
-GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.
+GPUSKU selects the GPU type to target. When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.
See [GPUSKUType](#gpuskutype).
**Validation:** Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300] Optional: \{\}
@@ -3810,12 +3810,12 @@ VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU
-TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.
+TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.
**Validation:** Optional: \{\}
-NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected from cluster GPU nodes.
+NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on.
**Validation:** Optional: \{\}
diff --git a/lib/bindings/python/tests/test_example_hello_world.py b/lib/bindings/python/tests/test_example_hello_world.py
index 4f4657185d60..175d67231da5 100644
--- a/lib/bindings/python/tests/test_example_hello_world.py
+++ b/lib/bindings/python/tests/test_example_hello_world.py
@@ -10,6 +10,7 @@
import subprocess
import pytest
+import pytest_asyncio
pytestmark = [
pytest.mark.gpu_0,
@@ -30,7 +31,7 @@ def example_dir():
)
-@pytest.fixture(scope="module")
+@pytest_asyncio.fixture(scope="module", loop_scope="module")
async def server_process(example_dir):
"""Start the hello_world server and clean up after test"""
server_proc = subprocess.Popen(
diff --git a/pyproject.toml b/pyproject.toml
index 9e0583619b80..036280102f97 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -254,9 +254,6 @@ xfail_strict = true
log_cli_level = "INFO"
filterwarnings = [
"error",
- # pytest 9.2.0 started raising errors for warnings from pytest_configure hooks.
- # pytest-asyncio warns there while asyncio_default_fixture_loop_scope is unset.
- 'ignore:The configuration option "asyncio_default_fixture_loop_scope" is unset:pytest.PytestDeprecationWarning',
# kr8s 0.20.13 predates Kubernetes 1.36; keep its version warning visible
# without aborting deploy tests. Remove when the pinned release supports 1.36.
# Upstream coverage: https://github.com/kr8s-org/kr8s/pull/744
@@ -337,6 +334,7 @@ filterwarnings = [
# NOTE: Can also manually mark tests with @pytest.mark.asyncio
asyncio_mode = "auto"
+asyncio_default_fixture_loop_scope = "function"
markers = [
"pre_merge: marks tests to run before merging",
"post_merge: marks tests to run after merge",