From 16bfadaf1c6f01157c2236fcf14f7b5d16f2c70c Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Sun, 25 Oct 2026 01:21:09 -0400 Subject: [PATCH 1/7] fix(operator): aggregate discovered GPU counts across matching nodes Signed-off-by: Pranjul Gupta --- ...dynamographdeploymentrequest_controller.go | 8 +- ...ploymentrequest_controller_envtest_test.go | 15 ++-- .../controller/enrich_hardware_test.go | 51 +++++++++++ deploy/operator/internal/gpu/discovery.go | 19 ++-- .../operator/internal/gpu/discovery_test.go | 86 +++++++++++++++++++ .../dynamo-graph-deployment-request.mdx | 4 +- 6 files changed, 166 insertions(+), 17 deletions(-) diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go index 7577d49df281..e93462cdf5e7 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go @@ -1900,7 +1900,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx logger.Info("GPU discovery completed successfully", "gpusPerNode", gpuInfo.GPUsPerNode, "nodesWithGPUs", gpuInfo.NodesWithGPUs, - "totalGpus", gpuInfo.GPUsPerNode*gpuInfo.NodesWithGPUs, + "totalGpus", gpuInfo.TotalGPUs, "model", gpuInfo.Model, "vramMiB", gpuInfo.VRAMPerGPU, "system", gpuInfo.System, @@ -1935,14 +1935,14 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx if hw.TotalGPUs == nil && gpuInfo != nil { // TODO: This is a temporary limit to prevent the profiler from using too many GPUs. // Will be removed once a fix is in the Profiler/AIC. - const defaultMaxAutoGPUs = int32(32) - total := int32(gpuInfo.GPUsPerNode * gpuInfo.NodesWithGPUs) + const defaultMaxAutoGPUs = 32 + total := gpuInfo.TotalGPUs if total > defaultMaxAutoGPUs { logger.Info("Capping auto-discovered TotalGPUs at default limit; set hardware.totalGpus to override", "discovered", total, "cap", defaultMaxAutoGPUs) total = defaultMaxAutoGPUs } - hw.TotalGPUs = &total + hw.TotalGPUs = new(int32(total)) changed = true } if hw.Interconnect == "" && gpuInfo != nil && gpuInfo.Interconnect != "" { diff --git a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go index 7658c07a3b0b..ba02c3b2fdb7 100644 --- a/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go +++ b/deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go @@ -23,6 +23,7 @@ import ( "context" "encoding/json" "errors" + "fmt" "slices" "strings" "time" @@ -3007,13 +3008,13 @@ spec: Expect(k8sClient.Create(ctx, dgdr)).Should(Succeed()) defer func() { _ = k8sClient.Delete(ctx, dgdr) }() - GinkgoT().Log("creating two eight-GPU nodes for node-label discovery") + GinkgoT().Log("creating eight-GPU and four-GPU nodes for node-label discovery") gpuNodes := make([]*corev1.Node, 0, 2) - for _, nodeName := range []string{"gpu-worker-budget-1", "gpu-worker-budget-2"} { + for i, count := range []string{"8", "4"} { node := &corev1.Node{ - Name: nodeName, + Name: fmt.Sprintf("gpu-worker-budget-%d", i), Labels: map[string]string{ - gpu.LabelGPUCount: "8", + gpu.LabelGPUCount: count, gpu.LabelGPUProduct: "H200-SXM5-141GB", gpu.LabelGPUMemory: "141312", }, @@ -3042,7 +3043,7 @@ spec: Expect(k8sClient.Get(ctx, request.NamespacedName, &persisted)).Should(Succeed()) Expect(persisted.Spec.Hardware).NotTo(BeNil()) Expect(persisted.Spec.Hardware.TotalGPUs).NotTo(BeNil()) - Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(16))) + Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(12))) GinkgoT().Log("creating the profiling Job from the persisted DGDR") _, err = reconciler.Reconcile(ctx, request) @@ -3110,6 +3111,7 @@ spec: VRAMPerGPU: 81920, System: "h100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3224,6 +3226,7 @@ spec: VRAMPerGPU: 81920, System: "h100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3339,6 +3342,7 @@ spec: VRAMPerGPU: 81920, System: "h100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) @@ -3961,6 +3965,7 @@ spec: VRAMPerGPU: 81920, System: "a100_sxm", NodesWithGPUs: 1, + TotalGPUs: 8, } cache := gpu.NewGPUDiscoveryCache() cache.Set("", mockGPU, 10*time.Minute) diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go index ce7b90a4225f..cf875c4a51a9 100644 --- a/deploy/operator/internal/controller/enrich_hardware_test.go +++ b/deploy/operator/internal/controller/enrich_hardware_test.go @@ -273,6 +273,54 @@ func TestEnrichHardwareFromDiscovery(t *testing.T) { } } +func TestEnrichHardwareFromDiscovery_TotalGPUs(t *testing.T) { + tests := []struct { + name string + counts []int + explicitTotal *int32 + wantTotal int32 + }{ + {name: "heterogeneous nodes", counts: []int{8, 4}, wantTotal: 12}, + {name: "uniform nodes", counts: []int{8, 8}, wantTotal: 16}, + {name: "aggregate below cap despite larger maximum times nodes", counts: []int{8, 8, 8, 4, 2}, wantTotal: 30}, + {name: "aggregate at cap", counts: []int{8, 8, 8, 4, 4}, wantTotal: 32}, + {name: "aggregate above cap", counts: []int{8, 8, 8, 8, 4}, wantTotal: 32}, + {name: "explicit total below discovery", counts: []int{8, 4}, explicitTotal: new(int32(4)), wantTotal: 4}, + {name: "explicit total above cap", counts: []int{8, 4}, explicitTotal: new(int32(64)), wantTotal: 64}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("creating H100 nodes and an unrelated A100 node") + objects := []client.Object{gpuNode("a100-node", "A100-SXM4-40GB", 8, 40960)} + for i, count := range tt.counts { + objects = append(objects, gpuNode(fmt.Sprintf("h100-node-%d", i), "H100-SXM5-80GB", count, 81920)) + } + r := newFakeReconciler(objects...) + dgdr := &nvidiacomv1beta1.DynamoGraphDeploymentRequest{ + Spec: nvidiacomv1beta1.DynamoGraphDeploymentRequestSpec{ + Hardware: &nvidiacomv1beta1.HardwareSpec{ + GPUSKU: nvidiacomv1beta1.GPUSKUTypeH100SXM, + TotalGPUs: tt.explicitTotal, + }, + }, + } + + t.Log("enriching missing hardware fields while preserving an explicit budget") + changed, err := r.enrichHardwareFromDiscovery(t.Context(), dgdr) + require.NoError(t, err) + require.True(t, changed) + require.NotNil(t, dgdr.Spec.Hardware.TotalGPUs) + assert.Equal(t, tt.wantTotal, *dgdr.Spec.Hardware.TotalGPUs) + assert.Equal(t, int32(8), *dgdr.Spec.Hardware.NumGPUsPerNode) + + t.Log("checking that enrichment preserves the resolved budget on another pass") + _, err = r.enrichHardwareFromDiscovery(t.Context(), dgdr) + require.NoError(t, err) + assert.Equal(t, tt.wantTotal, *dgdr.Spec.Hardware.TotalGPUs) + }) + } +} + func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T) { r := newFakeReconciler() r.GPUDiscovery = gpupkg.NewGPUDiscovery(nil) @@ -281,6 +329,7 @@ func TestEnrichHardwareFromDiscovery_WritesOptionalHardwareMetadata(t *testing.T NodeName: "n1", GPUsPerNode: 8, NodesWithGPUs: 2, + TotalGPUs: 16, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079, System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, @@ -313,6 +362,7 @@ func TestEnrichHardwareFromDiscovery_FillsOptionalMetadataWhenRequiredFieldsSet( NodeName: "n1", GPUsPerNode: 8, NodesWithGPUs: 4, + TotalGPUs: 32, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079, System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, @@ -367,6 +417,7 @@ func TestCreateProfilingJobPersistsDiscoveredHardware(t *testing.T) { NodeName: "n1", GPUsPerNode: 8, NodesWithGPUs: 2, + TotalGPUs: 16, Model: "NVIDIA H100 80GB HBM3", VRAMPerGPU: 81079, System: nvidiacomv1beta1.GPUSKUTypeH100PCIe, diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index 686ce3178d73..3841b67a8172 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -165,8 +165,9 @@ var gpuRules = []gpuRule{ // GPUInfo contains discovered GPU configuration from cluster nodes type GPUInfo struct { NodeName string // Name of the node with this GPU configuration - GPUsPerNode int // Maximum GPUs per node found in the cluster - NodesWithGPUs int // Number of nodes that have GPUs + GPUsPerNode int // Maximum GPUs per node with the selected SKU + NodesWithGPUs int // Number of discovered nodes with the selected SKU + TotalGPUs int // Sum of GPUs across discovered nodes with the selected SKU, before any budget cap Model string // GPU product name (e.g., "H100-SXM5-80GB") VRAMPerGPU int // VRAM in MiB per GPU System nvidiacomv1beta1.GPUSKUType // AIC hardware system identifier (e.g., "h100_sxm", "h200_sxm"), empty if unknown @@ -269,7 +270,7 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // 6. Selects the "best" GPU node (filtered by SKU when set) based on: // - Highest GPU count // - Highest VRAM per GPU (tie-breaker) -// 7. Counts only nodes matching the selected SKU for NodesWithGPUs. +// 7. Counts nodes and sums GPUs matching the selected SKU for NodesWithGPUs and TotalGPUs. // 8. Caches the result per SKU for a short duration to avoid repeated scraping. // // Behavior Notes: @@ -397,11 +398,12 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, return nil, fmt.Errorf("no GPU metrics could be parsed from any DCGM pod") } - // Count only nodes with the same SKU as the selected best node, + // Count nodes and GPUs with the same SKU as the selected best node, // and detect RDMA on matching nodes only. On a cold cache and a no-RDMA // cluster, this performs one Node read per matching node; that keeps a // single negative node from masking RDMA on another node. nodesWithGPUs := 0 + totalGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { @@ -409,6 +411,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, continue } nodesWithGPUs++ + totalGPUs += n.info.GPUsPerNode if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.nodeName) if rdma { @@ -433,6 +436,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, bestNode.System = bestSKU bestNode.CloudProvider = cloudProvider bestNode.NodesWithGPUs = nodesWithGPUs + bestNode.TotalGPUs = totalGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType @@ -854,16 +858,18 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK len(nodeList.Items), LabelGPUCount, LabelGPUProduct, LabelGPUMemory) } - // Count only nodes with the same SKU as the selected best node, + // Count nodes and GPUs with the same SKU as the selected best node, // and detect RDMA on matching nodes only. On a cold cache and a no-RDMA // cluster, this performs one Node read per matching node; that keeps a // single negative node from masking RDMA on another node. nodesWithGPUs := 0 + totalGPUs := 0 var rdmaDetected bool var rdmaType string for _, n := range allNodes { if n.sku == bestSKU { nodesWithGPUs++ + totalGPUs += n.info.GPUsPerNode if !rdmaDetected { rdma, rType := detectRDMAFromNode(ctx, k8sClient, n.info.NodeName) if rdma { @@ -884,12 +890,13 @@ func DiscoverGPUsFiltered(ctx context.Context, k8sClient client.Reader, filterSK bestNode.System = bestSKU bestNode.NodesWithGPUs = nodesWithGPUs + bestNode.TotalGPUs = totalGPUs bestNode.RDMAEnabled = rdmaDetected bestNode.RDMAType = rdmaType logger.Info("GPU discovery completed", "gpusPerNode", bestNode.GPUsPerNode, "nodesWithGPUs", bestNode.NodesWithGPUs, - "totalGpus", bestNode.GPUsPerNode*bestNode.NodesWithGPUs, + "totalGpus", bestNode.TotalGPUs, "model", bestNode.Model, "vram", bestNode.VRAMPerGPU, "system", bestNode.System, diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index 0c21526f8dd1..80547e3be436 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -50,6 +50,92 @@ func newFakeClient(objs ...client.Object) client.Reader { Build() } +func TestGPUDiscovery_TotalGPUs(t *testing.T) { + for _, source := range []string{"node labels", "DCGM"} { + t.Run(source, func(t *testing.T) { + tests := []struct { + name string + h100Counts []int + a100Counts []int + filterSKU nvidiacomv1beta1.GPUSKUType + wantSKU nvidiacomv1beta1.GPUSKUType + wantPerNode int + wantNodes int + wantTotal int + }{ + {name: "single node", h100Counts: []int{8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 1, wantTotal: 8}, + {name: "uniform nodes", h100Counts: []int{8, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 16}, + {name: "heterogeneous nodes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, + {name: "maximum selected after smaller node", h100Counts: []int{4, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, + {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, + {name: "filter selects other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, filterSKU: "a100_sxm", wantSKU: "a100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 10}, + {name: "discovery does not cap the total", h100Counts: []int{8, 8, 8, 8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 5, wantTotal: 36}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("creating GPU nodes and DCGM exporters for each SKU") + var objects []client.Object + metrics := make(map[string]GPUInfo) + for _, group := range []struct { + model string + vram int + counts []int + }{ + {model: "H100-SXM5-80GB", vram: 81920, counts: tt.h100Counts}, + {model: "A100-SXM4-40GB", vram: 40960, counts: tt.a100Counts}, + } { + for _, count := range group.counts { + name := fmt.Sprintf("gpu-node-%d", len(metrics)) + ip := fmt.Sprintf("10.0.0.%d", len(metrics)+1) + objects = append(objects, &corev1.Node{ + Name: name, + Labels: map[string]string{ + LabelGPUCount: fmt.Sprint(count), LabelGPUProduct: group.model, LabelGPUMemory: fmt.Sprint(group.vram), + }, + }, &corev1.Pod{ + Name: name, Namespace: "gpu-operator", + Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: name}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip}, + }) + metrics[buildDCGMEndpoint(ip)] = GPUInfo{NodeName: name, GPUsPerNode: count, Model: group.model, VRAMPerGPU: group.vram} + } + } + k8sClient := newFakeClient(objects...) + + t.Log("discovering the selected SKU's actual GPU total") + var info *GPUInfo + var err error + if source == "DCGM" { + discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { + info, ok := metrics[endpoint] + if !ok { + return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint) + } + return &info, nil + }) + cache := NewGPUDiscoveryCache() + info, err = discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, cache, tt.filterSKU) + require.NoError(t, err) + + t.Log("checking that the cached result retains the aggregate") + cached, err := discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, cache, tt.filterSKU) + require.NoError(t, err) + assert.Equal(t, tt.wantTotal, cached.TotalGPUs) + } else { + info, err = DiscoverGPUsFiltered(t.Context(), k8sClient, tt.filterSKU) + } + require.NoError(t, err) + assert.Equal(t, tt.wantSKU, info.System) + assert.Equal(t, tt.wantPerNode, info.GPUsPerNode) + assert.Equal(t, tt.wantNodes, info.NodesWithGPUs) + assert.Equal(t, tt.wantTotal, info.TotalGPUs) + }) + } + }) + } +} + func TestDiscoverGPUs_SingleNode(t *testing.T) { ctx := context.Background() diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index e771992fa0e4..aea6de984502 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -142,13 +142,13 @@ service. - GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. + GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. VRAM per GPU in MiB. Auto-detected from cluster GPU nodes when omitted. - GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. Set explicitly to override the cap. + GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. For example, matching nodes with 8 and 4 GPUs yield a budget of 12, while `numGpusPerNode` remains 8. Set explicitly to override the cap. GPUs per node. Auto-detected when omitted. From 65e8eef73caad489f5a41856a1341c83fc8c8d0a Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Sun, 25 Oct 2026 02:38:12 -0400 Subject: [PATCH 2/7] fix(operator): count duplicate exporters once and align GPU docs Signed-off-by: Pranjul Gupta --- .../profiler/utils/dgdr_v1beta1_types.py | 4 +- .../dynamographdeploymentrequest_types.go | 8 +-- ...dia.com_dynamographdeploymentrequests.yaml | 8 +-- .../controller/enrich_hardware_test.go | 3 -- deploy/operator/internal/gpu/discovery.go | 11 ++-- .../operator/internal/gpu/discovery_test.go | 52 ++++++++++++++----- .../additional-resources/api-reference-k8s.md | 4 +- .../kubernetes-api/full-api-reference.mdx | 4 +- 8 files changed, 62 insertions(+), 32 deletions(-) diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index d4ce2ad16f3e..425b031feb66 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -229,7 +229,7 @@ class HardwareSpec(BaseModel): gpuSku: Optional[GPUSKUType] = Field( default=None, - description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", + description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", ) vramMb: Optional[float] = Field( default=None, @@ -237,7 +237,7 @@ class HardwareSpec(BaseModel): ) totalGpus: Optional[int] = Field( default=None, - description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", + description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", ) numGpusPerNode: Optional[int] = Field( default=None, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index bbdc617b187e..fadaef947842 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -407,8 +407,8 @@ type FeaturesSpec struct { // If all four fields are set, discovery is skipped. type HardwareSpec struct { // GPUSKU selects the GPU type to target. - // When omitted, auto-detected by selecting the GPU with the highest - // node count, then highest VRAM. In mixed-GPU clusters, set this to + // When omitted, auto-detected by selecting the node with the most GPUs, + // then highest VRAM per GPU. In mixed-GPU clusters, set this to // choose which GPU type to use. Discovery and totalGpus are then // restricted to nodes matching this SKU. // The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -423,8 +423,8 @@ type HardwareSpec struct { // TotalGPUs is the GPU budget for profiling and deployment. // The profiler uses this to determine parallelism and replica count. - // When omitted, computed by counting GPUs on discovered nodes - // (filtered by gpuSku when set), temporarily capped at 32 to + // When omitted, computed by summing GPUs on discovered nodes with the + // selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to // limit profiler search space. This cap may be removed in a future // release. Set this field explicitly to override. // +optional diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 404cc63ec777..1363f9e54924 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -663,8 +663,8 @@ spec: - mi300 description: |- GPUSKU selects the GPU type to target. - When omitted, auto-detected by selecting the GPU with the highest - node count, then highest VRAM. In mixed-GPU clusters, set this to + When omitted, auto-detected by selecting the node with the most GPUs, + then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -721,8 +721,8 @@ spec: description: |- TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. - When omitted, computed by counting GPUs on discovered nodes - (filtered by gpuSku when set), temporarily capped at 32 to + When omitted, computed by summing GPUs on discovered nodes with the + selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. format: int32 diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go index cf875c4a51a9..003b5cd6fd34 100644 --- a/deploy/operator/internal/controller/enrich_hardware_test.go +++ b/deploy/operator/internal/controller/enrich_hardware_test.go @@ -281,11 +281,8 @@ func TestEnrichHardwareFromDiscovery_TotalGPUs(t *testing.T) { wantTotal int32 }{ {name: "heterogeneous nodes", counts: []int{8, 4}, wantTotal: 12}, - {name: "uniform nodes", counts: []int{8, 8}, wantTotal: 16}, {name: "aggregate below cap despite larger maximum times nodes", counts: []int{8, 8, 8, 4, 2}, wantTotal: 30}, - {name: "aggregate at cap", counts: []int{8, 8, 8, 4, 4}, wantTotal: 32}, {name: "aggregate above cap", counts: []int{8, 8, 8, 8, 4}, wantTotal: 32}, - {name: "explicit total below discovery", counts: []int{8, 4}, explicitTotal: new(int32(4)), wantTotal: 4}, {name: "explicit total above cap", counts: []int{8, 4}, explicitTotal: new(int32(64)), wantTotal: 64}, } for _, tt := range tests { diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index 3841b67a8172..27f14198936c 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -270,8 +270,7 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // 6. Selects the "best" GPU node (filtered by SKU when set) based on: // - Highest GPU count // - Highest VRAM per GPU (tie-breaker) -// 7. Counts nodes and sums GPUs matching the selected SKU for NodesWithGPUs and TotalGPUs. -// 8. Caches the result per SKU for a short duration to avoid repeated scraping. +// 7. Caches the result per SKU for a short duration to avoid repeated scraping. // // Behavior Notes: // @@ -279,7 +278,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // load-balancing ambiguity in multi-node clusters. // - If at least one pod is successfully scraped, partial failures are tolerated. // - If all pods fail to scrape, an aggregated error is returned. -// - Assumes DCGM exporter runs as a DaemonSet (one pod per GPU node). +// - Assumes each exporter reports metrics for its whole node; multiple +// exporters on the same node count only once. // // Returns: // - *GPUInfo for the selected node @@ -404,12 +404,17 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, // single negative node from masking RDMA on another node. nodesWithGPUs := 0 totalGPUs := 0 + seenNodes := make(map[string]struct{}) var rdmaDetected bool var rdmaType string for _, n := range allNodes { if n.sku != bestSKU { continue } + if _, seen := seenNodes[n.nodeName]; seen { + continue + } + seenNodes[n.nodeName] = struct{}{} nodesWithGPUs++ totalGPUs += n.info.GPUsPerNode if !rdmaDetected { diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index 80547e3be436..59b8ffffcd52 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -63,8 +63,6 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { wantNodes int wantTotal int }{ - {name: "single node", h100Counts: []int{8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 1, wantTotal: 8}, - {name: "uniform nodes", h100Counts: []int{8, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 16}, {name: "heterogeneous nodes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, {name: "maximum selected after smaller node", h100Counts: []int{4, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, @@ -108,20 +106,13 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { var err error if source == "DCGM" { discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { - info, ok := metrics[endpoint] + scraped, ok := metrics[endpoint] if !ok { return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint) } - return &info, nil + return &scraped, nil }) - cache := NewGPUDiscoveryCache() - info, err = discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, cache, tt.filterSKU) - require.NoError(t, err) - - t.Log("checking that the cached result retains the aggregate") - cached, err := discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, cache, tt.filterSKU) - require.NoError(t, err) - assert.Equal(t, tt.wantTotal, cached.TotalGPUs) + info, err = discovery.DiscoverGPUsFromDCGMFiltered(t.Context(), k8sClient, nil, tt.filterSKU) } else { info, err = DiscoverGPUsFiltered(t.Context(), k8sClient, tt.filterSKU) } @@ -155,6 +146,7 @@ func TestDiscoverGPUs_SingleNode(t *testing.T) { require.NotNil(t, gpuInfo) assert.Equal(t, 8, gpuInfo.GPUsPerNode) + assert.Equal(t, 8, gpuInfo.TotalGPUs) assert.Equal(t, "H100-SXM5-80GB", gpuInfo.Model) assert.Equal(t, 81920, gpuInfo.VRAMPerGPU) assert.Equal(t, "h100_sxm", string(gpuInfo.System)) @@ -188,6 +180,7 @@ func TestDiscoverGPUs_MultipleNodesHomogeneous(t *testing.T) { require.NotNil(t, gpuInfo) assert.Equal(t, 8, gpuInfo.GPUsPerNode) + assert.Equal(t, 16, gpuInfo.TotalGPUs) assert.Equal(t, "H100-SXM5-80GB", gpuInfo.Model) assert.Equal(t, 81920, gpuInfo.VRAMPerGPU) } @@ -1068,6 +1061,40 @@ func TestScrapeMetricsEndpoint(t *testing.T) { }) } +func TestDiscoverGPUsFromDCGM_DeduplicatesNodes(t *testing.T) { + for _, failFirstExporter := range []bool{false, true} { + t.Run(fmt.Sprintf("first exporter fails=%t", failFirstExporter), func(t *testing.T) { + t.Log("creating two exporters on one node and one on another node") + var objects []client.Object + for i, node := range []string{"node-a", "node-a", "node-b"} { + objects = append(objects, &corev1.Pod{ + Name: fmt.Sprintf("dcgm-%d", i), Namespace: "gpu-operator", + Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: node}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: fmt.Sprintf("10.0.0.%d", i+1)}, + }) + } + discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { + if failFirstExporter && endpoint == buildDCGMEndpoint("10.0.0.1") { + return nil, errors.New("exporter unavailable") + } + count := 8 + if endpoint == buildDCGMEndpoint("10.0.0.3") { + count = 4 + } + return &GPUInfo{GPUsPerNode: count, Model: "H100-SXM5-80GB", VRAMPerGPU: 81920}, nil + }) + + t.Log("counting each successfully discovered node once") + info, err := discovery.DiscoverGPUsFromDCGM(t.Context(), newFakeClient(objects...), nil) + require.NoError(t, err) + assert.Equal(t, 8, info.GPUsPerNode) + assert.Equal(t, 2, info.NodesWithGPUs) + assert.Equal(t, 12, info.TotalGPUs) + }) + } +} + func TestDiscoverGPUsFromDCGM_CacheHit(t *testing.T) { ctx := context.Background() @@ -1125,6 +1152,7 @@ func TestDiscoverGPUsFromDCGM_CacheHit(t *testing.T) { require.Equal(t, 1, callCount) require.Equal(t, info1, info2) + require.Equal(t, 4, info2.TotalGPUs) } func TestDiscoverGPUsFromDCGM_SharesConcurrentScrape(t *testing.T) { diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 6331c345674b..02c301b44359 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2543,9 +2543,9 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the GPU with the highest
node count, then highest VRAM. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| +| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the node with the most GPUs,
then highest VRAM per GPU. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| | `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| -| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by counting GPUs on discovered nodes
(filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| +| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs on discovered nodes with the
selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| | `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| | `interconnect` _string_ | Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT configure or enable any GPU interconnect; it only describes what is available/assumed.
- When omitted, the operator may attempt best-effort discovery (currently distinguishes "nvlink"
vs "pcie" based on DCGM NVLink link count). If discovery is unavailable, it may remain empty.
Impact of wrong / missing values:
- If set more optimistically than reality (e.g., "nvlink" when only PCIe is present), performance
models may overestimate intra-node bandwidth and choose overly aggressive parallelism or layouts,
resulting in degraded performance compared to expectations.
- If set more pessimistically than reality (e.g., "pcie" when NVLink is present), the system may
choose conservative plans and leave performance on the table.
- If unset and undiscovered, consumers should treat the interconnect as unknown and fall back to
conservative assumptions.
Example values: "pcie", "nvlink". Other values may be accepted but may not be auto-detected. | | Optional: \{\}
| | `rdma` _boolean_ | RDMA indicates whether the cluster has RDMA-capable networking available for Dynamo data movement.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT install, enable, or configure RDMA (e.g., drivers, SR-IOV, NVIDIA network operator,
GPUDirect settings). It only expresses availability/intent.
- When omitted, the operator may attempt best-effort discovery (e.g., via node labels indicating
RDMA/SR-IOV capability and/or presence of NVIDIA network-operator RDMA components). If discovery
is unavailable, it may remain unset.
Impact of wrong / missing values:
- False positive (set true when RDMA is not actually usable end-to-end) may cause plans or
deployments to assume RDMA is available; depending on the runtime transport selection and
fallback behavior, this can lead to connection/setup failures or performance regressions.
- False negative (set false when RDMA is available) will typically avoid RDMA-optimized paths and
fall back to non-RDMA transports, usually remaining functional but potentially slower.
- If unset and undiscovered, consumers should treat RDMA availability as unknown and use
conservative defaults / fallback transports. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 571fd8ba74d1..6eb36192d92c 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3799,7 +3799,7 @@ If all four fields are set, discovery is skipped. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta1-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta1-dynamographdeploymentrequestspec) -GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the GPU with the highest node count, then highest VRAM. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. +GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. See [GPUSKUType](#gpuskutype). **Validation:** Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300] Optional: \{\} @@ -3810,7 +3810,7 @@ VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU -TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by counting GPUs on discovered nodes (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. +TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. **Validation:** Optional: \{\} From 349f5e23677e2f8c308f99c59f179abb80d02175 Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Sun, 25 Oct 2026 02:59:48 -0400 Subject: [PATCH 3/7] test(operator): remove redundant enrichment pass Signed-off-by: Pranjul Gupta --- deploy/operator/internal/controller/enrich_hardware_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/deploy/operator/internal/controller/enrich_hardware_test.go b/deploy/operator/internal/controller/enrich_hardware_test.go index 003b5cd6fd34..4d365ca62e29 100644 --- a/deploy/operator/internal/controller/enrich_hardware_test.go +++ b/deploy/operator/internal/controller/enrich_hardware_test.go @@ -309,11 +309,6 @@ func TestEnrichHardwareFromDiscovery_TotalGPUs(t *testing.T) { require.NotNil(t, dgdr.Spec.Hardware.TotalGPUs) assert.Equal(t, tt.wantTotal, *dgdr.Spec.Hardware.TotalGPUs) assert.Equal(t, int32(8), *dgdr.Spec.Hardware.NumGPUsPerNode) - - t.Log("checking that enrichment preserves the resolved budget on another pass") - _, err = r.enrichHardwareFromDiscovery(t.Context(), dgdr) - require.NoError(t, err) - assert.Equal(t, tt.wantTotal, *dgdr.Spec.Hardware.TotalGPUs) }) } } From 55540d5a0ad6be22817de42eb79efdde09520101 Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Sun, 25 Oct 2026 19:47:15 -0400 Subject: [PATCH 4/7] fix(operator): reconcile exporter reports before selecting GPU nodes Signed-off-by: Pranjul Gupta --- deploy/operator/internal/gpu/discovery.go | 26 +++++++----- .../operator/internal/gpu/discovery_test.go | 40 +++++++++++++++---- 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/deploy/operator/internal/gpu/discovery.go b/deploy/operator/internal/gpu/discovery.go index 27f14198936c..419fcd0f16d8 100644 --- a/deploy/operator/internal/gpu/discovery.go +++ b/deploy/operator/internal/gpu/discovery.go @@ -258,7 +258,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // // When filterSKU is non-empty, only nodes whose inferred SKU matches are // considered. When empty, the best node is selected first (highest GPU count, -// then VRAM) and then only nodes with the same SKU are counted. +// then VRAM). Each node with the selected SKU contributes once to NodesWithGPUs, +// and its GPU count is summed into TotalGPUs. // // The function performs the following: // @@ -278,8 +279,8 @@ func (g *GPUDiscovery) DiscoverGPUsFromDCGM(ctx context.Context, k8sClient clien // load-balancing ambiguity in multi-node clusters. // - If at least one pod is successfully scraped, partial failures are tolerated. // - If all pods fail to scrape, an aggregated error is returned. -// - Assumes each exporter reports metrics for its whole node; multiple -// exporters on the same node count only once. +// - Multiple exporters on one node use the report with the highest GPU count, +// then highest VRAM, for both node selection and aggregation. // // Returns: // - *GPUInfo for the selected node @@ -352,6 +353,7 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, nodeName string } allNodes := make([]nodeInfo, 0, len(dcgmPods)) + nodeIndexes := make(map[string]int) var scrapeErrors []error for _, pod := range dcgmPods { @@ -365,7 +367,18 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, continue } - allNodes = append(allNodes, nodeInfo{info: info, sku: InferHardwareSystem(info.Model), nodeName: pod.Spec.NodeName}) + // Running pods have a scheduled node name; keep its strongest report in first-seen node order. + n := nodeInfo{info: info, sku: InferHardwareSystem(info.Model), nodeName: pod.Spec.NodeName} + if index, seen := nodeIndexes[n.nodeName]; seen { + previous := allNodes[index].info + if info.GPUsPerNode > previous.GPUsPerNode || + (info.GPUsPerNode == previous.GPUsPerNode && info.VRAMPerGPU > previous.VRAMPerGPU) { + allNodes[index] = n + } + continue + } + nodeIndexes[n.nodeName] = len(allNodes) + allNodes = append(allNodes, n) } if len(allNodes) == 0 { @@ -404,17 +417,12 @@ func (g *GPUDiscovery) discoverGPUsFromDCGMFilteredUncached(ctx context.Context, // single negative node from masking RDMA on another node. nodesWithGPUs := 0 totalGPUs := 0 - seenNodes := make(map[string]struct{}) var rdmaDetected bool var rdmaType string for _, n := range allNodes { if n.sku != bestSKU { continue } - if _, seen := seenNodes[n.nodeName]; seen { - continue - } - seenNodes[n.nodeName] = struct{}{} nodesWithGPUs++ totalGPUs += n.info.GPUsPerNode if !rdmaDetected { diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index 59b8ffffcd52..a6496766835d 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -1062,33 +1062,55 @@ func TestScrapeMetricsEndpoint(t *testing.T) { } func TestDiscoverGPUsFromDCGM_DeduplicatesNodes(t *testing.T) { - for _, failFirstExporter := range []bool{false, true} { - t.Run(fmt.Sprintf("first exporter fails=%t", failFirstExporter), func(t *testing.T) { + tests := []struct { + name string + counts [2]int + vram [2]int + failFirstExporter bool + wantVRAM int + }{ + {name: "matching reports", counts: [2]int{8, 8}, vram: [2]int{81920, 81920}, wantVRAM: 81920}, + {name: "first exporter fails", counts: [2]int{8, 8}, vram: [2]int{81920, 81920}, failFirstExporter: true, wantVRAM: 81920}, + {name: "larger count last", counts: [2]int{4, 8}, vram: [2]int{81920, 40960}, wantVRAM: 40960}, + {name: "larger count first", counts: [2]int{8, 4}, vram: [2]int{40960, 81920}, wantVRAM: 40960}, + {name: "higher VRAM last", counts: [2]int{8, 8}, vram: [2]int{40960, 81920}, wantVRAM: 81920}, + {name: "higher VRAM first", counts: [2]int{8, 8}, vram: [2]int{81920, 40960}, wantVRAM: 81920}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { t.Log("creating two exporters on one node and one on another node") var objects []client.Object + metrics := make(map[string]GPUInfo) for i, node := range []string{"node-a", "node-a", "node-b"} { + ip := fmt.Sprintf("10.0.0.%d", i+1) + count, vram := 4, 81920 + if i < 2 { + count, vram = tt.counts[i], tt.vram[i] + } + metrics[buildDCGMEndpoint(ip)] = GPUInfo{GPUsPerNode: count, Model: "H100-SXM5-80GB", VRAMPerGPU: vram} objects = append(objects, &corev1.Pod{ Name: fmt.Sprintf("dcgm-%d", i), Namespace: "gpu-operator", Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, Spec: corev1.PodSpec{NodeName: node}, - Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: fmt.Sprintf("10.0.0.%d", i+1)}, + Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: ip}, }) } discovery := NewGPUDiscovery(func(_ context.Context, endpoint string) (*GPUInfo, error) { - if failFirstExporter && endpoint == buildDCGMEndpoint("10.0.0.1") { + if tt.failFirstExporter && endpoint == buildDCGMEndpoint("10.0.0.1") { return nil, errors.New("exporter unavailable") } - count := 8 - if endpoint == buildDCGMEndpoint("10.0.0.3") { - count = 4 + scraped, ok := metrics[endpoint] + if !ok { + return nil, fmt.Errorf("unexpected DCGM endpoint %q", endpoint) } - return &GPUInfo{GPUsPerNode: count, Model: "H100-SXM5-80GB", VRAMPerGPU: 81920}, nil + return &scraped, nil }) t.Log("counting each successfully discovered node once") info, err := discovery.DiscoverGPUsFromDCGM(t.Context(), newFakeClient(objects...), nil) require.NoError(t, err) assert.Equal(t, 8, info.GPUsPerNode) + assert.Equal(t, tt.wantVRAM, info.VRAMPerGPU) assert.Equal(t, 2, info.NodesWithGPUs) assert.Equal(t, 12, info.TotalGPUs) }) @@ -1251,11 +1273,13 @@ func TestDiscoverGPUsFromDCGMFiltered_MixedSKU(t *testing.T) { &corev1.Pod{ Name: "dcgm-h100", Namespace: "gpu-operator", Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: "node-h100"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.1"}, }, &corev1.Pod{ Name: "dcgm-a100", Namespace: "gpu-operator", Labels: map[string]string{LabelApp: LabelValueNvidiaDCGMExporter}, + Spec: corev1.PodSpec{NodeName: "node-a100"}, Status: corev1.PodStatus{Phase: corev1.PodRunning, PodIP: "10.0.0.2"}, }, } From 1cb08d221ae581356dbdd4a3d172b40c7bba9d3e Mon Sep 17 00:00:00 2001 From: "Dr. Stefan Schimanski" Date: Mon, 26 Oct 2026 17:15:37 +0100 Subject: [PATCH 5/7] operator/dgdr: address aggregate GPU review comments Signed-off-by: Dr. Stefan Schimanski --- .../src/dynamo/profiler/utils/dgdr_v1beta1_types.py | 6 +++--- .../v1beta1/dynamographdeploymentrequest_types.go | 12 +++++++----- .../nvidia.com_dynamographdeploymentrequests.yaml | 12 +++++++----- deploy/operator/internal/gpu/discovery_test.go | 1 - .../additional-resources/api-reference-k8s.md | 6 +++--- .../dynamo-graph-deployment-request.mdx | 6 +++--- .../reference/kubernetes-api/full-api-reference.mdx | 6 +++--- 7 files changed, 26 insertions(+), 23 deletions(-) diff --git a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py index 425b031feb66..cd3205296343 100644 --- a/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py +++ b/components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py @@ -229,7 +229,7 @@ class HardwareSpec(BaseModel): gpuSku: Optional[GPUSKUType] = Field( default=None, - description="GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", + description="GPUSKU selects the GPU type to target. When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead.", ) vramMb: Optional[float] = Field( default=None, @@ -237,11 +237,11 @@ class HardwareSpec(BaseModel): ) totalGpus: Optional[int] = Field( default=None, - description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", + description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.", ) numGpusPerNode: Optional[int] = Field( default=None, - description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected from cluster GPU nodes.", + description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on.", ) interconnect: Optional[str] = Field( default=None, diff --git a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go index fadaef947842..3d4d7daf893f 100644 --- a/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go +++ b/deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go @@ -407,8 +407,8 @@ type FeaturesSpec struct { // If all four fields are set, discovery is skipped. type HardwareSpec struct { // GPUSKU selects the GPU type to target. - // When omitted, auto-detected by selecting the node with the most GPUs, - // then highest VRAM per GPU. In mixed-GPU clusters, set this to + // When omitted, auto-detected as the GPU type of the node with the most GPUs + // (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to // choose which GPU type to use. Discovery and totalGpus are then // restricted to nodes matching this SKU. // The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -423,15 +423,17 @@ type HardwareSpec struct { // TotalGPUs is the GPU budget for profiling and deployment. // The profiler uses this to determine parallelism and replica count. - // When omitted, computed by summing GPUs on discovered nodes with the - // selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to + // When omitted, computed by summing GPUs across discovered nodes of the + // selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to // limit profiler search space. This cap may be removed in a future // release. Set this field explicitly to override. // +optional TotalGPUs *int32 `json:"totalGpus,omitempty"` // NumGPUsPerNode is the number of GPUs per node. - // When omitted, auto-detected from cluster GPU nodes. + // When omitted, auto-detected as the highest GPU count on any discovered node + // of the selected GPU type. In clusters whose nodes have different GPU counts, + // set this explicitly to the node size you intend to schedule on. // +optional NumGPUsPerNode *int32 `json:"numGpusPerNode,omitempty"` // Interconnect describes the primary GPU-to-GPU interconnect *within a node*. diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml index 1363f9e54924..437a12d274ea 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml @@ -663,8 +663,8 @@ spec: - mi300 description: |- GPUSKU selects the GPU type to target. - When omitted, auto-detected by selecting the node with the most GPUs, - then highest VRAM per GPU. In mixed-GPU clusters, set this to + When omitted, auto-detected as the GPU type of the node with the most GPUs + (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. @@ -693,7 +693,9 @@ spec: numGpusPerNode: description: |- NumGPUsPerNode is the number of GPUs per node. - When omitted, auto-detected from cluster GPU nodes. + When omitted, auto-detected as the highest GPU count on any discovered node + of the selected GPU type. In clusters whose nodes have different GPU counts, + set this explicitly to the node size you intend to schedule on. format: int32 type: integer rdma: @@ -721,8 +723,8 @@ spec: description: |- TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. - When omitted, computed by summing GPUs on discovered nodes with the - selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to + When omitted, computed by summing GPUs across discovered nodes of the + selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. format: int32 diff --git a/deploy/operator/internal/gpu/discovery_test.go b/deploy/operator/internal/gpu/discovery_test.go index a6496766835d..6046d8daecb4 100644 --- a/deploy/operator/internal/gpu/discovery_test.go +++ b/deploy/operator/internal/gpu/discovery_test.go @@ -64,7 +64,6 @@ func TestGPUDiscovery_TotalGPUs(t *testing.T) { wantTotal int }{ {name: "heterogeneous nodes", h100Counts: []int{8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, - {name: "maximum selected after smaller node", h100Counts: []int{4, 8}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, {name: "unfiltered excludes other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 12}, {name: "filter selects other SKU", h100Counts: []int{8, 4}, a100Counts: []int{8, 2}, filterSKU: "a100_sxm", wantSKU: "a100_sxm", wantPerNode: 8, wantNodes: 2, wantTotal: 10}, {name: "discovery does not cap the total", h100Counts: []int{8, 8, 8, 8, 4}, wantSKU: "h100_sxm", wantPerNode: 8, wantNodes: 5, wantTotal: 36}, diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index 02c301b44359..92becf8a4230 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -2543,10 +2543,10 @@ _Appears in:_ | Field | Description | Default | Validation | | --- | --- | --- | --- | -| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected by selecting the node with the most GPUs,
then highest VRAM per GPU. In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| +| `gpuSku` _[GPUSKUType](#gpuskutype)_ | GPUSKU selects the GPU type to target.
When omitted, auto-detected as the GPU type of the node with the most GPUs
(ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to
choose which GPU type to use. Discovery and totalGpus are then
restricted to nodes matching this SKU.
The legacy value gb200_sxm is deprecated; use gb200 instead. | | Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300]
Optional: \{\}
| | `vramMb` _float_ | VRAMMB is the VRAM per GPU in MiB.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| -| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs on discovered nodes with the
selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| -| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected from cluster GPU nodes. | | Optional: \{\}
| +| `totalGpus` _integer_ | TotalGPUs is the GPU budget for profiling and deployment.
The profiler uses this to determine parallelism and replica count.
When omitted, computed by summing GPUs across discovered nodes of the
selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to
limit profiler search space. This cap may be removed in a future
release. Set this field explicitly to override. | | Optional: \{\}
| +| `numGpusPerNode` _integer_ | NumGPUsPerNode is the number of GPUs per node.
When omitted, auto-detected as the highest GPU count on any discovered node
of the selected GPU type. In clusters whose nodes have different GPU counts,
set this explicitly to the node size you intend to schedule on. | | Optional: \{\}
| | `interconnect` _string_ | Interconnect describes the primary GPU-to-GPU interconnect *within a node*.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT configure or enable any GPU interconnect; it only describes what is available/assumed.
- When omitted, the operator may attempt best-effort discovery (currently distinguishes "nvlink"
vs "pcie" based on DCGM NVLink link count). If discovery is unavailable, it may remain empty.
Impact of wrong / missing values:
- If set more optimistically than reality (e.g., "nvlink" when only PCIe is present), performance
models may overestimate intra-node bandwidth and choose overly aggressive parallelism or layouts,
resulting in degraded performance compared to expectations.
- If set more pessimistically than reality (e.g., "pcie" when NVLink is present), the system may
choose conservative plans and leave performance on the table.
- If unset and undiscovered, consumers should treat the interconnect as unknown and fall back to
conservative assumptions.
Example values: "pcie", "nvlink". Other values may be accepted but may not be auto-detected. | | Optional: \{\}
| | `rdma` _boolean_ | RDMA indicates whether the cluster has RDMA-capable networking available for Dynamo data movement.
Semantics / usage:
- This is capability metadata used for profiling, planning, and deployment decisions.
- It does NOT install, enable, or configure RDMA (e.g., drivers, SR-IOV, NVIDIA network operator,
GPUDirect settings). It only expresses availability/intent.
- When omitted, the operator may attempt best-effort discovery (e.g., via node labels indicating
RDMA/SR-IOV capability and/or presence of NVIDIA network-operator RDMA components). If discovery
is unavailable, it may remain unset.
Impact of wrong / missing values:
- False positive (set true when RDMA is not actually usable end-to-end) may cause plans or
deployments to assume RDMA is available; depending on the runtime transport selection and
fallback behavior, this can lead to connection/setup failures or performance regressions.
- False negative (set false when RDMA is available) will typically avoid RDMA-optimized paths and
fall back to non-RDMA transports, usually remaining functional but potentially slower.
- If unset and undiscovered, consumers should treat RDMA availability as unknown and use
conservative defaults / fallback transports. | | Optional: \{\}
| diff --git a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx index aea6de984502..214a8e7a2d43 100644 --- a/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx +++ b/docs/fern/pages/reference/kubernetes-api/dynamo-graph-deployment-request.mdx @@ -142,16 +142,16 @@ service. - GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. + GPU type to target (see [SKU format](#sku-format)). When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose the GPU type; discovery and `totalGpus` are then restricted to matching nodes. VRAM per GPU in MiB. Auto-detected from cluster GPU nodes when omitted. - GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by `gpuSku` when set), temporarily capped at 32 to limit the profiler search space. For example, matching nodes with 8 and 4 GPUs yield a budget of 12, while `numGpusPerNode` remains 8. Set explicitly to override the cap. + GPU budget for profiling and deployment; the profiler uses it to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (`gpuSku`, or the auto-detected type), temporarily capped at 32 to limit the profiler search space. This reflects discovered GPUs, not currently free capacity. For example, matching nodes with 8 and 4 GPUs yield a budget of 12, while `numGpusPerNode` remains 8. Set explicitly to override the cap. - GPUs per node. Auto-detected when omitted. + GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on. Primary GPU-to-GPU interconnect within a node, for example `nvlink` or `pcie`. Capability metadata for profiling and planning only — it does not enable any interconnect. Best-effort discovery distinguishes `nvlink` from `pcie` when omitted. An over-optimistic value can lead the planner to overestimate intra-node bandwidth and pick overly aggressive parallelism. diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 6eb36192d92c..29e20d2bafcb 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -3799,7 +3799,7 @@ If all four fields are set, discovery is skipped. **Appears in:** [DynamoGraphDeploymentRequestNativeSpec](#v1beta1-dynamographdeploymentrequestnativespec), [DynamoGraphDeploymentRequestSpec](#v1beta1-dynamographdeploymentrequestspec) -GPUSKU selects the GPU type to target. When omitted, auto-detected by selecting the node with the most GPUs, then highest VRAM per GPU. In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. +GPUSKU selects the GPU type to target. When omitted, auto-detected as the GPU type of the node with the most GPUs (ties broken by highest VRAM per GPU). In mixed-GPU clusters, set this to choose which GPU type to use. Discovery and totalGpus are then restricted to nodes matching this SKU. The legacy value gb200_sxm is deprecated; use gb200 instead. See [GPUSKUType](#gpuskutype). **Validation:** Enum: [vr_nvl72 gb200 gb200_sxm gb10 b300_sxm b200_sxm h200_sxm h100_sxm h100_pcie a100_sxm a100_pcie a30 l40s l40 l4 v100_sxm v100_pcie t4 mi200 mi300] Optional: \{\} @@ -3810,12 +3810,12 @@ VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU -TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on discovered nodes with the selected GPU type (filtered by gpuSku when set), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. +TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override. **Validation:** Optional: \{\} -NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected from cluster GPU nodes. +NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on. **Validation:** Optional: \{\} From eb25ba5ba392a9c85a28e76127a2181a260067cb Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Wed, 28 Oct 2026 10:26:39 -0400 Subject: [PATCH 6/7] test: ignore pytest-asyncio's unset loop-scope notice under pytest 9.2 Port the standalone CI fix from ai-dynamo/dynamo#17483 without its DGDR changes. Original fix by hongkuanz in 934ceb38fe1cc9173db24572c42156992672af89. Ignore only the unset fixture-loop-scope notice so pytest 9.2 can configure plugins under filterwarnings=error while preserving existing fixture scopes. (cherry picked from commit 934ceb38fe1cc9173db24572c42156992672af89) Signed-off-by: Pranjul Gupta --- pyproject.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 84f20120f317..094025a70df9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,6 +254,10 @@ xfail_strict = true log_cli_level = "INFO" filterwarnings = [ "error", + # pytest 9.2 applies these filters while plugins configure, so pytest-asyncio's + # notice that asyncio_default_fixture_loop_scope is unset aborts the session. + # Leaving it unset keeps each async fixture's loop at that fixture's own scope. + 'ignore:The configuration option "asyncio_default_fixture_loop_scope" is unset:pytest.PytestDeprecationWarning', # kr8s 0.20.13 predates Kubernetes 1.36; keep its version warning visible # without aborting deploy tests. Remove when the pinned release supports 1.36. # Upstream coverage: https://github.com/kr8s-org/kr8s/pull/744 From 7ca907b3070b3d069855714d17f9b1daca22cfbf Mon Sep 17 00:00:00 2001 From: Pranjul Gupta Date: Wed, 28 Oct 2026 17:54:01 -0400 Subject: [PATCH 7/7] test: configure explicit asyncio fixture loop scopes Set the default async fixture loop scope to function and preserve the hello-world server fixture's module-scoped loop explicitly. Remove the unset-loop-scope warning filter now that the configuration is explicit. Signed-off-by: Pranjul Gupta --- lib/bindings/python/tests/test_example_hello_world.py | 3 ++- pyproject.toml | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/bindings/python/tests/test_example_hello_world.py b/lib/bindings/python/tests/test_example_hello_world.py index 4f4657185d60..175d67231da5 100644 --- a/lib/bindings/python/tests/test_example_hello_world.py +++ b/lib/bindings/python/tests/test_example_hello_world.py @@ -10,6 +10,7 @@ import subprocess import pytest +import pytest_asyncio pytestmark = [ pytest.mark.gpu_0, @@ -30,7 +31,7 @@ def example_dir(): ) -@pytest.fixture(scope="module") +@pytest_asyncio.fixture(scope="module", loop_scope="module") async def server_process(example_dir): """Start the hello_world server and clean up after test""" server_proc = subprocess.Popen( diff --git a/pyproject.toml b/pyproject.toml index 094025a70df9..12503f5d64d3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -254,10 +254,6 @@ xfail_strict = true log_cli_level = "INFO" filterwarnings = [ "error", - # pytest 9.2 applies these filters while plugins configure, so pytest-asyncio's - # notice that asyncio_default_fixture_loop_scope is unset aborts the session. - # Leaving it unset keeps each async fixture's loop at that fixture's own scope. - 'ignore:The configuration option "asyncio_default_fixture_loop_scope" is unset:pytest.PytestDeprecationWarning', # kr8s 0.20.13 predates Kubernetes 1.36; keep its version warning visible # without aborting deploy tests. Remove when the pinned release supports 1.36. # Upstream coverage: https://github.com/kr8s-org/kr8s/pull/744 @@ -338,6 +334,7 @@ filterwarnings = [ # NOTE: Can also manually mark tests with @pytest.mark.asyncio asyncio_mode = "auto" +asyncio_default_fixture_loop_scope = "function" markers = [ "pre_merge: marks tests to run before merging", "post_merge: marks tests to run after merge",