fix(operator): separate inventory from selected-shape GPU capacity

components/src/dynamo/profiler/utils/dgdr_v1beta1_types.py

@@ -233,15 +233,15 @@ class HardwareSpec(BaseModel):
     )
     vramMb: Optional[float] = Field(
         default=None,
-        description="VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes.",
+        description="VRAMMB is the VRAM per GPU in MiB. When omitted, auto-detected from cluster GPU nodes. Nominal values within 10% of the measured value identify the same shape. When this value differs by more than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected.",
     )
     totalGpus: Optional[int] = Field(
         default=None,
-        description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs across discovered nodes of the selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to limit profiler search space. This cap may be removed in a future release. Set this field explicitly to override.",
+        description="TotalGPUs is the GPU budget for profiling and deployment. The profiler uses this to determine parallelism and replica count. When omitted, computed by summing GPUs on nodes matching the selected shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily capped at 32 to limit profiler search space. For example, same-type nodes with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU shape yields an automatic budget of 8. This cap may be removed in a future release. Set this field explicitly to override the derived budget or use a numGpusPerNode or vramMb value that differs from the selected shape. A shape override without an explicit totalGpus is rejected.",
     )
     numGpusPerNode: Optional[int] = Field(
         default=None,
-        description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on.",
+        description="NumGPUsPerNode is the number of GPUs per node. When omitted, auto-detected as the highest GPU count on any discovered node of the selected GPU type. In clusters whose nodes have different GPU counts, set this explicitly to the node size you intend to schedule on. When this value differs from the auto-selected shape and totalGpus is omitted, the request is rejected.",
     )
     interconnect: Optional[str] = Field(
         default=None,

deploy/operator/api/v1beta1/dynamographdeploymentrequest_types.go

@@ -417,23 +417,30 @@ type HardwareSpec struct {
 	GPUSKU GPUSKUType `json:"gpuSku,omitempty"`
 
 	// VRAMMB is the VRAM per GPU in MiB.
-	// When omitted, auto-detected from cluster GPU nodes.
+	// When omitted, auto-detected from cluster GPU nodes. Nominal values within 10%
+	// of the measured value identify the same shape. When this value differs by more
+	// than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected.
 	// +optional
 	VRAMMB *float64 `json:"vramMb,omitempty"`
 
 	// TotalGPUs is the GPU budget for profiling and deployment.
 	// The profiler uses this to determine parallelism and replica count.
-	// When omitted, computed by summing GPUs across discovered nodes of the
-	// selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to
-	// limit profiler search space. This cap may be removed in a future
-	// release. Set this field explicitly to override.
+	// When omitted, computed by summing GPUs on nodes matching the selected
+	// shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily
+	// capped at 32 to limit profiler search space. For example, same-type nodes
+	// with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU
+	// shape yields an automatic budget of 8. This cap may be removed in a future
+	// release. Set this field explicitly to override the derived budget or use a
+	// numGpusPerNode or vramMb value that differs from the selected shape. A shape
+	// override without an explicit totalGpus is rejected.
 	// +optional
 	TotalGPUs *int32 `json:"totalGpus,omitempty"`
 
 	// NumGPUsPerNode is the number of GPUs per node.
 	// When omitted, auto-detected as the highest GPU count on any discovered node
 	// of the selected GPU type. In clusters whose nodes have different GPU counts,
-	// set this explicitly to the node size you intend to schedule on.
+	// set this explicitly to the node size you intend to schedule on. When this value
+	// differs from the auto-selected shape and totalGpus is omitted, the request is rejected.
 	// +optional
 	NumGPUsPerNode *int32 `json:"numGpusPerNode,omitempty"`
 	// Interconnect describes the primary GPU-to-GPU interconnect *within a node*.

deploy/operator/config/crd/bases/nvidia.com_dynamographdeploymentrequests.yaml

@@ -695,7 +695,8 @@ spec:
                         NumGPUsPerNode is the number of GPUs per node.
                         When omitted, auto-detected as the highest GPU count on any discovered node
                         of the selected GPU type. In clusters whose nodes have different GPU counts,
-                        set this explicitly to the node size you intend to schedule on.
+                        set this explicitly to the node size you intend to schedule on. When this value
+                        differs from the auto-selected shape and totalGpus is omitted, the request is rejected.
                       format: int32
                       type: integer
                     rdma:
@@ -723,16 +724,22 @@ spec:
                       description: |-
                         TotalGPUs is the GPU budget for profiling and deployment.
                         The profiler uses this to determine parallelism and replica count.
-                        When omitted, computed by summing GPUs across discovered nodes of the
-                        selected GPU type (gpuSku, or the auto-detected type), temporarily capped at 32 to
-                        limit profiler search space. This cap may be removed in a future
-                        release. Set this field explicitly to override.
+                        When omitted, computed by summing GPUs on nodes matching the selected
+                        shape: GPU type, GPUs per node, and VRAM per GPU. The result is temporarily
+                        capped at 32 to limit profiler search space. For example, same-type nodes
+                        with 8 and 4 GPUs represent 12 GPUs of inventory, while the selected 8-GPU
+                        shape yields an automatic budget of 8. This cap may be removed in a future
+                        release. Set this field explicitly to override the derived budget or use a
+                        numGpusPerNode or vramMb value that differs from the selected shape. A shape
+                        override without an explicit totalGpus is rejected.
                       format: int32
                       type: integer
                     vramMb:
                       description: |-
                         VRAMMB is the VRAM per GPU in MiB.
-                        When omitted, auto-detected from cluster GPU nodes.
+                        When omitted, auto-detected from cluster GPU nodes. Nominal values within 10%
+                        of the measured value identify the same shape. When this value differs by more
+                        than 10% from the auto-selected shape and totalGpus is omitted, the request is rejected.
                       type: number
                   type: object
                 image:

deploy/operator/internal/controller/dynamographdeploymentrequest_controller.go

@@ -1901,6 +1901,7 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx
 		"gpusPerNode", gpuInfo.GPUsPerNode,
 		"nodesWithGPUs", gpuInfo.NodesWithGPUs,
 		"totalGpus", gpuInfo.TotalGPUs,
+		"selectedShapeGpus", gpuInfo.SelectedShapeGPUs,
 		"model", gpuInfo.Model,
 		"vramMiB", gpuInfo.VRAMPerGPU,
 		"system", gpuInfo.System,
@@ -1910,6 +1911,11 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx
 		"rdma", gpuInfo.RDMAEnabled,
 		"rdmaType", gpuInfo.RDMAType)
 
+	// An automatically derived budget is valid only for discovery's selected node shape.
+	if err := validateAutomaticGPUShape(hw, gpuInfo); err != nil {
+		return changed, err
+	}
+
 	if hw.GPUSKU == "" {
 		inferred := gpu.InferHardwareSystem(gpuInfo.Model)
 		switch {
@@ -1936,10 +1942,10 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx
 		// TODO: This is a temporary limit to prevent the profiler from using too many GPUs.
 		// Will be removed once a fix is in the Profiler/AIC.
 		const defaultMaxAutoGPUs = 32
-		total := gpuInfo.TotalGPUs
+		total := gpuInfo.SelectedShapeGPUs
 		if total > defaultMaxAutoGPUs {
-			logger.Info("Capping auto-discovered TotalGPUs at default limit; set hardware.totalGpus to override",
-				"discovered", total, "cap", defaultMaxAutoGPUs)
+			logger.Info("Capping auto-discovered GPU budget at default limit; set hardware.totalGpus to override",
+				"selectedShapeGpus", total, "cap", defaultMaxAutoGPUs)
 			total = defaultMaxAutoGPUs
 		}
 		hw.TotalGPUs = new(int32(total))
@@ -1962,6 +1968,25 @@ func (r *DynamoGraphDeploymentRequestReconciler) enrichHardwareFromDiscovery(ctx
 	return changed, nil
 }
 
+func validateAutomaticGPUShape(hw *nvidiacomv1beta1.HardwareSpec, gpuInfo *gpu.GPUInfo) error {
+	if hw.TotalGPUs != nil {
+		return nil
+	}
+	if hw.NumGPUsPerNode != nil && int(*hw.NumGPUsPerNode) != gpuInfo.GPUsPerNode {
+		return fmt.Errorf("cannot auto-detect hardware.totalGpus: hardware.numGpusPerNode=%d does not match the selected discovered shape with %d GPUs per node; set hardware.totalGpus explicitly", *hw.NumGPUsPerNode, gpuInfo.GPUsPerNode)
+	}
+	if hw.VRAMMB != nil && !sameVRAMShape(*hw.VRAMMB, gpuInfo.VRAMPerGPU) {
+		return fmt.Errorf("cannot auto-detect hardware.totalGpus: hardware.vramMb=%g does not match the selected discovered shape with %d MiB per GPU; set hardware.totalGpus explicitly", *hw.VRAMMB, gpuInfo.VRAMPerGPU)
+	}
+	return nil
+}
+
+// sameVRAMShape accepts nominal memory sizes while rejecting a materially different GPU variant.
+func sameVRAMShape(requestedMiB float64, discoveredMiB int) bool {
+	discovered := float64(discoveredMiB)
+	return max(requestedMiB-discovered, discovered-requestedMiB) <= discovered/10
+}
+
 func (r *DynamoGraphDeploymentRequestReconciler) discoverHardwareForEnrichment(ctx context.Context, hw *nvidiacomv1beta1.HardwareSpec, discoveryRequired bool) (*gpu.GPUInfo, error) {
 	logger := log.FromContext(ctx)
 	logger.Info("Attempting GPU discovery for profiling job")

deploy/operator/internal/controller/dynamographdeploymentrequest_controller_envtest_test.go

@@ -3043,7 +3043,7 @@ spec:
 			Expect(k8sClient.Get(ctx, request.NamespacedName, &persisted)).Should(Succeed())
 			Expect(persisted.Spec.Hardware).NotTo(BeNil())
 			Expect(persisted.Spec.Hardware.TotalGPUs).NotTo(BeNil())
-			Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(12)))
+			Expect(*persisted.Spec.Hardware.TotalGPUs).To(Equal(int32(8)))
 
 			GinkgoT().Log("creating the profiling Job from the persisted DGDR")
 			_, err = reconciler.Reconcile(ctx, request)
@@ -3107,11 +3107,12 @@ spec:
 			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()
 
 			mockGPU := &gpu.GPUInfo{
-				GPUsPerNode:   8,
-				VRAMPerGPU:    81920,
-				System:        "h100_sxm",
-				NodesWithGPUs: 1,
-				TotalGPUs:     8,
+				GPUsPerNode:       8,
+				VRAMPerGPU:        81920,
+				System:            "h100_sxm",
+				NodesWithGPUs:     1,
+				TotalGPUs:         8,
+				SelectedShapeGPUs: 8,
 			}
 			cache := gpu.NewGPUDiscoveryCache()
 			cache.Set("", mockGPU, 10*time.Minute)
@@ -3222,11 +3223,12 @@ spec:
 			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()
 
 			mockGPU := &gpu.GPUInfo{
-				GPUsPerNode:   8,
-				VRAMPerGPU:    81920,
-				System:        "h100_sxm",
-				NodesWithGPUs: 1,
-				TotalGPUs:     8,
+				GPUsPerNode:       8,
+				VRAMPerGPU:        81920,
+				System:            "h100_sxm",
+				NodesWithGPUs:     1,
+				TotalGPUs:         8,
+				SelectedShapeGPUs: 8,
 			}
 			cache := gpu.NewGPUDiscoveryCache()
 			cache.Set("", mockGPU, 10*time.Minute)
@@ -3338,11 +3340,12 @@ spec:
 			defer func() { _ = k8sClient.Delete(ctx, dgdr) }()
 
 			mockGPU := &gpu.GPUInfo{
-				GPUsPerNode:   8,
-				VRAMPerGPU:    81920,
-				System:        "h100_sxm",
-				NodesWithGPUs: 1,
-				TotalGPUs:     8,
+				GPUsPerNode:       8,
+				VRAMPerGPU:        81920,
+				System:            "h100_sxm",
+				NodesWithGPUs:     1,
+				TotalGPUs:         8,
+				SelectedShapeGPUs: 8,
 			}
 			cache := gpu.NewGPUDiscoveryCache()
 			cache.Set("", mockGPU, 10*time.Minute)
@@ -3961,11 +3964,12 @@ spec:
 
 			// Mock GPU discovery so validation and enrichment succeed.
 			mockGPU := &gpu.GPUInfo{
-				GPUsPerNode:   8,
-				VRAMPerGPU:    81920,
-				System:        "a100_sxm",
-				NodesWithGPUs: 1,
-				TotalGPUs:     8,
+				GPUsPerNode:       8,
+				VRAMPerGPU:        81920,
+				System:            "a100_sxm",
+				NodesWithGPUs:     1,
+				TotalGPUs:         8,
+				SelectedShapeGPUs: 8,
 			}
 			cache := gpu.NewGPUDiscoveryCache()
 			cache.Set("", mockGPU, 10*time.Minute)